def url_distance(preprocessor, url1, url2): url1 = urlparse(url1) url2 = urlparse(url2) process_fn = lambda s: preprocessor(unquote(s)) path1 = map(process_fn, url1.path.strip('/').split('/')) path2 = map(process_fn, url2.path.strip('/').split('/')) path_distance = levenshtein_array(path1, path2) query_distance = dict_distance(preprocessor, parse_qs(url1.query, True), parse_qs(url2.query, True) ) domain_distance = 4 * levenshtein_array( (url1.hostname or '').split('.'), (url2.hostname or '').split('.') ) return ( domain_distance + path_distance + query_distance + (url1.fragment != url2.fragment) )
def assert_url_equal(url, expected, compare_host=False): """Compare url paths and query strings.""" parsed = urlparse(six.text_type(url)) parsed_expected = urlparse(six.text_type(expected)) compare_url_part(parsed.path, parsed_expected.path) compare_url_part(parse_qs(parsed.query), parse_qs(parsed_expected.query)) if compare_host: compare_url_part(parsed.netloc, parsed_expected.netloc)
def _parse_request(self): self._buffer.seek(0) request_line = self._buffer.readline( ) # e.g. GET /v?v='http://www.nbc.com' HTTP/1.1 match = re.match("(GET|POST|PUT|DELETE|HEAD) (.+) .+", request_line) if match is None: raise ValueError("Invalid HTTP request %s" % request_line) self.method = match.group(1) # e.g. GET whole_url = match.group(2) # e.g. /?v='http://www.nbc.com' parsed_whole_url = urlparse.urlparse(whole_url) params = urlparse.parse_qs(parsed_whole_url.query) url_param = params.get('v', None) # e.g. http://www.nbc.com if url_param is None or len(url_param) == 0: logger.warning("Skipping %s" % whole_url.strip()) return # raise ValueError("No request url received in HTTP request") self.url = url_param[0] for line in self._buffer.readlines(): match = re.match("(.+):\\s*(.+)", line.strip()) if match is None: continue self.headers[match.group(1)] = match.group(2) return True
def get(self, url): images_dir = os.path.join(self.tm_env.images_dir, TAR_DIR) fs.mkdir_safe(images_dir) image = urllib_parse.urlparse(url) sha256 = urllib_parse.parse_qs(image.query).get('sha256', None) with tempfile.NamedTemporaryFile(dir=images_dir, delete=False, prefix='.tmp') as temp: if image.scheme == 'http': _download(url, temp) else: _copy(image.path, temp) if not tarfile.is_tarfile(temp.name): _LOGGER.error('File %r is not a tar file.', url) raise Exception('File {0} is not a tar file.'.format(url)) new_sha256 = _sha256sum(temp.name) if sha256 is not None and sha256[0] != new_sha256: _LOGGER.error('Hash does not match %r - %r', sha256[0], new_sha256) raise Exception('Hash of {0} does not match {1}.'.format( new_sha256, url)) # TODO: rename tar file to sha256 to allow for caching. return TarImage(self.tm_env, temp.name)
def parse_grip_uri(uri): parsed = urlparse(uri) # HACK: work around '+' character in base64-encoded values query = parsed.query.replace('+', '%2B') params = parse_qs(query) iss = None key = None if 'iss' in params: iss = params['iss'][0] del params['iss'] if 'key' in params: key = params['key'][0] del params['key'] if key is not None and key.startswith('base64:'): key = b64decode(key[7:]) qs = urlencode(params, True) path = parsed.path if path.endswith('/'): path = path[:-1] control_uri = parsed.scheme + '://' + parsed.netloc + path if qs: control_uri += '?' + qs out = {'control_uri': control_uri} if iss: out['control_iss'] = iss if key: out['key'] = key return out
def test_submit_service_result(self): with requests_mock.Mocker() as m: m.register_uri(requests_mock.POST, self.ws_arbiter_url + "/push_check_result") check_result = { "return_code": "0", "output": "TEST OUTPUT", "time_stamp": "1409149234" } response = self.post_json( "/v2/status/hosts/bogus-router/services/" + "service-example/results", params=check_result) self.assertEqual(response.status_int, 204) self.assertEqual( urllib_parse.parse_qs(m.last_request.body), { u'output': [u'TEST OUTPUT'], u'return_code': [u'0'], u'service_description': [u'service-example'], u'host_name': [u'bogus-router'], u'time_stamp': [u'1409149234'] })
def _build_url(self): url = list(urlsplit(self._path)) qs = parse_qs(url[3]) qs["count"] = self._count qs["page"] = self._page url[3] = urlencode(qs, doseq=True) return urlunsplit(url)
def depaginate(api, result): """Depaginate the first (or only) page of a paginated result""" meta = result['meta'] if meta['next'] is None: # No pages means we're done return result # make a copy of meta that we'll mess with and eventually return # since we'll be chewing on the 'meta' object with every new GET # same thing for objects, since we'll just be appending to it # while we pull more records ret_meta = meta.copy() ret_objects = result['objects'] while meta['next']: # parse out url bits for constructing the new api req next_url = urlparse(meta['next']) # ugh...need to find the word after 'api/' in the next URL to # get the resource endpoint name; not sure how to make this better next_endpoint = next_url.path.strip('/').split('/')[-1] next_params = {k: v[0] for k, v in parse_qs(next_url.query).items()} result = getattr(api, next_endpoint).get(**next_params) ret_objects.extend(result['objects']) meta = result['meta'] # fix meta up to not tell lies ret_meta['total_count'] = len(ret_objects) ret_meta['next'] = None ret_meta['limit'] = ret_meta['total_count'] return { 'meta': ret_meta, 'objects': ret_objects }
def on_request(self, env, start_response): method = env['REQUEST_METHOD'].upper() path = env['PATH_INFO'].lower().strip('/') if method not in self.method_handlers or path not in self.method_handlers[method]: start_response('404 Not Found', []) return if method == 'GET': request = urlparse.parse_qs(env.get('QUERY_STRING', ''), True) for query in request: if len(request[query]) == 1: request[query] = request[query][0] else: request = json.loads(env.get('wsgi.input', '{}')) response = self.method_handlers[method][path](request) if type(response) == tuple: response, options = response else: options = ResponseOptions() start_response('%d %s' % (options.status, options.reason), [('Content-Type', options.content_type)]) if options.send_json: yield json.dumps(response) else: yield response
def on_request(self, env, start_response): method = env['REQUEST_METHOD'].upper() path = env['PATH_INFO'].lower().strip('/') # If the API request is proxied, the path comes as a full url for some reason if '://' in path: path = urlparse.urlparse(path).path.strip('/') if method not in self.method_handlers or path not in self.method_handlers[method]: start_response('404 Not Found', []) return if method == 'GET': request = urlparse.parse_qs(env.get('QUERY_STRING', ''), True) for query in request: if len(request[query]) == 1: request[query] = request[query][0] else: inp = env.get('wsgi.input', None) request = json.loads(inp.read()) if inp is not None else {} response = self.method_handlers[method][path](request) if type(response) == tuple: response, options = response else: options = ResponseOptions() start_response('%d %s' % (options.status, options.reason), [('Content-Type', options.content_type)]) if options.send_json: return [json.dumps(response)] else: return [response]
def parse_url(url, warning=True): """Parse URLs especially for Google Drive links. file_id: ID of file on Google Drive. is_download_link: Flag if it is download link of Google Drive. """ parsed = urllib_parse.urlparse(url) query = urllib_parse.parse_qs(parsed.query) is_gdrive = parsed.hostname == "drive.google.com" is_download_link = parsed.path.endswith("/uc") file_id = None if is_gdrive and "id" in query: file_ids = query["id"] if len(file_ids) == 1: file_id = file_ids[0] match = re.match(r"^/file/d/(.*?)/view$", parsed.path) if match: file_id = match.groups()[0] if is_gdrive and not is_download_link: warnings.warn( "You specified Google Drive Link but it is not the correct link " "to download the file. Maybe you should try: {url}".format( url="https://drive.google.com/uc?id={}".format(file_id))) return file_id, is_download_link
def test_token_refresh_store_expired(self): expiration = datetime.datetime.utcnow() - datetime.timedelta(minutes=15) credentials = self._create_test_credentials(expiration=expiration) storage = file_module.Storage(FILENAME) storage.put(credentials) credentials = storage.get() new_cred = copy.copy(credentials) new_cred.access_token = "bar" storage.put(new_cred) access_token = "1/3w" token_response = {"access_token": access_token, "expires_in": 3600} response_content = json.dumps(token_response).encode("utf-8") http = http_mock.HttpMock(data=response_content) credentials._refresh(http) self.assertEquals(credentials.access_token, access_token) # Verify mocks. self.assertEqual(http.requests, 1) self.assertEqual(http.uri, credentials.token_uri) self.assertEqual(http.method, "POST") expected_body = { "grant_type": ["refresh_token"], "client_id": [credentials.client_id], "client_secret": [credentials.client_secret], "refresh_token": [credentials.refresh_token], } self.assertEqual(urllib_parse.parse_qs(http.body), expected_body) expected_headers = {"content-type": "application/x-www-form-urlencoded", "user-agent": credentials.user_agent} self.assertEqual(http.headers, expected_headers)
def assert_url(self, expected_url, url): """Check if two URL are same Assertions are called inside this this method. If anything is different, it will fail immediately. :param str expected_url: expected URL compare. :param str url: the URL to check if it is same as the expected URL. """ url = urlparse(url) expected_url = urlparse(expected_url) self.assertEqual(expected_url.scheme, url.scheme) self.assertEqual(expected_url.netloc, url.netloc) self.assertEqual(expected_url.path, url.path) self.assertEqual(parse_qs(expected_url.query), parse_qs(url.query))
def execute(self, subscription, messages, **kwargs): subscriber = urllib_parse.urlparse(subscription['subscriber']) params = urllib_parse.parse_qs(subscriber.query) params = dict((k.lower(), v) for k, v in params.items()) conf = kwargs.get('conf') try: for message in messages: p = subprocess.Popen(conf.notification.smtp_command.split(' '), stdin=subprocess.PIPE) # NOTE(Eva-i): Unfortunately this will add 'queue_name' key to # our original messages(dicts) which will be later consumed in # the storage controller. It seems safe though. message['queue_name'] = subscription['source'] msg = text.MIMEText(json.dumps(message)) msg["to"] = subscriber.path msg["from"] = subscription['options'].get('from', '') subject_opt = subscription['options'].get('subject', '') msg["subject"] = params.get('subject', subject_opt) p.communicate(msg.as_string()) except OSError as err: LOG.exception( _LE('Failed to create process for sendmail, ' 'because %s.') % str(err)) except Exception as exc: LOG.exception(_LE('Failed to send email because %s.') % str(exc))
def parse_url(url, warning=True): """Parse URLs especially for Google Drive links. file_id: ID of file on Google Drive. is_download_link: Flag if it is download link of Google Drive. """ parsed = urllib_parse.urlparse(url) query = urllib_parse.parse_qs(parsed.query) is_gdrive = parsed.hostname in ["drive.google.com", "docs.google.com"] is_download_link = parsed.path.endswith("/uc") if not is_gdrive: return is_gdrive, is_download_link file_id = None if "id" in query: file_ids = query["id"] if len(file_ids) == 1: file_id = file_ids[0] else: patterns = [r"^/file/d/(.*?)/view$", r"^/presentation/d/(.*?)/edit$"] for pattern in patterns: match = re.match(pattern, parsed.path) if match: file_id = match.groups()[0] break if warning and not is_download_link: warnings.warn( "You specified Google Drive Link but it is not the correct link " "to download the file. Maybe you should try: {url}".format( url="https://drive.google.com/uc?id={}".format(file_id))) return file_id, is_download_link
def test_comment_with_plain_text(self): bz = Bugzilla('http://localhost/bugzilla') url = bz.make_url(self.test_run, self.case_run_1, 1) url_parts = urlparse(url) expected_comment = '''Filed from caserun (INSERT URL HERE) Version-Release number of selected component (if applicable): {build_name} Steps to Reproduce: {setup} {action} Actual results: #FIXME Expected results: {effect} '''.format(build_name=self.case_run_1.build.name, setup='setup', action='action', effect='effect') self.assertEqual(expected_comment, parse_qs(url_parts.query)['comment'][0])
def callback(headers, params, request): status = 200 body = """ <directory name="_project" rev="10" vrev="" srcmd5="4adff519b4e14cf206bcc7bdce40a73c"> <entry name="_config" md5="11cfe7fdb2591b55435b312c156e17fe" size="1818" mtime="1542617696" /> </directory> """ parsed = urlparse(request.url) params.update(parse_qs(parsed.query)) if params.get("meta", ['0']) == ['1']: status = 200 body = """ <directory name="_project" rev="41" vrev="" srcmd5="f8bf4cb0e1edea467e34462742ca82ae"> <entry name="_attribute" md5="18fbe34f9ab146c9d73f7e5756f74d48" size="4172" mtime="1543406498" /> <entry name="_meta" md5="9392aa5e748eb15087c4db55ac1cb7f3" size="1875" mtime="1542635759" /> </directory> """ headers['request-id'] = '728d329e-0e86-11e4-a748-0c84dc037c13' return status, headers, body
def test_login(app): """Test Keycloak login.""" keycloak_config = app.config["OAUTHCLIENT_REMOTE_APPS"]["keycloak"] auth_url = keycloak_config["params"]["authorize_url"] client = app.test_client() resp = client.get( url_for( "invenio_oauthclient.login", remote_app="keycloak", next="/someurl/" ) ) assert resp.status_code == 302 comps = urlparse(resp.location) params = parse_qs(comps.query) url = "{}://{}{}".format(comps.scheme, comps.netloc, comps.path) assert url == auth_url assert params["response_type"] == ["code"] assert params["scope"] == ["openid"] assert params["redirect_uri"] assert params["client_id"] assert params["state"]
def test_fxa_login_url_requiring_two_factor_auth(): path = u'/en-US/addons/abp/?source=ddg' request = RequestFactory().get(path) request.session = {'fxa_state': 'myfxastate'} raw_url = utils.fxa_login_url(config=FXA_CONFIG['default'], state=request.session['fxa_state'], next_path=path, action='signin', force_two_factor=True) url = urlparse(raw_url) base = u'{scheme}://{netloc}{path}'.format(scheme=url.scheme, netloc=url.netloc, path=url.path) assert base == 'https://accounts.firefox.com/oauth/authorization' query = parse_qs(url.query) next_path = urlsafe_b64encode(path.encode('utf-8')).rstrip(b'=') assert query == { 'acr_values': ['AAL2'], 'action': ['signin'], 'client_id': ['foo'], 'redirect_url': ['https://testserver/fxa'], 'scope': ['profile'], 'state': ['myfxastate:{next_path}'.format(next_path=force_text(next_path))], }
def depaginate(api, result): """Depaginate the first (or only) page of a paginated result""" meta = result['meta'] if meta['next'] is None: # No pages means we're done return result # make a copy of meta that we'll mess with and eventually return # since we'll be chewing on the 'meta' object with every new GET # same thing for objects, since we'll just be appending to it # while we pull more records ret_meta = meta.copy() ret_objects = result['objects'] while meta['next']: # parse out url bits for constructing the new api req next_url = urlparse(meta['next']) # ugh...need to find the word after 'api/' in the next URL to # get the resource endpoint name; not sure how to make this better next_endpoint = next_url.path.strip('/').split('/')[-1] next_params = {k: v[0] for k, v in parse_qs(next_url.query).items()} result = getattr(api, next_endpoint).get(**next_params) ret_objects.extend(result['objects']) meta = result['meta'] # fix meta up to not tell lies ret_meta['total_count'] = len(ret_objects) ret_meta['next'] = None ret_meta['limit'] = ret_meta['total_count'] return {'meta': ret_meta, 'objects': ret_objects}
def _parse_request(self): self._buffer.seek(0) request_line = self._buffer.readline() # e.g. GET /v?v='http://www.nbc.com' HTTP/1.1 match = re.match("(GET|POST|PUT|DELETE|HEAD) (.+) .+", request_line) if match is None: raise ValueError("Invalid HTTP request %s" % request_line) self.method = match.group(1) # e.g. GET whole_url = match.group(2) # e.g. /?v='http://www.nbc.com' parsed_whole_url = urlparse.urlparse(whole_url) params = urlparse.parse_qs(parsed_whole_url.query) url_param = params.get('v', None) # e.g. http://www.nbc.com if url_param is None or len(url_param) == 0: logger.warning("Skipping %s" % whole_url.strip()) return # raise ValueError("No request url received in HTTP request") self.url = url_param[0] for line in self._buffer.readlines(): match = re.match("(.+):\\s*(.+)", line.strip()) if match is None: continue self.headers[match.group(1)] = match.group(2) return True
def _get_remote(self, url, components): parsed_url = urlparse(url) if 'token' not in parse_qs(parsed_url.query): components = set(components.split('+')) & {'db', 'media'} content = 'data' if components == {'db', 'media' } else components.pop() create_path = reverse('create_archive', kwargs={'data_type': content}) token = signer.sign(get_random_string()) query = 'token={}{}'.format( token, '&' + parsed_url.query if parsed_url.query else '') create_url = urlunparse([ parsed_url.scheme, parsed_url.netloc, create_path, '', query, '' ]) resp = self._get_url(create_url) archive_path = reverse('download_archive', kwargs={'data_type': content}) url = urlunparse([ parsed_url.scheme, parsed_url.netloc, archive_path, '', query, '' ]) resp = self._get_url(url, params={'token': ''}) return io.BytesIO(resp.content)
def test_fxa_login_url_requiring_two_factor_auth(): path = u'/en-US/addons/abp/?source=ddg' request = RequestFactory().get(path) request.session = {'fxa_state': 'myfxastate'} raw_url = utils.fxa_login_url( config=FXA_CONFIG['default'], state=request.session['fxa_state'], next_path=path, action='signin', force_two_factor=True) url = urlparse(raw_url) base = u'{scheme}://{netloc}{path}'.format( scheme=url.scheme, netloc=url.netloc, path=url.path) assert base == 'https://accounts.firefox.com/oauth/authorization' query = parse_qs(url.query) next_path = urlsafe_b64encode(path.encode('utf-8')).rstrip(b'=') assert query == { 'acr_values': ['AAL2'], 'action': ['signin'], 'client_id': ['foo'], 'redirect_url': ['https://testserver/fxa'], 'scope': ['profile'], 'state': ['myfxastate:{next_path}'.format( next_path=force_text(next_path))], }
def request(self, url, method="GET", body=None, headers={}): fail_state = { 'status':'400' }, "{}" parsed = urlparse(url) options = parse_qs(parsed.query) fn_name = str(active_call) if fn_name == 'get_authorize_login_url': return { 'status': '200', 'content-location':'http://example.com/redirect/login' }, None if not 'access_token' in options and not 'client_id' in options: fn_name += '_unauthorized' if 'self' in url and not 'access_token' in options: fn_name += '_no_auth_user' fl = open('fixtures/%s.json' % fn_name) content = fl.read() json_content = simplejson.loads(content) status = json_content['meta']['code'] fl.close() return { 'status': status }, content
def _parse_season(self, row, download_url, title): """Parse the torrent's detail page and return the season pack title.""" details_url = row.find('span').find_next( title='View torrent').get('href') torrent_id = parse_qs(download_url).get('id') if not all([details_url, torrent_id]): log.debug("Couldn't parse season pack details page for title: {0}", title) return title # Take a break before querying the provider again time.sleep(0.5) response = self.session.get(urljoin(self.url, details_url)) if not response or not response.text: log.debug("Couldn't open season pack details page for title: {0}", title) return title with BS4Parser(response.text, 'html5lib') as html: torrent_table = html.find('table', class_='torrent_table') torrent_row = torrent_table.find( 'tr', id='torrent_{0}'.format( torrent_id[0])) if torrent_table else None if not torrent_row: log.debug("Couldn't find season pack details for title: {0}", title) return title # Strip leading and trailing slash season_title = torrent_row.find('div', class_='filelist_path') if not season_title or not season_title.get_text(): log.debug("Couldn't parse season pack title for: {0}", title) return title return season_title.get_text(strip=True).strip('/')
def test_submit_service_result(self): with requests_mock.Mocker() as m: m.register_uri(requests_mock.POST, self.ws_arbiter_url + "/push_check_result") check_result = { "return_code": "0", "output": "TEST OUTPUT", "time_stamp": "1409149234" } response = self.app.post_json( "/v1/hosts/bogus-router/services/service-example/results", params=check_result ) self.assertEqual(response.status_int, 204) self.assertEqual( urllib_parse.parse_qs(m.last_request.body), { u'output': [u'TEST OUTPUT'], u'return_code': [u'0'], u'service_description': [u'service-example'], u'host_name': [u'bogus-router'], u'time_stamp': [u'1409149234'] } )
def test_url_simple(self): connection = Mock() connection.call.return_value = (None, {'start': 0, 'total_size': 0}) page = Page(connection, '/some-path', None) built_qs = parse_qs(urlsplit(page._build_url()).query) self.assertEqual( built_qs, dict(count=[str(DEFAULT_PAGE_ITEM_COUNT)], page=["1"]))
def __init(self, payment): """Initiate the given transaction with Paynow Args: payment (Payment): The payment object with details about transaction Returns: InitResponse: An object with misc information about the initiated transaction i.e redirect url (if available), status of initiation etc (see `InitResponse` declaration above) """ if payment.total() <= 0: raise ValueError('Transaction total cannot be less than 1') # Build up the object data = self.__build(payment) # Save response from Paynow response = requests.post(self.URL_INITIATE_TRANSACTION, data=data) # Reconstruct the response into key-value pairs response_object = self.__rebuild_response(parse_qs(response.text)) # If an error was encountered return a new InitResponse object without validating hash since hash is not # generated for error responses if str(response_object['status']).lower() == 'error': return InitResponse(response_object) # Verify the hash from Paynow with the locally generated one if not self.__verify_hash(response_object, self.integration_key): raise HashMismatchException("Hashes do not match") # Create a new InitResponse object object passing in the data from Paynow return InitResponse(response_object)
def _parse_season(self, row, download_url, title): """Parse the torrent's detail page and return the season pack title.""" details_url = row.find('span').find_next(title='View torrent').get('href') torrent_id = parse_qs(download_url).get('id') if not all([details_url, torrent_id]): log.debug("Couldn't parse season pack details page for title: {0}", title) return title # Take a break before querying the provider again time.sleep(0.5) response = self.session.get(urljoin(self.url, details_url)) if not response or not response.text: log.debug("Couldn't open season pack details page for title: {0}", title) return title with BS4Parser(response.text, 'html5lib') as html: torrent_table = html.find('table', class_='torrent_table') torrent_row = torrent_table.find('tr', id='torrent_{0}'.format(torrent_id[0])) if not torrent_row: log.debug("Couldn't find season pack details for title: {0}", title) return title # Strip leading and trailing slash season_title = torrent_row.find('div', class_='filelist_path') if not season_title or not season_title.get_text(): log.debug("Couldn't parse season pack title for: {0}", title) return title return season_title.get_text(strip=True).strip('/')
def execute(self, subscription, messages, **kwargs): subscriber = urllib_parse.urlparse(subscription['subscriber']) params = urllib_parse.parse_qs(subscriber.query) params = dict((k.lower(), v) for k, v in params.items()) conf_n = kwargs.get('conf').notification try: for message in messages: # Send confirmation email to subscriber. if (message.get('Message_Type') == MessageType.SubscriptionConfirmation.name): content = conf_n.subscription_confirmation_email_template msg = self._make_confirmation_email(content['body'], subscription, message, conf_n) msg["to"] = subscriber.path msg["from"] = content['sender'] msg["subject"] = content['topic'] elif (message.get('Message_Type') == MessageType.UnsubscribeConfirmation.name): content = conf_n.unsubscribe_confirmation_email_template msg = self._make_confirmation_email(content['body'], subscription, message, conf_n) msg["to"] = subscriber.path msg["from"] = content['sender'] msg["subject"] = content['topic'] else: # NOTE(Eva-i): Unfortunately this will add 'queue_name' key # to our original messages(dicts) which will be later # consumed in the storage controller. It seems safe though. message['queue_name'] = subscription['source'] msg = text.MIMEText(json.dumps(message)) msg["to"] = subscriber.path msg["from"] = subscription['options'].get('from', '') subject_opt = subscription['options'].get('subject', '') msg["subject"] = params.get('subject', subject_opt) if conf_n.smtp_mode == 'third_part': p = subprocess.Popen(conf_n.smtp_command.split(' '), stdin=subprocess.PIPE) p.communicate(msg.as_string()) elif conf_n.smtp_mode == 'self_local': sender = smtplib.SMTP_SSL(conf_n.smtp_host, conf_n.smtp_port) sender.set_debuglevel(1) sender.ehlo(conf_n.smtp_host) try: sender.login(conf_n.smtp_user_name, conf_n.smtp_user_password) except smtplib.SMTPException: LOG.error("Failed to connect to the SMTP service") continue sender.sendmail(msg['from'], msg['to'], msg.as_string()) LOG.debug("Send mail successfully: %s", msg.as_string()) except OSError as err: LOG.exception('Failed to create process for sendmail, ' 'because %s.', str(err)) except Exception as exc: LOG.exception('Failed to send email because %s.', str(exc))
def parse_taobao_id_from_url(url): url_obj = urllib_parse.urlparse(url) query_dict = urllib_parse.parse_qs(url_obj.query) accept_keys = ('id', 'item_id') for key in accept_keys: if key in query_dict: return query_dict[key][0] return None
def test_unicode_next_path(): path = u'/en-US/føø/bãr' request = RequestFactory().get(path) request.session = {} url = utils.default_fxa_login_url(request) state = parse_qs(urlparse(url).query)['state'][0] next_path = urlsafe_b64decode(state.split(':')[1] + '===') assert next_path.decode('utf-8') == path
def _get_params_from_url(url): """ Given a URL, return a dict of parameters and their values. If a parameter appears more than once all but the first value will be lost. """ parsed = urlparse.urlparse(url) params = urlparse.parse_qs(parsed.query, keep_blank_values=True) return dict((key, vals[0]) for key, vals in params.iteritems())
def test_short_desc(self): bz = Bugzilla('http://localhost/bugzilla') url = bz.make_url(self.test_run, self.case_run_1, 1) url_parts = urlparse(url) self.assertEqual( 'Test case failure: {}'.format(self.case_run_1.case.summary), parse_qs(url_parts.query)['short_desc'][0])
def test_url_simple(self): connection = Mock() connection.call.return_value = (None, {'start': 0, 'total_size': 0}) page = Page(connection, '/some-path', None) built_qs = parse_qs(urlsplit(page._build_url()).query) self.assertEqual(built_qs, dict( count=[str(DEFAULT_PAGE_ITEM_COUNT)], page=["1"]))
def url_distance(preprocessor, url1, url2): url1 = urlparse(url1) url2 = urlparse(url2) process_fn = lambda s: preprocessor(unquote(s)) path1 = map(process_fn, url1.path.strip('/').split('/')) path2 = map(process_fn, url2.path.strip('/').split('/')) path_distance = levenshtein_array(path1, path2) query_distance = dict_distance(preprocessor, parse_qs(url1.query, True), parse_qs(url2.query, True)) domain_distance = 4 * levenshtein_array((url1.hostname or '').split('.'), (url2.hostname or '').split('.')) return (domain_distance + path_distance + query_distance + (url1.fragment != url2.fragment))
def _validate_auth_request_url(url): parsed_url = urllib_parse.urlparse(url) params = urllib_parse.parse_qs(parsed_url.query) assert params.get("prompt") == ["select_account"], "Auth code request doesn't specify 'prompt=select_account'." # when used as a Mock's side_effect, this method's return value is the Mock's return value # (the real webbrowser.open returns a bool) return True
def testQueryRemapping(self): method_config = base_api.ApiMethodInfo( request_type_name="MessageWithRemappings", query_params=["remapped_field", "enum_field"] ) request = MessageWithRemappings(str_field="foo", enum_field=MessageWithRemappings.AnEnum.value_one) http_request = FakeService().PrepareHttpRequest(method_config, request) result_params = urllib_parse.parse_qs(urllib_parse.urlparse(http_request.url).query) expected_params = {"enum_field": "ONE%2FTWO", "remapped_field": "foo"} self.assertTrue(expected_params, result_params)
def __call__(self, request): if request.method == "POST": data = urllib_parse.parse_qs(request.body) data["username"] = [self.username] data["password"] = [self.password] request.prepare_body(data, None) return request
def episode(self, url, imdb, tvdb, title, premiered, season, episode): try: if url == None: return url = urllib_parse.parse_qs(url) url = dict([(i, url[i][0]) if url[i] else (i, '') for i in url]) url['imdb'], url['title'], url['premiered'], url['season'], url['episode'] = imdb, title, premiered, season, episode url = urllib_parse.urlencode(url) return url except: return None
def assertGetCallMatches(self, called_url, expected_path, params_dict): parsed_url = urlparse(called_url) query_dict = parse_qs(parsed_url.query) self.assertEqual(parsed_url.scheme, 'https') self.assertEqual(parsed_url.netloc, 'api.meetup.com') self.assertEqual(parsed_url.path, expected_path) self.assertDictEqual(params_dict, query_dict)
def callback(headers, params, request): status = 500 body = "" parsed = urlparse(request.url) params.update(parse_qs(parsed.query)) if not params: status = 200 body = """ <resultlist state="2c371787daedadb61dfd5fd8411ac6c2"> <result project="Some:Project" repository="SLE_15" arch="x86_64" code="published" state="published"> <status package="mypackage" code="disabled" /> <status package="anotherpackage" code="disabled" /> </result> <result project="Some:Project" repository="SLE_12_SP3" arch="x86_64" code="published" state="published"> <status package="mypackage" code="succeeded" /> <status package="anotherpackage" code="disabled" /> </result> <result project="Some:Project" repository="SLE_12_SP2" arch="x86_64" code="published" state="published"> <status package="mypackage" code="disabled" /> <status package="anotherpackage" code="disabled" /> </result> </resultlist> """ elif params.get("package", []): if params["package"][0] in ["mypackage", "anotherpackage"]: status = 200 body = """ <resultlist state="110f481609293ee149e28bbaced3b1b9"> <result project="Some:Project" repository="SLE_15" arch="x86_64" code="published" state="published"> <status package="{pkg}" code="disabled" /> </result> <result project="Some:Project" repository="SLE_12_SP3" arch="x86_64" code="published" state="published"> <status package="{pkg}" code="succeeded" /> </result> <result project="Some:Project" repository="SLE_12_SP2" arch="x86_64" code="published" state="published"> <status package="{pkg}" code="disabled" /> </result> </resultlist> """.format(pkg=params["package"][0]) else: status = 404 body = """ <status code="404"> <summary>unknown package '{pkg}'</summary> <details>404 unknown package '{pkg}'</details> </status> """.format(pkg=params["package"][0]) return status, headers, body
def __init__(self, url, **kwa): self.url = url urldata = urlparse.urlparse(url, **kwa) for key in self._components: val = getattr(urldata, key) setattr(self, key, val) self.query_str = urldata.query self.queryl = urlparse.parse_qs(urldata.query) self.query = dict(urlparse.parse_qsl(urldata.query))
def _custom_request_query_matcher(cls, r1, r2): """ Ensure method, path, and query parameters match. """ from six.moves.urllib_parse import urlparse, parse_qs # pylint: disable=import-error,relative-import url1 = urlparse(r1.uri) url2 = urlparse(r2.uri) q1 = parse_qs(url1.query) q2 = parse_qs(url2.query) shared_keys = set(q1.keys()).intersection(set(q2.keys())) if len(shared_keys) != len(q1) or len(shared_keys) != len(q2): return False for key in shared_keys: if q1[key][0].lower() != q2[key][0].lower(): return False return True
def test_redirect_uri(): """Test redirect uri.""" app = setup_app() with app.test_client() as client: # Test redirect resp = client.get( url_for("invenio_oauthclient.login", remote_app='test', next='http://invenio-software.org') ) assert resp.status_code == 302 # Verify parameters params = parse_qs(urlparse(resp.location).query) assert params['response_type'] == ['code'] assert params['client_id'] == ['testid'] assert params['redirect_uri'] assert params['state'] # Verify next parameter in state token does not allow blanco redirects state = serializer.loads(params['state'][0]) assert state['next'] is None # Assert redirect uri does not have any parameters. params = parse_qs(urlparse(params['redirect_uri'][0]).query) assert params == {} # Assert that local redirects are allowed test_urls = [ '/search', url_for('invenio_oauthclient.disconnect', remote_app='test', _external=True) ] for url in test_urls: resp = client.get( url_for("invenio_oauthclient.login", remote_app='test', next=url) ) assert resp.status_code == 302 state = serializer.loads( parse_qs(urlparse(resp.location).query)['state'][0] ) assert url == state['next']
def test_url_with_qs(self): connection = Mock() connection.call.return_value = (None, {'start': 0, 'total_size': 0}) page = Page(connection, '/some-path?with=a&query=string', None) built_qs = parse_qs(urlsplit(page._build_url()).query) self.assertEqual(built_qs, { "with": ["a"], "query": ["string"], "count": [str(DEFAULT_PAGE_ITEM_COUNT)], "page": ["1"], })
def testQueryRemapping(self): method_config = base_api.ApiMethodInfo( request_type_name='MessageWithRemappings', query_params=['remapped_field', 'enum_field']) request = MessageWithRemappings( str_field='foo', enum_field=MessageWithRemappings.AnEnum.value_one) http_request = FakeService().PrepareHttpRequest(method_config, request) result_params = urllib_parse.parse_qs( urllib_parse.urlparse(http_request.url).query) expected_params = {'enum_field': 'ONE%2FTWO', 'remapped_field': 'foo'} self.assertTrue(expected_params, result_params)
def test_login(self): """Test ORCID login.""" resp = self.client.get(url_for("oauthclient.login", remote_app="orcid", next="/someurl/")) self.assertStatus(resp, 302) params = parse_qs(urlparse(resp.location).query) self.assertEqual(params["response_type"], ["code"]) self.assertEqual(params["show_login"], ["true"]) self.assertEqual(params["scope"], ["/authenticate"]) assert params["redirect_uri"] assert params["client_id"] assert params["state"]
def test_manage_fxa_link(): user = mock.MagicMock(email='*****@*****.**', fxa_id='abcd1234') link = urlparse(manage_fxa_link({'user': user})) url = '{scheme}://{netloc}{path}'.format( scheme=link.scheme, netloc=link.netloc, path=link.path) assert url == 'https://stable.dev.lcip.org/settings' query = parse_qs(link.query) assert query == { 'uid': ['abcd1234'], 'email': ['*****@*****.**'], 'entrypoint': ['addons'], }
def mangle_url_m(url, include=None, exclude=None, add=None): """ Multivalue-version of the `mangle_url`; works with dict of lists only (`param -> [value1, …]`); sorts the resulting query """ url = to_bytes(url) url_parts = urlparse.urlparse(url) ## NOTE: the order of the fields is still lost. query = urlparse.parse_qs(url_parts.query, keep_blank_values=1) query_new = mangle_dict(query, include=include, exclude=exclude, add=add) query_new = [(k, v) for k, vl in query_new.items() for v in vl] query_new = [(to_bytes(k), to_bytes(v)) for k, v in query_new] query_new = sorted(query_new) # make the order stable since it's lost anyway return url_replace(url, query=query_new)
def test_redirect_uri(self): from invenio.modules.oauthclient.views.client import serializer # Test redirect resp = self.client.get( url_for("oauthclient.login", remote_app='test', next='http://invenio-software.org') ) self.assertStatus(resp, 302) # Verify parameters params = parse_qs(urlparse(resp.location).query) self.assertEqual(params['response_type'], ['code']) self.assertEqual(params['client_id'], ['testid']) assert params['redirect_uri'] assert params['state'] # Verify next parameter in state token does not allow blanco redirects state = serializer.loads(params['state'][0]) self.assertIsNone(state['next']) # Assert redirect uri does not have any parameters. params = parse_qs(urlparse(params['redirect_uri'][0]).query) self.assertEqual(params, {}) # Assert that local redirects are allowed test_urls = [ '/search', url_for('oauthclient.disconnect', remote_app='test', _external=True) ] for url in test_urls: resp = self.client.get( url_for("oauthclient.login", remote_app='test', next=url) ) self.assertStatus(resp, 302) state = serializer.loads( parse_qs(urlparse(resp.location).query)['state'][0] ) self.assertEqual(url, state['next'])
def _generate_helper(self, response_type=None, response_disposition=None, generation=None): endpoint = 'http://api.example.com' resource = '/name/path' credentials = _make_credentials( signing=True, signer_email='*****@*****.**') credentials.sign_bytes.return_value = b'DEADBEEF' signed = base64.b64encode(credentials.sign_bytes.return_value) signed = signed.decode('ascii') expiration = 1000 url = self._call_fut( credentials, resource, expiration, api_access_endpoint=endpoint, response_type=response_type, response_disposition=response_disposition, generation=generation, ) # Check the mock was called. string_to_sign = '\n'.join([ 'GET', '', '', str(expiration), resource, ]) credentials.sign_bytes.assert_called_once_with(string_to_sign) scheme, netloc, path, qs, frag = urllib_parse.urlsplit(url) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'api.example.com') self.assertEqual(path, resource) self.assertEqual(frag, '') # Check the URL parameters. params = urllib_parse.parse_qs(qs) expected_params = { 'GoogleAccessId': [credentials.signer_email], 'Expires': [str(expiration)], 'Signature': [signed], } if response_type is not None: expected_params['response-content-type'] = [response_type] if response_disposition is not None: expected_params['response-content-disposition'] = [ response_disposition] if generation is not None: expected_params['generation'] = [generation] self.assertEqual(params, expected_params)
def apply_to_request(self, req, service): parsed = urlparse.urlparse(req.url) if req.method == 'POST': # This is probably going to break when given multipart data. params = urlparse.parse_qs(req.body or '', keep_blank_values=True) else: params = urlparse.parse_qs(parsed.query, keep_blank_values=True) params = dict((key, vals[0]) for key, vals in params.iteritems()) params['AWSAccessKeyId'] = self.args['key_id'] params['SignatureVersion'] = 2 params['SignatureMethod'] = 'HmacSHA256' params['Timestamp'] = time.strftime(ISO8601, time.gmtime()) if self.args.get('security_token'): params['SecurityToken'] = self.args['security_token'] # Needed for retries so old signatures aren't included in to_sign params.pop('Signature', None) to_sign = '{method}\n{host}\n{path}\n'.format( method=req.method, host=parsed.netloc.lower(), path=(parsed.path or '/')) quoted_params = [] for key in sorted(params): val = six.text_type(params[key]) quoted_params.append(urlparse.quote(key, safe='') + '=' + urlparse.quote(val, safe='-_~')) query_string = '&'.join(quoted_params) to_sign += query_string # Redact passwords redacted_to_sign = re.sub('assword=[^&]*', 'assword=<redacted>', to_sign) self.log.debug('string to sign: %s', repr(redacted_to_sign)) signature = self.sign_string(to_sign) self.log.debug('b64-encoded signature: %s', signature) params['Signature'] = signature if req.method == 'POST': req.prepare_body(params, {}) else: req.prepare_url(_remove_params_from_url(req.url), params) return req