def url_distance(preprocessor, url1, url2):
url1 = urlparse(url1)
url2 = urlparse(url2)
process_fn = lambda s: preprocessor(unquote(s))
path1 = map(process_fn, url1.path.strip('/').split('/'))
path2 = map(process_fn, url2.path.strip('/').split('/'))
path_distance = levenshtein_array(path1, path2)
query_distance = dict_distance(preprocessor,
parse_qs(url1.query, True),
parse_qs(url2.query, True)
)
domain_distance = 4 * levenshtein_array(
(url1.hostname or '').split('.'),
(url2.hostname or '').split('.')
)
return (
domain_distance +
path_distance +
query_distance +
(url1.fragment != url2.fragment)
)
def assert_url_equal(url, expected, compare_host=False):
"""Compare url paths and query strings."""
parsed = urlparse(six.text_type(url))
parsed_expected = urlparse(six.text_type(expected))
compare_url_part(parsed.path, parsed_expected.path)
compare_url_part(parse_qs(parsed.query), parse_qs(parsed_expected.query))
if compare_host:
compare_url_part(parsed.netloc, parsed_expected.netloc)
def _parse_request(self):
self._buffer.seek(0)
request_line = self._buffer.readline(
) # e.g. GET /v?v='http://www.nbc.com' HTTP/1.1
match = re.match("(GET|POST|PUT|DELETE|HEAD) (.+) .+", request_line)
if match is None:
raise ValueError("Invalid HTTP request %s" % request_line)
self.method = match.group(1) # e.g. GET
whole_url = match.group(2) # e.g. /?v='http://www.nbc.com'
parsed_whole_url = urlparse.urlparse(whole_url)
params = urlparse.parse_qs(parsed_whole_url.query)
url_param = params.get('v', None) # e.g. http://www.nbc.com
if url_param is None or len(url_param) == 0:
logger.warning("Skipping %s" % whole_url.strip())
return
# raise ValueError("No request url received in HTTP request")
self.url = url_param[0]
for line in self._buffer.readlines():
match = re.match("(.+):\\s*(.+)", line.strip())
if match is None:
continue
self.headers[match.group(1)] = match.group(2)
return True
def get(self, url):
images_dir = os.path.join(self.tm_env.images_dir, TAR_DIR)
fs.mkdir_safe(images_dir)
image = urllib_parse.urlparse(url)
sha256 = urllib_parse.parse_qs(image.query).get('sha256', None)
with tempfile.NamedTemporaryFile(dir=images_dir,
delete=False,
prefix='.tmp') as temp:
if image.scheme == 'http':
_download(url, temp)
else:
_copy(image.path, temp)
if not tarfile.is_tarfile(temp.name):
_LOGGER.error('File %r is not a tar file.', url)
raise Exception('File {0} is not a tar file.'.format(url))
new_sha256 = _sha256sum(temp.name)
if sha256 is not None and sha256[0] != new_sha256:
_LOGGER.error('Hash does not match %r - %r', sha256[0], new_sha256)
raise Exception('Hash of {0} does not match {1}.'.format(
new_sha256, url))
# TODO: rename tar file to sha256 to allow for caching.
return TarImage(self.tm_env, temp.name)
def parse_grip_uri(uri):
parsed = urlparse(uri)
# HACK: work around '+' character in base64-encoded values
query = parsed.query.replace('+', '%2B')
params = parse_qs(query)
iss = None
key = None
if 'iss' in params:
iss = params['iss'][0]
del params['iss']
if 'key' in params:
key = params['key'][0]
del params['key']
if key is not None and key.startswith('base64:'):
key = b64decode(key[7:])
qs = urlencode(params, True)
path = parsed.path
if path.endswith('/'):
path = path[:-1]
control_uri = parsed.scheme + '://' + parsed.netloc + path
if qs:
control_uri += '?' + qs
out = {'control_uri': control_uri}
if iss:
out['control_iss'] = iss
if key:
out['key'] = key
return out
def test_submit_service_result(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
self.ws_arbiter_url + "/push_check_result")
check_result = {
"return_code": "0",
"output": "TEST OUTPUT",
"time_stamp": "1409149234"
}
response = self.post_json(
"/v2/status/hosts/bogus-router/services/" +
"service-example/results",
params=check_result)
self.assertEqual(response.status_int, 204)
self.assertEqual(
urllib_parse.parse_qs(m.last_request.body), {
u'output': [u'TEST OUTPUT'],
u'return_code': [u'0'],
u'service_description': [u'service-example'],
u'host_name': [u'bogus-router'],
u'time_stamp': [u'1409149234']
})
def _build_url(self):
url = list(urlsplit(self._path))
qs = parse_qs(url[3])
qs["count"] = self._count
qs["page"] = self._page
url[3] = urlencode(qs, doseq=True)
return urlunsplit(url)
def depaginate(api, result):
"""Depaginate the first (or only) page of a paginated result"""
meta = result['meta']
if meta['next'] is None:
# No pages means we're done
return result
# make a copy of meta that we'll mess with and eventually return
# since we'll be chewing on the 'meta' object with every new GET
# same thing for objects, since we'll just be appending to it
# while we pull more records
ret_meta = meta.copy()
ret_objects = result['objects']
while meta['next']:
# parse out url bits for constructing the new api req
next_url = urlparse(meta['next'])
# ugh...need to find the word after 'api/' in the next URL to
# get the resource endpoint name; not sure how to make this better
next_endpoint = next_url.path.strip('/').split('/')[-1]
next_params = {k: v[0] for k, v in parse_qs(next_url.query).items()}
result = getattr(api, next_endpoint).get(**next_params)
ret_objects.extend(result['objects'])
meta = result['meta']
# fix meta up to not tell lies
ret_meta['total_count'] = len(ret_objects)
ret_meta['next'] = None
ret_meta['limit'] = ret_meta['total_count']
return {
'meta': ret_meta,
'objects': ret_objects
}
def _build_url(self):
url = list(urlsplit(self._path))
qs = parse_qs(url[3])
qs["count"] = self._count
qs["page"] = self._page
url[3] = urlencode(qs, doseq=True)
return urlunsplit(url)
def on_request(self, env, start_response):
method = env['REQUEST_METHOD'].upper()
path = env['PATH_INFO'].lower().strip('/')
if method not in self.method_handlers or path not in self.method_handlers[method]:
start_response('404 Not Found', [])
return
if method == 'GET':
request = urlparse.parse_qs(env.get('QUERY_STRING', ''), True)
for query in request:
if len(request[query]) == 1:
request[query] = request[query][0]
else:
request = json.loads(env.get('wsgi.input', '{}'))
response = self.method_handlers[method][path](request)
if type(response) == tuple:
response, options = response
else:
options = ResponseOptions()
start_response('%d %s' % (options.status, options.reason), [('Content-Type', options.content_type)])
if options.send_json:
yield json.dumps(response)
else:
yield response
def on_request(self, env, start_response):
method = env['REQUEST_METHOD'].upper()
path = env['PATH_INFO'].lower().strip('/')
# If the API request is proxied, the path comes as a full url for some reason
if '://' in path:
path = urlparse.urlparse(path).path.strip('/')
if method not in self.method_handlers or path not in self.method_handlers[method]:
start_response('404 Not Found', [])
return
if method == 'GET':
request = urlparse.parse_qs(env.get('QUERY_STRING', ''), True)
for query in request:
if len(request[query]) == 1:
request[query] = request[query][0]
else:
inp = env.get('wsgi.input', None)
request = json.loads(inp.read()) if inp is not None else {}
response = self.method_handlers[method][path](request)
if type(response) == tuple:
response, options = response
else:
options = ResponseOptions()
start_response('%d %s' % (options.status, options.reason), [('Content-Type', options.content_type)])
if options.send_json:
return [json.dumps(response)]
else:
return [response]
def parse_url(url, warning=True):
"""Parse URLs especially for Google Drive links.
file_id: ID of file on Google Drive.
is_download_link: Flag if it is download link of Google Drive.
"""
parsed = urllib_parse.urlparse(url)
query = urllib_parse.parse_qs(parsed.query)
is_gdrive = parsed.hostname == "drive.google.com"
is_download_link = parsed.path.endswith("/uc")
file_id = None
if is_gdrive and "id" in query:
file_ids = query["id"]
if len(file_ids) == 1:
file_id = file_ids[0]
match = re.match(r"^/file/d/(.*?)/view$", parsed.path)
if match:
file_id = match.groups()[0]
if is_gdrive and not is_download_link:
warnings.warn(
"You specified Google Drive Link but it is not the correct link "
"to download the file. Maybe you should try: {url}".format(
url="https://drive.google.com/uc?id={}".format(file_id)))
return file_id, is_download_link
def test_token_refresh_store_expired(self):
expiration = datetime.datetime.utcnow() - datetime.timedelta(minutes=15)
credentials = self._create_test_credentials(expiration=expiration)
storage = file_module.Storage(FILENAME)
storage.put(credentials)
credentials = storage.get()
new_cred = copy.copy(credentials)
new_cred.access_token = "bar"
storage.put(new_cred)
access_token = "1/3w"
token_response = {"access_token": access_token, "expires_in": 3600}
response_content = json.dumps(token_response).encode("utf-8")
http = http_mock.HttpMock(data=response_content)
credentials._refresh(http)
self.assertEquals(credentials.access_token, access_token)
# Verify mocks.
self.assertEqual(http.requests, 1)
self.assertEqual(http.uri, credentials.token_uri)
self.assertEqual(http.method, "POST")
expected_body = {
"grant_type": ["refresh_token"],
"client_id": [credentials.client_id],
"client_secret": [credentials.client_secret],
"refresh_token": [credentials.refresh_token],
}
self.assertEqual(urllib_parse.parse_qs(http.body), expected_body)
expected_headers = {"content-type": "application/x-www-form-urlencoded", "user-agent": credentials.user_agent}
self.assertEqual(http.headers, expected_headers)
def assert_url(self, expected_url, url):
"""Check if two URL are same
Assertions are called inside this this method. If anything is
different, it will fail immediately.
:param str expected_url: expected URL compare.
:param str url: the URL to check if it is same as the expected URL.
"""
url = urlparse(url)
expected_url = urlparse(expected_url)
self.assertEqual(expected_url.scheme, url.scheme)
self.assertEqual(expected_url.netloc, url.netloc)
self.assertEqual(expected_url.path, url.path)
self.assertEqual(parse_qs(expected_url.query), parse_qs(url.query))
def execute(self, subscription, messages, **kwargs):
subscriber = urllib_parse.urlparse(subscription['subscriber'])
params = urllib_parse.parse_qs(subscriber.query)
params = dict((k.lower(), v) for k, v in params.items())
conf = kwargs.get('conf')
try:
for message in messages:
p = subprocess.Popen(conf.notification.smtp_command.split(' '),
stdin=subprocess.PIPE)
# NOTE(Eva-i): Unfortunately this will add 'queue_name' key to
# our original messages(dicts) which will be later consumed in
# the storage controller. It seems safe though.
message['queue_name'] = subscription['source']
msg = text.MIMEText(json.dumps(message))
msg["to"] = subscriber.path
msg["from"] = subscription['options'].get('from', '')
subject_opt = subscription['options'].get('subject', '')
msg["subject"] = params.get('subject', subject_opt)
p.communicate(msg.as_string())
except OSError as err:
LOG.exception(
_LE('Failed to create process for sendmail, '
'because %s.') % str(err))
except Exception as exc:
LOG.exception(_LE('Failed to send email because %s.') % str(exc))
def parse_url(url, warning=True):
"""Parse URLs especially for Google Drive links.
file_id: ID of file on Google Drive.
is_download_link: Flag if it is download link of Google Drive.
"""
parsed = urllib_parse.urlparse(url)
query = urllib_parse.parse_qs(parsed.query)
is_gdrive = parsed.hostname in ["drive.google.com", "docs.google.com"]
is_download_link = parsed.path.endswith("/uc")
if not is_gdrive:
return is_gdrive, is_download_link
file_id = None
if "id" in query:
file_ids = query["id"]
if len(file_ids) == 1:
file_id = file_ids[0]
else:
patterns = [r"^/file/d/(.*?)/view$", r"^/presentation/d/(.*?)/edit$"]
for pattern in patterns:
match = re.match(pattern, parsed.path)
if match:
file_id = match.groups()[0]
break
if warning and not is_download_link:
warnings.warn(
"You specified Google Drive Link but it is not the correct link "
"to download the file. Maybe you should try: {url}".format(
url="https://drive.google.com/uc?id={}".format(file_id)))
return file_id, is_download_link
def test_comment_with_plain_text(self):
bz = Bugzilla('http://localhost/bugzilla')
url = bz.make_url(self.test_run, self.case_run_1, 1)
url_parts = urlparse(url)
expected_comment = '''Filed from caserun (INSERT URL HERE)
Version-Release number of selected component (if applicable):
{build_name}
Steps to Reproduce:
{setup}
{action}
Actual results:
#FIXME
Expected results:
{effect}
'''.format(build_name=self.case_run_1.build.name,
setup='setup',
action='action',
effect='effect')
self.assertEqual(expected_comment,
parse_qs(url_parts.query)['comment'][0])
def callback(headers, params, request):
status = 200
body = """
<directory name="_project" rev="10" vrev=""
srcmd5="4adff519b4e14cf206bcc7bdce40a73c">
<entry name="_config" md5="11cfe7fdb2591b55435b312c156e17fe"
size="1818" mtime="1542617696" />
</directory>
"""
parsed = urlparse(request.url)
params.update(parse_qs(parsed.query))
if params.get("meta", ['0']) == ['1']:
status = 200
body = """
<directory name="_project" rev="41" vrev=""
srcmd5="f8bf4cb0e1edea467e34462742ca82ae">
<entry name="_attribute"
md5="18fbe34f9ab146c9d73f7e5756f74d48" size="4172"
mtime="1543406498" />
<entry name="_meta" md5="9392aa5e748eb15087c4db55ac1cb7f3"
size="1875" mtime="1542635759" />
</directory>
"""
headers['request-id'] = '728d329e-0e86-11e4-a748-0c84dc037c13'
return status, headers, body
def test_login(app):
"""Test Keycloak login."""
keycloak_config = app.config["OAUTHCLIENT_REMOTE_APPS"]["keycloak"]
auth_url = keycloak_config["params"]["authorize_url"]
client = app.test_client()
resp = client.get(
url_for(
"invenio_oauthclient.login",
remote_app="keycloak",
next="/someurl/"
)
)
assert resp.status_code == 302
comps = urlparse(resp.location)
params = parse_qs(comps.query)
url = "{}://{}{}".format(comps.scheme, comps.netloc, comps.path)
assert url == auth_url
assert params["response_type"] == ["code"]
assert params["scope"] == ["openid"]
assert params["redirect_uri"]
assert params["client_id"]
assert params["state"]
def test_fxa_login_url_requiring_two_factor_auth():
path = u'/en-US/addons/abp/?source=ddg'
request = RequestFactory().get(path)
request.session = {'fxa_state': 'myfxastate'}
raw_url = utils.fxa_login_url(config=FXA_CONFIG['default'],
state=request.session['fxa_state'],
next_path=path,
action='signin',
force_two_factor=True)
url = urlparse(raw_url)
base = u'{scheme}://{netloc}{path}'.format(scheme=url.scheme,
netloc=url.netloc,
path=url.path)
assert base == 'https://accounts.firefox.com/oauth/authorization'
query = parse_qs(url.query)
next_path = urlsafe_b64encode(path.encode('utf-8')).rstrip(b'=')
assert query == {
'acr_values': ['AAL2'],
'action': ['signin'],
'client_id': ['foo'],
'redirect_url': ['https://testserver/fxa'],
'scope': ['profile'],
'state':
['myfxastate:{next_path}'.format(next_path=force_text(next_path))],
}
def depaginate(api, result):
"""Depaginate the first (or only) page of a paginated result"""
meta = result['meta']
if meta['next'] is None:
# No pages means we're done
return result
# make a copy of meta that we'll mess with and eventually return
# since we'll be chewing on the 'meta' object with every new GET
# same thing for objects, since we'll just be appending to it
# while we pull more records
ret_meta = meta.copy()
ret_objects = result['objects']
while meta['next']:
# parse out url bits for constructing the new api req
next_url = urlparse(meta['next'])
# ugh...need to find the word after 'api/' in the next URL to
# get the resource endpoint name; not sure how to make this better
next_endpoint = next_url.path.strip('/').split('/')[-1]
next_params = {k: v[0] for k, v in parse_qs(next_url.query).items()}
result = getattr(api, next_endpoint).get(**next_params)
ret_objects.extend(result['objects'])
meta = result['meta']
# fix meta up to not tell lies
ret_meta['total_count'] = len(ret_objects)
ret_meta['next'] = None
ret_meta['limit'] = ret_meta['total_count']
return {'meta': ret_meta, 'objects': ret_objects}
def _parse_request(self):
self._buffer.seek(0)
request_line = self._buffer.readline() # e.g. GET /v?v='http://www.nbc.com' HTTP/1.1
match = re.match("(GET|POST|PUT|DELETE|HEAD) (.+) .+", request_line)
if match is None:
raise ValueError("Invalid HTTP request %s" % request_line)
self.method = match.group(1) # e.g. GET
whole_url = match.group(2) # e.g. /?v='http://www.nbc.com'
parsed_whole_url = urlparse.urlparse(whole_url)
params = urlparse.parse_qs(parsed_whole_url.query)
url_param = params.get('v', None) # e.g. http://www.nbc.com
if url_param is None or len(url_param) == 0:
logger.warning("Skipping %s" % whole_url.strip())
return
# raise ValueError("No request url received in HTTP request")
self.url = url_param[0]
for line in self._buffer.readlines():
match = re.match("(.+):\\s*(.+)", line.strip())
if match is None:
continue
self.headers[match.group(1)] = match.group(2)
return True
def _get_remote(self, url, components):
parsed_url = urlparse(url)
if 'token' not in parse_qs(parsed_url.query):
components = set(components.split('+')) & {'db', 'media'}
content = 'data' if components == {'db', 'media'
} else components.pop()
create_path = reverse('create_archive',
kwargs={'data_type': content})
token = signer.sign(get_random_string())
query = 'token={}{}'.format(
token, '&' + parsed_url.query if parsed_url.query else '')
create_url = urlunparse([
parsed_url.scheme, parsed_url.netloc, create_path, '', query,
''
])
resp = self._get_url(create_url)
archive_path = reverse('download_archive',
kwargs={'data_type': content})
url = urlunparse([
parsed_url.scheme, parsed_url.netloc, archive_path, '', query,
''
])
resp = self._get_url(url, params={'token': ''})
return io.BytesIO(resp.content)
def test_fxa_login_url_requiring_two_factor_auth():
path = u'/en-US/addons/abp/?source=ddg'
request = RequestFactory().get(path)
request.session = {'fxa_state': 'myfxastate'}
raw_url = utils.fxa_login_url(
config=FXA_CONFIG['default'],
state=request.session['fxa_state'], next_path=path, action='signin',
force_two_factor=True)
url = urlparse(raw_url)
base = u'{scheme}://{netloc}{path}'.format(
scheme=url.scheme, netloc=url.netloc, path=url.path)
assert base == 'https://accounts.firefox.com/oauth/authorization'
query = parse_qs(url.query)
next_path = urlsafe_b64encode(path.encode('utf-8')).rstrip(b'=')
assert query == {
'acr_values': ['AAL2'],
'action': ['signin'],
'client_id': ['foo'],
'redirect_url': ['https://testserver/fxa'],
'scope': ['profile'],
'state': ['myfxastate:{next_path}'.format(
next_path=force_text(next_path))],
}
def request(self, url, method="GET", body=None, headers={}):
fail_state = {
'status':'400'
}, "{}"
parsed = urlparse(url)
options = parse_qs(parsed.query)
fn_name = str(active_call)
if fn_name == 'get_authorize_login_url':
return {
'status': '200',
'content-location':'http://example.com/redirect/login'
}, None
if not 'access_token' in options and not 'client_id' in options:
fn_name += '_unauthorized'
if 'self' in url and not 'access_token' in options:
fn_name += '_no_auth_user'
fl = open('fixtures/%s.json' % fn_name)
content = fl.read()
json_content = simplejson.loads(content)
status = json_content['meta']['code']
fl.close()
return {
'status': status
}, content
def _parse_season(self, row, download_url, title):
"""Parse the torrent's detail page and return the season pack title."""
details_url = row.find('span').find_next(
title='View torrent').get('href')
torrent_id = parse_qs(download_url).get('id')
if not all([details_url, torrent_id]):
log.debug("Couldn't parse season pack details page for title: {0}",
title)
return title
# Take a break before querying the provider again
time.sleep(0.5)
response = self.session.get(urljoin(self.url, details_url))
if not response or not response.text:
log.debug("Couldn't open season pack details page for title: {0}",
title)
return title
with BS4Parser(response.text, 'html5lib') as html:
torrent_table = html.find('table', class_='torrent_table')
torrent_row = torrent_table.find(
'tr', id='torrent_{0}'.format(
torrent_id[0])) if torrent_table else None
if not torrent_row:
log.debug("Couldn't find season pack details for title: {0}",
title)
return title
# Strip leading and trailing slash
season_title = torrent_row.find('div', class_='filelist_path')
if not season_title or not season_title.get_text():
log.debug("Couldn't parse season pack title for: {0}", title)
return title
return season_title.get_text(strip=True).strip('/')
def test_submit_service_result(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
self.ws_arbiter_url + "/push_check_result")
check_result = {
"return_code": "0",
"output": "TEST OUTPUT",
"time_stamp": "1409149234"
}
response = self.app.post_json(
"/v1/hosts/bogus-router/services/service-example/results",
params=check_result
)
self.assertEqual(response.status_int, 204)
self.assertEqual(
urllib_parse.parse_qs(m.last_request.body),
{
u'output': [u'TEST OUTPUT'],
u'return_code': [u'0'],
u'service_description': [u'service-example'],
u'host_name': [u'bogus-router'],
u'time_stamp': [u'1409149234']
}
)
def parse_grip_uri(uri):
parsed = urlparse(uri)
# HACK: work around '+' character in base64-encoded values
query = parsed.query.replace('+', '%2B')
params = parse_qs(query)
iss = None
key = None
if 'iss' in params:
iss = params['iss'][0]
del params['iss']
if 'key' in params:
key = params['key'][0]
del params['key']
if key is not None and key.startswith('base64:'):
key = b64decode(key[7:])
qs = urlencode(params, True)
path = parsed.path
if path.endswith('/'):
path = path[:-1]
control_uri = parsed.scheme + '://' + parsed.netloc + path
if qs:
control_uri += '?' + qs
out = {'control_uri': control_uri}
if iss:
out['control_iss'] = iss
if key:
out['key'] = key
return out
def test_url_simple(self):
connection = Mock()
connection.call.return_value = (None, {'start': 0, 'total_size': 0})
page = Page(connection, '/some-path', None)
built_qs = parse_qs(urlsplit(page._build_url()).query)
self.assertEqual(
built_qs, dict(count=[str(DEFAULT_PAGE_ITEM_COUNT)], page=["1"]))
def __init(self, payment):
"""Initiate the given transaction with Paynow
Args:
payment (Payment): The payment object with details about transaction
Returns:
InitResponse: An object with misc information about the initiated transaction i.e
redirect url (if available), status of initiation etc (see `InitResponse` declaration above)
"""
if payment.total() <= 0:
raise ValueError('Transaction total cannot be less than 1')
# Build up the object
data = self.__build(payment)
# Save response from Paynow
response = requests.post(self.URL_INITIATE_TRANSACTION, data=data)
# Reconstruct the response into key-value pairs
response_object = self.__rebuild_response(parse_qs(response.text))
# If an error was encountered return a new InitResponse object without validating hash since hash is not
# generated for error responses
if str(response_object['status']).lower() == 'error':
return InitResponse(response_object)
# Verify the hash from Paynow with the locally generated one
if not self.__verify_hash(response_object, self.integration_key):
raise HashMismatchException("Hashes do not match")
# Create a new InitResponse object object passing in the data from Paynow
return InitResponse(response_object)
def _parse_season(self, row, download_url, title):
"""Parse the torrent's detail page and return the season pack title."""
details_url = row.find('span').find_next(title='View torrent').get('href')
torrent_id = parse_qs(download_url).get('id')
if not all([details_url, torrent_id]):
log.debug("Couldn't parse season pack details page for title: {0}", title)
return title
# Take a break before querying the provider again
time.sleep(0.5)
response = self.session.get(urljoin(self.url, details_url))
if not response or not response.text:
log.debug("Couldn't open season pack details page for title: {0}", title)
return title
with BS4Parser(response.text, 'html5lib') as html:
torrent_table = html.find('table', class_='torrent_table')
torrent_row = torrent_table.find('tr', id='torrent_{0}'.format(torrent_id[0]))
if not torrent_row:
log.debug("Couldn't find season pack details for title: {0}", title)
return title
# Strip leading and trailing slash
season_title = torrent_row.find('div', class_='filelist_path')
if not season_title or not season_title.get_text():
log.debug("Couldn't parse season pack title for: {0}", title)
return title
return season_title.get_text(strip=True).strip('/')
def execute(self, subscription, messages, **kwargs):
subscriber = urllib_parse.urlparse(subscription['subscriber'])
params = urllib_parse.parse_qs(subscriber.query)
params = dict((k.lower(), v) for k, v in params.items())
conf_n = kwargs.get('conf').notification
try:
for message in messages:
# Send confirmation email to subscriber.
if (message.get('Message_Type') ==
MessageType.SubscriptionConfirmation.name):
content = conf_n.subscription_confirmation_email_template
msg = self._make_confirmation_email(content['body'],
subscription,
message, conf_n)
msg["to"] = subscriber.path
msg["from"] = content['sender']
msg["subject"] = content['topic']
elif (message.get('Message_Type') ==
MessageType.UnsubscribeConfirmation.name):
content = conf_n.unsubscribe_confirmation_email_template
msg = self._make_confirmation_email(content['body'],
subscription,
message, conf_n)
msg["to"] = subscriber.path
msg["from"] = content['sender']
msg["subject"] = content['topic']
else:
# NOTE(Eva-i): Unfortunately this will add 'queue_name' key
# to our original messages(dicts) which will be later
# consumed in the storage controller. It seems safe though.
message['queue_name'] = subscription['source']
msg = text.MIMEText(json.dumps(message))
msg["to"] = subscriber.path
msg["from"] = subscription['options'].get('from', '')
subject_opt = subscription['options'].get('subject', '')
msg["subject"] = params.get('subject', subject_opt)
if conf_n.smtp_mode == 'third_part':
p = subprocess.Popen(conf_n.smtp_command.split(' '),
stdin=subprocess.PIPE)
p.communicate(msg.as_string())
elif conf_n.smtp_mode == 'self_local':
sender = smtplib.SMTP_SSL(conf_n.smtp_host,
conf_n.smtp_port)
sender.set_debuglevel(1)
sender.ehlo(conf_n.smtp_host)
try:
sender.login(conf_n.smtp_user_name,
conf_n.smtp_user_password)
except smtplib.SMTPException:
LOG.error("Failed to connect to the SMTP service")
continue
sender.sendmail(msg['from'], msg['to'], msg.as_string())
LOG.debug("Send mail successfully: %s", msg.as_string())
except OSError as err:
LOG.exception('Failed to create process for sendmail, '
'because %s.', str(err))
except Exception as exc:
LOG.exception('Failed to send email because %s.', str(exc))
def parse_taobao_id_from_url(url):
url_obj = urllib_parse.urlparse(url)
query_dict = urllib_parse.parse_qs(url_obj.query)
accept_keys = ('id', 'item_id')
for key in accept_keys:
if key in query_dict:
return query_dict[key][0]
return None
def test_unicode_next_path():
path = u'/en-US/føø/bãr'
request = RequestFactory().get(path)
request.session = {}
url = utils.default_fxa_login_url(request)
state = parse_qs(urlparse(url).query)['state'][0]
next_path = urlsafe_b64decode(state.split(':')[1] + '===')
assert next_path.decode('utf-8') == path
def _get_params_from_url(url):
"""
Given a URL, return a dict of parameters and their values. If a
parameter appears more than once all but the first value will be lost.
"""
parsed = urlparse.urlparse(url)
params = urlparse.parse_qs(parsed.query, keep_blank_values=True)
return dict((key, vals[0]) for key, vals in params.iteritems())
def test_short_desc(self):
bz = Bugzilla('http://localhost/bugzilla')
url = bz.make_url(self.test_run, self.case_run_1, 1)
url_parts = urlparse(url)
self.assertEqual(
'Test case failure: {}'.format(self.case_run_1.case.summary),
parse_qs(url_parts.query)['short_desc'][0])
def test_unicode_next_path():
path = u'/en-US/føø/bãr'
request = RequestFactory().get(path)
request.session = {}
url = utils.default_fxa_login_url(request)
state = parse_qs(urlparse(url).query)['state'][0]
next_path = urlsafe_b64decode(state.split(':')[1] + '===')
assert next_path.decode('utf-8') == path
def test_url_simple(self):
connection = Mock()
connection.call.return_value = (None, {'start': 0, 'total_size': 0})
page = Page(connection, '/some-path', None)
built_qs = parse_qs(urlsplit(page._build_url()).query)
self.assertEqual(built_qs, dict(
count=[str(DEFAULT_PAGE_ITEM_COUNT)],
page=["1"]))
def url_distance(preprocessor, url1, url2):
url1 = urlparse(url1)
url2 = urlparse(url2)
process_fn = lambda s: preprocessor(unquote(s))
path1 = map(process_fn, url1.path.strip('/').split('/'))
path2 = map(process_fn, url2.path.strip('/').split('/'))
path_distance = levenshtein_array(path1, path2)
query_distance = dict_distance(preprocessor, parse_qs(url1.query, True),
parse_qs(url2.query, True))
domain_distance = 4 * levenshtein_array((url1.hostname or '').split('.'),
(url2.hostname or '').split('.'))
return (domain_distance + path_distance + query_distance +
(url1.fragment != url2.fragment))
def _validate_auth_request_url(url):
parsed_url = urllib_parse.urlparse(url)
params = urllib_parse.parse_qs(parsed_url.query)
assert params.get("prompt") == ["select_account"], "Auth code request doesn't specify 'prompt=select_account'."
# when used as a Mock's side_effect, this method's return value is the Mock's return value
# (the real webbrowser.open returns a bool)
return True
def _get_params_from_url(url):
"""
Given a URL, return a dict of parameters and their values. If a
parameter appears more than once all but the first value will be lost.
"""
parsed = urlparse.urlparse(url)
params = urlparse.parse_qs(parsed.query, keep_blank_values=True)
return dict((key, vals[0]) for key, vals in params.iteritems())
def testQueryRemapping(self):
method_config = base_api.ApiMethodInfo(
request_type_name="MessageWithRemappings", query_params=["remapped_field", "enum_field"]
)
request = MessageWithRemappings(str_field="foo", enum_field=MessageWithRemappings.AnEnum.value_one)
http_request = FakeService().PrepareHttpRequest(method_config, request)
result_params = urllib_parse.parse_qs(urllib_parse.urlparse(http_request.url).query)
expected_params = {"enum_field": "ONE%2FTWO", "remapped_field": "foo"}
self.assertTrue(expected_params, result_params)
def __call__(self, request):
if request.method == "POST":
data = urllib_parse.parse_qs(request.body)
data["username"] = [self.username]
data["password"] = [self.password]
request.prepare_body(data, None)
return request
def __call__(self, request):
if request.method == "POST":
data = urllib_parse.parse_qs(request.body)
data["username"] = [self.username]
data["password"] = [self.password]
request.prepare_body(data, None)
return request
def episode(self, url, imdb, tvdb, title, premiered, season, episode):
try:
if url == None: return
url = urllib_parse.parse_qs(url)
url = dict([(i, url[i][0]) if url[i] else (i, '') for i in url])
url['imdb'], url['title'], url['premiered'], url['season'], url['episode'] = imdb, title, premiered, season, episode
url = urllib_parse.urlencode(url)
return url
except: return None
def assertGetCallMatches(self, called_url, expected_path, params_dict):
parsed_url = urlparse(called_url)
query_dict = parse_qs(parsed_url.query)
self.assertEqual(parsed_url.scheme, 'https')
self.assertEqual(parsed_url.netloc, 'api.meetup.com')
self.assertEqual(parsed_url.path, expected_path)
self.assertDictEqual(params_dict, query_dict)
def callback(headers, params, request):
status = 500
body = ""
parsed = urlparse(request.url)
params.update(parse_qs(parsed.query))
if not params:
status = 200
body = """
<resultlist state="2c371787daedadb61dfd5fd8411ac6c2">
<result project="Some:Project" repository="SLE_15"
arch="x86_64" code="published" state="published">
<status package="mypackage" code="disabled" />
<status package="anotherpackage" code="disabled" />
</result>
<result project="Some:Project" repository="SLE_12_SP3"
arch="x86_64" code="published" state="published">
<status package="mypackage" code="succeeded" />
<status package="anotherpackage" code="disabled" />
</result>
<result project="Some:Project" repository="SLE_12_SP2"
arch="x86_64" code="published" state="published">
<status package="mypackage" code="disabled" />
<status package="anotherpackage" code="disabled" />
</result>
</resultlist>
"""
elif params.get("package", []):
if params["package"][0] in ["mypackage", "anotherpackage"]:
status = 200
body = """
<resultlist state="110f481609293ee149e28bbaced3b1b9">
<result project="Some:Project" repository="SLE_15"
arch="x86_64" code="published" state="published">
<status package="{pkg}" code="disabled" />
</result>
<result project="Some:Project" repository="SLE_12_SP3"
arch="x86_64" code="published" state="published">
<status package="{pkg}" code="succeeded" />
</result>
<result project="Some:Project" repository="SLE_12_SP2"
arch="x86_64" code="published" state="published">
<status package="{pkg}" code="disabled" />
</result>
</resultlist>
""".format(pkg=params["package"][0])
else:
status = 404
body = """
<status code="404">
<summary>unknown package '{pkg}'</summary>
<details>404 unknown package '{pkg}'</details>
</status>
""".format(pkg=params["package"][0])
return status, headers, body
def __init__(self, url, **kwa):
self.url = url
urldata = urlparse.urlparse(url, **kwa)
for key in self._components:
val = getattr(urldata, key)
setattr(self, key, val)
self.query_str = urldata.query
self.queryl = urlparse.parse_qs(urldata.query)
self.query = dict(urlparse.parse_qsl(urldata.query))
def _custom_request_query_matcher(cls, r1, r2):
""" Ensure method, path, and query parameters match. """
from six.moves.urllib_parse import urlparse, parse_qs # pylint: disable=import-error,relative-import
url1 = urlparse(r1.uri)
url2 = urlparse(r2.uri)
q1 = parse_qs(url1.query)
q2 = parse_qs(url2.query)
shared_keys = set(q1.keys()).intersection(set(q2.keys()))
if len(shared_keys) != len(q1) or len(shared_keys) != len(q2):
return False
for key in shared_keys:
if q1[key][0].lower() != q2[key][0].lower():
return False
return True
def test_redirect_uri():
"""Test redirect uri."""
app = setup_app()
with app.test_client() as client:
# Test redirect
resp = client.get(
url_for("invenio_oauthclient.login", remote_app='test',
next='http://invenio-software.org')
)
assert resp.status_code == 302
# Verify parameters
params = parse_qs(urlparse(resp.location).query)
assert params['response_type'] == ['code']
assert params['client_id'] == ['testid']
assert params['redirect_uri']
assert params['state']
# Verify next parameter in state token does not allow blanco redirects
state = serializer.loads(params['state'][0])
assert state['next'] is None
# Assert redirect uri does not have any parameters.
params = parse_qs(urlparse(params['redirect_uri'][0]).query)
assert params == {}
# Assert that local redirects are allowed
test_urls = [
'/search',
url_for('invenio_oauthclient.disconnect', remote_app='test',
_external=True)
]
for url in test_urls:
resp = client.get(
url_for("invenio_oauthclient.login", remote_app='test',
next=url)
)
assert resp.status_code == 302
state = serializer.loads(
parse_qs(urlparse(resp.location).query)['state'][0]
)
assert url == state['next']
def test_url_with_qs(self):
connection = Mock()
connection.call.return_value = (None, {'start': 0, 'total_size': 0})
page = Page(connection, '/some-path?with=a&query=string', None)
built_qs = parse_qs(urlsplit(page._build_url()).query)
self.assertEqual(built_qs, {
"with": ["a"],
"query": ["string"],
"count": [str(DEFAULT_PAGE_ITEM_COUNT)],
"page": ["1"],
})
def testQueryRemapping(self):
method_config = base_api.ApiMethodInfo(
request_type_name='MessageWithRemappings',
query_params=['remapped_field', 'enum_field'])
request = MessageWithRemappings(
str_field='foo', enum_field=MessageWithRemappings.AnEnum.value_one)
http_request = FakeService().PrepareHttpRequest(method_config, request)
result_params = urllib_parse.parse_qs(
urllib_parse.urlparse(http_request.url).query)
expected_params = {'enum_field': 'ONE%2FTWO', 'remapped_field': 'foo'}
self.assertTrue(expected_params, result_params)
def test_login(self):
"""Test ORCID login."""
resp = self.client.get(url_for("oauthclient.login", remote_app="orcid", next="/someurl/"))
self.assertStatus(resp, 302)
params = parse_qs(urlparse(resp.location).query)
self.assertEqual(params["response_type"], ["code"])
self.assertEqual(params["show_login"], ["true"])
self.assertEqual(params["scope"], ["/authenticate"])
assert params["redirect_uri"]
assert params["client_id"]
assert params["state"]
def test_manage_fxa_link():
user = mock.MagicMock(email='*****@*****.**', fxa_id='abcd1234')
link = urlparse(manage_fxa_link({'user': user}))
url = '{scheme}://{netloc}{path}'.format(
scheme=link.scheme, netloc=link.netloc, path=link.path)
assert url == 'https://stable.dev.lcip.org/settings'
query = parse_qs(link.query)
assert query == {
'uid': ['abcd1234'],
'email': ['*****@*****.**'],
'entrypoint': ['addons'],
}
def mangle_url_m(url, include=None, exclude=None, add=None):
""" Multivalue-version of the `mangle_url`; works with dict of lists only
(`param -> [value1, …]`); sorts the resulting query """
url = to_bytes(url)
url_parts = urlparse.urlparse(url)
## NOTE: the order of the fields is still lost.
query = urlparse.parse_qs(url_parts.query, keep_blank_values=1)
query_new = mangle_dict(query, include=include, exclude=exclude, add=add)
query_new = [(k, v) for k, vl in query_new.items() for v in vl]
query_new = [(to_bytes(k), to_bytes(v)) for k, v in query_new]
query_new = sorted(query_new) # make the order stable since it's lost anyway
return url_replace(url, query=query_new)
def test_redirect_uri(self):
from invenio.modules.oauthclient.views.client import serializer
# Test redirect
resp = self.client.get(
url_for("oauthclient.login", remote_app='test',
next='http://invenio-software.org')
)
self.assertStatus(resp, 302)
# Verify parameters
params = parse_qs(urlparse(resp.location).query)
self.assertEqual(params['response_type'], ['code'])
self.assertEqual(params['client_id'], ['testid'])
assert params['redirect_uri']
assert params['state']
# Verify next parameter in state token does not allow blanco redirects
state = serializer.loads(params['state'][0])
self.assertIsNone(state['next'])
# Assert redirect uri does not have any parameters.
params = parse_qs(urlparse(params['redirect_uri'][0]).query)
self.assertEqual(params, {})
# Assert that local redirects are allowed
test_urls = [
'/search',
url_for('oauthclient.disconnect', remote_app='test',
_external=True)
]
for url in test_urls:
resp = self.client.get(
url_for("oauthclient.login", remote_app='test', next=url)
)
self.assertStatus(resp, 302)
state = serializer.loads(
parse_qs(urlparse(resp.location).query)['state'][0]
)
self.assertEqual(url, state['next'])
def _generate_helper(self, response_type=None, response_disposition=None,
generation=None):
endpoint = 'http://api.example.com'
resource = '/name/path'
credentials = _make_credentials(
signing=True, signer_email='*****@*****.**')
credentials.sign_bytes.return_value = b'DEADBEEF'
signed = base64.b64encode(credentials.sign_bytes.return_value)
signed = signed.decode('ascii')
expiration = 1000
url = self._call_fut(
credentials,
resource,
expiration,
api_access_endpoint=endpoint,
response_type=response_type,
response_disposition=response_disposition,
generation=generation,
)
# Check the mock was called.
string_to_sign = '\n'.join([
'GET',
'',
'',
str(expiration),
resource,
])
credentials.sign_bytes.assert_called_once_with(string_to_sign)
scheme, netloc, path, qs, frag = urllib_parse.urlsplit(url)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'api.example.com')
self.assertEqual(path, resource)
self.assertEqual(frag, '')
# Check the URL parameters.
params = urllib_parse.parse_qs(qs)
expected_params = {
'GoogleAccessId': [credentials.signer_email],
'Expires': [str(expiration)],
'Signature': [signed],
}
if response_type is not None:
expected_params['response-content-type'] = [response_type]
if response_disposition is not None:
expected_params['response-content-disposition'] = [
response_disposition]
if generation is not None:
expected_params['generation'] = [generation]
self.assertEqual(params, expected_params)
def apply_to_request(self, req, service):
parsed = urlparse.urlparse(req.url)
if req.method == 'POST':
# This is probably going to break when given multipart data.
params = urlparse.parse_qs(req.body or '', keep_blank_values=True)
else:
params = urlparse.parse_qs(parsed.query, keep_blank_values=True)
params = dict((key, vals[0]) for key, vals in params.iteritems())
params['AWSAccessKeyId'] = self.args['key_id']
params['SignatureVersion'] = 2
params['SignatureMethod'] = 'HmacSHA256'
params['Timestamp'] = time.strftime(ISO8601, time.gmtime())
if self.args.get('security_token'):
params['SecurityToken'] = self.args['security_token']
# Needed for retries so old signatures aren't included in to_sign
params.pop('Signature', None)
to_sign = '{method}\n{host}\n{path}\n'.format(
method=req.method, host=parsed.netloc.lower(),
path=(parsed.path or '/'))
quoted_params = []
for key in sorted(params):
val = six.text_type(params[key])
quoted_params.append(urlparse.quote(key, safe='') + '=' +
urlparse.quote(val, safe='-_~'))
query_string = '&'.join(quoted_params)
to_sign += query_string
# Redact passwords
redacted_to_sign = re.sub('assword=[^&]*', 'assword=<redacted>',
to_sign)
self.log.debug('string to sign: %s', repr(redacted_to_sign))
signature = self.sign_string(to_sign)
self.log.debug('b64-encoded signature: %s', signature)
params['Signature'] = signature
if req.method == 'POST':
req.prepare_body(params, {})
else:
req.prepare_url(_remove_params_from_url(req.url), params)
return req
