def _parse_url(self, url):
"""Create a url from test data.
If provided with a full URL, just return that. If SSL is requested
set the scheme appropriately.
Scheme and netloc are saved for later use in comparisons.
"""
query_params = self.test_data['query_parameters']
ssl = self.test_data['ssl']
parsed_url = urlparse.urlsplit(url)
if not parsed_url.scheme:
full_url = utils.create_url(url, self.host, port=self.port,
prefix=self.prefix, ssl=ssl)
# parse again to set updated netloc and scheme
parsed_url = urlparse.urlsplit(full_url)
self.scheme = parsed_url.scheme
self.netloc = parsed_url.netloc
if query_params:
query_string = self._update_query_params(parsed_url.query,
query_params)
else:
query_string = parsed_url.query
return urlparse.urlunsplit((parsed_url.scheme, parsed_url.netloc,
parsed_url.path, query_string, ''))
def test_authentication_disabled_app(authentication_disabled_app):
# app.auth should = false
assert mg_globals
assert mg_globals.app.auth is False
# Try to visit register page
template.clear_test_template_context()
response = authentication_disabled_app.get('/auth/register/')
response.follow()
# Correct redirect?
assert urlparse.urlsplit(response.location)[2] == '/'
assert 'mediagoblin/root.html' in template.TEMPLATE_TEST_CONTEXT
# Try to vist login page
template.clear_test_template_context()
response = authentication_disabled_app.get('/auth/login/')
response.follow()
# Correct redirect?
assert urlparse.urlsplit(response.location)[2] == '/'
assert 'mediagoblin/root.html' in template.TEMPLATE_TEST_CONTEXT
## Test check_login_simple should return None
assert auth_tools.check_login_simple('test', 'simple') is None
# Try to visit the forgot password page
template.clear_test_template_context()
response = authentication_disabled_app.get('/auth/register/')
response.follow()
# Correct redirect?
assert urlparse.urlsplit(response.location)[2] == '/'
assert 'mediagoblin/root.html' in template.TEMPLATE_TEST_CONTEXT
def copy_rec_files_s3(self, user, collection, recording, warc_files):
coll_warc_key = recording.COLL_WARC_KEY.format(coll=collection.my_id)
rec_warc_key = recording.REC_WARC_KEY.format(rec=recording.my_id)
# Copy WARCs
total_size = 0
coll_root = self.s3_root + collection.get_dir_path() + '/'
for n, url in warc_files.items():
if not url.startswith('s3://'):
print('Skipping: ' + url)
continue
src_parts = urlsplit(url)
src_bucket = src_parts.netloc
src_key = src_parts.path.lstrip('/')
try:
res = self.s3.head_object(Bucket=src_bucket,
Key=src_key)
except Exception as e:
print('Skipping: ' + url)
print(e)
continue
size = res['ContentLength']
if n != recording.INDEX_FILE_KEY:
target_file = coll_root + 'warcs/' + n
self.redis.hset(coll_warc_key, n, target_file)
self.redis.sadd(rec_warc_key, n)
total_size += size
else:
target_file = coll_root + 'indexes/' + os.path.basename(url)
recording.set_prop(n, target_file, update_ts=False)
# target
target_parts = urlsplit(target_file)
target_bucket = target_parts.netloc
target_key = target_parts.path.lstrip('/')
params = dict(Bucket=target_bucket,
Key=target_key,
CopySource=dict(Bucket=src_bucket, Key=src_key))
print(' Copying:')
print(' From: s3://' + src_bucket + '/' + src_key)
print(' To: s3://' + target_bucket + '/' + target_key)
print(' Size: ' + str(size))
try:
if not self.dry_run:
res = self.s3.copy_object(**params)
except Exception as e:
print(' ERROR:')
print(e)
return total_size
def _update_link_prefix(self, orig_url, prefix):
if not prefix:
return orig_url
url_parts = list(parse.urlsplit(orig_url))
prefix_parts = list(parse.urlsplit(prefix))
url_parts[0:2] = prefix_parts[0:2]
return parse.urlunsplit(url_parts)
def __init__(self, base, relative=None):
self._has_token = False
self._url = None
self._url_parts = None
self._loaded = False
self._xml = None
self._url_parts = None
self._headers = None
self._config = None
if isinstance(base, six.string_types):
base_url = base
self._url_parts = list(parse.urlsplit(base_url))
elif isinstance(base, RequestBase):
base_url = base.url
self._has_token = base.has_token
self._url_parts = base._url_parts[:]
self._headers = base._headers
self._config = base.config
if relative:
scheme, netloc, path, qs, fragment = parse.urlsplit(relative)
if path:
self._url_parts[2] = _join_plex(self._url_parts[2], path)
if qs:
data = parse.parse_qsl(self._url_parts[3]) + parse.parse_qsl(qs)
self._url_parts[3] = parse.urlencode(data)
else:
# Strip of all non-token parts
data = parse.parse_qsl(self._url_parts[3])
self._url_parts[3] = parse.urlencode([(x, y) for x, y in data if x == 'X-Plex-Token'])
if not self._has_token:
self._has_token = 'X-Plex-Token' in parse.parse_qs(self._url_parts[3])
self._url = parse.urlunsplit(self._url_parts)
def test_notify_alarm_with_batch_listener(self, logger):
data1 = {
"actions": ["test://"],
"alarm_id": "foobar",
"alarm_name": "testalarm",
"severity": "critical",
"previous": "OK",
"current": "ALARM",
"reason": "Everything is on fire",
"reason_data": {"fire": "everywhere"},
}
data2 = {
"actions": ["test://"],
"alarm_id": "foobar2",
"alarm_name": "testalarm2",
"severity": "low",
"previous": "ALARM",
"current": "OK",
"reason": "Everything is fine",
"reason_data": {"fine": "fine"},
}
self.service.terminate()
self.CONF.set_override("batch_size", 2, "notifier")
# Init a new service with new configuration
self.svc = notifier.AlarmNotifierService(0, self.CONF)
self.addCleanup(self.svc.terminate)
self._msg_notifier.sample({}, "alarm.update", data1)
self._msg_notifier.sample({}, "alarm.update", data2)
time.sleep(1)
notifications = self.svc.notifiers["test"].obj.notifications
self.assertEqual(2, len(notifications))
self.assertEqual(
(
urlparse.urlsplit(data1["actions"][0]),
data1["alarm_id"],
data1["alarm_name"],
data1["severity"],
data1["previous"],
data1["current"],
data1["reason"],
data1["reason_data"],
),
notifications[0],
)
self.assertEqual(
(
urlparse.urlsplit(data2["actions"][0]),
data2["alarm_id"],
data2["alarm_name"],
data2["severity"],
data2["previous"],
data2["current"],
data2["reason"],
data2["reason_data"],
),
notifications[1],
)
self.assertEqual(mock.call("Received %s messages in batch.", 2), logger.call_args_list[0])
def get_bottom_url(t_ver, t_url, b_ver, b_endpoint):
"""get_bottom_url
convert url received by Tricircle service to bottom OpenStack
request url through the configured endpoint in the KeyStone
:param t_ver: version of top service
:param t_url: request url to the top service
:param b_ver: version of bottom service
:param b_endpoint: endpoint registered in keystone for bottom service
:return: request url to bottom service
"""
t_parse = urlparse.urlsplit(t_url)
after_ver = t_parse.path
remove_ver = '/' + t_ver + '/'
pos = after_ver.find(remove_ver)
if pos == 0:
after_ver = after_ver[len(remove_ver):]
else:
remove_ver = t_ver + '/'
pos = after_ver.find(remove_ver)
if pos == 0:
after_ver = after_ver[len(remove_ver):]
if after_ver == t_parse.path:
# wrong t_url
return ''
b_parse = urlparse.urlsplit(b_endpoint)
scheme = b_parse.scheme
netloc = b_parse.netloc
path = '/' + b_ver + '/' + after_ver
if b_ver == '':
path = '/' + after_ver
# Remove availability_zone filter since it is handled by VolumeController.
# VolumeController will send GET request only to bottom pods whose AZ
# is specified in availability_zone filter.
query_filters = []
for k, v in urlparse.parse_qsl(t_parse.query):
if k == 'availability_zone':
continue
query_filters.append((k, v))
query = urlparse.urlencode(query_filters)
fragment = t_parse.fragment
b_url = urlparse.urlunsplit((scheme,
netloc,
path,
query,
fragment))
return b_url
def __clean_external_url(url):
if url.startswith(('http://', 'https://', '//')):
# Do the domains and ports match?
pnext = urlsplit(url)
preq = urlsplit(request.url)
if pnext.port != preq.port:
return ''
if not (pnext.hostname == preq.hostname or pnext.hostname.endswith('.' + preq.hostname)):
return ''
return url
def create_cookie(trans, key_name, key, email, age=DEFAULT_BIOSTAR_COOKIE_AGE, override_never_authenticate=False):
if trans.app.config.biostar_never_authenticate and not override_never_authenticate:
log.debug('A BioStar link was clicked, but never authenticate has been enabled, so we will not create the login cookie.')
return
digest = hmac.new(key, email).hexdigest()
value = "%s:%s" % (email, digest)
trans.set_cookie(value, name=key_name, path='/', age=age, version='1')
# We need to explicitly set the domain here, in order to allow for biostar in a subdomain to work
galaxy_hostname = urlsplit(url_for('/', qualified=True)).hostname
biostar_hostname = urlsplit(trans.app.config.biostar_url).hostname
trans.response.cookies[key_name]['domain'] = determine_cookie_domain(galaxy_hostname, biostar_hostname)
def prepare_url(orig_url):
"""Takes a link and swaps in network_link_prefix if set."""
prefix = cfg.CONF.network_link_prefix
# Copied directly from nova/api/openstack/common.py
if not prefix:
return orig_url
url_parts = list(parse.urlsplit(orig_url))
prefix_parts = list(parse.urlsplit(prefix))
url_parts[0:2] = prefix_parts[0:2]
url_parts[2] = prefix_parts[2] + url_parts[2]
return parse.urlunsplit(url_parts).rstrip('/')
def _test_edit_persona():
# Try and delete only Persona email address
template.clear_test_template_context()
res = persona_plugin_app.post(
'/edit/persona/',
{'email': '*****@*****.**'})
assert 'mediagoblin/plugins/persona/edit.html' in template.TEMPLATE_TEST_CONTEXT
context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/plugins/persona/edit.html']
form = context['form']
assert form.email.errors == [u"You can't delete your only Persona email address unless you have a password set."]
template.clear_test_template_context()
res = persona_plugin_app.post(
'/edit/persona/', {})
assert 'mediagoblin/plugins/persona/edit.html' in template.TEMPLATE_TEST_CONTEXT
context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/plugins/persona/edit.html']
form = context['form']
assert form.email.errors == [u'This field is required.']
# Try and delete Persona not owned by the user
template.clear_test_template_context()
res = persona_plugin_app.post(
'/edit/persona/',
{'email': '*****@*****.**'})
assert 'mediagoblin/plugins/persona/edit.html' in template.TEMPLATE_TEST_CONTEXT
context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/plugins/persona/edit.html']
form = context['form']
assert form.email.errors == [u'That Persona email address is not registered to this account.']
res = persona_plugin_app.get('/edit/persona/add/')
assert urlparse.urlsplit(res.location)[2] == '/edit/persona/'
# Add Persona email address
template.clear_test_template_context()
res = persona_plugin_app.post(
'/edit/persona/add/')
res.follow()
assert urlparse.urlsplit(res.location)[2] == '/edit/account/'
# Delete a Persona
res = persona_plugin_app.post(
'/edit/persona/',
{'email': '*****@*****.**'})
res.follow()
assert urlparse.urlsplit(res.location)[2] == '/edit/account/'
def assert_valid_temp_url(self, name):
url = self.backend.url(name)
split_url = urlparse.urlsplit(url)
query_params = urlparse.parse_qs(split_url[3])
split_base_url = urlparse.urlsplit(base_url(container=self.backend.container_name, path=name))
# ensure scheme, netloc, and path are same as to non-temporary URL
self.assertEqual(split_base_url[0:2], split_url[0:2])
# ensure query string contains signature and expiry
self.assertIn('temp_url_sig', query_params)
self.assertIn('temp_url_expires', query_params)
def extra_clean(self, doc):
for el in doc.xpath('//*[@href]'):
scheme, netloc, path, query, fragment = urlsplit(el.attrib['href'])
if scheme and scheme not in self.allowed_protocols:
el.drop_tag()
for attr in self.attr_val_is_uri:
if attr == 'href':
continue
for el in doc.xpath('//*[@'+attr+']'):
scheme, netloc, path, query, fragment = urlsplit(el.attrib[attr])
scheme_fail = scheme and scheme not in self.allowed_protocols
netloc_fail = not self.allow_external_src and netloc
if scheme_fail or netloc_fail:
if attr == 'src':
el.drop_tag()
else:
el.attrib.pop(attr)
if self.a_without_href:
for link in doc.xpath('//a[not(@href)]'):
link.drop_tag()
if self.allow_classes is not None:
for el in doc.xpath('//*[@class]'):
classes = filter(None, el.attrib['class'].split())
if el.tag in self.allow_classes:
allowed = self.allow_classes[el.tag]
condition = allowed if callable(allowed) else \
(lambda cls: cls in allowed)
classes = filter(condition, classes)
else:
classes = []
if classes:
el.attrib['class'] = ' '.join(classes)
else:
el.attrib.pop('class')
for callback in self.dom_callbacks:
callback(doc)
if self.wrap_inline_tags is not False and self.tags_to_wrap:
self.clean_top(doc)
if self.split_paragraphs_by_br:
self.remove_brs_from_pars(doc)
for tag in self.drop_empty_tags:
for el in doc.xpath('//'+tag):
if not el.attrib and self.is_element_empty(el):
el.drop_tree()
def parse_host(host):
if host.startswith('http://') or host.startswith('https://'):
parsed = urlsplit(host)
else:
scheme = 'https' if settings.INTRACLUSTER_HTTPS else 'http'
parsed = urlsplit(scheme + '://' + host)
return {
'host': parsed.netloc,
'url': '%s://%s%s' % (parsed.scheme, parsed.netloc, parsed.path),
'params': {key: value[-1] for (key, value) in parse_qs(parsed.query).items()},
}
def test_notify_alarm_with_batch_listener(self, logger):
data1 = {
'actions': ['test://'],
'alarm_id': 'foobar',
'alarm_name': 'testalarm',
'severity': 'critical',
'previous': 'OK',
'current': 'ALARM',
'reason': 'Everything is on fire',
'reason_data': {'fire': 'everywhere'}
}
data2 = {
'actions': ['test://'],
'alarm_id': 'foobar2',
'alarm_name': 'testalarm2',
'severity': 'low',
'previous': 'ALARM',
'current': 'OK',
'reason': 'Everything is fine',
'reason_data': {'fine': 'fine'}
}
self.service.stop()
self.CONF.set_override("batch_size", 2, 'notifier')
# Init a new service with new configuration
self.svc = notifier.AlarmNotifierService(self.CONF)
self.svc.start()
self._msg_notifier.sample({}, 'alarm.update', data1)
self._msg_notifier.sample({}, 'alarm.update', data2)
time.sleep(1)
notifications = self.svc.notifiers['test'].obj.notifications
self.assertEqual(2, len(notifications))
self.assertEqual((urlparse.urlsplit(data1['actions'][0]),
data1['alarm_id'],
data1['alarm_name'],
data1['severity'],
data1['previous'],
data1['current'],
data1['reason'],
data1['reason_data']),
notifications[0])
self.assertEqual((urlparse.urlsplit(data2['actions'][0]),
data2['alarm_id'],
data2['alarm_name'],
data2['severity'],
data2['previous'],
data2['current'],
data2['reason'],
data2['reason_data']),
notifications[1])
self.assertEqual(mock.call('Received %s messages in batch.', 2),
logger.call_args_list[0])
self.svc.stop()
def UrlMustHaveSchemeAndPath( sURL ):
#
#from Utils.Both2n3 import urlsplit, urlunsplit
#
sScheme, sLocation, sPath, sQuery, sFragmentID = urlsplit( sURL )
#
if sScheme not in ( 'http', 'https', 'ftp' ): # if you got no scheme, urlsplit gets confused
#
sScheme, sLocation, sPath, sQuery, sFragmentID = urlsplit( 'http://' + sURL )
#
#
if not sPath: sPath = '/'
#
return urlunsplit( ( sScheme, sLocation, sPath, sQuery, sFragmentID ) )
def _test_new_user():
openid_plugin_app.post(
'/auth/openid/login/', {
'openid': u'http://real.myopenid.com'})
# Right place?
assert 'mediagoblin/auth/register.html' in template.TEMPLATE_TEST_CONTEXT
context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html']
register_form = context['register_form']
# Register User
res = openid_plugin_app.post(
'/auth/openid/register/', {
'openid': register_form.openid.data,
'username': u'chris',
'email': u'*****@*****.**'})
res.follow()
# Correct place?
assert urlparse.urlsplit(res.location)[2] == '/u/chris/'
assert 'mediagoblin/user_pages/user_nonactive.html' in template.TEMPLATE_TEST_CONTEXT
# No need to test if user is in logged in and verification email
# awaits, since openid uses the register_user function which is
# tested in test_auth
# Logout User
openid_plugin_app.get('/auth/logout')
# Get user and detach from session
test_user = mg_globals.database.LocalUser.query.filter(
LocalUser.username==u'chris'
).first()
Session.expunge(test_user)
# Log back in
# Could not get it to work by 'POST'ing to /auth/openid/login/
template.clear_test_template_context()
res = openid_plugin_app.post(
'/auth/openid/login/finish/', {
'openid': u'http://real.myopenid.com'})
res.follow()
assert urlparse.urlsplit(res.location)[2] == '/'
assert 'mediagoblin/root.html' in template.TEMPLATE_TEST_CONTEXT
# Make sure user is in the session
context = template.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html']
session = context['request'].session
assert session['user_id'] == six.text_type(test_user.id)
def _get_version_match(self, endpoint, profile_version, service_type):
"""Return the best matching version
Look through each version trying to find the best match for
the version specified in this profile.
* The best match will only ever be found within the same
major version, meaning a v2 profile will never match if
only v3 is available on the server.
* The search for the best match is fuzzy if needed.
* If the profile specifies v2 and the server has
v2.0, v2.1, and v2.2, the match will be v2.2.
* When an exact major/minor is specified, e.g., v2.0,
it will only match v2.0.
"""
match_version = None
for version in endpoint.versions:
api_version = self._parse_version(version["id"])
if profile_version.major != api_version.major:
continue
if profile_version.minor <= api_version.minor:
for link in version["links"]:
if link["rel"] == "self":
resp_link = link['href']
match_version = parse.urlsplit(resp_link).path
# Only break out of the loop on an exact match,
# otherwise keep trying.
if profile_version.minor == api_version.minor:
break
if match_version is None:
raise exceptions.EndpointNotFound(
"Unable to determine endpoint for %s" % service_type)
# Make sure the root endpoint has no overlap with match_version
root_parts = parse.urlsplit(endpoint.uri)
match_version = match_version.replace(root_parts.path, "", 1)
match = utils.urljoin(endpoint.uri, match_version)
# For services that require the project id in the request URI,
# add them in here.
if endpoint.needs_project_id:
match = utils.urljoin(match, endpoint.project_id)
return match
def remove_version_from_href(href):
"""Removes the first api version from the href.
Given: 'http://www.nova.com/v1.1/123'
Returns: 'http://www.nova.com/123'
Given: 'http://www.nova.com/v1.1'
Returns: 'http://www.nova.com'
"""
parsed_url = urlparse.urlsplit(href)
url_parts = parsed_url.path.split('/', 2)
# NOTE: this should match vX.X or vX
expression = re.compile(r'^v([0-9]+|[0-9]+\.[0-9]+)(/.*|$)')
if expression.match(url_parts[1]):
del url_parts[1]
new_path = '/'.join(url_parts)
if new_path == parsed_url.path:
LOG.debug('href %s does not contain version' % href)
raise ValueError(_('href %s does not contain version') % href)
parsed_url = list(parsed_url)
parsed_url[2] = new_path
return urlparse.urlunsplit(parsed_url)
def _get_parameters_from_request(self, request, exception=False):
"""Get parameters to log in OPERATION_LOG."""
user = request.user
referer_url = None
try:
referer_dic = urlparse.urlsplit(
urlparse.unquote(request.META.get('HTTP_REFERER')))
referer_url = referer_dic[2]
if referer_dic[3]:
referer_url += "?" + referer_dic[3]
if isinstance(referer_url, str):
referer_url = referer_url.decode('utf-8')
except Exception:
pass
return {
'domain_name': getattr(user, 'domain_name', None),
'domain_id': getattr(user, 'domain_id', None),
'project_name': getattr(user, 'project_name', None),
'project_id': getattr(user, 'project_id', None),
'user_name': getattr(user, 'username', None),
'user_id': request.session.get('user_id', None),
'request_scheme': request.scheme,
'referer_url': referer_url,
'request_url': urlparse.unquote(request.path),
'method': request.method if not exception else None,
'param': self._get_request_param(request),
}
def _get_safe_url(url):
"""Gets version of *url* with basic auth passwords obscured. This function
returns results suitable for printing and logging.
E.g.: https://user:[email protected] => https://user:********@example.com
.. note::
The number of astrisks is invariant in the length of the basic auth
password, so minimal information is leaked.
:param url: a url
:type url: ``str``
:return: *url* with password obscured
:rtype: ``str``
"""
safe_url = url
url, username, _ = _strip_basic_auth(url)
if username is not None:
# case: url contained basic auth creds; obscure password
url_parts = parse.urlsplit(url)
safe_netloc = '{0}@{1}'.format(username, url_parts.hostname)
# replace original netloc w/ obscured version
frags = list(url_parts)
frags[1] = safe_netloc
safe_url = parse.urlunsplit(frags)
return safe_url
def _strip_basic_auth(url):
"""Returns *url* with basic auth credentials removed. Also returns the
basic auth username and password if they're present in *url*.
E.g.: https://user:[email protected] => https://example.com
*url* need not include basic auth credentials.
:param url: url which may or may not contain basic auth credentials
:type url: ``str``
:return: 3-``tuple`` of:
* (``str``) -- *url* with any basic auth creds removed
* (``str`` or ``NoneType``) -- basic auth username or ``None`` if basic
auth username not given
* (``str`` or ``NoneType``) -- basic auth password or ``None`` if basic
auth password not given
:rtype: ``tuple``
"""
url_parts = parse.urlsplit(url)
username = url_parts.username
password = url_parts.password
frags = list(url_parts)
# swap out "user[:pass]@hostname" for "hostname"
frags[1] = url_parts.hostname
url = parse.urlunsplit(frags)
return (url, username, password)
def save(self, *args, **kwargs):
if self.image:
try:
tmp_filename, headers = urlretrieve(self.image)
max_filename_length = self.image_cache.field.max_length # Usually 100
template_needs = len(self.image_cache_file_path_template % '')
# We'll use the actual instance label size rather than the maximum that it could be
# instance_label_length = self.instance._meta.get_field_by_name('label')[0].max_length
instance_label_length = len(self.instance.label)
truncate_to = max_filename_length - template_needs - instance_label_length - 8
image_filename = os.path.basename(urlsplit(self.image).path)
image_filename = url_to_unicode(image_filename)
filename_root, extension = os.path.splitext(image_filename)
truncated_filename = filename_root[:(truncate_to - len(extension))] + extension
self.image_cache.save(
truncated_filename,
File(open(tmp_filename, 'rb')),
# Calling save on a FieldFile causes a save on the model instance
# we don't need that as we're about to do it below (without this
# we get an infinite recursion).
save=False,
)
except HTTPError:
pass
return super(Speaker, self).save(*args, **kwargs)
def test_notify_alarm(self):
data = {
"actions": ["test://"],
"alarm_id": "foobar",
"alarm_name": "testalarm",
"severity": "critical",
"previous": "OK",
"current": "ALARM",
"reason": "Everything is on fire",
"reason_data": {"fire": "everywhere"},
}
self._msg_notifier.sample({}, "alarm.update", data)
time.sleep(1)
notifications = self.service.notifiers["test"].obj.notifications
self.assertEqual(1, len(notifications))
self.assertEqual(
(
urlparse.urlsplit(data["actions"][0]),
data["alarm_id"],
data["alarm_name"],
data["severity"],
data["previous"],
data["current"],
data["reason"],
data["reason_data"],
),
notifications[0],
)
def remove_version_from_href(href):
"""Removes the first api version from the href.
Given: 'http://manila.example.com/v1.1/123'
Returns: 'http://manila.example.com/123'
Given: 'http://www.manila.com/v1.1'
Returns: 'http://www.manila.com'
Given: 'http://manila.example.com/share/v1.1/123'
Returns: 'http://manila.example.com/share/123'
"""
parsed_url = parse.urlsplit(href)
url_parts = parsed_url.path.split('/')
# NOTE: this should match vX.X or vX
expression = re.compile(r'^v([0-9]+|[0-9]+\.[0-9]+)(/.*|$)')
for x in range(len(url_parts)):
if expression.match(url_parts[x]):
del url_parts[x]
break
new_path = '/'.join(url_parts)
if new_path == parsed_url.path:
msg = 'href %s does not contain version' % href
LOG.debug(msg)
raise ValueError(msg)
parsed_url = list(parsed_url)
parsed_url[2] = new_path
return parse.urlunsplit(parsed_url)
def _info(r):
m = re.search(r, svn_info, re.M)
if m:
return urlsplit(m.group(1))
else:
return None
def generate_module(src):
"""
Converts a url into a made-up module name by doing the following:
* Extract just the path name ignoring querystrings
* Trimming off the initial /
* Trimming off the file extension
* Removes off useless folder prefixes
e.g. http://google.com/js/v1.0/foo/bar/baz.js -> foo/bar/baz
"""
if not src:
return UNKNOWN_MODULE
filename, ext = splitext(urlsplit(src).path)
if filename.endswith('.min'):
filename = filename[:-4]
# TODO(dcramer): replace CLEAN_MODULE_RE with tokenizer completely
tokens = filename.split('/')
for idx, token in enumerate(tokens):
# a SHA
if VERSION_RE.match(token):
return '/'.join(tokens[idx + 1:])
return CLEAN_MODULE_RE.sub('', filename) or UNKNOWN_MODULE
def _parse_url(self, url, ssl=False):
"""Create a url from test data.
If provided with a full URL, just return that. If SSL is requested
set the scheme appropriately.
Scheme and netloc are saved for later use in comparisons.
"""
parsed_url = urlparse.urlsplit(url)
url_scheme = parsed_url[0]
scheme = 'http'
netloc = self.host
if not url_scheme:
if self.port:
netloc = '%s:%s' % (self.host, self.port)
if ssl:
scheme = 'https'
path = parsed_url[2]
if self.prefix:
path = '%s%s' % (self.prefix, path)
full_url = urlparse.urlunsplit((scheme, netloc, path,
parsed_url[3], ''))
self.scheme = scheme
self.netloc = netloc
else:
full_url = url
self.scheme = url_scheme
self.netloc = parsed_url[1]
return full_url
def resolve(self, url):
"""Resolve given URL and use regitered loader."""
if self.url_map is None:
self._build_url_map()
parts = urlsplit(url)
loader, args = self.url_map.bind(parts.hostname).match(parts.path)
return loader(**args)
def open(self, target_uri, **kwargs):
"""Open target uri.
:param target_uri: Uri to open
:type target_uri: string
:returns: Target object
"""
target = urlsplit(target_uri, scheme=self.default_opener)
opener = self.get_opener(target.scheme)
query = opener.conform_query(target.query)
target = opener.get_target(
target.scheme,
target.path,
target.fragment,
target.username,
target.password,
target.hostname,
target.port,
query,
**kwargs
)
target.opener_path = target_uri
return target
def __init__(self, url, client_dir, client_cert=None):
self._url = url
self._server_url = self._extract_server_url(url)
self._client_dir = client_dir
self._client_cert = client_cert
self.use(urlsplit(url)[2])
def _extract_server_url(self, url):
parts = urlsplit(url)
return urlunsplit((parts.scheme, parts.netloc, '', '', ''))
def url_path_append(url, suffix):
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
path = (path + suffix).replace('//', '/')
return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def check_parsed(repo_root, path, f):
source_file = SourceFile(repo_root, path, "/", contents=f.read())
errors = []
if path.startswith("css/"):
if (source_file.type == "support" and not source_file.name_is_non_test
and not source_file.name_is_reference):
return [rules.SupportWrongDir.error(path)]
if (source_file.type != "support" and not source_file.name_is_reference
and not source_file.spec_links):
return [rules.MissingLink.error(path)]
if source_file.name_is_non_test:
return []
if source_file.markup_type is None:
return []
if source_file.root is None:
return [rules.ParseFailed.error(path)]
if source_file.type == "manual" and not source_file.name_is_manual:
errors.append(rules.ContentManual.error(path))
if source_file.type == "visual" and not source_file.name_is_visual:
errors.append(rules.ContentVisual.error(path))
about_blank_parts = urlsplit("about:blank")
for reftest_node in source_file.reftest_nodes:
href = reftest_node.attrib.get("href", "").strip(space_chars)
parts = urlsplit(href)
if parts == about_blank_parts:
continue
if (parts.scheme or parts.netloc):
errors.append(rules.AbsoluteUrlRef.error(path, (href, )))
continue
ref_url = urljoin(source_file.url, href)
ref_parts = urlsplit(ref_url)
if source_file.url == ref_url:
errors.append(rules.SameFileRef.error(path))
continue
assert ref_parts.path != ""
reference_file = os.path.join(repo_root, ref_parts.path[1:])
reference_rel = reftest_node.attrib.get("rel", "")
if not os.path.isfile(reference_file):
errors.append(
rules.NonexistentRef.error(path, (reference_rel, href)))
if len(source_file.timeout_nodes) > 1:
errors.append(rules.MultipleTimeout.error(path))
for timeout_node in source_file.timeout_nodes:
timeout_value = timeout_node.attrib.get("content", "").lower()
if timeout_value != "long":
errors.append(rules.InvalidTimeout.error(path, (timeout_value, )))
if source_file.testharness_nodes:
if len(source_file.testharness_nodes) > 1:
errors.append(rules.MultipleTestharness.error(path))
testharnessreport_nodes = source_file.root.findall(
".//{http://www.w3.org/1999/xhtml}script[@src='/resources/testharnessreport.js']"
)
if not testharnessreport_nodes:
errors.append(rules.MissingTestharnessReport.error(path))
else:
if len(testharnessreport_nodes) > 1:
errors.append(rules.MultipleTestharnessReport.error(path))
testharnesscss_nodes = source_file.root.findall(
".//{http://www.w3.org/1999/xhtml}link[@href='/resources/testharness.css']"
)
if testharnesscss_nodes:
errors.append(rules.PresentTestharnessCSS.error(path))
for element in source_file.variant_nodes:
if "content" not in element.attrib:
errors.append(rules.VariantMissing.error(path))
else:
variant = element.attrib["content"]
if variant != "" and variant[0] not in ("?", "#"):
errors.append(rules.MalformedVariant.error(path, (path, )))
seen_elements = {
"timeout": False,
"testharness": False,
"testharnessreport": False
}
required_elements = [
key for key, value in {
"testharness": True,
"testharnessreport": len(testharnessreport_nodes) > 0,
"timeout": len(source_file.timeout_nodes) > 0
}.items() if value
]
for elem in source_file.root.iter():
if source_file.timeout_nodes and elem == source_file.timeout_nodes[
0]:
seen_elements["timeout"] = True
if seen_elements["testharness"]:
errors.append(rules.LateTimeout.error(path))
elif elem == source_file.testharness_nodes[0]:
seen_elements["testharness"] = True
elif testharnessreport_nodes and elem == testharnessreport_nodes[0]:
seen_elements["testharnessreport"] = True
if not seen_elements["testharness"]:
errors.append(rules.EarlyTestharnessReport.error(path))
if all(seen_elements[name] for name in required_elements):
break
if source_file.testdriver_nodes:
if len(source_file.testdriver_nodes) > 1:
errors.append(rules.MultipleTestdriver.error(path))
testdriver_vendor_nodes = source_file.root.findall(
".//{http://www.w3.org/1999/xhtml}script[@src='/resources/testdriver-vendor.js']"
)
if not testdriver_vendor_nodes:
errors.append(rules.MissingTestdriverVendor.error(path))
else:
if len(testdriver_vendor_nodes) > 1:
errors.append(rules.MultipleTestdriverVendor.error(path))
for element in source_file.root.findall(
".//{http://www.w3.org/1999/xhtml}script[@src]"):
src = element.attrib["src"]
def incorrect_path(script, src):
return (script == src or ("/%s" % script in src
and src != "/resources/%s" % script))
if incorrect_path("testharness.js", src):
errors.append(rules.TestharnessPath.error(path))
if incorrect_path("testharnessreport.js", src):
errors.append(rules.TestharnessReportPath.error(path))
if incorrect_path("testdriver.js", src):
errors.append(rules.TestdriverPath.error(path))
if incorrect_path("testdriver-vendor.js", src):
errors.append(rules.TestdriverVendorPath.error(path))
return errors
def check_parsed(repo_root, path, f):
source_file = SourceFile(repo_root, path, "/", contents=f.read())
errors = []
if path.startswith("css/"):
if (source_file.type == "support" and
not source_file.name_is_non_test and
not source_file.name_is_reference):
return [("SUPPORT-WRONG-DIR", "Support file not in support directory", path, None)]
if (source_file.type != "support" and
not source_file.name_is_reference and
not source_file.spec_links):
return [("MISSING-LINK", "Testcase file must have a link to a spec", path, None)]
if source_file.name_is_non_test or source_file.name_is_manual:
return []
if source_file.markup_type is None:
return []
if source_file.root is None:
return [("PARSE-FAILED", "Unable to parse file", path, None)]
if source_file.type == "manual" and not source_file.name_is_manual:
return [("CONTENT-MANUAL", "Manual test whose filename doesn't end in '-manual'", path, None)]
if source_file.type == "visual" and not source_file.name_is_visual:
return [("CONTENT-VISUAL", "Visual test whose filename doesn't end in '-visual'", path, None)]
for reftest_node in source_file.reftest_nodes:
href = reftest_node.attrib.get("href", "").strip(space_chars)
parts = urlsplit(href)
if (parts.scheme or parts.netloc) and parts != urlsplit("about:blank"):
errors.append(("ABSOLUTE-URL-REF",
"Reference test with a reference file specified via an absolute URL: '%s'" % href, path, None))
continue
ref_url = urljoin(source_file.url, href)
ref_parts = urlsplit(ref_url)
if source_file.url == ref_url:
errors.append(("SAME-FILE-REF",
"Reference test which points at itself as a reference",
path,
None))
continue
assert ref_parts.path != ""
reference_file = os.path.join(repo_root, ref_parts.path[1:])
reference_rel = reftest_node.attrib.get("rel", "")
if not os.path.isfile(reference_file):
errors.append(("NON-EXISTENT-REF",
"Reference test with a non-existent '%s' relationship reference: '%s'" % (reference_rel, href), path, None))
if len(source_file.timeout_nodes) > 1:
errors.append(("MULTIPLE-TIMEOUT", "More than one meta name='timeout'", path, None))
for timeout_node in source_file.timeout_nodes:
timeout_value = timeout_node.attrib.get("content", "").lower()
if timeout_value != "long":
errors.append(("INVALID-TIMEOUT", "Invalid timeout value %s" % timeout_value, path, None))
if source_file.testharness_nodes:
if len(source_file.testharness_nodes) > 1:
errors.append(("MULTIPLE-TESTHARNESS",
"More than one <script src='/resources/testharness.js'>", path, None))
testharnessreport_nodes = source_file.root.findall(".//{http://www.w3.org/1999/xhtml}script[@src='/resources/testharnessreport.js']")
if not testharnessreport_nodes:
errors.append(("MISSING-TESTHARNESSREPORT",
"Missing <script src='/resources/testharnessreport.js'>", path, None))
else:
if len(testharnessreport_nodes) > 1:
errors.append(("MULTIPLE-TESTHARNESSREPORT",
"More than one <script src='/resources/testharnessreport.js'>", path, None))
testharnesscss_nodes = source_file.root.findall(".//{http://www.w3.org/1999/xhtml}link[@href='/resources/testharness.css']")
if testharnesscss_nodes:
errors.append(("PRESENT-TESTHARNESSCSS",
"Explicit link to testharness.css present", path, None))
for element in source_file.variant_nodes:
if "content" not in element.attrib:
errors.append(("VARIANT-MISSING",
"<meta name=variant> missing 'content' attribute", path, None))
else:
variant = element.attrib["content"]
if variant != "" and variant[0] not in ("?", "#"):
errors.append(("MALFORMED-VARIANT",
"%s <meta name=variant> 'content' attribute must be the empty string or start with '?' or '#'" % path, None))
seen_elements = {"timeout": False,
"testharness": False,
"testharnessreport": False}
required_elements = [key for key, value in {"testharness": True,
"testharnessreport": len(testharnessreport_nodes) > 0,
"timeout": len(source_file.timeout_nodes) > 0}.items()
if value]
for elem in source_file.root.iter():
if source_file.timeout_nodes and elem == source_file.timeout_nodes[0]:
seen_elements["timeout"] = True
if seen_elements["testharness"]:
errors.append(("LATE-TIMEOUT",
"<meta name=timeout> seen after testharness.js script", path, None))
elif elem == source_file.testharness_nodes[0]:
seen_elements["testharness"] = True
elif testharnessreport_nodes and elem == testharnessreport_nodes[0]:
seen_elements["testharnessreport"] = True
if not seen_elements["testharness"]:
errors.append(("EARLY-TESTHARNESSREPORT",
"testharnessreport.js script seen before testharness.js script", path, None))
if all(seen_elements[name] for name in required_elements):
break
if source_file.testdriver_nodes:
if len(source_file.testdriver_nodes) > 1:
errors.append(("MULTIPLE-TESTDRIVER",
"More than one <script src='/resources/testdriver.js'>", path, None))
testdriver_vendor_nodes = source_file.root.findall(".//{http://www.w3.org/1999/xhtml}script[@src='/resources/testdriver-vendor.js']")
if not testdriver_vendor_nodes:
errors.append(("MISSING-TESTDRIVER-VENDOR",
"Missing <script src='/resources/testdriver-vendor.js'>", path, None))
else:
if len(testdriver_vendor_nodes) > 1:
errors.append(("MULTIPLE-TESTDRIVER-VENDOR",
"More than one <script src='/resources/testdriver-vendor.js'>", path, None))
for element in source_file.root.findall(".//{http://www.w3.org/1999/xhtml}script[@src]"):
src = element.attrib["src"]
for name in ["testharness", "testharnessreport", "testdriver", "testdriver-vendor"]:
if "%s.js" % name == src or ("/%s.js" % name in src and src != "/resources/%s.js" % name):
errors.append(("%s-PATH" % name.upper(), "%s.js script seen with incorrect path" % name, path, None))
return errors
def _get_url_path(url):
# Remove schema from url
_, _, path, query, _ = urlsplit(url)
url = urlunsplit(("", "", path, query, ""))
return url
elif len(git_head_tags) > 1:
raise Exception('Git HEAD has multiple tags: '
+ ', '.join('"{}"'.format(t) for t in git_head_tags))
args.tag = git_head_tags[0]
assert args.tag
print('\nPublishing version "{}" on PyPI\n'.format(args.tag))
# get internal artifacts repository url
with io.open(Path.home() / '.pip' / 'pip.conf', 'r', encoding='utf-8') as fin:
repo_url = re.search(
r'^index-url = (?P<url>.+)$',
fin.read(),
flags=re.MULTILINE
).group('url')
repo_url = urlunsplit(
urlsplit(repo_url)._replace(path='', query='', fragment='')
)
# get artifact url corresponding to the version to publish
artifact_url = None
components_api = '/service/rest/beta/components?repository=bromine'
r = requests.get(urljoin(repo_url, components_api))
while not artifact_url:
r = r.json()
match = [i for i in r['items'] if i['version'] == args.tag]
if match:
artifact_data = match[0]['assets'][0]
artifact_url = urljoin(repo_url, '/repository/bromine/' + artifact_data['path'])
artifact_sha256 = artifact_data['checksum']['sha256']
else:
token = r['continuationToken']
def add_command(config, tasks):
keyed_by_args = [
"channel",
"archive-prefix",
"previous-archive-prefix",
"aus-server",
"override-certs",
"include-version",
"mar-channel-id-override",
"last-watershed",
]
optional_args = [
"updater-platform",
]
release_config = get_release_config(config)
for task in tasks:
task["description"] = "generate update verify config for {}".format(
task["attributes"]["build_platform"])
command = [
"python",
"testing/mozharness/scripts/release/update-verify-config-creator.py",
"--product",
task["extra"]["product"],
"--stage-product",
task["shipping-product"],
"--app-name",
task["extra"]["app-name"],
"--branch-prefix",
task["extra"]["branch-prefix"],
"--platform",
task["extra"]["platform"],
"--to-version",
release_config["version"],
"--to-app-version",
release_config["appVersion"],
"--to-build-number",
str(release_config["build_number"]),
"--to-buildid",
config.params["moz_build_date"],
"--to-revision",
get_branch_rev(config),
"--output-file",
"update-verify.cfg",
]
repo_path = urlparse.urlsplit(get_branch_repo(config)).path.lstrip("/")
command.extend(["--repo-path", repo_path])
if release_config.get("partial_versions"):
for partial in release_config["partial_versions"].split(","):
command.extend(
["--partial-version",
partial.split("build")[0]])
for arg in optional_args:
if task["extra"].get(arg):
command.append("--{}".format(arg))
command.append(task["extra"][arg])
for arg in keyed_by_args:
thing = "extra.{}".format(arg)
resolve_keyed_by(task,
thing,
item_name=task["name"],
platform=task["attributes"]["build_platform"],
**{
"release-type": config.params["release_type"],
"release-level":
config.params.release_level(),
})
# ignore things that resolved to null
if not task["extra"].get(arg):
continue
if arg == "include-version":
task["extra"][arg] = INCLUDE_VERSION_REGEXES[task["extra"]
[arg]]
if arg == "mar-channel-id-override":
task["extra"][arg] = MAR_CHANNEL_ID_OVERRIDE_REGEXES[
task["extra"][arg]]
command.append("--{}".format(arg))
command.append(task["extra"][arg])
task["run"].update({
"using": "mach",
"mach": " ".join(command),
})
yield task
def __init__(self):
normalize = lambda url: urlparse.urlsplit(url).geturl()
self.fetched = NormDict(normalize)
self.resolved = NormDict(normalize)
self.resolving = NormDict(normalize)
self.index = NormDict()
def get_driver(conf):
"""Return the configured driver."""
split = parse.urlsplit(conf.indexer.url)
d = driver.DriverManager('gnocchi.indexer', split.scheme).driver
return d(conf)
def has_in_url_path(url, sub):
"""Test if the `sub` string is in the `url` path."""
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
return sub in path
def redirect_tor(self, request):
_, _, path, query, frag = urlsplit(request.uri)
redirect_url = urlunsplit(
(b'http', State.tenant_cache[request.tid].onionnames[0], path,
query, frag))
self.redirect(request, redirect_url)
def redirect_https(self, request):
_, _, path, query, frag = urlsplit(request.uri)
redirect_url = urlunsplit(
(b'https', request.hostname, path, query, frag))
self.redirect(request, redirect_url)
def _get_url_base(self, url):
"""
Return the base of a URL
"""
s = urlsplit(url)
return urlunsplit([s.scheme, s.netloc, '', '', ''])
def parse_components(url):
parsed = parse.urlsplit(url)
query = parse.parse_qs(parsed.query)
return parsed._replace(query=''), query
def get_volume_api_version_from_endpoint(self):
magic_tuple = urlparse.urlsplit(self.management_url)
scheme, netloc, path, query, frag = magic_tuple
return path.lstrip('/').split('/')[0][1:]
def to_python(cls, data):
is_valid, errors = validate_and_default_interface(data, cls.path)
if not is_valid:
raise InterfaceValidationError("Invalid interface data")
kwargs = {}
if data.get('method'):
method = data['method'].upper()
# Optimize for the common path here, where it's a GET/POST, falling
# back to a regular expresion test
if method not in ('GET',
'POST') and not http_method_re.match(method):
raise InterfaceValidationError("Invalid value for 'method'")
kwargs['method'] = method
else:
kwargs['method'] = None
scheme, netloc, path, query_bit, fragment_bit = urlsplit(data['url'])
query_string = data.get('query_string') or query_bit
if query_string:
# if querystring was a dict, convert it to a string
if isinstance(query_string, dict):
query_string = urlencode([(to_bytes(k), to_bytes(v))
for k, v in query_string.items()])
else:
if query_string[0] == '?':
# remove '?' prefix
query_string = query_string[1:]
kwargs['query_string'] = trim(query_string, 4096)
else:
kwargs['query_string'] = ''
fragment = data.get('fragment') or fragment_bit
cookies = data.get('cookies')
# if cookies were [also] included in headers we
# strip them out
headers = data.get('headers')
if headers:
headers, cookie_header = format_headers(headers)
if not cookies and cookie_header:
cookies = cookie_header
else:
headers = ()
# We prefer the body to be a string, since we can then attempt to parse it
# as JSON OR decode it as a URL encoded query string, without relying on
# the correct content type header being passed.
body = data.get('data')
content_type = next((v for k, v in headers if k == 'Content-Type'),
None)
# Remove content type parameters
if content_type is not None:
content_type = content_type.partition(';')[0].rstrip()
# We process request data once during ingestion and again when
# requesting the http interface over the API. Avoid overwriting
# decoding the body again.
inferred_content_type = data.get('inferred_content_type', content_type)
if 'inferred_content_type' not in data and not isinstance(body, dict):
body, inferred_content_type = heuristic_decode(body, content_type)
if body:
body = trim(body, settings.SENTRY_MAX_HTTP_BODY_SIZE)
kwargs['inferred_content_type'] = inferred_content_type
kwargs['cookies'] = trim_pairs(format_cookies(cookies))
kwargs['env'] = trim_dict(data.get('env') or {})
kwargs['headers'] = trim_pairs(headers)
kwargs['data'] = fix_broken_encoding(body)
kwargs['url'] = urlunsplit((scheme, netloc, path, '', ''))
kwargs['fragment'] = trim(fragment, 1024)
return cls(**kwargs)
def _endpoint_from_url(self, url):
app, name = urlsplit(url).path.split("/")[2:4]
return getattr(pynetbox.core.app.App(self.api, app), name)
def url(self, url):
if urlparse.urlsplit(url).netloc is None:
return self.url(url)
body = {"url": url}
return self.send_session_command("POST", "url", body)
def load_resource(self, cdx, params):
if cdx.get('filename') and cdx.get('offset') is not None:
return None
load_url = cdx.get('load_url')
if not load_url:
return None
if params.get('content_type') == VideoLoader.CONTENT_TYPE:
return None
if self.forward_proxy_prefix and not cdx.get('is_live'):
load_url = self.forward_proxy_prefix + load_url
input_req = params['_input_req']
req_headers = input_req.get_req_headers()
dt = timestamp_to_datetime(cdx['timestamp'])
if cdx.get('memento_url'):
req_headers['Accept-Datetime'] = datetime_to_http_date(dt)
method = input_req.get_req_method()
data = input_req.get_req_body()
p = PreparedRequest()
try:
p.prepare_url(load_url, None)
except Exception:
raise LiveResourceException(load_url)
p.prepare_headers(None)
p.prepare_auth(None, load_url)
auth = p.headers.get('Authorization')
if auth:
req_headers['Authorization'] = auth
load_url = p.url
# host is set to the actual host for live loading
# ensure it is set to the load_url host
if not cdx.get('is_live'):
#req_headers.pop('Host', '')
req_headers['Host'] = urlsplit(p.url).netloc
referrer = cdx.get('set_referrer')
if referrer:
req_headers['Referer'] = referrer
upstream_res = self._do_request_with_redir_check(
method, load_url, data, req_headers, params, cdx)
memento_dt = upstream_res.headers.get('Memento-Datetime')
if memento_dt:
dt = http_date_to_datetime(memento_dt)
cdx['timestamp'] = datetime_to_timestamp(dt)
elif cdx.get('memento_url'):
# if 'memento_url' set and no Memento-Datetime header present
# then its an error
no_except_close(upstream_res)
return None
agg_type = upstream_res.headers.get('Warcserver-Type')
if agg_type == 'warc':
cdx['source'] = unquote(
upstream_res.headers.get('Warcserver-Source-Coll'))
return None, upstream_res.headers, upstream_res
if upstream_res.version == 11:
version = '1.1'
else:
version = '1.0'
status = 'HTTP/{version} {status} {reason}\r\n'
status = status.format(version=version,
status=upstream_res.status,
reason=upstream_res.reason)
http_headers_buff = status
orig_resp = upstream_res._original_response
try: #pragma: no cover
#PY 3
resp_headers = orig_resp.headers._headers
for n, v in resp_headers:
nl = n.lower()
if nl in self.SKIP_HEADERS:
continue
if nl in self.UNREWRITE_HEADERS:
v = self.unrewrite_header(cdx, v)
http_headers_buff += n + ': ' + v + '\r\n'
http_headers_buff += '\r\n'
try:
# http headers could be encoded as utf-8 (though non-standard)
# first try utf-8 encoding
http_headers_buff = http_headers_buff.encode('utf-8')
except:
# then, fall back to latin-1
http_headers_buff = http_headers_buff.encode('latin-1')
except: #pragma: no cover
#PY 2
resp_headers = orig_resp.msg.headers
for line in resp_headers:
n, v = line.split(':', 1)
n = n.lower()
v = v.strip()
if n in self.SKIP_HEADERS:
continue
new_v = v
if n in self.UNREWRITE_HEADERS:
new_v = self.unrewrite_header(cdx, v)
if new_v != v:
http_headers_buff += n + ': ' + new_v + '\r\n'
else:
http_headers_buff += line
# if python2, already byte headers, so leave as is
http_headers_buff += '\r\n'
try:
fp = upstream_res._fp.fp
if hasattr(fp, 'raw'): #pragma: no cover
fp = fp.raw
remote_ip = fp._sock.getpeername()[0]
except: #pragma: no cover
remote_ip = None
warc_headers = {}
warc_headers['WARC-Type'] = 'response'
warc_headers['WARC-Record-ID'] = self._make_warc_id()
warc_headers['WARC-Target-URI'] = cdx['url']
warc_headers['WARC-Date'] = datetime_to_iso_date(dt)
if not cdx.get('is_live'):
now = datetime.datetime.utcnow()
warc_headers['WARC-Source-URI'] = cdx.get('load_url')
warc_headers['WARC-Creation-Date'] = datetime_to_iso_date(now)
if remote_ip:
warc_headers['WARC-IP-Address'] = remote_ip
ct = upstream_res.headers.get('Content-Type')
if ct:
metadata = self.get_custom_metadata(ct, dt)
if metadata:
warc_headers['WARC-JSON-Metadata'] = json.dumps(metadata)
warc_headers['Content-Type'] = 'application/http; msgtype=response'
if method == 'HEAD':
content_len = 0
else:
content_len = upstream_res.headers.get('Content-Length', -1)
self._set_content_len(content_len, warc_headers,
len(http_headers_buff))
warc_headers = StatusAndHeaders('WARC/1.0', warc_headers.items())
return (warc_headers, http_headers_buff, upstream_res)
def __init__(self, url=''):
if url:
self._elem = list(urllib_parse.urlsplit(url))
else:
self._elem = []
def resolve_image(self, src):
description = ''
if urlsplit(src)[0]:
# We have a scheme
return None, None, src, description
base = self.context
subpath = src
appendix = ''
def traversal_stack(base, path):
if path.startswith('/'):
base = getSite()
path = path[1:]
obj = base
stack = [obj]
components = path.split('/')
while components:
child_id = unquote(components.pop(0))
try:
if hasattr(aq_base(obj), 'scale'):
if components:
child = obj.scale(child_id, components.pop())
else:
child = obj.scale(child_id)
else:
# Do not use restrictedTraverse here; the path to the
# image may lead over containers that lack the View
# permission for the current user!
# Also, if the image itself is not viewable, we rather
# show a broken image than hide it or raise
# unauthorized here (for the referring document).
child = obj.unrestrictedTraverse(str(child_id))
except ConflictError:
raise
except (AttributeError, KeyError, NotFound, ztkNotFound):
return
obj = child
stack.append(obj)
return stack
def traverse_path(base, path):
stack = traversal_stack(base, path)
if stack is None:
return
return stack[-1]
obj, subpath, appendix = self.resolve_link(src)
if obj is not None:
# resolved uid
fullimage = obj
image = traverse_path(fullimage, subpath)
elif '/@@' in subpath:
# split on view
pos = subpath.find('/@@')
fullimage = traverse_path(base, subpath[:pos])
if fullimage is None:
return None, None, src, description
image = traverse_path(fullimage, subpath[pos + 1:])
else:
stack = traversal_stack(base, subpath)
if stack is None:
return None, None, src, description
image = stack.pop()
# if it's a scale, find the full image by traversing one less
fullimage = image
if not IContentish.providedBy(fullimage):
stack.reverse()
for parent in stack:
if hasattr(aq_base(parent), 'tag'):
fullimage = parent
break
if image is None:
return None, None, src, description
try:
url = image.absolute_url()
except AttributeError:
return None, None, src, description
src = url + appendix
description = safe_unicode(aq_acquire(fullimage, 'Description')())
return image, fullimage, src, description
def payment_domain_name(self):
if self.enable_microfrontend_for_basket_page:
return urlsplit(self.payment_microfrontend_url).netloc
return self.site.domain
def __init__(self,
hostname,
username=None,
password=None,
authinfo=None,
protocol=None,
port=None,
verify_ssl=None):
super(Session, self).__init__()
# Determine whether or not server SSL certificates should be verified.
if verify_ssl is None:
verify_ssl = os.environ.get('SSLREQCERT', 'yes')
verify_ssl = str(verify_ssl).lower() not in ('no', 'false')
self._id = uuid4().hex
self.message_log = logger.getChild('session.%s' % self._id)
# If certificate path has already been set for SWAT package, make
# Requests module reuse it.
for k in ['SSLCALISTLOC', 'CAS_CLIENT_SSL_CA_LIST']:
if k in os.environ:
os.environ['REQUESTS_CA_BUNDLE'] = os.environ[k]
break
# If certificate path hasn't been specified in either environment
# variable, replace the default adapter with one that will use the
# machine's default SSL _settings.
if 'REQUESTS_CA_BUNDLE' not in os.environ:
if verify_ssl:
# Skip hostname verification if IP address specified instead
# of DNS name. Prevents error from urllib3.
try:
from urllib3.util.ssl_ import is_ipaddress
except ImportError:
# is_ipaddres not present in older versions of urllib3
def is_ipaddress(hst):
return re.match(r"^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$",
hst)
verify_hostname = not is_ipaddress(hostname)
adapter = SSLContextAdapter(assert_hostname=verify_hostname)
self.mount('https://', adapter)
else:
# Every request will generate an InsecureRequestWarning
from urllib3.exceptions import InsecureRequestWarning
warnings.simplefilter('default', InsecureRequestWarning)
self.filters = DEFAULT_FILTERS
# Used for context manager
self._old_session = None
# Reuse an existing CAS connection if possible
if swat and isinstance(hostname, swat.CAS):
if isinstance(hostname._sw_connection,
swat.cas.rest.connection.REST_CASConnection):
import base64
# Use the httpAddress action to retieve information about
# REST endpoints
httpAddress = hostname.get_action('builtins.httpAddress')
address = httpAddress()
domain = address.virtualHost
# httpAddress action may return virtualHost = ''
# if this happens, try the CAS host
if not domain:
domain = hostname._sw_connection._current_hostname
protocol = address.protocol
port = address.port
auth = hostname._sw_connection._auth.decode('utf-8').replace(
'Basic ', '')
username, password = base64.b64decode(auth).decode(
'utf-8').split(':')
else:
raise ValueError("A 'swat.CAS' session can only be reused "
"when it's connected via the REST APIs.")
else:
url = urlsplit(hostname)
# Extract http/https from domain name if present and protocl not
# explicitly given
protocol = protocol or url.scheme
domain = url.hostname or str(hostname)
self._settings = {
'protocol': protocol or 'https',
'domain': domain,
'port': port,
'username': username,
'password': password
}
if self._settings['password'] is None:
# Try to get credentials from .authinfo or .netrc files.
# If no file path was specified, the default locations will
# be checked.
if 'swat' in sys.modules:
auth = swat.utils.authinfo.query_authinfo(domain,
user=username,
path=authinfo)
self._settings['username'] = auth.get('user')
self._settings['password'] = auth.get('password')
# Not able to load credentials using SWAT. Try Netrc.
# TODO: IF a username was specified, verify that the credentials
# found are for that username.
if self._settings['password'] is None:
try:
parser = netrc.netrc(authinfo)
values = parser.authenticators(domain)
if values:
self._settings['username'], \
_, \
self._settings['password'] = values
except (OSError, IOError):
pass # netrc throws if $HOME is not set
self.verify = verify_ssl
self.auth = HTTPBearerAuth(self.get_token())
if current_session() is None:
current_session(self)
def __init__(self, **settings):
# check if some of the settings provided as class attributes
# should be overwritten
for name, value in settings.items():
if hasattr(self, name):
setattr(self, name, value)
validate_settings(self)
self.last_headers_name = None
self.last_headers_value = None
self.os_options = {
'tenant_id': self.tenant_id,
'tenant_name': self.tenant_name,
'user_domain_id': self.user_domain_id,
'user_domain_name': self.user_domain_name,
'project_domain_id': self.project_domain_id,
'project_domain_name': self.project_domain_name,
'region_name': self.region_name,
}
self.os_options.update(self.os_extra_options)
# Get authentication token
self.storage_url, self.token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options=self.os_options)
self.http_conn = swiftclient.http_connection(self.storage_url)
# Check container
try:
swiftclient.head_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn)
except swiftclient.ClientException:
headers = {}
if self.auto_create_container:
if self.auto_create_container_public:
headers['X-Container-Read'] = '.r:*'
if self.auto_create_container_allow_orgin:
headers['X-Container-Meta-Access-Control-Allow-Origin'] = \
self.auto_create_container_allow_orgin
swiftclient.put_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn,
headers=headers)
else:
raise ImproperlyConfigured("Container %s does not exist." %
self.container_name)
if self.auto_base_url:
# Derive a base URL based on the authentication information from
# the server, optionally overriding the protocol, host/port and
# potentially adding a path fragment before the auth information.
self.base_url = self.storage_url + '/'
if self.override_base_url is not None:
# override the protocol and host, append any path fragments
split_derived = urlparse.urlsplit(self.base_url)
split_override = urlparse.urlsplit(self.override_base_url)
split_result = [''] * 5
split_result[0:2] = split_override[0:2]
split_result[2] = (split_override[2] +
split_derived[2]).replace('//', '/')
self.base_url = urlparse.urlunsplit(split_result)
self.base_url = urlparse.urljoin(self.base_url,
self.container_name)
self.base_url += '/'
else:
self.base_url = self.override_base_url
def setAttribute(self, name, value):
if log.ThugOpts.features_logging:
log.ThugLogging.Features.increase_setattribute_count()
if not isinstance(name, six.string_types):
name = str(name)
if log.ThugOpts.Personality.isFirefox():
if name in ('style', ):
svalue = value.split('-')
_value = svalue[0]
if len(svalue) > 1:
_value = '{}{}'.format(
_value, ''.join([s.capitalize() for s in svalue[1:]]))
for css in [
p for p in FF_STYLES
if log.ThugOpts.Personality.browserMajorVersion >= p[0]
]:
if css[1] in value:
self.tag[name] = _value
return
if name in ('type', ):
for _input in [
p for p in FF_INPUTS
if log.ThugOpts.Personality.browserMajorVersion > p[0]
]:
if _input[1] in value:
self.tag[name] = value
return
self.tag[name] = value
if name.lower() in ('src', 'archive'):
s = urlparse.urlsplit(value)
handler = getattr(log.SchemeHandler, 'handle_%s' % (s.scheme, ),
None)
if handler:
handler(self.doc.window, value)
return
try:
response = self.doc.window._navigator.fetch(
value, redirect_type="element workaround")
except Exception:
return
if response is None:
return
if response.status_code == 404:
return
ctype = response.headers.get('content-type', None)
if ctype is None:
return
handler = log.MIMEHandler.get_handler(ctype)
if handler:
handler(self.doc.window.url, response.content)
def test_upload_from_file_resumable(self):
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _Monkey
from gcloud._testing import _NamedTemporaryFile
from gcloud.streaming import http_wrapper
from gcloud.streaming import transfer
BLOB_NAME = 'blob-name'
UPLOAD_URL = 'http://example.com/upload/name/key'
DATA = b'ABCDEF'
loc_response = {'status': OK, 'location': UPLOAD_URL}
chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE,
'range': 'bytes 0-4'}
chunk2_response = {'status': OK}
# Need valid JSON on last response, since resumable.
connection = _Connection(
(loc_response, b''),
(chunk1_response, b''),
(chunk2_response, b'{}'),
)
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
# Set the threshhold low enough that we force a resumable uploada.
with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
with _NamedTemporaryFile() as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
with open(temp.name, 'rb') as file_obj:
blob.upload_from_file(file_obj, rewind=True)
rq = connection.http._requested
self.assertEqual(len(rq), 3)
# Requested[0]
headers = dict(
[(x.title(), str(y)) for x, y in rq[0].pop('headers').items()])
self.assertEqual(headers['X-Upload-Content-Length'], '6')
self.assertEqual(headers['X-Upload-Content-Type'],
'application/octet-stream')
uri = rq[0].pop('uri')
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/b/name/o')
self.assertEqual(dict(parse_qsl(qs)),
{'uploadType': 'resumable', 'name': BLOB_NAME})
self.assertEqual(rq[0], {
'method': 'POST',
'body': '',
'connection_type': None,
'redirections': 5,
})
# Requested[1]
headers = dict(
[(x.title(), str(y)) for x, y in rq[1].pop('headers').items()])
self.assertEqual(headers['Content-Range'], 'bytes 0-4/6')
self.assertEqual(rq[1], {
'method': 'PUT',
'uri': UPLOAD_URL,
'body': DATA[:5],
'connection_type': None,
'redirections': 5,
})
# Requested[2]
headers = dict(
[(x.title(), str(y)) for x, y in rq[2].pop('headers').items()])
self.assertEqual(headers['Content-Range'], 'bytes 5-5/6')
self.assertEqual(rq[2], {
'method': 'PUT',
'uri': UPLOAD_URL,
'body': DATA[5:],
'connection_type': None,
'redirections': 5,
})
def get_path(url):
p = urlsplit(url)
return urlunsplit(['', '', p.path or '/', p.query, p.fragment])
def get_link_from_line(cls, line):
"""Parse link information from given requirement line.
Return a 6-tuple:
- `vcs_type` indicates the VCS to use (e.g. "git"), or None.
- `prefer` is either "file", "path" or "uri", indicating how the
information should be used in later stages.
- `relpath` is the relative path to use when recording the dependency,
instead of the absolute path/URI used to perform installation.
This can be None (to prefer the absolute path or URI).
- `path` is the absolute file path to the package. This will always use
forward slashes. Can be None if the line is a remote URI.
- `uri` is the absolute URI to the package. Can be None if the line is
not a URI.
- `link` is an instance of :class:`pip._internal.index.Link`,
representing a URI parse result based on the value of `uri`.
This function is provided to deal with edge cases concerning URIs
without a valid netloc. Those URIs are problematic to a straight
``urlsplit` call because they cannot be reliably reconstructed with
``urlunsplit`` due to a bug in the standard library:
>>> from urllib.parse import urlsplit, urlunsplit
>>> urlunsplit(urlsplit('git+file:///this/breaks'))
'git+file:/this/breaks'
>>> urlunsplit(urlsplit('file:///this/works'))
'file:///this/works'
See `https://bugs.python.org/issue23505#msg277350`.
"""
# Git allows `[email protected]...` lines that are not really URIs.
# Add "ssh://" so we can parse correctly, and restore afterwards.
fixed_line = add_ssh_scheme_to_git_uri(line)
added_ssh_scheme = fixed_line != line
# We can assume a lot of things if this is a local filesystem path.
if "://" not in fixed_line:
p = Path(fixed_line).absolute()
path = p.as_posix()
uri = p.as_uri()
link = Link(uri)
try:
relpath = get_converted_relative_path(path)
except ValueError:
relpath = None
return LinkInfo(None, "path", relpath, path, uri, link)
# This is an URI. We'll need to perform some elaborated parsing.
parsed_url = urllib_parse.urlsplit(fixed_line)
if added_ssh_scheme and ':' in parsed_url.netloc:
original_netloc, original_path_start = parsed_url.netloc.rsplit(':', 1)
uri_path = '/{0}{1}'.format(original_path_start, parsed_url.path)
original_url = parsed_url
parsed_url = original_url._replace(netloc=original_netloc, path=uri_path)
# Split the VCS part out if needed.
original_scheme = parsed_url.scheme
if "+" in original_scheme:
vcs_type, scheme = original_scheme.split("+", 1)
parsed_url = parsed_url._replace(scheme=scheme)
prefer = "uri"
else:
vcs_type = None
prefer = "file"
if parsed_url.scheme == "file" and parsed_url.path:
# This is a "file://" URI. Use url_to_path and path_to_url to
# ensure the path is absolute. Also we need to build relpath.
path = Path(url_to_path(urllib_parse.urlunsplit(parsed_url))).as_posix()
try:
relpath = get_converted_relative_path(path)
except ValueError:
relpath = None
uri = path_to_url(path)
else:
# This is a remote URI. Simply use it.
path = None
relpath = None
# Cut the fragment, but otherwise this is fixed_line.
uri = urllib_parse.urlunsplit(
parsed_url._replace(scheme=original_scheme, fragment="")
)
if added_ssh_scheme:
original_uri = urllib_parse.urlunsplit(original_url._replace(scheme=original_scheme, fragment=""))
uri = strip_ssh_from_git_uri(original_uri)
# Re-attach VCS prefix to build a Link.
link = Link(
urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme))
)
return LinkInfo(vcs_type, prefer, relpath, path, uri, link)
def has_in_url_path(url, subs):
"""Test if any of `subs` strings is present in the `url` path."""
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
return any([sub in path for sub in subs])
