def to_python(self, value):
def split_url(url):
"""
Returns a list of url parts via ``urlparse.urlsplit`` (or raises a
``ValidationError`` exception for certain).
"""
try:
return list(urlsplit(url))
except ValueError:
# urlparse.urlsplit can raise a ValueError with some
# misformatted URLs.
raise ValidationError(self.error_messages['invalid'], code='invalid')
value = super(URLField, self).to_python(value)
if value:
url_fields = split_url(value)
if not url_fields[0]:
# If no URL scheme given, assume http://
url_fields[0] = 'http'
if not url_fields[1]:
# Assume that if no domain is provided, that the path segment
# contains the domain.
url_fields[1] = url_fields[2]
url_fields[2] = ''
# Rebuild the url_fields list, since the domain segment may now
# contain the path too.
url_fields = split_url(urlunsplit(url_fields))
value = urlunsplit(url_fields)
return value
def set_language(request):
"""
Since this view changes how the user will see the rest of the site, it must
only be accessed as a POST request. If called as a GET request, it will
error.
"""
lang_code = request.POST.get(LANGUAGE_QUERY_PARAMETER)
next_url = urlsplit(request.POST.get("next")) if request.POST.get("next") else None
if lang_code and check_for_language(lang_code):
if next_url and is_valid_path(next_url.path):
# If it is a recognized Kolibri path, then translate it to the new language and return it.
next_path = urlunsplit(
(
next_url[0],
next_url[1],
translate_url(next_url[2], lang_code),
next_url[3],
next_url[4],
)
)
else:
next_path = translate_url(reverse("kolibri:core:redirect_user"), lang_code)
response = HttpResponse(next_path)
if hasattr(request, "session"):
request.session[LANGUAGE_SESSION_KEY] = lang_code
# Always set cookie
response.set_cookie(
settings.LANGUAGE_COOKIE_NAME,
lang_code,
max_age=settings.LANGUAGE_COOKIE_AGE,
path=settings.LANGUAGE_COOKIE_PATH,
domain=settings.LANGUAGE_COOKIE_DOMAIN,
)
else:
lang_code = (
get_device_language()
or get_accept_headers_language(request)
or get_settings_language()
)
if next_url and is_valid_path(next_url.path):
# If it is a recognized Kolibri path, then translate it using the default language code for this device
next_path = urlunsplit(
(
next_url[0],
next_url[1],
translate_url(next_url[2], lang_code),
next_url[3],
next_url[4],
)
)
else:
next_path = translate_url(reverse("kolibri:core:redirect_user"), lang_code)
response = HttpResponse(next_path)
if hasattr(request, "session"):
request.session.pop(LANGUAGE_SESSION_KEY, "")
response.delete_cookie(settings.LANGUAGE_COOKIE_NAME)
return response
def hashed_name(self, name, content=None, filename=None):
parsed_name = urlsplit(unquote(name))
clean_name = parsed_name.path.strip()
opened = False
if content is None:
absolute_path = finders.find(clean_name)
try:
content = open(absolute_path, 'rb')
except (IOError, OSError) as e:
if e.errno == errno.ENOENT:
raise ValueError("The file '%s' could not be found with %r." % (clean_name, self))
else:
raise
content = File(content)
opened = True
try:
file_hash = self.file_hash(clean_name, content)
finally:
if opened:
content.close()
path, filename = os.path.split(clean_name)
root, ext = os.path.splitext(filename)
if file_hash is not None:
file_hash = ".%s" % file_hash
hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext))
unparsed_name = list(parsed_name)
unparsed_name[2] = hashed_name
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
if '?#' in name and not unparsed_name[3]:
unparsed_name[2] += '?'
return urlunsplit(unparsed_name)
def translate_url(url, lang_code):
"""
Given a URL (absolute or relative), try to get its translated version in
the `lang_code` language (either by i18n_patterns or by translated regex).
Return the original URL if no translated version is found.
"""
parsed = urlsplit(url)
try:
match = resolve(parsed.path)
except Resolver404:
pass
else:
to_be_reversed = "%s:%s" % (match.namespace, match.url_name
) if match.namespace else match.url_name
with override(lang_code):
try:
url = reverse(to_be_reversed,
args=match.args,
kwargs=match.kwargs)
except NoReverseMatch:
pass
else:
url = urlunsplit((parsed.scheme, parsed.netloc, url,
parsed.query, parsed.fragment))
return url
def hashed_name(self, name, content=None):
parsed_name = urlsplit(unquote(name))
clean_name = parsed_name.path.strip()
opened = False
if content is None:
if not self.exists(clean_name):
raise ValueError("The file '%s' could not be found with %r." %
(clean_name, self))
try:
content = self.open(clean_name)
except IOError:
# Handle directory paths and fragments
return name
opened = True
try:
file_hash = self.file_hash(clean_name, content)
finally:
if opened:
content.close()
path, filename = os.path.split(clean_name)
root, ext = os.path.splitext(filename)
if file_hash is not None:
file_hash = ".%s" % file_hash
hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext))
unparsed_name = list(parsed_name)
unparsed_name[2] = hashed_name
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
if '?#' in name and not unparsed_name[3]:
unparsed_name[2] += '?'
return urlunsplit(unparsed_name)
def __call__(self, value):
value = force_text(value)
# Check first if the scheme is valid
scheme = value.split('://')[0].lower()
if scheme not in self.schemes:
raise ValidationError(self.message, code=self.code)
# Then check full URL
try:
super(URLValidator, self).__call__(value)
except ValidationError as e:
# Trivial case failed. Try for possible IDN domain
if value:
scheme, netloc, path, query, fragment = urlsplit(value)
try:
netloc = netloc.encode('idna').decode(
'ascii') # IDN -> ACE
except UnicodeError: # invalid domain part
raise e
url = urlunsplit((scheme, netloc, path, query, fragment))
super(URLValidator, self).__call__(url)
else:
raise
else:
# Now verify IPv6 in the netloc part
host_match = re.search(r'^\[(.+)\](?::\d{2,5})?$',
urlsplit(value).netloc)
if host_match:
potential_ip = host_match.groups()[0]
try:
validate_ipv6_address(potential_ip)
except ValidationError:
raise ValidationError(self.message, code=self.code)
url = value
def is_valid_repository(self):
"""Checks if this is a valid Git repository."""
url_parts = urlsplit(self.path)
if (url_parts.scheme.lower() in ('http', 'https') and
url_parts.username is None and self.username):
# Git URLs, especially HTTP(s), that require authentication should
# be entered without the authentication info in the URL (because
# then it would be visible), but we need it in the URL when testing
# to make sure it exists. Reformat the path here to include them.
new_netloc = urlquote(self.username, safe='')
if self.password:
new_netloc += ':' + urlquote(self.password, safe='')
new_netloc += '@' + url_parts.netloc
path = urlunsplit((url_parts[0], new_netloc, url_parts[2],
url_parts[3], url_parts[4]))
else:
path = self.path
p = self._run_git(['ls-remote', path, 'HEAD'])
errmsg = p.stderr.read()
failure = p.wait()
if failure:
logging.error("Git: Failed to find valid repository %s: %s" %
(self.path, errmsg))
return False
return True
def url(self, name, force=False):
"""
Return the real URL in DEBUG mode.
"""
if settings.DEBUG and not force:
hashed_name, fragment = name, ''
else:
clean_name, fragment = urldefrag(name)
if urlsplit(clean_name).path.endswith('/'): # don't hash paths
hashed_name = name
else:
hashed_name = self.stored_name(clean_name)
final_url = super(HashedFilesMixin, self).url(hashed_name)
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
query_fragment = '?#' in name # [sic!]
if fragment or query_fragment:
urlparts = list(urlsplit(final_url))
if fragment and not urlparts[4]:
urlparts[4] = fragment
if query_fragment and not urlparts[3]:
urlparts[2] += '?'
final_url = urlunsplit(urlparts)
return unquote(final_url)
def hashed_name(self, name, content=None, filename=None):
# `filename` is the name of file to hash if `content` isn't given.
# `name` is the base name to construct the new hashed filename from.
parsed_name = urlsplit(unquote(name))
clean_name = parsed_name.path.strip()
if filename:
filename = urlsplit(unquote(filename)).path.strip()
filename = filename or clean_name
opened = False
if content is None:
try:
content = self.open(filename)
except IOError:
# Handle directory paths and fragments
return name
opened = True
try:
file_hash = self.file_hash(clean_name, content)
finally:
if opened:
content.close()
path, filename = os.path.split(clean_name)
root, ext = os.path.splitext(filename)
if file_hash is not None:
file_hash = ".%s" % file_hash
hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext))
unparsed_name = list(parsed_name)
unparsed_name[2] = hashed_name
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
if '?#' in name and not unparsed_name[3]:
unparsed_name[2] += '?'
return urlunsplit(unparsed_name)
def hashed_name(self, name, content=None):
parsed_name = urlsplit(unquote(name))
clean_name = parsed_name.path.strip()
opened = False
if content is None:
if not self.exists(clean_name):
raise ValueError("The file '%s' could not be found with %r." %
(clean_name, self))
try:
content = self.open(clean_name)
except IOError:
# Handle directory paths and fragments
return name
opened = True
try:
file_hash = self.file_hash(clean_name, content)
finally:
if opened:
content.close()
path, filename = os.path.split(clean_name)
root, ext = os.path.splitext(filename)
if file_hash is not None:
file_hash = ".%s" % file_hash
hashed_name = os.path.join(path, "%s%s%s" %
(root, file_hash, ext))
unparsed_name = list(parsed_name)
unparsed_name[2] = hashed_name
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
if '?#' in name and not unparsed_name[3]:
unparsed_name[2] += '?'
return urlunsplit(unparsed_name)
def _add_query_params(self, url, new_query_params):
"""Add query parameters onto the given URL.
Args:
url (unicode):
The URL to add query parameters to.
new_query_params (dict):
The query parameters to add.
Returns:
unicode:
The resulting URL.
"""
scheme, netloc, path, query_string, fragment = urlsplit(url)
query_params = parse_qs(query_string)
query_params.update(new_query_params)
new_query_string = urlencode(
[
(key, value)
for key, value in sorted(six.iteritems(query_params),
key=lambda i: i[0])
],
doseq=True)
return urlunsplit((scheme, netloc, path, new_query_string, fragment))
def url(self, name, force=False):
"""
Returns the real URL in DEBUG mode.
"""
if settings.DEBUG and not force:
hashed_name, fragment = name, ""
else:
clean_name, fragment = urldefrag(name)
if urlsplit(clean_name).path.endswith("/"): # don't hash paths
hashed_name = name
else:
cache_key = self.cache_key(name)
hashed_name = self.cache.get(cache_key)
if hashed_name is None:
hashed_name = self.hashed_name(clean_name).replace("\\", "/")
# set the cache if there was a miss
# (e.g. if cache server goes down)
self.cache.set(cache_key, hashed_name)
final_url = super(CachedFilesMixin, self).url(hashed_name)
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
query_fragment = "?#" in name # [sic!]
if fragment or query_fragment:
urlparts = list(urlsplit(final_url))
if fragment and not urlparts[4]:
urlparts[4] = fragment
if query_fragment and not urlparts[3]:
urlparts[2] += "?"
final_url = urlunsplit(urlparts)
return unquote(final_url)
def assertRedirects(self, response, expected_url, status_code=302,
target_status_code=200, host=None, msg_prefix='',
fetch_redirect_response=True):
"""Asserts that a response redirected to a specific URL, and that the
redirect URL can be loaded.
Note that assertRedirects won't work for external links since it uses
TestClient to do a request (use fetch_redirect_response=False to check
such links without fetching thtem).
"""
if msg_prefix:
msg_prefix += ": "
if hasattr(response, 'redirect_chain'):
# The request was a followed redirect
self.assertTrue(len(response.redirect_chain) > 0,
msg_prefix + "Response didn't redirect as expected: Response"
" code was %d (expected %d)" %
(response.status_code, status_code))
self.assertEqual(response.redirect_chain[0][1], status_code,
msg_prefix + "Initial response didn't redirect as expected:"
" Response code was %d (expected %d)" %
(response.redirect_chain[0][1], status_code))
url, status_code = response.redirect_chain[-1]
self.assertEqual(response.status_code, target_status_code,
msg_prefix + "Response didn't redirect as expected: Final"
" Response code was %d (expected %d)" %
(response.status_code, target_status_code))
else:
# Not a followed redirect
self.assertEqual(response.status_code, status_code,
msg_prefix + "Response didn't redirect as expected: Response"
" code was %d (expected %d)" %
(response.status_code, status_code))
url = response.url
scheme, netloc, path, query, fragment = urlsplit(url)
if fetch_redirect_response:
redirect_response = response.client.get(path, QueryDict(query))
# Get the redirection page, using the same client that was used
# to obtain the original response.
self.assertEqual(redirect_response.status_code, target_status_code,
msg_prefix + "Couldn't retrieve redirection page '%s':"
" response code was %d (expected %d)" %
(path, redirect_response.status_code, target_status_code))
e_scheme, e_netloc, e_path, e_query, e_fragment = urlsplit(expected_url)
if not (e_scheme or e_netloc):
expected_url = urlunsplit(('http', host or 'testserver', e_path,
e_query, e_fragment))
self.assertEqual(url, expected_url,
msg_prefix + "Response redirected to '%s', expected '%s'" %
(url, expected_url))
def _do_on_path(self, cb, path, revision=HEAD):
if not path:
raise FileNotFoundError(path, revision)
try:
normpath = self.normalize_path(path)
# SVN expects to have URLs escaped. Take care to only
# escape the path part of the URL.
if self.client.is_url(normpath):
pathtuple = urlsplit(normpath)
path = pathtuple[2]
if isinstance(path, six.text_type):
path = path.encode("utf-8", "ignore")
normpath = urlunsplit((pathtuple[0], pathtuple[1], quote(path), "", ""))
normrev = self._normalize_revision(revision)
return cb(normpath, normrev)
except ClientError as e:
exc = bytes(e).decode("utf-8")
if "File not found" in exc or "path not found" in exc:
raise FileNotFoundError(path, revision, detail=exc)
elif "callback_ssl_server_trust_prompt required" in exc:
raise SCMError(
_(
"HTTPS certificate not accepted. Please ensure that "
"the proper certificate exists in %s "
"for the user that reviewboard is running as."
)
% os.path.join(self.config_dir, "auth")
)
else:
raise SVNTool.normalize_error(e)
def url(self, name, force=False):
"""
Returns the real URL in DEBUG mode.
"""
if settings.DEBUG and not force:
hashed_name, fragment = name, ''
else:
clean_name, fragment = urldefrag(name)
if urlsplit(clean_name).path.endswith('/'): # don't hash paths
hashed_name = name
else:
cache_key = self.cache_key(name)
hashed_name = self.cache.get(cache_key)
if hashed_name is None:
hashed_name = self.hashed_name(clean_name).replace(
'\\', '/')
# set the cache if there was a miss
# (e.g. if cache server goes down)
self.cache.set(cache_key, hashed_name)
final_url = super(CachedFilesMixin, self).url(hashed_name)
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
query_fragment = '?#' in name # [sic!]
if fragment or query_fragment:
urlparts = list(urlsplit(final_url))
if fragment and not urlparts[4]:
urlparts[4] = fragment
if query_fragment and not urlparts[3]:
urlparts[2] += '?'
final_url = urlunsplit(urlparts)
return unquote(final_url)
def url(self, name, force=False):
"""
Return the real URL in DEBUG mode.
"""
if settings.DEBUG and not force:
hashed_name, fragment = name, ''
else:
clean_name, fragment = urldefrag(name)
if urlsplit(clean_name).path.endswith('/'): # don't hash paths
hashed_name = name
else:
hashed_name = self.stored_name(clean_name)
final_url = super(HashedFilesMixin, self).url(hashed_name)
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
query_fragment = '?#' in name # [sic!]
if fragment or query_fragment:
urlparts = list(urlsplit(final_url))
if fragment and not urlparts[4]:
urlparts[4] = fragment
if query_fragment and not urlparts[3]:
urlparts[2] += '?'
final_url = urlunsplit(urlparts)
return unquote(final_url)
def hashed_name(self, name, content=None, filename=None):
parsed_name = urlsplit(unquote(name))
clean_name = parsed_name.path.strip()
opened = False
if content is None:
absolute_path = finders.find(clean_name)
try:
content = open(absolute_path, 'rb')
except (IOError, OSError) as e:
if e.errno == errno.ENOENT:
raise ValueError(
"The file '%s' could not be found with %r." %
(clean_name, self))
else:
raise
content = File(content)
opened = True
try:
file_hash = self.file_hash(clean_name, content)
finally:
if opened:
content.close()
path, filename = os.path.split(clean_name)
root, ext = os.path.splitext(filename)
if file_hash is not None:
file_hash = ".%s" % file_hash
hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext))
unparsed_name = list(parsed_name)
unparsed_name[2] = hashed_name
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
if '?#' in name and not unparsed_name[3]:
unparsed_name[2] += '?'
return urlunsplit(unparsed_name)
def __call__(self, value):
value = force_text(value)
# Check first if the scheme is valid
scheme = value.split('://')[0].lower()
if scheme not in self.schemes:
raise ValidationError(self.message, code=self.code)
# Then check full URL
try:
super(URLValidator, self).__call__(value)
except ValidationError as e:
# Trivial case failed. Try for possible IDN domain
if value:
scheme, netloc, path, query, fragment = urlsplit(value)
try:
netloc = netloc.encode('idna').decode(
'ascii') # IDN -> ACE
except UnicodeError: # invalid domain part
raise e
url = urlunsplit((scheme, netloc, path, query, fragment))
super(URLValidator, self).__call__(url)
else:
raise
else:
url = value
def _do_on_path(self, cb, path, revision=HEAD):
if not path:
raise FileNotFoundError(path, revision)
try:
normpath = self.normalize_path(path)
# SVN expects to have URLs escaped. Take care to only
# escape the path part of the URL.
if self.client.is_url(normpath):
pathtuple = urlsplit(normpath)
path = pathtuple[2]
if isinstance(path, six.text_type):
path = path.encode('utf-8', 'ignore')
normpath = urlunsplit((pathtuple[0],
pathtuple[1],
quote(path),
'', ''))
normrev = self._normalize_revision(revision)
return cb(normpath, normrev)
except ClientError as e:
exc = force_text(e)
if 'File not found' in exc or 'path not found' in exc:
raise FileNotFoundError(path, revision, detail=exc)
elif 'callback_ssl_server_trust_prompt required' in exc:
raise SCMError(
_('HTTPS certificate not accepted. Please ensure that '
'the proper certificate exists in %s '
'for the user that reviewboard is running as.')
% os.path.join(self.config_dir, 'auth'))
else:
raise SVNTool.normalize_error(e)
def __call__(self, value):
value = force_text(value)
# Check first if the scheme is valid
scheme = value.split('://')[0].lower()
if scheme not in self.schemes:
raise ValidationError(self.message, code=self.code)
# Then check full URL
try:
super(URLValidator, self).__call__(value)
except ValidationError as e:
# Trivial case failed. Try for possible IDN domain
if value:
scheme, netloc, path, query, fragment = urlsplit(value)
try:
netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE
except UnicodeError: # invalid domain part
raise e
url = urlunsplit((scheme, netloc, path, query, fragment))
super(URLValidator, self).__call__(url)
else:
raise
else:
# Now verify IPv6 in the netloc part
host_match = re.search(r'^\[(.+)\](?::\d{2,5})?$', urlsplit(value).netloc)
if host_match:
potential_ip = host_match.groups()[0]
try:
validate_ipv6_address(potential_ip)
except ValidationError:
raise ValidationError(self.message, code=self.code)
url = value
def is_valid_repository(self):
"""Checks if this is a valid Git repository."""
url_parts = urlsplit(self.path)
if (url_parts.scheme.lower() in ('http', 'https')
and url_parts.username is None and self.username):
# Git URLs, especially HTTP(s), that require authentication should
# be entered without the authentication info in the URL (because
# then it would be visible), but we need it in the URL when testing
# to make sure it exists. Reformat the path here to include them.
new_netloc = urlquote(self.username, safe='')
if self.password:
new_netloc += ':' + urlquote(self.password, safe='')
new_netloc += '@' + url_parts.netloc
path = urlunsplit((url_parts[0], new_netloc, url_parts[2],
url_parts[3], url_parts[4]))
else:
path = self.path
p = self._run_git(['ls-remote', path, 'HEAD'])
errmsg = p.stderr.read()
failure = p.wait()
if failure:
logging.error("Git: Failed to find valid repository %s: %s" %
(self.path, errmsg))
return False
return True
def smart_urlquote(url):
"Quotes a URL if it isn't already quoted."
def unquote_quote(segment):
segment = unquote(force_str(segment))
# Tilde is part of RFC3986 Unreserved Characters
# http://tools.ietf.org/html/rfc3986#section-2.3
# See also http://bugs.python.org/issue16285
segment = quote(segment, safe=RFC3986_SUBDELIMS + RFC3986_GENDELIMS + str('~'))
return force_text(segment)
# Handle IDN before quoting.
try:
scheme, netloc, path, query, fragment = urlsplit(url)
except ValueError:
# invalid IPv6 URL (normally square brackets in hostname part).
return unquote_quote(url)
try:
netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE
except UnicodeError: # invalid domain part
return unquote_quote(url)
if query:
# Separately unquoting key/value, so as to not mix querystring separators
# included in query values. See #22267.
query_parts = [(unquote(force_str(q[0])), unquote(force_str(q[1])))
for q in parse_qsl(query, keep_blank_values=True)]
# urlencode will take care of quoting
query = urlencode(query_parts)
path = unquote_quote(path)
fragment = unquote_quote(fragment)
return urlunsplit((scheme, netloc, path, query, fragment))
def smart_urlquote(url):
"Quotes a URL if it isn't already quoted."
def unquote_quote(segment):
segment = unquote(force_str(segment))
# Tilde is part of RFC3986 Unreserved Characters
# http://tools.ietf.org/html/rfc3986#section-2.3
# See also http://bugs.python.org/issue16285
segment = quote(segment, safe=RFC3986_SUBDELIMS + RFC3986_GENDELIMS + str('~'))
return force_text(segment)
# Handle IDN before quoting.
try:
scheme, netloc, path, query, fragment = urlsplit(url)
except ValueError:
# invalid IPv6 URL (normally square brackets in hostname part).
return unquote_quote(url)
try:
netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE
except UnicodeError: # invalid domain part
return unquote_quote(url)
if query:
# Separately unquoting key/value, so as to not mix querystring separators
# included in query values. See #22267.
query_parts = [(unquote(force_str(q[0])), unquote(force_str(q[1])))
for q in parse_qsl(query, keep_blank_values=True)]
# urlencode will take care of quoting
query = urlencode(query_parts)
path = unquote_quote(path)
fragment = unquote_quote(fragment)
return urlunsplit((scheme, netloc, path, query, fragment))
def set_language(request):
"""
Since this view changes how the user will see the rest of the site, it must
only be accessed as a POST request. If called as a GET request, it will
error.
"""
payload = json.loads(request.body)
lang_code = payload.get(LANGUAGE_QUERY_PARAMETER)
next_url = urlsplit(payload.get("next")) if payload.get("next") else None
if lang_code and lang_code in SUPPORTED_LANGUAGES:
if next_url and is_valid_path(next_url.path):
# If it is a recognized path, then translate it to the new language and return it.
next_path = urlunsplit(
(
next_url[0],
next_url[1],
translate_url(next_url[2], lang_code),
next_url[3],
next_url[4],
)
)
else:
# Just redirect to the base URL w/ the lang_code
next_path = translate_url(reverse('base'), lang_code)
response = HttpResponse(next_path)
if hasattr(request, "session"):
request.session[LANGUAGE_SESSION_KEY] = lang_code
else:
lang_code = get_language()
if next_url and is_valid_path(next_url.path):
# If it is a recognized path, then translate it using the default language code for this device
next_path = urlunsplit(
(
next_url[0],
next_url[1],
translate_url(next_url[2], lang_code),
next_url[3],
next_url[4],
)
)
else:
# Just redirect to the base URL w/ the lang_code, likely the default language
next_path = translate_url(reverse('base'), lang_code)
response = HttpResponse(next_path)
if hasattr(request, "session"):
request.session.pop(LANGUAGE_SESSION_KEY, "")
return response
def assertRedirects(self, response, expected_url, status_code=302,
target_status_code=200, host=None, msg_prefix=''):
"""Asserts that a response redirected to a specific URL, and that the
redirect URL can be loaded.
Note that assertRedirects won't work for external links since it uses
TestClient to do a request.
"""
if msg_prefix:
msg_prefix += ": "
if hasattr(response, 'redirect_chain'):
# The request was a followed redirect
self.assertTrue(len(response.redirect_chain) > 0,
msg_prefix + "Response didn't redirect as expected: Response"
" code was %d (expected %d)" %
(response.status_code, status_code))
self.assertEqual(response.redirect_chain[0][1], status_code,
msg_prefix + "Initial response didn't redirect as expected:"
" Response code was %d (expected %d)" %
(response.redirect_chain[0][1], status_code))
url, status_code = response.redirect_chain[-1]
self.assertEqual(response.status_code, target_status_code,
msg_prefix + "Response didn't redirect as expected: Final"
" Response code was %d (expected %d)" %
(response.status_code, target_status_code))
else:
# Not a followed redirect
self.assertEqual(response.status_code, status_code,
msg_prefix + "Response didn't redirect as expected: Response"
" code was %d (expected %d)" %
(response.status_code, status_code))
url = response.url
scheme, netloc, path, query, fragment = urlsplit(url)
redirect_response = response.client.get(path, QueryDict(query))
# Get the redirection page, using the same client that was used
# to obtain the original response.
self.assertEqual(redirect_response.status_code, target_status_code,
msg_prefix + "Couldn't retrieve redirection page '%s':"
" response code was %d (expected %d)" %
(path, redirect_response.status_code, target_status_code))
e_scheme, e_netloc, e_path, e_query, e_fragment = urlsplit(
expected_url)
if not (e_scheme or e_netloc):
expected_url = urlunsplit(('http', host or 'testserver', e_path,
e_query, e_fragment))
self.assertEqual(url, expected_url,
msg_prefix + "Response redirected to '%s', expected '%s'" %
(url, expected_url))
def remote_url(self, local_name):
if local_name[:len(self.source_dir)] != self.source_dir:
raise DistillPublishError('File {} is not in source dir {}'.format(
local_name, self.source_dir))
truncated = local_name[len(self.source_dir):]
remote_file = '/'.join(truncated.split(os.sep))
remote_uri = self.remote_url_parts.path + remote_file
return urlunsplit((self.remote_url_parts.scheme,
self.remote_url_parts.netloc, remote_uri, '', ''))
def remote_url(self, local_name):
if local_name[:len(self.source_dir)] != self.source_dir:
raise DistillPublishError('File {} is not in source dir {}'.format(
local_name, self.source_dir))
truncated = local_name[len(self.source_dir):]
remote_file = '/'.join(truncated.split(os.sep))
remote_uri = self.remote_url_parts.path + remote_file
return urlunsplit((self.remote_url_parts.scheme,
self.remote_url_parts.netloc, remote_uri, '', ''))
def render(self, context):
url = self.url.render(context)
parts = urlsplit(url)
if parts.query:
qs = parts.query + '&tag=' + refcache.get_ref()
else:
qs = 'tag=' + refcache.get_ref()
return urlunsplit(
(parts.scheme, parts.netloc, parts.path, qs, parts.fragment))
def render(self, context):
url = self.url.render(context)
parts = urlsplit(url)
if parts.query:
qs = parts.query + '&tag=' + refcache.get_ref()
else:
qs = 'tag=' + refcache.get_ref()
return urlunsplit((parts.scheme, parts.netloc, parts.path, qs,
parts.fragment))
def _add_query_params(self, url, new_query_params):
"""Adds query parameters onto the given URL."""
scheme, netloc, path, query_string, fragment = urlsplit(url)
query_params = parse_qs(query_string)
query_params.update(new_query_params)
new_query_string = urlencode([(key, value) for key, value in sorted(
six.iteritems(query_params), key=lambda i: i[0])],
doseq=True)
return urlunsplit((scheme, netloc, path, new_query_string, fragment))
def replace_query_param(url, key, val):
"""
Given a URL and a key/val pair, set or replace an item in the query
parameters of the URL, and return the new URL.
"""
(scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
query_dict = QueryDict(query).copy()
query_dict[key] = val
query = query_dict.urlencode()
return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def replace_query_param(url, key, val):
"""
Given a URL and a key/val pair, set or replace an item in the query
parameters of the URL, and return the new URL.
"""
(scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
query_dict = QueryDict(query).copy()
query_dict[key] = val
query = query_dict.urlencode()
return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def remove_query_param(url, key):
"""
Given a URL and a key/val pair, remove an item in the query
parameters of the URL, and return the new URL.
"""
(scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
query_dict = urlparse.parse_qs(query)
query_dict.pop(key, None)
query = urlparse.urlencode(sorted(list(query_dict.items())), doseq=True)
return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def __call__(self, request):
if request.get_host() in settings.LEGACY_HOSTS:
site_parts = urlsplit(settings.SITE_URL)
legacy_parts = urlsplit(request.get_full_path())
# Construct the destination URL with the scheme and domain from the
# SITE_URL, and the path, querystring, etc from the legacy URL.
dest_url = urlunsplit(site_parts[:2] + legacy_parts[2:])
return HttpResponsePermanentRedirect(dest_url)
return self.get_response(request)
def replace_query_param(url, key, val):
"""
Given a URL and a key/val pair, set or replace an item in the query
parameters of the URL, and return the new URL.
"""
(scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
query_dict = urlparse.parse_qs(query, keep_blank_values=True)
query_dict[key] = [val]
query = urlparse.urlencode(sorted(list(query_dict.items())), doseq=True)
return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def remove_query_param(url, key):
"""
Given a URL and a key/val pair, remove an item in the query
parameters of the URL, and return the new URL.
"""
(scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
query_dict = urlparse.parse_qs(query)
query_dict.pop(key, None)
query = urlparse.urlencode(sorted(list(query_dict.items())), doseq=True)
return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def __call__(self, request):
if request.get_host() in settings.LEGACY_HOSTS:
site_parts = urlsplit(settings.SITE_URL)
legacy_parts = urlsplit(request.get_full_path())
# Construct the destination URL with the scheme and domain from the
# SITE_URL, and the path, querystring, etc from the legacy URL.
dest_url = urlunsplit(site_parts[:2] + legacy_parts[2:])
return HttpResponsePermanentRedirect(dest_url)
return self.get_response(request)
def testSyncMetaWithGzip(self):
content = b"foo" * 1000
with self.settings(AWS_S3_GZIP=True):
with self.save_file(name="foo/bar.txt", content=content):
meta = default_storage.meta("foo/bar.txt")
self.assertEqual(meta["CacheControl"], "private,max-age=3600")
self.assertEqual(meta["ContentType"], "text/plain")
self.assertEqual(meta["ContentEncoding"], "gzip")
self.assertEqual(meta.get("ContentDisposition"), None)
self.assertEqual(meta.get("ContentLanguage"), None)
self.assertEqual(meta["Metadata"],
{"uncompressed_size": str(len(content))})
self.assertEqual(meta.get("StorageClass"), None)
self.assertEqual(meta.get("ServerSideEncryption"), None)
# Store new metadata.
with self.settings(
AWS_S3_BUCKET_AUTH=False,
AWS_S3_MAX_AGE_SECONDS=9999,
AWS_S3_CONTENT_DISPOSITION=lambda name:
"attachment; filename={}".format(name),
AWS_S3_CONTENT_LANGUAGE="eo",
AWS_S3_METADATA={
"foo": "bar",
"baz": lambda name: name,
},
AWS_S3_REDUCED_REDUNDANCY=True,
AWS_S3_ENCRYPT_KEY=True,
):
default_storage.sync_meta()
# Check metadata changed.
meta = default_storage.meta("foo/bar.txt")
self.assertEqual(meta["CacheControl"], "public,max-age=9999")
self.assertEqual(meta["ContentType"], "text/plain")
self.assertEqual(meta["ContentEncoding"], "gzip")
self.assertEqual(meta.get("ContentDisposition"),
"attachment; filename=foo/bar.txt")
self.assertEqual(meta.get("ContentLanguage"), "eo")
self.assertEqual(
meta.get("Metadata"), {
"foo": "bar",
"baz": "foo/bar.txt",
"uncompressed_size": str(len(content)),
})
self.assertEqual(meta["StorageClass"], "REDUCED_REDUNDANCY")
self.assertEqual(meta["ServerSideEncryption"], "AES256")
# Check ACL changed by removing the query string.
url_unauthenticated = urlunsplit(
urlsplit(default_storage.url("foo/bar.txt"))[:3] + (
"",
"",
))
response = requests.get(url_unauthenticated)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content, b"foo" * 1000)
def clean_url(url):
parsed = urlsplit(url)
try:
parts = parsed.path.split('/')
except:
return url
lang = get_language_from_path(parsed.path)
if len(parts) > 1 and parts[1] == lang:
del parts[1]
return urlunsplit((parsed.scheme, parsed.netloc, '/'.join(parts),
parsed.query, parsed.fragment))
def clean_url(url):
parsed = urlsplit(url)
try:
parts = parsed.path.split('/')
except:
return url
lang = get_language_from_path(parsed.path)
if len(parts) > 1 and parts[1] == lang:
del parts[1]
return urlunsplit((
parsed.scheme, parsed.netloc, '/'.join(parts), parsed.query,
parsed.fragment))
def get_canonical_url(self):
"""Determine whether to send a canonical url for the blog list.
A blog list view without any query should be the same as a blog
list view with query `page=1`. The `page=1` view should have a canonical
link to the simpler URL.
"""
if self.request.GET.get('page') == '1':
querystring = self.request.GET.copy()
del querystring['page']
return parse.urlunsplit(('', '', self.request.path, querystring.urlencode(), ''))
else:
return None
def _data_with_signed_urls(self, data):
thumbnail_keys = [
'thumbnail_xs_path', 'thumbnail_sm_path', 'thumbnail_md_path',
'thumbnail_lg_path'
]
for key, value in data.items():
if key in thumbnail_keys and value:
split_results = list(tuple(parse.urlsplit(value)))
split_results[-2] = ''
cleaned_url = parse.unquote(parse.urlunsplit(split_results))
data[key] = sign_s3_url(cleaned_url)
return data
def get_canonical_url(self):
"""Determine whether to send a canonical url for the blog list.
A blog list view without any query should be the same as a blog
list view with query `page=1`. The `page=1` view should have a canonical
link to the simpler URL.
"""
if self.request.GET.get('page') == '1':
querystring = self.request.GET.copy()
del querystring['page']
return parse.urlunsplit(
('', '', self.request.path, querystring.urlencode(), ''))
else:
return None
def testUrl(self):
with self.save_file():
url = default_storage.url("foo.txt")
# The URL should contain query string authentication.
self.assertTrue(urlsplit(url).query)
response = requests.get(url)
# The URL should be accessible, but be marked as private.
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content, b"foo")
self.assertEqual(response.headers["cache-control"], "private,max-age=3600")
# With the query string removed, the URL should not be accessible.
url_unauthenticated = urlunsplit(urlsplit(url)[:3] + ("", "",))
response_unauthenticated = requests.get(url_unauthenticated)
self.assertEqual(response_unauthenticated.status_code, 403)
def _add_query_params(self, url, new_query_params):
"""Adds query parameters onto the given URL."""
scheme, netloc, path, query_string, fragment = urlsplit(url)
query_params = parse_qs(query_string)
query_params.update(new_query_params)
new_query_string = urlencode(
[
(key, value)
for key, value in sorted(six.iteritems(query_params),
key=lambda i: i[0])
],
doseq=True)
return urlunsplit((scheme, netloc, path, new_query_string, fragment))
def replace_query_param(url, key, val):
"""
Given a URL and a key/val pair, set or replace an item in the query
parameters of the URL, and return the new URL.
Forked from rest_framework.utils.urls; overwriten here because we
need to pass keep_blank_values=True to urlparse.parse_qs() so that
it doesn't remove the ?filter[id__in]= blank query parameter from
our links in the case of an empty remote to-many link.
"""
(scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
query_dict = urlparse.parse_qs(query, keep_blank_values=True)
query_dict[key] = [val]
query = urlparse.urlencode(sorted(list(query_dict.items())), doseq=True)
return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def replace_query_param(url, key, val):
"""
Given a URL and a key/val pair, set or replace an item in the query
parameters of the URL, and return the new URL.
Forked from rest_framework.utils.urls; overwriten here because we
need to pass keep_blank_values=True to urlparse.parse_qs() so that
it doesn't remove the ?filter[id__in]= blank query parameter from
our links in the case of an empty remote to-many link.
"""
(scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
query_dict = urlparse.parse_qs(query, keep_blank_values=True)
query_dict[key] = [val]
query = urlparse.urlencode(sorted(list(query_dict.items())), doseq=True)
return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def url(self, name, extra_params={}):
# Use a public URL, if specified.
if self.settings.AWS_S3_PUBLIC_URL:
return urljoin(self.settings.AWS_S3_PUBLIC_URL, filepath_to_uri(name))
# Otherwise, generate the URL.
url = self.s3_connection.generate_presigned_url(
ClientMethod="get_object",
Params=self._presigned_url_params(name, **extra_params),
ExpiresIn=self.settings.AWS_S3_MAX_AGE_SECONDS,
)
# Strip off the query params if we're not interested in bucket auth.
if not self.settings.AWS_S3_BUCKET_AUTH:
url = urlunsplit(urlsplit(url)[:3] + ("", "",))
# All done!
return url
def stored_name(self, name):
parsed_name = urlsplit(unquote(name))
clean_name = parsed_name.path.strip()
hash_key = self.hash_key(clean_name)
cache_name = self.hashed_files.get(hash_key)
if cache_name is None:
if self.manifest_strict:
raise ValueError("Missing staticfiles manifest entry for '%s'" % clean_name)
cache_name = self.clean_name(self.hashed_name(name))
unparsed_name = list(parsed_name)
unparsed_name[2] = cache_name
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
if '?#' in name and not unparsed_name[3]:
unparsed_name[2] += '?'
return urlunsplit(unparsed_name)
def _data_with_signed_urls(self, data):
thumbnail_keys = [
'thumbnail_xs_path',
'thumbnail_sm_path',
'thumbnail_md_path',
'thumbnail_lg_path'
]
for key, value in data.items():
if key in thumbnail_keys and value:
split_results = list(tuple(parse.urlsplit(value)))
split_results[-2] = ''
cleaned_url = parse.unquote(parse.urlunsplit(split_results))
data[key] = sign_s3_url(cleaned_url)
return data
def switch_language_url(context, lang_code: str):
request = context['request']
match = request.resolver_match
parsed_url = urlsplit(request.get_full_path())
to_be_reversed = "%s:%s" % (match.namespace, match.url_name) \
if match.namespace else match.url_name
with override(lang_code):
try:
url = reverse(to_be_reversed, args=match.args, kwargs=match.kwargs)
except NoReverseMatch:
pass
else:
url = urlunsplit((parsed_url.scheme, parsed_url.netloc,
url, parsed_url.query, parsed_url.fragment))
return url
def __call__(self, value):
try:
super(URIValidator, self).__call__(value)
except ValidationError as e:
# Trivial case failed. Try for possible IDN domain
if value:
value = force_text(value)
scheme, netloc, path, query, fragment = urlsplit(value)
try:
netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE
except UnicodeError: # invalid domain part
raise e
url = urlunsplit((scheme, netloc, path, query, fragment))
super(URIValidator, self).__call__(url)
else:
raise
else:
url = value
def dmpui_static_url(name):
hashed_name, fragment = name, ''
final_url = reverse("dmp_ui:dmpui_static",
kwargs={"path": hashed_name})
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
query_fragment = '?#' in name # [sic!]
if fragment or query_fragment:
urlparts = list(urlsplit(final_url))
if fragment and not urlparts[4]:
urlparts[4] = fragment
if query_fragment and not urlparts[3]:
urlparts[2] += '?'
final_url = urlunsplit(urlparts)
return unquote(final_url)
def __call__(self, value):
value = force_text(value)
# Check first if the scheme is valid
scheme = value.split('://')[0].lower()
if scheme not in self.schemes:
raise ValidationError(self.message, code=self.code)
# Then check full URL
try:
super(URLValidator, self).__call__(value)
except ValidationError as e:
# Trivial case failed. Try for possible IDN domain
if value:
try:
scheme, netloc, path, query, fragment = urlsplit(value)
except ValueError: # for example, "Invalid IPv6 URL"
raise ValidationError(self.message, code=self.code)
try:
netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE
except UnicodeError: # invalid domain part
raise e
url = urlunsplit((scheme, netloc, path, query, fragment))
super(URLValidator, self).__call__(url)
else:
raise
else:
# Now verify IPv6 in the netloc part
host_match = re.search(r'^\[(.+)\](?::\d{2,5})?$', urlsplit(value).netloc)
if host_match:
potential_ip = host_match.groups()[0]
try:
validate_ipv6_address(potential_ip)
except ValidationError:
raise ValidationError(self.message, code=self.code)
url = value
# The maximum length of a full host name is 253 characters per RFC 1034
# section 3.1. It's defined to be 255 bytes or less, but this includes
# one byte for the length of the name and one byte for the trailing dot
# that's used to indicate absolute names in DNS.
if len(urlsplit(value).netloc) > 253:
raise ValidationError(self.message, code=self.code)
def url(self, name, force=False):
final_url = super(ManifestFilesMixinUrlEncoded, self).url(name, force)
urlparts = list(urlsplit(final_url))
urlparts[1] = getenv('AWS_CLOUDFRONT_URL')
url_queries = urlparts[3]
query_pairs = url_queries.split('&')
encoded_queries = []
for query in query_pairs:
key, val = query.split('=', 1)
encoded_queries.append(
key + '=' + quote(val)
)
urlparts[3] ='&'.join(encoded_queries)
final_url = urlunsplit(urlparts)
return final_url
def smart_urlquote(url):
"Quotes a URL if it isn't already quoted."
# Handle IDN before quoting.
try:
scheme, netloc, path, query, fragment = urlsplit(url)
try:
netloc = netloc.encode("idna").decode("ascii") # IDN -> ACE
except UnicodeError: # invalid domain part
pass
else:
url = urlunsplit((scheme, netloc, path, query, fragment))
except ValueError:
# invalid IPv6 URL (normally square brackets in hostname part).
pass
url = unquote(force_str(url))
# See http://bugs.python.org/issue2637
url = quote(url, safe=b"!*'();:@&=+$,/?#[]~")
return force_text(url)
def translate_url(url, lang_code):
"""
Given a URL (absolute or relative), try to get its translated version in
the `lang_code` language (either by i18n_patterns or by translated regex).
Return the original URL if no translated version is found.
"""
parsed = urlsplit(url)
try:
match = resolve(parsed.path)
except Resolver404:
pass
else:
to_be_reversed = "%s:%s" % (match.namespace, match.url_name) if match.namespace else match.url_name
with override(lang_code):
try:
url = reverse(to_be_reversed, args=match.args, kwargs=match.kwargs)
except NoReverseMatch:
pass
else:
url = urlunsplit((parsed.scheme, parsed.netloc, url, parsed.query, parsed.fragment))
return url
def smart_urlquote(url):
"""Quotes a URL if it isn't already quoted. Returns None on errors."""
# Handle IDN before quoting.
try:
scheme, netloc, path, query, fragment = urlsplit(url)
except ValueError:
# invalid IPv6 URL (normally square brackets in hostname part).
return None
try:
netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE
except UnicodeError: # invalid domain part
return None
if not url_path_re.search(path):
path = quote(force_str(path), safe=SAFE_PATH_CHARS)
if not url_query_fragment_re.search(query):
query = quote(force_str(query), safe=SAFE_QUERY_FRAGMENT_CHARS)
if not url_query_fragment_re.search(fragment):
fragment = quote(force_str(fragment), safe=SAFE_QUERY_FRAGMENT_CHARS)
url = urlunsplit((scheme, netloc, path, query, fragment))
return force_text(url)