def parse_header(line): """ Parse the header into a key-value. Input (line): bytes, output: unicode for key/name, bytes for value which will be decoded later """ plist = _parse_header_params(b';' + line) key = plist.pop(0).lower().decode('ascii') pdict = {} for p in plist: i = p.find(b'=') if i >= 0: has_encoding = False name = p[:i].strip().lower().decode('ascii') if name.endswith('*'): # Lang/encoding embedded in the value (like "filename*=UTF-8''file.ext") # http://tools.ietf.org/html/rfc2231#section-4 name = name[:-1] if p.count(b"'") == 2: has_encoding = True value = p[i + 1:].strip() if has_encoding: encoding, lang, value = value.split(b"'") if six.PY3: value = unquote(value.decode(), encoding=encoding.decode()) else: value = unquote(value).decode(encoding) if len(value) >= 2 and value[:1] == value[-1:] == b'"': value = value[1:-1] value = value.replace(b'\\\\', b'\\').replace(b'\\"', b'"') pdict[name] = value return key, pdict
def parse_header(line): """ Parse the header into a key-value. Input (line): bytes, output: unicode for key/name, bytes for value which will be decoded later """ plist = _parse_header_params(b';' + line) key = plist.pop(0).lower().decode('ascii') pdict = {} for p in plist: i = p.find(b'=') if i >= 0: has_encoding = False name = p[:i].strip().lower().decode('ascii') if name.endswith('*'): # Lang/encoding embedded in the value (like "filename*=UTF-8''file.ext") # http://tools.ietf.org/html/rfc2231#section-4 name = name[:-1] if p.count(b"'") == 2: has_encoding = True value = p[i + 1:].strip() if has_encoding: encoding, lang, value = value.split(b"'") if six.PY3: value = unquote(value.decode(), encoding=encoding.decode()) else: value = unquote(value).decode(encoding) if len(value) >= 2 and value[:1] == value[-1:] == b'"': value = value[1:-1] value = value.replace(b'\\\\', b'\\').replace(b'\\"', b'"') pdict[name] = value return key, pdict
def smart_urlquote(url): "Quotes a URL if it isn't already quoted." def unquote_quote(segment): segment = unquote(force_str(segment)) # Tilde is part of RFC3986 Unreserved Characters # http://tools.ietf.org/html/rfc3986#section-2.3 # See also http://bugs.python.org/issue16285 segment = quote(segment, safe=RFC3986_SUBDELIMS + RFC3986_GENDELIMS + str('~')) return force_text(segment) # Handle IDN before quoting. try: scheme, netloc, path, query, fragment = urlsplit(url) except ValueError: # invalid IPv6 URL (normally square brackets in hostname part). return unquote_quote(url) try: netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE except UnicodeError: # invalid domain part return unquote_quote(url) if query: # Separately unquoting key/value, so as to not mix querystring separators # included in query values. See #22267. query_parts = [(unquote(force_str(q[0])), unquote(force_str(q[1]))) for q in parse_qsl(query, keep_blank_values=True)] # urlencode will take care of quoting query = urlencode(query_parts) path = unquote_quote(path) fragment = unquote_quote(fragment) return urlunsplit((scheme, netloc, path, query, fragment))
def smart_urlquote(url): "Quotes a URL if it isn't already quoted." def unquote_quote(segment): segment = unquote(force_str(segment)) # Tilde is part of RFC3986 Unreserved Characters # http://tools.ietf.org/html/rfc3986#section-2.3 # See also http://bugs.python.org/issue16285 segment = quote(segment, safe=RFC3986_SUBDELIMS + RFC3986_GENDELIMS + str('~')) return force_text(segment) # Handle IDN before quoting. try: scheme, netloc, path, query, fragment = urlsplit(url) except ValueError: # invalid IPv6 URL (normally square brackets in hostname part). return unquote_quote(url) try: netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE except UnicodeError: # invalid domain part return unquote_quote(url) if query: # Separately unquoting key/value, so as to not mix querystring separators # included in query values. See #22267. query_parts = [(unquote(force_str(q[0])), unquote(force_str(q[1]))) for q in parse_qsl(query, keep_blank_values=True)] # urlencode will take care of quoting query = urlencode(query_parts) path = unquote_quote(path) fragment = unquote_quote(fragment) return urlunsplit((scheme, netloc, path, query, fragment))
def converter(matchobj): """ Converts the matched URL depending on the parent level (`..`) and returns the normalized and hashed URL using the url method of the storage. """ matched, url = matchobj.groups() # Completely ignore http(s) prefixed URLs, # fragments and data-uri URLs if url.startswith(('#', 'http:', 'https:', 'data:', '//')): return matched name_parts = name.split(os.sep) # Using posix normpath here to remove duplicates url = posixpath.normpath(url) url_parts = url.split('/') parent_level, sub_level = url.count('..'), url.count('/') if url.startswith('/'): sub_level -= 1 url_parts = url_parts[1:] if parent_level or not url.startswith('/'): start, end = parent_level + 1, parent_level else: if sub_level: if sub_level == 1: parent_level -= 1 start, end = parent_level, 1 else: start, end = 1, sub_level - 1 joined_result = '/'.join(name_parts[:-start] + url_parts[end:]) hashed_url = self.url(unquote(joined_result), force=True) file_name = hashed_url.split('/')[-1:] relative_url = '/'.join(url.split('/')[:-1] + file_name) # Return the hashed version to the file return template % unquote(relative_url)
def converter(matchobj): """ Converts the matched URL depending on the parent level (`..`) and returns the normalized and hashed URL using the url method of the storage. """ matched, url = matchobj.groups() # Completely ignore http(s) prefixed URLs, # fragments and data-uri URLs if url.startswith(('#', 'http:', 'https:', 'data:', '//')): return matched name_parts = name.split(os.sep) # Using posix normpath here to remove duplicates url = posixpath.normpath(url) url_parts = url.split('/') parent_level, sub_level = url.count('..'), url.count('/') if url.startswith('/'): sub_level -= 1 url_parts = url_parts[1:] if parent_level or not url.startswith('/'): start, end = parent_level + 1, parent_level else: if sub_level: if sub_level == 1: parent_level -= 1 start, end = parent_level, 1 else: start, end = 1, sub_level - 1 joined_result = '/'.join(name_parts[:-start] + url_parts[end:]) hashed_url = self.url(unquote(joined_result), force=True) file_name = hashed_url.split('/')[-1:] relative_url = '/'.join(url.split('/')[:-1] + file_name) # Return the hashed version to the file return template % unquote(relative_url)
def urlunquote(quoted_url): """ A wrapper for Python's ``urllib.unquote()`` function that can operate on unicode strings. """ if six.PY3: return unquote(quoted_url) else: return latin_to_unicode(unquote(force_bytes(quoted_url)))
def limited_parse_qsl(qs, keep_blank_values=False, encoding='utf-8', errors='replace', fields_limit=None): """ Return a list of key/value tuples parsed from query string. Copied from urlparse with an additional "fields_limit" argument. Copyright (C) 2013 Python Software Foundation (see LICENSE.python). Arguments: qs: percent-encoded query string to be parsed keep_blank_values: flag indicating whether blank values in percent-encoded queries should be treated as blank strings. A true value indicates that blanks should be retained as blank strings. The default false value indicates that blank values are to be ignored and treated as if they were not included. encoding and errors: specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() method. fields_limit: maximum number of fields parsed or an exception is raised. None means no limit and is the default. """ if fields_limit: pairs = FIELDS_MATCH.split(qs, fields_limit) if len(pairs) > fields_limit: raise TooManyFieldsSent( 'The number of GET/POST parameters exceeded ' 'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.') else: pairs = FIELDS_MATCH.split(qs) r = [] for name_value in pairs: if not name_value: continue nv = name_value.split(str('='), 1) if len(nv) != 2: # Handle case of a control-name with no equal sign if keep_blank_values: nv.append('') else: continue if len(nv[1]) or keep_blank_values: if six.PY3: name = nv[0].replace('+', ' ') name = unquote(name, encoding=encoding, errors=errors) value = nv[1].replace('+', ' ') value = unquote(value, encoding=encoding, errors=errors) else: name = unquote(nv[0].replace(b'+', b' ')) value = unquote(nv[1].replace(b'+', b' ')) r.append((name, value)) return r
def limited_parse_qsl(qs, keep_blank_values=False, encoding='utf-8', errors='replace', fields_limit=None): """ Return a list of key/value tuples parsed from query string. Copied from urlparse with an additional "fields_limit" argument. Copyright (C) 2013 Python Software Foundation (see LICENSE.python). Arguments: qs: percent-encoded query string to be parsed keep_blank_values: flag indicating whether blank values in percent-encoded queries should be treated as blank strings. A true value indicates that blanks should be retained as blank strings. The default false value indicates that blank values are to be ignored and treated as if they were not included. encoding and errors: specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() method. fields_limit: maximum number of fields parsed or an exception is raised. None means no limit and is the default. """ if fields_limit: pairs = FIELDS_MATCH.split(qs, fields_limit) if len(pairs) > fields_limit: raise TooManyFieldsSent( 'The number of GET/POST parameters exceeded ' 'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.' ) else: pairs = FIELDS_MATCH.split(qs) r = [] for name_value in pairs: if not name_value: continue nv = name_value.split(str('='), 1) if len(nv) != 2: # Handle case of a control-name with no equal sign if keep_blank_values: nv.append('') else: continue if len(nv[1]) or keep_blank_values: if six.PY3: name = nv[0].replace('+', ' ') name = unquote(name, encoding=encoding, errors=errors) value = nv[1].replace('+', ' ') value = unquote(value, encoding=encoding, errors=errors) else: name = unquote(nv[0].replace(b'+', b' ')) value = unquote(nv[1].replace(b'+', b' ')) r.append((name, value)) return r
def converter(matchobj): """ Convert the matched URL to a normalized and hashed URL. This requires figuring out which files the matched URL resolves to and calling the url() method of the storage. """ matched, url = matchobj.groups() # Ignore absolute/protocol-relative and data-uri URLs. if re.match(r'^[a-z]+:', url): return matched # Ignore absolute URLs that don't point to a static file (dynamic # CSS / JS?). Note that STATIC_URL cannot be empty. if url.startswith('/') and not url.startswith(settings.STATIC_URL): return matched # Strip off the fragment so a path-like fragment won't interfere. url_path, fragment = urldefrag(url) if url_path.startswith('/'): # Otherwise the condition above would have returned prematurely. assert url_path.startswith(settings.STATIC_URL) target_name = url_path[len(settings.STATIC_URL):] else: # We're using the posixpath module to mix paths and URLs conveniently. source_name = name if os.sep == '/' else name.replace( os.sep, '/') target_name = posixpath.join(posixpath.dirname(source_name), url_path) # Determine the hashed name of the target file with the storage backend. hashed_url = self._url( self._stored_name, unquote(target_name), force=True, hashed_files=hashed_files, ) # NOTE: # The line below was commented out so that absolute urls are used instead of relative urls to make themed # assets work correctly. # # The line is commented and not removed to make future django upgrade easier and show exactly what is # changed in this method override # #transformed_url = '/'.join(url_path.split('/')[:-1] + hashed_url.split('/')[-1:]) transformed_url = hashed_url # This line was added. # Restore the fragment that was stripped off earlier. if fragment: transformed_url += ('?#' if '?#' in url else '#') + fragment # Return the hashed version to the file return template % unquote(transformed_url)
def converter(matchobj): """ Convert the matched URL to a normalized and hashed URL. This requires figuring out which files the matched URL resolves to and calling the url() method of the storage. """ matched, url = matchobj.groups() # Ignore absolute/protocol-relative and data-uri URLs. if re.match(r'^[a-z]+:', url): return matched # Ignore absolute URLs that don't point to a static file (dynamic # CSS / JS?). Note that STATIC_URL cannot be empty. if url.startswith('/') and not url.startswith(settings.STATIC_URL): return matched # Strip off the fragment so a path-like fragment won't interfere. url_path, fragment = urldefrag(url) if url_path.startswith('/'): # Otherwise the condition above would have returned prematurely. assert url_path.startswith(settings.STATIC_URL) target_name = url_path[len(settings.STATIC_URL):] else: # We're using the posixpath module to mix paths and URLs conveniently. source_name = name if os.sep == '/' else name.replace(os.sep, '/') target_name = posixpath.join(posixpath.dirname(source_name), url_path) # Determine the hashed name of the target file with the storage backend. hashed_url = self._url( self._stored_name, unquote(target_name), force=True, hashed_files=hashed_files, ) # NOTE: # The line below was commented out so that absolute urls are used instead of relative urls to make themed # assets work correctly. # # The line is commented and not removed to make future django upgrade easier and show exactly what is # changed in this method override # #transformed_url = '/'.join(url_path.split('/')[:-1] + hashed_url.split('/')[-1:]) transformed_url = hashed_url # This line was added. # Restore the fragment that was stripped off earlier. if fragment: transformed_url += ('?#' if '?#' in url else '#') + fragment # Return the hashed version to the file return template % unquote(transformed_url)
def _processed_asset_name(self, name): """ Returns either a themed or unthemed version of the given asset name, depending on several factors. See the class docstring for more info. """ theme = ThemingConfiguration.theming_helpers.get_current_theme() if theme and theme.theme_dir_name not in name: # during server run, append theme name to the asset name if it is not already there # this is ensure that correct hash is created and default asset is not always # used to create hash of themed assets. name = os.path.join(theme.theme_dir_name, name) parsed_name = urlsplit(unquote(name)) clean_name = parsed_name.path.strip() asset_name = name if not self.exists(clean_name): # if themed asset does not exists then use default asset theme_name = name.split("/", 1)[0] # verify that themed asset was accessed if theme_name in [theme.theme_dir_name for theme in ThemingConfiguration.theming_helpers.get_themes()]: asset_name = "/".join(name.split("/")[1:]) elif theme and theme.theme_dir_name in asset_name: return asset_name # Try the same with default theme parent_theme = ThemingConfiguration.get_parent_or_default_theme() if theme and parent_theme.name == theme.name: return asset_name theme = parent_theme if theme and theme.theme_dir_name not in asset_name: # during server run, append theme name to the asset name if it is not already there # this is ensure that correct hash is created and default asset is not always # used to create hash of themed assets. name = os.path.join(theme.theme_dir_name, asset_name) parsed_name = urlsplit(unquote(name)) clean_name = parsed_name.path.strip() asset_name = name if not self.exists(clean_name): # if themed asset does not exists then use default asset theme = name.split("/", 1)[0] # verify that themed asset was accessed if theme in [theme.theme_dir_name for theme in ThemingConfiguration.theming_helpers.get_themes()]: asset_name = "/".join(name.split("/")[1:]) return asset_name
def serve(request, path, insecure=False, **kwargs): """ Serve static files below a given point in the directory structure or from locations inferred from the staticfiles finders. To use, put a URL pattern such as:: (r'^(?P<path>.*)$', 'django.contrib.staticfiles.views.serve') in your URLconf. It uses the django.views.static view to serve the found files. """ if not settings.DEBUG and not insecure: raise ImproperlyConfigured("The staticfiles view can only be used in " "debug mode or if the --insecure " "option of 'runserver' is used") normalized_path = posixpath.normpath(unquote(path)).lstrip('/') absolute_path = finders.find(normalized_path) if not absolute_path: if path.endswith('/') or path == '': raise Http404("Directory indexes are not allowed here.") raise Http404("'%s' could not be found" % path) document_root, path = os.path.split(absolute_path) return static.serve(request, path, document_root=document_root, **kwargs)
def serve_static(request, path, insecure=False, **kwargs): """ Serve static files below a given point in the directory structure or from locations inferred from the staticfiles finders. To use, put a URL pattern such as: (r'^static/(?P<path>.*)$', 'annalist.views.statichack.serve_static') in your URLconf. It uses the django.views.static.serve() view to serve the found files. """ # log.info("serve_static %s"%(path)) try: normalized_path = posixpath.normpath(unquote(path)).lstrip('/') absolute_path = finders.find(normalized_path) if not absolute_path: if path.endswith('/') or path == '': raise Http404("Directory indexes are not allowed here.") raise Http404("'%s' could not be found" % path) document_root, path = os.path.split(absolute_path) # log.info("document_root %s, path %s"%(document_root, path)) except Exception, e: log.info(str(e)) raise
def ftp(request, path): document_root = os.path.dirname(path) path = posixpath.normpath(unquote(os.path.basename(path))).lstrip('/') fullpath = safe_join(document_root, path) if os.path.isdir(fullpath): files = [] for f in os.listdir(fullpath): f_stat = {} if not f.startswith('.'): p = os.path.join(fullpath, f) if os.path.isdir(p): f_stat['type'] = 'directory' f_stat['size'] = '' else: f_stat['type'] = 'file' f_stat['size'] = convert_bytes(os.path.getsize(p)) f_stat['name'] = f f_stat['lastModified'] = ctime(os.path.getmtime(p)) files.append(f_stat) return render(request, 'gallery/gallery.html', { 'directory': fullpath, 'file_list': files }) if not os.path.exists(fullpath): raise Http404(('"%(path)s" does not exist') % {'path': fullpath}) return stream(request, fullpath)
def test_owner_or_mod_required_passes_url_parameters(self): @owner_or_moderator_required def mock_view(request, user, context): return None request = Mock(spec=('path', 'REQUEST', 'user')) request.user = AnonymousUser() request.REQUEST = {'abra': 'cadabra', 'foo': 'bar'} request.path = '/some/path/' user = self.create_user('user') response = mock_view(request, user, {}) self.assertEqual(isinstance(response, HttpResponseRedirect), True) url = response['location'] parsed_url = urlparse(url) self.assertEqual(parsed_url.path, reverse('user_signin')) next = dict(parse_qsl(parsed_url.query))['next'] next_url = unquote(next) parsed_url = urlparse(next_url) self.assertEqual(parsed_url.path, request.path) query = dict(parse_qsl(parsed_url.query)) self.assertEqual(set(query.keys()), set(['foo', 'abra'])) self.assertEqual(set(query.values()), set(['bar', 'cadabra'])) self.assertEqual(query['abra'], 'cadabra')
def serve(request, path, insecure=False, **kwargs): """ Serve static files below a given point in the directory structure or from locations inferred from the staticfiles finders. To use, put a URL pattern such as:: (r'^(?P<path>.*)$', 'django.contrib.staticfiles.views.serve') in your URLconf. It uses the django.views.static view to serve the found files. """ if not settings.DEBUG and not insecure: raise ImproperlyConfigured("The staticfiles view can only be used in " "debug mode or if the --insecure " "option of 'runserver' is used") normalized_path = posixpath.normpath(unquote(path)).lstrip('/') absolute_path = finders.find(normalized_path) if not absolute_path: if path.endswith('/') or path == '': raise Http404("Directory indexes are not allowed here.") raise Http404("'%s' could not be found" % path) document_root, path = os.path.split(absolute_path) return static.serve(request, path, document_root=document_root, **kwargs)
def get_object(self, queryset=None): if queryset is None: queryset = self.get_queryset() pk = self.kwargs.get('pk', None) doi = self.kwargs.get('doi', None) if doi: doi = unquote(doi) doi = to_doi(doi) paper = None try: if pk is not None: paper = queryset.get(pk=pk) elif doi is not None: paper = Paper.get_by_doi(doi) else: raise Http404(_("Paper view expects a DOI or a pk")) except ObjectDoesNotExist: pass if not paper: paper = Paper.create_by_doi(doi) if paper is None or paper.is_orphan(): raise Http404(_("No %(verbose_name)s found matching the query") % {'verbose_name': Paper._meta.verbose_name}) if not paper.visible: raise Http404(_("This paper has been deleted.")) return paper
def get_object(self, queryset=None): if queryset is None: queryset = self.get_queryset() pk = self.kwargs.get('pk', None) doi = self.kwargs.get('doi', None) if doi: doi = unquote(doi) doi = to_doi(doi) paper = None try: if pk is not None: paper = queryset.get(pk=pk) elif doi is not None: paper = Paper.get_by_doi(doi) else: raise Http404(_("Paper view expects a DOI or a pk")) except ObjectDoesNotExist: pass if paper is None or paper.is_orphan(): raise Http404( _("No %(verbose_name)s found matching the query") % {'verbose_name': Paper._meta.verbose_name}) if not paper.visible: raise Http404(_("This paper has been deleted.")) paper = queryset.prefetch_related('oairecord_set').get(pk=paper.pk) return paper
def hashed_name(self, name, content=None): parsed_name = urlsplit(unquote(name)) clean_name = parsed_name.path.strip() opened = False if content is None: if not self.exists(clean_name): raise ValueError("The file '%s' could not be found with %r." % (clean_name, self)) try: content = self.open(clean_name) except IOError: # Handle directory paths and fragments return name opened = True try: file_hash = self.file_hash(clean_name, content) finally: if opened: content.close() path, filename = os.path.split(clean_name) root, ext = os.path.splitext(filename) if file_hash is not None: file_hash = ".%s" % file_hash hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext)) unparsed_name = list(parsed_name) unparsed_name[2] = hashed_name # Special casing for a @font-face hack, like url(myfont.eot?#iefix") # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax if '?#' in name and not unparsed_name[3]: unparsed_name[2] += '?' return urlunsplit(unparsed_name)
def static_file_handler(request, path, extension, insecure=False, **kwargs): """ Note: This is static handler is only used during development. In production, the Docker image uses NGINX to serve static content. Serve static files below a given point in the directory structure or from locations inferred from the staticfiles finders. To use, put a URL pattern such as:: from django.contrib.staticfiles import views url(r'^(?P<path>.*)$', views.serve) in your URLconf. It uses the django.views.static.serve() view to serve the found files. """ append_path = '' if not settings.DEBUG and not insecure: raise Http404 normalized_path = posixpath.normpath(unquote(path)).lstrip('/') absolute_path = settings.EXTERNAL_TEMPLATE_DIR + '/' + append_path + normalized_path + '.' + extension if not absolute_path: if path.endswith('/') or path == '': raise Http404('Directory indexes are not allowed here.') raise Http404('\'%s\' could not be found' % path) document_root, path = os.path.split(absolute_path) return static.serve(request, path, document_root=document_root, **kwargs)
def serve(request, path, insecure=False, **kwargs): """ Serve static files below a given point in the directory structure or from locations inferred from the staticfiles finders. To use, put a URL pattern such as:: from django.contrib.staticfiles import views url(r'^(?P<path>.*)$', views.serve) in your URLconf. It uses the django.views.static.serve() view to serve the found files. """ if not settings.DEBUG and not insecure: raise Http404 normalized_path = posixpath.normpath(unquote(path)).lstrip('/') absolute_path = finders.find(normalized_path) if not absolute_path: if path.endswith('/') or path == '': raise Http404("Directory indexes are not allowed here.") raise Http404("'%s' could not be found" % path) document_root, path = os.path.split(absolute_path) return static.serve(request, path, document_root=document_root, **kwargs)
def hashed_name(self, name, content=None, filename=None): parsed_name = urlsplit(unquote(name)) clean_name = parsed_name.path.strip() opened = False if content is None: absolute_path = finders.find(clean_name) try: content = open(absolute_path, 'rb') except (IOError, OSError) as e: if e.errno == errno.ENOENT: raise ValueError( "The file '%s' could not be found with %r." % (clean_name, self)) else: raise content = File(content) opened = True try: file_hash = self.file_hash(clean_name, content) finally: if opened: content.close() path, filename = os.path.split(clean_name) root, ext = os.path.splitext(filename) if file_hash is not None: file_hash = ".%s" % file_hash hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext)) unparsed_name = list(parsed_name) unparsed_name[2] = hashed_name # Special casing for a @font-face hack, like url(myfont.eot?#iefix") # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax if '?#' in name and not unparsed_name[3]: unparsed_name[2] += '?' return urlunsplit(unparsed_name)
def cache(request, cache, path): path = unquote(path) if cache not in fEs.CACHE_TYPES: raise Http404 entity = fEs.by_path(path) if entity is None: raise Http404 user = request.user if request.user.is_authenticated() else None if not entity.may_view(user): if user is None: # user is not logged in return redirect_to_login(request.get_full_path()) raise PermissionDenied entity.ensure_cached(cache) st = os.stat(entity.get_cache_path(cache)) lm = strftime("%a, %d %b %Y %H:%M:%S GMT", gmtime(st.st_mtime)) if request.META.get('HTTP_IF_MODIFIED_SINCE', None) == lm: return HttpResponseNotModified() cc = 'max-age=30780000' # Cache-Control header if not entity.may_view(None): # not publicly cacheable cc = 'private, ' + cc resp = HttpResponse(FileWrapper(open(entity.get_cache_path(cache), 'rb')), content_type=entity.get_cache_mimetype(cache)) resp['Content-Length'] = str(st.st_size) resp['Last-Modified'] = lm resp['Cache-Control'] = cc return resp
def url(self, name, force=False): """ Returns the real URL in DEBUG mode. """ if settings.DEBUG and not force: hashed_name, fragment = name, '' else: clean_name, fragment = urldefrag(name) if urlsplit(clean_name).path.endswith('/'): # don't hash paths hashed_name = name else: cache_key = self.cache_key(name) hashed_name = self.cache.get(cache_key) if hashed_name is None: hashed_name = self.hashed_name(clean_name).replace( '\\', '/') # set the cache if there was a miss # (e.g. if cache server goes down) self.cache.set(cache_key, hashed_name) final_url = super(CachedFilesMixin, self).url(hashed_name) # Special casing for a @font-face hack, like url(myfont.eot?#iefix") # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax query_fragment = '?#' in name # [sic!] if fragment or query_fragment: urlparts = list(urlsplit(final_url)) if fragment and not urlparts[4]: urlparts[4] = fragment if query_fragment and not urlparts[3]: urlparts[2] += '?' final_url = urlunsplit(urlparts) return unquote(final_url)
def app_index(request, app_name, path): if app_name not in configured_clients: raise Http404('{} not configured'.format(app_name)) normalized_path = unquote(path).lstrip('/') if not normalized_path: return render_index(request, app_name, path) return render_index(request, app_name, normalized_path)
def serve_pages(request, coll_id, page_ref, insecure=False, **kwargs): """ Serve static files below a given point in the directory structure or from locations inferred from the staticfiles finders. To use, put a URL pattern such as: url(r'^c/(?P<coll_id>\w{1,128})/p/(?P<page_ref>[\w/.-]{1,250})$', 'annalist.views.statichack.serve_pages`), in your `annalist_root/annalist/urls.py` URL configuration file. It uses the django.views.static.serve() view to serve the found files. """ # log.info("serve_pages %s"%(path)) try: page_path = settings.BASE_SITE_DIR+"/c/"+coll_id+"/p/"+page_ref log.info("statichack.serve_pages %s"%(page_path,)) normalized_path = posixpath.normpath(unquote(page_path)) if not os.path.exists(normalized_path): if page_path.endswith('/') or page_path == '': raise Http404("Directory indexes are not allowed here.") raise Http404("'%s' could not be found" % page_path) document_root, path = os.path.split(normalized_path) # log.info("document_root %s, path %s"%(document_root, path)) except Exception as e: log.info(str(e)) raise return static.serve(request, path, document_root=document_root, **kwargs)
def hashed_name(self, name, content=None): parsed_name = urlsplit(unquote(name)) clean_name = parsed_name.path.strip() opened = False if content is None: if not self.exists(clean_name): raise ValueError("The file '%s' could not be found with %r." % (clean_name, self)) try: content = self.open(clean_name) except IOError: # Handle directory paths and fragments return name opened = True try: file_hash = self.file_hash(clean_name, content) finally: if opened: content.close() path, filename = os.path.split(clean_name) root, ext = os.path.splitext(filename) if file_hash is not None: file_hash = ".%s" % file_hash hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext)) unparsed_name = list(parsed_name) unparsed_name[2] = hashed_name # Special casing for a @font-face hack, like url(myfont.eot?#iefix") # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax if '?#' in name and not unparsed_name[3]: unparsed_name[2] += '?' return urlunsplit(unparsed_name)
def unquote_quote(segment): segment = unquote(force_str(segment)) # Tilde is part of RFC3986 Unreserved Characters # http://tools.ietf.org/html/rfc3986#section-2.3 # See also http://bugs.python.org/issue16285 segment = quote(segment, safe=RFC3986_SUBDELIMS + RFC3986_GENDELIMS + str('~')) return force_text(segment)
def serve_static(request, path, insecure=False, **kwargs): """ Serve static files below a given point in the directory structure or from locations inferred from the staticfiles finders. To use, put a URL pattern such as: (r'^static/(?P<path>.*)$', 'annalist.views.statichack.serve_static') in your `annalist_root/urls.py` URL configuration file. It uses the django.views.static.serve() view to serve the found files. """ # log.info("serve_static %s"%(path)) try: normalized_path = posixpath.normpath(unquote(path)).lstrip('/') absolute_path = finders.find(normalized_path) if not absolute_path: if path.endswith('/') or path == '': raise Http404("Directory indexes are not allowed here.") raise Http404("'%s' could not be found" % path) document_root, path = os.path.split(absolute_path) # log.info("document_root %s, path %s"%(document_root, path)) except Exception as e: log.info(str(e)) raise return static.serve(request, path, document_root=document_root, **kwargs)
def cache(request, cache, path): path = unquote(path) if cache not in fEs.CACHE_TYPES: raise Http404 entity = fEs.by_path(path) if entity is None: raise Http404 user = request.user if request.user.is_authenticated() else None if not entity.may_view(user): if user is None: # user is not logged in return redirect_to_login(request.get_full_path()) raise PermissionDenied entity.ensure_cached(cache) st = os.stat(entity.get_cache_path(cache)) lm = strftime("%a, %d %b %Y %H:%M:%S GMT", gmtime(st.st_mtime)) if request.META.get('HTTP_IF_MODIFIED_SINCE', None) == lm: return HttpResponseNotModified() cc = 'max-age=30780000' # Cache-Control header if not entity.may_view(None): # not publicly cacheable cc = 'private, ' + cc resp = HttpResponse(FileWrapper(open(entity.get_cache_path(cache), 'rb')), content_type=entity.get_cache_mimetype(cache)) resp['Content-Length'] = str(st.st_size) resp['Last-Modified'] = lm resp['Cache-Control'] = cc return resp
def serve(request, path, document_root=None, show_indexes=False): """ Note: Modified to use X-Sendfile Serve static files below a given point in the directory structure. To use, put a URL pattern such as:: from django.views.static import serve url(r'^(?P<path>.*)$', serve, {'document_root': '/path/to/my/files/'}) in your URLconf. You must provide the ``document_root`` param. You may also set ``show_indexes`` to ``True`` if you'd like to serve a basic index of the directory. This index view will use the template hardcoded below, but if you'd like to override it, you can create a template called ``static/directory_index.html``. """ path = posixpath.normpath(unquote(path)).lstrip("/") fullpath = safe_join(document_root, path) if os.path.isdir(fullpath): if show_indexes: return directory_index(path, fullpath) raise Http404("Directory indexes are not allowed here.") if not os.path.exists(fullpath): raise Http404('"%(path)s" does not exist' % {"path": fullpath}) # Respect the If-Modified-Since header. statobj = os.stat(fullpath) if not was_modified_since( request.META.get("HTTP_IF_MODIFIED_SINCE"), statobj.st_mtime, statobj.st_size ): return HttpResponseNotModified() response = HttpResponse() response["X-Sendfile"] = fullpath return response
def url(self, name, force=False): """ Return the real URL in DEBUG mode. """ if settings.DEBUG and not force: hashed_name, fragment = name, '' else: clean_name, fragment = urldefrag(name) if urlsplit(clean_name).path.endswith('/'): # don't hash paths hashed_name = name else: hashed_name = self.stored_name(clean_name) final_url = super(HashedFilesMixin, self).url(hashed_name) # Special casing for a @font-face hack, like url(myfont.eot?#iefix") # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax query_fragment = '?#' in name # [sic!] if fragment or query_fragment: urlparts = list(urlsplit(final_url)) if fragment and not urlparts[4]: urlparts[4] = fragment if query_fragment and not urlparts[3]: urlparts[2] += '?' final_url = urlunsplit(urlparts) return unquote(final_url)
def unquote_quote(segment): segment = unquote(force_str(segment)) # Tilde is part of RFC3986 Unreserved Characters # http://tools.ietf.org/html/rfc3986#section-2.3 # See also http://bugs.python.org/issue16285 segment = quote(segment, safe=RFC3986_SUBDELIMS + RFC3986_GENDELIMS + str('~')) return force_text(segment)
def url(self, name, force=False): """ Returns the real URL in DEBUG mode. """ if settings.DEBUG and not force: hashed_name, fragment = name, "" else: clean_name, fragment = urldefrag(name) if urlsplit(clean_name).path.endswith("/"): # don't hash paths hashed_name = name else: cache_key = self.cache_key(name) hashed_name = self.cache.get(cache_key) if hashed_name is None: hashed_name = self.hashed_name(clean_name).replace("\\", "/") # set the cache if there was a miss # (e.g. if cache server goes down) self.cache.set(cache_key, hashed_name) final_url = super(CachedFilesMixin, self).url(hashed_name) # Special casing for a @font-face hack, like url(myfont.eot?#iefix") # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax query_fragment = "?#" in name # [sic!] if fragment or query_fragment: urlparts = list(urlsplit(final_url)) if fragment and not urlparts[4]: urlparts[4] = fragment if query_fragment and not urlparts[3]: urlparts[2] += "?" final_url = urlunsplit(urlparts) return unquote(final_url)
def hashed_name(self, name, content=None, filename=None): parsed_name = urlsplit(unquote(name)) clean_name = parsed_name.path.strip() opened = False if content is None: absolute_path = finders.find(clean_name) try: content = open(absolute_path, 'rb') except (IOError, OSError) as e: if e.errno == errno.ENOENT: raise ValueError("The file '%s' could not be found with %r." % (clean_name, self)) else: raise content = File(content) opened = True try: file_hash = self.file_hash(clean_name, content) finally: if opened: content.close() path, filename = os.path.split(clean_name) root, ext = os.path.splitext(filename) if file_hash is not None: file_hash = ".%s" % file_hash hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext)) unparsed_name = list(parsed_name) unparsed_name[2] = hashed_name # Special casing for a @font-face hack, like url(myfont.eot?#iefix") # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax if '?#' in name and not unparsed_name[3]: unparsed_name[2] += '?' return urlunsplit(unparsed_name)
def url(self, name, force=False): """ Return the real URL in DEBUG mode. """ if settings.DEBUG and not force: hashed_name, fragment = name, '' else: clean_name, fragment = urldefrag(name) if urlsplit(clean_name).path.endswith('/'): # don't hash paths hashed_name = name else: hashed_name = self.stored_name(clean_name) final_url = super(HashedFilesMixin, self).url(hashed_name) # Special casing for a @font-face hack, like url(myfont.eot?#iefix") # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax query_fragment = '?#' in name # [sic!] if fragment or query_fragment: urlparts = list(urlsplit(final_url)) if fragment and not urlparts[4]: urlparts[4] = fragment if query_fragment and not urlparts[3]: urlparts[2] += '?' final_url = urlunsplit(urlparts) return unquote(final_url)
def construct_absolute_url(url, asset): cwd = os.path.dirname(asset.full_path) url_path = os.path.abspath(os.path.join(cwd, url)) # Get asset url_path = url_path[asset.full_path.index(asset.path):] url_asset = Asset(url_path) return unquote(url_asset.url)
def render(self, context): url = unquote(self.url(context)) if context.autoescape: url = conditional_escape(url) if self.varname is None: return url context[self.varname] = url return ''
def test_post_not_urlencoded(self): url = ( 'https://developer.mozilla.org/en-US/docs/' 'Web/CSS/%3A-moz-ui-invalid') fp = FeaturePage.objects.create(url=url, feature_id=self.feature.id) response = self.client.get(self.url, {'url': unquote(url)}) next_url = reverse('feature_page_detail', kwargs={'pk': fp.id}) self.assertRedirects(response, next_url)
def render(self, context): url = unquote(self.url(context)) if context.autoescape: url = conditional_escape(url) if self.varname is None: return url context[self.varname] = url return ''
def test_post_not_urlencoded(self): url = ( "https://developer.mozilla.org/en-US/docs/" "Web/CSS/%3A-moz-ui-invalid") fp = FeaturePage.objects.create(url=url, feature_id=self.feature.id) response = self.client.get(self.url, {'url': unquote(url)}) next_url = reverse('feature_page_detail', kwargs={'pk': fp.id}) self.assertRedirects(response, next_url)
def style(request, app_name, path): if app_name not in configured_clients: raise Http404('{} not configured'.format(app_name)) client_root = configured_clients[app_name]['root'] normalized_path = unquote(path).lstrip('/') fullpath = safe_join(client_root, normalized_path) return serve_static(fullpath)
def __init__(self, path, **options): super(DocumentBaseSource, self).__init__(path, **options) if path != unquote(path): raise ValueError('URL-encoded path "%s"' % path) try: self.locale, self.slug = self.locale_and_slug(path) except ValueError: self.locale, self.slug = None, None
def __init__(self, path, **options): super(DocumentBaseSource, self).__init__(path, **options) if path != unquote(path): raise ValueError('URL-encoded path "%s"' % path) try: self.locale, self.slug = self.locale_and_slug(path) except ValueError: self.locale, self.slug = None, None
def serve(self, request): os_rel_path = self.file_path(request.path) os_rel_path = posixpath.normpath(unquote(os_rel_path)) # Emulate behavior of django.contrib.staticfiles.views.serve() when it # invokes staticfiles' finders functionality. # TODO: Modify if/when that internal API is refactored final_rel_path = os_rel_path.replace('\\', '/').lstrip('/') return serve(request, final_rel_path, document_root=self.get_base_dir())
def serve(self, request): os_rel_path = self.file_path(request.path) os_rel_path = posixpath.normpath(unquote(os_rel_path)) # Emulate behavior of django.contrib.staticfiles.views.serve() when it # invokes staticfiles' finders functionality. # TODO: Modify if/when that internal API is refactored final_rel_path = os_rel_path.replace('\\', '/').lstrip('/') return serve(request, final_rel_path, document_root=self.get_base_dir())
def converter(matchobj): """ Convert the matched URL to a normalized and hashed URL. This requires figuring out which files the matched URL resolves to and calling the url() method of the storage. """ matched, url = matchobj.groups() # Ignore absolute/protocol-relative, fragments and data-uri URLs. if url.startswith(('http:', 'https:', '//', '#', 'data:')): return matched # Ignore absolute URLs that don't point to a static file (dynamic # CSS / JS?). Note that STATIC_URL cannot be empty. if url.startswith('/') and not url.startswith(settings.STATIC_URL): return matched # Strip off the fragment so a path-like fragment won't interfere. url_path, fragment = urldefrag(url) if url_path.startswith('/'): # Otherwise the condition above would have returned prematurely. assert url_path.startswith(settings.STATIC_URL) target_name = url_path[len(settings.STATIC_URL):] else: # We're using the posixpath module to mix paths and URLs conveniently. source_name = name if os.sep == '/' else name.replace( os.sep, '/') target_name = posixpath.join(posixpath.dirname(source_name), url_path) # Determine the hashed name of the target file with the storage backend. hashed_url = self.url(unquote(target_name), force=True) transformed_url = '/'.join( url_path.split('/')[:-1] + hashed_url.split('/')[-1:]) # Restore the fragment that was stripped off earlier. if fragment: transformed_url += ('?#' if '?#' in url else '#') + fragment # Return the hashed version to the file return template % unquote(transformed_url)
def get_page_from_request(request, use_path=None): """ Gets the current page from a request object. URLs can be of the following form (this should help understand the code): http://server.whatever.com/<some_path>/"pages-root"/some/page/slug <some_path>: This can be anything, and should be stripped when resolving pages names. This means the CMS is not installed at the root of the server's URLs. "pages-root" This is the root of Django urls for the CMS. It is, in essence an empty page slug (slug == '') The page slug can then be resolved to a Page model object """ # The following is used by cms.middleware.page.CurrentPageMiddleware if hasattr(request, '_current_page_cache'): return request._current_page_cache draft = use_draft(request) preview = 'preview' in request.GET # If use_path is given, someone already did the path cleaning if use_path is not None: path = use_path else: path = request.path_info pages_root = unquote(reverse("pages-root")) # otherwise strip off the non-cms part of the URL if is_installed('django.contrib.admin'): admin_base = admin_reverse('index') else: admin_base = None if path.startswith(pages_root) and (not admin_base or not path.startswith(admin_base)): path = path[len(pages_root):] # and strip any final slash if path.endswith("/"): path = path[:-1] page = get_page_from_path(path, preview, draft) if draft and page and not user_can_change_page(request.user, page): page = get_page_from_path(path, preview, draft=False) # For public pages we check if any parent is hidden due to published dates # In this case the selected page is not reachable if page and not draft: ancestors = page.get_ancestors().filter( Q(publication_date__gt=timezone.now()) | Q(publication_end_date__lt=timezone.now()), ) if ancestors.exists(): page = None request._current_page_cache = page return page
def __init__(self, path, **options): super(ZoneRootSource, self).__init__(path, **options) if path != unquote(path): raise ValueError('URL-encoded path "%s"' % path) try: self.locale, self.slug = self.locale_and_zone(path) except ValueError as exception: self.locale, self.slug = None, None logger.warn(exception) self.state = self.STATE_ERROR
def _get_path(self, parsed): path = force_str(parsed[2]) # If there are parameters, add them if parsed[3]: path += str(";") + force_str(parsed[3]) path = unquote(path) # WSGI requires latin-1 encoded strings. See get_path_info(). if six.PY3: path = path.encode('utf-8').decode('iso-8859-1') return path
def resolve(path): # Mostly yanked from Django core and changed to return the path: # See: https://github.com/django/django/blob/1.6.11/django/contrib/staticfiles/views.py normalized_path = posixpath.normpath(unquote(path)).lstrip('/') absolute_path = finders.find(normalized_path) if not absolute_path: raise Http404("'%s' could not be found" % path) if path[-1] == '/' or os.path.isdir(absolute_path): raise Http404('Directory indexes are not allowed here.') return os.path.split(absolute_path)
def converter(matchobj): """ Converts the matched URL depending on the parent level (`..`) and returns the normalized and hashed URL using the url method of the storage. """ matched, url = matchobj.groups() # Completely ignore http(s) prefixed URLs, # fragments and data-uri URLs if url.startswith(('#', 'http:', 'https:', 'data:', '//')): return matched name_parts = name.split(os.sep) # Using posix normpath here to remove duplicates url = posixpath.normpath(url) url_parts = url.split('/') parent_level, sub_level = url.count('..'), url.count('/') if url.startswith('/'): sub_level -= 1 url_parts = url_parts[1:] if parent_level or not url.startswith('/'): start, end = parent_level + 1, parent_level else: if sub_level: if sub_level == 1: parent_level -= 1 start, end = parent_level, 1 else: start, end = 1, sub_level - 1 joined_result = '/'.join(name_parts[:-start] + url_parts[end:]) hashed_url = self.url(unquote(joined_result), force=True) # NOTE: # following two lines are commented out so that absolute urls are used instead of relative urls # to make themed assets work correctly. # # The lines are commented and not removed to make future django upgrade easier and # show exactly what is changed in this method override # # file_name = hashed_url.split('/')[-1:] # relative_url = '/'.join(url.split('/')[:-1] + file_name) # Return the hashed version to the file return template % unquote(hashed_url)
def _get_path(self, parsed): path = force_str(parsed[2]) # If there are parameters, add them if parsed[3]: path += str(";") + force_str(parsed[3]) path = unquote(path) # WSGI requires latin-1 encoded strings. See get_path_info(). if six.PY3: path = path.encode("utf-8").decode("iso-8859-1") return path
def get_page_from_request(request, use_path=None): """ Gets the current page from a request object. URLs can be of the following form (this should help understand the code): http://server.whatever.com/<some_path>/"pages-root"/some/page/slug <some_path>: This can be anything, and should be stripped when resolving pages names. This means the CMS is not installed at the root of the server's URLs. "pages-root" This is the root of Django urls for the CMS. It is, in essence an empty page slug (slug == '') The page slug can then be resolved to a Page model object """ # The following is used by cms.middleware.page.CurrentPageMiddleware if hasattr(request, '_current_page_cache'): return request._current_page_cache draft = use_draft(request) preview = 'preview' in request.GET # If use_path is given, someone already did the path cleaning if use_path is not None: path = use_path else: path = request.path_info pages_root = unquote(reverse("pages-root")) # otherwise strip off the non-cms part of the URL if is_installed('django.contrib.admin'): admin_base = admin_reverse('index') else: admin_base = None if path.startswith(pages_root) and (not admin_base or not path.startswith(admin_base)): path = path[len(pages_root):] # and strip any final slash if path.endswith("/"): path = path[:-1] page = get_page_from_path(path, preview, draft) if draft and page and not user_can_change_page(request.user, page): page = get_page_from_path(path, preview, draft=False) # For public pages we check if any parent is hidden due to published dates # In this case the selected page is not reachable if page and not draft: ancestors = page.get_ancestors().filter( Q(publication_date__gt=timezone.now()) | Q(publication_end_date__lt=timezone.now()), ) if ancestors.exists(): page = None request._current_page_cache = page return page
def converter(matchobj): """ Convert the matched URL to a normalized and hashed URL. This requires figuring out which files the matched URL resolves to and calling the url() method of the storage. """ matched, url = matchobj.groups() # Ignore absolute/protocol-relative, fragments and data-uri URLs. if url.startswith(('http:', 'https:', '//', '#', 'data:')): return matched # Ignore absolute URLs that don't point to a static file (dynamic # CSS / JS?). Note that STATIC_URL cannot be empty. if url.startswith('/') and not url.startswith(settings.STATIC_URL): return matched # Strip off the fragment so a path-like fragment won't interfere. url_path, fragment = urldefrag(url) if url_path.startswith('/'): # Otherwise the condition above would have returned prematurely. assert url_path.startswith(settings.STATIC_URL) target_name = url_path[len(settings.STATIC_URL):] else: # We're using the posixpath module to mix paths and URLs conveniently. source_name = name if os.sep == '/' else name.replace(os.sep, '/') target_name = posixpath.join(posixpath.dirname(source_name), url_path) # Determine the hashed name of the target file with the storage backend. hashed_url = self.url(unquote(target_name), force=True) transformed_url = '/'.join(url_path.split('/')[:-1] + hashed_url.split('/')[-1:]) # Restore the fragment that was stripped off earlier. if fragment: transformed_url += ('?#' if '?#' in url else '#') + fragment # Return the hashed version to the file return template % unquote(transformed_url)
def decode_href(self, href): """Convert URL-escaped href attributes to unicode.""" if PY3: # pragma: no cover # TODO: Remove the PY3 condition once we don't run Python 2 anymore # In Python 3, unquote returns unicode if isinstance(href, binary_type): uhref = href.decode('ascii') else: uhref = href decoded = unquote(uhref) assert isinstance(decoded, text_type) else: # In Python 2, unquote takes and returns binary if isinstance(href, binary_type): bhref = href else: bhref = href.encode('utf-8') decoded = unquote(bhref) assert isinstance(decoded, binary_type) decoded = decoded.decode('utf8') return decoded
def get_encoded_filename(self, filename_parm): """ Handle encoded filenames per RFC6266. See also: https://tools.ietf.org/html/rfc2231#section-4 """ encoded_filename = force_text(filename_parm['filename*']) try: charset, lang, filename = encoded_filename.split('\'', 2) filename = urlparse.unquote(filename) except (ValueError, LookupError): filename = force_text(filename_parm['filename']) return filename