def test_resolver_force_version(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='index.html', version_slug='foo') self.assertEqual(url, '/docs/pip/en/foo/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.pip, filename='index.html', version_slug='foo') self.assertEqual(url, '/en/foo/')
def test_resolver_force_language(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='index.html', language='cz') self.assertEqual(url, '/docs/pip/cz/latest/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.pip, filename='index.html', language='cz') self.assertEqual(url, '/cz/latest/')
def test_resolver_filename_index(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='foo/bar/index.html') self.assertEqual(url, '/docs/pip/en/latest/foo/bar/') url = resolve_path( project=self.pip, filename='foo/index/index.html') self.assertEqual(url, '/docs/pip/en/latest/foo/index/')
def test_resolver_force_domain(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='index.html', cname=True) self.assertEqual(url, '/en/latest/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.pip, filename='index.html', cname=True) self.assertEqual(url, '/en/latest/')
def test_resolver_translation(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.translation, filename='index.html') self.assertEqual(url, '/docs/pip/ja/latest/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.translation, filename='index.html') self.assertEqual(url, '/ja/latest/')
def test_resolver_subproject_subdomain(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.subproject, filename='index.html') self.assertEqual(url, '/docs/pip/projects/sub/ja/latest/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.subproject, filename='index.html') self.assertEqual(url, '/projects/sub/ja/latest/')
def test_resolver_no_force_translation_with_version(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.translation, filename='index.html', language='cz', version_slug='foo') self.assertEqual(url, '/docs/pip/ja/foo/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.translation, filename='index.html', language='cz', version_slug='foo') self.assertEqual(url, '/ja/foo/')
def test_resolver_domain_object_not_canonical(self): self.domain = get(Domain, domain='http://docs.foobar.com', project=self.pip, canonical=False) with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='') self.assertEqual(url, '/docs/pip/en/latest/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.pip, filename='') self.assertEqual(url, '/en/latest/')
def test_resolver_force_domain_single_version(self): self.pip.single_version = False with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='index.html', single_version=True, cname=True) self.assertEqual(url, '/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.pip, filename='index.html', single_version=True, cname=True) self.assertEqual(url, '/')
def test_resolver_subproject_both_single_version(self): self.pip.single_version = True self.subproject.single_version = True with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.subproject, filename='index.html') self.assertEqual(url, '/docs/pip/projects/sub/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.subproject, filename='index.html') self.assertEqual(url, '/projects/sub/')
def test_resolver_filename_sphinx(self): self.pip.documentation_type = 'sphinx' with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='foo/bar') self.assertEqual(url, '/docs/pip/en/latest/foo/bar.html') url = resolve_path(project=self.pip, filename='foo/index') self.assertEqual(url, '/docs/pip/en/latest/foo/') self.pip.documentation_type = 'mkdocs' with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='foo/bar') self.assertEqual(url, '/docs/pip/en/latest/foo/bar/') url = resolve_path(project=self.pip, filename='foo/index') self.assertEqual(url, '/docs/pip/en/latest/foo/')
def test_resolver_domain_object_not_canonical(self): self.domain = fixture.get( Domain, domain='http://docs.foobar.com', project=self.pip, canonical=False, ) with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='') self.assertEqual(url, '/docs/pip/en/latest/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.pip, filename='') self.assertEqual(url, '/en/latest/')
def test_resolver_force_domain_single_version(self): self.pip.single_version = False with override_settings(USE_SUBDOMAIN=False): url = resolve_path( project=self.pip, filename='index.html', single_version=True, cname=True, ) self.assertEqual(url, '/index.html') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( project=self.pip, filename='index.html', single_version=True, cname=True, ) self.assertEqual(url, '/index.html')
def test_resolver_filename(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='/foo/bar/blah.html') self.assertEqual(url, '/docs/pip/en/latest/foo/bar/blah.html') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.pip, filename='/foo/bar/blah.html') self.assertEqual(url, '/en/latest/foo/bar/blah.html') with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='') self.assertEqual(url, '/docs/pip/en/latest/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path(project=self.pip, filename='') self.assertEqual(url, '/en/latest/')
def test_resolver_force_language(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path( project=self.pip, filename='index.html', language='cz', ) self.assertEqual(url, '/docs/pip/cz/latest/index.html') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( project=self.pip, filename='index.html', language='cz', ) self.assertEqual(url, '/cz/latest/index.html')
def test_resolver_force_domain(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path( project=self.pip, filename='index.html', cname=True, ) self.assertEqual(url, '/en/latest/index.html') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( project=self.pip, filename='index.html', cname=True, ) self.assertEqual(url, '/en/latest/index.html')
def test_resolver_force_version(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path( project=self.pip, filename='index.html', version_slug='foo', ) self.assertEqual(url, '/docs/pip/en/foo/index.html') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( project=self.pip, filename='index.html', version_slug='foo', ) self.assertEqual(url, '/en/foo/index.html')
def serve_docs(request, project, subproject, lang_slug=None, version_slug=None, filename=''): """ This exists mainly to map existing proj, lang, version, filename views to the file format. """ if not version_slug: version_slug = project.get_default_version() try: version = project.versions.public(request.user).get(slug=version_slug) except Version.DoesNotExist: # Properly raise a 404 if the version doesn't exist & a 401 if it does if project.versions.filter(slug=version_slug).exists(): return _serve_401(request, project) raise Http404('Version does not exist.') filename = resolve_path( subproject or project, # Resolve the subproject if it exists version_slug=version_slug, language=lang_slug, filename=filename, subdomain=True, # subdomain will make it a "full" path without a URL prefix ) if ( version.privacy_level == constants.PRIVATE and not AdminPermission.is_member(user=request.user, obj=project) ): return _serve_401(request, project) return _serve_symlink_docs(request, filename=filename, project=project, privacy_level=version.privacy_level)
def serve_docs(request, project, subproject, lang_slug=None, version_slug=None, filename=''): """Exists to map existing proj, lang, version, filename views to the file format.""" if not version_slug: version_slug = project.get_default_version() try: version = project.versions.public(request.user).get(slug=version_slug) except Version.DoesNotExist: # Properly raise a 404 if the version doesn't exist & a 401 if it does if project.versions.filter(slug=version_slug).exists(): return _serve_401(request, project) raise Http404('Version does not exist.') filename = resolve_path( subproject or project, # Resolve the subproject if it exists version_slug=version_slug, language=lang_slug, filename=filename, subdomain= True, # subdomain will make it a "full" path without a URL prefix ) if (version.privacy_level == constants.PRIVATE and not AdminPermission.is_member(user=request.user, obj=project)): return _serve_401(request, project) return _serve_symlink_docs( request, filename=filename, project=project, privacy_level=version.privacy_level, )
def _manage_imported_files(version, path, commit): """Update imported files for version :param version: Version instance :param path: Path to search :param commit: Commit that updated path """ changed_files = set() for root, __, filenames in os.walk(path): for filename in filenames: dirpath = os.path.join(root.replace(path, "").lstrip("/"), filename.lstrip("/")) full_path = os.path.join(root, filename) md5 = hashlib.md5(open(full_path, "rb").read()).hexdigest() try: obj, __ = ImportedFile.objects.get_or_create( project=version.project, version=version, path=dirpath, name=filename ) except ImportedFile.MultipleObjectsReturned: log.exception("Error creating ImportedFile") continue if obj.md5 != md5: obj.md5 = md5 changed_files.add(dirpath) if obj.commit != commit: obj.commit = commit obj.save() # Delete ImportedFiles from previous versions ImportedFile.objects.filter(project=version.project, version=version).exclude(commit=commit).delete() # Purge Cache changed_files = [resolve_path(version.project, filename=file, version_slug=version.slug) for file in changed_files] cdn_ids = getattr(settings, "CDN_IDS", None) if cdn_ids: if version.project.slug in cdn_ids: purge(cdn_ids[version.project.slug], changed_files)
def _get_subprojects_and_urls(self): """ Get a tuple of subprojects and its absolute URls. All subprojects share the domain from the parent, so instead of resolving the domain and path for each subproject, we resolve only the path of each one. """ subprojects_and_urls = [] project = self.get_project() subprojects = project.subprojects.select_related('child') if not subprojects.exists(): return subprojects_and_urls main_domain = resolve(project) parsed_main_domain = urlparse(main_domain) for subproject in subprojects: subproject_path = resolve_path(subproject.child) parsed_subproject_domain = parsed_main_domain._replace( path=subproject_path, ) subprojects_and_urls.append(( subproject, parsed_subproject_domain.geturl(), )) return subprojects_and_urls
def _get_hidden_paths(self, project): """Get the absolute paths of the public hidden versions of `project`.""" hidden_versions = (Version.internal.public(project=project).filter( hidden=True)) hidden_paths = [ resolve_path(project, version_slug=version.slug) for version in hidden_versions ] return hidden_paths
def get_full_path(self, filename, language=None, version_slug=None): """ Return a full path for a given filename. This will include version and language information. No protocol/domain is returned. """ # Handle explicit http redirects if re.match("^https?://", filename): return filename return resolve_path(project=self.project, language=language, version_slug=version_slug, filename=filename)
def robots_txt(request, project): """ Serve custom user's defined ``/robots.txt``. If the user added a ``robots.txt`` in the "default version" of the project, we serve it directly. """ # Use the ``robots.txt`` file from the default version configured version_slug = project.get_default_version() version = project.versions.get(slug=version_slug) no_serve_robots_txt = any([ # If project is private or, project.privacy_level == constants.PRIVATE, # default version is private or, version.privacy_level == constants.PRIVATE, # default version is not active or, not version.active, # default version is not built not version.built, ]) if no_serve_robots_txt: # ... we do return a 404 raise Http404() filename = resolve_path( project, version_slug=version_slug, filename='robots.txt', subdomain= True, # subdomain will make it a "full" path without a URL prefix ) # This breaks path joining, by ignoring the root when given an "absolute" path if filename[0] == '/': filename = filename[1:] basepath = PublicSymlink(project).project_root fullpath = os.path.join(basepath, filename) if os.path.exists(fullpath): return HttpResponse(open(fullpath).read(), content_type='text/plain') sitemap_url = '{scheme}://{domain}/sitemap.xml'.format( scheme='https', domain=project.subdomain(), ) return HttpResponse( 'User-agent: *\nAllow: /\nSitemap: {}\n'.format(sitemap_url), content_type='text/plain', )
def robots_txt(request, project): """ Serve custom user's defined ``/robots.txt``. If the user added a ``robots.txt`` in the "default version" of the project, we serve it directly. """ # Use the ``robots.txt`` file from the default version configured version_slug = project.get_default_version() version = project.versions.get(slug=version_slug) no_serve_robots_txt = any([ # If project is private or, project.privacy_level == constants.PRIVATE, # default version is private or, version.privacy_level == constants.PRIVATE, # default version is not active or, not version.active, # default version is not built not version.built, ]) if no_serve_robots_txt: # ... we do return a 404 raise Http404() filename = resolve_path( project, version_slug=version_slug, filename='robots.txt', subdomain=True, # subdomain will make it a "full" path without a URL prefix ) # This breaks path joining, by ignoring the root when given an "absolute" path if filename[0] == '/': filename = filename[1:] basepath = PublicSymlink(project).project_root fullpath = os.path.join(basepath, filename) if os.path.exists(fullpath): return HttpResponse(open(fullpath).read(), content_type='text/plain') sitemap_url = '{scheme}://{domain}/sitemap.xml'.format( scheme='https', domain=project.subdomain(), ) return HttpResponse( 'User-agent: *\nAllow: /\nSitemap: {}\n'.format(sitemap_url), content_type='text/plain', )
def get_full_path(self, filename, language=None, version_slug=None): """ Return a full path for a given filename. This will include version and language information. No protocol/domain is returned. """ # Handle explicit http redirects if re.match('^https?://', filename): return filename return resolve_path(project=self.project, language=language, version_slug=version_slug, filename=filename)
def _manage_imported_files(version, path, commit): """ Update imported files for version. :param version: Version instance :param path: Path to search :param commit: Commit that updated path """ changed_files = set() for root, __, filenames in os.walk(path): for filename in filenames: dirpath = os.path.join( root.replace(path, '').lstrip('/'), filename.lstrip('/'), ) full_path = os.path.join(root, filename) md5 = hashlib.md5(open(full_path, 'rb').read()).hexdigest() try: obj, __ = ImportedFile.objects.get_or_create( project=version.project, version=version, path=dirpath, name=filename, ) except ImportedFile.MultipleObjectsReturned: log.warning('Error creating ImportedFile') continue if obj.md5 != md5: obj.md5 = md5 changed_files.add(dirpath) if obj.commit != commit: obj.commit = commit obj.save() # Delete ImportedFiles from previous versions ImportedFile.objects.filter( project=version.project, version=version, ).exclude(commit=commit).delete() changed_files = [ resolve_path( version.project, filename=file, version_slug=version.slug, ) for file in changed_files ] files_changed.send( sender=Project, project=version.project, files=changed_files, )
def _manage_imported_files(version, path, commit): """Update imported files for version :param version: Version instance :param path: Path to search :param commit: Commit that updated path """ changed_files = set() for root, __, filenames in os.walk(path): for filename in filenames: dirpath = os.path.join( root.replace(path, '').lstrip('/'), filename.lstrip('/')) full_path = os.path.join(root, filename) md5 = hashlib.md5(open(full_path, 'rb').read()).hexdigest() try: obj, __ = ImportedFile.objects.get_or_create( project=version.project, version=version, path=dirpath, name=filename, ) except ImportedFile.MultipleObjectsReturned: log.exception('Error creating ImportedFile') continue if obj.md5 != md5: obj.md5 = md5 changed_files.add(dirpath) if obj.commit != commit: obj.commit = commit obj.save() # Delete ImportedFiles from previous versions ImportedFile.objects.filter( project=version.project, version=version).exclude(commit=commit).delete() # Purge Cache changed_files = [ resolve_path( version.project, filename=fname, version_slug=version.slug, ) for fname in changed_files ] cdn_ids = getattr(settings, 'CDN_IDS', None) if cdn_ids: if version.project.slug in cdn_ids: purge(cdn_ids[version.project.slug], changed_files)
def resolve_404_path(project, version_slug=None, language=None, filename='404.html'): """ Helper to resolve the path of ``404.html`` for project. The resolution is based on ``project`` object, version slug and language. :returns: tuple containing the (basepath, filename) :rtype: tuple """ filename = resolve_path( project, version_slug=version_slug, language=language, filename=filename, subdomain= True, # subdomain will make it a "full" path without a URL prefix ) # This breaks path joining, by ignoring the root when given an "absolute" path if filename[0] == '/': filename = filename[1:] version = None if version_slug: version_qs = project.versions.filter(slug=version_slug) if version_qs.exists(): version = version_qs.first() private = any([ version and version.privacy_level == PRIVATE, not version and project.privacy_level == PRIVATE, ]) if private: symlink = PrivateSymlink(project) else: symlink = PublicSymlink(project) basepath = symlink.project_root fullpath = os.path.join(basepath, filename) return (basepath, filename, fullpath)
def _manage_imported_files(version, path, commit): """ Update imported files for version. :param version: Version instance :param path: Path to search :param commit: Commit that updated path """ changed_files = set() for root, __, filenames in os.walk(path): for filename in filenames: dirpath = os.path.join(root.replace(path, '').lstrip('/'), filename.lstrip('/')) full_path = os.path.join(root, filename) md5 = hashlib.md5(open(full_path, 'rb').read()).hexdigest() try: obj, __ = ImportedFile.objects.get_or_create( project=version.project, version=version, path=dirpath, name=filename, ) except ImportedFile.MultipleObjectsReturned: log.warning('Error creating ImportedFile') continue if obj.md5 != md5: obj.md5 = md5 changed_files.add(dirpath) if obj.commit != commit: obj.commit = commit obj.save() # Delete ImportedFiles from previous versions ImportedFile.objects.filter(project=version.project, version=version ).exclude(commit=commit).delete() changed_files = [ resolve_path( version.project, filename=file, version_slug=version.slug, ) for file in changed_files ] files_changed.send(sender=Project, project=version.project, files=changed_files)
def resolve_404_path(project, version_slug=None, language=None): """ Helper to resolve the path of ``404.html`` for project. The resolution is based on ``project`` object, version slug and language. :returns: tuple containing the (basepath, filename) :rtype: tuple """ filename = resolve_path( project, version_slug=version_slug, language=language, filename='404.html', subdomain=True, # subdomain will make it a "full" path without a URL prefix ) # This breaks path joining, by ignoring the root when given an "absolute" path if filename[0] == '/': filename = filename[1:] version = None if version_slug: version_qs = project.versions.filter(slug=version_slug) if version_qs.exists(): version = version_qs.first() private = any([ version and version.privacy_level == PRIVATE, not version and project.privacy_level == PRIVATE, ]) if private: symlink = PrivateSymlink(project) else: symlink = PublicSymlink(project) basepath = symlink.project_root fullpath = os.path.join(basepath, filename) return (basepath, filename, fullpath)
def make_document_url(project, version=None, page=""): if not project: return "" return resolve_path(project=project, version_slug=version, filename=page)
def test_resolver_urlconf_extra(self): url = resolve_path(project=self.translation, filename='index.html', urlconf='foo/bar/$version/$filename') self.assertEqual(url, 'foo/bar/latest/index.html')
def _manage_imported_files(version, path, commit): """ Update imported files for version. :param version: Version instance :param path: Path to search :param commit: Commit that updated path """ changed_files = set() created_html_files = [] for root, __, filenames in os.walk(path): for filename in filenames: if fnmatch.fnmatch(filename, '*.html'): model_class = HTMLFile else: model_class = ImportedFile dirpath = os.path.join( root.replace(path, '').lstrip('/'), filename.lstrip('/')) full_path = os.path.join(root, filename) md5 = hashlib.md5(open(full_path, 'rb').read()).hexdigest() try: # pylint: disable=unpacking-non-sequence obj, __ = model_class.objects.get_or_create( project=version.project, version=version, path=dirpath, name=filename, ) except model_class.MultipleObjectsReturned: log.warning('Error creating ImportedFile') continue if obj.md5 != md5: obj.md5 = md5 changed_files.add(dirpath) if obj.commit != commit: obj.commit = commit obj.save() if model_class == HTMLFile: # the `obj` is HTMLFile, so add it to the list created_html_files.append(obj) # Send bulk_post_create signal for bulk indexing to Elasticsearch bulk_post_create.send(sender=HTMLFile, instance_list=created_html_files) # Delete the HTMLFile first from previous commit and # send bulk_post_delete signal for bulk removing from Elasticsearch delete_queryset = (HTMLFile.objects.filter( project=version.project, version=version).exclude(commit=commit)) # Keep the objects into memory to send it to signal instance_list = list(delete_queryset) # Safely delete from database delete_queryset.delete() # Always pass the list of instance, not queryset. bulk_post_delete.send(sender=HTMLFile, instance_list=instance_list) # Delete ImportedFiles from previous versions (ImportedFile.objects.filter( project=version.project, version=version).exclude(commit=commit).delete()) changed_files = [ resolve_path( version.project, filename=file, version_slug=version.slug, ) for file in changed_files ] files_changed.send( sender=Project, project=version.project, files=changed_files, )
def test_resolver_filename_index(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path(project=self.pip, filename='foo/bar/index.html') self.assertEqual(url, '/docs/pip/en/latest/foo/bar/') url = resolve_path(project=self.pip, filename='foo/index/index.html') self.assertEqual(url, '/docs/pip/en/latest/foo/index/')