def test_gen_path_info(): input_path = "/home/user/swh-environment/swh-web/" expected_result = [ { "name": "home", "path": "home" }, { "name": "user", "path": "home/user" }, { "name": "swh-environment", "path": "home/user/swh-environment" }, { "name": "swh-web", "path": "home/user/swh-environment/swh-web" }, ] path_info = utils.gen_path_info(input_path) assert path_info == expected_result input_path = "home/user/swh-environment/swh-web" path_info = utils.gen_path_info(input_path) assert path_info == expected_result
def test_gen_path_info(self): input_path = '/home/user/swh-environment/swh-web/' expected_result = [{ 'name': 'home', 'path': 'home' }, { 'name': 'user', 'path': 'home/user' }, { 'name': 'swh-environment', 'path': 'home/user/swh-environment' }, { 'name': 'swh-web', 'path': 'home/user/swh-environment/swh-web' }] path_info = utils.gen_path_info(input_path) self.assertEqual(path_info, expected_result) input_path = 'home/user/swh-environment/swh-web' path_info = utils.gen_path_info(input_path) self.assertEqual(path_info, expected_result)
def test_content_view_text_with_path(client, archive_data, content): path = content["path"] url = reverse( "browse-content", url_args={"query_string": content["sha1"]}, query_params={"path": path}, ) resp = check_html_get_response(client, url, status_code=200, template_used="browse/content.html") assert_contains(resp, '<nav class="bread-crumbs') content_display = _process_content_for_display(archive_data, content) mimetype = content_display["mimetype"] if mimetype.startswith("text/"): hljs_language = content["hljs_language"] assert_contains(resp, '<code class="%s">' % hljs_language) assert_contains(resp, escape(content_display["content_data"])) split_path = path.split("/") root_dir_sha1 = split_path[0] filename = split_path[-1] path = path.replace(root_dir_sha1 + "/", "").replace(filename, "") swhid_context = { "anchor": gen_swhid(DIRECTORY, root_dir_sha1), "path": f"/{path}{filename}", } swh_cnt_id = gen_swhid(CONTENT, content["sha1_git"], metadata=swhid_context) swh_cnt_id_url = reverse("browse-swhid", url_args={"swhid": swh_cnt_id}) assert_contains(resp, swh_cnt_id) assert_contains(resp, swh_cnt_id_url) path_info = gen_path_info(path) root_dir_url = reverse("browse-directory", url_args={"sha1_git": root_dir_sha1}) assert_contains(resp, '<li class="swh-path">', count=len(path_info) + 1) assert_contains( resp, '<a href="' + root_dir_url + '">' + root_dir_sha1[:7] + "</a>") for p in path_info: dir_url = reverse( "browse-directory", url_args={"sha1_git": root_dir_sha1}, query_params={"path": p["path"]}, ) assert_contains(resp, '<a href="' + dir_url + '">' + p["name"] + "</a>") assert_contains(resp, "<li>" + filename + "</li>") url_raw = reverse( "browse-content-raw", url_args={"query_string": content["sha1"]}, query_params={"filename": filename}, ) assert_contains(resp, url_raw) url = reverse( "browse-content", url_args={"query_string": content["sha1"]}, query_params={"path": filename}, ) resp = check_html_get_response(client, url, status_code=200, template_used="browse/content.html") assert_not_contains(resp, '<nav class="bread-crumbs') invalid_path = "%s/foo/bar/baz" % root_dir_sha1 url = reverse( "browse-content", url_args={"query_string": content["sha1"]}, query_params={"path": invalid_path}, ) resp = check_html_get_response(client, url, status_code=404, template_used="error.html")
def _origin_content_view_test_helper( client, archive_data, origin_info, origin_visit, snapshot_sizes, origin_branches, origin_releases, root_dir_sha1, content, visit_id=None, timestamp=None, snapshot_id=None, ): content_path = "/".join(content["path"].split("/")[1:]) if not visit_id and not snapshot_id: visit_id = origin_visit["visit"] query_params = {"origin_url": origin_info["url"], "path": content_path} if timestamp: query_params["timestamp"] = timestamp if visit_id: query_params["visit_id"] = visit_id elif snapshot_id: query_params["snapshot"] = snapshot_id url = reverse("browse-origin-content", query_params=query_params) resp = check_html_get_response( client, url, status_code=200, template_used="browse/content.html" ) assert type(content["data"]) == str assert_contains(resp, '<code class="%s">' % content["hljs_language"]) assert_contains(resp, escape(content["data"])) split_path = content_path.split("/") filename = split_path[-1] path = content_path.replace(filename, "")[:-1] path_info = gen_path_info(path) del query_params["path"] if timestamp: query_params["timestamp"] = format_utc_iso_date( parse_iso8601_date_to_utc(timestamp).isoformat(), "%Y-%m-%dT%H:%M:%SZ" ) root_dir_url = reverse("browse-origin-directory", query_params=query_params) assert_contains(resp, '<li class="swh-path">', count=len(path_info) + 1) assert_contains(resp, '<a href="%s">%s</a>' % (root_dir_url, root_dir_sha1[:7])) for p in path_info: query_params["path"] = p["path"] dir_url = reverse("browse-origin-directory", query_params=query_params) assert_contains(resp, '<a href="%s">%s</a>' % (dir_url, p["name"])) assert_contains(resp, "<li>%s</li>" % filename) query_string = "sha1_git:" + content["sha1_git"] url_raw = reverse( "browse-content-raw", url_args={"query_string": query_string}, query_params={"filename": filename}, ) assert_contains(resp, url_raw) if "path" in query_params: del query_params["path"] origin_branches_url = reverse("browse-origin-branches", query_params=query_params) assert_contains(resp, f'href="{escape(origin_branches_url)}"') assert_contains(resp, f"Branches ({snapshot_sizes['revision']})") origin_releases_url = reverse("browse-origin-releases", query_params=query_params) assert_contains(resp, f'href="{escape(origin_releases_url)}">') assert_contains(resp, f"Releases ({snapshot_sizes['release']})") assert_contains(resp, '<li class="swh-branch">', count=len(origin_branches)) query_params["path"] = content_path for branch in origin_branches: root_dir_branch_url = reverse( "browse-origin-content", query_params={"branch": branch["name"], **query_params}, ) assert_contains(resp, '<a href="%s">' % root_dir_branch_url) assert_contains(resp, '<li class="swh-release">', count=len(origin_releases)) query_params["branch"] = None for release in origin_releases: root_dir_release_url = reverse( "browse-origin-content", query_params={"release": release["name"], **query_params}, ) assert_contains(resp, '<a href="%s">' % root_dir_release_url) url = reverse("browse-origin-content", query_params=query_params) resp = check_html_get_response( client, url, status_code=200, template_used="browse/content.html" ) snapshot = archive_data.snapshot_get(origin_visit["snapshot"]) head_rev_id = archive_data.snapshot_get_head(snapshot) swhid_context = { "origin": origin_info["url"], "visit": gen_swhid(SNAPSHOT, snapshot["id"]), "anchor": gen_swhid(REVISION, head_rev_id), "path": f"/{content_path}", } swh_cnt_id = gen_swhid(CONTENT, content["sha1_git"], metadata=swhid_context) swh_cnt_id_url = reverse("browse-swhid", url_args={"swhid": swh_cnt_id}) assert_contains(resp, swh_cnt_id) assert_contains(resp, swh_cnt_id_url) assert_contains(resp, "swh-take-new-snapshot") _check_origin_link(resp, origin_info["url"]) assert_not_contains(resp, "swh-metadata-popover")
def test_content_view_with_path(self, content): path = content['path'] url = reverse('browse-content', url_args={'query_string': content['sha1']}, query_params={'path': path}) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/content.html') self.assertContains(resp, '<nav class="bread-crumbs') content_display = self._process_content_for_display(content) mimetype = content_display['mimetype'] if mimetype.startswith('text/'): hljs_language = content['hljs_language'] self.assertContains(resp, '<code class="%s">' % hljs_language) self.assertContains(resp, escape(content_display['content_data'])) split_path = path.split('/') root_dir_sha1 = split_path[0] filename = split_path[-1] path = path.replace(root_dir_sha1 + '/', '').replace(filename, '') path_info = gen_path_info(path) root_dir_url = reverse('browse-directory', url_args={'sha1_git': root_dir_sha1}) self.assertContains(resp, '<li class="swh-path">', count=len(path_info) + 1) self.assertContains( resp, '<a href="' + root_dir_url + '">' + root_dir_sha1[:7] + '</a>') for p in path_info: dir_url = reverse('browse-directory', url_args={ 'sha1_git': root_dir_sha1, 'path': p['path'] }) self.assertContains( resp, '<a href="' + dir_url + '">' + p['name'] + '</a>') self.assertContains(resp, '<li>' + filename + '</li>') url_raw = reverse('browse-content-raw', url_args={'query_string': content['sha1']}, query_params={'filename': filename}) self.assertContains(resp, url_raw) url = reverse('browse-content', url_args={'query_string': content['sha1']}, query_params={'path': filename}) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/content.html') self.assertNotContains(resp, '<nav class="bread-crumbs')
def browse_snapshot_content(request, snapshot_id=None, origin_type=None, origin_url=None, timestamp=None, path=None): """ Django view implementation for browsing a content in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, origin_type, origin_url, timestamp, path, browse_context='content') root_sha1_git = snapshot_context['root_sha1_git'] sha1_git = None query_string = None content_data = None if root_sha1_git: content_info = service.lookup_directory_with_path( root_sha1_git, path) sha1_git = content_info['target'] query_string = 'sha1_git:' + sha1_git content_data = request_content(query_string, raise_if_unavailable=False) except Exception as exc: return handle_view_exception(request, exc) swh_type = snapshot_context['swh_type'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] revision_id = snapshot_context['revision_id'] origin_info = snapshot_context['origin_info'] visit_info = snapshot_context['visit_info'] snapshot_id = snapshot_context['snapshot_id'] content = None language = None mimetype = None if content_data and content_data['raw_data'] is not None: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] filename = None path_info = None browse_view_name = 'browse-' + swh_type + '-directory' breadcrumbs = [] split_path = path.split('/') filename = split_path[-1] path_info = gen_path_info(path[:-len(filename)]) if root_sha1_git: breadcrumbs.append({ 'name': root_sha1_git[:7], 'url': reverse(browse_view_name, url_args=url_args, query_params=query_params) }) for pi in path_info: bc_url_args = dict(url_args) bc_url_args['path'] = pi['path'] breadcrumbs.append({ 'name': pi['name'], 'url': reverse(browse_view_name, url_args=bc_url_args, query_params=query_params) }) breadcrumbs.append({'name': filename, 'url': None}) browse_content_link = \ gen_content_link(sha1_git, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}) content_raw_url = None if query_string: content_raw_url = reverse('browse-content-raw', url_args={'query_string': query_string}, query_params={'filename': filename}) browse_rev_link = \ gen_revision_link(revision_id, snapshot_context=snapshot_context, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}) content_metadata = { 'context-independent content': browse_content_link, 'path': None, 'filename': None, 'revision id': revision_id, 'revision': browse_rev_link, 'snapshot id': snapshot_id } cnt_sha1_git = None content_size = None error_code = 200 error_description = '' error_message = '' if content_data: content_metadata['sha1 checksum'] = \ content_data['checksums']['sha1'] content_metadata['sha1_git checksum'] = \ content_data['checksums']['sha1_git'] content_metadata['sha256 checksum'] = \ content_data['checksums']['sha256'] content_metadata['blake2s256 checksum'] = \ content_data['checksums']['blake2s256'] content_metadata['mime type'] = content_data['mimetype'] content_metadata['encoding'] = content_data['encoding'] content_metadata['size'] = filesizeformat(content_data['length']) content_metadata['language'] = content_data['language'] content_metadata['licenses'] = content_data['licenses'] content_metadata['path'] = '/' + path[:-len(filename)] content_metadata['filename'] = filename cnt_sha1_git = content_data['checksums']['sha1_git'] content_size = content_data['length'] error_code = content_data['error_code'] error_message = content_data['error_message'] error_description = content_data['error_description'] if origin_info: content_metadata['origin id'] = origin_info['id'] content_metadata['origin type'] = origin_info['type'] content_metadata['origin url'] = origin_info['url'] content_metadata['origin visit date'] = format_utc_iso_date( visit_info['date']) # noqa content_metadata['origin visit id'] = visit_info['visit'] browse_snapshot_url = reverse('browse-snapshot-content', url_args={ 'snapshot_id': snapshot_id, 'path': path }, query_params=request.GET) browse_snapshot_link = \ gen_link(browse_snapshot_url, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}) content_metadata['snapshot context'] = browse_snapshot_link swh_objects = [{ 'type': 'content', 'id': cnt_sha1_git }, { 'type': 'revision', 'id': revision_id }, { 'type': 'snapshot', 'id': snapshot_id }] release_id = snapshot_context['release_id'] if release_id: swh_objects.append({'type': 'release', 'id': release_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) content_path = '/'.join([bc['name'] for bc in breadcrumbs]) context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading = 'Content - %s - %s - %s' %\ (content_path, snapshot_context['branch'], context_found) return render(request, 'browse/content.html', { 'heading': heading, 'swh_object_name': 'Content', 'swh_object_metadata': content_metadata, 'content': content, 'content_size': content_size, 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'breadcrumbs': breadcrumbs if root_sha1_git else [], 'top_right_link': { 'url': content_raw_url, 'icon': swh_object_icons['content'], 'text': 'Raw File' }, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': error_code, 'error_message': error_message, 'error_description': error_description }, status=error_code)
def browse_snapshot_directory(request, snapshot_id=None, origin_type=None, origin_url=None, timestamp=None, path=None): """ Django view implementation for browsing a directory in a snapshot context. """ try: snapshot_context = _process_snapshot_request( request, snapshot_id, origin_type, origin_url, timestamp, path, browse_context='directory') # noqa root_sha1_git = snapshot_context['root_sha1_git'] sha1_git = root_sha1_git if root_sha1_git and path: dir_info = service.lookup_directory_with_path(root_sha1_git, path) # some readme files can reference assets reachable from the # browsed directory, handle that special case in order to # correctly displayed them if dir_info and dir_info['type'] == 'file': file_raw_url = reverse( 'browse-content-raw', url_args={'query_string': dir_info['checksums']['sha1']}) return redirect(file_raw_url) sha1_git = dir_info['target'] dirs = [] files = [] if sha1_git: dirs, files = get_directory_entries(sha1_git) except Exception as exc: return handle_view_exception(request, exc) swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] visit_info = snapshot_context['visit_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] revision_id = snapshot_context['revision_id'] snapshot_id = snapshot_context['snapshot_id'] path_info = gen_path_info(path) browse_view_name = 'browse-' + swh_type + '-directory' breadcrumbs = [] if root_sha1_git: breadcrumbs.append({ 'name': root_sha1_git[:7], 'url': reverse(browse_view_name, url_args=url_args, query_params=query_params) }) for pi in path_info: bc_url_args = dict(url_args) bc_url_args['path'] = pi['path'] breadcrumbs.append({ 'name': pi['name'], 'url': reverse(browse_view_name, url_args=bc_url_args, query_params=query_params) }) path = '' if path is None else (path + '/') for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}) else: bc_url_args = dict(url_args) bc_url_args['path'] = path + d['name'] d['url'] = reverse(browse_view_name, url_args=bc_url_args, query_params=query_params) sum_file_sizes = 0 readmes = {} browse_view_name = 'browse-' + swh_type + '-content' for f in files: bc_url_args = dict(url_args) bc_url_args['path'] = path + f['name'] f['url'] = reverse(browse_view_name, url_args=bc_url_args, query_params=query_params) if f['length'] is not None: sum_file_sizes += f['length'] f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) browse_view_name = 'browse-' + swh_type + '-log' history_url = None if snapshot_id != _empty_snapshot_id: history_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) nb_files = None nb_dirs = None sum_file_sizes = None dir_path = None if root_sha1_git: nb_files = len(files) nb_dirs = len(dirs) sum_file_sizes = filesizeformat(sum_file_sizes) dir_path = '/' + path browse_dir_link = \ gen_directory_link(sha1_git, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}) browse_rev_link = \ gen_revision_link(revision_id, snapshot_context=snapshot_context, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}) dir_metadata = { 'id': sha1_git, 'context-independent directory': browse_dir_link, 'number of regular files': nb_files, 'number of subdirectories': nb_dirs, 'sum of regular file sizes': sum_file_sizes, 'path': dir_path, 'revision id': revision_id, 'revision': browse_rev_link, 'snapshot id': snapshot_id } if origin_info: dir_metadata['origin id'] = origin_info['id'] dir_metadata['origin type'] = origin_info['type'] dir_metadata['origin url'] = origin_info['url'] dir_metadata['origin visit date'] = format_utc_iso_date( visit_info['date']) # noqa dir_metadata['origin visit id'] = visit_info['visit'] snapshot_context_url = reverse('browse-snapshot-directory', url_args={'snapshot_id': snapshot_id}, query_params=request.GET) browse_snapshot_link = \ gen_link(snapshot_context_url, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}) dir_metadata['snapshot context'] = browse_snapshot_link vault_cooking = { 'directory_context': True, 'directory_id': sha1_git, 'revision_context': True, 'revision_id': revision_id } swh_objects = [{ 'type': 'directory', 'id': sha1_git }, { 'type': 'revision', 'id': revision_id }, { 'type': 'snapshot', 'id': snapshot_id }] release_id = snapshot_context['release_id'] if release_id: swh_objects.append({'type': 'release', 'id': release_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/' context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading = 'Directory - %s - %s - %s' %\ (dir_path, snapshot_context['branch'], context_found) return render( request, 'browse/directory.html', { 'heading': heading, 'swh_object_name': 'Directory', 'swh_object_metadata': dir_metadata, 'dirs': dirs, 'files': files, 'breadcrumbs': breadcrumbs if root_sha1_git else [], 'top_right_link': { 'url': history_url, 'icon': swh_object_icons['revisions history'], 'text': 'History' }, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'snapshot_context': snapshot_context, 'vault_cooking': vault_cooking, 'show_actions_menu': True, 'swh_ids': swh_ids })
def content_display(request, query_string): """Django view that produces an HTML display of a content identified by its hash value. The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/` """ algo, checksum = query.parse_hash(query_string) checksum = hash_to_hex(checksum) origin_url = request.GET.get("origin_url") selected_language = request.GET.get("language") if not origin_url: origin_url = request.GET.get("origin") snapshot_id = request.GET.get("snapshot") path = request.GET.get("path") content_data = {} error_info = {"status_code": 200, "description": None} try: content_data = request_content(query_string) except NotFoundExc as e: error_info["status_code"] = 404 error_info["description"] = f"NotFoundExc: {str(e)}" snapshot_context = None if origin_url is not None or snapshot_id is not None: try: snapshot_context = get_snapshot_context( origin_url=origin_url, snapshot_id=snapshot_id, branch_name=request.GET.get("branch"), release_name=request.GET.get("release"), revision_id=request.GET.get("revision"), path=path, browse_context=CONTENT, ) except NotFoundExc as e: if str(e).startswith("Origin"): raw_cnt_url = reverse("browse-content", url_args={"query_string": query_string}) error_message = ( "The Software Heritage archive has a content " "with the hash you provided but the origin " "mentioned in your request appears broken: %s. " "Please check the URL and try again.\n\n" "Nevertheless, you can still browse the content " "without origin information: %s" % (gen_link(origin_url), gen_link(raw_cnt_url))) raise NotFoundExc(error_message) else: raise e content = None language = None mimetype = None if content_data.get("raw_data") is not None: content_display_data = prepare_content_for_display( content_data["raw_data"], content_data["mimetype"], path) content = content_display_data["content_data"] language = content_display_data["language"] mimetype = content_display_data["mimetype"] # Override language with user-selected language if selected_language is not None: language = selected_language available_languages = None if mimetype and "text/" in mimetype: available_languages = highlightjs.get_supported_languages() filename = None path_info = None directory_id = None root_dir = None if snapshot_context: root_dir = snapshot_context.get("root_directory") query_params = snapshot_context["query_params"] if snapshot_context else {} breadcrumbs = [] if path: split_path = path.split("/") root_dir = root_dir or split_path[0] filename = split_path[-1] if root_dir != path: path = path.replace(root_dir + "/", "") path = path[:-len(filename)] path_info = gen_path_info(path) query_params.pop("path", None) dir_url = reverse( "browse-directory", url_args={"sha1_git": root_dir}, query_params=query_params, ) breadcrumbs.append({"name": root_dir[:7], "url": dir_url}) for pi in path_info: query_params["path"] = pi["path"] dir_url = reverse( "browse-directory", url_args={"sha1_git": root_dir}, query_params=query_params, ) breadcrumbs.append({"name": pi["name"], "url": dir_url}) breadcrumbs.append({"name": filename, "url": None}) if path and root_dir != path: dir_info = archive.lookup_directory_with_path(root_dir, path) directory_id = dir_info["target"] elif root_dir != path: directory_id = root_dir else: root_dir = None query_params = {"filename": filename} content_checksums = content_data.get("checksums", {}) content_url = reverse( "browse-content", url_args={"query_string": query_string}, ) content_raw_url = reverse( "browse-content-raw", url_args={"query_string": query_string}, query_params=query_params, ) content_metadata = ContentMetadata( object_type=CONTENT, object_id=content_checksums.get("sha1_git"), sha1=content_checksums.get("sha1"), sha1_git=content_checksums.get("sha1_git"), sha256=content_checksums.get("sha256"), blake2s256=content_checksums.get("blake2s256"), content_url=content_url, mimetype=content_data.get("mimetype"), encoding=content_data.get("encoding"), size=filesizeformat(content_data.get("length", 0)), language=content_data.get("language"), root_directory=root_dir, path=f"/{path}" if path else None, filename=filename or "", directory=directory_id, revision=None, release=None, snapshot=None, origin_url=origin_url, ) swh_objects = [ SWHObjectInfo(object_type=CONTENT, object_id=content_checksums.get("sha1_git")) ] if directory_id: swh_objects.append( SWHObjectInfo(object_type=DIRECTORY, object_id=directory_id)) if snapshot_context: swh_objects.append( SWHObjectInfo(object_type=REVISION, object_id=snapshot_context["revision_id"])) swh_objects.append( SWHObjectInfo(object_type=SNAPSHOT, object_id=snapshot_context["snapshot_id"])) if snapshot_context["release_id"]: swh_objects.append( SWHObjectInfo(object_type=RELEASE, object_id=snapshot_context["release_id"])) swhids_info = get_swhids_info( swh_objects, snapshot_context, extra_context=content_metadata, ) heading = "Content - %s" % content_checksums.get("sha1_git") if breadcrumbs: content_path = "/".join([bc["name"] for bc in breadcrumbs]) heading += " - %s" % content_path return render( request, "browse/content.html", { "heading": heading, "swh_object_id": swhids_info[0]["swhid"], "swh_object_name": "Content", "swh_object_metadata": content_metadata, "content": content, "content_size": content_data.get("length"), "max_content_size": content_display_max_size, "filename": filename, "encoding": content_data.get("encoding"), "mimetype": mimetype, "language": language, "available_languages": available_languages, "breadcrumbs": breadcrumbs, "top_right_link": { "url": content_raw_url, "icon": swh_object_icons["content"], "text": "Raw File", }, "snapshot_context": snapshot_context, "vault_cooking": None, "show_actions": True, "swhids_info": swhids_info, "error_code": error_info["status_code"], "error_message": http_status_code_message.get(error_info["status_code"]), "error_description": error_info["description"], }, status=error_info["status_code"], )
def revision_browse(request, sha1_git, extra_path=None): """ Django view that produces an HTML display of a revision identified by its id. The url that points to it is :http:get:`/browse/revision/(sha1_git)/`. """ try: revision = service.lookup_revision(sha1_git) # some readme files can reference assets reachable from the # browsed directory, handle that special case in order to # correctly displayed them if extra_path: dir_info = \ service.lookup_directory_with_path(revision['directory'], extra_path) if dir_info and dir_info['type'] == 'file': file_raw_url = reverse( 'browse-content-raw', url_args={'query_string': dir_info['checksums']['sha1']}) return redirect(file_raw_url) origin_info = None snapshot_context = None origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) timestamp = request.GET.get('timestamp', None) visit_id = request.GET.get('visit_id', None) snapshot_id = request.GET.get('snapshot_id', None) path = request.GET.get('path', None) dir_id = None dirs, files = None, None content_data = None if origin_url: try: snapshot_context = get_snapshot_context( None, origin_type, origin_url, timestamp, visit_id) except Exception: raw_rev_url = reverse('browse-revision', url_args={'sha1_git': sha1_git}) error_message = \ ('The Software Heritage archive has a revision ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the revision ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_rev_url))) raise NotFoundExc(error_message) origin_info = snapshot_context['origin_info'] snapshot_id = snapshot_context['snapshot_id'] elif snapshot_id: snapshot_context = get_snapshot_context(snapshot_id) if path: file_info = \ service.lookup_directory_with_path(revision['directory'], path) if file_info['type'] == 'dir': dir_id = file_info['target'] else: query_string = 'sha1_git:' + file_info['target'] content_data = request_content(query_string, raise_if_unavailable=False) else: dir_id = revision['directory'] if dir_id: path = '' if path is None else (path + '/') dirs, files = get_directory_entries(dir_id) except Exception as exc: return handle_view_exception(request, exc) revision_data = {} author_name = 'None' revision_data['author'] = 'None' if revision['author']: author_name = revision['author']['name'] or \ revision['author']['fullname'] revision_data['author'] = \ gen_person_link(revision['author']['id'], author_name, snapshot_context) revision_data['committer'] = 'None' if revision['committer']: revision_data['committer'] = \ gen_person_link(revision['committer']['id'], revision['committer']['name'], snapshot_context) revision_data['committer date'] = \ format_utc_iso_date(revision['committer_date']) revision_data['date'] = format_utc_iso_date(revision['date']) if snapshot_context: revision_data['snapshot id'] = snapshot_id revision_data['directory'] = \ gen_snapshot_directory_link(snapshot_context, sha1_git, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', # noqa 'role': 'button'}) else: revision_data['directory'] = \ gen_directory_link(revision['directory'], link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}) revision_data['id'] = sha1_git revision_data['merge'] = revision['merge'] revision_data['metadata'] = escape( json.dumps(revision['metadata'], sort_keys=True, indent=4, separators=(',', ': '))) if origin_info: revision_data['context-independent revision'] = \ gen_revision_link(sha1_git, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}) revision_data['origin id'] = origin_info['id'] revision_data['origin type'] = origin_info['type'] revision_data['origin url'] = gen_link(origin_info['url'], origin_info['url']) browse_snapshot_link = \ gen_snapshot_link(snapshot_id, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}) revision_data['snapshot'] = browse_snapshot_link parents = '' for p in revision['parents']: parent_link = gen_revision_link(p, snapshot_context=snapshot_context) parents += parent_link + '<br/>' revision_data['parents'] = mark_safe(parents) revision_data['synthetic'] = revision['synthetic'] revision_data['type'] = revision['type'] message_lines = ['None'] if revision['message']: message_lines = revision['message'].split('\n') parents = [] for p in revision['parents']: parent_url = gen_revision_url(p, snapshot_context) parents.append({'id': p, 'url': parent_url}) path_info = gen_path_info(path) query_params = { 'snapshot_id': snapshot_id, 'origin_type': origin_type, 'origin': origin_url, 'timestamp': timestamp, 'visit_id': visit_id } breadcrumbs = [] breadcrumbs.append({ 'name': revision['directory'][:7], 'url': reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params) }) for pi in path_info: query_params['path'] = pi['path'] breadcrumbs.append({ 'name': pi['name'], 'url': reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params) }) vault_cooking = { 'directory_context': False, 'directory_id': None, 'revision_context': True, 'revision_id': sha1_git } swh_objects = [{'type': 'revision', 'id': sha1_git}] content = None content_size = None mimetype = None language = None readme_name = None readme_url = None readme_html = None readmes = {} error_code = 200 error_message = '' error_description = '' if content_data: breadcrumbs[-1]['url'] = None content_size = content_data['length'] mimetype = content_data['mimetype'] if content_data['raw_data']: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] query_params = {} if path: filename = path_info[-1]['name'] query_params['filename'] = path_info[-1]['name'] revision_data['filename'] = filename top_right_link = { 'url': reverse('browse-content-raw', url_args={'query_string': query_string}, query_params=query_params), 'icon': swh_object_icons['content'], 'text': 'Raw File' } swh_objects.append({'type': 'content', 'id': file_info['target']}) error_code = content_data['error_code'] error_message = content_data['error_message'] error_description = content_data['error_description'] else: for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}) else: query_params['path'] = path + d['name'] d['url'] = reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params) for f in files: query_params['path'] = path + f['name'] f['url'] = reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params) if f['length'] is not None: f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) top_right_link = { 'url': get_revision_log_url(sha1_git, snapshot_context), 'icon': swh_object_icons['revisions history'], 'text': 'History' } vault_cooking['directory_context'] = True vault_cooking['directory_id'] = dir_id swh_objects.append({'type': 'directory', 'id': dir_id}) diff_revision_url = reverse('diff-revision', url_args={'sha1_git': sha1_git}, query_params={ 'origin_type': origin_type, 'origin': origin_url, 'timestamp': timestamp, 'visit_id': visit_id }) if snapshot_id: swh_objects.append({'type': 'snapshot', 'id': snapshot_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) heading = 'Revision - %s - %s' %\ (sha1_git[:7], textwrap.shorten(message_lines[0], width=70)) if snapshot_context: context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading += ' - %s' % context_found return render(request, 'browse/revision.html', { 'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Revision', 'swh_object_metadata': revision_data, 'message_header': message_lines[0], 'message_body': '\n'.join(message_lines[1:]), 'parents': parents, 'snapshot_context': snapshot_context, 'dirs': dirs, 'files': files, 'content': content, 'content_size': content_size, 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'breadcrumbs': breadcrumbs, 'top_right_link': top_right_link, 'vault_cooking': vault_cooking, 'diff_revision_url': diff_revision_url, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': error_code, 'error_message': error_message, 'error_description': error_description }, status=error_code)
def _directory_view_checks( client, root_directory_sha1, directory_entries, path=None, origin_url=None, snapshot_id=None, revision_id=None, ): dirs = [e for e in directory_entries if e["type"] in ("dir", "rev")] files = [e for e in directory_entries if e["type"] == "file"] url_args = {"sha1_git": root_directory_sha1} query_params = {"origin_url": origin_url, "snapshot": snapshot_id} url = reverse( "browse-directory", url_args=url_args, query_params={ **query_params, "path": path }, ) root_dir_url = reverse( "browse-directory", url_args=url_args, query_params=query_params, ) resp = check_html_get_response(client, url, status_code=200, template_used="browse/directory.html") assert_contains( resp, '<a href="' + root_dir_url + '">' + root_directory_sha1[:7] + "</a>", ) assert_contains(resp, '<td class="swh-directory">', count=len(dirs)) assert_contains(resp, '<td class="swh-content">', count=len(files)) for d in dirs: if d["type"] == "rev": dir_url = reverse("browse-revision", url_args={"sha1_git": d["target"]}) else: dir_path = d["name"] if path: dir_path = "%s/%s" % (path, d["name"]) dir_url = reverse( "browse-directory", url_args={"sha1_git": root_directory_sha1}, query_params={ **query_params, "path": dir_path }, ) assert_contains(resp, dir_url) for f in files: file_path = "%s/%s" % (root_directory_sha1, f["name"]) if path: file_path = "%s/%s/%s" % (root_directory_sha1, path, f["name"]) query_string = "sha1_git:" + f["target"] file_url = reverse( "browse-content", url_args={"query_string": query_string}, query_params={ **query_params, "path": file_path }, ) assert_contains(resp, file_url) path_info = gen_path_info(path) assert_contains(resp, '<li class="swh-path">', count=len(path_info) + 1) assert_contains( resp, '<a href="%s">%s</a>' % (root_dir_url, root_directory_sha1[:7])) for p in path_info: dir_url = reverse( "browse-directory", url_args={"sha1_git": root_directory_sha1}, query_params={ **query_params, "path": p["path"] }, ) assert_contains(resp, '<a href="%s">%s</a>' % (dir_url, p["name"])) assert_contains(resp, "vault-cook-directory") swh_dir_id = gen_swhid(DIRECTORY, directory_entries[0]["dir_id"]) swh_dir_id_url = reverse("browse-swhid", url_args={"swhid": swh_dir_id}) swhid_context = {} if origin_url: swhid_context["origin"] = origin_url if snapshot_id: swhid_context["visit"] = gen_swhid(SNAPSHOT, snapshot_id) if root_directory_sha1 != directory_entries[0]["dir_id"]: swhid_context["anchor"] = gen_swhid(DIRECTORY, root_directory_sha1) if root_directory_sha1 != directory_entries[0]["dir_id"]: swhid_context["anchor"] = gen_swhid(DIRECTORY, root_directory_sha1) if revision_id: swhid_context["anchor"] = gen_swhid(REVISION, revision_id) swhid_context["path"] = f"/{path}/" if path else None swh_dir_id = gen_swhid(DIRECTORY, directory_entries[0]["dir_id"], metadata=swhid_context) swh_dir_id_url = reverse("browse-swhid", url_args={"swhid": swh_dir_id}) assert_contains(resp, swh_dir_id) assert_contains(resp, swh_dir_id_url) assert_not_contains(resp, "swh-metadata-popover")
def make_api_response(request, data, doc_data={}, options={}): """Generates an API response based on the requested mimetype. Args: request: a DRF Request object data: raw data to return in the API response doc_data: documentation data for HTML response options: optional data that can be used to generate the response Returns: a DRF Response a object """ if data: options['headers'] = compute_link_header(data, options) data = transform(data) data = filter_by_fields(request, data) doc_env = doc_data headers = {} if 'headers' in options: doc_env['headers_data'] = options['headers'] headers = options['headers'] # get request status code doc_env['status_code'] = options.get('status', 200) response_args = { 'status': doc_env['status_code'], 'headers': headers, 'content_type': request.accepted_media_type } # when requesting HTML, typically when browsing the API through its # documented views, we need to enrich the input data with documentation # related ones and inform DRF that we request HTML template rendering if request.accepted_media_type == 'text/html': if data: data = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')) doc_env['response_data'] = data doc_env['request'] = { 'path': request.path, 'method': request.method, 'absolute_uri': request.build_absolute_uri(), } doc_env['heading'] = shorten_path(str(request.path)) if 'route' in doc_env: doc_env['endpoint_path'] = gen_path_info(doc_env['route']) response_args['data'] = doc_env response_args['template_name'] = 'api/apidoc.html' # otherwise simply return the raw data and let DRF picks # the correct renderer (JSON or YAML) else: response_args['data'] = data return Response(**response_args)
def content_display(request, query_string): """Django view that produces an HTML display of a content identified by its hash value. The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/` """ # noqa try: algo, checksum = query.parse_hash(query_string) checksum = hash_to_hex(checksum) content_data = request_content(query_string, raise_if_unavailable=False) origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) snapshot_context = None if origin_url: try: snapshot_context = get_snapshot_context( None, origin_type, origin_url) except Exception: raw_cnt_url = reverse('browse-content', url_args={'query_string': query_string}) error_message = \ ('The Software Heritage archive has a content ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the content ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_cnt_url))) raise NotFoundExc(error_message) if snapshot_context: snapshot_context['visit_info'] = None except Exception as exc: return handle_view_exception(request, exc) path = request.GET.get('path', None) content = None language = None mimetype = None if content_data['raw_data'] is not None: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] root_dir = None filename = None path_info = None query_params = {'origin': origin_url} breadcrumbs = [] if path: split_path = path.split('/') root_dir = split_path[0] filename = split_path[-1] if root_dir != path: path = path.replace(root_dir + '/', '') path = path[:-len(filename)] path_info = gen_path_info(path) dir_url = reverse('browse-directory', url_args={'sha1_git': root_dir}, query_params=query_params) breadcrumbs.append({'name': root_dir[:7], 'url': dir_url}) for pi in path_info: dir_url = reverse('browse-directory', url_args={ 'sha1_git': root_dir, 'path': pi['path'] }, query_params=query_params) breadcrumbs.append({'name': pi['name'], 'url': dir_url}) breadcrumbs.append({'name': filename, 'url': None}) query_params = {'filename': filename} content_raw_url = reverse('browse-content-raw', url_args={'query_string': query_string}, query_params=query_params) content_metadata = { 'sha1 checksum': content_data['checksums']['sha1'], 'sha1_git checksum': content_data['checksums']['sha1_git'], 'sha256 checksum': content_data['checksums']['sha256'], 'blake2s256 checksum': content_data['checksums']['blake2s256'], 'mime type': content_data['mimetype'], 'encoding': content_data['encoding'], 'size': filesizeformat(content_data['length']), 'language': content_data['language'], 'licenses': content_data['licenses'], 'filename': filename } if filename: content_metadata['filename'] = filename sha1_git = content_data['checksums']['sha1_git'] swh_ids = get_swh_persistent_ids([{'type': 'content', 'id': sha1_git}]) heading = 'Content - %s' % sha1_git if breadcrumbs: content_path = '/'.join([bc['name'] for bc in breadcrumbs]) heading += ' - %s' % content_path return render(request, 'browse/content.html', { 'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Content', 'swh_object_metadata': content_metadata, 'content': content, 'content_size': content_data['length'], 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'breadcrumbs': breadcrumbs, 'top_right_link': { 'url': content_raw_url, 'icon': swh_object_icons['content'], 'text': 'Raw File' }, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': content_data['error_code'], 'error_message': content_data['error_message'], 'error_description': content_data['error_description'] }, status=content_data['error_code'])
def _directory_browse(request, sha1_git, path=None): root_sha1_git = sha1_git error_info = {"status_code": 200, "description": None} if path: try: dir_info = archive.lookup_directory_with_path(sha1_git, path) sha1_git = dir_info["target"] except NotFoundExc as e: error_info["status_code"] = 404 error_info["description"] = f"NotFoundExc: {str(e)}" sha1_git = None dirs, files = [], [] if sha1_git is not None: dirs, files = get_directory_entries(sha1_git) origin_url = request.GET.get("origin_url") if not origin_url: origin_url = request.GET.get("origin") snapshot_id = request.GET.get("snapshot") snapshot_context = None if origin_url is not None or snapshot_id is not None: try: snapshot_context = get_snapshot_context( snapshot_id=snapshot_id, origin_url=origin_url, branch_name=request.GET.get("branch"), release_name=request.GET.get("release"), revision_id=request.GET.get("revision"), path=path, ) except NotFoundExc as e: if str(e).startswith("Origin"): raw_dir_url = reverse( "browse-directory", url_args={"sha1_git": sha1_git} ) error_message = ( "The Software Heritage archive has a directory " "with the hash you provided but the origin " "mentioned in your request appears broken: %s. " "Please check the URL and try again.\n\n" "Nevertheless, you can still browse the directory " "without origin information: %s" % (gen_link(origin_url), gen_link(raw_dir_url)) ) raise NotFoundExc(error_message) else: raise e path_info = gen_path_info(path) query_params = snapshot_context["query_params"] if snapshot_context else {} breadcrumbs = [] breadcrumbs.append( { "name": root_sha1_git[:7], "url": reverse( "browse-directory", url_args={"sha1_git": root_sha1_git}, query_params={**query_params, "path": None}, ), } ) for pi in path_info: breadcrumbs.append( { "name": pi["name"], "url": reverse( "browse-directory", url_args={"sha1_git": root_sha1_git}, query_params={**query_params, "path": pi["path"],}, ), } ) path = "" if path is None else (path + "/") for d in dirs: if d["type"] == "rev": d["url"] = reverse( "browse-revision", url_args={"sha1_git": d["target"]}, query_params=query_params, ) else: d["url"] = reverse( "browse-directory", url_args={"sha1_git": root_sha1_git}, query_params={**query_params, "path": path + d["name"],}, ) sum_file_sizes = 0 readmes = {} for f in files: query_string = "sha1_git:" + f["target"] f["url"] = reverse( "browse-content", url_args={"query_string": query_string}, query_params={ **query_params, "path": root_sha1_git + "/" + path + f["name"], }, ) if f["length"] is not None: sum_file_sizes += f["length"] f["length"] = filesizeformat(f["length"]) if f["name"].lower().startswith("readme"): readmes[f["name"]] = f["checksums"]["sha1"] readme_name, readme_url, readme_html = get_readme_to_display(readmes) sum_file_sizes = filesizeformat(sum_file_sizes) dir_metadata = DirectoryMetadata( object_type=DIRECTORY, object_id=sha1_git, directory=root_sha1_git, nb_files=len(files), nb_dirs=len(dirs), sum_file_sizes=sum_file_sizes, root_directory=root_sha1_git, path=f"/{path}" if path else None, revision=None, revision_found=None, release=None, snapshot=None, ) vault_cooking = { "directory_context": True, "directory_id": sha1_git, "revision_context": False, "revision_id": None, } swh_objects = [SWHObjectInfo(object_type=DIRECTORY, object_id=sha1_git)] if snapshot_context: swh_objects.append( SWHObjectInfo( object_type=REVISION, object_id=snapshot_context["revision_id"] ) ) swh_objects.append( SWHObjectInfo( object_type=SNAPSHOT, object_id=snapshot_context["snapshot_id"] ) ) if snapshot_context["release_id"]: swh_objects.append( SWHObjectInfo( object_type=RELEASE, object_id=snapshot_context["release_id"] ) ) swhids_info = get_swhids_info(swh_objects, snapshot_context, dir_metadata) heading = "Directory - %s" % sha1_git if breadcrumbs: dir_path = "/".join([bc["name"] for bc in breadcrumbs]) + "/" heading += " - %s" % dir_path top_right_link = None if snapshot_context is not None and not snapshot_context["is_empty"]: history_url = reverse( "browse-revision-log", url_args={"sha1_git": snapshot_context["revision_id"]}, query_params=query_params, ) top_right_link = { "url": history_url, "icon": swh_object_icons["revisions history"], "text": "History", } return render( request, "browse/directory.html", { "heading": heading, "swh_object_id": swhids_info[0]["swhid"], "swh_object_name": "Directory", "swh_object_metadata": dir_metadata, "dirs": dirs, "files": files, "breadcrumbs": breadcrumbs, "top_right_link": top_right_link, "readme_name": readme_name, "readme_url": readme_url, "readme_html": readme_html, "snapshot_context": snapshot_context, "vault_cooking": vault_cooking, "show_actions": True, "swhids_info": swhids_info, "error_code": error_info["status_code"], "error_message": http_status_code_message.get(error_info["status_code"]), "error_description": error_info["description"], }, status=error_info["status_code"], )
def directory_browse(request, sha1_git, path=None): """Django view for browsing the content of a directory identified by its sha1_git value. The url that points to it is :http:get:`/browse/directory/(sha1_git)/[(path)/]` """ # noqa root_sha1_git = sha1_git try: if path: dir_info = service.lookup_directory_with_path(sha1_git, path) # some readme files can reference assets reachable from the # browsed directory, handle that special case in order to # correctly displayed them if dir_info and dir_info['type'] == 'file': file_raw_url = reverse( 'browse-content-raw', url_args={'query_string': dir_info['checksums']['sha1']}) return redirect(file_raw_url) sha1_git = dir_info['target'] dirs, files = get_directory_entries(sha1_git) origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) snapshot_context = None if origin_url: try: snapshot_context = get_snapshot_context( None, origin_type, origin_url) except Exception: raw_dir_url = reverse('browse-directory', url_args={'sha1_git': sha1_git}) error_message = \ ('The Software Heritage archive has a directory ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the directory ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_dir_url))) raise NotFoundExc(error_message) if snapshot_context: snapshot_context['visit_info'] = None except Exception as exc: return handle_view_exception(request, exc) path_info = gen_path_info(path) query_params = {'origin': origin_url} breadcrumbs = [] breadcrumbs.append({ 'name': root_sha1_git[:7], 'url': reverse('browse-directory', url_args={'sha1_git': root_sha1_git}, query_params=query_params) }) for pi in path_info: breadcrumbs.append({ 'name': pi['name'], 'url': reverse('browse-directory', url_args={ 'sha1_git': root_sha1_git, 'path': pi['path'] }, query_params=query_params) }) path = '' if path is None else (path + '/') for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}, query_params=query_params) else: d['url'] = reverse('browse-directory', url_args={ 'sha1_git': root_sha1_git, 'path': path + d['name'] }, query_params=query_params) sum_file_sizes = 0 readmes = {} for f in files: query_string = 'sha1_git:' + f['target'] f['url'] = reverse('browse-content', url_args={'query_string': query_string}, query_params={ 'path': root_sha1_git + '/' + path + f['name'], 'origin': origin_url }) if f['length'] is not None: sum_file_sizes += f['length'] f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) sum_file_sizes = filesizeformat(sum_file_sizes) dir_metadata = { 'id': sha1_git, 'number of regular files': len(files), 'number of subdirectories': len(dirs), 'sum of regular file sizes': sum_file_sizes } vault_cooking = { 'directory_context': True, 'directory_id': sha1_git, 'revision_context': False, 'revision_id': None } swh_ids = get_swh_persistent_ids([{'type': 'directory', 'id': sha1_git}]) heading = 'Directory - %s' % sha1_git if breadcrumbs: dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/' heading += ' - %s' % dir_path return render( request, 'browse/directory.html', { 'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Directory', 'swh_object_metadata': dir_metadata, 'dirs': dirs, 'files': files, 'breadcrumbs': breadcrumbs, 'top_right_link': None, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'snapshot_context': snapshot_context, 'vault_cooking': vault_cooking, 'show_actions_menu': True, 'swh_ids': swh_ids })
def origin_content_view_helper(self, origin_info, origin_visits, origin_branches, origin_releases, root_dir_sha1, content, visit_id=None, timestamp=None): content_path = '/'.join(content['path'].split('/')[1:]) url_args = {'origin_type': origin_info['type'], 'origin_url': origin_info['url'], 'path': content_path} if not visit_id: visit_id = origin_visits[-1]['visit'] query_params = {} if timestamp: url_args['timestamp'] = timestamp if visit_id: query_params['visit_id'] = visit_id url = reverse('browse-origin-content', url_args=url_args, query_params=query_params) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('content.html') self.assertContains(resp, '<code class="%s">' % content['hljs_language']) self.assertContains(resp, escape(content['data'])) split_path = content_path.split('/') filename = split_path[-1] path = content_path.replace(filename, '')[:-1] path_info = gen_path_info(path) del url_args['path'] if timestamp: url_args['timestamp'] = \ format_utc_iso_date(parse_timestamp(timestamp).isoformat(), '%Y-%m-%dT%H:%M:%S') root_dir_url = reverse('browse-origin-directory', url_args=url_args, query_params=query_params) self.assertContains(resp, '<li class="swh-path">', count=len(path_info)+1) self.assertContains(resp, '<a href="%s">%s</a>' % (root_dir_url, root_dir_sha1[:7])) for p in path_info: url_args['path'] = p['path'] dir_url = reverse('browse-origin-directory', url_args=url_args, query_params=query_params) self.assertContains(resp, '<a href="%s">%s</a>' % (dir_url, p['name'])) self.assertContains(resp, '<li>%s</li>' % filename) query_string = 'sha1_git:' + content['sha1_git'] url_raw = reverse('browse-content-raw', url_args={'query_string': query_string}, query_params={'filename': filename}) self.assertContains(resp, url_raw) if 'args' in url_args: del url_args['path'] origin_branches_url = reverse('browse-origin-branches', url_args=url_args, query_params=query_params) self.assertContains(resp, '<a href="%s">Branches (%s)</a>' % (origin_branches_url, len(origin_branches))) origin_releases_url = reverse('browse-origin-releases', url_args=url_args, query_params=query_params) self.assertContains(resp, '<a href="%s">Releases (%s)</a>' % (origin_releases_url, len(origin_releases))) self.assertContains(resp, '<li class="swh-branch">', count=len(origin_branches)) url_args['path'] = content_path for branch in origin_branches: query_params['branch'] = branch['name'] root_dir_branch_url = reverse('browse-origin-content', url_args=url_args, query_params=query_params) self.assertContains(resp, '<a href="%s">' % root_dir_branch_url) self.assertContains(resp, '<li class="swh-release">', count=len(origin_releases)) query_params['branch'] = None for release in origin_releases: query_params['release'] = release['name'] root_dir_release_url = reverse('browse-origin-content', url_args=url_args, query_params=query_params) self.assertContains(resp, '<a href="%s">' % root_dir_release_url) del url_args['origin_type'] url = reverse('browse-origin-content', url_args=url_args, query_params=query_params) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('content.html') swh_cnt_id = get_swh_persistent_id('content', content['sha1_git']) swh_cnt_id_url = reverse('browse-swh-id', url_args={'swh_id': swh_cnt_id}) self.assertContains(resp, swh_cnt_id) self.assertContains(resp, swh_cnt_id_url) self.assertContains(resp, 'swh-take-new-snapshot')
def revision_browse(request, sha1_git): """ Django view that produces an HTML display of a revision identified by its id. The url that points to it is :http:get:`/browse/revision/(sha1_git)/`. """ revision = archive.lookup_revision(sha1_git) origin_info = None snapshot_context = None origin_url = request.GET.get("origin_url") if not origin_url: origin_url = request.GET.get("origin") timestamp = request.GET.get("timestamp") visit_id = request.GET.get("visit_id") snapshot_id = request.GET.get("snapshot_id") if not snapshot_id: snapshot_id = request.GET.get("snapshot") path = request.GET.get("path") dir_id = None dirs, files = [], [] content_data = {} if origin_url: try: snapshot_context = get_snapshot_context( snapshot_id=snapshot_id, origin_url=origin_url, timestamp=timestamp, visit_id=visit_id, branch_name=request.GET.get("branch"), release_name=request.GET.get("release"), revision_id=sha1_git, path=path, ) except NotFoundExc as e: raw_rev_url = reverse("browse-revision", url_args={"sha1_git": sha1_git}) error_message = ("The Software Heritage archive has a revision " "with the hash you provided but the origin " "mentioned in your request appears broken: %s. " "Please check the URL and try again.\n\n" "Nevertheless, you can still browse the revision " "without origin information: %s" % (gen_link(origin_url), gen_link(raw_rev_url))) if str(e).startswith("Origin"): raise NotFoundExc(error_message) else: raise e origin_info = snapshot_context["origin_info"] snapshot_id = snapshot_context["snapshot_id"] elif snapshot_id: snapshot_context = get_snapshot_context(snapshot_id) error_info = {"status_code": 200, "description": None} if path: try: file_info = archive.lookup_directory_with_path( revision["directory"], path) if file_info["type"] == "dir": dir_id = file_info["target"] else: query_string = "sha1_git:" + file_info["target"] content_data = request_content(query_string) except NotFoundExc as e: error_info["status_code"] = 404 error_info["description"] = f"NotFoundExc: {str(e)}" else: dir_id = revision["directory"] if dir_id: path = "" if path is None else (path + "/") dirs, files = get_directory_entries(dir_id) revision_metadata = RevisionMetadata( object_type=REVISION, object_id=sha1_git, revision=sha1_git, author=revision["author"]["fullname"] if revision["author"] else "None", author_url=gen_person_mail_link(revision["author"]) if revision["author"] else "None", committer=revision["committer"]["fullname"] if revision["committer"] else "None", committer_url=gen_person_mail_link(revision["committer"]) if revision["committer"] else "None", committer_date=format_utc_iso_date(revision["committer_date"]), date=format_utc_iso_date(revision["date"]), directory=revision["directory"], merge=revision["merge"], metadata=json.dumps(revision["metadata"], sort_keys=True, indent=4, separators=(",", ": ")), parents=revision["parents"], synthetic=revision["synthetic"], type=revision["type"], snapshot=snapshot_id, origin_url=origin_url, ) message_lines = ["None"] if revision["message"]: message_lines = revision["message"].split("\n") parents = [] for p in revision["parents"]: parent_url = gen_revision_url(p, snapshot_context) parents.append({"id": p, "url": parent_url}) path_info = gen_path_info(path) query_params = snapshot_context["query_params"] if snapshot_context else {} breadcrumbs = [] breadcrumbs.append({ "name": revision["directory"][:7], "url": reverse( "browse-revision", url_args={"sha1_git": sha1_git}, query_params=query_params, ), }) for pi in path_info: query_params["path"] = pi["path"] breadcrumbs.append({ "name": pi["name"], "url": reverse( "browse-revision", url_args={"sha1_git": sha1_git}, query_params=query_params, ), }) vault_cooking = { "directory_context": False, "directory_id": None, "revision_context": True, "revision_id": sha1_git, } swh_objects = [SWHObjectInfo(object_type=REVISION, object_id=sha1_git)] content = None content_size = None filename = None mimetype = None language = None readme_name = None readme_url = None readme_html = None readmes = {} extra_context = dict(revision_metadata) extra_context["path"] = f"/{path}" if path else None if content_data: breadcrumbs[-1]["url"] = None content_size = content_data["length"] mimetype = content_data["mimetype"] if content_data["raw_data"]: content_display_data = prepare_content_for_display( content_data["raw_data"], content_data["mimetype"], path) content = content_display_data["content_data"] language = content_display_data["language"] mimetype = content_display_data["mimetype"] if path: filename = path_info[-1]["name"] query_params["filename"] = filename filepath = "/".join(pi["name"] for pi in path_info[:-1]) extra_context["path"] = f"/{filepath}/" if filepath else "/" extra_context["filename"] = filename top_right_link = { "url": reverse( "browse-content-raw", url_args={"query_string": query_string}, query_params={"filename": filename}, ), "icon": swh_object_icons["content"], "text": "Raw File", } swh_objects.append( SWHObjectInfo(object_type=CONTENT, object_id=file_info["target"])) else: for d in dirs: if d["type"] == "rev": d["url"] = reverse("browse-revision", url_args={"sha1_git": d["target"]}) else: query_params["path"] = path + d["name"] d["url"] = reverse( "browse-revision", url_args={"sha1_git": sha1_git}, query_params=query_params, ) for f in files: query_params["path"] = path + f["name"] f["url"] = reverse( "browse-revision", url_args={"sha1_git": sha1_git}, query_params=query_params, ) if f["length"] is not None: f["length"] = filesizeformat(f["length"]) if f["name"].lower().startswith("readme"): readmes[f["name"]] = f["checksums"]["sha1"] readme_name, readme_url, readme_html = get_readme_to_display(readmes) top_right_link = { "url": get_revision_log_url(sha1_git, snapshot_context), "icon": swh_object_icons["revisions history"], "text": "History", } vault_cooking["directory_context"] = True vault_cooking["directory_id"] = dir_id swh_objects.append( SWHObjectInfo(object_type=DIRECTORY, object_id=dir_id)) query_params.pop("path", None) diff_revision_url = reverse( "diff-revision", url_args={"sha1_git": sha1_git}, query_params=query_params, ) if snapshot_id: swh_objects.append( SWHObjectInfo(object_type=SNAPSHOT, object_id=snapshot_id)) swhids_info = get_swhids_info(swh_objects, snapshot_context, extra_context) heading = "Revision - %s - %s" % ( sha1_git[:7], textwrap.shorten(message_lines[0], width=70), ) if snapshot_context: context_found = "snapshot: %s" % snapshot_context["snapshot_id"] if origin_info: context_found = "origin: %s" % origin_info["url"] heading += " - %s" % context_found return render( request, "browse/revision.html", { "heading": heading, "swh_object_id": swhids_info[0]["swhid"], "swh_object_name": "Revision", "swh_object_metadata": revision_metadata, "message_header": message_lines[0], "message_body": "\n".join(message_lines[1:]), "parents": parents, "snapshot_context": snapshot_context, "dirs": dirs, "files": files, "content": content, "content_size": content_size, "max_content_size": content_display_max_size, "filename": filename, "encoding": content_data.get("encoding"), "mimetype": mimetype, "language": language, "readme_name": readme_name, "readme_url": readme_url, "readme_html": readme_html, "breadcrumbs": breadcrumbs, "top_right_link": top_right_link, "vault_cooking": vault_cooking, "diff_revision_url": diff_revision_url, "show_actions": True, "swhids_info": swhids_info, "error_code": error_info["status_code"], "error_message": http_status_code_message.get(error_info["status_code"]), "error_description": error_info["description"], }, status=error_info["status_code"], )
def directory_view(self, root_directory_sha1, directory_entries, path=None): dirs = [e for e in directory_entries if e['type'] in ('dir', 'rev')] files = [e for e in directory_entries if e['type'] == 'file'] url_args = {'sha1_git': root_directory_sha1} if path: url_args['path'] = path url = reverse('browse-directory', url_args=url_args) root_dir_url = reverse('browse-directory', url_args={'sha1_git': root_directory_sha1}) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/directory.html') self.assertContains( resp, '<a href="' + root_dir_url + '">' + root_directory_sha1[:7] + '</a>') self.assertContains(resp, '<td class="swh-directory">', count=len(dirs)) self.assertContains(resp, '<td class="swh-content">', count=len(files)) for d in dirs: if d['type'] == 'rev': dir_url = reverse('browse-revision', url_args={'sha1_git': d['target']}) else: dir_path = d['name'] if path: dir_path = "%s/%s" % (path, d['name']) dir_url = reverse('browse-directory', url_args={ 'sha1_git': root_directory_sha1, 'path': dir_path }) self.assertContains(resp, dir_url) for f in files: file_path = "%s/%s" % (root_directory_sha1, f['name']) if path: file_path = "%s/%s/%s" % (root_directory_sha1, path, f['name']) query_string = 'sha1_git:' + f['target'] file_url = reverse('browse-content', url_args={'query_string': query_string}, query_params={'path': file_path}) self.assertContains(resp, file_url) path_info = gen_path_info(path) self.assertContains(resp, '<li class="swh-path">', count=len(path_info) + 1) self.assertContains( resp, '<a href="%s">%s</a>' % (root_dir_url, root_directory_sha1[:7])) for p in path_info: dir_url = reverse('browse-directory', url_args={ 'sha1_git': root_directory_sha1, 'path': p['path'] }) self.assertContains(resp, '<a href="%s">%s</a>' % (dir_url, p['name'])) self.assertContains(resp, 'vault-cook-directory') swh_dir_id = get_swh_persistent_id( 'directory', directory_entries[0]['dir_id']) # noqa swh_dir_id_url = reverse('browse-swh-id', url_args={'swh_id': swh_dir_id}) self.assertContains(resp, swh_dir_id) self.assertContains(resp, swh_dir_id_url)