def test_gen_path_info():
input_path = "/home/user/swh-environment/swh-web/"
expected_result = [
{
"name": "home",
"path": "home"
},
{
"name": "user",
"path": "home/user"
},
{
"name": "swh-environment",
"path": "home/user/swh-environment"
},
{
"name": "swh-web",
"path": "home/user/swh-environment/swh-web"
},
]
path_info = utils.gen_path_info(input_path)
assert path_info == expected_result
input_path = "home/user/swh-environment/swh-web"
path_info = utils.gen_path_info(input_path)
assert path_info == expected_result
def test_gen_path_info(self):
input_path = '/home/user/swh-environment/swh-web/'
expected_result = [{
'name': 'home',
'path': 'home'
}, {
'name': 'user',
'path': 'home/user'
}, {
'name': 'swh-environment',
'path': 'home/user/swh-environment'
}, {
'name': 'swh-web',
'path': 'home/user/swh-environment/swh-web'
}]
path_info = utils.gen_path_info(input_path)
self.assertEqual(path_info, expected_result)
input_path = 'home/user/swh-environment/swh-web'
path_info = utils.gen_path_info(input_path)
self.assertEqual(path_info, expected_result)
def test_content_view_text_with_path(client, archive_data, content):
path = content["path"]
url = reverse(
"browse-content",
url_args={"query_string": content["sha1"]},
query_params={"path": path},
)
resp = check_html_get_response(client,
url,
status_code=200,
template_used="browse/content.html")
assert_contains(resp, '<nav class="bread-crumbs')
content_display = _process_content_for_display(archive_data, content)
mimetype = content_display["mimetype"]
if mimetype.startswith("text/"):
hljs_language = content["hljs_language"]
assert_contains(resp, '<code class="%s">' % hljs_language)
assert_contains(resp, escape(content_display["content_data"]))
split_path = path.split("/")
root_dir_sha1 = split_path[0]
filename = split_path[-1]
path = path.replace(root_dir_sha1 + "/", "").replace(filename, "")
swhid_context = {
"anchor": gen_swhid(DIRECTORY, root_dir_sha1),
"path": f"/{path}{filename}",
}
swh_cnt_id = gen_swhid(CONTENT,
content["sha1_git"],
metadata=swhid_context)
swh_cnt_id_url = reverse("browse-swhid", url_args={"swhid": swh_cnt_id})
assert_contains(resp, swh_cnt_id)
assert_contains(resp, swh_cnt_id_url)
path_info = gen_path_info(path)
root_dir_url = reverse("browse-directory",
url_args={"sha1_git": root_dir_sha1})
assert_contains(resp, '<li class="swh-path">', count=len(path_info) + 1)
assert_contains(
resp, '<a href="' + root_dir_url + '">' + root_dir_sha1[:7] + "</a>")
for p in path_info:
dir_url = reverse(
"browse-directory",
url_args={"sha1_git": root_dir_sha1},
query_params={"path": p["path"]},
)
assert_contains(resp,
'<a href="' + dir_url + '">' + p["name"] + "</a>")
assert_contains(resp, "<li>" + filename + "</li>")
url_raw = reverse(
"browse-content-raw",
url_args={"query_string": content["sha1"]},
query_params={"filename": filename},
)
assert_contains(resp, url_raw)
url = reverse(
"browse-content",
url_args={"query_string": content["sha1"]},
query_params={"path": filename},
)
resp = check_html_get_response(client,
url,
status_code=200,
template_used="browse/content.html")
assert_not_contains(resp, '<nav class="bread-crumbs')
invalid_path = "%s/foo/bar/baz" % root_dir_sha1
url = reverse(
"browse-content",
url_args={"query_string": content["sha1"]},
query_params={"path": invalid_path},
)
resp = check_html_get_response(client,
url,
status_code=404,
template_used="error.html")
def _origin_content_view_test_helper(
client,
archive_data,
origin_info,
origin_visit,
snapshot_sizes,
origin_branches,
origin_releases,
root_dir_sha1,
content,
visit_id=None,
timestamp=None,
snapshot_id=None,
):
content_path = "/".join(content["path"].split("/")[1:])
if not visit_id and not snapshot_id:
visit_id = origin_visit["visit"]
query_params = {"origin_url": origin_info["url"], "path": content_path}
if timestamp:
query_params["timestamp"] = timestamp
if visit_id:
query_params["visit_id"] = visit_id
elif snapshot_id:
query_params["snapshot"] = snapshot_id
url = reverse("browse-origin-content", query_params=query_params)
resp = check_html_get_response(
client, url, status_code=200, template_used="browse/content.html"
)
assert type(content["data"]) == str
assert_contains(resp, '<code class="%s">' % content["hljs_language"])
assert_contains(resp, escape(content["data"]))
split_path = content_path.split("/")
filename = split_path[-1]
path = content_path.replace(filename, "")[:-1]
path_info = gen_path_info(path)
del query_params["path"]
if timestamp:
query_params["timestamp"] = format_utc_iso_date(
parse_iso8601_date_to_utc(timestamp).isoformat(), "%Y-%m-%dT%H:%M:%SZ"
)
root_dir_url = reverse("browse-origin-directory", query_params=query_params)
assert_contains(resp, '<li class="swh-path">', count=len(path_info) + 1)
assert_contains(resp, '<a href="%s">%s</a>' % (root_dir_url, root_dir_sha1[:7]))
for p in path_info:
query_params["path"] = p["path"]
dir_url = reverse("browse-origin-directory", query_params=query_params)
assert_contains(resp, '<a href="%s">%s</a>' % (dir_url, p["name"]))
assert_contains(resp, "<li>%s</li>" % filename)
query_string = "sha1_git:" + content["sha1_git"]
url_raw = reverse(
"browse-content-raw",
url_args={"query_string": query_string},
query_params={"filename": filename},
)
assert_contains(resp, url_raw)
if "path" in query_params:
del query_params["path"]
origin_branches_url = reverse("browse-origin-branches", query_params=query_params)
assert_contains(resp, f'href="{escape(origin_branches_url)}"')
assert_contains(resp, f"Branches ({snapshot_sizes['revision']})")
origin_releases_url = reverse("browse-origin-releases", query_params=query_params)
assert_contains(resp, f'href="{escape(origin_releases_url)}">')
assert_contains(resp, f"Releases ({snapshot_sizes['release']})")
assert_contains(resp, '<li class="swh-branch">', count=len(origin_branches))
query_params["path"] = content_path
for branch in origin_branches:
root_dir_branch_url = reverse(
"browse-origin-content",
query_params={"branch": branch["name"], **query_params},
)
assert_contains(resp, '<a href="%s">' % root_dir_branch_url)
assert_contains(resp, '<li class="swh-release">', count=len(origin_releases))
query_params["branch"] = None
for release in origin_releases:
root_dir_release_url = reverse(
"browse-origin-content",
query_params={"release": release["name"], **query_params},
)
assert_contains(resp, '<a href="%s">' % root_dir_release_url)
url = reverse("browse-origin-content", query_params=query_params)
resp = check_html_get_response(
client, url, status_code=200, template_used="browse/content.html"
)
snapshot = archive_data.snapshot_get(origin_visit["snapshot"])
head_rev_id = archive_data.snapshot_get_head(snapshot)
swhid_context = {
"origin": origin_info["url"],
"visit": gen_swhid(SNAPSHOT, snapshot["id"]),
"anchor": gen_swhid(REVISION, head_rev_id),
"path": f"/{content_path}",
}
swh_cnt_id = gen_swhid(CONTENT, content["sha1_git"], metadata=swhid_context)
swh_cnt_id_url = reverse("browse-swhid", url_args={"swhid": swh_cnt_id})
assert_contains(resp, swh_cnt_id)
assert_contains(resp, swh_cnt_id_url)
assert_contains(resp, "swh-take-new-snapshot")
_check_origin_link(resp, origin_info["url"])
assert_not_contains(resp, "swh-metadata-popover")
def test_content_view_with_path(self, content):
path = content['path']
url = reverse('browse-content',
url_args={'query_string': content['sha1']},
query_params={'path': path})
resp = self.client.get(url)
self.assertEqual(resp.status_code, 200)
self.assertTemplateUsed('browse/content.html')
self.assertContains(resp, '<nav class="bread-crumbs')
content_display = self._process_content_for_display(content)
mimetype = content_display['mimetype']
if mimetype.startswith('text/'):
hljs_language = content['hljs_language']
self.assertContains(resp, '<code class="%s">' % hljs_language)
self.assertContains(resp, escape(content_display['content_data']))
split_path = path.split('/')
root_dir_sha1 = split_path[0]
filename = split_path[-1]
path = path.replace(root_dir_sha1 + '/', '').replace(filename, '')
path_info = gen_path_info(path)
root_dir_url = reverse('browse-directory',
url_args={'sha1_git': root_dir_sha1})
self.assertContains(resp,
'<li class="swh-path">',
count=len(path_info) + 1)
self.assertContains(
resp,
'<a href="' + root_dir_url + '">' + root_dir_sha1[:7] + '</a>')
for p in path_info:
dir_url = reverse('browse-directory',
url_args={
'sha1_git': root_dir_sha1,
'path': p['path']
})
self.assertContains(
resp, '<a href="' + dir_url + '">' + p['name'] + '</a>')
self.assertContains(resp, '<li>' + filename + '</li>')
url_raw = reverse('browse-content-raw',
url_args={'query_string': content['sha1']},
query_params={'filename': filename})
self.assertContains(resp, url_raw)
url = reverse('browse-content',
url_args={'query_string': content['sha1']},
query_params={'path': filename})
resp = self.client.get(url)
self.assertEqual(resp.status_code, 200)
self.assertTemplateUsed('browse/content.html')
self.assertNotContains(resp, '<nav class="bread-crumbs')
def browse_snapshot_content(request,
snapshot_id=None,
origin_type=None,
origin_url=None,
timestamp=None,
path=None):
"""
Django view implementation for browsing a content in a snapshot context.
"""
try:
snapshot_context = _process_snapshot_request(request,
snapshot_id,
origin_type,
origin_url,
timestamp,
path,
browse_context='content')
root_sha1_git = snapshot_context['root_sha1_git']
sha1_git = None
query_string = None
content_data = None
if root_sha1_git:
content_info = service.lookup_directory_with_path(
root_sha1_git, path)
sha1_git = content_info['target']
query_string = 'sha1_git:' + sha1_git
content_data = request_content(query_string,
raise_if_unavailable=False)
except Exception as exc:
return handle_view_exception(request, exc)
swh_type = snapshot_context['swh_type']
url_args = snapshot_context['url_args']
query_params = snapshot_context['query_params']
revision_id = snapshot_context['revision_id']
origin_info = snapshot_context['origin_info']
visit_info = snapshot_context['visit_info']
snapshot_id = snapshot_context['snapshot_id']
content = None
language = None
mimetype = None
if content_data and content_data['raw_data'] is not None:
content_display_data = prepare_content_for_display(
content_data['raw_data'], content_data['mimetype'], path)
content = content_display_data['content_data']
language = content_display_data['language']
mimetype = content_display_data['mimetype']
filename = None
path_info = None
browse_view_name = 'browse-' + swh_type + '-directory'
breadcrumbs = []
split_path = path.split('/')
filename = split_path[-1]
path_info = gen_path_info(path[:-len(filename)])
if root_sha1_git:
breadcrumbs.append({
'name':
root_sha1_git[:7],
'url':
reverse(browse_view_name,
url_args=url_args,
query_params=query_params)
})
for pi in path_info:
bc_url_args = dict(url_args)
bc_url_args['path'] = pi['path']
breadcrumbs.append({
'name':
pi['name'],
'url':
reverse(browse_view_name,
url_args=bc_url_args,
query_params=query_params)
})
breadcrumbs.append({'name': filename, 'url': None})
browse_content_link = \
gen_content_link(sha1_git, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
content_raw_url = None
if query_string:
content_raw_url = reverse('browse-content-raw',
url_args={'query_string': query_string},
query_params={'filename': filename})
browse_rev_link = \
gen_revision_link(revision_id,
snapshot_context=snapshot_context,
link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
content_metadata = {
'context-independent content': browse_content_link,
'path': None,
'filename': None,
'revision id': revision_id,
'revision': browse_rev_link,
'snapshot id': snapshot_id
}
cnt_sha1_git = None
content_size = None
error_code = 200
error_description = ''
error_message = ''
if content_data:
content_metadata['sha1 checksum'] = \
content_data['checksums']['sha1']
content_metadata['sha1_git checksum'] = \
content_data['checksums']['sha1_git']
content_metadata['sha256 checksum'] = \
content_data['checksums']['sha256']
content_metadata['blake2s256 checksum'] = \
content_data['checksums']['blake2s256']
content_metadata['mime type'] = content_data['mimetype']
content_metadata['encoding'] = content_data['encoding']
content_metadata['size'] = filesizeformat(content_data['length'])
content_metadata['language'] = content_data['language']
content_metadata['licenses'] = content_data['licenses']
content_metadata['path'] = '/' + path[:-len(filename)]
content_metadata['filename'] = filename
cnt_sha1_git = content_data['checksums']['sha1_git']
content_size = content_data['length']
error_code = content_data['error_code']
error_message = content_data['error_message']
error_description = content_data['error_description']
if origin_info:
content_metadata['origin id'] = origin_info['id']
content_metadata['origin type'] = origin_info['type']
content_metadata['origin url'] = origin_info['url']
content_metadata['origin visit date'] = format_utc_iso_date(
visit_info['date']) # noqa
content_metadata['origin visit id'] = visit_info['visit']
browse_snapshot_url = reverse('browse-snapshot-content',
url_args={
'snapshot_id': snapshot_id,
'path': path
},
query_params=request.GET)
browse_snapshot_link = \
gen_link(browse_snapshot_url, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
content_metadata['snapshot context'] = browse_snapshot_link
swh_objects = [{
'type': 'content',
'id': cnt_sha1_git
}, {
'type': 'revision',
'id': revision_id
}, {
'type': 'snapshot',
'id': snapshot_id
}]
release_id = snapshot_context['release_id']
if release_id:
swh_objects.append({'type': 'release', 'id': release_id})
swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context)
content_path = '/'.join([bc['name'] for bc in breadcrumbs])
context_found = 'snapshot: %s' % snapshot_context['snapshot_id']
if origin_info:
context_found = 'origin: %s' % origin_info['url']
heading = 'Content - %s - %s - %s' %\
(content_path, snapshot_context['branch'], context_found)
return render(request,
'browse/content.html', {
'heading': heading,
'swh_object_name': 'Content',
'swh_object_metadata': content_metadata,
'content': content,
'content_size': content_size,
'max_content_size': content_display_max_size,
'mimetype': mimetype,
'language': language,
'breadcrumbs': breadcrumbs if root_sha1_git else [],
'top_right_link': {
'url': content_raw_url,
'icon': swh_object_icons['content'],
'text': 'Raw File'
},
'snapshot_context': snapshot_context,
'vault_cooking': None,
'show_actions_menu': True,
'swh_ids': swh_ids,
'error_code': error_code,
'error_message': error_message,
'error_description': error_description
},
status=error_code)
def browse_snapshot_directory(request,
snapshot_id=None,
origin_type=None,
origin_url=None,
timestamp=None,
path=None):
"""
Django view implementation for browsing a directory in a snapshot context.
"""
try:
snapshot_context = _process_snapshot_request(
request,
snapshot_id,
origin_type,
origin_url,
timestamp,
path,
browse_context='directory') # noqa
root_sha1_git = snapshot_context['root_sha1_git']
sha1_git = root_sha1_git
if root_sha1_git and path:
dir_info = service.lookup_directory_with_path(root_sha1_git, path)
# some readme files can reference assets reachable from the
# browsed directory, handle that special case in order to
# correctly displayed them
if dir_info and dir_info['type'] == 'file':
file_raw_url = reverse(
'browse-content-raw',
url_args={'query_string': dir_info['checksums']['sha1']})
return redirect(file_raw_url)
sha1_git = dir_info['target']
dirs = []
files = []
if sha1_git:
dirs, files = get_directory_entries(sha1_git)
except Exception as exc:
return handle_view_exception(request, exc)
swh_type = snapshot_context['swh_type']
origin_info = snapshot_context['origin_info']
visit_info = snapshot_context['visit_info']
url_args = snapshot_context['url_args']
query_params = snapshot_context['query_params']
revision_id = snapshot_context['revision_id']
snapshot_id = snapshot_context['snapshot_id']
path_info = gen_path_info(path)
browse_view_name = 'browse-' + swh_type + '-directory'
breadcrumbs = []
if root_sha1_git:
breadcrumbs.append({
'name':
root_sha1_git[:7],
'url':
reverse(browse_view_name,
url_args=url_args,
query_params=query_params)
})
for pi in path_info:
bc_url_args = dict(url_args)
bc_url_args['path'] = pi['path']
breadcrumbs.append({
'name':
pi['name'],
'url':
reverse(browse_view_name,
url_args=bc_url_args,
query_params=query_params)
})
path = '' if path is None else (path + '/')
for d in dirs:
if d['type'] == 'rev':
d['url'] = reverse('browse-revision',
url_args={'sha1_git': d['target']})
else:
bc_url_args = dict(url_args)
bc_url_args['path'] = path + d['name']
d['url'] = reverse(browse_view_name,
url_args=bc_url_args,
query_params=query_params)
sum_file_sizes = 0
readmes = {}
browse_view_name = 'browse-' + swh_type + '-content'
for f in files:
bc_url_args = dict(url_args)
bc_url_args['path'] = path + f['name']
f['url'] = reverse(browse_view_name,
url_args=bc_url_args,
query_params=query_params)
if f['length'] is not None:
sum_file_sizes += f['length']
f['length'] = filesizeformat(f['length'])
if f['name'].lower().startswith('readme'):
readmes[f['name']] = f['checksums']['sha1']
readme_name, readme_url, readme_html = get_readme_to_display(readmes)
browse_view_name = 'browse-' + swh_type + '-log'
history_url = None
if snapshot_id != _empty_snapshot_id:
history_url = reverse(browse_view_name,
url_args=url_args,
query_params=query_params)
nb_files = None
nb_dirs = None
sum_file_sizes = None
dir_path = None
if root_sha1_git:
nb_files = len(files)
nb_dirs = len(dirs)
sum_file_sizes = filesizeformat(sum_file_sizes)
dir_path = '/' + path
browse_dir_link = \
gen_directory_link(sha1_git, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
browse_rev_link = \
gen_revision_link(revision_id,
snapshot_context=snapshot_context,
link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
dir_metadata = {
'id': sha1_git,
'context-independent directory': browse_dir_link,
'number of regular files': nb_files,
'number of subdirectories': nb_dirs,
'sum of regular file sizes': sum_file_sizes,
'path': dir_path,
'revision id': revision_id,
'revision': browse_rev_link,
'snapshot id': snapshot_id
}
if origin_info:
dir_metadata['origin id'] = origin_info['id']
dir_metadata['origin type'] = origin_info['type']
dir_metadata['origin url'] = origin_info['url']
dir_metadata['origin visit date'] = format_utc_iso_date(
visit_info['date']) # noqa
dir_metadata['origin visit id'] = visit_info['visit']
snapshot_context_url = reverse('browse-snapshot-directory',
url_args={'snapshot_id': snapshot_id},
query_params=request.GET)
browse_snapshot_link = \
gen_link(snapshot_context_url, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
dir_metadata['snapshot context'] = browse_snapshot_link
vault_cooking = {
'directory_context': True,
'directory_id': sha1_git,
'revision_context': True,
'revision_id': revision_id
}
swh_objects = [{
'type': 'directory',
'id': sha1_git
}, {
'type': 'revision',
'id': revision_id
}, {
'type': 'snapshot',
'id': snapshot_id
}]
release_id = snapshot_context['release_id']
if release_id:
swh_objects.append({'type': 'release', 'id': release_id})
swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context)
dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/'
context_found = 'snapshot: %s' % snapshot_context['snapshot_id']
if origin_info:
context_found = 'origin: %s' % origin_info['url']
heading = 'Directory - %s - %s - %s' %\
(dir_path, snapshot_context['branch'], context_found)
return render(
request, 'browse/directory.html', {
'heading': heading,
'swh_object_name': 'Directory',
'swh_object_metadata': dir_metadata,
'dirs': dirs,
'files': files,
'breadcrumbs': breadcrumbs if root_sha1_git else [],
'top_right_link': {
'url': history_url,
'icon': swh_object_icons['revisions history'],
'text': 'History'
},
'readme_name': readme_name,
'readme_url': readme_url,
'readme_html': readme_html,
'snapshot_context': snapshot_context,
'vault_cooking': vault_cooking,
'show_actions_menu': True,
'swh_ids': swh_ids
})
def content_display(request, query_string):
"""Django view that produces an HTML display of a content identified
by its hash value.
The url that points to it is
:http:get:`/browse/content/[(algo_hash):](hash)/`
"""
algo, checksum = query.parse_hash(query_string)
checksum = hash_to_hex(checksum)
origin_url = request.GET.get("origin_url")
selected_language = request.GET.get("language")
if not origin_url:
origin_url = request.GET.get("origin")
snapshot_id = request.GET.get("snapshot")
path = request.GET.get("path")
content_data = {}
error_info = {"status_code": 200, "description": None}
try:
content_data = request_content(query_string)
except NotFoundExc as e:
error_info["status_code"] = 404
error_info["description"] = f"NotFoundExc: {str(e)}"
snapshot_context = None
if origin_url is not None or snapshot_id is not None:
try:
snapshot_context = get_snapshot_context(
origin_url=origin_url,
snapshot_id=snapshot_id,
branch_name=request.GET.get("branch"),
release_name=request.GET.get("release"),
revision_id=request.GET.get("revision"),
path=path,
browse_context=CONTENT,
)
except NotFoundExc as e:
if str(e).startswith("Origin"):
raw_cnt_url = reverse("browse-content",
url_args={"query_string": query_string})
error_message = (
"The Software Heritage archive has a content "
"with the hash you provided but the origin "
"mentioned in your request appears broken: %s. "
"Please check the URL and try again.\n\n"
"Nevertheless, you can still browse the content "
"without origin information: %s" %
(gen_link(origin_url), gen_link(raw_cnt_url)))
raise NotFoundExc(error_message)
else:
raise e
content = None
language = None
mimetype = None
if content_data.get("raw_data") is not None:
content_display_data = prepare_content_for_display(
content_data["raw_data"], content_data["mimetype"], path)
content = content_display_data["content_data"]
language = content_display_data["language"]
mimetype = content_display_data["mimetype"]
# Override language with user-selected language
if selected_language is not None:
language = selected_language
available_languages = None
if mimetype and "text/" in mimetype:
available_languages = highlightjs.get_supported_languages()
filename = None
path_info = None
directory_id = None
root_dir = None
if snapshot_context:
root_dir = snapshot_context.get("root_directory")
query_params = snapshot_context["query_params"] if snapshot_context else {}
breadcrumbs = []
if path:
split_path = path.split("/")
root_dir = root_dir or split_path[0]
filename = split_path[-1]
if root_dir != path:
path = path.replace(root_dir + "/", "")
path = path[:-len(filename)]
path_info = gen_path_info(path)
query_params.pop("path", None)
dir_url = reverse(
"browse-directory",
url_args={"sha1_git": root_dir},
query_params=query_params,
)
breadcrumbs.append({"name": root_dir[:7], "url": dir_url})
for pi in path_info:
query_params["path"] = pi["path"]
dir_url = reverse(
"browse-directory",
url_args={"sha1_git": root_dir},
query_params=query_params,
)
breadcrumbs.append({"name": pi["name"], "url": dir_url})
breadcrumbs.append({"name": filename, "url": None})
if path and root_dir != path:
dir_info = archive.lookup_directory_with_path(root_dir, path)
directory_id = dir_info["target"]
elif root_dir != path:
directory_id = root_dir
else:
root_dir = None
query_params = {"filename": filename}
content_checksums = content_data.get("checksums", {})
content_url = reverse(
"browse-content",
url_args={"query_string": query_string},
)
content_raw_url = reverse(
"browse-content-raw",
url_args={"query_string": query_string},
query_params=query_params,
)
content_metadata = ContentMetadata(
object_type=CONTENT,
object_id=content_checksums.get("sha1_git"),
sha1=content_checksums.get("sha1"),
sha1_git=content_checksums.get("sha1_git"),
sha256=content_checksums.get("sha256"),
blake2s256=content_checksums.get("blake2s256"),
content_url=content_url,
mimetype=content_data.get("mimetype"),
encoding=content_data.get("encoding"),
size=filesizeformat(content_data.get("length", 0)),
language=content_data.get("language"),
root_directory=root_dir,
path=f"/{path}" if path else None,
filename=filename or "",
directory=directory_id,
revision=None,
release=None,
snapshot=None,
origin_url=origin_url,
)
swh_objects = [
SWHObjectInfo(object_type=CONTENT,
object_id=content_checksums.get("sha1_git"))
]
if directory_id:
swh_objects.append(
SWHObjectInfo(object_type=DIRECTORY, object_id=directory_id))
if snapshot_context:
swh_objects.append(
SWHObjectInfo(object_type=REVISION,
object_id=snapshot_context["revision_id"]))
swh_objects.append(
SWHObjectInfo(object_type=SNAPSHOT,
object_id=snapshot_context["snapshot_id"]))
if snapshot_context["release_id"]:
swh_objects.append(
SWHObjectInfo(object_type=RELEASE,
object_id=snapshot_context["release_id"]))
swhids_info = get_swhids_info(
swh_objects,
snapshot_context,
extra_context=content_metadata,
)
heading = "Content - %s" % content_checksums.get("sha1_git")
if breadcrumbs:
content_path = "/".join([bc["name"] for bc in breadcrumbs])
heading += " - %s" % content_path
return render(
request,
"browse/content.html",
{
"heading":
heading,
"swh_object_id":
swhids_info[0]["swhid"],
"swh_object_name":
"Content",
"swh_object_metadata":
content_metadata,
"content":
content,
"content_size":
content_data.get("length"),
"max_content_size":
content_display_max_size,
"filename":
filename,
"encoding":
content_data.get("encoding"),
"mimetype":
mimetype,
"language":
language,
"available_languages":
available_languages,
"breadcrumbs":
breadcrumbs,
"top_right_link": {
"url": content_raw_url,
"icon": swh_object_icons["content"],
"text": "Raw File",
},
"snapshot_context":
snapshot_context,
"vault_cooking":
None,
"show_actions":
True,
"swhids_info":
swhids_info,
"error_code":
error_info["status_code"],
"error_message":
http_status_code_message.get(error_info["status_code"]),
"error_description":
error_info["description"],
},
status=error_info["status_code"],
)
def revision_browse(request, sha1_git, extra_path=None):
"""
Django view that produces an HTML display of a revision
identified by its id.
The url that points to it is :http:get:`/browse/revision/(sha1_git)/`.
"""
try:
revision = service.lookup_revision(sha1_git)
# some readme files can reference assets reachable from the
# browsed directory, handle that special case in order to
# correctly displayed them
if extra_path:
dir_info = \
service.lookup_directory_with_path(revision['directory'],
extra_path)
if dir_info and dir_info['type'] == 'file':
file_raw_url = reverse(
'browse-content-raw',
url_args={'query_string': dir_info['checksums']['sha1']})
return redirect(file_raw_url)
origin_info = None
snapshot_context = None
origin_type = request.GET.get('origin_type', None)
origin_url = request.GET.get('origin_url', None)
if not origin_url:
origin_url = request.GET.get('origin', None)
timestamp = request.GET.get('timestamp', None)
visit_id = request.GET.get('visit_id', None)
snapshot_id = request.GET.get('snapshot_id', None)
path = request.GET.get('path', None)
dir_id = None
dirs, files = None, None
content_data = None
if origin_url:
try:
snapshot_context = get_snapshot_context(
None, origin_type, origin_url, timestamp, visit_id)
except Exception:
raw_rev_url = reverse('browse-revision',
url_args={'sha1_git': sha1_git})
error_message = \
('The Software Heritage archive has a revision '
'with the hash you provided but the origin '
'mentioned in your request appears broken: %s. '
'Please check the URL and try again.\n\n'
'Nevertheless, you can still browse the revision '
'without origin information: %s'
% (gen_link(origin_url), gen_link(raw_rev_url)))
raise NotFoundExc(error_message)
origin_info = snapshot_context['origin_info']
snapshot_id = snapshot_context['snapshot_id']
elif snapshot_id:
snapshot_context = get_snapshot_context(snapshot_id)
if path:
file_info = \
service.lookup_directory_with_path(revision['directory'], path)
if file_info['type'] == 'dir':
dir_id = file_info['target']
else:
query_string = 'sha1_git:' + file_info['target']
content_data = request_content(query_string,
raise_if_unavailable=False)
else:
dir_id = revision['directory']
if dir_id:
path = '' if path is None else (path + '/')
dirs, files = get_directory_entries(dir_id)
except Exception as exc:
return handle_view_exception(request, exc)
revision_data = {}
author_name = 'None'
revision_data['author'] = 'None'
if revision['author']:
author_name = revision['author']['name'] or \
revision['author']['fullname']
revision_data['author'] = \
gen_person_link(revision['author']['id'], author_name,
snapshot_context)
revision_data['committer'] = 'None'
if revision['committer']:
revision_data['committer'] = \
gen_person_link(revision['committer']['id'],
revision['committer']['name'], snapshot_context)
revision_data['committer date'] = \
format_utc_iso_date(revision['committer_date'])
revision_data['date'] = format_utc_iso_date(revision['date'])
if snapshot_context:
revision_data['snapshot id'] = snapshot_id
revision_data['directory'] = \
gen_snapshot_directory_link(snapshot_context, sha1_git,
link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm', # noqa
'role': 'button'})
else:
revision_data['directory'] = \
gen_directory_link(revision['directory'], link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
revision_data['id'] = sha1_git
revision_data['merge'] = revision['merge']
revision_data['metadata'] = escape(
json.dumps(revision['metadata'],
sort_keys=True,
indent=4,
separators=(',', ': ')))
if origin_info:
revision_data['context-independent revision'] = \
gen_revision_link(sha1_git, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
revision_data['origin id'] = origin_info['id']
revision_data['origin type'] = origin_info['type']
revision_data['origin url'] = gen_link(origin_info['url'],
origin_info['url'])
browse_snapshot_link = \
gen_snapshot_link(snapshot_id, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
revision_data['snapshot'] = browse_snapshot_link
parents = ''
for p in revision['parents']:
parent_link = gen_revision_link(p, snapshot_context=snapshot_context)
parents += parent_link + '<br/>'
revision_data['parents'] = mark_safe(parents)
revision_data['synthetic'] = revision['synthetic']
revision_data['type'] = revision['type']
message_lines = ['None']
if revision['message']:
message_lines = revision['message'].split('\n')
parents = []
for p in revision['parents']:
parent_url = gen_revision_url(p, snapshot_context)
parents.append({'id': p, 'url': parent_url})
path_info = gen_path_info(path)
query_params = {
'snapshot_id': snapshot_id,
'origin_type': origin_type,
'origin': origin_url,
'timestamp': timestamp,
'visit_id': visit_id
}
breadcrumbs = []
breadcrumbs.append({
'name':
revision['directory'][:7],
'url':
reverse('browse-revision',
url_args={'sha1_git': sha1_git},
query_params=query_params)
})
for pi in path_info:
query_params['path'] = pi['path']
breadcrumbs.append({
'name':
pi['name'],
'url':
reverse('browse-revision',
url_args={'sha1_git': sha1_git},
query_params=query_params)
})
vault_cooking = {
'directory_context': False,
'directory_id': None,
'revision_context': True,
'revision_id': sha1_git
}
swh_objects = [{'type': 'revision', 'id': sha1_git}]
content = None
content_size = None
mimetype = None
language = None
readme_name = None
readme_url = None
readme_html = None
readmes = {}
error_code = 200
error_message = ''
error_description = ''
if content_data:
breadcrumbs[-1]['url'] = None
content_size = content_data['length']
mimetype = content_data['mimetype']
if content_data['raw_data']:
content_display_data = prepare_content_for_display(
content_data['raw_data'], content_data['mimetype'], path)
content = content_display_data['content_data']
language = content_display_data['language']
mimetype = content_display_data['mimetype']
query_params = {}
if path:
filename = path_info[-1]['name']
query_params['filename'] = path_info[-1]['name']
revision_data['filename'] = filename
top_right_link = {
'url':
reverse('browse-content-raw',
url_args={'query_string': query_string},
query_params=query_params),
'icon':
swh_object_icons['content'],
'text':
'Raw File'
}
swh_objects.append({'type': 'content', 'id': file_info['target']})
error_code = content_data['error_code']
error_message = content_data['error_message']
error_description = content_data['error_description']
else:
for d in dirs:
if d['type'] == 'rev':
d['url'] = reverse('browse-revision',
url_args={'sha1_git': d['target']})
else:
query_params['path'] = path + d['name']
d['url'] = reverse('browse-revision',
url_args={'sha1_git': sha1_git},
query_params=query_params)
for f in files:
query_params['path'] = path + f['name']
f['url'] = reverse('browse-revision',
url_args={'sha1_git': sha1_git},
query_params=query_params)
if f['length'] is not None:
f['length'] = filesizeformat(f['length'])
if f['name'].lower().startswith('readme'):
readmes[f['name']] = f['checksums']['sha1']
readme_name, readme_url, readme_html = get_readme_to_display(readmes)
top_right_link = {
'url': get_revision_log_url(sha1_git, snapshot_context),
'icon': swh_object_icons['revisions history'],
'text': 'History'
}
vault_cooking['directory_context'] = True
vault_cooking['directory_id'] = dir_id
swh_objects.append({'type': 'directory', 'id': dir_id})
diff_revision_url = reverse('diff-revision',
url_args={'sha1_git': sha1_git},
query_params={
'origin_type': origin_type,
'origin': origin_url,
'timestamp': timestamp,
'visit_id': visit_id
})
if snapshot_id:
swh_objects.append({'type': 'snapshot', 'id': snapshot_id})
swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context)
heading = 'Revision - %s - %s' %\
(sha1_git[:7], textwrap.shorten(message_lines[0], width=70))
if snapshot_context:
context_found = 'snapshot: %s' % snapshot_context['snapshot_id']
if origin_info:
context_found = 'origin: %s' % origin_info['url']
heading += ' - %s' % context_found
return render(request,
'browse/revision.html', {
'heading': heading,
'swh_object_id': swh_ids[0]['swh_id'],
'swh_object_name': 'Revision',
'swh_object_metadata': revision_data,
'message_header': message_lines[0],
'message_body': '\n'.join(message_lines[1:]),
'parents': parents,
'snapshot_context': snapshot_context,
'dirs': dirs,
'files': files,
'content': content,
'content_size': content_size,
'max_content_size': content_display_max_size,
'mimetype': mimetype,
'language': language,
'readme_name': readme_name,
'readme_url': readme_url,
'readme_html': readme_html,
'breadcrumbs': breadcrumbs,
'top_right_link': top_right_link,
'vault_cooking': vault_cooking,
'diff_revision_url': diff_revision_url,
'show_actions_menu': True,
'swh_ids': swh_ids,
'error_code': error_code,
'error_message': error_message,
'error_description': error_description
},
status=error_code)
def _directory_view_checks(
client,
root_directory_sha1,
directory_entries,
path=None,
origin_url=None,
snapshot_id=None,
revision_id=None,
):
dirs = [e for e in directory_entries if e["type"] in ("dir", "rev")]
files = [e for e in directory_entries if e["type"] == "file"]
url_args = {"sha1_git": root_directory_sha1}
query_params = {"origin_url": origin_url, "snapshot": snapshot_id}
url = reverse(
"browse-directory",
url_args=url_args,
query_params={
**query_params, "path": path
},
)
root_dir_url = reverse(
"browse-directory",
url_args=url_args,
query_params=query_params,
)
resp = check_html_get_response(client,
url,
status_code=200,
template_used="browse/directory.html")
assert_contains(
resp,
'<a href="' + root_dir_url + '">' + root_directory_sha1[:7] + "</a>",
)
assert_contains(resp, '<td class="swh-directory">', count=len(dirs))
assert_contains(resp, '<td class="swh-content">', count=len(files))
for d in dirs:
if d["type"] == "rev":
dir_url = reverse("browse-revision",
url_args={"sha1_git": d["target"]})
else:
dir_path = d["name"]
if path:
dir_path = "%s/%s" % (path, d["name"])
dir_url = reverse(
"browse-directory",
url_args={"sha1_git": root_directory_sha1},
query_params={
**query_params, "path": dir_path
},
)
assert_contains(resp, dir_url)
for f in files:
file_path = "%s/%s" % (root_directory_sha1, f["name"])
if path:
file_path = "%s/%s/%s" % (root_directory_sha1, path, f["name"])
query_string = "sha1_git:" + f["target"]
file_url = reverse(
"browse-content",
url_args={"query_string": query_string},
query_params={
**query_params, "path": file_path
},
)
assert_contains(resp, file_url)
path_info = gen_path_info(path)
assert_contains(resp, '<li class="swh-path">', count=len(path_info) + 1)
assert_contains(
resp, '<a href="%s">%s</a>' % (root_dir_url, root_directory_sha1[:7]))
for p in path_info:
dir_url = reverse(
"browse-directory",
url_args={"sha1_git": root_directory_sha1},
query_params={
**query_params, "path": p["path"]
},
)
assert_contains(resp, '<a href="%s">%s</a>' % (dir_url, p["name"]))
assert_contains(resp, "vault-cook-directory")
swh_dir_id = gen_swhid(DIRECTORY, directory_entries[0]["dir_id"])
swh_dir_id_url = reverse("browse-swhid", url_args={"swhid": swh_dir_id})
swhid_context = {}
if origin_url:
swhid_context["origin"] = origin_url
if snapshot_id:
swhid_context["visit"] = gen_swhid(SNAPSHOT, snapshot_id)
if root_directory_sha1 != directory_entries[0]["dir_id"]:
swhid_context["anchor"] = gen_swhid(DIRECTORY, root_directory_sha1)
if root_directory_sha1 != directory_entries[0]["dir_id"]:
swhid_context["anchor"] = gen_swhid(DIRECTORY, root_directory_sha1)
if revision_id:
swhid_context["anchor"] = gen_swhid(REVISION, revision_id)
swhid_context["path"] = f"/{path}/" if path else None
swh_dir_id = gen_swhid(DIRECTORY,
directory_entries[0]["dir_id"],
metadata=swhid_context)
swh_dir_id_url = reverse("browse-swhid", url_args={"swhid": swh_dir_id})
assert_contains(resp, swh_dir_id)
assert_contains(resp, swh_dir_id_url)
assert_not_contains(resp, "swh-metadata-popover")
def make_api_response(request, data, doc_data={}, options={}):
"""Generates an API response based on the requested mimetype.
Args:
request: a DRF Request object
data: raw data to return in the API response
doc_data: documentation data for HTML response
options: optional data that can be used to generate the response
Returns:
a DRF Response a object
"""
if data:
options['headers'] = compute_link_header(data, options)
data = transform(data)
data = filter_by_fields(request, data)
doc_env = doc_data
headers = {}
if 'headers' in options:
doc_env['headers_data'] = options['headers']
headers = options['headers']
# get request status code
doc_env['status_code'] = options.get('status', 200)
response_args = {
'status': doc_env['status_code'],
'headers': headers,
'content_type': request.accepted_media_type
}
# when requesting HTML, typically when browsing the API through its
# documented views, we need to enrich the input data with documentation
# related ones and inform DRF that we request HTML template rendering
if request.accepted_media_type == 'text/html':
if data:
data = json.dumps(data,
sort_keys=True,
indent=4,
separators=(',', ': '))
doc_env['response_data'] = data
doc_env['request'] = {
'path': request.path,
'method': request.method,
'absolute_uri': request.build_absolute_uri(),
}
doc_env['heading'] = shorten_path(str(request.path))
if 'route' in doc_env:
doc_env['endpoint_path'] = gen_path_info(doc_env['route'])
response_args['data'] = doc_env
response_args['template_name'] = 'api/apidoc.html'
# otherwise simply return the raw data and let DRF picks
# the correct renderer (JSON or YAML)
else:
response_args['data'] = data
return Response(**response_args)
def content_display(request, query_string):
"""Django view that produces an HTML display of a content identified
by its hash value.
The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/`
""" # noqa
try:
algo, checksum = query.parse_hash(query_string)
checksum = hash_to_hex(checksum)
content_data = request_content(query_string,
raise_if_unavailable=False)
origin_type = request.GET.get('origin_type', None)
origin_url = request.GET.get('origin_url', None)
if not origin_url:
origin_url = request.GET.get('origin', None)
snapshot_context = None
if origin_url:
try:
snapshot_context = get_snapshot_context(
None, origin_type, origin_url)
except Exception:
raw_cnt_url = reverse('browse-content',
url_args={'query_string': query_string})
error_message = \
('The Software Heritage archive has a content '
'with the hash you provided but the origin '
'mentioned in your request appears broken: %s. '
'Please check the URL and try again.\n\n'
'Nevertheless, you can still browse the content '
'without origin information: %s'
% (gen_link(origin_url), gen_link(raw_cnt_url)))
raise NotFoundExc(error_message)
if snapshot_context:
snapshot_context['visit_info'] = None
except Exception as exc:
return handle_view_exception(request, exc)
path = request.GET.get('path', None)
content = None
language = None
mimetype = None
if content_data['raw_data'] is not None:
content_display_data = prepare_content_for_display(
content_data['raw_data'], content_data['mimetype'], path)
content = content_display_data['content_data']
language = content_display_data['language']
mimetype = content_display_data['mimetype']
root_dir = None
filename = None
path_info = None
query_params = {'origin': origin_url}
breadcrumbs = []
if path:
split_path = path.split('/')
root_dir = split_path[0]
filename = split_path[-1]
if root_dir != path:
path = path.replace(root_dir + '/', '')
path = path[:-len(filename)]
path_info = gen_path_info(path)
dir_url = reverse('browse-directory',
url_args={'sha1_git': root_dir},
query_params=query_params)
breadcrumbs.append({'name': root_dir[:7], 'url': dir_url})
for pi in path_info:
dir_url = reverse('browse-directory',
url_args={
'sha1_git': root_dir,
'path': pi['path']
},
query_params=query_params)
breadcrumbs.append({'name': pi['name'], 'url': dir_url})
breadcrumbs.append({'name': filename, 'url': None})
query_params = {'filename': filename}
content_raw_url = reverse('browse-content-raw',
url_args={'query_string': query_string},
query_params=query_params)
content_metadata = {
'sha1 checksum': content_data['checksums']['sha1'],
'sha1_git checksum': content_data['checksums']['sha1_git'],
'sha256 checksum': content_data['checksums']['sha256'],
'blake2s256 checksum': content_data['checksums']['blake2s256'],
'mime type': content_data['mimetype'],
'encoding': content_data['encoding'],
'size': filesizeformat(content_data['length']),
'language': content_data['language'],
'licenses': content_data['licenses'],
'filename': filename
}
if filename:
content_metadata['filename'] = filename
sha1_git = content_data['checksums']['sha1_git']
swh_ids = get_swh_persistent_ids([{'type': 'content', 'id': sha1_git}])
heading = 'Content - %s' % sha1_git
if breadcrumbs:
content_path = '/'.join([bc['name'] for bc in breadcrumbs])
heading += ' - %s' % content_path
return render(request,
'browse/content.html', {
'heading': heading,
'swh_object_id': swh_ids[0]['swh_id'],
'swh_object_name': 'Content',
'swh_object_metadata': content_metadata,
'content': content,
'content_size': content_data['length'],
'max_content_size': content_display_max_size,
'mimetype': mimetype,
'language': language,
'breadcrumbs': breadcrumbs,
'top_right_link': {
'url': content_raw_url,
'icon': swh_object_icons['content'],
'text': 'Raw File'
},
'snapshot_context': snapshot_context,
'vault_cooking': None,
'show_actions_menu': True,
'swh_ids': swh_ids,
'error_code': content_data['error_code'],
'error_message': content_data['error_message'],
'error_description': content_data['error_description']
},
status=content_data['error_code'])
def _directory_browse(request, sha1_git, path=None):
root_sha1_git = sha1_git
error_info = {"status_code": 200, "description": None}
if path:
try:
dir_info = archive.lookup_directory_with_path(sha1_git, path)
sha1_git = dir_info["target"]
except NotFoundExc as e:
error_info["status_code"] = 404
error_info["description"] = f"NotFoundExc: {str(e)}"
sha1_git = None
dirs, files = [], []
if sha1_git is not None:
dirs, files = get_directory_entries(sha1_git)
origin_url = request.GET.get("origin_url")
if not origin_url:
origin_url = request.GET.get("origin")
snapshot_id = request.GET.get("snapshot")
snapshot_context = None
if origin_url is not None or snapshot_id is not None:
try:
snapshot_context = get_snapshot_context(
snapshot_id=snapshot_id,
origin_url=origin_url,
branch_name=request.GET.get("branch"),
release_name=request.GET.get("release"),
revision_id=request.GET.get("revision"),
path=path,
)
except NotFoundExc as e:
if str(e).startswith("Origin"):
raw_dir_url = reverse(
"browse-directory", url_args={"sha1_git": sha1_git}
)
error_message = (
"The Software Heritage archive has a directory "
"with the hash you provided but the origin "
"mentioned in your request appears broken: %s. "
"Please check the URL and try again.\n\n"
"Nevertheless, you can still browse the directory "
"without origin information: %s"
% (gen_link(origin_url), gen_link(raw_dir_url))
)
raise NotFoundExc(error_message)
else:
raise e
path_info = gen_path_info(path)
query_params = snapshot_context["query_params"] if snapshot_context else {}
breadcrumbs = []
breadcrumbs.append(
{
"name": root_sha1_git[:7],
"url": reverse(
"browse-directory",
url_args={"sha1_git": root_sha1_git},
query_params={**query_params, "path": None},
),
}
)
for pi in path_info:
breadcrumbs.append(
{
"name": pi["name"],
"url": reverse(
"browse-directory",
url_args={"sha1_git": root_sha1_git},
query_params={**query_params, "path": pi["path"],},
),
}
)
path = "" if path is None else (path + "/")
for d in dirs:
if d["type"] == "rev":
d["url"] = reverse(
"browse-revision",
url_args={"sha1_git": d["target"]},
query_params=query_params,
)
else:
d["url"] = reverse(
"browse-directory",
url_args={"sha1_git": root_sha1_git},
query_params={**query_params, "path": path + d["name"],},
)
sum_file_sizes = 0
readmes = {}
for f in files:
query_string = "sha1_git:" + f["target"]
f["url"] = reverse(
"browse-content",
url_args={"query_string": query_string},
query_params={
**query_params,
"path": root_sha1_git + "/" + path + f["name"],
},
)
if f["length"] is not None:
sum_file_sizes += f["length"]
f["length"] = filesizeformat(f["length"])
if f["name"].lower().startswith("readme"):
readmes[f["name"]] = f["checksums"]["sha1"]
readme_name, readme_url, readme_html = get_readme_to_display(readmes)
sum_file_sizes = filesizeformat(sum_file_sizes)
dir_metadata = DirectoryMetadata(
object_type=DIRECTORY,
object_id=sha1_git,
directory=root_sha1_git,
nb_files=len(files),
nb_dirs=len(dirs),
sum_file_sizes=sum_file_sizes,
root_directory=root_sha1_git,
path=f"/{path}" if path else None,
revision=None,
revision_found=None,
release=None,
snapshot=None,
)
vault_cooking = {
"directory_context": True,
"directory_id": sha1_git,
"revision_context": False,
"revision_id": None,
}
swh_objects = [SWHObjectInfo(object_type=DIRECTORY, object_id=sha1_git)]
if snapshot_context:
swh_objects.append(
SWHObjectInfo(
object_type=REVISION, object_id=snapshot_context["revision_id"]
)
)
swh_objects.append(
SWHObjectInfo(
object_type=SNAPSHOT, object_id=snapshot_context["snapshot_id"]
)
)
if snapshot_context["release_id"]:
swh_objects.append(
SWHObjectInfo(
object_type=RELEASE, object_id=snapshot_context["release_id"]
)
)
swhids_info = get_swhids_info(swh_objects, snapshot_context, dir_metadata)
heading = "Directory - %s" % sha1_git
if breadcrumbs:
dir_path = "/".join([bc["name"] for bc in breadcrumbs]) + "/"
heading += " - %s" % dir_path
top_right_link = None
if snapshot_context is not None and not snapshot_context["is_empty"]:
history_url = reverse(
"browse-revision-log",
url_args={"sha1_git": snapshot_context["revision_id"]},
query_params=query_params,
)
top_right_link = {
"url": history_url,
"icon": swh_object_icons["revisions history"],
"text": "History",
}
return render(
request,
"browse/directory.html",
{
"heading": heading,
"swh_object_id": swhids_info[0]["swhid"],
"swh_object_name": "Directory",
"swh_object_metadata": dir_metadata,
"dirs": dirs,
"files": files,
"breadcrumbs": breadcrumbs,
"top_right_link": top_right_link,
"readme_name": readme_name,
"readme_url": readme_url,
"readme_html": readme_html,
"snapshot_context": snapshot_context,
"vault_cooking": vault_cooking,
"show_actions": True,
"swhids_info": swhids_info,
"error_code": error_info["status_code"],
"error_message": http_status_code_message.get(error_info["status_code"]),
"error_description": error_info["description"],
},
status=error_info["status_code"],
)
def directory_browse(request, sha1_git, path=None):
"""Django view for browsing the content of a directory identified
by its sha1_git value.
The url that points to it is :http:get:`/browse/directory/(sha1_git)/[(path)/]`
""" # noqa
root_sha1_git = sha1_git
try:
if path:
dir_info = service.lookup_directory_with_path(sha1_git, path)
# some readme files can reference assets reachable from the
# browsed directory, handle that special case in order to
# correctly displayed them
if dir_info and dir_info['type'] == 'file':
file_raw_url = reverse(
'browse-content-raw',
url_args={'query_string': dir_info['checksums']['sha1']})
return redirect(file_raw_url)
sha1_git = dir_info['target']
dirs, files = get_directory_entries(sha1_git)
origin_type = request.GET.get('origin_type', None)
origin_url = request.GET.get('origin_url', None)
if not origin_url:
origin_url = request.GET.get('origin', None)
snapshot_context = None
if origin_url:
try:
snapshot_context = get_snapshot_context(
None, origin_type, origin_url)
except Exception:
raw_dir_url = reverse('browse-directory',
url_args={'sha1_git': sha1_git})
error_message = \
('The Software Heritage archive has a directory '
'with the hash you provided but the origin '
'mentioned in your request appears broken: %s. '
'Please check the URL and try again.\n\n'
'Nevertheless, you can still browse the directory '
'without origin information: %s'
% (gen_link(origin_url), gen_link(raw_dir_url)))
raise NotFoundExc(error_message)
if snapshot_context:
snapshot_context['visit_info'] = None
except Exception as exc:
return handle_view_exception(request, exc)
path_info = gen_path_info(path)
query_params = {'origin': origin_url}
breadcrumbs = []
breadcrumbs.append({
'name':
root_sha1_git[:7],
'url':
reverse('browse-directory',
url_args={'sha1_git': root_sha1_git},
query_params=query_params)
})
for pi in path_info:
breadcrumbs.append({
'name':
pi['name'],
'url':
reverse('browse-directory',
url_args={
'sha1_git': root_sha1_git,
'path': pi['path']
},
query_params=query_params)
})
path = '' if path is None else (path + '/')
for d in dirs:
if d['type'] == 'rev':
d['url'] = reverse('browse-revision',
url_args={'sha1_git': d['target']},
query_params=query_params)
else:
d['url'] = reverse('browse-directory',
url_args={
'sha1_git': root_sha1_git,
'path': path + d['name']
},
query_params=query_params)
sum_file_sizes = 0
readmes = {}
for f in files:
query_string = 'sha1_git:' + f['target']
f['url'] = reverse('browse-content',
url_args={'query_string': query_string},
query_params={
'path': root_sha1_git + '/' + path + f['name'],
'origin': origin_url
})
if f['length'] is not None:
sum_file_sizes += f['length']
f['length'] = filesizeformat(f['length'])
if f['name'].lower().startswith('readme'):
readmes[f['name']] = f['checksums']['sha1']
readme_name, readme_url, readme_html = get_readme_to_display(readmes)
sum_file_sizes = filesizeformat(sum_file_sizes)
dir_metadata = {
'id': sha1_git,
'number of regular files': len(files),
'number of subdirectories': len(dirs),
'sum of regular file sizes': sum_file_sizes
}
vault_cooking = {
'directory_context': True,
'directory_id': sha1_git,
'revision_context': False,
'revision_id': None
}
swh_ids = get_swh_persistent_ids([{'type': 'directory', 'id': sha1_git}])
heading = 'Directory - %s' % sha1_git
if breadcrumbs:
dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/'
heading += ' - %s' % dir_path
return render(
request, 'browse/directory.html', {
'heading': heading,
'swh_object_id': swh_ids[0]['swh_id'],
'swh_object_name': 'Directory',
'swh_object_metadata': dir_metadata,
'dirs': dirs,
'files': files,
'breadcrumbs': breadcrumbs,
'top_right_link': None,
'readme_name': readme_name,
'readme_url': readme_url,
'readme_html': readme_html,
'snapshot_context': snapshot_context,
'vault_cooking': vault_cooking,
'show_actions_menu': True,
'swh_ids': swh_ids
})
def origin_content_view_helper(self, origin_info, origin_visits,
origin_branches, origin_releases,
root_dir_sha1, content,
visit_id=None, timestamp=None):
content_path = '/'.join(content['path'].split('/')[1:])
url_args = {'origin_type': origin_info['type'],
'origin_url': origin_info['url'],
'path': content_path}
if not visit_id:
visit_id = origin_visits[-1]['visit']
query_params = {}
if timestamp:
url_args['timestamp'] = timestamp
if visit_id:
query_params['visit_id'] = visit_id
url = reverse('browse-origin-content',
url_args=url_args,
query_params=query_params)
resp = self.client.get(url)
self.assertEqual(resp.status_code, 200)
self.assertTemplateUsed('content.html')
self.assertContains(resp, '<code class="%s">' %
content['hljs_language'])
self.assertContains(resp, escape(content['data']))
split_path = content_path.split('/')
filename = split_path[-1]
path = content_path.replace(filename, '')[:-1]
path_info = gen_path_info(path)
del url_args['path']
if timestamp:
url_args['timestamp'] = \
format_utc_iso_date(parse_timestamp(timestamp).isoformat(),
'%Y-%m-%dT%H:%M:%S')
root_dir_url = reverse('browse-origin-directory',
url_args=url_args,
query_params=query_params)
self.assertContains(resp, '<li class="swh-path">',
count=len(path_info)+1)
self.assertContains(resp, '<a href="%s">%s</a>' %
(root_dir_url, root_dir_sha1[:7]))
for p in path_info:
url_args['path'] = p['path']
dir_url = reverse('browse-origin-directory',
url_args=url_args,
query_params=query_params)
self.assertContains(resp, '<a href="%s">%s</a>' %
(dir_url, p['name']))
self.assertContains(resp, '<li>%s</li>' % filename)
query_string = 'sha1_git:' + content['sha1_git']
url_raw = reverse('browse-content-raw',
url_args={'query_string': query_string},
query_params={'filename': filename})
self.assertContains(resp, url_raw)
if 'args' in url_args:
del url_args['path']
origin_branches_url = reverse('browse-origin-branches',
url_args=url_args,
query_params=query_params)
self.assertContains(resp, '<a href="%s">Branches (%s)</a>' %
(origin_branches_url, len(origin_branches)))
origin_releases_url = reverse('browse-origin-releases',
url_args=url_args,
query_params=query_params)
self.assertContains(resp, '<a href="%s">Releases (%s)</a>' %
(origin_releases_url, len(origin_releases)))
self.assertContains(resp, '<li class="swh-branch">',
count=len(origin_branches))
url_args['path'] = content_path
for branch in origin_branches:
query_params['branch'] = branch['name']
root_dir_branch_url = reverse('browse-origin-content',
url_args=url_args,
query_params=query_params)
self.assertContains(resp, '<a href="%s">' % root_dir_branch_url)
self.assertContains(resp, '<li class="swh-release">',
count=len(origin_releases))
query_params['branch'] = None
for release in origin_releases:
query_params['release'] = release['name']
root_dir_release_url = reverse('browse-origin-content',
url_args=url_args,
query_params=query_params)
self.assertContains(resp, '<a href="%s">' % root_dir_release_url)
del url_args['origin_type']
url = reverse('browse-origin-content',
url_args=url_args,
query_params=query_params)
resp = self.client.get(url)
self.assertEqual(resp.status_code, 200)
self.assertTemplateUsed('content.html')
swh_cnt_id = get_swh_persistent_id('content', content['sha1_git'])
swh_cnt_id_url = reverse('browse-swh-id',
url_args={'swh_id': swh_cnt_id})
self.assertContains(resp, swh_cnt_id)
self.assertContains(resp, swh_cnt_id_url)
self.assertContains(resp, 'swh-take-new-snapshot')
def revision_browse(request, sha1_git):
"""
Django view that produces an HTML display of a revision
identified by its id.
The url that points to it is :http:get:`/browse/revision/(sha1_git)/`.
"""
revision = archive.lookup_revision(sha1_git)
origin_info = None
snapshot_context = None
origin_url = request.GET.get("origin_url")
if not origin_url:
origin_url = request.GET.get("origin")
timestamp = request.GET.get("timestamp")
visit_id = request.GET.get("visit_id")
snapshot_id = request.GET.get("snapshot_id")
if not snapshot_id:
snapshot_id = request.GET.get("snapshot")
path = request.GET.get("path")
dir_id = None
dirs, files = [], []
content_data = {}
if origin_url:
try:
snapshot_context = get_snapshot_context(
snapshot_id=snapshot_id,
origin_url=origin_url,
timestamp=timestamp,
visit_id=visit_id,
branch_name=request.GET.get("branch"),
release_name=request.GET.get("release"),
revision_id=sha1_git,
path=path,
)
except NotFoundExc as e:
raw_rev_url = reverse("browse-revision",
url_args={"sha1_git": sha1_git})
error_message = ("The Software Heritage archive has a revision "
"with the hash you provided but the origin "
"mentioned in your request appears broken: %s. "
"Please check the URL and try again.\n\n"
"Nevertheless, you can still browse the revision "
"without origin information: %s" %
(gen_link(origin_url), gen_link(raw_rev_url)))
if str(e).startswith("Origin"):
raise NotFoundExc(error_message)
else:
raise e
origin_info = snapshot_context["origin_info"]
snapshot_id = snapshot_context["snapshot_id"]
elif snapshot_id:
snapshot_context = get_snapshot_context(snapshot_id)
error_info = {"status_code": 200, "description": None}
if path:
try:
file_info = archive.lookup_directory_with_path(
revision["directory"], path)
if file_info["type"] == "dir":
dir_id = file_info["target"]
else:
query_string = "sha1_git:" + file_info["target"]
content_data = request_content(query_string)
except NotFoundExc as e:
error_info["status_code"] = 404
error_info["description"] = f"NotFoundExc: {str(e)}"
else:
dir_id = revision["directory"]
if dir_id:
path = "" if path is None else (path + "/")
dirs, files = get_directory_entries(dir_id)
revision_metadata = RevisionMetadata(
object_type=REVISION,
object_id=sha1_git,
revision=sha1_git,
author=revision["author"]["fullname"]
if revision["author"] else "None",
author_url=gen_person_mail_link(revision["author"])
if revision["author"] else "None",
committer=revision["committer"]["fullname"]
if revision["committer"] else "None",
committer_url=gen_person_mail_link(revision["committer"])
if revision["committer"] else "None",
committer_date=format_utc_iso_date(revision["committer_date"]),
date=format_utc_iso_date(revision["date"]),
directory=revision["directory"],
merge=revision["merge"],
metadata=json.dumps(revision["metadata"],
sort_keys=True,
indent=4,
separators=(",", ": ")),
parents=revision["parents"],
synthetic=revision["synthetic"],
type=revision["type"],
snapshot=snapshot_id,
origin_url=origin_url,
)
message_lines = ["None"]
if revision["message"]:
message_lines = revision["message"].split("\n")
parents = []
for p in revision["parents"]:
parent_url = gen_revision_url(p, snapshot_context)
parents.append({"id": p, "url": parent_url})
path_info = gen_path_info(path)
query_params = snapshot_context["query_params"] if snapshot_context else {}
breadcrumbs = []
breadcrumbs.append({
"name":
revision["directory"][:7],
"url":
reverse(
"browse-revision",
url_args={"sha1_git": sha1_git},
query_params=query_params,
),
})
for pi in path_info:
query_params["path"] = pi["path"]
breadcrumbs.append({
"name":
pi["name"],
"url":
reverse(
"browse-revision",
url_args={"sha1_git": sha1_git},
query_params=query_params,
),
})
vault_cooking = {
"directory_context": False,
"directory_id": None,
"revision_context": True,
"revision_id": sha1_git,
}
swh_objects = [SWHObjectInfo(object_type=REVISION, object_id=sha1_git)]
content = None
content_size = None
filename = None
mimetype = None
language = None
readme_name = None
readme_url = None
readme_html = None
readmes = {}
extra_context = dict(revision_metadata)
extra_context["path"] = f"/{path}" if path else None
if content_data:
breadcrumbs[-1]["url"] = None
content_size = content_data["length"]
mimetype = content_data["mimetype"]
if content_data["raw_data"]:
content_display_data = prepare_content_for_display(
content_data["raw_data"], content_data["mimetype"], path)
content = content_display_data["content_data"]
language = content_display_data["language"]
mimetype = content_display_data["mimetype"]
if path:
filename = path_info[-1]["name"]
query_params["filename"] = filename
filepath = "/".join(pi["name"] for pi in path_info[:-1])
extra_context["path"] = f"/{filepath}/" if filepath else "/"
extra_context["filename"] = filename
top_right_link = {
"url":
reverse(
"browse-content-raw",
url_args={"query_string": query_string},
query_params={"filename": filename},
),
"icon":
swh_object_icons["content"],
"text":
"Raw File",
}
swh_objects.append(
SWHObjectInfo(object_type=CONTENT, object_id=file_info["target"]))
else:
for d in dirs:
if d["type"] == "rev":
d["url"] = reverse("browse-revision",
url_args={"sha1_git": d["target"]})
else:
query_params["path"] = path + d["name"]
d["url"] = reverse(
"browse-revision",
url_args={"sha1_git": sha1_git},
query_params=query_params,
)
for f in files:
query_params["path"] = path + f["name"]
f["url"] = reverse(
"browse-revision",
url_args={"sha1_git": sha1_git},
query_params=query_params,
)
if f["length"] is not None:
f["length"] = filesizeformat(f["length"])
if f["name"].lower().startswith("readme"):
readmes[f["name"]] = f["checksums"]["sha1"]
readme_name, readme_url, readme_html = get_readme_to_display(readmes)
top_right_link = {
"url": get_revision_log_url(sha1_git, snapshot_context),
"icon": swh_object_icons["revisions history"],
"text": "History",
}
vault_cooking["directory_context"] = True
vault_cooking["directory_id"] = dir_id
swh_objects.append(
SWHObjectInfo(object_type=DIRECTORY, object_id=dir_id))
query_params.pop("path", None)
diff_revision_url = reverse(
"diff-revision",
url_args={"sha1_git": sha1_git},
query_params=query_params,
)
if snapshot_id:
swh_objects.append(
SWHObjectInfo(object_type=SNAPSHOT, object_id=snapshot_id))
swhids_info = get_swhids_info(swh_objects, snapshot_context, extra_context)
heading = "Revision - %s - %s" % (
sha1_git[:7],
textwrap.shorten(message_lines[0], width=70),
)
if snapshot_context:
context_found = "snapshot: %s" % snapshot_context["snapshot_id"]
if origin_info:
context_found = "origin: %s" % origin_info["url"]
heading += " - %s" % context_found
return render(
request,
"browse/revision.html",
{
"heading":
heading,
"swh_object_id":
swhids_info[0]["swhid"],
"swh_object_name":
"Revision",
"swh_object_metadata":
revision_metadata,
"message_header":
message_lines[0],
"message_body":
"\n".join(message_lines[1:]),
"parents":
parents,
"snapshot_context":
snapshot_context,
"dirs":
dirs,
"files":
files,
"content":
content,
"content_size":
content_size,
"max_content_size":
content_display_max_size,
"filename":
filename,
"encoding":
content_data.get("encoding"),
"mimetype":
mimetype,
"language":
language,
"readme_name":
readme_name,
"readme_url":
readme_url,
"readme_html":
readme_html,
"breadcrumbs":
breadcrumbs,
"top_right_link":
top_right_link,
"vault_cooking":
vault_cooking,
"diff_revision_url":
diff_revision_url,
"show_actions":
True,
"swhids_info":
swhids_info,
"error_code":
error_info["status_code"],
"error_message":
http_status_code_message.get(error_info["status_code"]),
"error_description":
error_info["description"],
},
status=error_info["status_code"],
)
def directory_view(self,
root_directory_sha1,
directory_entries,
path=None):
dirs = [e for e in directory_entries if e['type'] in ('dir', 'rev')]
files = [e for e in directory_entries if e['type'] == 'file']
url_args = {'sha1_git': root_directory_sha1}
if path:
url_args['path'] = path
url = reverse('browse-directory', url_args=url_args)
root_dir_url = reverse('browse-directory',
url_args={'sha1_git': root_directory_sha1})
resp = self.client.get(url)
self.assertEqual(resp.status_code, 200)
self.assertTemplateUsed('browse/directory.html')
self.assertContains(
resp, '<a href="' + root_dir_url + '">' + root_directory_sha1[:7] +
'</a>')
self.assertContains(resp,
'<td class="swh-directory">',
count=len(dirs))
self.assertContains(resp, '<td class="swh-content">', count=len(files))
for d in dirs:
if d['type'] == 'rev':
dir_url = reverse('browse-revision',
url_args={'sha1_git': d['target']})
else:
dir_path = d['name']
if path:
dir_path = "%s/%s" % (path, d['name'])
dir_url = reverse('browse-directory',
url_args={
'sha1_git': root_directory_sha1,
'path': dir_path
})
self.assertContains(resp, dir_url)
for f in files:
file_path = "%s/%s" % (root_directory_sha1, f['name'])
if path:
file_path = "%s/%s/%s" % (root_directory_sha1, path, f['name'])
query_string = 'sha1_git:' + f['target']
file_url = reverse('browse-content',
url_args={'query_string': query_string},
query_params={'path': file_path})
self.assertContains(resp, file_url)
path_info = gen_path_info(path)
self.assertContains(resp,
'<li class="swh-path">',
count=len(path_info) + 1)
self.assertContains(
resp,
'<a href="%s">%s</a>' % (root_dir_url, root_directory_sha1[:7]))
for p in path_info:
dir_url = reverse('browse-directory',
url_args={
'sha1_git': root_directory_sha1,
'path': p['path']
})
self.assertContains(resp,
'<a href="%s">%s</a>' % (dir_url, p['name']))
self.assertContains(resp, 'vault-cook-directory')
swh_dir_id = get_swh_persistent_id(
'directory', directory_entries[0]['dir_id']) # noqa
swh_dir_id_url = reverse('browse-swh-id',
url_args={'swh_id': swh_dir_id})
self.assertContains(resp, swh_dir_id)
self.assertContains(resp, swh_dir_id_url)
