def _revision_diff(request, sha1_git):
"""
Browse internal endpoint to compute revision diff
"""
revision = archive.lookup_revision(sha1_git)
snapshot_context = None
origin_url = request.GET.get("origin_url", None)
if not origin_url:
origin_url = request.GET.get("origin", None)
timestamp = request.GET.get("timestamp", None)
visit_id = request.GET.get("visit_id", None)
if origin_url:
snapshot_context = get_snapshot_context(origin_url=origin_url,
timestamp=timestamp,
visit_id=visit_id)
changes = archive.diff_revision(sha1_git)
changes_msg = _gen_revision_changes_list(revision, changes,
snapshot_context)
diff_data = {
"total_nb_changes": len(changes),
"changes": changes[:_max_displayed_file_diffs],
"changes_msg": changes_msg,
}
return JsonResponse(diff_data)
def test_lookup_revision_invalid_msg(archive_data, new_revision):
new_revision = new_revision.to_dict()
new_revision["message"] = b"elegant fix for bug \xff"
archive_data.revision_add([Revision.from_dict(new_revision)])
revision = archive.lookup_revision(hash_to_hex(new_revision["id"]))
assert revision["message"] == "elegant fix for bug \\xff"
assert revision["decoding_failures"] == ["message"]
def _gen_revision_changes_list(revision, changes, snapshot_context):
"""
Returns a HTML string describing the file changes
introduced in a revision.
As this string will be displayed in the browse revision view,
links to adequate file diffs are also generated.
Args:
revision (str): hexadecimal representation of a revision identifier
changes (list): list of file changes in the revision
snapshot_context (dict): optional origin context used to reverse
the content urls
Returns:
A string to insert in a revision HTML view.
"""
changes_msg = []
for i, change in enumerate(changes):
hasher = hashlib.sha1()
from_query_string = ""
to_query_string = ""
diff_id = "diff-"
if change["from"]:
from_query_string = "sha1_git:" + change["from"]["target"]
diff_id += change["from"]["target"] + "-" + change["from_path"]
diff_id += "-"
if change["to"]:
to_query_string = "sha1_git:" + change["to"]["target"]
diff_id += change["to"]["target"] + change["to_path"]
change["path"] = change["to_path"] or change["from_path"]
url_args = {
"from_query_string": from_query_string,
"to_query_string": to_query_string,
}
query_params = {"path": change["path"]}
change["diff_url"] = reverse("diff-contents",
url_args=url_args,
query_params=query_params)
hasher.update(diff_id.encode("utf-8"))
diff_id = hasher.hexdigest()
change["id"] = diff_id
diff_link = "#diff_" + diff_id
if change["type"] == "modify":
change["content_url"] = _gen_content_url(revision, to_query_string,
change["to_path"],
snapshot_context)
changes_msg.append("modified: %s" %
_gen_diff_link(i, diff_link, change["to_path"]))
elif change["type"] == "insert":
change["content_url"] = _gen_content_url(revision, to_query_string,
change["to_path"],
snapshot_context)
changes_msg.append("new file: %s" %
_gen_diff_link(i, diff_link, change["to_path"]))
elif change["type"] == "delete":
parent = archive.lookup_revision(revision["parents"][0])
change["content_url"] = _gen_content_url(parent, from_query_string,
change["from_path"],
snapshot_context)
changes_msg.append(
"deleted: %s" %
_gen_diff_link(i, diff_link, change["from_path"]))
elif change["type"] == "rename":
change["content_url"] = _gen_content_url(revision, to_query_string,
change["to_path"],
snapshot_context)
link_text = change["from_path"] + " → " + change["to_path"]
changes_msg.append("renamed: %s" %
_gen_diff_link(i, diff_link, link_text))
if not changes:
changes_msg.append("No changes")
return mark_safe("\n".join(changes_msg))
def revision_browse(request, sha1_git):
"""
Django view that produces an HTML display of a revision
identified by its id.
The url that points to it is :http:get:`/browse/revision/(sha1_git)/`.
"""
revision = archive.lookup_revision(sha1_git)
origin_info = None
snapshot_context = None
origin_url = request.GET.get("origin_url")
if not origin_url:
origin_url = request.GET.get("origin")
timestamp = request.GET.get("timestamp")
visit_id = request.GET.get("visit_id")
snapshot_id = request.GET.get("snapshot_id")
if not snapshot_id:
snapshot_id = request.GET.get("snapshot")
path = request.GET.get("path")
dir_id = None
dirs, files = [], []
content_data = {}
if origin_url:
try:
snapshot_context = get_snapshot_context(
snapshot_id=snapshot_id,
origin_url=origin_url,
timestamp=timestamp,
visit_id=visit_id,
branch_name=request.GET.get("branch"),
release_name=request.GET.get("release"),
revision_id=sha1_git,
path=path,
)
except NotFoundExc as e:
raw_rev_url = reverse("browse-revision",
url_args={"sha1_git": sha1_git})
error_message = ("The Software Heritage archive has a revision "
"with the hash you provided but the origin "
"mentioned in your request appears broken: %s. "
"Please check the URL and try again.\n\n"
"Nevertheless, you can still browse the revision "
"without origin information: %s" %
(gen_link(origin_url), gen_link(raw_rev_url)))
if str(e).startswith("Origin"):
raise NotFoundExc(error_message)
else:
raise e
origin_info = snapshot_context["origin_info"]
snapshot_id = snapshot_context["snapshot_id"]
elif snapshot_id:
snapshot_context = get_snapshot_context(snapshot_id)
error_info = {"status_code": 200, "description": None}
if path:
try:
file_info = archive.lookup_directory_with_path(
revision["directory"], path)
if file_info["type"] == "dir":
dir_id = file_info["target"]
else:
query_string = "sha1_git:" + file_info["target"]
content_data = request_content(query_string)
except NotFoundExc as e:
error_info["status_code"] = 404
error_info["description"] = f"NotFoundExc: {str(e)}"
else:
dir_id = revision["directory"]
if dir_id:
path = "" if path is None else (path + "/")
dirs, files = get_directory_entries(dir_id)
revision_metadata = RevisionMetadata(
object_type=REVISION,
object_id=sha1_git,
revision=sha1_git,
author=revision["author"]["fullname"]
if revision["author"] else "None",
author_url=gen_person_mail_link(revision["author"])
if revision["author"] else "None",
committer=revision["committer"]["fullname"]
if revision["committer"] else "None",
committer_url=gen_person_mail_link(revision["committer"])
if revision["committer"] else "None",
committer_date=format_utc_iso_date(revision["committer_date"]),
date=format_utc_iso_date(revision["date"]),
directory=revision["directory"],
merge=revision["merge"],
metadata=json.dumps(revision["metadata"],
sort_keys=True,
indent=4,
separators=(",", ": ")),
parents=revision["parents"],
synthetic=revision["synthetic"],
type=revision["type"],
snapshot=snapshot_id,
origin_url=origin_url,
)
message_lines = ["None"]
if revision["message"]:
message_lines = revision["message"].split("\n")
parents = []
for p in revision["parents"]:
parent_url = gen_revision_url(p, snapshot_context)
parents.append({"id": p, "url": parent_url})
path_info = gen_path_info(path)
query_params = snapshot_context["query_params"] if snapshot_context else {}
breadcrumbs = []
breadcrumbs.append({
"name":
revision["directory"][:7],
"url":
reverse(
"browse-revision",
url_args={"sha1_git": sha1_git},
query_params=query_params,
),
})
for pi in path_info:
query_params["path"] = pi["path"]
breadcrumbs.append({
"name":
pi["name"],
"url":
reverse(
"browse-revision",
url_args={"sha1_git": sha1_git},
query_params=query_params,
),
})
vault_cooking = {
"directory_context": False,
"directory_id": None,
"revision_context": True,
"revision_id": sha1_git,
}
swh_objects = [SWHObjectInfo(object_type=REVISION, object_id=sha1_git)]
content = None
content_size = None
filename = None
mimetype = None
language = None
readme_name = None
readme_url = None
readme_html = None
readmes = {}
extra_context = dict(revision_metadata)
extra_context["path"] = f"/{path}" if path else None
if content_data:
breadcrumbs[-1]["url"] = None
content_size = content_data["length"]
mimetype = content_data["mimetype"]
if content_data["raw_data"]:
content_display_data = prepare_content_for_display(
content_data["raw_data"], content_data["mimetype"], path)
content = content_display_data["content_data"]
language = content_display_data["language"]
mimetype = content_display_data["mimetype"]
if path:
filename = path_info[-1]["name"]
query_params["filename"] = filename
filepath = "/".join(pi["name"] for pi in path_info[:-1])
extra_context["path"] = f"/{filepath}/" if filepath else "/"
extra_context["filename"] = filename
top_right_link = {
"url":
reverse(
"browse-content-raw",
url_args={"query_string": query_string},
query_params={"filename": filename},
),
"icon":
swh_object_icons["content"],
"text":
"Raw File",
}
swh_objects.append(
SWHObjectInfo(object_type=CONTENT, object_id=file_info["target"]))
else:
for d in dirs:
if d["type"] == "rev":
d["url"] = reverse("browse-revision",
url_args={"sha1_git": d["target"]})
else:
query_params["path"] = path + d["name"]
d["url"] = reverse(
"browse-revision",
url_args={"sha1_git": sha1_git},
query_params=query_params,
)
for f in files:
query_params["path"] = path + f["name"]
f["url"] = reverse(
"browse-revision",
url_args={"sha1_git": sha1_git},
query_params=query_params,
)
if f["length"] is not None:
f["length"] = filesizeformat(f["length"])
if f["name"].lower().startswith("readme"):
readmes[f["name"]] = f["checksums"]["sha1"]
readme_name, readme_url, readme_html = get_readme_to_display(readmes)
top_right_link = {
"url": get_revision_log_url(sha1_git, snapshot_context),
"icon": swh_object_icons["revisions history"],
"text": "History",
}
vault_cooking["directory_context"] = True
vault_cooking["directory_id"] = dir_id
swh_objects.append(
SWHObjectInfo(object_type=DIRECTORY, object_id=dir_id))
query_params.pop("path", None)
diff_revision_url = reverse(
"diff-revision",
url_args={"sha1_git": sha1_git},
query_params=query_params,
)
if snapshot_id:
swh_objects.append(
SWHObjectInfo(object_type=SNAPSHOT, object_id=snapshot_id))
swhids_info = get_swhids_info(swh_objects, snapshot_context, extra_context)
heading = "Revision - %s - %s" % (
sha1_git[:7],
textwrap.shorten(message_lines[0], width=70),
)
if snapshot_context:
context_found = "snapshot: %s" % snapshot_context["snapshot_id"]
if origin_info:
context_found = "origin: %s" % origin_info["url"]
heading += " - %s" % context_found
return render(
request,
"browse/revision.html",
{
"heading":
heading,
"swh_object_id":
swhids_info[0]["swhid"],
"swh_object_name":
"Revision",
"swh_object_metadata":
revision_metadata,
"message_header":
message_lines[0],
"message_body":
"\n".join(message_lines[1:]),
"parents":
parents,
"snapshot_context":
snapshot_context,
"dirs":
dirs,
"files":
files,
"content":
content,
"content_size":
content_size,
"max_content_size":
content_display_max_size,
"filename":
filename,
"encoding":
content_data.get("encoding"),
"mimetype":
mimetype,
"language":
language,
"readme_name":
readme_name,
"readme_url":
readme_url,
"readme_html":
readme_html,
"breadcrumbs":
breadcrumbs,
"top_right_link":
top_right_link,
"vault_cooking":
vault_cooking,
"diff_revision_url":
diff_revision_url,
"show_actions":
True,
"swhids_info":
swhids_info,
"error_code":
error_info["status_code"],
"error_message":
http_status_code_message.get(error_info["status_code"]),
"error_description":
error_info["description"],
},
status=error_info["status_code"],
)
def release_browse(request, sha1_git):
"""
Django view that produces an HTML display of a release
identified by its id.
The url that points to it is :http:get:`/browse/release/(sha1_git)/`.
"""
release = archive.lookup_release(sha1_git)
snapshot_context = {}
origin_info = None
snapshot_id = request.GET.get("snapshot_id")
if not snapshot_id:
snapshot_id = request.GET.get("snapshot")
origin_url = request.GET.get("origin_url")
if not origin_url:
origin_url = request.GET.get("origin")
timestamp = request.GET.get("timestamp")
visit_id = request.GET.get("visit_id")
if origin_url:
try:
snapshot_context = get_snapshot_context(
snapshot_id,
origin_url,
timestamp,
visit_id,
release_name=release["name"],
)
except NotFoundExc as e:
raw_rel_url = reverse("browse-release", url_args={"sha1_git": sha1_git})
error_message = (
"The Software Heritage archive has a release "
"with the hash you provided but the origin "
"mentioned in your request appears broken: %s. "
"Please check the URL and try again.\n\n"
"Nevertheless, you can still browse the release "
"without origin information: %s"
% (gen_link(origin_url), gen_link(raw_rel_url))
)
if str(e).startswith("Origin"):
raise NotFoundExc(error_message)
else:
raise e
origin_info = snapshot_context["origin_info"]
elif snapshot_id:
snapshot_context = get_snapshot_context(
snapshot_id, release_name=release["name"]
)
snapshot_id = snapshot_context.get("snapshot_id", None)
release_metadata = ReleaseMetadata(
object_type=RELEASE,
object_id=sha1_git,
release=sha1_git,
author=release["author"]["fullname"] if release["author"] else "None",
author_url=gen_person_mail_link(release["author"])
if release["author"]
else "None",
date=format_utc_iso_date(release["date"]),
name=release["name"],
synthetic=release["synthetic"],
target=release["target"],
target_type=release["target_type"],
snapshot=snapshot_id,
origin_url=origin_url,
)
release_note_lines = []
if release["message"]:
release_note_lines = release["message"].split("\n")
swh_objects = [SWHObjectInfo(object_type=RELEASE, object_id=sha1_git)]
vault_cooking = None
rev_directory = None
target_link = None
if release["target_type"] == REVISION:
target_link = gen_revision_link(
release["target"],
snapshot_context=snapshot_context,
link_text=None,
link_attrs=None,
)
try:
revision = archive.lookup_revision(release["target"])
rev_directory = revision["directory"]
vault_cooking = {
"directory_context": True,
"directory_id": rev_directory,
"revision_context": True,
"revision_id": release["target"],
}
swh_objects.append(
SWHObjectInfo(object_type=REVISION, object_id=release["target"])
)
swh_objects.append(
SWHObjectInfo(object_type=DIRECTORY, object_id=rev_directory)
)
except Exception as exc:
sentry_sdk.capture_exception(exc)
elif release["target_type"] == DIRECTORY:
target_link = gen_directory_link(
release["target"],
snapshot_context=snapshot_context,
link_text=None,
link_attrs=None,
)
try:
# check directory exists
archive.lookup_directory(release["target"])
vault_cooking = {
"directory_context": True,
"directory_id": release["target"],
"revision_context": False,
"revision_id": None,
}
swh_objects.append(
SWHObjectInfo(object_type=DIRECTORY, object_id=release["target"])
)
except Exception as exc:
sentry_sdk.capture_exception(exc)
elif release["target_type"] == CONTENT:
target_link = gen_content_link(
release["target"],
snapshot_context=snapshot_context,
link_text=None,
link_attrs=None,
)
swh_objects.append(
SWHObjectInfo(object_type=CONTENT, object_id=release["target"])
)
elif release["target_type"] == RELEASE:
target_link = gen_release_link(
release["target"],
snapshot_context=snapshot_context,
link_text=None,
link_attrs=None,
)
rev_directory_url = None
if rev_directory is not None:
if origin_info:
rev_directory_url = reverse(
"browse-origin-directory",
query_params={
"origin_url": origin_info["url"],
"release": release["name"],
"snapshot": snapshot_id,
},
)
elif snapshot_id:
rev_directory_url = reverse(
"browse-snapshot-directory",
url_args={"snapshot_id": snapshot_id},
query_params={"release": release["name"]},
)
else:
rev_directory_url = reverse(
"browse-directory", url_args={"sha1_git": rev_directory}
)
directory_link = None
if rev_directory_url is not None:
directory_link = gen_link(rev_directory_url, rev_directory)
release["directory_link"] = directory_link
release["target_link"] = target_link
if snapshot_context:
snapshot_id = snapshot_context["snapshot_id"]
if snapshot_id:
swh_objects.append(SWHObjectInfo(object_type=SNAPSHOT, object_id=snapshot_id))
swhids_info = get_swhids_info(swh_objects, snapshot_context)
note_header = "None"
if len(release_note_lines) > 0:
note_header = release_note_lines[0]
release["note_header"] = note_header
release["note_body"] = "\n".join(release_note_lines[1:])
heading = "Release - %s" % release["name"]
if snapshot_context:
context_found = "snapshot: %s" % snapshot_context["snapshot_id"]
if origin_info:
context_found = "origin: %s" % origin_info["url"]
heading += " - %s" % context_found
return render(
request,
"browse/release.html",
{
"heading": heading,
"swh_object_id": swhids_info[0]["swhid"],
"swh_object_name": "Release",
"swh_object_metadata": release_metadata,
"release": release,
"snapshot_context": snapshot_context,
"show_actions": True,
"breadcrumbs": None,
"vault_cooking": vault_cooking,
"top_right_link": None,
"swhids_info": swhids_info,
},
)
def test_lookup_revision_through_with_revision(revision):
assert archive.lookup_revision_through(
{"sha1_git": revision}) == archive.lookup_revision(revision)
def test_lookup_revision(archive_data, revision):
actual_revision = archive.lookup_revision(revision)
assert actual_revision == archive_data.revision_get(revision)
def resolve_swhid(
swhid: str, query_params: Optional[QueryParameters] = None
) -> ResolvedSWHID:
"""
Try to resolve a SoftWare Heritage persistent IDentifier into an url for
browsing the targeted object.
Args:
swhid: a SoftWare Heritage persistent IDentifier
query_params: optional dict filled with
query parameters to append to the browse url
Returns:
a dict with the following keys:
* **swhid_parsed**: the parsed identifier
* **browse_url**: the url for browsing the targeted object
"""
swhid_parsed = get_swhid(swhid)
object_type = swhid_parsed.object_type
object_id = swhid_parsed.object_id
browse_url = None
url_args = {}
query_dict = QueryDict("", mutable=True)
fragment = ""
process_lines = object_type == ObjectType.CONTENT
if query_params and len(query_params) > 0:
for k in sorted(query_params.keys()):
query_dict[k] = query_params[k]
if swhid_parsed.origin:
origin_url = unquote(swhid_parsed.origin)
origin_url = archive.lookup_origin({"url": origin_url})["url"]
query_dict["origin_url"] = origin_url
if swhid_parsed.path and swhid_parsed.path != b"/":
query_dict["path"] = swhid_parsed.path.decode("utf8", errors="replace")
if swhid_parsed.anchor:
directory = b""
if swhid_parsed.anchor.object_type == ObjectType.DIRECTORY:
directory = swhid_parsed.anchor.object_id
elif swhid_parsed.anchor.object_type == ObjectType.REVISION:
revision = archive.lookup_revision(
hash_to_hex(swhid_parsed.anchor.object_id)
)
directory = revision["directory"]
elif swhid_parsed.anchor.object_type == ObjectType.RELEASE:
release = archive.lookup_release(
hash_to_hex(swhid_parsed.anchor.object_id)
)
if release["target_type"] == REVISION:
revision = archive.lookup_revision(release["target"])
directory = revision["directory"]
if object_type == ObjectType.CONTENT:
if not swhid_parsed.origin:
# when no origin context, content objects need to have their
# path prefixed by root directory id for proper breadcrumbs display
query_dict["path"] = hash_to_hex(directory) + query_dict["path"]
else:
# remove leading slash from SWHID content path
query_dict["path"] = query_dict["path"][1:]
elif object_type == ObjectType.DIRECTORY:
object_id = directory
# remove leading and trailing slashes from SWHID directory path
if query_dict["path"].endswith("/"):
query_dict["path"] = query_dict["path"][1:-1]
else:
query_dict["path"] = query_dict["path"][1:]
# snapshot context
if swhid_parsed.visit:
if swhid_parsed.visit.object_type != ObjectType.SNAPSHOT:
raise BadInputExc("Visit must be a snapshot SWHID.")
query_dict["snapshot"] = hash_to_hex(swhid_parsed.visit.object_id)
if swhid_parsed.anchor:
if swhid_parsed.anchor.object_type == ObjectType.REVISION:
# check if the anchor revision is the tip of a branch
branch_name = archive.lookup_snapshot_branch_name_from_tip_revision(
hash_to_hex(swhid_parsed.visit.object_id),
hash_to_hex(swhid_parsed.anchor.object_id),
)
if branch_name:
query_dict["branch"] = branch_name
elif object_type != ObjectType.REVISION:
query_dict["revision"] = hash_to_hex(swhid_parsed.anchor.object_id)
elif swhid_parsed.anchor.object_type == ObjectType.RELEASE:
release = archive.lookup_release(
hash_to_hex(swhid_parsed.anchor.object_id)
)
if release:
query_dict["release"] = release["name"]
if object_type == ObjectType.REVISION and "release" not in query_dict:
branch_name = archive.lookup_snapshot_branch_name_from_tip_revision(
hash_to_hex(swhid_parsed.visit.object_id), hash_to_hex(object_id)
)
if branch_name:
query_dict["branch"] = branch_name
# browsing content or directory without snapshot context
elif (
object_type in (ObjectType.CONTENT, ObjectType.DIRECTORY)
and swhid_parsed.anchor
):
if swhid_parsed.anchor.object_type == ObjectType.REVISION:
# anchor revision, objects are browsed from its view
object_type = ObjectType.REVISION
object_id = swhid_parsed.anchor.object_id
elif (
object_type == ObjectType.DIRECTORY
and swhid_parsed.anchor.object_type == ObjectType.DIRECTORY
):
# a directory is browsed from its root
object_id = swhid_parsed.anchor.object_id
if object_type == ObjectType.CONTENT:
url_args["query_string"] = f"sha1_git:{hash_to_hex(object_id)}"
elif object_type in (ObjectType.DIRECTORY, ObjectType.RELEASE, ObjectType.REVISION):
url_args["sha1_git"] = hash_to_hex(object_id)
elif object_type == ObjectType.SNAPSHOT:
url_args["snapshot_id"] = hash_to_hex(object_id)
if swhid_parsed.lines and process_lines:
lines = swhid_parsed.lines
fragment += "#L" + str(lines[0])
if lines[1]:
fragment += "-L" + str(lines[1])
if url_args:
browse_url = (
reverse(
f"browse-{object_type.name.lower()}",
url_args=url_args,
query_params=query_dict,
)
+ fragment
)
return ResolvedSWHID(swhid_parsed=swhid_parsed, browse_url=browse_url)