def test_get_origin_visits(self, mock_service):
mock_service.MAX_LIMIT = 2
def _lookup_origin_visits(*args, **kwargs):
if kwargs['last_visit'] is None:
return [{
'visit': 1,
'date': '2017-05-06T00:59:10+00:00',
'metadata': {}
}, {
'visit': 2,
'date': '2017-08-06T00:59:10+00:00',
'metadata': {}
}]
else:
return [{
'visit': 3,
'date': '2017-09-06T00:59:10+00:00',
'metadata': {}
}]
mock_service.lookup_origin_visits.side_effect = _lookup_origin_visits
origin_info = {
'id': 1,
'type': 'git',
'url': 'https://github.com/foo/bar',
}
origin_visits = get_origin_visits(origin_info)
self.assertEqual(len(origin_visits), 3)
def _get_visit_info_for_save_request(save_request):
visit_date = None
visit_status = None
time_now = datetime.now(tz=timezone.utc)
time_delta = time_now - save_request.request_date
# stop trying to find a visit date one month after save request submission
# as those requests to storage are expensive and associated loading task
# surely ended up with errors
if time_delta.days <= 30:
try:
origin = {"url": save_request.origin_url}
origin_info = archive.lookup_origin(origin)
origin_visits = get_origin_visits(origin_info)
visit_dates = [
parse_iso8601_date_to_utc(v["date"]) for v in origin_visits
]
i = bisect_right(visit_dates, save_request.request_date)
if i != len(visit_dates):
visit_date = visit_dates[i]
visit_status = origin_visits[i]["status"]
if origin_visits[i]["status"] not in ("full", "partial",
"not_found"):
visit_date = None
except Exception as exc:
sentry_sdk.capture_exception(exc)
return visit_date, visit_status
def test_api_lookup_origin_visits(api_client, archive_data, new_origin,
visit_dates, new_snapshots):
archive_data.origin_add([new_origin])
for i, visit_date in enumerate(visit_dates):
origin_visit = archive_data.origin_visit_add([
OriginVisit(
origin=new_origin.url,
date=visit_date,
type="git",
)
])[0]
archive_data.snapshot_add([new_snapshots[i]])
visit_status = OriginVisitStatus(
origin=new_origin.url,
visit=origin_visit.visit,
date=now(),
status="full",
snapshot=new_snapshots[i].id,
)
archive_data.origin_visit_status_add([visit_status])
all_visits = list(reversed(get_origin_visits(new_origin.to_dict())))
for last_visit, expected_visits in (
(None, all_visits[:2]),
(all_visits[1]["visit"], all_visits[2:]),
):
url = reverse(
"api-1-origin-visits",
url_args={"origin_url": new_origin.url},
query_params={
"per_page": 2,
"last_visit": last_visit
},
)
rv = check_api_get_responses(api_client, url, status_code=200)
for i in range(len(expected_visits)):
expected_visits[i] = enrich_origin_visit(
expected_visits[i],
with_origin_link=False,
with_origin_visit_link=True,
request=rv.wsgi_request,
)
assert rv.data == expected_visits
def _lookup_origin_visits(origin_id,
last_visit=last_visit,
per_page=per_page):
all_visits = get_origin_visits({'id': origin_id})
all_visits.reverse()
visits = []
if not last_visit:
visits = all_visits[:per_page]
else:
for i, v in enumerate(all_visits):
if v['visit'] == last_visit:
visits = all_visits[i + 1:i + 1 + per_page]
break
for v in visits:
yield v
def test_get_origin_visit_latest_snapshot(mocker, origin):
origin_visits = get_origin_visits(origin)
first_visit = origin_visits[0]
latest_visit = origin_visits[-1]
mock_get_origin_visits = mocker.patch(
"swh.web.common.origin_visits.get_origin_visits")
mock_get_origin_visits.return_value = origin_visits
visit = get_origin_visit(origin, snapshot_id=latest_visit["snapshot"])
assert visit == latest_visit
assert not mock_get_origin_visits.called
visit = get_origin_visit(origin, snapshot_id=first_visit["snapshot"])
assert visit == first_visit
assert mock_get_origin_visits.called
def origin_visits_browse(request, origin_url, origin_type=None):
"""Django view that produces an HTML display of visits reporting
for a swh origin identified by its id or its url.
The url that points to it is :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visits/`.
""" # noqa
try:
origin_info = get_origin_info(origin_url, origin_type)
origin_visits = get_origin_visits(origin_info)
snapshot_context = get_snapshot_context(origin_type=origin_type,
origin_url=origin_url)
except Exception as exc:
return handle_view_exception(request, exc)
for i, visit in enumerate(origin_visits):
url_date = format_utc_iso_date(visit['date'], '%Y-%m-%dT%H:%M:%SZ')
visit['fmt_date'] = format_utc_iso_date(visit['date'])
query_params = {}
if i < len(origin_visits) - 1:
if visit['date'] == origin_visits[i+1]['date']:
query_params = {'visit_id': visit['visit']}
if i > 0:
if visit['date'] == origin_visits[i-1]['date']:
query_params = {'visit_id': visit['visit']}
snapshot = visit['snapshot'] if visit['snapshot'] else ''
visit['browse_url'] = reverse('browse-origin-directory',
url_args={'origin_type': origin_type,
'origin_url': origin_url,
'timestamp': url_date},
query_params=query_params)
if not snapshot:
visit['snapshot'] = ''
visit['date'] = parse_timestamp(visit['date']).timestamp()
heading = 'Origin visits - %s' % origin_url
return render(request, 'browse/origin-visits.html',
{'heading': heading,
'swh_object_name': 'Visits',
'swh_object_metadata': origin_info,
'origin_visits': origin_visits,
'origin_info': origin_info,
'snapshot_context': snapshot_context,
'vault_cooking': None,
'show_actions_menu': False})
def _origin_visits_browse(request, origin_url):
if origin_url is None:
raise BadInputExc("An origin URL must be provided as query parameter.")
origin_info = archive.lookup_origin({"url": origin_url})
origin_visits = get_origin_visits(origin_info)
snapshot_context = get_snapshot_context(origin_url=origin_url)
for i, visit in enumerate(origin_visits):
url_date = format_utc_iso_date(visit["date"], "%Y-%m-%dT%H:%M:%SZ")
visit["formatted_date"] = format_utc_iso_date(visit["date"])
query_params = {"origin_url": origin_url, "timestamp": url_date}
if i < len(origin_visits) - 1:
if visit["date"] == origin_visits[i + 1]["date"]:
query_params = {"visit_id": visit["visit"]}
if i > 0:
if visit["date"] == origin_visits[i - 1]["date"]:
query_params = {"visit_id": visit["visit"]}
snapshot = visit["snapshot"] if visit["snapshot"] else ""
visit["url"] = reverse(
"browse-origin-directory",
query_params=query_params,
)
if not snapshot:
visit["snapshot"] = ""
visit["date"] = parse_iso8601_date_to_utc(visit["date"]).timestamp()
heading = "Origin visits - %s" % origin_url
return render(
request,
"browse/origin-visits.html",
{
"heading": heading,
"swh_object_name": "Visits",
"swh_object_metadata": origin_info,
"origin_visits": origin_visits,
"origin_info": origin_info,
"snapshot_context": snapshot_context,
"vault_cooking": None,
"show_actions": False,
},
)
def test_api_lookup_origin_visit_not_found(self, origin):
all_visits = list(reversed(get_origin_visits(origin)))
max_visit_id = max([v['visit'] for v in all_visits])
url = reverse('api-origin-visit',
url_args={'origin_id': origin['id'],
'visit_id': max_visit_id + 1})
rv = self.client.get(url)
self.assertEqual(rv.status_code, 404)
self.assertEqual(rv['Content-Type'], 'application/json')
self.assertEqual(rv.data, {
'exception': 'NotFoundExc',
'reason': 'Origin with id %s or its visit with id %s not found!' %
(origin['id'], max_visit_id+1)
})
def _get_visit_info_for_save_request(save_request):
visit_date = None
visit_status = None
try:
origin = {
'type': save_request.origin_type,
'url': save_request.origin_url
}
origin_info = service.lookup_origin(origin)
origin_visits = get_origin_visits(origin_info)
visit_dates = [parse_timestamp(v['date']) for v in origin_visits]
i = bisect_right(visit_dates, save_request.request_date)
if i != len(visit_dates):
visit_date = visit_dates[i]
visit_status = origin_visits[i]['status']
if origin_visits[i]['status'] == 'ongoing':
visit_date = None
except Exception:
pass
return visit_date, visit_status
def test_enrich_origin_visit(api_request_factory, origin):
origin_visit = random.choice(get_origin_visits(origin))
url = reverse(
"api-1-origin-visit",
url_args={
"origin_url": origin["url"],
"visit_id": origin_visit["visit"]
},
)
request = api_request_factory.get(url)
actual_origin_visit = utils.enrich_origin_visit(
origin_visit,
with_origin_link=True,
with_origin_visit_link=True,
request=request,
)
origin_visit["origin_url"] = reverse(
"api-1-origin",
url_args={"origin_url": origin["url"]},
request=request)
origin_visit["origin_visit_url"] = reverse(
"api-1-origin-visit",
url_args={
"origin_url": origin["url"],
"visit_id": origin_visit["visit"]
},
request=request,
)
origin_visit["snapshot_url"] = reverse(
"api-1-snapshot",
url_args={"snapshot_id": origin_visit["snapshot"]},
request=request,
)
assert actual_origin_visit == origin_visit
def test_get_origin_visits(mocker, snapshots):
mock_archive = mocker.patch("swh.web.common.archive")
mock_archive.MAX_LIMIT = 2
def _lookup_origin_visits(*args, **kwargs):
if kwargs["last_visit"] is None:
return [
{
"visit": 1,
"date": "2017-05-06T00:59:10+00:00",
"status": "full",
"snapshot": hash_to_hex(snapshots[0].id),
"type": "git",
},
{
"visit": 2,
"date": "2017-08-06T00:59:10+00:00",
"status": "full",
"snapshot": hash_to_hex(snapshots[1].id),
"type": "git",
},
]
else:
return [{
"visit": 3,
"date": "2017-09-06T00:59:10+00:00",
"status": "full",
"snapshot": hash_to_hex(snapshots[2].id),
"type": "git",
}]
mock_archive.lookup_origin_visits.side_effect = _lookup_origin_visits
origin_info = {
"url": "https://github.com/foo/bar",
}
origin_visits = get_origin_visits(origin_info)
assert len(origin_visits) == 3
def test_api_lookup_origin_visit_not_found(api_client, origin):
all_visits = list(reversed(get_origin_visits(origin)))
max_visit_id = max([v["visit"] for v in all_visits])
url = reverse(
"api-1-origin-visit",
url_args={
"origin_url": origin["url"],
"visit_id": max_visit_id + 1
},
)
rv = check_api_get_responses(api_client, url, status_code=404)
assert rv.data == {
"exception":
"NotFoundExc",
"reason":
"Origin %s or its visit with id %s not found!" %
(origin["url"], max_visit_id + 1),
}
def test_api_lookup_origin_visits(self, new_origin, visit_dates,
new_snapshots):
origin_id = self.storage.origin_add_one(new_origin)
new_origin['id'] = origin_id
for i, visit_date in enumerate(visit_dates):
origin_visit = self.storage.origin_visit_add(origin_id, visit_date)
self.storage.snapshot_add(origin_id, origin_visit['visit'],
new_snapshots[i])
all_visits = list(reversed(get_origin_visits(new_origin)))
for last_visit, expected_visits in (
(None, all_visits[:2]),
(all_visits[1]['visit'], all_visits[2:4])):
url = reverse('api-origin-visits',
url_args={'origin_id': origin_id},
query_params={'per_page': 2,
'last_visit': last_visit})
rv = self.client.get(url)
self.assertEqual(rv.status_code, 200)
self.assertEqual(rv['Content-Type'], 'application/json')
for expected_visit in expected_visits:
origin_visit_url = reverse(
'api-origin-visit',
url_args={'origin_id': origin_id,
'visit_id': expected_visit['visit']})
snapshot_url = reverse(
'api-snapshot',
url_args={'snapshot_id': expected_visit['snapshot']})
expected_visit['origin_visit_url'] = origin_visit_url
expected_visit['snapshot_url'] = snapshot_url
self.assertEqual(rv.data, expected_visits)
def test_get_snapshot_context_with_origin(archive_data, origin):
origin_visits = get_origin_visits(origin)
timestamp = format_utc_iso_date(origin_visits[0]["date"],
"%Y-%m-%dT%H:%M:%SZ")
visit_id = origin_visits[1]["visit"]
for browse_context, kwargs in (
("content", {
"origin_url": origin["url"],
"path": "/some/path"
}),
("directory", {
"origin_url": origin["url"]
}),
("log", {
"origin_url": origin["url"]
}),
(
"directory",
{
"origin_url": origin["url"],
"timestamp": timestamp,
},
),
(
"directory",
{
"origin_url": origin["url"],
"visit_id": visit_id,
},
),
):
visit_id = kwargs["visit_id"] if "visit_id" in kwargs else None
visit_ts = kwargs["timestamp"] if "timestamp" in kwargs else None
visit_info = get_origin_visit({"url": kwargs["origin_url"]},
visit_ts=visit_ts,
visit_id=visit_id)
snapshot = visit_info["snapshot"]
snapshot_context = get_snapshot_context(**kwargs,
browse_context=browse_context)
query_params = dict(kwargs)
branches, releases, _ = get_snapshot_content(snapshot)
releases = list(reversed(releases))
revision_id = None
root_directory = None
for branch in branches:
if branch["name"] == "HEAD":
revision_id = branch["revision"]
root_directory = branch["directory"]
branch["url"] = reverse(
f"browse-origin-{browse_context}",
query_params={
"branch": branch["name"],
**query_params
},
)
for release in releases:
release["url"] = reverse(
f"browse-origin-{browse_context}",
query_params={
"release": release["name"],
**query_params
},
)
query_params.pop("path", None)
branches_url = reverse("browse-origin-branches",
query_params=query_params)
releases_url = reverse("browse-origin-releases",
query_params=query_params)
origin_visits_url = reverse(
"browse-origin-visits",
query_params={"origin_url": kwargs["origin_url"]})
is_empty = not branches and not releases
snapshot_swhid = gen_swhid("snapshot", snapshot)
snapshot_sizes = archive_data.snapshot_count_branches(snapshot)
visit_info["url"] = reverse("browse-origin-directory",
query_params=query_params)
visit_info["formatted_date"] = format_utc_iso_date(visit_info["date"])
if "path" in kwargs:
query_params["path"] = kwargs["path"]
expected = SnapshotContext(
branch="HEAD",
branch_alias=True,
branches=branches,
branches_url=branches_url,
is_empty=is_empty,
origin_info={"url": origin["url"]},
origin_visits_url=origin_visits_url,
release=None,
release_alias=False,
release_id=None,
query_params=query_params,
releases=releases,
releases_url=releases_url,
revision_id=revision_id,
revision_info=_get_revision_info(archive_data, revision_id),
root_directory=root_directory,
snapshot_id=snapshot,
snapshot_sizes=snapshot_sizes,
snapshot_swhid=snapshot_swhid,
url_args={},
visit_info=visit_info,
)
if revision_id:
expected["revision_info"]["revision_url"] = gen_revision_url(
revision_id, snapshot_context)
assert snapshot_context == expected
_check_branch_release_revision_parameters(archive_data, expected,
browse_context, kwargs,
branches, releases)