def test_get_origin_visits(self, mock_service): mock_service.MAX_LIMIT = 2 def _lookup_origin_visits(*args, **kwargs): if kwargs['last_visit'] is None: return [{ 'visit': 1, 'date': '2017-05-06T00:59:10+00:00', 'metadata': {} }, { 'visit': 2, 'date': '2017-08-06T00:59:10+00:00', 'metadata': {} }] else: return [{ 'visit': 3, 'date': '2017-09-06T00:59:10+00:00', 'metadata': {} }] mock_service.lookup_origin_visits.side_effect = _lookup_origin_visits origin_info = { 'id': 1, 'type': 'git', 'url': 'https://github.com/foo/bar', } origin_visits = get_origin_visits(origin_info) self.assertEqual(len(origin_visits), 3)
def _get_visit_info_for_save_request(save_request): visit_date = None visit_status = None time_now = datetime.now(tz=timezone.utc) time_delta = time_now - save_request.request_date # stop trying to find a visit date one month after save request submission # as those requests to storage are expensive and associated loading task # surely ended up with errors if time_delta.days <= 30: try: origin = {"url": save_request.origin_url} origin_info = archive.lookup_origin(origin) origin_visits = get_origin_visits(origin_info) visit_dates = [ parse_iso8601_date_to_utc(v["date"]) for v in origin_visits ] i = bisect_right(visit_dates, save_request.request_date) if i != len(visit_dates): visit_date = visit_dates[i] visit_status = origin_visits[i]["status"] if origin_visits[i]["status"] not in ("full", "partial", "not_found"): visit_date = None except Exception as exc: sentry_sdk.capture_exception(exc) return visit_date, visit_status
def test_api_lookup_origin_visits(api_client, archive_data, new_origin, visit_dates, new_snapshots): archive_data.origin_add([new_origin]) for i, visit_date in enumerate(visit_dates): origin_visit = archive_data.origin_visit_add([ OriginVisit( origin=new_origin.url, date=visit_date, type="git", ) ])[0] archive_data.snapshot_add([new_snapshots[i]]) visit_status = OriginVisitStatus( origin=new_origin.url, visit=origin_visit.visit, date=now(), status="full", snapshot=new_snapshots[i].id, ) archive_data.origin_visit_status_add([visit_status]) all_visits = list(reversed(get_origin_visits(new_origin.to_dict()))) for last_visit, expected_visits in ( (None, all_visits[:2]), (all_visits[1]["visit"], all_visits[2:]), ): url = reverse( "api-1-origin-visits", url_args={"origin_url": new_origin.url}, query_params={ "per_page": 2, "last_visit": last_visit }, ) rv = check_api_get_responses(api_client, url, status_code=200) for i in range(len(expected_visits)): expected_visits[i] = enrich_origin_visit( expected_visits[i], with_origin_link=False, with_origin_visit_link=True, request=rv.wsgi_request, ) assert rv.data == expected_visits
def _lookup_origin_visits(origin_id, last_visit=last_visit, per_page=per_page): all_visits = get_origin_visits({'id': origin_id}) all_visits.reverse() visits = [] if not last_visit: visits = all_visits[:per_page] else: for i, v in enumerate(all_visits): if v['visit'] == last_visit: visits = all_visits[i + 1:i + 1 + per_page] break for v in visits: yield v
def test_get_origin_visit_latest_snapshot(mocker, origin): origin_visits = get_origin_visits(origin) first_visit = origin_visits[0] latest_visit = origin_visits[-1] mock_get_origin_visits = mocker.patch( "swh.web.common.origin_visits.get_origin_visits") mock_get_origin_visits.return_value = origin_visits visit = get_origin_visit(origin, snapshot_id=latest_visit["snapshot"]) assert visit == latest_visit assert not mock_get_origin_visits.called visit = get_origin_visit(origin, snapshot_id=first_visit["snapshot"]) assert visit == first_visit assert mock_get_origin_visits.called
def origin_visits_browse(request, origin_url, origin_type=None): """Django view that produces an HTML display of visits reporting for a swh origin identified by its id or its url. The url that points to it is :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visits/`. """ # noqa try: origin_info = get_origin_info(origin_url, origin_type) origin_visits = get_origin_visits(origin_info) snapshot_context = get_snapshot_context(origin_type=origin_type, origin_url=origin_url) except Exception as exc: return handle_view_exception(request, exc) for i, visit in enumerate(origin_visits): url_date = format_utc_iso_date(visit['date'], '%Y-%m-%dT%H:%M:%SZ') visit['fmt_date'] = format_utc_iso_date(visit['date']) query_params = {} if i < len(origin_visits) - 1: if visit['date'] == origin_visits[i+1]['date']: query_params = {'visit_id': visit['visit']} if i > 0: if visit['date'] == origin_visits[i-1]['date']: query_params = {'visit_id': visit['visit']} snapshot = visit['snapshot'] if visit['snapshot'] else '' visit['browse_url'] = reverse('browse-origin-directory', url_args={'origin_type': origin_type, 'origin_url': origin_url, 'timestamp': url_date}, query_params=query_params) if not snapshot: visit['snapshot'] = '' visit['date'] = parse_timestamp(visit['date']).timestamp() heading = 'Origin visits - %s' % origin_url return render(request, 'browse/origin-visits.html', {'heading': heading, 'swh_object_name': 'Visits', 'swh_object_metadata': origin_info, 'origin_visits': origin_visits, 'origin_info': origin_info, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': False})
def _origin_visits_browse(request, origin_url): if origin_url is None: raise BadInputExc("An origin URL must be provided as query parameter.") origin_info = archive.lookup_origin({"url": origin_url}) origin_visits = get_origin_visits(origin_info) snapshot_context = get_snapshot_context(origin_url=origin_url) for i, visit in enumerate(origin_visits): url_date = format_utc_iso_date(visit["date"], "%Y-%m-%dT%H:%M:%SZ") visit["formatted_date"] = format_utc_iso_date(visit["date"]) query_params = {"origin_url": origin_url, "timestamp": url_date} if i < len(origin_visits) - 1: if visit["date"] == origin_visits[i + 1]["date"]: query_params = {"visit_id": visit["visit"]} if i > 0: if visit["date"] == origin_visits[i - 1]["date"]: query_params = {"visit_id": visit["visit"]} snapshot = visit["snapshot"] if visit["snapshot"] else "" visit["url"] = reverse( "browse-origin-directory", query_params=query_params, ) if not snapshot: visit["snapshot"] = "" visit["date"] = parse_iso8601_date_to_utc(visit["date"]).timestamp() heading = "Origin visits - %s" % origin_url return render( request, "browse/origin-visits.html", { "heading": heading, "swh_object_name": "Visits", "swh_object_metadata": origin_info, "origin_visits": origin_visits, "origin_info": origin_info, "snapshot_context": snapshot_context, "vault_cooking": None, "show_actions": False, }, )
def test_api_lookup_origin_visit_not_found(self, origin): all_visits = list(reversed(get_origin_visits(origin))) max_visit_id = max([v['visit'] for v in all_visits]) url = reverse('api-origin-visit', url_args={'origin_id': origin['id'], 'visit_id': max_visit_id + 1}) rv = self.client.get(url) self.assertEqual(rv.status_code, 404) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'NotFoundExc', 'reason': 'Origin with id %s or its visit with id %s not found!' % (origin['id'], max_visit_id+1) })
def _get_visit_info_for_save_request(save_request): visit_date = None visit_status = None try: origin = { 'type': save_request.origin_type, 'url': save_request.origin_url } origin_info = service.lookup_origin(origin) origin_visits = get_origin_visits(origin_info) visit_dates = [parse_timestamp(v['date']) for v in origin_visits] i = bisect_right(visit_dates, save_request.request_date) if i != len(visit_dates): visit_date = visit_dates[i] visit_status = origin_visits[i]['status'] if origin_visits[i]['status'] == 'ongoing': visit_date = None except Exception: pass return visit_date, visit_status
def test_enrich_origin_visit(api_request_factory, origin): origin_visit = random.choice(get_origin_visits(origin)) url = reverse( "api-1-origin-visit", url_args={ "origin_url": origin["url"], "visit_id": origin_visit["visit"] }, ) request = api_request_factory.get(url) actual_origin_visit = utils.enrich_origin_visit( origin_visit, with_origin_link=True, with_origin_visit_link=True, request=request, ) origin_visit["origin_url"] = reverse( "api-1-origin", url_args={"origin_url": origin["url"]}, request=request) origin_visit["origin_visit_url"] = reverse( "api-1-origin-visit", url_args={ "origin_url": origin["url"], "visit_id": origin_visit["visit"] }, request=request, ) origin_visit["snapshot_url"] = reverse( "api-1-snapshot", url_args={"snapshot_id": origin_visit["snapshot"]}, request=request, ) assert actual_origin_visit == origin_visit
def test_get_origin_visits(mocker, snapshots): mock_archive = mocker.patch("swh.web.common.archive") mock_archive.MAX_LIMIT = 2 def _lookup_origin_visits(*args, **kwargs): if kwargs["last_visit"] is None: return [ { "visit": 1, "date": "2017-05-06T00:59:10+00:00", "status": "full", "snapshot": hash_to_hex(snapshots[0].id), "type": "git", }, { "visit": 2, "date": "2017-08-06T00:59:10+00:00", "status": "full", "snapshot": hash_to_hex(snapshots[1].id), "type": "git", }, ] else: return [{ "visit": 3, "date": "2017-09-06T00:59:10+00:00", "status": "full", "snapshot": hash_to_hex(snapshots[2].id), "type": "git", }] mock_archive.lookup_origin_visits.side_effect = _lookup_origin_visits origin_info = { "url": "https://github.com/foo/bar", } origin_visits = get_origin_visits(origin_info) assert len(origin_visits) == 3
def test_api_lookup_origin_visit_not_found(api_client, origin): all_visits = list(reversed(get_origin_visits(origin))) max_visit_id = max([v["visit"] for v in all_visits]) url = reverse( "api-1-origin-visit", url_args={ "origin_url": origin["url"], "visit_id": max_visit_id + 1 }, ) rv = check_api_get_responses(api_client, url, status_code=404) assert rv.data == { "exception": "NotFoundExc", "reason": "Origin %s or its visit with id %s not found!" % (origin["url"], max_visit_id + 1), }
def test_api_lookup_origin_visits(self, new_origin, visit_dates, new_snapshots): origin_id = self.storage.origin_add_one(new_origin) new_origin['id'] = origin_id for i, visit_date in enumerate(visit_dates): origin_visit = self.storage.origin_visit_add(origin_id, visit_date) self.storage.snapshot_add(origin_id, origin_visit['visit'], new_snapshots[i]) all_visits = list(reversed(get_origin_visits(new_origin))) for last_visit, expected_visits in ( (None, all_visits[:2]), (all_visits[1]['visit'], all_visits[2:4])): url = reverse('api-origin-visits', url_args={'origin_id': origin_id}, query_params={'per_page': 2, 'last_visit': last_visit}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200) self.assertEqual(rv['Content-Type'], 'application/json') for expected_visit in expected_visits: origin_visit_url = reverse( 'api-origin-visit', url_args={'origin_id': origin_id, 'visit_id': expected_visit['visit']}) snapshot_url = reverse( 'api-snapshot', url_args={'snapshot_id': expected_visit['snapshot']}) expected_visit['origin_visit_url'] = origin_visit_url expected_visit['snapshot_url'] = snapshot_url self.assertEqual(rv.data, expected_visits)
def test_get_snapshot_context_with_origin(archive_data, origin): origin_visits = get_origin_visits(origin) timestamp = format_utc_iso_date(origin_visits[0]["date"], "%Y-%m-%dT%H:%M:%SZ") visit_id = origin_visits[1]["visit"] for browse_context, kwargs in ( ("content", { "origin_url": origin["url"], "path": "/some/path" }), ("directory", { "origin_url": origin["url"] }), ("log", { "origin_url": origin["url"] }), ( "directory", { "origin_url": origin["url"], "timestamp": timestamp, }, ), ( "directory", { "origin_url": origin["url"], "visit_id": visit_id, }, ), ): visit_id = kwargs["visit_id"] if "visit_id" in kwargs else None visit_ts = kwargs["timestamp"] if "timestamp" in kwargs else None visit_info = get_origin_visit({"url": kwargs["origin_url"]}, visit_ts=visit_ts, visit_id=visit_id) snapshot = visit_info["snapshot"] snapshot_context = get_snapshot_context(**kwargs, browse_context=browse_context) query_params = dict(kwargs) branches, releases, _ = get_snapshot_content(snapshot) releases = list(reversed(releases)) revision_id = None root_directory = None for branch in branches: if branch["name"] == "HEAD": revision_id = branch["revision"] root_directory = branch["directory"] branch["url"] = reverse( f"browse-origin-{browse_context}", query_params={ "branch": branch["name"], **query_params }, ) for release in releases: release["url"] = reverse( f"browse-origin-{browse_context}", query_params={ "release": release["name"], **query_params }, ) query_params.pop("path", None) branches_url = reverse("browse-origin-branches", query_params=query_params) releases_url = reverse("browse-origin-releases", query_params=query_params) origin_visits_url = reverse( "browse-origin-visits", query_params={"origin_url": kwargs["origin_url"]}) is_empty = not branches and not releases snapshot_swhid = gen_swhid("snapshot", snapshot) snapshot_sizes = archive_data.snapshot_count_branches(snapshot) visit_info["url"] = reverse("browse-origin-directory", query_params=query_params) visit_info["formatted_date"] = format_utc_iso_date(visit_info["date"]) if "path" in kwargs: query_params["path"] = kwargs["path"] expected = SnapshotContext( branch="HEAD", branch_alias=True, branches=branches, branches_url=branches_url, is_empty=is_empty, origin_info={"url": origin["url"]}, origin_visits_url=origin_visits_url, release=None, release_alias=False, release_id=None, query_params=query_params, releases=releases, releases_url=releases_url, revision_id=revision_id, revision_info=_get_revision_info(archive_data, revision_id), root_directory=root_directory, snapshot_id=snapshot, snapshot_sizes=snapshot_sizes, snapshot_swhid=snapshot_swhid, url_args={}, visit_info=visit_info, ) if revision_id: expected["revision_info"]["revision_url"] = gen_revision_url( revision_id, snapshot_context) assert snapshot_context == expected _check_branch_release_revision_parameters(archive_data, expected, browse_context, kwargs, branches, releases)