def db_to_raw_extrinsic_metadata(row) -> RawExtrinsicMetadata:
target = row["raw_extrinsic_metadata.target"]
if not target.startswith("swh:1:"):
warnings.warn("Fetching raw_extrinsic_metadata row with URL target",
DeprecationWarning)
target = str(Origin(url=target).swhid())
return RawExtrinsicMetadata(
target=ExtendedSWHID.from_string(target),
authority=MetadataAuthority(
type=MetadataAuthorityType(row["metadata_authority.type"]),
url=row["metadata_authority.url"],
),
fetcher=MetadataFetcher(
name=row["metadata_fetcher.name"],
version=row["metadata_fetcher.version"],
),
discovery_date=row["discovery_date"],
format=row["format"],
metadata=row["raw_extrinsic_metadata.metadata"],
origin=row["origin"],
visit=row["visit"],
snapshot=map_optional(CoreSWHID.from_string, row["snapshot"]),
release=map_optional(CoreSWHID.from_string, row["release"]),
revision=map_optional(CoreSWHID.from_string, row["revision"]),
path=row["path"],
directory=map_optional(CoreSWHID.from_string, row["directory"]),
)
def test_db_to_raw_extrinsic_metadata_raw_target():
row = {
"raw_extrinsic_metadata.target": "https://example.com/origin",
"metadata_authority.type": "forge",
"metadata_authority.url": "https://example.com",
"metadata_fetcher.name": "swh.lister",
"metadata_fetcher.version": "1.0.0",
"discovery_date": datetime.datetime(
2021, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc
),
"format": "text/plain",
"raw_extrinsic_metadata.metadata": b"metadata",
"origin": None,
"visit": None,
"snapshot": None,
"release": None,
"revision": None,
"path": None,
"directory": None,
}
with pytest.deprecated_call():
computed_rem = converters.db_to_raw_extrinsic_metadata(row)
assert computed_rem.target == ExtendedSWHID.from_string(
"swh:1:ori:5a7439b0b93a5d230b6a67b8e7e0f7dc3c9f6c70"
)
EMPTY_SNAPSHOT_ID = "1a8893e6a86f444e8be8e7bda6cb34fb1735a00e"
FULL_SNAPSHOT_ID = "4ac5730a9393f5099b63a35a17b6c33d36d70c3a"
AUTHORITY = MetadataAuthority(
type=MetadataAuthorityType.FORGE,
url="http://example.org/",
)
ORIGIN_URL = "http://example.org/archive.tgz"
ORIGIN_SWHID = Origin(ORIGIN_URL).swhid()
REVISION_ID = hash_to_bytes("8ff44f081d43176474b267de5451f2c2e88089d0")
RELEASE_ID = hash_to_bytes("9477a708196b44e59efb4e47b7d979a4146bd428")
RELEASE_SWHID = CoreSWHID.from_string(f"swh:1:rel:{RELEASE_ID.hex()}")
DIRECTORY_ID = hash_to_bytes("aa" * 20)
DIRECTORY_SWHID = ExtendedSWHID.from_string(f"swh:1:dir:{DIRECTORY_ID.hex()}")
FETCHER = MetadataFetcher(
name="swh.loader.package.tests.test_loader_metadata.MetadataTestLoader",
version=__version__,
)
DISCOVERY_DATE = datetime.datetime.now(tz=datetime.timezone.utc)
DIRECTORY_METADATA = [
RawExtrinsicMetadata(
target=DIRECTORY_SWHID,
discovery_date=DISCOVERY_DATE,
authority=AUTHORITY,
fetcher=FETCHER,
format="test-format1",
def test_put_update_metadata_done_deposit_nominal(
tmp_path,
authenticated_client,
complete_deposit,
deposit_collection,
atom_dataset,
sample_data,
swh_storage,
):
"""Nominal scenario, client send an update of metadata on a deposit with status "done"
with an existing swhid. Such swhid has its metadata updated accordingly both in
the deposit backend and in the metadata storage.
Response: 204
"""
deposit_swhid = CoreSWHID.from_string(complete_deposit.swhid)
assert deposit_swhid.object_type == ObjectType.DIRECTORY
directory_id = hash_to_bytes(deposit_swhid.object_id)
# directory targeted by the complete_deposit does not exist in the storage
assert list(swh_storage.directory_missing([directory_id
])) == [directory_id]
# so let's create a directory reference in the storage (current deposit targets an
# unknown swhid)
existing_directory = sample_data.directory
swh_storage.directory_add([existing_directory])
assert list(swh_storage.directory_missing([existing_directory.id])) == []
# and patch one complete deposit swhid so it targets said reference
complete_deposit.swhid = str(existing_directory.swhid())
complete_deposit.save()
actual_existing_requests_archive = DepositRequest.objects.filter(
deposit=complete_deposit, type="archive")
nb_archives = len(actual_existing_requests_archive)
actual_existing_requests_metadata = DepositRequest.objects.filter(
deposit=complete_deposit, type="metadata")
nb_metadata = len(actual_existing_requests_metadata)
update_uri = reverse(EDIT_IRI,
args=[deposit_collection.name, complete_deposit.id])
response = put_atom(
authenticated_client,
update_uri,
data=atom_dataset["entry-data1"],
HTTP_X_CHECK_SWHID=complete_deposit.swhid,
)
assert response.status_code == status.HTTP_204_NO_CONTENT
new_requests_meta = DepositRequest.objects.filter(deposit=complete_deposit,
type="metadata")
assert len(new_requests_meta) == nb_metadata + 1
request_meta1 = new_requests_meta[0]
raw_metadata1 = request_meta1.raw_metadata
assert raw_metadata1 == atom_dataset["entry-data1"]
# check we did not touch the other parts
requests_archive1 = DepositRequest.objects.filter(deposit=complete_deposit,
type="archive")
assert len(requests_archive1) == nb_archives
assert set(actual_existing_requests_archive) == set(requests_archive1)
# Ensure metadata stored in the metadata storage is consistent
metadata_authority = MetadataAuthority(
type=MetadataAuthorityType.DEPOSIT_CLIENT,
url=complete_deposit.client.provider_url,
)
actual_authority = swh_storage.metadata_authority_get(
MetadataAuthorityType.DEPOSIT_CLIENT,
url=complete_deposit.client.provider_url)
assert actual_authority == metadata_authority
config = APIConfig()
metadata_fetcher = MetadataFetcher(
name=config.tool["name"],
version=config.tool["version"],
)
actual_fetcher = swh_storage.metadata_fetcher_get(config.tool["name"],
config.tool["version"])
assert actual_fetcher == metadata_fetcher
directory_swhid = ExtendedSWHID.from_string(complete_deposit.swhid)
page_results = swh_storage.raw_extrinsic_metadata_get(
directory_swhid, metadata_authority)
assert page_results == PagedResult(
results=[
RawExtrinsicMetadata(
target=directory_swhid,
discovery_date=request_meta1.date,
authority=metadata_authority,
fetcher=metadata_fetcher,
format="sword-v2-atom-codemeta",
metadata=raw_metadata1.encode(),
origin=complete_deposit.origin_url,
)
],
next_page_token=None,
)
def extended_swhid_from_qualified(swhid: QualifiedSWHID) -> ExtendedSWHID:
"""Used to get the target of a metadata object from a <swh:reference>,
as the latter uses a QualifiedSWHID."""
return ExtendedSWHID.from_string(str(swhid).split(";")[0])
version="1.0.0",
metadata={},
)
]
RAW_EXTRINSIC_METADATA = [
RawExtrinsicMetadata(
target=Origin("http://example.org/foo.git").swhid(),
discovery_date=datetime.datetime(2020, 7, 30, 17, 8, 20, tzinfo=UTC),
authority=attr.evolve(METADATA_AUTHORITIES[0], metadata=None),
fetcher=attr.evolve(METADATA_FETCHERS[0], metadata=None),
format="json",
metadata=b'{"foo": "bar"}',
),
RawExtrinsicMetadata(
target=ExtendedSWHID.from_string(str(CONTENTS[0].swhid())),
discovery_date=datetime.datetime(2020, 7, 30, 17, 8, 20, tzinfo=UTC),
authority=attr.evolve(METADATA_AUTHORITIES[0], metadata=None),
fetcher=attr.evolve(METADATA_FETCHERS[0], metadata=None),
format="json",
metadata=b'{"foo": "bar"}',
),
]
TEST_OBJECTS: Dict[str, Sequence[BaseModel]] = {
"content": CONTENTS,
"directory": DIRECTORIES,
"extid": EXTIDS,
"metadata_authority": METADATA_AUTHORITIES,
"metadata_fetcher": METADATA_FETCHERS,
"origin": ORIGINS,