def build_release(
self,
p_info: DebianPackageInfo,
uncompressed_path: str,
directory: Sha1Git,
) -> Optional[Release]:
dsc_url, dsc_name = dsc_information(p_info)
if not dsc_name:
raise ValueError("dsc name for url %s should not be None" %
dsc_url)
dsc_path = path.join(path.dirname(uncompressed_path), dsc_name)
intrinsic_metadata = get_intrinsic_package_metadata(
p_info, dsc_path, uncompressed_path)
logger.debug("intrinsic_metadata: %s", intrinsic_metadata)
logger.debug("p_info: %s", p_info)
msg = (f"Synthetic release for Debian source package {p_info.name} "
f"version {p_info.intrinsic_version}\n")
author = prepare_person(intrinsic_metadata.changelog.person)
date = TimestampWithTimezone.from_iso8601(
intrinsic_metadata.changelog.date)
# inspired from swh.loader.debian.converters.package_metadata_to_revision
return Release(
name=p_info.intrinsic_version.encode(),
message=msg.encode(),
author=author,
date=date,
target=directory,
target_type=ObjectType.DIRECTORY,
synthetic=True,
)
def build_release(self, p_info: PyPIPackageInfo, uncompressed_path: str,
directory: Sha1Git) -> Optional[Release]:
i_metadata = extract_intrinsic_metadata(uncompressed_path)
if not i_metadata:
return None
# from intrinsic metadata
version_ = i_metadata.get("version", p_info.version)
author_ = author(i_metadata)
if p_info.comment_text:
msg = p_info.comment_text
else:
msg = (f"Synthetic release for PyPI source package {p_info.name} "
f"version {version_}\n")
date = TimestampWithTimezone.from_iso8601(p_info.upload_time)
return Release(
name=p_info.version.encode(),
message=msg.encode(),
author=author_,
date=date,
target=directory,
target_type=ObjectType.DIRECTORY,
synthetic=True,
)
def build_release(self, p_info: ArchPackageInfo, uncompressed_path: str,
directory: Sha1Git) -> Optional[Release]:
intrinsic_metadata = extract_intrinsic_metadata(
Path(uncompressed_path))
author = Person.from_fullname(intrinsic_metadata["packager"].encode())
description = intrinsic_metadata["pkgdesc"]
message = (
f"Synthetic release for Arch Linux source package {p_info.name} "
f"version {p_info.version}\n\n"
f"{description}\n")
return Release(
name=p_info.version.encode(),
author=author,
date=TimestampWithTimezone.from_iso8601(p_info.last_modified),
message=message.encode(),
target_type=ObjectType.DIRECTORY,
target=directory,
synthetic=True,
)
def build_release(self, p_info: NpmPackageInfo, uncompressed_path: str,
directory: Sha1Git) -> Optional[Release]:
# Metadata from NPM is not intrinsic to tarballs.
# This means two package versions can have the same tarball, but different
# metadata. To avoid mixing up releases, every field used to build the
# release object must be part of NpmPackageInfo.MANIFEST_FORMAT.
i_metadata = extract_intrinsic_metadata(uncompressed_path)
if not i_metadata:
return None
author = extract_npm_package_author(i_metadata)
assert self.package_name == p_info.package_name
msg = (
f"Synthetic release for NPM source package {p_info.package_name} "
f"version {p_info.version}\n")
if p_info.date is None:
url = p_info.url
artifact_name = os.path.basename(url)
raise ValueError(
"Origin %s: Cannot determine upload time for artifact %s." %
(p_info.url, artifact_name))
date = TimestampWithTimezone.from_iso8601(p_info.date)
# FIXME: this is to remain bug-compatible with earlier versions:
date = attr.evolve(date,
timestamp=attr.evolve(date.timestamp,
microseconds=0))
r = Release(
name=p_info.version.encode(),
message=msg.encode(),
author=author,
date=date,
target=directory,
target_type=ObjectType.DIRECTORY,
synthetic=True,
)
return r
def build_release(self, p_info: CratesPackageInfo, uncompressed_path: str,
directory: Sha1Git) -> Optional[Release]:
# Extract intrinsic metadata from dir_path/Cargo.toml
name = p_info.name
version = p_info.version
dir_path = Path(uncompressed_path, f"{name}-{version}")
i_metadata_raw = extract_intrinsic_metadata(dir_path)
# Get only corresponding key of IntrinsicPackageMetadata
i_metadata_keys = [
k for k in IntrinsicPackageMetadata.__annotations__.keys()
]
# We use data only from "package" entry
i_metadata = {
k: v
for k, v in i_metadata_raw["package"].items()
if k in i_metadata_keys
}
p_info.i_metadata = IntrinsicPackageMetadata(
**i_metadata) # type: ignore[misc]
author = extract_author(p_info)
description = extract_description(p_info)
message = (f"Synthetic release for Crate source package {p_info.name} "
f"version {p_info.version}\n\n"
f"{description}\n")
# The only way to get a value for updated_at is through extrinsic metadata
updated_at = p_info.e_metadata_version.get("updated_at")
return Release(
name=version.encode(),
author=author,
date=TimestampWithTimezone.from_iso8601(updated_at),
message=message.encode(),
target_type=ObjectType.DIRECTORY,
target=directory,
synthetic=True,
)
def test_arch_loader_load_one_version(datadir, requests_mock_datadir,
swh_storage):
loader = ArchLoader(
swh_storage,
url=EXPECTED_PACKAGES[1]["url"],
artifacts=EXPECTED_PACKAGES[1]["artifacts"],
)
actual_load_status = loader.load()
assert actual_load_status["status"] == "eventful"
assert actual_load_status["snapshot_id"] is not None
expected_snapshot_id = "4020d0a278027550e336b5481a4159a913c91aa4"
expected_release_id = "7681098c9e381f9cc8bd1724d57eeee2182982dc"
assert expected_snapshot_id == actual_load_status["snapshot_id"]
expected_snapshot = Snapshot(
id=hash_to_bytes(actual_load_status["snapshot_id"]),
branches={
b"releases/1.12-1/gzip-1.12-1-aarch64.pkg.tar.xz":
SnapshotBranch(
target=hash_to_bytes(expected_release_id),
target_type=TargetType.RELEASE,
),
b"HEAD":
SnapshotBranch(
target=b"releases/1.12-1/gzip-1.12-1-aarch64.pkg.tar.xz",
target_type=TargetType.ALIAS,
),
},
)
check_snapshot(expected_snapshot, swh_storage)
stats = get_stats(swh_storage)
assert {
"content": 1,
"directory": 1,
"origin": 1,
"origin_visit": 1,
"release": 1,
"revision": 0,
"skipped_content": 0,
"snapshot": 1,
} == stats
assert swh_storage.release_get([
hash_to_bytes(expected_release_id)
])[0] == Release(
name=b"1.12-1",
message=b"Synthetic release for Arch Linux source package gzip version "
b"1.12-1\n\nGNU compression utility\n",
target=hash_to_bytes("bd742aaf422953a1f7a5e084ec4a7477491d63fb"),
target_type=ObjectType.DIRECTORY,
synthetic=True,
author=Person.from_fullname(
b"Arch Linux ARM Build System <*****@*****.**>"),
date=TimestampWithTimezone.from_iso8601("2022-04-07T21:08:14+00:00"),
id=hash_to_bytes(expected_release_id),
)
assert_last_visit_matches(
swh_storage,
url=EXPECTED_PACKAGES[1]["url"],
status="full",
type="arch",
snapshot=expected_snapshot.id,
)