def test_nixguix(): extrinsic_metadata = { "url": "https://files.pythonhosted.org/packages/source/a/alerta/alerta-7.4.5.tar.gz", "integrity": "sha256-km8RAaG1ep+tYR8eHVr3UWk+/MNEqdsBr1Di/g02LYQ=", } original_artifacts = [{ "length": 34903, "filename": "alerta-7.4.5.tar.gz", "checksums": { "sha1": "66db4398b664de272fd5aa6610caa776b5e64651", "sha256": "926f1101a1b57a9fad611f1e1d5af751693efcc344a9db01af50e2fe0d362d84", }, }] row = { "id": b"\x00\x01\xbaM\xd0S\x94\x85\x02\x11\xd7\xb3\x85M\x99\x13\xd2:\xe3y", "directory": DIRECTORY_ID, "date": None, "committer_date": None, "type": "tar", "message": b"", "metadata": { "extrinsic": { "raw": extrinsic_metadata, "when": "2020-06-03T11:25:05.259341+00:00", "provider": "https://nix-community.github.io/nixpkgs-swh/sources-unstable.json", }, "original_artifact": original_artifacts, }, } origin_url = "https://nix-community.github.io/nixpkgs-swh/sources-unstable.json" storage = Mock() def origin_get(urls): assert urls == [origin_url] return [Origin(url=origin_url)] storage.origin_get.side_effect = origin_get deposit_cur = None handle_row(row, storage, deposit_cur, dry_run=False) assert storage.method_calls == [ call.origin_get([origin_url]), call.raw_extrinsic_metadata_add([ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime(2020, 6, 3, 11, 25, 5, 259341, tzinfo=datetime.timezone.utc), authority=NIX_UNSTABLE_AUTHORITY, fetcher=FETCHER, format="nixguix-sources-json", metadata=json.dumps(extrinsic_metadata).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:0001ba4dd05394850211d7b3854d9913d23ae379"), ), ]), call.raw_extrinsic_metadata_add([ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime(2020, 6, 3, 11, 25, 5, 259341, tzinfo=datetime.timezone.utc), authority=SWH_AUTHORITY, fetcher=FETCHER, format="original-artifacts-json", metadata=json.dumps(original_artifacts).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:0001ba4dd05394850211d7b3854d9913d23ae379"), ), ]), ]
def test_debian_without_extrinsic(): source_original_artifacts = [ { "name": "pymongo_1.10-1.dsc", "sha1": "81877c1ae4406c2519b9cc9c4557cf6b0775a241", "length": 99, "sha256": "40269a73f38ee4c2f9cc021f1d5d091cc59ca6e778c339684b7be030e29e282f", "sha1_git": "0ac7bdb8e4d10926c5d3e51baa2be7bb29a3966b", }, { "name": "pymongo_1.10.orig.tar.gz", "sha1": "4f4c97641b86ac8f21396281bd1a7369236693c3", "length": 99, "sha256": "0b6bffb310782ffaeb3916c75790742ec5830c63a758fc711cd1f557eb5a4b5f", "sha1_git": "19ef0adda8868520d1ef9d4164b3ace4df1d62ad", }, { "name": "pymongo_1.10-1.debian.tar.gz", "sha1": "fbf378296613c8d55e043aec98896b3e50a94971", "length": 99, "sha256": "3970cc70fe3ba6499a9c56ba4b4c6c3782f56433d0d17d72b7a0e2ceae31b513", "sha1_git": "2eea9904806050a8fda95edd5d4fa60d29c1fdec", }, ] dest_original_artifacts = [ { "length": 99, "filename": "pymongo_1.10-1.dsc", "checksums": { "sha1": "81877c1ae4406c2519b9cc9c4557cf6b0775a241", "sha256": "40269a73f38ee4c2f9cc021f1d5d091cc59ca6e778c339684b7be030e29e282f", "sha1_git": "0ac7bdb8e4d10926c5d3e51baa2be7bb29a3966b", }, }, { "length": 99, "filename": "pymongo_1.10.orig.tar.gz", "checksums": { "sha1": "4f4c97641b86ac8f21396281bd1a7369236693c3", "sha256": "0b6bffb310782ffaeb3916c75790742ec5830c63a758fc711cd1f557eb5a4b5f", "sha1_git": "19ef0adda8868520d1ef9d4164b3ace4df1d62ad", }, }, { "length": 99, "filename": "pymongo_1.10-1.debian.tar.gz", "checksums": { "sha1": "fbf378296613c8d55e043aec98896b3e50a94971", "sha256": "3970cc70fe3ba6499a9c56ba4b4c6c3782f56433d0d17d72b7a0e2ceae31b513", "sha1_git": "2eea9904806050a8fda95edd5d4fa60d29c1fdec", }, }, ] row = { "id": b"\x00\x00\x01\xc2\x8c\x8f\xca\x01\xb9\x04\xde\x92\xa2d\n\x86l\xe0<\xb7", "directory": DIRECTORY_ID, "date": datetime.datetime(2011, 3, 31, 20, 17, 41, tzinfo=datetime.timezone.utc), "date_offset": 0, "type": "dsc", "message": b"Synthetic revision for Debian source package pymongo version 1.10-1", "metadata": { "package_info": { "name": "pymongo", "version": "1.10-1", "changelog": { # ... }, "maintainers": [ { "name": "Federico Ceratto", "email": "*****@*****.**" }, { "name": "Janos Guljas", "email": "*****@*****.**" }, ], "pgp_signature": { "date": "2011-03-31T21:02:44+00:00", "keyid": "2BABC6254E66E7B8450AC3E1E6AA90171392B174", "person": { "name": "David Paleino", "email": "*****@*****.**" }, }, "lister_metadata": { "id": 244296, "lister": "snapshot.debian.org" }, }, "original_artifact": source_original_artifacts, }, } storage = Mock() origin_url = "http://snapshot.debian.org/package/pymongo" deposit_cur = None with patch("debian_origins_from_row", return_value=[origin_url]): handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) assert storage.method_calls == [ call.raw_extrinsic_metadata_add([ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime(2011, 3, 31, 20, 17, 41, tzinfo=datetime.timezone.utc), authority=SWH_AUTHORITY, fetcher=FETCHER, format="original-artifacts-json", metadata=json.dumps(dest_original_artifacts).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:000001c28c8fca01b904de92a2640a866ce03cb7"), ), ]) ]
def test_gnu(): original_artifacts = [{ "length": 842501, "filename": "gperf-3.0.1.tar.gz", "checksums": { "sha1": "c4453ee492032b369006ee464f4dd4e2c0c0e650", "sha256": "5be283ef62e1bd26abdaaf88b416dbea4b14c360b09befcda2f055656dc43f87", "sha1_git": "bf1d5bb57d571101dd7b6acab2b78ae11bb861de", "blake2s256": "661f84afeb1e0b914defe2b249d424af1dfe380a96016b3282ae758c70e19a70", }, }] row = { "id": b"\x00\x1cqE\x8e@[%\xba\xcc\xc8\x0b\x99\xf6cM\xff\x9d+\x18", "directory": DIRECTORY_ID, "date": datetime.datetime(2003, 6, 13, 0, 11, tzinfo=datetime.timezone.utc), "committer_date": datetime.datetime(2003, 6, 13, 0, 11, tzinfo=datetime.timezone.utc), "type": "tar", "message": b"swh-loader-package: synthetic revision message", "metadata": { "extrinsic": { "raw": { "url": "https://ftp.gnu.org/gnu/gperf/gperf-3.0.1.tar.gz", "time": "2003-06-13T00:11:00+00:00", "length": 842501, "version": "3.0.1", "filename": "gperf-3.0.1.tar.gz", }, "when": "2019-11-27T11:17:38.318997+00:00", "provider": "https://ftp.gnu.org/gnu/gperf/", }, "intrinsic": {}, "original_artifact": original_artifacts, }, } origin_url = "https://ftp.gnu.org/gnu/gperf/" storage = Mock() def origin_get(urls): assert urls == [origin_url] return [Origin(url=origin_url)] storage.origin_get.side_effect = origin_get deposit_cur = None handle_row(row, storage, deposit_cur, dry_run=False) assert storage.method_calls == [ call.origin_get([origin_url]), call.raw_extrinsic_metadata_add([ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime(2019, 11, 27, 11, 17, 38, 318997, tzinfo=datetime.timezone.utc), authority=SWH_AUTHORITY, fetcher=FETCHER, format="original-artifacts-json", metadata=json.dumps(original_artifacts).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:001c71458e405b25baccc80b99f6634dff9d2b18"), ), ]), ]
def test_npm_1(): """Tests loading a revision generated by a new NPM loader that has a provider.""" extrinsic_metadata = { "_id": "@l3ilkojr/[email protected]", "dist": { "shasum": "b7f0d66090e0285f4e95d082d39bcb0c1b8f4ec8", "tarball": "https://registry.npmjs.org/@l3ilkojr/jdinsults/-/jdinsults-3.0.0.tgz", "fileCount": 4, "integrity": "sha512-qpv8Zg51g0l51VjODEooMUGSGanGUuQpzX5msfR7ZzbgTsgPbpDNyTIsQ0wQzI9RzCCUjS84Ii2VhMISEQcEUA==", "unpackedSize": 1583, "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJeUMS5CRA9TVsSAnZWagAAXpgP/0YgNOWN0U/Fz2RGeQhR\nVIKPvfGqZ2UfFxxUXWIc4QHvwyLCNUedCctpVdqnqmGJ9m/hj3K2zbRPD7Tm\n3nPl0HfzE7v3T8TDZfGhzW3c9mWxig+syr+sjo0EKyAgZVJ0mxbjOl4KHt+U\nQEwl/4falBsyYtK/pkCXWmmuC606QmPn/c6ZRD1Fw4vJjT9i5qi1KaBkIf6M\nnFmpOFxTcwxGGltOk3s3TKDtr8CIeWmdm3VkgsP2ErkPKAOcu12AT4/5tkg0\nDU+m1XmJb67rskb4Ncjvic/VutnPkEfNrk1IRXrmjDZBQbHtCJ7hd5ETmb9S\nE5WmMV8cpaGiW7AZvGTmkn5WETwQQU7po914zYiMg9+ozdwc7yC8cpGj/UoF\niKxsc1uxdfwWk/p3dShegEYM7sveloIXYsPaxbd84WRIfnwkWFZV82op96E3\neX+FRkhMfsHlK8OjZsBPXkppaB48jnZdm3GOOzT9YgyphV33j3J9GnNcDMDe\nriyCLV1BNSKDHElCDrvl1cBGg+C5qn/cTYjQdfEPPY2Hl2MgW9s4UV2s+YSx\n0BBd2A3j80wncP+Y7HFeC4Pv0SM0Pdq6xJaf3ELhj6j0rVZeTW1O3E/PFLXK\nnn/DZcsFXgIzjY+eBIMQgAhqyeJve8LeQNnGt3iNW10E2nZMpfc+dn0ESiwV\n2Gw4\r\n=8uqZ\r\n-----END PGP SIGNATURE-----\r\n", }, "name": "@l3ilkojr/jdinsults", "version": "3.0.0", "_npmUser": {"name": "l3ilkojr", "email": "*****@*****.**"}, "_npmVersion": "6.13.6", "description": "Generates insults", "directories": {}, "maintainers": [{"name": "l3ilkojr", "email": "*****@*****.**"}], "_nodeVersion": "10.14.0", "_hasShrinkwrap": False, "_npmOperationalInternal": { "tmp": "tmp/jdinsults_3.0.0_1582351545285_0.2614827716102821", "host": "s3://npm-registry-packages", }, } original_artifacts = [ { "length": 1033, "filename": "jdinsults-3.0.0.tgz", "checksums": { "sha1": "b7f0d66090e0285f4e95d082d39bcb0c1b8f4ec8", "sha256": "42f22795ac883b02fded0b2bf3d8a77f6507d40bc67f28eea6b1b73eb59c515f", }, } ] row = { "id": b"\x00\x00\x02\xa4\x9b\xba\x17\xca\x8c\xf3\x7f_=\x16\xaa\xac\xf9S`\xfc", "directory": DIRECTORY_ID, "date": datetime.datetime(2020, 2, 22, 6, 5, 45, tzinfo=datetime.timezone.utc), "committer_date": datetime.datetime( 2020, 2, 22, 6, 5, 45, tzinfo=datetime.timezone.utc ), "type": "tar", "message": b"3.0.0", "metadata": { "extrinsic": { "raw": extrinsic_metadata, "when": "2020-02-27T01:35:47.965375+00:00", "provider": "https://replicate.npmjs.com/%40l3ilkojr%2Fjdinsults/", }, "intrinsic": { "raw": {"name": "@l3ilkojr/jdinsults", "version": "3.0.0"}, "tool": "package.json", }, "original_artifact": original_artifacts, }, } origin_url = "https://www.npmjs.com/package/@l3ilkojr/jdinsults" storage = Mock() def origin_get(urls): assert urls == [origin_url] return [Origin(url=origin_url)] storage.origin_get.side_effect = origin_get deposit_cur = None handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) assert storage.method_calls == [ call.origin_get([origin_url]), call.raw_extrinsic_metadata_add( [ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime( 2020, 2, 27, 1, 35, 47, 965375, tzinfo=datetime.timezone.utc, ), authority=NPM_AUTHORITY, fetcher=FETCHER, format="replicate-npm-package-json", metadata=json.dumps(extrinsic_metadata).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:000002a49bba17ca8cf37f5f3d16aaacf95360fc" ), ), ] ), call.raw_extrinsic_metadata_add( [ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime( 2020, 2, 27, 1, 35, 47, 965375, tzinfo=datetime.timezone.utc, ), authority=SWH_AUTHORITY, fetcher=FETCHER, format="original-artifacts-json", metadata=json.dumps(original_artifacts).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:000002a49bba17ca8cf37f5f3d16aaacf95360fc" ), ), ] ), ]
def test_debian_with_extrinsic(): dest_original_artifacts = [ { "length": 2936, "filename": "kalgebra_19.12.1-1.dsc", "checksums": { "sha1": "f869e9f1155b1ee6d28ae3b40060570152a358cd", "sha256": "75f77150aefdaa4bcf8bc5b1e9b8b90b5cb1651b76a068c5e58e5b83658d5d11", }, "url": "http://deb.debian.org/debian//pool/main/k/kalgebra/kalgebra_19.12.1-1.dsc", }, { "length": 1156408, "filename": "kalgebra_19.12.1.orig.tar.xz", "checksums": { "sha1": "e496032962212983a5359aebadfe13c4026fd45c", "sha256": "49d623186800eb8f6fbb91eb43fb14dff78e112624c9cda6b331d494d610b16a", }, "url": "http://deb.debian.org/debian//pool/main/k/kalgebra/kalgebra_19.12.1.orig.tar.xz", }, { "length": 10044, "filename": "kalgebra_19.12.1-1.debian.tar.xz", "checksums": { "sha1": "b518bfc2ac708b40577c595bd539faa8b84572db", "sha256": "1a30acd2699c3769da302f7a0c63a7d7b060f80925b38c8c43ce3bec92744d67", }, "url": "http://deb.debian.org/debian//pool/main/k/kalgebra/kalgebra_19.12.1-1.debian.tar.xz", }, { "length": 488, "filename": "kalgebra_19.12.1.orig.tar.xz.asc", "checksums": { "sha1": "ff53a5c21c1aef2b9caa38a02fa3488f43df4c20", "sha256": "a37e0b95bb1f16b19b0587bc5d3b99ba63a195d7f6335c4a359122ad96d682dd", }, "url": "http://deb.debian.org/debian//pool/main/k/kalgebra/kalgebra_19.12.1.orig.tar.xz.asc", }, ] source_original_artifacts = [{k: v for (k, v) in d.items() if k != "url"} for d in dest_original_artifacts] row = { "id": b"\x00\x00\x03l1\x1e\xf3:(\x1b\x05h\x8fn\xad\xcf\xc0\x94:\xee", "directory": DIRECTORY_ID, "date": datetime.datetime( 2020, 1, 26, 22, 3, 24, tzinfo=datetime.timezone.utc, ), "date_offset": 60, "type": "dsc", "message": b"Synthetic revision for Debian source package kalgebra version 4:19.12.1-1", "metadata": { "extrinsic": { "raw": { "id": 2718802, "name": "kalgebra", "files": { "kalgebra_19.12.1-1.dsc": { "uri": "http://deb.debian.org/debian//pool/main/k/kalgebra/kalgebra_19.12.1-1.dsc", "name": "kalgebra_19.12.1-1.dsc", "size": 2936, "md5sum": "fd28f604d4cc31a0a305543230f1622a", "sha256": "75f77150aefdaa4bcf8bc5b1e9b8b90b5cb1651b76a068c5e58e5b83658d5d11", }, "kalgebra_19.12.1.orig.tar.xz": { "uri": "http://deb.debian.org/debian//pool/main/k/kalgebra/kalgebra_19.12.1.orig.tar.xz", "name": "kalgebra_19.12.1.orig.tar.xz", "size": 1156408, "md5sum": "34e09ed152da762d53101ea33634712b", "sha256": "49d623186800eb8f6fbb91eb43fb14dff78e112624c9cda6b331d494d610b16a", }, "kalgebra_19.12.1-1.debian.tar.xz": { "uri": "http://deb.debian.org/debian//pool/main/k/kalgebra/kalgebra_19.12.1-1.debian.tar.xz", "name": "kalgebra_19.12.1-1.debian.tar.xz", "size": 10044, "md5sum": "4f639f36143898d97d044f273f038e58", "sha256": "1a30acd2699c3769da302f7a0c63a7d7b060f80925b38c8c43ce3bec92744d67", }, "kalgebra_19.12.1.orig.tar.xz.asc": { "uri": "http://deb.debian.org/debian//pool/main/k/kalgebra/kalgebra_19.12.1.orig.tar.xz.asc", "name": "kalgebra_19.12.1.orig.tar.xz.asc", "size": 488, "md5sum": "3c29291e4e6f0c294de80feb8e9fce4c", "sha256": "a37e0b95bb1f16b19b0587bc5d3b99ba63a195d7f6335c4a359122ad96d682dd", }, }, "version": "4:19.12.1-1", "revision_id": None, }, "when": "2020-01-27T19:32:03.925498+00:00", "provider": "http://deb.debian.org/debian//pool/main/k/kalgebra/kalgebra_19.12.1-1.dsc", }, "intrinsic": { "raw": { "name": "kalgebra", "version": "4:19.12.1-1", # ... }, "tool": "dsc", }, "original_artifact": source_original_artifacts, }, } origin_url = "deb://Debian/packages/kalgebra" storage = Mock() deposit_cur = None with patch("debian_origins_from_row", return_value=[origin_url]): handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) assert storage.method_calls == [ call.raw_extrinsic_metadata_add([ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime( 2020, 1, 26, 22, 3, 24, tzinfo=datetime.timezone.utc, ), authority=SWH_AUTHORITY, fetcher=FETCHER, format="original-artifacts-json", metadata=json.dumps(dest_original_artifacts).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:0000036c311ef33a281b05688f6eadcfc0943aee"), ), ]), ]
def test_npm_2_scoped(): """Tests loading a revision generated by an old NPM loader that doesn't have a provider; and the package name is scoped (ie. in the format @org/name).""" extrinsic_metadata = { "bugs": {"url": "https://github.com/piximi/components/issues"}, "name": "@piximi/components", # ... } package_source = { "url": "https://registry.npmjs.org/@piximi/components/-/components-0.1.11.tgz", "date": "2019-06-07T19:56:04.753Z", "name": "@piximi/components", "sha1": "4ab74e563cb61bb5b2022601a5133a2dd19d19ec", "sha256": "69bb980bd6de3277b6bca86fd79c91f1c28db6910c8d03ecd05b32b78a35188f", "version": "0.1.11", "filename": "components-0.1.11.tgz", "blake2s256": "ce33181d5eff25b70ffdd6f1a18acd472a1707ede23cd2adc6af272dfc40dbfd", } original_artifacts = [ { "filename": "components-0.1.11.tgz", "checksums": { "sha1": "4ab74e563cb61bb5b2022601a5133a2dd19d19ec", "sha256": "69bb980bd6de3277b6bca86fd79c91f1c28db6910c8d03ecd05b32b78a35188f", "blake2s256": "ce33181d5eff25b70ffdd6f1a18acd472a1707ede23cd2adc6af272dfc40dbfd", }, "url": "https://registry.npmjs.org/@piximi/components/-/components-0.1.11.tgz", } ] row = { "id": b"\x00\x00 \x19\xc5wXt\xbc\xed\x00zR\x9b\xd3\xb7\x8b\xf6\x04W", "directory": DIRECTORY_ID, "date": datetime.datetime(2019, 6, 7, 19, 56, 4, tzinfo=datetime.timezone.utc), "committer_date": datetime.datetime( 2019, 6, 7, 19, 56, 4, tzinfo=datetime.timezone.utc ), "type": "tar", "message": b"0.1.11", "metadata": { "package": extrinsic_metadata, "package_source": package_source, }, } origin_url = "https://www.npmjs.com/package/@piximi/components" storage = Mock() def origin_get(urls): assert urls == [origin_url] return [Origin(url=origin_url)] storage.origin_get.side_effect = origin_get deposit_cur = None handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) assert storage.method_calls == [ call.origin_get([origin_url]), call.raw_extrinsic_metadata_add( [ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime( 2019, 6, 7, 19, 56, 4, tzinfo=datetime.timezone.utc, ), authority=NPM_AUTHORITY, fetcher=FETCHER, format="replicate-npm-package-json", metadata=json.dumps(extrinsic_metadata).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:00002019c5775874bced007a529bd3b78bf60457" ), ), ] ), call.raw_extrinsic_metadata_add( [ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime( 2019, 6, 7, 19, 56, 4, tzinfo=datetime.timezone.utc, ), authority=SWH_AUTHORITY, fetcher=FETCHER, format="original-artifacts-json", metadata=json.dumps(original_artifacts).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:00002019c5775874bced007a529bd3b78bf60457" ), ), ] ), ]
def test_npm_2_unscoped(): """Tests loading a revision generated by an old NPM loader that doesn't have a provider; and the package name is unscoped (ie. doesn't contain a slash).""" extrinsic_metadata = { "bugs": {"url": "https://github.com/niwasawa/simplemaps/issues"}, "name": "simplemaps", "author": "Naoki Iwasawa", "license": "MIT", # ... } package_source = { "url": "https://registry.npmjs.org/simplemaps/-/simplemaps-0.0.6.tgz", "date": "2016-12-23T07:21:29.733Z", "name": "simplemaps", "sha1": "e2b8222930196def764527f5c61048c5b28fe3c4", "sha256": "3ce94927bab5feafea5695d1fa4c2b8131413e53e249b32f9ac2ccff4d865a0b", "version": "0.0.6", "filename": "simplemaps-0.0.6.tgz", "blake2s256": "6769b4009f8162be2e745604b153443d4907a85781d31a724217a3e2d42a7462", } original_artifacts = [ { "filename": "simplemaps-0.0.6.tgz", "checksums": { "sha1": "e2b8222930196def764527f5c61048c5b28fe3c4", "sha256": "3ce94927bab5feafea5695d1fa4c2b8131413e53e249b32f9ac2ccff4d865a0b", "blake2s256": "6769b4009f8162be2e745604b153443d4907a85781d31a724217a3e2d42a7462", }, "url": "https://registry.npmjs.org/simplemaps/-/simplemaps-0.0.6.tgz", } ] row = { "id": b"\x00\x00\x04\xae\xed\t\xee\x08\x9cx\x12d\xc0M%d\xfdX\xfe\xb5", "directory": DIRECTORY_ID, "date": datetime.datetime( 2016, 12, 23, 7, 21, 29, tzinfo=datetime.timezone.utc ), "committer_date": datetime.datetime( 2016, 12, 23, 7, 21, 29, tzinfo=datetime.timezone.utc ), "type": "tar", "message": b"0.0.6", "metadata": { "package": extrinsic_metadata, "package_source": package_source, }, } origin_url = "https://www.npmjs.com/package/simplemaps" storage = Mock() def origin_get(urls): assert urls == [origin_url] return [Origin(url=origin_url)] storage.origin_get.side_effect = origin_get deposit_cur = None handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) assert storage.method_calls == [ call.origin_get([origin_url]), call.raw_extrinsic_metadata_add( [ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime( 2016, 12, 23, 7, 21, 29, tzinfo=datetime.timezone.utc, ), authority=NPM_AUTHORITY, fetcher=FETCHER, format="replicate-npm-package-json", metadata=json.dumps(extrinsic_metadata).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:000004aeed09ee089c781264c04d2564fd58feb5" ), ), ] ), call.raw_extrinsic_metadata_add( [ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime( 2016, 12, 23, 7, 21, 29, tzinfo=datetime.timezone.utc, ), authority=SWH_AUTHORITY, fetcher=FETCHER, format="original-artifacts-json", metadata=json.dumps(original_artifacts).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:000004aeed09ee089c781264c04d2564fd58feb5" ), ), ] ), ]
def test_cran(): source_original_artifacts = [{ "length": 170623, "filename": "ExtremeRisks_0.0.3.tar.gz", "checksums": { "sha1": "f2f19fc0f24b66b5ea9413366c632f3c229f7f3f", "sha256": "6f232556313019809dde3554149a1399bb1901a366b4965af49dc007d01945c9", }, }] dest_original_artifacts = [{ "length": 170623, "filename": "ExtremeRisks_0.0.3.tar.gz", "checksums": { "sha1": "f2f19fc0f24b66b5ea9413366c632f3c229f7f3f", "sha256": "6f232556313019809dde3554149a1399bb1901a366b4965af49dc007d01945c9", }, "url": "https://cran.r-project.org/src/contrib/ExtremeRisks_0.0.3.tar.gz", }] row = { "id": b"\x00\x03a\xaa3\x84,\xbd\xea_\xa6\xe7}\xb6\x96\xb97\xeb\xd2i", "directory": DIRECTORY_ID, "date": datetime.datetime( 2020, 5, 5, 0, 0, tzinfo=datetime.timezone.utc, ), "committer_date": datetime.datetime( 2020, 5, 5, 0, 0, tzinfo=datetime.timezone.utc, ), "type": "tar", "message": b"0.0.3", "metadata": { "extrinsic": { "raw": { "url": "https://cran.r-project.org/src/contrib/ExtremeRisks_0.0.3.tar.gz", "version": "0.0.3", }, "when": "2020-05-07T15:27:38.652281+00:00", "provider": "https://cran.r-project.org/package=ExtremeRisks", }, "intrinsic": { "raw": { "URL": "mypage.unibocconi.it/simonepadoan/", "Date": "2020-05-05", "Title": "Extreme Risk Measures", "Author": "Simone Padoan [cre, aut],\n Gilles Stupfler [aut]", # ... "Date/Publication": "2020-05-07 10:20:02 UTC", }, "tool": "DESCRIPTION", }, "original_artifact": source_original_artifacts, }, } origin_url = "https://cran.r-project.org/package=ExtremeRisks" storage = Mock() def origin_get(urls): assert urls == [origin_url] return [Origin(url=origin_url)] storage.origin_get.side_effect = origin_get deposit_cur = None handle_row(row, storage, deposit_cur, dry_run=False) assert storage.method_calls == [ call.origin_get([origin_url]), call.raw_extrinsic_metadata_add([ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime( 2020, 5, 7, 15, 27, 38, 652281, tzinfo=datetime.timezone.utc, ), authority=SWH_AUTHORITY, fetcher=FETCHER, format="original-artifacts-json", metadata=json.dumps(dest_original_artifacts).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:000361aa33842cbdea5fa6e77db696b937ebd269"), ), ]), ]
def test_cran_with_new_original_artifacts_format(): original_artifacts = [{ "url": "https://cran.r-project.org/src/contrib/r2mlm_0.1.0.tar.gz", "length": 346563, "filename": "r2mlm_0.1.0.tar.gz", "checksums": { "sha1": "25c06b4af523c35a7813b58dd0db414e79848501", "sha256": "c887fe6c4f78c94b2279759052e12d639cf80225b444c1f67931c6aa6f0faf23", }, }] row = { "id": b'."7\x82\xeeK\xa1R\xe4\xc8\x86\xf7\x97\x97bA\xc3\x9a\x9a\xab', "directory": DIRECTORY_ID, "date": None, "committer_date": None, "type": "tar", "message": b"0.1.0", "metadata": { "extrinsic": { "raw": { "url": "https://cran.r-project.org/src/contrib/r2mlm_0.1.0.tar.gz" }, "when": "2020-09-25T14:04:20.926667+00:00", "provider": "https://cran.r-project.org/package=r2mlm", }, "intrinsic": { "raw": { "URL": "https://github.com/mkshaw/r2mlm", "Type": "Package", "Title": "R-Squared Measures for Multilevel Models", "Author": "Mairead Shaw [aut, cre],\n Jason Rights [aut],\n Sonya Sterba [aut],\n Jessica Flake [aut]", # ... }, "tool": "DESCRIPTION", }, "original_artifact": original_artifacts, }, } origin_url = "https://cran.r-project.org/package=r2mlm" storage = Mock() def origin_get(urls): assert urls == [origin_url] return [Origin(url=origin_url)] storage.origin_get.side_effect = origin_get deposit_cur = None handle_row(row, storage, deposit_cur, dry_run=False) assert storage.method_calls == [ call.origin_get([origin_url]), call.raw_extrinsic_metadata_add([ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime( 2020, 9, 25, 14, 4, 20, 926667, tzinfo=datetime.timezone.utc, ), authority=SWH_AUTHORITY, fetcher=FETCHER, format="original-artifacts-json", metadata=json.dumps(original_artifacts).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:2e223782ee4ba152e4c886f797976241c39a9aab"), ), ]), ]
def test_cran_without_revision_date(): """Tests a CRAN revision with a date in the metadata but not as revision date""" source_original_artifacts = [{ "length": 8018, "filename": "gofgamma_1.0.tar.gz", "checksums": { "sha1": "58f2993140f9e9e1a136554f0af0174a252f2c7b", "sha256": "55408f004642b5043bb01de831a7e7a0b9f24a30cb0151e70c2d37abdc508d03", }, }] dest_original_artifacts = [{ "length": 8018, "filename": "gofgamma_1.0.tar.gz", "checksums": { "sha1": "58f2993140f9e9e1a136554f0af0174a252f2c7b", "sha256": "55408f004642b5043bb01de831a7e7a0b9f24a30cb0151e70c2d37abdc508d03", }, "url": "https://cran.r-project.org/src/contrib/gofgamma_1.0.tar.gz", }] row = { "id": b'\x00\x00\xd4\xef^\x16a"\xae\xe6\x86*\xd3\x8a\x18\xceS\x86\xcc>', "directory": DIRECTORY_ID, "date": None, "committer_date": None, "type": "tar", "message": b"1.0", "metadata": { "extrinsic": { "raw": { "url": "https://cran.r-project.org/src/contrib/gofgamma_1.0.tar.gz", "version": "1.0", }, "when": "2020-04-30T11:01:57.832481+00:00", "provider": "https://cran.r-project.org/package=gofgamma", }, "intrinsic": { "raw": { "Type": "Package", "Title": "Goodness-of-Fit Tests for the Gamma Distribution", "Author": "Lucas Butsch [aut],\n Bruno Ebner [aut, cre],\n Steffen Betsch [aut]", # ... }, "tool": "DESCRIPTION", }, "original_artifact": source_original_artifacts, }, } origin_url = "https://cran.r-project.org/package=gofgamma" storage = Mock() def origin_get(urls): assert urls == [origin_url] return [Origin(url=origin_url)] storage.origin_get.side_effect = origin_get deposit_cur = None handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) assert storage.method_calls == [ call.origin_get([origin_url]), call.raw_extrinsic_metadata_add([ RawExtrinsicMetadata( target=DIRECTORY_SWHID, discovery_date=datetime.datetime( 2020, 4, 30, 11, 1, 57, 832481, tzinfo=datetime.timezone.utc, ), authority=SWH_AUTHORITY, fetcher=FETCHER, format="original-artifacts-json", metadata=json.dumps(dest_original_artifacts).encode(), origin=origin_url, revision=CoreSWHID.from_string( "swh:1:rev:0000d4ef5e166122aee6862ad38a18ce5386cc3e"), ), ]), ]