def test_write(self, _format, loader): rc = ReplicaCatalog() rc.add_replica("site1", "lfn1", "pfn1", True).add_replica("site2", "lfn2", "pfn2", True) expected = { "pegasus": PEGASUS_VERSION, "replicas": [ { "lfn": "lfn1", "pfn": "pfn1", "site": "site1", "regex": True }, { "lfn": "lfn2", "pfn": "pfn2", "site": "site2", "regex": True }, ], } expected["replicas"] = sorted(expected["replicas"], key=lambda d: d["lfn"]) with NamedTemporaryFile(mode="r+") as f: rc.write(f, _format=_format) f.seek(0) result = loader(f) result["replicas"] = sorted(result["replicas"], key=lambda d: d["lfn"]) assert result == expected
def test_add_duplicate_replica(self): rc = ReplicaCatalog() with pytest.raises(DuplicateError) as e: rc.add_replica("site", "lfn", "pfn", True) rc.add_replica("site", File("lfn"), "pfn", True) assert ( "entry: ReplicaEntry(site=site, lfn=lfn, pfn=pfn, regex=True, checksum_type=None, checksum_value=None)" in str(e))
def test_add_replica_str_as_lfn(self): rc = ReplicaCatalog() rc.add_replica("local", "f.a", "/f.a") assert _tojson(rc.entries[("f.a", False)]) == { "lfn": "f.a", "pfns": [{ "site": "local", "pfn": "/f.a" }], }
def test_tojson(self, convert_yaml_schemas_to_json, load_schema): rc = ReplicaCatalog() rc.add_replica( "local", "f.a", "/f.a", checksum={"sha256": "123"}, metadata={ "size": 1024, "㐦": "㐦" }, ) rc.add_regex_replica("local", "*.txt", "/path", metadata={"creator": "ryan"}) result = _tojson(rc) expected = { "pegasus": "5.0", "replicas": [ { "lfn": "f.a", "pfns": [{ "site": "local", "pfn": "/f.a" }], "checksum": { "sha256": "123" }, "metadata": { "size": 1024, "㐦": "㐦" }, }, { "lfn": "*.txt", "pfns": [{ "site": "local", "pfn": "/path" }], "metadata": { "creator": "ryan" }, "regex": True, }, ], } assert result == expected rc_schema = load_schema("rc-5.0.json") validate(instance=result, schema=rc_schema)
def test_add_replica_multiple_pfns(self): rc = ReplicaCatalog() rc.add_replica("local", "f.a", "/f.a") rc.add_replica("condorpool", "f.a", "/f.a") rc.add_replica("condorpool", "f.a", "/f.a") assert len(rc.entries) == 1 assert rc.entries[("f.a", False)].pfns == { _PFN("local", "/f.a"), _PFN("condorpool", "/f.a"), } assert rc.entries[("f.a", False)].metadata == {}
def test_add_replica_multiple_pfns_checksums_and_metadata(self): rc = ReplicaCatalog() rc.add_replica( "local", "f.a", "/f.a", checksum={"sha256": "abc"}, metadata={"creator": "ryan"}, ) rc.add_replica( "condorpool", "f.a", "/f.a", metadata={"size": 1024}, ) f_a_entry = rc.entries[("f.a", False)] assert f_a_entry.metadata == {"size": 1024, "creator": "ryan"} assert f_a_entry.checksum == {"sha256": "abc"}
def test_add_multiple_replicas(self): rc = ReplicaCatalog() rc.add_replica("local", "f.a", "/f.a") rc.add_replica("local", "f.b", "/f.b") assert _tojson(rc.entries[("f.a", False)]) == { "lfn": "f.a", "pfns": [{ "site": "local", "pfn": "/f.a" }], } assert _tojson(rc.entries[("f.b", False)]) == { "lfn": "f.b", "pfns": [{ "site": "local", "pfn": "/f.b" }], } assert len(rc.entries) == 2
def _to_rc(d: dict) -> ReplicaCatalog: """Convert dict to ReplicaCatalog :param d: ReplicaCatalog represented as a dict :type d: dict :raises PegasusError: encountered error parsing :return: a ReplicaCatalog object based on d :rtype: ReplicaCatalog """ rc = ReplicaCatalog() try: for r in d["replicas"]: site = r["site"] lfn = r["lfn"] pfn = r["pfn"] regex = r.get("regex") if not regex: regex = False checksum = r.get("checksum") _type = None value = None if checksum: _type = checksum["type"] value = checksum["value"] rc.add_replica(site, lfn, pfn, regex=regex, checksum_type=_type, checksum_value=value) except KeyError: raise PegasusError("error parsing {}".format(d)) return rc
def _to_rc(d: dict) -> ReplicaCatalog: """Convert dict to ReplicaCatalog :param d: ReplicaCatalog represented as a dict :type d: dict :raises PegasusError: encountered error parsing :return: a ReplicaCatalog object based on d :rtype: ReplicaCatalog """ rc = ReplicaCatalog() try: for r in d["replicas"]: lfn = r["lfn"] pfns = {_PFN(i["site"], i["pfn"]) for i in r["pfns"]} checksum = r.get("checksum") if r.get("checksum") else {} metadata = r.get("metadata") if r.get("metadata") else {} regex = r.get("regex") for pfn in pfns: if regex: rc.add_regex_replica(pfn.site, lfn, pfn.pfn, metadata=metadata) else: rc.add_replica(pfn.site, lfn, pfn.pfn, metadata=metadata, checksum=checksum) except KeyError: raise PegasusError("error parsing {}".format(d)) return rc
def test_dumps(rc_as_dict): rc = ReplicaCatalog() rc.add_replica("local", "a", "/a", regex=True) rc.add_replica("local", "b", "/b") rc.add_replica("local", "c", "/c", checksum_type="sha256", checksum_value="abc123") rc_as_dict["replicas"] = sorted(rc_as_dict["replicas"], key=lambda r: r["lfn"]) result = yaml.load(dumps(rc)) result["replicas"] = sorted(result["replicas"], key=lambda r: r["lfn"]) assert result["replicas"] == rc_as_dict["replicas"]
def test_to_rc(rc_as_dict): expected = ReplicaCatalog() expected.add_replica("local", "a", "/a", regex=True) expected.add_replica("local", "b", "/b") expected.add_replica( "local", "c", "/c", checksum_type="sha256", checksum_value="abc123" ) result = _to_rc(rc_as_dict) assert result.replicas == expected.replicas
def test_tojson(self, convert_yaml_schemas_to_json, load_schema): rc = ReplicaCatalog() rc.add_replica("site1", "lfn1", "pfn1") rc.add_replica("site2", "lfn2", "pfn2", True) rc.add_replica("site3", "lfn3", "pfn3", checksum_type="sha256", checksum_value="abc123") expected = { "pegasus": PEGASUS_VERSION, "replicas": [ { "lfn": "lfn1", "pfn": "pfn1", "site": "site1" }, { "lfn": "lfn2", "pfn": "pfn2", "site": "site2", "regex": True }, { "lfn": "lfn3", "pfn": "pfn3", "site": "site3", "checksum": { "type": "sha256", "value": "abc123" }, }, ], } expected["replicas"] = sorted(expected["replicas"], key=lambda d: d["lfn"]) result = json.loads(json.dumps(rc, cls=_CustomEncoder)) result["replicas"] = sorted(result["replicas"], key=lambda d: d["lfn"]) rc_schema = load_schema("rc-5.0.json") validate(instance=result, schema=rc_schema) assert result == expected
def test_write(self, _format, loader): rc = ReplicaCatalog() f_a = File("f.a", size=1024).add_metadata(creator="ryan") rc.add_replica( "local", f_a, "/f.a", checksum={"sha256": "123"}, metadata={"extra": "metadata"}, ) rc.add_replica("condorpool", f_a, "/f.a") rc.add_replica("local", "f.b", "/f.b") rc.add_regex_replica("local", "*.txt", "/path", metadata={"creator": "ryan"}) expected = { "pegasus": "5.0", "replicas": [ { "lfn": "f.a", "pfns": [ { "site": "local", "pfn": "/f.a" }, { "site": "condorpool", "pfn": "/f.a" }, ], "checksum": { "sha256": "123" }, "metadata": { "extra": "metadata", "size": 1024, "creator": "ryan" }, }, { "lfn": "f.b", "pfns": [{ "site": "local", "pfn": "/f.b" }], "metadata": { "size": 1024, "creator": "ryan" }, }, { "lfn": "*.txt", "pfns": [{ "site": "local", "pfn": "/path" }], "metadata": { "creator": "ryan" }, "regex": True, }, ], } expected["replicas"][0]["pfns"] = sorted( expected["replicas"][0]["pfns"], key=lambda pfn: pfn["site"]) with NamedTemporaryFile(mode="r+") as f: rc.write(f, _format=_format) f.seek(0) result = loader(f) result["replicas"][0]["pfns"] = sorted(result["replicas"][0]["pfns"], key=lambda pfn: pfn["site"]) assert "createdOn" in result["x-pegasus"] assert result["x-pegasus"]["createdBy"] == getpass.getuser() assert result["x-pegasus"]["apiLang"] == "python" del result["x-pegasus"] assert result == expected
def test_add_replica_with_invalid_checksum(self): rc = ReplicaCatalog() with pytest.raises(ValueError) as e: rc.add_replica("local", "f.a", "/f.a", checksum={"md5": "123"}) assert "Invalid checksum: md5" in str(e)
def test_add_replica(self, replica: tuple): rc = ReplicaCatalog() rc.add_replica(*replica) assert len(rc.replicas) == 1
def test_add_invalid_replica(self): rc = ReplicaCatalog() with pytest.raises(TypeError) as e: rc.add_replica("site", set(), "pfn") assert "invalid lfn: {lfn}".format(lfn=set()) in str(e)
def test_add_replica_pfn_with_invalid_path_object(self): rc = ReplicaCatalog() with pytest.raises(ValueError) as e: rc.add_replica("local", "f.a", Path("file")) assert "Invalid pfn: file" in str(e)
def test_add_replica_pfn_with_path_obj(self): rc = ReplicaCatalog() rc.add_replica("local", "test_replica_catalog", Path("/file")) assert rc.entries[("test_replica_catalog", False)].pfns.pop().pfn == "/file"