def test_write(self, _format, loader):
rc = ReplicaCatalog()
rc.add_replica("site1", "lfn1", "pfn1",
True).add_replica("site2", "lfn2", "pfn2", True)
expected = {
"pegasus":
PEGASUS_VERSION,
"replicas": [
{
"lfn": "lfn1",
"pfn": "pfn1",
"site": "site1",
"regex": True
},
{
"lfn": "lfn2",
"pfn": "pfn2",
"site": "site2",
"regex": True
},
],
}
expected["replicas"] = sorted(expected["replicas"],
key=lambda d: d["lfn"])
with NamedTemporaryFile(mode="r+") as f:
rc.write(f, _format=_format)
f.seek(0)
result = loader(f)
result["replicas"] = sorted(result["replicas"], key=lambda d: d["lfn"])
assert result == expected
def test_add_duplicate_replica(self):
rc = ReplicaCatalog()
with pytest.raises(DuplicateError) as e:
rc.add_replica("site", "lfn", "pfn", True)
rc.add_replica("site", File("lfn"), "pfn", True)
assert (
"entry: ReplicaEntry(site=site, lfn=lfn, pfn=pfn, regex=True, checksum_type=None, checksum_value=None)"
in str(e))
def test_add_replica_str_as_lfn(self):
rc = ReplicaCatalog()
rc.add_replica("local", "f.a", "/f.a")
assert _tojson(rc.entries[("f.a", False)]) == {
"lfn": "f.a",
"pfns": [{
"site": "local",
"pfn": "/f.a"
}],
}
def test_tojson(self, convert_yaml_schemas_to_json, load_schema):
rc = ReplicaCatalog()
rc.add_replica(
"local",
"f.a",
"/f.a",
checksum={"sha256": "123"},
metadata={
"size": 1024,
"㐦": "㐦"
},
)
rc.add_regex_replica("local",
"*.txt",
"/path",
metadata={"creator": "ryan"})
result = _tojson(rc)
expected = {
"pegasus":
"5.0",
"replicas": [
{
"lfn": "f.a",
"pfns": [{
"site": "local",
"pfn": "/f.a"
}],
"checksum": {
"sha256": "123"
},
"metadata": {
"size": 1024,
"㐦": "㐦"
},
},
{
"lfn": "*.txt",
"pfns": [{
"site": "local",
"pfn": "/path"
}],
"metadata": {
"creator": "ryan"
},
"regex": True,
},
],
}
assert result == expected
rc_schema = load_schema("rc-5.0.json")
validate(instance=result, schema=rc_schema)
def test_add_replica_multiple_pfns(self):
rc = ReplicaCatalog()
rc.add_replica("local", "f.a", "/f.a")
rc.add_replica("condorpool", "f.a", "/f.a")
rc.add_replica("condorpool", "f.a", "/f.a")
assert len(rc.entries) == 1
assert rc.entries[("f.a", False)].pfns == {
_PFN("local", "/f.a"),
_PFN("condorpool", "/f.a"),
}
assert rc.entries[("f.a", False)].metadata == {}
def test_add_replica_multiple_pfns_checksums_and_metadata(self):
rc = ReplicaCatalog()
rc.add_replica(
"local",
"f.a",
"/f.a",
checksum={"sha256": "abc"},
metadata={"creator": "ryan"},
)
rc.add_replica(
"condorpool",
"f.a",
"/f.a",
metadata={"size": 1024},
)
f_a_entry = rc.entries[("f.a", False)]
assert f_a_entry.metadata == {"size": 1024, "creator": "ryan"}
assert f_a_entry.checksum == {"sha256": "abc"}
def test_add_multiple_replicas(self):
rc = ReplicaCatalog()
rc.add_replica("local", "f.a", "/f.a")
rc.add_replica("local", "f.b", "/f.b")
assert _tojson(rc.entries[("f.a", False)]) == {
"lfn": "f.a",
"pfns": [{
"site": "local",
"pfn": "/f.a"
}],
}
assert _tojson(rc.entries[("f.b", False)]) == {
"lfn": "f.b",
"pfns": [{
"site": "local",
"pfn": "/f.b"
}],
}
assert len(rc.entries) == 2
def _to_rc(d: dict) -> ReplicaCatalog:
"""Convert dict to ReplicaCatalog
:param d: ReplicaCatalog represented as a dict
:type d: dict
:raises PegasusError: encountered error parsing
:return: a ReplicaCatalog object based on d
:rtype: ReplicaCatalog
"""
rc = ReplicaCatalog()
try:
for r in d["replicas"]:
site = r["site"]
lfn = r["lfn"]
pfn = r["pfn"]
regex = r.get("regex")
if not regex:
regex = False
checksum = r.get("checksum")
_type = None
value = None
if checksum:
_type = checksum["type"]
value = checksum["value"]
rc.add_replica(site,
lfn,
pfn,
regex=regex,
checksum_type=_type,
checksum_value=value)
except KeyError:
raise PegasusError("error parsing {}".format(d))
return rc
def _to_rc(d: dict) -> ReplicaCatalog:
"""Convert dict to ReplicaCatalog
:param d: ReplicaCatalog represented as a dict
:type d: dict
:raises PegasusError: encountered error parsing
:return: a ReplicaCatalog object based on d
:rtype: ReplicaCatalog
"""
rc = ReplicaCatalog()
try:
for r in d["replicas"]:
lfn = r["lfn"]
pfns = {_PFN(i["site"], i["pfn"]) for i in r["pfns"]}
checksum = r.get("checksum") if r.get("checksum") else {}
metadata = r.get("metadata") if r.get("metadata") else {}
regex = r.get("regex")
for pfn in pfns:
if regex:
rc.add_regex_replica(pfn.site,
lfn,
pfn.pfn,
metadata=metadata)
else:
rc.add_replica(pfn.site,
lfn,
pfn.pfn,
metadata=metadata,
checksum=checksum)
except KeyError:
raise PegasusError("error parsing {}".format(d))
return rc
def test_dumps(rc_as_dict):
rc = ReplicaCatalog()
rc.add_replica("local", "a", "/a", regex=True)
rc.add_replica("local", "b", "/b")
rc.add_replica("local", "c", "/c", checksum_type="sha256", checksum_value="abc123")
rc_as_dict["replicas"] = sorted(rc_as_dict["replicas"], key=lambda r: r["lfn"])
result = yaml.load(dumps(rc))
result["replicas"] = sorted(result["replicas"], key=lambda r: r["lfn"])
assert result["replicas"] == rc_as_dict["replicas"]
def test_to_rc(rc_as_dict):
expected = ReplicaCatalog()
expected.add_replica("local", "a", "/a", regex=True)
expected.add_replica("local", "b", "/b")
expected.add_replica(
"local", "c", "/c", checksum_type="sha256", checksum_value="abc123"
)
result = _to_rc(rc_as_dict)
assert result.replicas == expected.replicas
def test_tojson(self, convert_yaml_schemas_to_json, load_schema):
rc = ReplicaCatalog()
rc.add_replica("site1", "lfn1", "pfn1")
rc.add_replica("site2", "lfn2", "pfn2", True)
rc.add_replica("site3",
"lfn3",
"pfn3",
checksum_type="sha256",
checksum_value="abc123")
expected = {
"pegasus":
PEGASUS_VERSION,
"replicas": [
{
"lfn": "lfn1",
"pfn": "pfn1",
"site": "site1"
},
{
"lfn": "lfn2",
"pfn": "pfn2",
"site": "site2",
"regex": True
},
{
"lfn": "lfn3",
"pfn": "pfn3",
"site": "site3",
"checksum": {
"type": "sha256",
"value": "abc123"
},
},
],
}
expected["replicas"] = sorted(expected["replicas"],
key=lambda d: d["lfn"])
result = json.loads(json.dumps(rc, cls=_CustomEncoder))
result["replicas"] = sorted(result["replicas"], key=lambda d: d["lfn"])
rc_schema = load_schema("rc-5.0.json")
validate(instance=result, schema=rc_schema)
assert result == expected
def test_write(self, _format, loader):
rc = ReplicaCatalog()
f_a = File("f.a", size=1024).add_metadata(creator="ryan")
rc.add_replica(
"local",
f_a,
"/f.a",
checksum={"sha256": "123"},
metadata={"extra": "metadata"},
)
rc.add_replica("condorpool", f_a, "/f.a")
rc.add_replica("local", "f.b", "/f.b")
rc.add_regex_replica("local",
"*.txt",
"/path",
metadata={"creator": "ryan"})
expected = {
"pegasus":
"5.0",
"replicas": [
{
"lfn":
"f.a",
"pfns": [
{
"site": "local",
"pfn": "/f.a"
},
{
"site": "condorpool",
"pfn": "/f.a"
},
],
"checksum": {
"sha256": "123"
},
"metadata": {
"extra": "metadata",
"size": 1024,
"creator": "ryan"
},
},
{
"lfn": "f.b",
"pfns": [{
"site": "local",
"pfn": "/f.b"
}],
"metadata": {
"size": 1024,
"creator": "ryan"
},
},
{
"lfn": "*.txt",
"pfns": [{
"site": "local",
"pfn": "/path"
}],
"metadata": {
"creator": "ryan"
},
"regex": True,
},
],
}
expected["replicas"][0]["pfns"] = sorted(
expected["replicas"][0]["pfns"], key=lambda pfn: pfn["site"])
with NamedTemporaryFile(mode="r+") as f:
rc.write(f, _format=_format)
f.seek(0)
result = loader(f)
result["replicas"][0]["pfns"] = sorted(result["replicas"][0]["pfns"],
key=lambda pfn: pfn["site"])
assert "createdOn" in result["x-pegasus"]
assert result["x-pegasus"]["createdBy"] == getpass.getuser()
assert result["x-pegasus"]["apiLang"] == "python"
del result["x-pegasus"]
assert result == expected
def test_add_replica_with_invalid_checksum(self):
rc = ReplicaCatalog()
with pytest.raises(ValueError) as e:
rc.add_replica("local", "f.a", "/f.a", checksum={"md5": "123"})
assert "Invalid checksum: md5" in str(e)
def test_add_replica(self, replica: tuple):
rc = ReplicaCatalog()
rc.add_replica(*replica)
assert len(rc.replicas) == 1
def test_add_invalid_replica(self):
rc = ReplicaCatalog()
with pytest.raises(TypeError) as e:
rc.add_replica("site", set(), "pfn")
assert "invalid lfn: {lfn}".format(lfn=set()) in str(e)
def test_add_replica_pfn_with_invalid_path_object(self):
rc = ReplicaCatalog()
with pytest.raises(ValueError) as e:
rc.add_replica("local", "f.a", Path("file"))
assert "Invalid pfn: file" in str(e)
def test_add_replica_pfn_with_path_obj(self):
rc = ReplicaCatalog()
rc.add_replica("local", "test_replica_catalog", Path("/file"))
assert rc.entries[("test_replica_catalog",
False)].pfns.pop().pfn == "/file"
