def test_add_duplicate_regex_replica(self):
rc = ReplicaCatalog()
rc.add_regex_replica("local", "*.txt", "/path")
with pytest.raises(DuplicateError) as e:
rc.add_regex_replica("local", "*.txt", "/path")
assert "Pattern: *.txt already exists" in str(e)
def test_tojson(self, convert_yaml_schemas_to_json, load_schema):
rc = ReplicaCatalog()
rc.add_replica(
"local",
"f.a",
"/f.a",
checksum={"sha256": "123"},
metadata={
"size": 1024,
"㐦": "㐦"
},
)
rc.add_regex_replica("local",
"*.txt",
"/path",
metadata={"creator": "ryan"})
result = _tojson(rc)
expected = {
"pegasus":
"5.0",
"replicas": [
{
"lfn": "f.a",
"pfns": [{
"site": "local",
"pfn": "/f.a"
}],
"checksum": {
"sha256": "123"
},
"metadata": {
"size": 1024,
"㐦": "㐦"
},
},
{
"lfn": "*.txt",
"pfns": [{
"site": "local",
"pfn": "/path"
}],
"metadata": {
"creator": "ryan"
},
"regex": True,
},
],
}
assert result == expected
rc_schema = load_schema("rc-5.0.json")
validate(instance=result, schema=rc_schema)
def test_add_regex_replica(self):
rc = ReplicaCatalog()
rc.add_regex_replica("local", "*.txt", "/path")
assert _tojson(rc.entries[("*.txt", True)]) == {
"lfn": "*.txt",
"pfns": [{
"site": "local",
"pfn": "/path"
}],
"regex": True,
}
def test_add_regex_replica_with_metadata(self):
rc = ReplicaCatalog()
rc.add_regex_replica("local",
"*.txt",
"/path",
metadata={"creator": "ryan"})
assert _tojson(rc.entries[("*.txt", True)]) == {
"lfn": "*.txt",
"pfns": [{
"site": "local",
"pfn": "/path"
}],
"metadata": {
"creator": "ryan"
},
"regex": True,
}
def _to_rc(d: dict) -> ReplicaCatalog:
"""Convert dict to ReplicaCatalog
:param d: ReplicaCatalog represented as a dict
:type d: dict
:raises PegasusError: encountered error parsing
:return: a ReplicaCatalog object based on d
:rtype: ReplicaCatalog
"""
rc = ReplicaCatalog()
try:
for r in d["replicas"]:
lfn = r["lfn"]
pfns = {_PFN(i["site"], i["pfn"]) for i in r["pfns"]}
checksum = r.get("checksum") if r.get("checksum") else {}
metadata = r.get("metadata") if r.get("metadata") else {}
regex = r.get("regex")
for pfn in pfns:
if regex:
rc.add_regex_replica(pfn.site,
lfn,
pfn.pfn,
metadata=metadata)
else:
rc.add_replica(pfn.site,
lfn,
pfn.pfn,
metadata=metadata,
checksum=checksum)
except KeyError:
raise PegasusError("error parsing {}".format(d))
return rc
def test_write(self, _format, loader):
rc = ReplicaCatalog()
f_a = File("f.a", size=1024).add_metadata(creator="ryan")
rc.add_replica(
"local",
f_a,
"/f.a",
checksum={"sha256": "123"},
metadata={"extra": "metadata"},
)
rc.add_replica("condorpool", f_a, "/f.a")
rc.add_replica("local", "f.b", "/f.b")
rc.add_regex_replica("local",
"*.txt",
"/path",
metadata={"creator": "ryan"})
expected = {
"pegasus":
"5.0",
"replicas": [
{
"lfn":
"f.a",
"pfns": [
{
"site": "local",
"pfn": "/f.a"
},
{
"site": "condorpool",
"pfn": "/f.a"
},
],
"checksum": {
"sha256": "123"
},
"metadata": {
"extra": "metadata",
"size": 1024,
"creator": "ryan"
},
},
{
"lfn": "f.b",
"pfns": [{
"site": "local",
"pfn": "/f.b"
}],
"metadata": {
"size": 1024,
"creator": "ryan"
},
},
{
"lfn": "*.txt",
"pfns": [{
"site": "local",
"pfn": "/path"
}],
"metadata": {
"creator": "ryan"
},
"regex": True,
},
],
}
expected["replicas"][0]["pfns"] = sorted(
expected["replicas"][0]["pfns"], key=lambda pfn: pfn["site"])
with NamedTemporaryFile(mode="r+") as f:
rc.write(f, _format=_format)
f.seek(0)
result = loader(f)
result["replicas"][0]["pfns"] = sorted(result["replicas"][0]["pfns"],
key=lambda pfn: pfn["site"])
assert "createdOn" in result["x-pegasus"]
assert result["x-pegasus"]["createdBy"] == getpass.getuser()
assert result["x-pegasus"]["apiLang"] == "python"
del result["x-pegasus"]
assert result == expected
