def test_add_duplicate_site(self):
with pytest.raises(DuplicateError) as e:
t = Transformation("test")
t.add_sites(TransformationSite("local", "/pfn", True))
t.add_sites(TransformationSite("isi", "/pfn", True))
t.add_sites(TransformationSite("local", "/pfn", True))
assert "local" in str(e)
def test_example_transformation_catalog(
self, convert_yaml_schemas_to_json, load_schema, _format, loader
):
# validates the sample tc in pegasus/etc/sample-5.0-data/tc.yml
tc = TransformationCatalog()
foo = (
Transformation("foo")
.add_globus_profile(max_time=2)
.add_dagman_profile(retry=2)
.add_metadata(size=2048)
.add_sites(
TransformationSite(
"local",
"/nfs/u2/ryan/bin/foo",
True,
arch=Arch.X86_64,
os_type=OS.LINUX,
)
.add_env(JAVA_HOME="/usr/bin/java")
.add_metadata(size=2048)
)
.add_requirement("bar")
.add_shell_hook(EventType.START, "/bin/echo 'starting'")
)
bar = Transformation("bar").add_sites(
TransformationSite(
"local",
"/nfs/u2/ryan/bin/bar",
True,
arch=Arch.X86_64,
os_type=OS.LINUX,
)
)
centos_pegasus_container = Container(
"centos-pegasus",
Container.DOCKER,
"docker:///ryan/centos-pegasus:latest",
arguments="--shm-size 123",
mounts=["/Volumes/Work/lfs1:/shared-data/:ro"],
).add_env(JAVA_HOME="/usr/bin/java")
(tc.add_transformations(foo, bar).add_containers(centos_pegasus_container))
with NamedTemporaryFile(mode="r+") as f:
tc.write(f, _format=_format)
f.seek(0)
tc_json = loader(f)
tc_schema = load_schema("tc-5.0.json")
validate(instance=tc_json, schema=tc_schema)
def test_tojson_no_profiles_or_metadata(
self,
transformation_site: TransformationSite,
expected_json: dict,
convert_yaml_schemas_to_json,
load_schema,
):
result = transformation_site.__json__()
transformation_site_schema = load_schema("tc-5.0.json")["$defs"][
"transformation"]["properties"]["sites"]["items"]
validate(instance=result, schema=transformation_site_schema)
assert transformation_site.__json__() == expected_json
def test_tojson_no_containers(self, convert_yaml_schemas_to_json,
load_schema):
tc = TransformationCatalog()
(tc.add_transformations(
Transformation("t1").add_sites(
TransformationSite("local", "/pfn",
False))).add_transformations(
Transformation("t2").add_sites(
TransformationSite(
"local2", "/pfn", True))))
expected = {
"pegasus":
PEGASUS_VERSION,
"transformations": [
{
"name":
"t1",
"sites": [{
"name": "local",
"pfn": "/pfn",
"type": "installed"
}],
},
{
"name":
"t2",
"sites": [{
"name": "local2",
"pfn": "/pfn",
"type": "stageable"
}],
},
],
}
expected["transformations"] = sorted(expected["transformations"],
key=lambda t: t["name"])
result = json.loads(json.dumps(tc, cls=_CustomEncoder))
result["transformations"] = sorted(result["transformations"],
key=lambda t: t["name"])
tc_schema = load_schema("tc-5.0.json")
validate(instance=result, schema=tc_schema)
assert expected == result
def test_tojson_with_profiles_hooks_metadata(
self, convert_yaml_schemas_to_json, load_schema
):
t = Transformation("test", namespace="pegasus")
t.add_sites(
TransformationSite("local", "/pfn", True).add_env(JAVA_HOME="/java/home")
)
t.add_requirement("required")
t.add_env(JAVA_HOME="/java/home")
t.add_shell_hook(EventType.START, "/bin/echo hi")
t.add_metadata(key="value")
result = json.loads(json.dumps(t, cls=_CustomEncoder))
expected = {
"name": "test",
"namespace": "pegasus",
"requires": ["required"],
"sites": [
{
"name": "local",
"pfn": "/pfn",
"type": "stageable",
"profiles": {"env": {"JAVA_HOME": "/java/home"}},
}
],
"metadata": {"key": "value"},
"profiles": {Namespace.ENV.value: {"JAVA_HOME": "/java/home"}},
"hooks": {"shell": [{"_on": EventType.START.value, "cmd": "/bin/echo hi"}]},
}
transformation_schema = load_schema("tc-5.0.json")["$defs"]["transformation"]
validate(instance=result, schema=transformation_schema)
assert result == expected
def test_tojson_with_profiles_and_metadata(
self, convert_yaml_schemas_to_json, load_schema
):
t = (
TransformationSite("local", "/pfn", False)
.add_env(JAVA_HOME="/java/home")
.add_metadata(key="value")
)
result = t.__json__()
expected = {
"name": "local",
"pfn": "/pfn",
"type": "installed",
"profiles": {Namespace.ENV.value: {"JAVA_HOME": "/java/home"}},
"metadata": {"key": "value"},
}
transformation_site_schema = load_schema("tc-5.0.json")["$defs"][
"transformation"
]["properties"]["sites"]["items"]
validate(instance=result, schema=transformation_site_schema)
assert result == expected
def test_invalid_use_of_bypass_staging(self):
with pytest.raises(ValueError) as e:
TransformationSite("local", "/pfn", False, bypass_staging=True)
assert (
"bypass_staging can only be used when is_stageable is set to True" in str(e)
)
def test_invalid_transformation_site(
self, name: str, pfn: str, transformation_type: bool, kwargs: dict
):
with pytest.raises(TypeError) as e:
TransformationSite(name, pfn, transformation_type, **kwargs)
assert "invalid" in str(e)
def test_chaining(self):
t = (Transformation("test").add_sites(
TransformationSite("local", "/pfn", True).add_env(
JAVA_HOME="/java/home")).add_requirement("required"))
assert "local" in t.sites
assert t.sites["local"].profiles["env"]["JAVA_HOME"] == "/java/home"
assert "required" in t.requires
def tc2():
return (TransformationCatalog().add_transformations(
Transformation("t1", namespace="test", version="1.0").add_sites(
TransformationSite(
"local",
"/pfn",
True,
))).add_containers(
Container(
"cont",
Container.DOCKER,
"docker:///ryan/centos-pegasus:latest",
mounts=["/Volumes/Work/lfs1:/shared-data/:ro"],
image_site="local",
)))
def tc1():
return (TransformationCatalog().add_transformations(
Transformation("t1", namespace="test", version="1.0").add_sites(
TransformationSite(
"local",
"/pfn",
True,
arch=Arch.X86_64,
os_type=OS.LINUX,
os_release="1",
os_version="1",
container="cont",
).add_dagman_profile(retry="3").add_metadata(
JAVA_HOME="/usr/bin/java")).add_requirement(
"t2", namespace="test", version="1.0").add_shell_hook(
EventType.START, "echo hello")).add_containers(
Container(
"cont",
Container.DOCKER,
"docker:///ryan/centos-pegasus:latest",
mounts=["/Volumes/Work/lfs1:/shared-data/:ro"],
image_site="local",
).add_env(JAVA_HOME="/usr/bin/java")))
def test_tojson_without_profiles_hooks_metadata(
self, convert_yaml_schemas_to_json, load_schema):
t = Transformation("test", namespace="pegasus")
t.add_sites(TransformationSite("local", "/pfn", True))
t.add_requirement("required")
result = json.loads(json.dumps(t, cls=_CustomEncoder))
expected = {
"name": "test",
"namespace": "pegasus",
"requires": ["required"],
"sites": [{
"name": "local",
"pfn": "/pfn",
"type": "stageable"
}],
}
transformation_schema = load_schema(
"tc-5.0.json")["$defs"]["transformation"]
validate(instance=result, schema=transformation_schema)
assert result == expected
def test_tojson(self, convert_yaml_schemas_to_json, load_schema):
tc = TransformationCatalog()
(
tc.add_transformations(
Transformation("t1").add_sites(
TransformationSite("local", "/pfn", False)
)
)
.add_transformations(
Transformation("t2").add_sites(
TransformationSite("local", "/pfn", False)
)
)
.add_containers(
Container(
"container1",
Container.DOCKER,
"image",
arguments="--shm-size 123",
mounts=["mount1"],
bypass_staging=True,
)
)
.add_containers(
Container("container2", Container.DOCKER, "image", mounts=["mount1"])
)
)
expected = {
"pegasus": PEGASUS_VERSION,
"transformations": [
{
"name": "t1",
"sites": [{"name": "local", "pfn": "/pfn", "type": "installed"}],
},
{
"name": "t2",
"sites": [{"name": "local", "pfn": "/pfn", "type": "installed"}],
},
],
"containers": [
{
"name": "container1",
"type": "docker",
"image": "image",
"mounts": ["mount1"],
"bypass": True,
"profiles": {"pegasus": {"container.arguments": "--shm-size 123"}},
},
{
"name": "container2",
"type": "docker",
"image": "image",
"mounts": ["mount1"],
},
],
}
expected["transformations"] = sorted(
expected["transformations"], key=lambda t: t["name"]
)
expected["containers"] = sorted(expected["containers"], key=lambda c: c["name"])
result = json.loads(json.dumps(tc, cls=_CustomEncoder))
result["transformations"] = sorted(
result["transformations"], key=lambda t: t["name"]
)
result["containers"] = sorted(result["containers"], key=lambda c: c["name"])
tc_schema = load_schema("tc-5.0.json")
validate(instance=result, schema=tc_schema)
assert result == expected
def test_add_site(self):
t = Transformation("test")
t.add_sites(TransformationSite("local", "/pfn", True))
assert "local" in t.sites
class TestTransformationSite:
@pytest.mark.parametrize(
"name, pfn, transformation_type, kwargs",
[
(
"condorpool",
"/pfn",
False,
{
"bypass_staging": False,
"arch": Arch.X86_64,
"os_type": None,
"os_release": None,
"os_version": None,
"container": None,
},
),
(
"local",
"/pfn",
True,
{
"bypass_staging": False,
"arch": Arch.X86_64,
"os_type": None,
"os_release": None,
"os_version": None,
"container": None,
},
),
(
"local",
"/pfn",
True,
{
"arch": Arch.X86_64,
"os_type": OS.LINUX,
"os_release": None,
"os_version": None,
"container": None,
},
),
(
"local",
"/pfn",
True,
{
"arch": Arch.X86_64,
"os_type": OS.LINUX,
"os_release": "release",
"os_version": "1.1.1",
"container": "centos-pegasus",
},
),
(
"local",
"/pfn",
True,
{
"arch": Arch.X86_64,
"os_type": OS.LINUX,
"os_release": "release",
"os_version": "1.1.1",
"container": Container(
"centos-pegasus",
Container.DOCKER,
"docker:///ryan/centos-pegasus:latest",
),
},
),
],
)
def test_valid_transformation_site(
self, name: str, pfn: str, transformation_type: bool, kwargs: dict
):
assert TransformationSite(name, pfn, **kwargs)
@pytest.mark.parametrize(
"name, pfn, transformation_type, kwargs",
[
(
"local",
"/pfn",
True,
{
"arch": "should be one of Arch",
"os_type": None,
"os_release": None,
"os_version": None,
"container": None,
},
),
(
"local",
"/pfn",
True,
{
"arch": Arch.X86_64,
"os_type": "should be one of OS",
"os_release": None,
"os_version": None,
"container": None,
},
),
(
"local",
"/pfn",
True,
{
"arch": Arch.X86_64,
"os_type": "should be one of OS",
"os_release": None,
"os_version": None,
"container": 123,
},
),
],
)
def test_invalid_transformation_site(
self, name: str, pfn: str, transformation_type: bool, kwargs: dict
):
with pytest.raises(TypeError) as e:
TransformationSite(name, pfn, transformation_type, **kwargs)
assert "invalid" in str(e)
def test_invalid_use_of_bypass_staging(self):
with pytest.raises(ValueError) as e:
TransformationSite("local", "/pfn", False, bypass_staging=True)
assert (
"bypass_staging can only be used when is_stageable is set to True" in str(e)
)
def test_invalid_pfn(self):
with pytest.raises(ValueError) as e:
TransformationSite("local", Path("."), False)
assert "invalid pfn" in str(e)
@pytest.mark.parametrize(
"transformation_site, expected_json",
[
(
TransformationSite("local", "/pfn", True),
{"name": "local", "pfn": "/pfn", "type": "stageable"},
),
(
TransformationSite("local", "/pfn", True, bypass_staging=True),
{"name": "local", "pfn": "/pfn", "type": "stageable", "bypass": True},
),
(
TransformationSite("local", Path("/pfn"), True, bypass_staging=True),
{"name": "local", "pfn": "/pfn", "type": "stageable", "bypass": True},
),
(
TransformationSite(
"local",
"/pfn",
False,
bypass_staging=False,
arch=Arch.X86_64,
os_type=OS.LINUX,
os_release="release",
os_version="1.1.1",
container="centos-pegasus",
),
{
"name": "local",
"pfn": "/pfn",
"type": "installed",
"arch": "x86_64",
"os.type": "linux",
"os.release": "release",
"os.version": "1.1.1",
"container": "centos-pegasus",
},
),
(
TransformationSite(
"local",
"/pfn",
False,
arch=Arch.X86_64,
os_type=OS.LINUX,
os_release="release",
os_version="1.1.1",
container=Container(
"centos-pegasus",
Container.DOCKER,
"docker:///ryan/centos-pegasus:latest",
),
),
{
"name": "local",
"pfn": "/pfn",
"type": "installed",
"arch": "x86_64",
"os.type": "linux",
"os.release": "release",
"os.version": "1.1.1",
"container": "centos-pegasus",
},
),
],
)
def test_tojson_no_profiles_or_metadata(
self,
transformation_site: TransformationSite,
expected_json: dict,
convert_yaml_schemas_to_json,
load_schema,
):
result = transformation_site.__json__()
transformation_site_schema = load_schema("tc-5.0.json")["$defs"][
"transformation"
]["properties"]["sites"]["items"]
validate(instance=result, schema=transformation_site_schema)
assert transformation_site.__json__() == expected_json
def test_tojson_with_profiles_and_metadata(
self, convert_yaml_schemas_to_json, load_schema
):
t = (
TransformationSite("local", "/pfn", False)
.add_env(JAVA_HOME="/java/home")
.add_metadata(key="value")
)
result = t.__json__()
expected = {
"name": "local",
"pfn": "/pfn",
"type": "installed",
"profiles": {Namespace.ENV.value: {"JAVA_HOME": "/java/home"}},
"metadata": {"key": "value"},
}
transformation_site_schema = load_schema("tc-5.0.json")["$defs"][
"transformation"
]["properties"]["sites"]["items"]
validate(instance=result, schema=transformation_site_schema)
assert result == expected
def _to_tc(d: dict) -> TransformationCatalog:
"""Convert dict to TransformationCatalog
:param d: TransformationCatalog represented as a dict
:type d: dict
:raises PegasusError: encountered error parsing
:return: a TransformationCatalog object based on d
:rtype: TransformationCatalog
"""
try:
tc = TransformationCatalog()
# add transformations
for tr in d["transformations"]:
tr_to_add = Transformation(
tr["name"],
tr.get("namespace"),
tr.get("version"),
checksum=tr.get("checksum"),
)
# add transformation sites
for s in tr["sites"]:
site_to_add = TransformationSite(
s["name"],
s["pfn"],
True if s["type"] == "stageable" else False,
bypass_staging=s.get("bypass"),
arch=getattr(Arch,
s.get("arch").upper())
if s.get("arch") else None,
os_type=getattr(OS,
s.get("os.type").upper())
if s.get("os.type") else None,
os_release=s.get("os.release"),
os_version=s.get("os.version"),
container=s.get("container"),
)
# add profiles
if s.get("profiles"):
site_to_add.profiles = defaultdict(dict, s.get("profiles"))
# add metadata
if s.get("metadata"):
site_to_add.metadata = s.get("metadata")
# add site to this tr
tr_to_add.add_sites(site_to_add)
# add requires
if tr.get("requires"):
tr_to_add.requires = set(tr.get("requires"))
# add profiles
if tr.get("profiles"):
tr_to_add.profiles = defaultdict(dict, tr.get("profiles"))
# add hooks
if tr.get("hooks"):
tr_to_add.hooks = defaultdict(list, tr.get("hooks"))
# add metadata
if tr.get("metadata"):
tr_to_add.metadata = tr.get("metadata")
# add tr to tc
tc.add_transformations(tr_to_add)
# add containers
if "containers" in d:
for cont in d["containers"]:
cont_to_add = Container(
cont["name"],
getattr(Container, cont["type"].upper()),
cont["image"],
mounts=cont.get("mounts"),
image_site=cont.get("image.site"),
checksum=cont.get("checksum"),
bypass_staging=cont.get("bypass"),
)
# add profiles
if cont.get("profiles"):
cont_to_add.profiles = defaultdict(dict,
cont.get("profiles"))
# add cont to tc
tc.add_containers(cont_to_add)
return tc
except KeyError:
raise PegasusError("error parsing {}".format(d))
def test_valid_transformation_site(
self, name: str, pfn: str, transformation_type: bool, kwargs: dict
):
assert TransformationSite(name, pfn, **kwargs)
def test_invalid_pfn(self):
with pytest.raises(ValueError) as e:
TransformationSite("local", Path("."), False)
assert "invalid pfn" in str(e)