def test_add_duplicate_site(self): with pytest.raises(DuplicateError) as e: t = Transformation("test") t.add_sites(TransformationSite("local", "/pfn", True)) t.add_sites(TransformationSite("isi", "/pfn", True)) t.add_sites(TransformationSite("local", "/pfn", True)) assert "local" in str(e)
def test_example_transformation_catalog( self, convert_yaml_schemas_to_json, load_schema, _format, loader ): # validates the sample tc in pegasus/etc/sample-5.0-data/tc.yml tc = TransformationCatalog() foo = ( Transformation("foo") .add_globus_profile(max_time=2) .add_dagman_profile(retry=2) .add_metadata(size=2048) .add_sites( TransformationSite( "local", "/nfs/u2/ryan/bin/foo", True, arch=Arch.X86_64, os_type=OS.LINUX, ) .add_env(JAVA_HOME="/usr/bin/java") .add_metadata(size=2048) ) .add_requirement("bar") .add_shell_hook(EventType.START, "/bin/echo 'starting'") ) bar = Transformation("bar").add_sites( TransformationSite( "local", "/nfs/u2/ryan/bin/bar", True, arch=Arch.X86_64, os_type=OS.LINUX, ) ) centos_pegasus_container = Container( "centos-pegasus", Container.DOCKER, "docker:///ryan/centos-pegasus:latest", arguments="--shm-size 123", mounts=["/Volumes/Work/lfs1:/shared-data/:ro"], ).add_env(JAVA_HOME="/usr/bin/java") (tc.add_transformations(foo, bar).add_containers(centos_pegasus_container)) with NamedTemporaryFile(mode="r+") as f: tc.write(f, _format=_format) f.seek(0) tc_json = loader(f) tc_schema = load_schema("tc-5.0.json") validate(instance=tc_json, schema=tc_schema)
def test_tojson_no_profiles_or_metadata( self, transformation_site: TransformationSite, expected_json: dict, convert_yaml_schemas_to_json, load_schema, ): result = transformation_site.__json__() transformation_site_schema = load_schema("tc-5.0.json")["$defs"][ "transformation"]["properties"]["sites"]["items"] validate(instance=result, schema=transformation_site_schema) assert transformation_site.__json__() == expected_json
def test_tojson_no_containers(self, convert_yaml_schemas_to_json, load_schema): tc = TransformationCatalog() (tc.add_transformations( Transformation("t1").add_sites( TransformationSite("local", "/pfn", False))).add_transformations( Transformation("t2").add_sites( TransformationSite( "local2", "/pfn", True)))) expected = { "pegasus": PEGASUS_VERSION, "transformations": [ { "name": "t1", "sites": [{ "name": "local", "pfn": "/pfn", "type": "installed" }], }, { "name": "t2", "sites": [{ "name": "local2", "pfn": "/pfn", "type": "stageable" }], }, ], } expected["transformations"] = sorted(expected["transformations"], key=lambda t: t["name"]) result = json.loads(json.dumps(tc, cls=_CustomEncoder)) result["transformations"] = sorted(result["transformations"], key=lambda t: t["name"]) tc_schema = load_schema("tc-5.0.json") validate(instance=result, schema=tc_schema) assert expected == result
def test_tojson_with_profiles_hooks_metadata( self, convert_yaml_schemas_to_json, load_schema ): t = Transformation("test", namespace="pegasus") t.add_sites( TransformationSite("local", "/pfn", True).add_env(JAVA_HOME="/java/home") ) t.add_requirement("required") t.add_env(JAVA_HOME="/java/home") t.add_shell_hook(EventType.START, "/bin/echo hi") t.add_metadata(key="value") result = json.loads(json.dumps(t, cls=_CustomEncoder)) expected = { "name": "test", "namespace": "pegasus", "requires": ["required"], "sites": [ { "name": "local", "pfn": "/pfn", "type": "stageable", "profiles": {"env": {"JAVA_HOME": "/java/home"}}, } ], "metadata": {"key": "value"}, "profiles": {Namespace.ENV.value: {"JAVA_HOME": "/java/home"}}, "hooks": {"shell": [{"_on": EventType.START.value, "cmd": "/bin/echo hi"}]}, } transformation_schema = load_schema("tc-5.0.json")["$defs"]["transformation"] validate(instance=result, schema=transformation_schema) assert result == expected
def test_tojson_with_profiles_and_metadata( self, convert_yaml_schemas_to_json, load_schema ): t = ( TransformationSite("local", "/pfn", False) .add_env(JAVA_HOME="/java/home") .add_metadata(key="value") ) result = t.__json__() expected = { "name": "local", "pfn": "/pfn", "type": "installed", "profiles": {Namespace.ENV.value: {"JAVA_HOME": "/java/home"}}, "metadata": {"key": "value"}, } transformation_site_schema = load_schema("tc-5.0.json")["$defs"][ "transformation" ]["properties"]["sites"]["items"] validate(instance=result, schema=transformation_site_schema) assert result == expected
def test_invalid_use_of_bypass_staging(self): with pytest.raises(ValueError) as e: TransformationSite("local", "/pfn", False, bypass_staging=True) assert ( "bypass_staging can only be used when is_stageable is set to True" in str(e) )
def test_invalid_transformation_site( self, name: str, pfn: str, transformation_type: bool, kwargs: dict ): with pytest.raises(TypeError) as e: TransformationSite(name, pfn, transformation_type, **kwargs) assert "invalid" in str(e)
def test_chaining(self): t = (Transformation("test").add_sites( TransformationSite("local", "/pfn", True).add_env( JAVA_HOME="/java/home")).add_requirement("required")) assert "local" in t.sites assert t.sites["local"].profiles["env"]["JAVA_HOME"] == "/java/home" assert "required" in t.requires
def tc2(): return (TransformationCatalog().add_transformations( Transformation("t1", namespace="test", version="1.0").add_sites( TransformationSite( "local", "/pfn", True, ))).add_containers( Container( "cont", Container.DOCKER, "docker:///ryan/centos-pegasus:latest", mounts=["/Volumes/Work/lfs1:/shared-data/:ro"], image_site="local", )))
def tc1(): return (TransformationCatalog().add_transformations( Transformation("t1", namespace="test", version="1.0").add_sites( TransformationSite( "local", "/pfn", True, arch=Arch.X86_64, os_type=OS.LINUX, os_release="1", os_version="1", container="cont", ).add_dagman_profile(retry="3").add_metadata( JAVA_HOME="/usr/bin/java")).add_requirement( "t2", namespace="test", version="1.0").add_shell_hook( EventType.START, "echo hello")).add_containers( Container( "cont", Container.DOCKER, "docker:///ryan/centos-pegasus:latest", mounts=["/Volumes/Work/lfs1:/shared-data/:ro"], image_site="local", ).add_env(JAVA_HOME="/usr/bin/java")))
def test_tojson_without_profiles_hooks_metadata( self, convert_yaml_schemas_to_json, load_schema): t = Transformation("test", namespace="pegasus") t.add_sites(TransformationSite("local", "/pfn", True)) t.add_requirement("required") result = json.loads(json.dumps(t, cls=_CustomEncoder)) expected = { "name": "test", "namespace": "pegasus", "requires": ["required"], "sites": [{ "name": "local", "pfn": "/pfn", "type": "stageable" }], } transformation_schema = load_schema( "tc-5.0.json")["$defs"]["transformation"] validate(instance=result, schema=transformation_schema) assert result == expected
def test_tojson(self, convert_yaml_schemas_to_json, load_schema): tc = TransformationCatalog() ( tc.add_transformations( Transformation("t1").add_sites( TransformationSite("local", "/pfn", False) ) ) .add_transformations( Transformation("t2").add_sites( TransformationSite("local", "/pfn", False) ) ) .add_containers( Container( "container1", Container.DOCKER, "image", arguments="--shm-size 123", mounts=["mount1"], bypass_staging=True, ) ) .add_containers( Container("container2", Container.DOCKER, "image", mounts=["mount1"]) ) ) expected = { "pegasus": PEGASUS_VERSION, "transformations": [ { "name": "t1", "sites": [{"name": "local", "pfn": "/pfn", "type": "installed"}], }, { "name": "t2", "sites": [{"name": "local", "pfn": "/pfn", "type": "installed"}], }, ], "containers": [ { "name": "container1", "type": "docker", "image": "image", "mounts": ["mount1"], "bypass": True, "profiles": {"pegasus": {"container.arguments": "--shm-size 123"}}, }, { "name": "container2", "type": "docker", "image": "image", "mounts": ["mount1"], }, ], } expected["transformations"] = sorted( expected["transformations"], key=lambda t: t["name"] ) expected["containers"] = sorted(expected["containers"], key=lambda c: c["name"]) result = json.loads(json.dumps(tc, cls=_CustomEncoder)) result["transformations"] = sorted( result["transformations"], key=lambda t: t["name"] ) result["containers"] = sorted(result["containers"], key=lambda c: c["name"]) tc_schema = load_schema("tc-5.0.json") validate(instance=result, schema=tc_schema) assert result == expected
def test_add_site(self): t = Transformation("test") t.add_sites(TransformationSite("local", "/pfn", True)) assert "local" in t.sites
class TestTransformationSite: @pytest.mark.parametrize( "name, pfn, transformation_type, kwargs", [ ( "condorpool", "/pfn", False, { "bypass_staging": False, "arch": Arch.X86_64, "os_type": None, "os_release": None, "os_version": None, "container": None, }, ), ( "local", "/pfn", True, { "bypass_staging": False, "arch": Arch.X86_64, "os_type": None, "os_release": None, "os_version": None, "container": None, }, ), ( "local", "/pfn", True, { "arch": Arch.X86_64, "os_type": OS.LINUX, "os_release": None, "os_version": None, "container": None, }, ), ( "local", "/pfn", True, { "arch": Arch.X86_64, "os_type": OS.LINUX, "os_release": "release", "os_version": "1.1.1", "container": "centos-pegasus", }, ), ( "local", "/pfn", True, { "arch": Arch.X86_64, "os_type": OS.LINUX, "os_release": "release", "os_version": "1.1.1", "container": Container( "centos-pegasus", Container.DOCKER, "docker:///ryan/centos-pegasus:latest", ), }, ), ], ) def test_valid_transformation_site( self, name: str, pfn: str, transformation_type: bool, kwargs: dict ): assert TransformationSite(name, pfn, **kwargs) @pytest.mark.parametrize( "name, pfn, transformation_type, kwargs", [ ( "local", "/pfn", True, { "arch": "should be one of Arch", "os_type": None, "os_release": None, "os_version": None, "container": None, }, ), ( "local", "/pfn", True, { "arch": Arch.X86_64, "os_type": "should be one of OS", "os_release": None, "os_version": None, "container": None, }, ), ( "local", "/pfn", True, { "arch": Arch.X86_64, "os_type": "should be one of OS", "os_release": None, "os_version": None, "container": 123, }, ), ], ) def test_invalid_transformation_site( self, name: str, pfn: str, transformation_type: bool, kwargs: dict ): with pytest.raises(TypeError) as e: TransformationSite(name, pfn, transformation_type, **kwargs) assert "invalid" in str(e) def test_invalid_use_of_bypass_staging(self): with pytest.raises(ValueError) as e: TransformationSite("local", "/pfn", False, bypass_staging=True) assert ( "bypass_staging can only be used when is_stageable is set to True" in str(e) ) def test_invalid_pfn(self): with pytest.raises(ValueError) as e: TransformationSite("local", Path("."), False) assert "invalid pfn" in str(e) @pytest.mark.parametrize( "transformation_site, expected_json", [ ( TransformationSite("local", "/pfn", True), {"name": "local", "pfn": "/pfn", "type": "stageable"}, ), ( TransformationSite("local", "/pfn", True, bypass_staging=True), {"name": "local", "pfn": "/pfn", "type": "stageable", "bypass": True}, ), ( TransformationSite("local", Path("/pfn"), True, bypass_staging=True), {"name": "local", "pfn": "/pfn", "type": "stageable", "bypass": True}, ), ( TransformationSite( "local", "/pfn", False, bypass_staging=False, arch=Arch.X86_64, os_type=OS.LINUX, os_release="release", os_version="1.1.1", container="centos-pegasus", ), { "name": "local", "pfn": "/pfn", "type": "installed", "arch": "x86_64", "os.type": "linux", "os.release": "release", "os.version": "1.1.1", "container": "centos-pegasus", }, ), ( TransformationSite( "local", "/pfn", False, arch=Arch.X86_64, os_type=OS.LINUX, os_release="release", os_version="1.1.1", container=Container( "centos-pegasus", Container.DOCKER, "docker:///ryan/centos-pegasus:latest", ), ), { "name": "local", "pfn": "/pfn", "type": "installed", "arch": "x86_64", "os.type": "linux", "os.release": "release", "os.version": "1.1.1", "container": "centos-pegasus", }, ), ], ) def test_tojson_no_profiles_or_metadata( self, transformation_site: TransformationSite, expected_json: dict, convert_yaml_schemas_to_json, load_schema, ): result = transformation_site.__json__() transformation_site_schema = load_schema("tc-5.0.json")["$defs"][ "transformation" ]["properties"]["sites"]["items"] validate(instance=result, schema=transformation_site_schema) assert transformation_site.__json__() == expected_json def test_tojson_with_profiles_and_metadata( self, convert_yaml_schemas_to_json, load_schema ): t = ( TransformationSite("local", "/pfn", False) .add_env(JAVA_HOME="/java/home") .add_metadata(key="value") ) result = t.__json__() expected = { "name": "local", "pfn": "/pfn", "type": "installed", "profiles": {Namespace.ENV.value: {"JAVA_HOME": "/java/home"}}, "metadata": {"key": "value"}, } transformation_site_schema = load_schema("tc-5.0.json")["$defs"][ "transformation" ]["properties"]["sites"]["items"] validate(instance=result, schema=transformation_site_schema) assert result == expected
def _to_tc(d: dict) -> TransformationCatalog: """Convert dict to TransformationCatalog :param d: TransformationCatalog represented as a dict :type d: dict :raises PegasusError: encountered error parsing :return: a TransformationCatalog object based on d :rtype: TransformationCatalog """ try: tc = TransformationCatalog() # add transformations for tr in d["transformations"]: tr_to_add = Transformation( tr["name"], tr.get("namespace"), tr.get("version"), checksum=tr.get("checksum"), ) # add transformation sites for s in tr["sites"]: site_to_add = TransformationSite( s["name"], s["pfn"], True if s["type"] == "stageable" else False, bypass_staging=s.get("bypass"), arch=getattr(Arch, s.get("arch").upper()) if s.get("arch") else None, os_type=getattr(OS, s.get("os.type").upper()) if s.get("os.type") else None, os_release=s.get("os.release"), os_version=s.get("os.version"), container=s.get("container"), ) # add profiles if s.get("profiles"): site_to_add.profiles = defaultdict(dict, s.get("profiles")) # add metadata if s.get("metadata"): site_to_add.metadata = s.get("metadata") # add site to this tr tr_to_add.add_sites(site_to_add) # add requires if tr.get("requires"): tr_to_add.requires = set(tr.get("requires")) # add profiles if tr.get("profiles"): tr_to_add.profiles = defaultdict(dict, tr.get("profiles")) # add hooks if tr.get("hooks"): tr_to_add.hooks = defaultdict(list, tr.get("hooks")) # add metadata if tr.get("metadata"): tr_to_add.metadata = tr.get("metadata") # add tr to tc tc.add_transformations(tr_to_add) # add containers if "containers" in d: for cont in d["containers"]: cont_to_add = Container( cont["name"], getattr(Container, cont["type"].upper()), cont["image"], mounts=cont.get("mounts"), image_site=cont.get("image.site"), checksum=cont.get("checksum"), bypass_staging=cont.get("bypass"), ) # add profiles if cont.get("profiles"): cont_to_add.profiles = defaultdict(dict, cont.get("profiles")) # add cont to tc tc.add_containers(cont_to_add) return tc except KeyError: raise PegasusError("error parsing {}".format(d))
def test_valid_transformation_site( self, name: str, pfn: str, transformation_type: bool, kwargs: dict ): assert TransformationSite(name, pfn, **kwargs)
def test_invalid_pfn(self): with pytest.raises(ValueError) as e: TransformationSite("local", Path("."), False) assert "invalid pfn" in str(e)