def _to_rc(d: dict) -> ReplicaCatalog:
"""Convert dict to ReplicaCatalog
:param d: ReplicaCatalog represented as a dict
:type d: dict
:raises PegasusError: encountered error parsing
:return: a ReplicaCatalog object based on d
:rtype: ReplicaCatalog
"""
rc = ReplicaCatalog()
try:
for r in d["replicas"]:
site = r["site"]
lfn = r["lfn"]
pfn = r["pfn"]
regex = r.get("regex")
if not regex:
regex = False
checksum = r.get("checksum")
_type = None
value = None
if checksum:
_type = checksum["type"]
value = checksum["value"]
rc.add_replica(site,
lfn,
pfn,
regex=regex,
checksum_type=_type,
checksum_value=value)
except KeyError:
raise PegasusError("error parsing {}".format(d))
return rc
def _to_rc(d: dict) -> ReplicaCatalog:
"""Convert dict to ReplicaCatalog
:param d: ReplicaCatalog represented as a dict
:type d: dict
:raises PegasusError: encountered error parsing
:return: a ReplicaCatalog object based on d
:rtype: ReplicaCatalog
"""
rc = ReplicaCatalog()
try:
for r in d["replicas"]:
lfn = r["lfn"]
pfns = {_PFN(i["site"], i["pfn"]) for i in r["pfns"]}
checksum = r.get("checksum") if r.get("checksum") else {}
metadata = r.get("metadata") if r.get("metadata") else {}
regex = r.get("regex")
for pfn in pfns:
if regex:
rc.add_regex_replica(pfn.site,
lfn,
pfn.pfn,
metadata=metadata)
else:
rc.add_replica(pfn.site,
lfn,
pfn.pfn,
metadata=metadata,
checksum=checksum)
except KeyError:
raise PegasusError("error parsing {}".format(d))
return rc
def _to_wf(d: dict) -> Workflow:
"""Convert dict to Workflow
:param d: Workflow represented as a dict
:type d: dict
:raises PegasusError: encountered error parsing
:return: a Workflow object based on d
:rtype: Workflow
"""
try:
#
wf = Workflow(d["name"], infer_dependencies=False)
# add rc
if "replicaCatalog" in d:
wf.replica_catalog = _to_rc(d["replicaCatalog"])
# add tc
if "transformationCatalog" in d:
wf.transformation_catalog = _to_tc(d["transformationCatalog"])
# add sc
if "siteCatalog" in d:
wf.site_catalog = _to_sc(d["siteCatalog"])
# add jobs
for j in d["jobs"]:
# create appropriate job based on type
if j["type"] == "job":
job = Job(
j["name"],
_id=j["id"],
node_label=j.get("nodeLabel"),
namespace=j.get("namespace"),
version=j.get("version"),
)
elif j["type"] in {"pegasusWorkflow", "condorWorkflow"}:
f = File(j["file"])
is_planned = False if j["type"] == "pegasusWorkflow" else True
job = SubWorkflow(
f, is_planned, _id=j["id"], node_label=j.get("nodeLabel")
)
else:
raise ValueError
# add args
args = list()
for a in j["arguments"]:
args.append(a)
job.args = args
# add uses
uses = set()
for u in j["uses"]:
f = File(u["lfn"], size=u.get("size"))
try:
f.metadata = u["metadata"]
except KeyError:
pass
uses.add(
_Use(
f,
getattr(_LinkType, u["type"].upper()),
stage_out=u.get("stageOut"),
register_replica=u.get("registerReplica"),
bypass_staging=u.get("bypass"),
)
)
job.uses = uses
# set stdin
if "stdin" in j:
for u in job.uses:
if u.file.lfn == j["stdin"]:
job.stdin = u.file
break
# set stdout
if "stdout" in j:
for u in job.uses:
if u.file.lfn == j["stdout"]:
job.stdout = u.file
break
# set stderr
if "stderr" in j:
for u in job.uses:
if u.file.lfn == j["stderr"]:
job.stderr = u.file
break
# add profiles
if j.get("profiles"):
job.profiles = defaultdict(dict, j.get("profiles"))
# add metadata
if j.get("metadata"):
job.metadata = j.get("metadata")
# add hooks
if j.get("hooks"):
job.hooks = defaultdict(list, j.get("hooks"))
# add job to wf
wf.add_jobs(job)
# add dependencies
if d.get("jobDependencies"):
dependencies = defaultdict(_JobDependency)
for item in d.get("jobDependencies"):
dependencies[item["id"]] = _JobDependency(
item["id"], {child for child in item["children"]}
)
wf.dependencies = dependencies
# add profiles
if d.get("profiles"):
wf.profiles = defaultdict(dict, d.get("profiles"))
# add metadata
if d.get("metadata"):
wf.metadata = d.get("metadata")
# add hooks
if d.get("hooks"):
wf.hooks = defaultdict(list, d.get("hooks"))
return wf
except (KeyError, ValueError):
raise PegasusError("error parsing {}".format(d))
def _to_sc(d: dict) -> SiteCatalog:
"""Convert dict to SiteCatalog
:param d: SiteCatalog represented as a dict
:type d: dict
:raises PegasusError: encountered error parsing
:return: a SiteCatalog object based on d
:rtype: SiteCatalog
"""
try:
sc = SiteCatalog()
for s in d["sites"]:
site = Site(
s["name"],
arch=getattr(Arch,
s.get("arch").upper()) if s.get("arch") else None,
os_type=getattr(OS,
s.get("os.type").upper())
if s.get("os.type") else None,
os_release=s.get("os.release"),
os_version=s.get("os.version"),
)
# add directories
for _dir in s["directories"]:
dir_type = None
for enum_name, enum in _DirectoryType.__members__.items():
if _dir["type"] == enum.value:
dir_type = enum_name
break
directory = Directory(getattr(Directory, dir_type),
_dir["path"])
# add file servers
for fs in _dir["fileServers"]:
file_server = FileServer(
fs["url"], getattr(Operation, fs["operation"].upper()))
# add profiles
if fs.get("profiles"):
file_server.profiles = defaultdict(
dict, fs.get("profiles"))
# add file server to this directory
directory.add_file_servers(file_server)
# add directory to this site
site.add_directories(directory)
# add grids
if s.get("grids"):
for gr in s.get("grids"):
grid = Grid(
getattr(Grid, gr["type"].upper()),
gr["contact"],
getattr(Scheduler, gr["scheduler"].upper()),
job_type=getattr(SupportedJobs,
gr.get("jobtype").upper())
if gr.get("jobtype") else None,
)
# add grid to this site
site.add_grids(grid)
# add profiles
if s.get("profiles"):
site.profiles = defaultdict(dict, s.get("profiles"))
# add site to sc
sc.add_sites(site)
return sc
except KeyError:
raise PegasusError("error parsing {}".format(d))
def _to_tc(d: dict) -> TransformationCatalog:
"""Convert dict to TransformationCatalog
:param d: TransformationCatalog represented as a dict
:type d: dict
:raises PegasusError: encountered error parsing
:return: a TransformationCatalog object based on d
:rtype: TransformationCatalog
"""
try:
tc = TransformationCatalog()
# add transformations
for tr in d["transformations"]:
tr_to_add = Transformation(
tr["name"],
tr.get("namespace"),
tr.get("version"),
checksum=tr.get("checksum"),
)
# add transformation sites
for s in tr["sites"]:
site_to_add = TransformationSite(
s["name"],
s["pfn"],
True if s["type"] == "stageable" else False,
bypass_staging=s.get("bypass"),
arch=getattr(Arch,
s.get("arch").upper())
if s.get("arch") else None,
os_type=getattr(OS,
s.get("os.type").upper())
if s.get("os.type") else None,
os_release=s.get("os.release"),
os_version=s.get("os.version"),
container=s.get("container"),
)
# add profiles
if s.get("profiles"):
site_to_add.profiles = defaultdict(dict, s.get("profiles"))
# add metadata
if s.get("metadata"):
site_to_add.metadata = s.get("metadata")
# add site to this tr
tr_to_add.add_sites(site_to_add)
# add requires
if tr.get("requires"):
tr_to_add.requires = set(tr.get("requires"))
# add profiles
if tr.get("profiles"):
tr_to_add.profiles = defaultdict(dict, tr.get("profiles"))
# add hooks
if tr.get("hooks"):
tr_to_add.hooks = defaultdict(list, tr.get("hooks"))
# add metadata
if tr.get("metadata"):
tr_to_add.metadata = tr.get("metadata")
# add tr to tc
tc.add_transformations(tr_to_add)
# add containers
if "containers" in d:
for cont in d["containers"]:
cont_to_add = Container(
cont["name"],
getattr(Container, cont["type"].upper()),
cont["image"],
mounts=cont.get("mounts"),
image_site=cont.get("image.site"),
checksum=cont.get("checksum"),
bypass_staging=cont.get("bypass"),
)
# add profiles
if cont.get("profiles"):
cont_to_add.profiles = defaultdict(dict,
cont.get("profiles"))
# add cont to tc
tc.add_containers(cont_to_add)
return tc
except KeyError:
raise PegasusError("error parsing {}".format(d))