def simplify_services(self, services):
"""
Simplify Services attribute and record the Name -> ID mapping
"""
if not isinstance(services, dict):
return None
new_services = simplify_attr_list(services["Service"], "Name")
for name, data in new_services.items():
if not is_null(data, "Details"):
new_details = dict()
for key, val in data["Details"].items():
if not is_null(val):
new_details[key] = val
if len(new_details) > 0:
data["Details"] = dict(new_details)
else:
del data["Details"]
else:
data.pop("Details", None)
if "ID" in data:
if name not in self.services:
self.services[name] = data["ID"]
else:
if data["ID"] != self.services[name]:
raise RuntimeError(
"Identical services with different IDs: %s" % name)
del data["ID"]
return new_services
def simplify_contacttypes(contacttypes):
"""Simplify ContactTypes attribute
Turn e.g.
{"ContactType":
[{"Contacts":
{"Contact": [{"Name": "Steve Timm"},
{"Name": "Joe Boyd"}]},
"Type": "Miscellaneous Contact"}
]
}
into
{"Miscellanous Contact":
[ "Steve Timm", "Joe Boyd" ]
}
"""
if is_null(contacttypes, "ContactType"):
return None
new_contacttypes = {}
for ct in ensure_list(contacttypes["ContactType"]):
if is_null(ct, "Contacts", "Contact"):
continue
type_ = ct["Type"]
# Remove duplicates but keep ordering
contacts = []
for c in ensure_list(ct["Contacts"]["Contact"]):
if c["Name"] not in contacts:
contacts.append(c["Name"])
new_contacttypes[type_] = contacts
return new_contacttypes
def simplify_fields_of_science(fos: Dict) -> Union[Dict, None]:
"""Turn
{"PrimaryFields": {"Field": ["P1", "P2", ...]},
"SecondaryFields": {"Field": ["S1", "S2", ...]}}
into
{"PrimaryFields": ["P1", "P2", ...],
"SecondaryFields": ["S1", "S2", ...]}
"""
if is_null(fos, "PrimaryFields") or is_null(fos["PrimaryFields"], "Field"):
return None
new_fields = {"PrimaryFields": ensure_list(fos["PrimaryFields"]["Field"])}
if not is_null(fos, "SecondaryFields", "Field"):
new_fields["SecondaryFields"] = ensure_list(fos["SecondaryFields"]["Field"])
return new_fields
def expand_resource(name: str, res: Dict,
service_name_to_id: Dict[str, int]) -> OrderedDict:
"""Expand a single Resource from the format in a yaml file to the xml format.
Services, VOOwnership, FQDNAliases, ContactLists are expanded;
``name`` is inserted into the Resource as the "Name" attribute;
Defaults are added for VOOwnership, FQDNAliases, and WLCGInformation if they're missing from the yaml file.
Return the data structure for the expanded Resource as an OrderedDict to fit the xml schema.
"""
defaults = {
"ContactLists": None,
"FQDNAliases": None,
"Services": "no applicable service exists",
"VOOwnership": "(Information not available)",
"WLCGInformation": "(Information not available)",
}
res = dict(res)
if not is_null(res, "Services"):
res["Services"] = expand_services(res["Services"], service_name_to_id)
else:
res.pop("Services", None)
if "VOOwnership" in res:
res["VOOwnership"] = expand_voownership(res["VOOwnership"])
if "FQDNAliases" in res:
res["FQDNAliases"] = {
"FQDNAlias": singleton_list_to_value(res["FQDNAliases"])
}
if not is_null(res, "ContactLists"):
res["ContactLists"] = expand_contactlists(res["ContactLists"])
res["Name"] = name
if "WLCGInformation" in res and isinstance(res["WLCGInformation"], dict):
res["WLCGInformation"] = expand_wlcginformation(res["WLCGInformation"])
new_res = OrderedDict()
for elem in [
"ID", "Name", "Active", "Disable", "Services", "Description",
"FQDN", "FQDNAliases", "VOOwnership", "WLCGInformation",
"ContactLists"
]:
if elem in res:
new_res[elem] = res[elem]
elif elem in defaults:
new_res[elem] = defaults[elem]
return new_res
def simplify_oasis_managers(managers):
"""Simplify OASIS/Managers attributes
Turn
{"Manager": [{"Name": "a", "DNs": {"DN": [...]}}]}
into
{"a": {"DNs": [...]}}
"""
if is_null(managers, "Manager"):
return None
new_managers = simplify_attr_list(managers["Manager"], "Name")
for manager, data in new_managers.items():
if not is_null(data, "DNs"):
data["DNs"] = data["DNs"]["DN"]
if not is_null(data, "ContactID"):
data["ContactID"] = int(data["ContactID"])
return new_managers
def simplify_reportinggroups(reportinggroups):
"""Simplify ReportingGroups attributes
Turn e.g.
{"ReportingGroup": [{"Contacts": {"Contact": [{"Name": "a"},
{"Name": "b"}
},
"FQANs": {"FQAN": [{"GroupName": "XXX",
"Role": "YYY"}]
}
"Name": "ZZZ"
}]
}
into
{"ZZZ": {"Contacts": ["a", "b"],
"FQANs": [{"GroupName": "XXX", "Role": "YYY"}]
}
}
"""
if is_null(reportinggroups, "ReportingGroup"):
return None
# [{"Name": "XXX", <...>}, {"Name": "YYY", <...>}] becomes
# {"XXX": {<...>}, "YYY": {<...>}>
new_reportinggroups = simplify_attr_list(reportinggroups["ReportingGroup"], "Name")
if not new_reportinggroups: # only null entries found
return None
for rgname, rgdata in new_reportinggroups.items():
if not is_null(rgdata["Contacts"], "Contact"):
# {"Contacts": {"Contact": [{"Name": "a"}, {"Name": "b"}]}} becomes
# {"Contacts": ["a", "b"]}
new_contacts = []
for c in ensure_list(rgdata["Contacts"]["Contact"]):
if not is_null(c, "Name") and c["Name"] not in new_contacts:
new_contacts.append(c["Name"])
rgdata["Contacts"] = new_contacts
if not is_null(rgdata["FQANs"], "FQAN"):
rgdata["FQANs"] = ensure_list(rgdata["FQANs"]["FQAN"])
return new_reportinggroups
def expand_fields_of_science(fields_of_science):
"""Turn
{"PrimaryFields": ["P1", "P2", ...],
"SecondaryFields": ["S1", "S2", ...]}
into
{"PrimaryFields": {"Field": ["P1", "P2", ...]},
"SecondaryFields": {"Field": ["S1", "S2", ...]}}
"""
if is_null(fields_of_science, "PrimaryFields"):
return None
new_fields = OrderedDict()
new_fields["PrimaryFields"] = {
"Field": singleton_list_to_value(fields_of_science["PrimaryFields"])
}
if not is_null(fields_of_science, "SecondaryFields"):
new_fields["SecondaryFields"] = {
"Field":
singleton_list_to_value(fields_of_science["SecondaryFields"])
}
return new_fields
def expand_reportinggroups(reportinggroups_list: List,
reportinggroups_data: Dict) -> Dict:
"""Expand
["XXX", "YYY", "ZZZ"]
using data from reportinggroups_data into
{"ReportingGroup": [{"Contacts": {"Contact": [{"Name": "a"},
{"Name": "b"}
},
"FQANs": {"FQAN": [{"GroupName": "...",
"Role": "..."}]
}
"Name": "XXX"
}]
}
"""
new_reportinggroups = {}
for name, data in reportinggroups_data.items():
if name not in reportinggroups_list: continue
new_reportinggroups[name] = {}
newdata = new_reportinggroups[name]
if not is_null(data, "Contacts"):
new_contact = [{"Name": x} for x in data["Contacts"]]
newdata["Contacts"] = {
"Contact": singleton_list_to_value(new_contact)
}
else:
newdata["Contacts"] = None
if not is_null(data, "FQANs"):
fqans = []
for fqan in data["FQANs"]:
fqans.append(
OrderedDict([("GroupName", fqan["GroupName"]),
("Role", fqan["Role"])]))
newdata["FQANs"] = {"FQAN": singleton_list_to_value(fqans)}
else:
newdata["FQANs"] = None
new_reportinggroups = expand_attr_list(
new_reportinggroups, "Name", ordering=["Name", "FQANs", "Contacts"])
return {"ReportingGroup": new_reportinggroups}
def simplify_voownership(self, voownership):
"""
Simplify VOOwnership attribute
"""
if is_null(voownership):
return None
voownership = dict(voownership) # copy
del voownership["ChartURL"] # can be derived from the other attributes
new_voownership = simplify_attr_list(voownership["Ownership"], "VO")
new_voownership.pop("(Other)",
None) # can be derived from the other attributes
for vo in new_voownership:
new_voownership[vo] = int(new_voownership[vo]["Percent"])
return new_voownership
def simplify_resource(self, res: Dict) -> Dict:
res = dict(res)
services = self.simplify_services(res["Services"])
if is_null(services):
res.pop("Services", None)
else:
res["Services"] = services
res["VOOwnership"] = self.simplify_voownership(res.get("VOOwnership"))
if not res["VOOwnership"]:
del res["VOOwnership"]
if not is_null(res, "WLCGInformation"):
new_wlcg = {}
for key, val in res["WLCGInformation"].items():
if not is_null(val):
new_wlcg[key] = val
if len(new_wlcg) > 0:
res["WLCGInformation"] = new_wlcg
else:
del res["WLCGInformation"]
else:
res.pop("WLCGInformation", None)
if not is_null(res, "FQDNAliases"):
aliases = []
for a in ensure_list(res["FQDNAliases"]["FQDNAlias"]):
aliases.append(a)
res["FQDNAliases"] = aliases
else:
res.pop("FQDNAliases", None)
if not is_null(res, "ContactLists"):
new_contactlists = self.simplify_contactlists(res["ContactLists"])
if new_contactlists:
res["ContactLists"] = new_contactlists
else:
res.pop("ContactLists", None)
return res
def simplify_contactlists(self, contactlists):
"""Simplify ContactLists attribute
Turn e.g.
{"ContactList":
[{"ContactType": "Administrative Contact",
{"Contacts":
{"Contact":
[{"Name": "Andrew Malone Melo", "ContactRank": "Primary"},
{"Name": "Paul Sheldon", "ContactRank": "Secondary"}]
}
}
}]
}
into
{"Administrative Contact":
{"Primary": "Andrew Malone Melo",
"Secondary": "Paul Sheldon"}
}
"""
if is_null(contactlists):
return None
contactlists_simple = simplify_attr_list(contactlists["ContactList"],
"ContactType")
new_contactlists = {}
for contact_type, contact_data in contactlists_simple.items():
contacts = simplify_attr_list(contact_data["Contacts"]["Contact"],
"ContactRank")
new_contacts = {}
for contact_rank in contacts:
if contact_rank in new_contacts and contacts[contact_rank][
"Name"] != new_contacts[contact_rank]:
# Multiple people with the same rank -- hope this never happens.
# Duplicates are fine though -- we collapse them into one.
raise RuntimeError("dammit %s" %
contacts[contact_rank]["Name"])
new_contacts[contact_rank] = contacts[contact_rank]["Name"]
new_contactlists[contact_type] = new_contacts
return new_contactlists
def expand_oasis_managers(managers):
"""Expand
{"a": {"DNs": [...]}}
into
{"Manager": [{"Name": "a", "DNs": {"DN": [...]}}]}
"""
new_managers = managers.copy()
for name, data in managers.items():
if not is_null(data, "DNs"):
new_managers[name]["DNs"] = {
"DN": singleton_list_to_value(data["DNs"])
}
else:
new_managers[name]["DNs"] = None
return {
"Manager":
expand_attr_list(new_managers,
"Name",
ordering=["ContactID", "Name", "DNs"],
ignore_missing=True)
}
if "ID" in vo:
vo["ID"] = int(vo["ID"])
vo["Active"] = is_true_str(vo.get("Active", ""))
vo["CertificateOnly"] = is_true_str(vo.get("CertificateOnly", ""))
vo["Disable"] = is_true_str(vo.get("Disable", ""))
if "ContactTypes" in vo:
vo["Contacts"] = simplify_contacttypes(vo["ContactTypes"])
del vo["ContactTypes"]
if "ReportingGroups" in vo:
rgs = simplify_reportinggroups(vo["ReportingGroups"])
if rgs is not None:
vo["ReportingGroups"] = sorted(set(rgs.keys()))
reportinggroup_data.update(rgs)
if "OASIS" in vo:
if not is_null(vo["OASIS"], "Managers"):
vo["OASIS"]["Managers"] = simplify_oasis_managers(vo["OASIS"]["Managers"])
else:
vo["OASIS"].pop("Managers", None)
if not is_null(vo["OASIS"], "OASISRepoURLs", "URL"):
vo["OASIS"]["OASISRepoURLs"] = ensure_list(vo["OASIS"]["OASISRepoURLs"]["URL"])
else:
vo["OASIS"].pop("OASISRepoURLs")
vo["OASIS"]["UseOASIS"] = is_true_str(vo["OASIS"].get("UseOASIS", ""))
if not is_null(vo, "FieldsOfScience"):
vo["FieldsOfScience"] = simplify_fields_of_science(vo["FieldsOfScience"])
if not is_null(vo, "ParentVO"):
vo["ParentVO"]["ID"] = int(vo["ParentVO"]["ID"])
vo.pop("MemeberResources", None) # will recreate MemeberResources [sic] from RG data
# delete empty fields
def expand_vo(vo, reportinggroups_data):
vo = vo.copy()
if is_null(vo, "Contacts"):
vo["ContactTypes"] = None
else:
vo["ContactTypes"] = expand_contacttypes(vo["Contacts"])
vo.pop("Contacts", None)
if is_null(vo, "ReportingGroups"):
vo["ReportingGroups"] = None
else:
vo["ReportingGroups"] = expand_reportinggroups(vo["ReportingGroups"],
reportinggroups_data)
if is_null(vo, "OASIS"):
vo["OASIS"] = None
else:
oasis = OrderedDict()
oasis["UseOASIS"] = vo["OASIS"].get("UseOASIS", False)
if is_null(vo["OASIS"], "Managers"):
oasis["Managers"] = None
else:
oasis["Managers"] = expand_oasis_managers(vo["OASIS"]["Managers"])
if is_null(vo["OASIS"], "OASISRepoURLs"):
oasis["OASISRepoURLs"] = None
else:
oasis["OASISRepoURLs"] = {
"URL": singleton_list_to_value(vo["OASIS"]["OASISRepoURLs"])
}
vo["OASIS"] = oasis
if is_null(vo, "FieldsOfScience"):
vo["FieldsOfScience"] = None
else:
vo["FieldsOfScience"] = expand_fields_of_science(vo["FieldsOfScience"])
# Restore ordering
if not is_null(vo, "ParentVO"):
parentvo = OrderedDict()
for elem in ["ID", "Name"]:
if elem in vo["ParentVO"]:
parentvo[elem] = vo["ParentVO"][elem]
vo["ParentVO"] = parentvo
else:
vo["ParentVO"] = None
for key in [
"MembershipServicesURL", "PrimaryURL", "PurposeURL", "SupportURL"
]:
if key not in vo:
vo[key] = None
# TODO: Recreate <MemeberResources> [sic]
# should look like
# <MemeberResources>
# <Resource><ID>75</ID><Name>NERSC-PDSF</Name></Resource>
# ...
# </MemeberResources>
# Restore ordering
new_vo = OrderedDict()
for elem in [
"ID",
"Name",
"LongName",
"CertificateOnly",
"PrimaryURL",
"MembershipServicesURL",
"PurposeURL",
"SupportURL",
"AppDescription",
"Community",
# TODO "MemeberResources",
"FieldsOfScience",
"ParentVO",
"ReportingGroups",
"Active",
"Disable",
"ContactTypes",
"OASIS"
]:
if elem in vo:
new_vo[elem] = vo[elem]
return new_vo
