def simplify_services(self, services): """ Simplify Services attribute and record the Name -> ID mapping """ if not isinstance(services, dict): return None new_services = simplify_attr_list(services["Service"], "Name") for name, data in new_services.items(): if not is_null(data, "Details"): new_details = dict() for key, val in data["Details"].items(): if not is_null(val): new_details[key] = val if len(new_details) > 0: data["Details"] = dict(new_details) else: del data["Details"] else: data.pop("Details", None) if "ID" in data: if name not in self.services: self.services[name] = data["ID"] else: if data["ID"] != self.services[name]: raise RuntimeError( "Identical services with different IDs: %s" % name) del data["ID"] return new_services
def simplify_contacttypes(contacttypes): """Simplify ContactTypes attribute Turn e.g. {"ContactType": [{"Contacts": {"Contact": [{"Name": "Steve Timm"}, {"Name": "Joe Boyd"}]}, "Type": "Miscellaneous Contact"} ] } into {"Miscellanous Contact": [ "Steve Timm", "Joe Boyd" ] } """ if is_null(contacttypes, "ContactType"): return None new_contacttypes = {} for ct in ensure_list(contacttypes["ContactType"]): if is_null(ct, "Contacts", "Contact"): continue type_ = ct["Type"] # Remove duplicates but keep ordering contacts = [] for c in ensure_list(ct["Contacts"]["Contact"]): if c["Name"] not in contacts: contacts.append(c["Name"]) new_contacttypes[type_] = contacts return new_contacttypes
def simplify_fields_of_science(fos: Dict) -> Union[Dict, None]: """Turn {"PrimaryFields": {"Field": ["P1", "P2", ...]}, "SecondaryFields": {"Field": ["S1", "S2", ...]}} into {"PrimaryFields": ["P1", "P2", ...], "SecondaryFields": ["S1", "S2", ...]} """ if is_null(fos, "PrimaryFields") or is_null(fos["PrimaryFields"], "Field"): return None new_fields = {"PrimaryFields": ensure_list(fos["PrimaryFields"]["Field"])} if not is_null(fos, "SecondaryFields", "Field"): new_fields["SecondaryFields"] = ensure_list(fos["SecondaryFields"]["Field"]) return new_fields
def expand_resource(name: str, res: Dict, service_name_to_id: Dict[str, int]) -> OrderedDict: """Expand a single Resource from the format in a yaml file to the xml format. Services, VOOwnership, FQDNAliases, ContactLists are expanded; ``name`` is inserted into the Resource as the "Name" attribute; Defaults are added for VOOwnership, FQDNAliases, and WLCGInformation if they're missing from the yaml file. Return the data structure for the expanded Resource as an OrderedDict to fit the xml schema. """ defaults = { "ContactLists": None, "FQDNAliases": None, "Services": "no applicable service exists", "VOOwnership": "(Information not available)", "WLCGInformation": "(Information not available)", } res = dict(res) if not is_null(res, "Services"): res["Services"] = expand_services(res["Services"], service_name_to_id) else: res.pop("Services", None) if "VOOwnership" in res: res["VOOwnership"] = expand_voownership(res["VOOwnership"]) if "FQDNAliases" in res: res["FQDNAliases"] = { "FQDNAlias": singleton_list_to_value(res["FQDNAliases"]) } if not is_null(res, "ContactLists"): res["ContactLists"] = expand_contactlists(res["ContactLists"]) res["Name"] = name if "WLCGInformation" in res and isinstance(res["WLCGInformation"], dict): res["WLCGInformation"] = expand_wlcginformation(res["WLCGInformation"]) new_res = OrderedDict() for elem in [ "ID", "Name", "Active", "Disable", "Services", "Description", "FQDN", "FQDNAliases", "VOOwnership", "WLCGInformation", "ContactLists" ]: if elem in res: new_res[elem] = res[elem] elif elem in defaults: new_res[elem] = defaults[elem] return new_res
def simplify_oasis_managers(managers): """Simplify OASIS/Managers attributes Turn {"Manager": [{"Name": "a", "DNs": {"DN": [...]}}]} into {"a": {"DNs": [...]}} """ if is_null(managers, "Manager"): return None new_managers = simplify_attr_list(managers["Manager"], "Name") for manager, data in new_managers.items(): if not is_null(data, "DNs"): data["DNs"] = data["DNs"]["DN"] if not is_null(data, "ContactID"): data["ContactID"] = int(data["ContactID"]) return new_managers
def simplify_reportinggroups(reportinggroups): """Simplify ReportingGroups attributes Turn e.g. {"ReportingGroup": [{"Contacts": {"Contact": [{"Name": "a"}, {"Name": "b"} }, "FQANs": {"FQAN": [{"GroupName": "XXX", "Role": "YYY"}] } "Name": "ZZZ" }] } into {"ZZZ": {"Contacts": ["a", "b"], "FQANs": [{"GroupName": "XXX", "Role": "YYY"}] } } """ if is_null(reportinggroups, "ReportingGroup"): return None # [{"Name": "XXX", <...>}, {"Name": "YYY", <...>}] becomes # {"XXX": {<...>}, "YYY": {<...>}> new_reportinggroups = simplify_attr_list(reportinggroups["ReportingGroup"], "Name") if not new_reportinggroups: # only null entries found return None for rgname, rgdata in new_reportinggroups.items(): if not is_null(rgdata["Contacts"], "Contact"): # {"Contacts": {"Contact": [{"Name": "a"}, {"Name": "b"}]}} becomes # {"Contacts": ["a", "b"]} new_contacts = [] for c in ensure_list(rgdata["Contacts"]["Contact"]): if not is_null(c, "Name") and c["Name"] not in new_contacts: new_contacts.append(c["Name"]) rgdata["Contacts"] = new_contacts if not is_null(rgdata["FQANs"], "FQAN"): rgdata["FQANs"] = ensure_list(rgdata["FQANs"]["FQAN"]) return new_reportinggroups
def expand_fields_of_science(fields_of_science): """Turn {"PrimaryFields": ["P1", "P2", ...], "SecondaryFields": ["S1", "S2", ...]} into {"PrimaryFields": {"Field": ["P1", "P2", ...]}, "SecondaryFields": {"Field": ["S1", "S2", ...]}} """ if is_null(fields_of_science, "PrimaryFields"): return None new_fields = OrderedDict() new_fields["PrimaryFields"] = { "Field": singleton_list_to_value(fields_of_science["PrimaryFields"]) } if not is_null(fields_of_science, "SecondaryFields"): new_fields["SecondaryFields"] = { "Field": singleton_list_to_value(fields_of_science["SecondaryFields"]) } return new_fields
def expand_reportinggroups(reportinggroups_list: List, reportinggroups_data: Dict) -> Dict: """Expand ["XXX", "YYY", "ZZZ"] using data from reportinggroups_data into {"ReportingGroup": [{"Contacts": {"Contact": [{"Name": "a"}, {"Name": "b"} }, "FQANs": {"FQAN": [{"GroupName": "...", "Role": "..."}] } "Name": "XXX" }] } """ new_reportinggroups = {} for name, data in reportinggroups_data.items(): if name not in reportinggroups_list: continue new_reportinggroups[name] = {} newdata = new_reportinggroups[name] if not is_null(data, "Contacts"): new_contact = [{"Name": x} for x in data["Contacts"]] newdata["Contacts"] = { "Contact": singleton_list_to_value(new_contact) } else: newdata["Contacts"] = None if not is_null(data, "FQANs"): fqans = [] for fqan in data["FQANs"]: fqans.append( OrderedDict([("GroupName", fqan["GroupName"]), ("Role", fqan["Role"])])) newdata["FQANs"] = {"FQAN": singleton_list_to_value(fqans)} else: newdata["FQANs"] = None new_reportinggroups = expand_attr_list( new_reportinggroups, "Name", ordering=["Name", "FQANs", "Contacts"]) return {"ReportingGroup": new_reportinggroups}
def simplify_voownership(self, voownership): """ Simplify VOOwnership attribute """ if is_null(voownership): return None voownership = dict(voownership) # copy del voownership["ChartURL"] # can be derived from the other attributes new_voownership = simplify_attr_list(voownership["Ownership"], "VO") new_voownership.pop("(Other)", None) # can be derived from the other attributes for vo in new_voownership: new_voownership[vo] = int(new_voownership[vo]["Percent"]) return new_voownership
def simplify_resource(self, res: Dict) -> Dict: res = dict(res) services = self.simplify_services(res["Services"]) if is_null(services): res.pop("Services", None) else: res["Services"] = services res["VOOwnership"] = self.simplify_voownership(res.get("VOOwnership")) if not res["VOOwnership"]: del res["VOOwnership"] if not is_null(res, "WLCGInformation"): new_wlcg = {} for key, val in res["WLCGInformation"].items(): if not is_null(val): new_wlcg[key] = val if len(new_wlcg) > 0: res["WLCGInformation"] = new_wlcg else: del res["WLCGInformation"] else: res.pop("WLCGInformation", None) if not is_null(res, "FQDNAliases"): aliases = [] for a in ensure_list(res["FQDNAliases"]["FQDNAlias"]): aliases.append(a) res["FQDNAliases"] = aliases else: res.pop("FQDNAliases", None) if not is_null(res, "ContactLists"): new_contactlists = self.simplify_contactlists(res["ContactLists"]) if new_contactlists: res["ContactLists"] = new_contactlists else: res.pop("ContactLists", None) return res
def simplify_contactlists(self, contactlists): """Simplify ContactLists attribute Turn e.g. {"ContactList": [{"ContactType": "Administrative Contact", {"Contacts": {"Contact": [{"Name": "Andrew Malone Melo", "ContactRank": "Primary"}, {"Name": "Paul Sheldon", "ContactRank": "Secondary"}] } } }] } into {"Administrative Contact": {"Primary": "Andrew Malone Melo", "Secondary": "Paul Sheldon"} } """ if is_null(contactlists): return None contactlists_simple = simplify_attr_list(contactlists["ContactList"], "ContactType") new_contactlists = {} for contact_type, contact_data in contactlists_simple.items(): contacts = simplify_attr_list(contact_data["Contacts"]["Contact"], "ContactRank") new_contacts = {} for contact_rank in contacts: if contact_rank in new_contacts and contacts[contact_rank][ "Name"] != new_contacts[contact_rank]: # Multiple people with the same rank -- hope this never happens. # Duplicates are fine though -- we collapse them into one. raise RuntimeError("dammit %s" % contacts[contact_rank]["Name"]) new_contacts[contact_rank] = contacts[contact_rank]["Name"] new_contactlists[contact_type] = new_contacts return new_contactlists
def expand_oasis_managers(managers): """Expand {"a": {"DNs": [...]}} into {"Manager": [{"Name": "a", "DNs": {"DN": [...]}}]} """ new_managers = managers.copy() for name, data in managers.items(): if not is_null(data, "DNs"): new_managers[name]["DNs"] = { "DN": singleton_list_to_value(data["DNs"]) } else: new_managers[name]["DNs"] = None return { "Manager": expand_attr_list(new_managers, "Name", ordering=["ContactID", "Name", "DNs"], ignore_missing=True) }
if "ID" in vo: vo["ID"] = int(vo["ID"]) vo["Active"] = is_true_str(vo.get("Active", "")) vo["CertificateOnly"] = is_true_str(vo.get("CertificateOnly", "")) vo["Disable"] = is_true_str(vo.get("Disable", "")) if "ContactTypes" in vo: vo["Contacts"] = simplify_contacttypes(vo["ContactTypes"]) del vo["ContactTypes"] if "ReportingGroups" in vo: rgs = simplify_reportinggroups(vo["ReportingGroups"]) if rgs is not None: vo["ReportingGroups"] = sorted(set(rgs.keys())) reportinggroup_data.update(rgs) if "OASIS" in vo: if not is_null(vo["OASIS"], "Managers"): vo["OASIS"]["Managers"] = simplify_oasis_managers(vo["OASIS"]["Managers"]) else: vo["OASIS"].pop("Managers", None) if not is_null(vo["OASIS"], "OASISRepoURLs", "URL"): vo["OASIS"]["OASISRepoURLs"] = ensure_list(vo["OASIS"]["OASISRepoURLs"]["URL"]) else: vo["OASIS"].pop("OASISRepoURLs") vo["OASIS"]["UseOASIS"] = is_true_str(vo["OASIS"].get("UseOASIS", "")) if not is_null(vo, "FieldsOfScience"): vo["FieldsOfScience"] = simplify_fields_of_science(vo["FieldsOfScience"]) if not is_null(vo, "ParentVO"): vo["ParentVO"]["ID"] = int(vo["ParentVO"]["ID"]) vo.pop("MemeberResources", None) # will recreate MemeberResources [sic] from RG data # delete empty fields
def expand_vo(vo, reportinggroups_data): vo = vo.copy() if is_null(vo, "Contacts"): vo["ContactTypes"] = None else: vo["ContactTypes"] = expand_contacttypes(vo["Contacts"]) vo.pop("Contacts", None) if is_null(vo, "ReportingGroups"): vo["ReportingGroups"] = None else: vo["ReportingGroups"] = expand_reportinggroups(vo["ReportingGroups"], reportinggroups_data) if is_null(vo, "OASIS"): vo["OASIS"] = None else: oasis = OrderedDict() oasis["UseOASIS"] = vo["OASIS"].get("UseOASIS", False) if is_null(vo["OASIS"], "Managers"): oasis["Managers"] = None else: oasis["Managers"] = expand_oasis_managers(vo["OASIS"]["Managers"]) if is_null(vo["OASIS"], "OASISRepoURLs"): oasis["OASISRepoURLs"] = None else: oasis["OASISRepoURLs"] = { "URL": singleton_list_to_value(vo["OASIS"]["OASISRepoURLs"]) } vo["OASIS"] = oasis if is_null(vo, "FieldsOfScience"): vo["FieldsOfScience"] = None else: vo["FieldsOfScience"] = expand_fields_of_science(vo["FieldsOfScience"]) # Restore ordering if not is_null(vo, "ParentVO"): parentvo = OrderedDict() for elem in ["ID", "Name"]: if elem in vo["ParentVO"]: parentvo[elem] = vo["ParentVO"][elem] vo["ParentVO"] = parentvo else: vo["ParentVO"] = None for key in [ "MembershipServicesURL", "PrimaryURL", "PurposeURL", "SupportURL" ]: if key not in vo: vo[key] = None # TODO: Recreate <MemeberResources> [sic] # should look like # <MemeberResources> # <Resource><ID>75</ID><Name>NERSC-PDSF</Name></Resource> # ... # </MemeberResources> # Restore ordering new_vo = OrderedDict() for elem in [ "ID", "Name", "LongName", "CertificateOnly", "PrimaryURL", "MembershipServicesURL", "PurposeURL", "SupportURL", "AppDescription", "Community", # TODO "MemeberResources", "FieldsOfScience", "ParentVO", "ReportingGroups", "Active", "Disable", "ContactTypes", "OASIS" ]: if elem in vo: new_vo[elem] = vo[elem] return new_vo