def get_vos_data(indir, contacts_data, strict=False) -> VOsData:
reporting_groups_data = load_yaml_file(
os.path.join(indir, "REPORTING_GROUPS.yaml"))
vos_data = VOsData(contacts_data=contacts_data,
reporting_groups_data=reporting_groups_data)
for file in os.listdir(indir):
if file == "REPORTING_GROUPS.yaml": continue
if not file.endswith(".yaml"): continue
name = file[:-5]
data = None
try:
data = load_yaml_file(os.path.join(indir, file))
vos_data.add_vo(name, data)
except yaml.YAMLError:
if strict:
raise
else:
# load_yaml_file() already logs the specific error
log.error("skipping (non-strict mode)")
continue
except Exception as e:
log.error("%r adding VO %s", e, name)
log.error("Data:\n%s", pprint.pformat(data))
if strict:
raise
log.exception("Skipping (non-strict mode); exception info follows")
continue
return vos_data
def get_one_project(file: str, campus_grid_ids: Dict,
vos_data: VOsData) -> Dict:
project = OrderedDict.fromkeys([
"ID", "Name", "Description", "PIName", "Organization", "Department",
"FieldOfScience", "Sponsor", "ResourceAllocations"
])
data = None
try:
data = load_yaml_file(file)
if 'CampusGrid' in data['Sponsor']:
name = data['Sponsor']['CampusGrid']['Name']
ID = campus_grid_ids[name]
data['Sponsor']['CampusGrid'] = OrderedDict([("ID", ID),
("Name", name)])
elif 'VirtualOrganization' in data['Sponsor']:
name = data['Sponsor']['VirtualOrganization']['Name']
ID = vos_data.vos[name]['ID']
data['Sponsor']['VirtualOrganization'] = OrderedDict([("ID", ID),
("Name",
name)])
if 'ResourceAllocations' in data:
resource_allocations = [
get_resource_allocation(ra, idx)
for idx, ra in enumerate(data['ResourceAllocations'])
]
data['ResourceAllocations'] = {
"ResourceAllocation": resource_allocations
}
except Exception as e:
log.error("%r adding project %s", e, file)
log.error("Data:\n%s", pprint.pformat(data))
raise
project.update(data)
return project
def get_projects(indir="../projects", strict=False):
to_output = {"Projects": {"Project": []}}
projects = []
mapping = load_yaml_file(os.path.join(indir, "_CAMPUS_GRIDS.yaml"))
for file in os.listdir(indir):
if not file.endswith(".yaml"):
continue
elif file.endswith("_CAMPUS_GRIDS.yaml"):
continue
project = OrderedDict.fromkeys([
"ID", "Name", "Description", "PIName", "Organization",
"Department", "FieldOfScience", "Sponsor"
])
data = None
try:
data = load_yaml_file(os.path.join(indir, file))
if 'CampusGrid' in data['Sponsor']:
name = data['Sponsor']['CampusGrid']['Name']
ID = mapping[name]
data['Sponsor']['CampusGrid'] = OrderedDict([("ID", ID),
("Name", name)])
except yaml.YAMLError:
if strict:
raise
else:
# load_yaml_file() already logs the specific error
log.error("skipping (non-strict mode)")
continue
except Exception as e:
log.error("%r adding project %s", e, file)
log.error("Data:\n%s", pprint.pformat(data))
if strict:
raise
log.exception("Skipping (non-strict mode); exception info follows")
continue
project.update(data)
projects.append(project)
to_output["Projects"]["Project"] = projects
return to_output
def get_nsfscience(indir: str, strict: bool) -> Dict:
nsfscience = {}
try:
nsfscience = load_yaml_file(os.path.join(indir, "nsfscience.yaml"))
except yaml.YAMLError:
if strict:
raise
else:
# load_yaml_file() already logs the specific error
log.error("skipping (non-strict mode)")
return nsfscience
def get_project_institution(indir: str, strict: bool) -> Dict:
project_institution = {}
try:
project_institution = load_yaml_file(
os.path.join(indir, "project_institution.yaml"))
except yaml.YAMLError:
if strict:
raise
else:
# load_yaml_file() already logs the specific error
log.error("skipping (non-strict mode)")
return project_institution
def get_one_project(file: str, campus_grid_ids: Dict,
vos_data: VOsData) -> Dict:
project = OrderedDict.fromkeys([
"ID", "Name", "Description", "PIName", "Organization", "Department",
"FieldOfScience", "Sponsor", "ResourceAllocations"
])
data = None
try:
data = load_yaml_file(file)
if 'CampusGrid' in data['Sponsor']:
name = data['Sponsor']['CampusGrid']['Name']
ID = campus_grid_ids[name]
data['Sponsor']['CampusGrid'] = OrderedDict([("ID", ID),
("Name", name)])
elif 'VirtualOrganization' in data['Sponsor']:
name = data['Sponsor']['VirtualOrganization']['Name']
ID = vos_data.vos[name]['ID']
data['Sponsor']['VirtualOrganization'] = OrderedDict([("ID", ID),
("Name",
name)])
if 'ResourceAllocations' in data:
resource_allocations = [
get_resource_allocation(ra, idx)
for idx, ra in enumerate(data['ResourceAllocations'])
]
data['ResourceAllocations'] = {
"ResourceAllocation": resource_allocations
}
if 'ID' not in data:
del project['ID']
name_from_filename = os.path.basename(file)[:-5] # strip '.yaml'
if not is_null(data, 'Name'):
if data['Name'] != name_from_filename:
log.warning("%s: 'Name' %r does not match filename" %
(file, data['Name']))
else:
data['Name'] = name_from_filename
except Exception as e:
log.error("%r adding project %s", e, file)
log.error("Data:\n%s", pprint.pformat(data))
raise
project.update(data)
return project
def get_topology(indir="../topology", contacts_data=None, strict=False):
root = Path(indir)
support_centers = load_yaml_file(root / "support-centers.yaml")
service_types = load_yaml_file(root / "services.yaml")
tables = CommonData(contacts=contacts_data, service_types=service_types, support_centers=support_centers)
topology = Topology(tables)
for facility_path in root.glob("*/FACILITY.yaml"):
name = facility_path.parts[-2]
id_ = load_yaml_file(facility_path)["ID"]
topology.add_facility(name, id_)
for site_path in root.glob("*/*/SITE.yaml"):
facility, name = site_path.parts[-3:-1]
site_info = load_yaml_file(site_path)
id_ = site_info["ID"]
topology.add_site(facility, name, id_, site_info)
for yaml_path in root.glob("*/*/*.yaml"):
facility, site, name = yaml_path.parts[-3:]
if name == "SITE.yaml": continue
if name.endswith("_downtime.yaml"): continue
name = name.replace(".yaml", "")
try:
rg = load_yaml_file(yaml_path)
except yaml.YAMLError:
if strict:
raise
else:
# load_yaml_file() already logs the specific error
log.error("skipping (non-strict mode)")
continue
downtime_yaml_path = yaml_path.with_name(name + "_downtime.yaml")
downtimes = None
if downtime_yaml_path.exists():
try:
downtimes = ensure_list(load_yaml_file(downtime_yaml_path))
except yaml.YAMLError:
if strict:
raise
# load_yaml_file() already logs the specific error
log.error("skipping (non-strict mode)")
# keep going with downtimes=None
topology.add_rg(facility, site, name, rg)
if downtimes:
for downtime in downtimes:
topology.add_downtime(site, name, downtime)
return topology
def get_projects(indir="../projects", strict=False):
to_output = {"Projects":{"Project": []}}
projects = []
for file in os.listdir(indir):
if not file.endswith(".yaml"):
continue
project = OrderedDict.fromkeys(["ID", "Name", "Description", "PIName", "Organization", "Department",
"FieldOfScience", "Sponsor"])
try:
data = load_yaml_file(os.path.join(indir, file))
except yaml.YAMLError:
if strict:
raise
else:
# load_yaml_file() already logs the specific error
log.error("skipping (non-strict mode)")
continue
project.update(data)
projects.append(project)
to_output["Projects"]["Project"] = projects
return to_output
def get_campus_grid_ids(indir="."):
return load_yaml_file(os.path.join(indir, "_CAMPUS_GRIDS.yaml"))
def get_contacts_data(infile) -> ContactsData:
if infile:
return ContactsData(load_yaml_file(infile))
else:
return ContactsData({})
def get_contacts_data(infile) -> ContactsData:
return ContactsData(load_yaml_file(infile))