def get_vos_data(indir, contacts_data, strict=False) -> VOsData:
reporting_groups_data = load_yaml_file(
os.path.join(indir, "REPORTING_GROUPS.yaml"))
vos_data = VOsData(contacts_data=contacts_data,
reporting_groups_data=reporting_groups_data)
for file in os.listdir(indir):
if file == "REPORTING_GROUPS.yaml": continue
if not file.endswith(".yaml"): continue
name = file[:-5]
data = None
try:
data = load_yaml_file(os.path.join(indir, file))
vos_data.add_vo(name, data)
except yaml.YAMLError:
if strict:
raise
else:
# load_yaml_file() already logs the specific error
log.error("skipping (non-strict mode)")
continue
except Exception as e:
log.error("%r adding VO %s", e, name)
log.error("Data:\n%s", pprint.pformat(data))
if strict:
raise
log.exception("Skipping (non-strict mode); exception info follows")
continue
return vos_data
def generate_public_cache_authfile(vo_data: VOsData, resource_groups: List[ResourceGroup], fqdn=None, legacy=True, suppress_errors=True) -> str:
"""
Generate the Xrootd authfile needed for public caches
"""
if legacy:
authfile = "u * /user/ligo -rl \\\n"
else:
authfile = "u * \\\n"
resource = _get_cache_resource(fqdn, resource_groups, suppress_errors)
if fqdn and not resource:
return ""
public_dirs = set()
for vo_name, vo_data in vo_data.vos.items():
stashcache_data = vo_data.get('DataFederations', {}).get('StashCache')
if not stashcache_data:
continue
if resource and not _cache_is_allowed(resource, vo_name, stashcache_data, True, suppress_errors):
continue
for dirname, authz_list in stashcache_data.get("Namespaces", {}).items():
if "PUBLIC" in authz_list:
public_dirs.add(dirname)
for dirname in sorted(public_dirs):
authfile += " {} rl \\\n".format(dirname)
if authfile.endswith("\\\n"):
authfile = authfile[:-2] + "\n"
return authfile
def generate_cache_authfile(vo_data: VOsData, resource_groups: List[ResourceGroup], fqdn=None, legacy=True, suppress_errors=True) -> str:
"""
Generate the Xrootd authfile needed by a StashCache cache server.
"""
authfile = ""
id_to_dir = defaultdict(set)
resource = _get_cache_resource(fqdn, resource_groups, suppress_errors)
if fqdn and not resource:
return ""
for vo_name, vo_data in vo_data.vos.items():
stashcache_data = vo_data.get('DataFederations', {}).get('StashCache')
if not stashcache_data:
continue
namespaces = stashcache_data.get("Namespaces")
if not namespaces:
if suppress_errors:
continue
else:
raise DataError("VO {} in StashCache does not provide a Namespaces list.".format(vo_name))
needs_authz = False
for namespace, authz_list in namespaces.items():
if not authz_list:
if suppress_errors:
continue
else:
raise DataError("Namespace {} (VO {}) does not provide any authorizations.".format(namespace, vo_name))
if authz_list != ["PUBLIC"]:
needs_authz = True
break
if not needs_authz:
continue
if resource and not _cache_is_allowed(resource, vo_name, stashcache_data, False, suppress_errors):
continue
for namespace, authz_list in namespaces.items():
for authz in authz_list:
if not isinstance(authz, str):
continue
if authz.startswith("FQAN:"):
id_to_dir["g {}".format(authz[5:])].add(namespace)
elif authz.startswith("DN:"):
hash = _generate_dn_hash(authz[3:])
id_to_dir["u {}".format(hash)].add(namespace)
if legacy:
for dn in _generate_ligo_dns():
hash = _generate_dn_hash(dn)
id_to_dir["u {}".format(hash)].add("/user/ligo")
for id, dir_list in id_to_dir.items():
if dir_list:
authfile += "{} {}\n".format(id,
" ".join([i + " rl" for i in sorted(dir_list)]))
return authfile
def get_vos_data(indir, contacts_data) -> VOsData:
reporting_groups_data = anymarkup.parse_file(
os.path.join(indir, "REPORTING_GROUPS.yaml"))
vos_data = VOsData(contacts_data=contacts_data,
reporting_groups_data=reporting_groups_data)
for file in os.listdir(indir):
if file == "REPORTING_GROUPS.yaml": continue
if not file.endswith(".yaml"): continue
name = file[:-5]
data = anymarkup.parse_file(os.path.join(indir, file))
try:
vos_data.add_vo(name, data)
except Exception:
pprint.pprint(name, data)
raise
return vos_data
def generate_origin_scitokens(vo_data: VOsData,
resource_groups: List[ResourceGroup],
fqdn: str,
suppress_errors=True) -> str:
template = """\
[Global]
audience = {allowed_vos_str}
{issuer_blocks_str}
"""
allowed_vos = []
issuer_blocks = []
for vo_name, vo_data in vo_data.vos.items():
stashcache_data = vo_data.get('DataFederations', {}).get('StashCache')
if not stashcache_data:
continue
if not _origin_is_allowed(fqdn,
vo_name,
stashcache_data,
resource_groups,
suppress_errors=suppress_errors):
continue
namespaces = stashcache_data.get("Namespaces")
if not namespaces:
if suppress_errors:
continue
else:
raise DataError(
"VO {} in StashCache does not provide a Namespaces list.".
format(vo_name))
for dirname, authz_list in namespaces.items():
if not authz_list:
if suppress_errors:
continue
else:
raise DataError(
"Namespace {} (VO {}) does not provide any authorizations."
.format(dirname, vo_name))
for authz in authz_list:
if not isinstance(authz, dict) or not isinstance(
authz.get("SciTokens"), dict):
continue
issuer_blocks.append(
_get_scitokens_issuer_block(vo_name, authz['SciTokens'],
dirname, suppress_errors))
allowed_vos.append(vo_name)
issuer_blocks_str = "\n".join(issuer_blocks)
allowed_vos_str = ", ".join(allowed_vos)
return template.format(**locals()).rstrip() + "\n"
def generate_cache_scitokens(vo_data: VOsData,
resource_groups: List[ResourceGroup],
fqdn: str,
suppress_errors=True) -> str:
"""
Generate the SciTokens needed by a StashCache cache server, given the fqdn
of the cache server.
The scitokens config for a StashCache namespace is in the VO YAML and looks like:
DataFederations:
StashCache:
Namespaces:
/store:
- SciTokens:
Issuer: https://scitokens.org/cms
Base Path: /
Restricted Path: /store
"Restricted Path" is optional.
`fqdn` must belong to a registered cache resource.
If suppress_errors is True, returns an empty string on various error conditions (e.g. no fqdn,
no resource matching fqdn, resource does not contain a cache server, etc.). Otherwise, raises
ValueError or DataError.
"""
template = """\
[Global]
audience = {allowed_vos_str}
{issuer_blocks_str}
"""
if not fqdn:
if suppress_errors:
return ""
else:
raise ValueError("fqdn: empty")
resource = _get_cache_resource(fqdn, resource_groups, suppress_errors)
if not resource:
return ""
log.debug(f"Generating stashcache cache config for {fqdn}")
allowed_vos = []
issuer_blocks = []
for vo_name, vo_data in vo_data.vos.items():
stashcache_data = vo_data.get('DataFederations', {}).get('StashCache')
if not stashcache_data:
continue
namespaces = stashcache_data.get("Namespaces", {})
if not namespaces:
continue
log.debug(f"Namespaces found for {vo_name}")
needs_authz = False
for authz_list in namespaces.values():
for authz in authz_list:
if authz != "PUBLIC":
needs_authz = True
break
if not needs_authz:
log.debug(f"\tAuth not needed for {vo_name}")
continue
if not _cache_is_allowed(resource,
vo_name,
stashcache_data,
public=False,
suppress_errors=suppress_errors):
continue
for dirname, authz_list in namespaces.items():
for authz in authz_list:
if not isinstance(authz, dict) or not isinstance(
authz.get("SciTokens"), dict):
log.debug(
f"\tNo SciTokens info for {dirname} in {vo_name}")
continue
issuer_blocks.append(
_get_scitokens_issuer_block(vo_name, authz['SciTokens'],
dirname, suppress_errors))
allowed_vos.append(vo_name)
issuer_blocks_str = "\n".join(issuer_blocks)
allowed_vos_str = ", ".join(allowed_vos)
return template.format(**locals()).rstrip() + "\n"
def generate_origin_scitokens(vo_data: VOsData, resource_groups: List[ResourceGroup], fqdn: str, suppress_errors=True) -> str:
"""
Generate the SciTokens needed by a StashCache origin server, given the fqdn
of the origin server.
The scitokens config for a StashCache namespace is in the VO YAML and looks like:
DataFederations:
StashCache:
Namespaces:
/store:
- SciTokens:
Issuer: https://scitokens.org/cms
Base Path: /
Restricted Path: /store
"Restricted Path" is optional.
`fqdn` must belong to a registered cache resource.
You may have multiple `- SciTokens:` blocks
Returns a file with a dummy "issuer" block if there are no `- SciTokens:` blocks.
If suppress_errors is True, returns an empty string on various error conditions (e.g. no fqdn,
no resource matching fqdn, resource does not contain an origin server, etc.). Otherwise, raises
ValueError or DataError.
"""
template = """\
[Global]
audience = {allowed_vos_str}
{issuer_blocks_str}
"""
allowed_vos = []
issuer_blocks = []
for vo_name, vo_data in vo_data.vos.items():
stashcache_data = vo_data.get('DataFederations', {}).get('StashCache')
if not stashcache_data:
continue
if not _origin_is_allowed(fqdn, vo_name, stashcache_data, resource_groups, suppress_errors=suppress_errors):
continue
namespaces = stashcache_data.get("Namespaces")
if not namespaces:
if suppress_errors:
continue
else:
raise DataError("VO {} in StashCache does not provide a Namespaces list.".format(vo_name))
for dirname, authz_list in namespaces.items():
if not authz_list:
if suppress_errors:
continue
else:
raise DataError("Namespace {} (VO {}) does not provide any authorizations.".format(dirname, vo_name))
for authz in authz_list:
if not isinstance(authz, dict) or not isinstance(authz.get("SciTokens"), dict):
continue
issuer_blocks.append(_get_scitokens_issuer_block(vo_name, authz['SciTokens'],
dirname, suppress_errors))
allowed_vos.append(vo_name)
# Older plugin versions require at least one issuer block (SOFTWARE-4389)
if not issuer_blocks:
issuer_blocks.append(
_get_scitokens_issuer_block(vo_name="nonexistent",
scitokens={"Issuer": "https://scitokens.org/nonexistent",
"Base Path": "/no-issuers-found"},
dirname="/no-issuers-found",
suppress_errors=suppress_errors))
issuer_blocks_str = "\n".join(issuer_blocks)
allowed_vos_str = ", ".join(allowed_vos)
return template.format(**locals()).rstrip() + "\n"