def get_final_context(info, context):
"""Based on env CRDS_CONTEXT, the `context` parameter, and the server's reported,
cached, or defaulted `operational_context`, choose the pipeline mapping which
defines the reference selection rules.
Returns a .pmap name
"""
env_context = config.get_crds_env_context()
if context: # context parameter trumps all, <observatory>-operational is default
input_context = context
log.verbose("Using reference file selection rules",
srepr(input_context), "defined by caller.")
info.status = "context parameter"
elif env_context:
input_context = env_context
log.verbose("Using reference file selection rules",
srepr(input_context),
"defined by environment CRDS_CONTEXT.")
info.status = "env var CRDS_CONTEXT"
else:
input_context = str(info.operational_context)
log.verbose("Using reference selection rules", srepr(input_context),
"defined by", info.status + ".")
final_context = translate_date_based_context(input_context,
info.observatory)
return final_context
def get_config_info(observatory):
"""Get the operational context and server s/w version from (in order of priority):
1. The server.
2. The cache from a prior server access.
3. The basic CRDS installation.
Return ConfigInfo
"""
try:
info = ConfigInfo(api.get_server_info())
info.status = "server"
info.connected = True
log.verbose("Connected to server at", srepr(api.get_crds_server()))
if info.effective_mode != "remote":
if not config.writable_cache_or_verbose("Using cached configuration and default context."):
info = load_server_info(observatory)
info.status = "cache"
info.connected = True
log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced))
except CrdsError as exc:
if "serverless" not in api.get_crds_server():
log.verbose_warning("Couldn't contact CRDS server:", srepr(api.get_crds_server()), ":", str(exc))
info = load_server_info(observatory)
info.status = "cache"
info.connected = False
log.verbose("Using CACHED CRDS reference assignment rules last updated on", repr(info.last_synced))
return info
def rmap_apply(self, func, *args, **keys):
"""Apply `func()` to *args and **keys, adding the pmap, imap, and rmap values
associated with the elaboration of args.source_context, args.instruments, args.types.
"""
keywords = dict(keys)
self._setup_source_context()
if self.args.rmaps:
for rmap_name in self.args.rmaps:
with log.error_on_exception("Failed processing rmap", srepr(rmap_name)):
log.info("="*20, "Refactoring rmap", srepr(rmap_name), "="*20)
rmapping = rmap.load_mapping(rmap_name)
new_filename = self._process_rmap(func, rmapping=rmapping, **keywords)
self._diff_and_certify(rmapping=rmapping, new_filename=new_filename,
source_context=self.source_context, **keywords)
else:
pmapping = rmap.load_mapping(self.source_context)
instruments = pmapping.selections.keys() if "all" in self.args.instruments else self.args.instruments
for instr in instruments:
with log.augment_exception("Failed loading imap for", repr(instr), "from",
repr(self.source_context)):
imapping = pmapping.get_imap(instr)
types = imapping.selections.keys() if "all" in self.args.types else self.args.types
for filekind in types:
with log.error_on_exception("Failed processing rmap for", repr(filekind)):
#, "from",
# repr(imapping.basename), "of", repr(self.source_context)):
try:
rmapping = imapping.get_rmap(filekind).copy()
except crds.exceptions.IrrelevantReferenceTypeError as exc:
log.info("Skipping type", srepr(filekind), "as N/A")
continue
log.info("="*20, "Refactoring rmap", srepr(rmapping.basename), "="*20)
new_filename = self._process_rmap(func, rmapping=rmapping, **keywords)
self._diff_and_certify(rmapping=rmapping, source_context=self.source_context,
new_filename=new_filename, **keywords)
def get_best_references(pipeline_context, header, reftypes=None):
"""Get best references for dict-like `header` relative to
`pipeline_context`.
pipeline_context CRDS context for lookup, e.g. 'hst_0001.pmap'
header dict-like mapping { lookup_parameter : value }
reftypes If None, return all reference types; otherwise return
best refs for the specified list of reftypes.
Returns { reftype : reference_basename ... }
Raises CrdsLookupError, typically for problems with header values
"""
header = { str(key):str(value) for (key,value) in header.items() }
try:
bestrefs = S.get_best_references(pipeline_context, dict(header), reftypes)
except Exception as exc:
raise CrdsLookupError(str(exc))
# Due to limitations of jsonrpc, exception handling is kludged in here.
for filetype, refname in bestrefs.items():
if "NOT FOUND" in refname:
if refname == "NOT FOUND n/a":
log.verbose("Reference type", srepr(filetype), "not applicable.", verbosity=80)
else:
raise CrdsLookupError("Error determining best reference for " +
srepr(filetype) + " = " +
str(refname)[len("NOT FOUND"):])
return bestrefs
def rmap_apply(self, func, *args, **keys):
"""Apply `func()` to *args and **keys, adding the pmap, imap, and rmap values
associated with the elaboration of args.source_context, args.instruments, args.types.
"""
keywords = dict(keys)
self._setup_source_context()
if self.args.rmaps:
for rmap_name in self.args.rmaps:
with log.error_on_exception("Failed processing rmap", srepr(rmap_name)):
log.info("="*20, "Refactoring rmap", srepr(rmap_name), "="*20)
rmapping = rmap.load_mapping(rmap_name)
new_filename = self._process_rmap(func, rmapping=rmapping, **keywords)
self._diff_and_certify(rmapping=rmapping, new_filename=new_filename,
source_context=self.source_context, **keywords)
else:
pmapping = rmap.load_mapping(self.source_context)
instruments = pmapping.selections.keys() if "all" in self.args.instruments else self.args.instruments
for instr in instruments:
with log.augment_exception("Failed loading imap for", repr(instr), "from",
repr(self.source_context)):
imapping = pmapping.get_imap(instr)
types = imapping.selections.keys() if "all" in self.args.types else self.args.types
for filekind in types:
with log.error_on_exception("Failed processing rmap for", repr(filekind)):
#, "from",
# repr(imapping.basename), "of", repr(self.source_context)):
try:
rmapping = imapping.get_rmap(filekind).copy()
except crds.exceptions.IrrelevantReferenceTypeError as exc:
log.info("Skipping type", srepr(filekind), "as N/A")
continue
log.info("="*20, "Refactoring rmap", srepr(rmapping.basename), "="*20)
new_filename = self._process_rmap(func, rmapping=rmapping, **keywords)
self._diff_and_certify(rmapping=rmapping, source_context=self.source_context,
new_filename=new_filename, **keywords)
def get_config_info(observatory):
"""Get the operational context and server s/w version from (in order of priority):
1. The server.
2. The cache from a prior server access.
3. The basic CRDS installation.
Return ConfigInfo
"""
try:
info = ConfigInfo(api.get_server_info())
info.status = "server"
info.connected = True
log.verbose("Connected to server at", srepr(api.get_crds_server()))
if info.effective_mode != "remote":
if not config.writable_cache_or_verbose(
"Using cached configuration and default context."):
info = load_server_info(observatory)
info.status = "cache"
info.connected = True
log.verbose(
"Using CACHED CRDS reference assignment rules last updated on",
repr(info.last_synced))
except CrdsError as exc:
if "serverless" not in api.get_crds_server():
log.verbose_warning("Couldn't contact CRDS server:",
srepr(api.get_crds_server()), ":", str(exc))
info = load_server_info(observatory)
info.status = "cache"
info.connected = False
log.verbose(
"Using CACHED CRDS reference assignment rules last updated on",
repr(info.last_synced))
return info
def set_rmap_substitution(rmapping, new_filename, parameter_name, old_text, new_text, *args, **keys):
log.info("Adding substitution for", srepr(parameter_name),
"from", srepr(old_text), "to", srepr(new_text), "in", srepr(rmapping.basename))
new_mapping = rmapping.copy()
if "substitutions" not in new_mapping.header:
new_mapping.header["substitutions"] = {}
new_mapping.header["substitutions"][parameter_name] = { old_text : new_text }
new_mapping.write(new_filename)
def set_rmap_substitution(rmapping, new_filename, parameter_name, old_text, new_text, *args, **keys):
log.info("Adding substitution for", srepr(parameter_name),
"from", srepr(old_text), "to", srepr(new_text), "in", srepr(rmapping.basename))
new_mapping = rmapping.copy()
if "substitutions" not in new_mapping.header:
new_mapping.header["substitutions"] = {}
new_mapping.header["substitutions"][parameter_name] = { old_text : new_text }
new_mapping.write(new_filename)
def del_rmap_header(rmapping, new_filename, header_key):
"""Set the value of `key` in `filename` to `new_value` and rewrite the rmap.
This is potentially lossy since rewriting the rmap may/will lose comments and
formatting quirks.
"""
log.verbose("Deleting header value in", srepr(rmapping.basename), "for", srepr(header_key))
del rmapping.header[header_key]
rmapping.write(new_filename)
def del_rmap_header(rmapping, new_filename, header_key):
"""Set the value of `key` in `filename` to `new_value` and rewrite the rmap.
This is potentially lossy since rewriting the rmap may/will lose comments and
formatting quirks.
"""
log.verbose("Deleting header value in", srepr(rmapping.basename), "for", srepr(header_key))
del rmapping.header[header_key]
rmapping.write(new_filename)
def replace_rmap_text(rmapping, new_filename, old_text, new_text, *args, **keys):
"""Do simple text replacement from `old_text` to `new_text` in `rmapping`.
"""
log.info("Replacing", srepr(old_text), "with", srepr(new_text), "in",
srepr(rmapping.basename), "to", srepr(new_filename))
original_rmap = str(rmapping)
new_rmap = original_rmap.replace(old_text, new_text)
new_mapping = rmap.ReferenceMapping.from_string(new_rmap, ignore_checksum=True)
new_mapping.write(new_filename)
def cat_rmap(rmapping, new_filename, header_key, *args, **keys):
"""Cat/print rmapping's source text or the value of `header_key` in the rmap header."""
if header_key is not None:
log.info("In", srepr(rmapping.basename), "parameter", srepr(header_key), "=", srepr(rmapping.header[header_key]))
else:
log.info("-"*80)
log.info("Rmap", srepr(rmapping.basename), "is:")
log.info("-"*80)
log.write(str(rmapping))
def cat_rmap(rmapping, new_filename, header_key, *args, **keys):
"""Cat/print rmapping's source text or the value of `header_key` in the rmap header."""
if header_key is not None:
log.info("In", srepr(rmapping.basename), "parameter", srepr(header_key), "=", srepr(rmapping.header[header_key]))
else:
log.info("-"*80)
log.info("Rmap", srepr(rmapping.basename), "is:")
log.info("-"*80)
log.write(str(rmapping))
def set_rmap_header(rmapping, new_filename, header_key, header_value, *args, **keys):
"""Set the value of `key` in `filename` to `new_value` and rewrite the rmap.
This is potentially lossy since rewriting the rmap may/will lose comments and
formatting quirks.
"""
log.verbose("Setting header value in", srepr(rmapping.basename), "for", srepr(header_key),
"=", srepr(header_value))
try:
rmapping.header[header_key] = eval(header_value)
except Exception:
rmapping.header[header_key] = header_value
rmapping.write(new_filename)
def set_rmap_header(rmapping, new_filename, header_key, header_value, *args, **keys):
"""Set the value of `key` in `filename` to `new_value` and rewrite the rmap.
This is potentially lossy since rewriting the rmap may/will lose comments and
formatting quirks.
"""
log.verbose("Setting header value in", srepr(rmapping.basename), "for", srepr(header_key),
"=", srepr(header_value))
try:
rmapping.header[header_key] = eval(header_value)
except Exception:
rmapping.header[header_key] = header_value
rmapping.write(new_filename)
def del_rmap_parameter(rmapping, new_filename, parameter, *args, **keys):
"""Delete `parameter_name` from the parkey item of the `types` of the specified
`instruments` in `context`.
"""
log.info("Deleting parameter", repr(parameter), "from",repr(rmapping.basename))
parkey = rmapping.parkey
i, j = get_parameter_index(parkey, parameter)
del_parkey = parkey[:i] + ((parkey[i][:j] + parkey[i][j+1:]),) + parkey[i+1:]
log.verbose("Replacing", srepr(parkey), "with", srepr(del_parkey), "in", srepr(rmapping.basename))
rmapping.header["parkey"] = del_parkey
rmapping.selector.delete_match_param(parameter)
rmapping.write(new_filename)
def del_rmap_parameter(rmapping, new_filename, parameter, *args, **keys):
"""Delete `parameter_name` from the parkey item of the `types` of the specified
`instruments` in `context`.
"""
log.info("Deleting parameter", repr(parameter), "from",repr(rmapping.basename))
parkey = rmapping.parkey
i, j = get_parameter_index(parkey, parameter)
del_parkey = parkey[:i] + ((parkey[i][:j] + parkey[i][j+1:]),) + parkey[i+1:]
log.verbose("Replacing", srepr(parkey), "with", srepr(del_parkey), "in", srepr(rmapping.basename))
rmapping.header["parkey"] = del_parkey
rmapping.selector.delete_match_param(parameter)
rmapping.write(new_filename)
def _poll_status(self):
"""Use network API to pull status messages from server."""
try:
messages = api.jpoll_pull_messages(self.args.key, since_id=str(self._last_id))
if messages:
self._last_id = np.max([int(msg.id) for msg in messages])
return messages
except exceptions.StatusChannelNotFoundError:
log.verbose("Channel", srepr(self.args.key),
"not found. Waiting for processing to start.")
return []
except exceptions.ServiceError as exc:
log.verbose("Unhandled RPC exception for", srepr(self.args.key), "is", str(exc))
raise
def set_rmap_parkey(rmapping, new_filename, parkey, *args, **keys):
"""Set the parkey of `rmapping` to `parkey` and write out to `new_filename`.
"""
log.info("Setting parkey, removing all references from", srepr(rmapping.basename))
pktuple = eval(parkey)
required_keywords = tuple(utils.flatten(pktuple))
refnames = rmapping.reference_names()
references_headers = { refname : get_refactoring_header(rmapping.filename, refname, required_keywords)
for refname in refnames }
rmapping = rmap_delete_references(rmapping.filename, new_filename, refnames)
log.info("Setting parkey", srepr(parkey), "in", srepr(rmapping.basename))
rmapping.header["parkey"] = pktuple
rmapping.write(new_filename)
rmapping = rmap.load_mapping(new_filename)
rmapping = rmap_insert_references_by_matches(new_filename, new_filename, references_headers)
return rmapping
def verify_file(self, file, info, bytes_so_far, total_bytes, nth_file, total_files):
"""Check one `file` against the provided CRDS database `info` dictionary."""
path = rmap.locate_file(file, observatory=self.observatory)
base = os.path.basename(file)
n_bytes = int(info["size"])
log.verbose(api.file_progress("Verifying", base, path, n_bytes, bytes_so_far, total_bytes, nth_file, total_files),
verbosity=10)
if not os.path.exists(path):
log.error("File", repr(base), "doesn't exist at", repr(path))
return
size = os.stat(path).st_size
if int(info["size"]) != size:
self.error_and_repair(path, "File", repr(base), "length mismatch LOCAL size=" + srepr(size),
"CRDS size=" + srepr(info["size"]))
elif self.args.check_sha1sum:
log.verbose("Computing checksum for", repr(base), "of size", repr(size), verbosity=100)
sha1sum = utils.checksum(path)
if info["sha1sum"] == "none":
log.warning("CRDS doesn't know the checksum for", repr(base))
elif info["sha1sum"] != sha1sum:
self.error_and_repair(path, "File", repr(base), "checksum mismatch CRDS=" + repr(info["sha1sum"]),
"LOCAL=" + repr(sha1sum))
if info["state"] not in ["archived", "operational"]:
log.warning("File", repr(base), "has an unusual CRDS file state", repr(info["state"]))
if info["rejected"] != "false":
log.warning("File", repr(base), "has been explicitly rejected.")
if self.args.purge_rejected:
self.remove_files([path], "files")
return
if info["blacklisted"] != "false":
log.warning("File", repr(base), "has been blacklisted or is dependent on a blacklisted file.")
if self.args.purge_blacklisted:
self.remove_files([path], "files")
return
return
def verify_file(self, file, info, bytes_so_far, total_bytes, nth_file, total_files):
"""Check one `file` against the provided CRDS database `info` dictionary."""
path = rmap.locate_file(file, observatory=self.observatory)
base = os.path.basename(file)
n_bytes = int(info["size"])
log.verbose(api.file_progress("Verifying", base, path, n_bytes, bytes_so_far, total_bytes, nth_file, total_files),
verbosity=10)
if not os.path.exists(path):
log.error("File", repr(base), "doesn't exist at", repr(path))
return
size = os.stat(path).st_size
if int(info["size"]) != size:
self.error_and_repair(path, "File", repr(base), "length mismatch LOCAL size=" + srepr(size),
"CRDS size=" + srepr(info["size"]))
elif self.args.check_sha1sum:
log.verbose("Computing checksum for", repr(base), "of size", repr(size), verbosity=100)
sha1sum = utils.checksum(path)
if info["sha1sum"] == "none":
log.warning("CRDS doesn't know the checksum for", repr(base))
elif info["sha1sum"] != sha1sum:
self.error_and_repair(path, "File", repr(base), "checksum mismatch CRDS=" + repr(info["sha1sum"]),
"LOCAL=" + repr(sha1sum))
if info["state"] not in ["archived", "operational"]:
log.warning("File", repr(base), "has an unusual CRDS file state", repr(info["state"]))
if info["rejected"] != "false":
log.warning("File", repr(base), "has been explicitly rejected.")
if self.args.purge_rejected:
self.remove_files([path], "files")
return
if info["blacklisted"] != "false":
log.warning("File", repr(base), "has been blacklisted or is dependent on a blacklisted file.")
if self.args.purge_blacklisted:
self.remove_files([path], "files")
return
return
def _poll_status(self):
"""Use network API to pull status messages from server."""
try:
messages = api.jpoll_pull_messages(self.args.key,
since_id=str(self._last_id))
if messages:
self._last_id = np.max([int(msg.id) for msg in messages])
return messages
except exceptions.StatusChannelNotFoundError:
log.verbose("Channel", srepr(self.args.key),
"not found. Waiting for processing to start.")
return []
except exceptions.ServiceError as exc:
log.verbose("Unhandled RPC exception for", srepr(self.args.key),
"is", str(exc))
raise
def set_rmap_parkey(rmapping, new_filename, parkey, *args, **keys):
"""Set the parkey of `rmapping` to `parkey` and write out to `new_filename`.
"""
log.info("Setting parkey, removing all references from", srepr(rmapping.basename))
pktuple = eval(parkey)
required_keywords = tuple(utils.flatten(pktuple))
refnames = rmapping.reference_names()
references_headers = { refname : get_refactoring_header(rmapping.filename, refname, required_keywords)
for refname in refnames }
rmapping = rmap_delete_references(rmapping.filename, new_filename, refnames)
log.info("Setting parkey", srepr(parkey), "in", srepr(rmapping.basename))
rmapping.header["parkey"] = pktuple
rmapping.write(new_filename)
rmapping = rmap.load_mapping(new_filename)
rmapping = rmap_insert_references_by_matches(new_filename, new_filename, references_headers)
return rmapping
def scan_for_nonsubmitted_ingests(self, ingest_info):
"""Check for junk in the submitter's ingest directory, left over files not
in the current submission and fail if found.
"""
submitted_basenames = [ os.path.basename(filepath) for filepath in self.files ]
for ingested in ingest_info.keys():
if ingested not in submitted_basenames:
log.fatal_error(
"Non-submitted file", log.srepr(ingested),
"is already in the CRDS server's ingest directory. Delete it (--wipe-files?) or submit it.")
def download(self, name, localpath):
"""Download a single file."""
# This code is complicated by the desire to blow away failed downloads. For the specific
# case of KeyboardInterrupt, the file needs to be blown away, but the interrupt should not
# be re-characterized so it is still un-trapped elsewhere under normal idioms which try *not*
# to trap KeyboardInterrupt.
assert not config.get_cache_readonly(), "Readonly cache, cannot download files " + repr(name)
try:
utils.ensure_dir_exists(localpath)
return proxy.apply_with_retries(self.download_core, name, localpath)
except Exception as exc:
self.remove_file(localpath)
raise CrdsDownloadError("Error fetching data for " + srepr(name) +
" from context " + srepr(self.pipeline_context) +
" at CRDS server " + srepr(get_crds_server()) +
" with mode " + srepr(config.get_download_mode()) +
" : " + str(exc))
except: # mainly for control-c, catch it and throw it.
self.remove_file(localpath)
raise
def rmap_check_modifications(old_rmap, new_rmap, old_ref, new_ref, expected=("add",)):
"""Check the differences between `old_rmap` and `new_rmap` and make sure they're
limited to the types listed in `expected`.
expected should be "add" or "replace".
Returns as_expected, True IFF all rmap modifications match `expected`.
"""
diffs = diff.mapping_diffs(old_rmap, new_rmap)
as_expected = True
for difference in diffs:
actual = diff.diff_action(difference)
if actual in expected:
pass # white-list so it will fail when expected is bogus.
else:
log.error("Expected one of", srepr(expected), "but got", srepr(actual),
"from change", srepr(difference))
as_expected = False
with open(old_rmap) as pfile:
old_count = len([line for line in pfile.readlines() if os.path.basename(old_ref) in line])
with open(new_rmap) as pfile:
new_count = len([line for line in pfile.readlines() if os.path.basename(new_ref) in line])
if "replace" in expected and old_count != new_count:
log.error("Replacement COUNT DIFFERENCE replacing", srepr(old_ref), "with",
srepr(new_ref), "in", srepr(old_rmap),
old_count, "vs.", new_count)
as_expected = False
return as_expected
def rmap_check_modifications(old_rmap, new_rmap, old_ref, new_ref, expected=("add",)):
"""Check the differences between `old_rmap` and `new_rmap` and make sure they're
limited to the types listed in `expected`.
expected should be "add" or "replace".
Returns as_expected, True IFF all rmap modifications match `expected`.
"""
diffs = diff.mapping_diffs(old_rmap, new_rmap)
as_expected = True
for difference in diffs:
actual = diff.diff_action(difference)
if actual in expected:
pass # white-list so it will fail when expected is bogus.
else:
log.error("Expected one of", srepr(expected), "but got", srepr(actual),
"from change", srepr(difference))
as_expected = False
with open(old_rmap) as pfile:
old_count = len([line for line in pfile.readlines() if os.path.basename(old_ref) in line])
with open(new_rmap) as pfile:
new_count = len([line for line in pfile.readlines() if os.path.basename(new_ref) in line])
if "replace" in expected and old_count != new_count:
log.error("Replacement COUNT DIFFERENCE replacing", srepr(old_ref), "with",
srepr(new_ref), "in", srepr(old_rmap),
old_count, "vs.", new_count)
as_expected = False
return as_expected
def _setup_source_context(self):
"""Default the --source-context if necessary and then translate any symbolic name to a literal .pmap
name. e.g. jwst-edit --> jwst_0109.pmap. Then optionally sync the files to a local cache.
"""
if self.args.source_context is None:
self.source_context = self.observatory + "-edit"
log.info("Defaulting --source-context to", srepr(self.source_context))
else:
self.source_context = self.args.source_context
self.source_context = self.resolve_context(self.source_context)
if self.args.sync_files:
errs = sync.SyncScript("crds.sync --contexts {}".format(self.source_context))()
assert not errs, "Errors occurred while syncing all rules to CRDS cache."
def rmap_insert_references_by_matches(old_rmap, new_rmap, references_headers):
"""Given the full path of starting rmap `old_rmap`, modify it by inserting
or replacing all files in dict `references_headers` which maps a reference file basename
onto a list of headers under which it should be matched. Write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.load_mapping(old_rmap, ignore_checksum=True)
for baseref, header in references_headers.items():
with log.augment_exception("In reference", srepr(baseref)):
log.info("Inserting", srepr(baseref), "match case", srepr(header), "into", srepr(baseref))
new = new.insert_header_reference(header, baseref)
new.header["derived_from"] = old.basename
log.verbose("Writing", srepr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for baseref in references_headers:
assert baseref in formatted, \
"Rules update failure. " + srepr(baseref) + " does not appear in new rmap." \
" May be identical match with other submitted references."
return new
def _setup_source_context(self):
"""Default the --source-context if necessary and then translate any symbolic name to a literal .pmap
name. e.g. jwst-edit --> jwst_0109.pmap. Then optionally sync the files to a local cache.
"""
if self.args.source_context is None:
self.source_context = self.observatory + "-edit"
log.info("Defaulting --source-context to", srepr(self.source_context))
else:
self.source_context = self.args.source_context
self.source_context = self.resolve_context(self.source_context)
if self.args.sync_files:
errs = sync.SyncScript("crds.sync --contexts {}".format(self.source_context))()
assert not errs, "Errors occurred while syncing all rules to CRDS cache."
def get_final_context(info, context):
"""Based on env CRDS_CONTEXT, the `context` parameter, and the server's reported,
cached, or defaulted `operational_context`, choose the pipeline mapping which
defines the reference selection rules.
Returns a .pmap name
"""
env_context = config.get_crds_env_context()
if context: # context parameter trumps all, <observatory>-operational is default
input_context = context
log.verbose("Using reference file selection rules", srepr(input_context), "defined by caller.")
info.status = "context parameter"
elif env_context:
input_context = env_context
log.verbose("Using reference file selection rules", srepr(input_context),
"defined by environment CRDS_CONTEXT.")
info.status = "env var CRDS_CONTEXT"
else:
input_context = str(info.operational_context)
log.verbose("Using reference selection rules", srepr(input_context), "defined by", info.status + ".")
final_context = translate_date_based_context(info, input_context)
return final_context
def rmap_delete_references(old_rmap, new_rmap, deleted_references):
"""Given the full path of starting rmap `old_rmap`, modify it by deleting
all files in `deleted_references` and write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.load_mapping(old_rmap, ignore_checksum=True)
for reference in deleted_references:
baseref = os.path.basename(reference)
log.info("Deleting", srepr(baseref), "from", srepr(new.name))
with log.augment_exception("In reference", srepr(baseref)):
new = new.delete(reference)
new.header["derived_from"] = old.basename
log.verbose("Writing", srepr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for reference in deleted_references:
reference = os.path.basename(reference)
assert reference not in formatted, \
"Rules update failure. Deleted " + srepr(reference) + " still appears in new rmap."
return new
def scan_for_nonsubmitted_ingests(self, ingest_info):
"""Check for junk in the submitter's ingest directory, left over files not
in the current submission and fail if found.
"""
submitted_basenames = [
os.path.basename(filepath) for filepath in self.files
]
for ingested in ingest_info.keys():
if ingested not in submitted_basenames:
log.fatal_error(
"Non-submitted file", log.srepr(ingested),
"is already in the CRDS server's ingest directory. Delete it (--wipe-files?) or submit it."
)
def rmap_delete_references(old_rmap, new_rmap, deleted_references):
"""Given the full path of starting rmap `old_rmap`, modify it by deleting
all files in `deleted_references` and write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.load_mapping(old_rmap, ignore_checksum=True)
for reference in deleted_references:
baseref = os.path.basename(reference)
log.info("Deleting", srepr(baseref), "from", srepr(new.name))
with log.augment_exception("In reference", srepr(baseref)):
new = new.delete(reference)
new.header["derived_from"] = old.basename
log.verbose("Writing", srepr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for reference in deleted_references:
reference = os.path.basename(reference)
assert reference not in formatted, \
"Rules update failure. Deleted " + srepr(reference) + " still appears in new rmap."
return new
def rmap_insert_references(old_rmap, new_rmap, inserted_references):
"""Given the full path of starting rmap `old_rmap`, modify it by inserting
or replacing all files in `inserted_references` and write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.fetch_mapping(old_rmap, ignore_checksum=True)
for reference in inserted_references:
baseref = os.path.basename(reference)
with log.augment_exception("In reference", srepr(baseref)):
log.info("Inserting", srepr(baseref), "into", srepr(new.name))
new = new.insert_reference(reference)
new.header["derived_from"] = old.basename
log.verbose("Writing", srepr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for reference in inserted_references:
reference = os.path.basename(reference)
assert reference in formatted, \
"Rules update failure. " + srepr(reference) + " does not appear in new rmap." \
" May be identical match with other submitted references."
return new
def rmap_insert_references(old_rmap, new_rmap, inserted_references):
"""Given the full path of starting rmap `old_rmap`, modify it by inserting
or replacing all files in `inserted_references` and write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.fetch_mapping(old_rmap, ignore_checksum=True)
for reference in inserted_references:
baseref = os.path.basename(reference)
with log.augment_exception("In reference", srepr(baseref)):
log.info("Inserting", srepr(baseref), "into", srepr(new.name))
new = new.insert_reference(reference)
new.header["derived_from"] = old.basename
log.verbose("Writing", srepr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for reference in inserted_references:
reference = os.path.basename(reference)
assert reference in formatted, \
"Rules update failure. " + srepr(reference) + " does not appear in new rmap." \
" May be identical match with other submitted references."
return new
def get_refactoring_header(rmapping, refname, required_keywords):
"""Create a composite header which is derived from the file contents overidden by any values
as they appear in the rmap.
"""
rmapping = rmap.asmapping(rmapping)
# A fallback source of information is the reference file headers
header = rmapping.get_refactor_header(
rmap.locate_file(refname, rmapping.observatory),
extra_keys=("META.OBSERVATION.DATE", "META.OBSERVATION.TIME", "DATE-OBS","TIME-OBS") + required_keywords)
# The primary source of information is the original rmap and the matching values defined there
headers2 = matches.find_match_paths_as_dict(rmapping.filename, refname)
# Combine the two, using the rmap values to override anything duplicated in the reffile header
assert len(headers2) == 1, "Can't refactor file with more than one match: " + srepr(refname)
header.update(headers2[0])
return header
def get_refactoring_header(rmapping, refname, required_keywords):
"""Create a composite header which is derived from the file contents overidden by any values
as they appear in the rmap.
"""
rmapping = rmap.asmapping(rmapping)
# A fallback source of information is the reference file headers
header = rmapping.get_refactor_header(
rmap.locate_file(refname, rmapping.observatory),
extra_keys=("META.OBSERVATION.DATE", "META.OBSERVATION.TIME", "DATE-OBS","TIME-OBS") + required_keywords)
# The primary source of information is the original rmap and the matching values defined there
headers2 = matches.find_match_paths_as_dict(rmapping.filename, refname)
# Combine the two, using the rmap values to override anything duplicated in the reffile header
assert len(headers2) == 1, "Can't refactor file with more than one match: " + srepr(refname)
header.update(headers2[0])
return header
def _process_rmap(self, func, rmapping, *args, **keys):
"""Execute `func` on a single `rmapping` passing along *args and **keys"""
keywords = dict(keys)
rmapping_org = rmapping
new_filename = rmapping.filename if self.args.inplace else os.path.join(".", rmapping.basename)
if os.path.exists(new_filename):
log.info("Continuing refactoring from local copy", srepr(new_filename))
rmapping = rmap.load_mapping(new_filename)
keywords.update(locals())
fixers = self.args.fixers
if fixers:
rmapping = rmap.load_mapping(rmapping.filename)
keywords.update(locals())
apply_rmap_fixers(*args, **keywords)
func(*args, **keywords)
return new_filename
def _process_rmap(self, func, rmapping, *args, **keys):
"""Execute `func` on a single `rmapping` passing along *args and **keys"""
keywords = dict(keys)
rmapping_org = rmapping
new_filename = rmapping.filename if self.args.inplace else os.path.join(".", rmapping.basename)
if os.path.exists(new_filename):
log.info("Continuing refactoring from local copy", srepr(new_filename))
rmapping = rmap.load_mapping(new_filename)
keywords.update(locals())
fixers = self.args.fixers
if fixers:
rmapping = rmap.load_mapping(rmapping.filename)
keywords.update(locals())
apply_rmap_fixers(*args, **keywords)
func(*args, **keywords)
return new_filename
def _submission(self, relative_url):
"""Do a generic submission re-post to the specified relative_url."""
assert self.args.description is not None, "You must supply a --description for this function."
self.ingest_files()
log.info("Posting web request for", srepr(relative_url))
completion_args = self.connection.repost_start(
relative_url,
pmap_mode=self.pmap_mode,
pmap_name=self.pmap_name,
instrument=self.instrument,
change_level=self.args.change_level,
creator=self.args.creator,
description=self.args.description,
auto_rename=not self.args.dont_auto_rename,
compare_old_reference=not self.args.dont_compare_old_reference,
)
# give POST time to complete send, not response
time.sleep(10)
return completion_args
def dump_references(pipeline_context, baserefs=None, ignore_cache=False, raise_exceptions=True, api=1):
"""Given a pipeline `pipeline_context` and list of `baserefs` reference
file basenames, obtain the set of reference files and cache them on the
local file system.
If `basrefs` is None, sync the closure of references referred to by `pipeline_context`.
Returns: { ref_basename : reference_local_filepath ... } (api=1)
{ ref_basename : reference_local_path }, downloads, bytes (api=2)
"""
if baserefs is None:
baserefs = get_reference_names(pipeline_context)
baserefs = list(baserefs)
for refname in baserefs:
if "NOT FOUND" in refname:
log.verbose("Skipping " + srepr(refname))
baserefs.remove(refname)
baserefs = sorted(set(baserefs))
return FileCacher(pipeline_context, ignore_cache, raise_exceptions, api).get_local_files(baserefs)
def _submission(self, relative_url):
"""Do a generic submission re-post to the specified relative_url."""
assert self.args.description is not None, "You must supply a --description for this function."
self.ingest_files()
log.info("Posting web request for", srepr(relative_url))
completion_args = self.connection.repost_start(
relative_url,
pmap_mode = self.pmap_mode,
pmap_name = self.pmap_name,
instrument = self.instrument,
change_level=self.args.change_level,
creator=self.args.creator,
description=self.args.description,
auto_rename=not self.args.dont_auto_rename,
compare_old_reference=not self.args.dont_compare_old_reference,
)
# give POST time to complete send, not response
time.sleep(10)
return completion_args
def local_bestrefs(parameters, reftypes, context, ignore_cache=False):
"""Perform bestref computations locally, contacting the network only to
obtain references or mappings which are not already cached locally.
In the case of the default "auto" mode, assuming it has an up-to-date client
CRDS will only use the server for status and to transfer files.
"""
# Make sure pmap_name is actually present in the local machine's cache.
# First assume the context files are already here and try to load them.
# If that fails, attempt to get them from the network, then load them.
try:
if ignore_cache:
raise IOError("explicitly ignoring cache.")
# Finally do the best refs computation using pmap methods from local code.
return hv_best_references(context, parameters, reftypes)
except IOError as exc:
log.verbose("Caching mapping files for context", srepr(context))
try:
api.dump_mappings(context, ignore_cache=ignore_cache)
except CrdsError as exc:
traceback.print_exc()
raise CrdsNetworkError("Failed caching mapping files: " + str(exc))
return hv_best_references(context, parameters, reftypes)
def get_data_http(self, filename):
"""Yield the data returned from `filename` of `pipeline_context` in manageable chunks."""
url = self.get_url(filename)
try:
infile = urlopen(url)
file_size = utils.human_format_number(self.catalog_file_size(filename)).strip()
stats = utils.TimingStats()
data = infile.read(config.CRDS_DATA_CHUNK_SIZE)
while data:
stats.increment("bytes", len(data))
status = stats.status("bytes")
bytes_so_far = " ".join(status[0].split()[:-1])
log.verbose("Transferred HTTP", repr(url), bytes_so_far, "/", file_size, "bytes at", status[1], verbosity=20)
yield data
data = infile.read(config.CRDS_DATA_CHUNK_SIZE)
except Exception as exc:
raise CrdsDownloadError("Failed downloading", srepr(filename), "from url", srepr(url), ":", str(exc))
finally:
try:
infile.close()
except UnboundLocalError: # maybe the open failed.
pass
def get_server_info():
"""Return a dictionary of critical parameters about the server such as:
operational_context - the context in use in the operational pipeline
edit_context - the context which was last edited, not
necessarily archived or operational yet.
crds* - the CRDS package versions on the server.
This is intended as a single flexible network call which can be used to
initialize a higher level getreferences() call, providing information on
what context, software, and network mode should be used for processing.
"""
try:
info = S.get_server_info()
info["server"] = get_crds_server()
info["reference_url"] = info.pop("reference_url")["unchecked"]
info["mapping_url"] = info.pop("mapping_url")["unchecked"]
return info
except ServiceError as exc:
raise CrdsNetworkError("network connection failed: " + srepr(get_crds_server()) + " : " + str(exc))
def local_bestrefs(parameters, reftypes, context, ignore_cache=False):
"""Perform bestref computations locally, contacting the network only to
obtain references or mappings which are not already cached locally.
In the case of the default "auto" mode, assuming it has an up-to-date client
CRDS will only use the server for status and to transfer files.
"""
# Make sure pmap_name is actually present in the local machine's cache.
# First assume the context files are already here and try to load them.
# If that fails, attempt to get them from the network, then load them.
try:
if ignore_cache:
raise IOError("explicitly ignoring cache.")
# Finally do the best refs computation using pmap methods from local code.
return rmap.get_best_references(context, parameters, reftypes)
except IOError as exc:
log.verbose("Caching mapping files for context", srepr(context))
try:
api.dump_mappings(context, ignore_cache=ignore_cache)
except CrdsError as exc:
traceback.print_exc()
raise CrdsNetworkError("Failed caching mapping files: " + str(exc))
return rmap.get_best_references(context, parameters, reftypes)
def rmap_insert_references_by_matches(old_rmap, new_rmap, references_headers):
"""Given the full path of starting rmap `old_rmap`, modify it by inserting
or replacing all files in dict `references_headers` which maps a reference file basename
onto a list of headers under which it should be matched. Write out the result to
`new_rmap`. If no actions are performed, don't write out `new_rmap`.
Return new ReferenceMapping named `new_rmap`
"""
new = old = rmap.load_mapping(old_rmap, ignore_checksum=True)
for baseref, header in references_headers.items():
with log.augment_exception("In reference", srepr(baseref)):
log.info("Inserting", srepr(baseref), "into", srepr(old_rmap))
log.verbose("Inserting", srepr(baseref), "match case", srepr(header), "into", srepr(old_rmap))
new = new.insert_header_reference(header, baseref)
new.header["derived_from"] = old.basename
log.verbose("Writing", srepr(new_rmap))
new.write(new_rmap)
formatted = new.format()
for baseref in references_headers:
assert baseref in formatted, \
"Rules update failure. " + srepr(baseref) + " does not appear in new rmap." \
" May be identical match with other submitted references."
return new
def copytree(src, dst, symlinks=False, fnc_directory=_no_message,
fnc_file=_no_message, fnc_symlink=_no_message):
"""Derived from shutil.copytree() example with added function hooks called
on a per-directory, per-file, and per-symlink basis with (src, dest)
parameters. Removes exception trapping since partial copies are useless
for CRDS. Cannot handle devices or sockets, only regular files and
directories. File stats not preserved.
"""
os.makedirs(dst)
for name in os.listdir(src):
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
if symlinks and os.path.islink(srcname):
linkto = os.readlink(srcname)
fnc_symlink("Linking", log.srepr(linkto), "to", log.srepr(dstname))
os.symlink(linkto, dstname)
elif os.path.isdir(srcname):
fnc_directory("Copying dir", log.srepr(srcname), "to", log.srepr(dstname))
copytree(srcname, dstname, symlinks)
else:
fnc_file("Coping", log.srepr(srcname), "to", log.srepr(dstname))
shutil.copy(srcname, dstname)
def plugin_download(self, filename, localpath):
"""Run an external program defined by CRDS_DOWNLOAD_PLUGIN to download filename to localpath."""
url = self.get_url(filename)
plugin_cmd = config.get_download_plugin()
plugin_cmd = plugin_cmd.replace("${SOURCE_URL}", url)
plugin_cmd = plugin_cmd.replace("${OUTPUT_PATH}", localpath)
log.verbose("Running download plugin:", repr(plugin_cmd))
status = os.system(plugin_cmd)
if status != 0:
if status == 2:
raise KeyboardInterrupt("Interrupted plugin.")
else:
raise CrdsDownloadError("Plugin download fail status = {} with command: {}".format(status, srepr(plugin_cmd)))
