def _get_config_refpath(context, cal_ver):
"""Given CRDS `context` and calibration s/w version `cal_ver`, identify the applicable
SYSTEM CRDSCFG reference file, cache it, and return the file path.
"""
i = 0
while (i < len(REFPATHS)-1 and not _versions_lt(cal_ver, REFPATHS[i+1][0])):
i += 1
refpath = os.path.join(HERE, REFPATHS[i][1])
try: # Use a normal try/except because exceptions are expected.
header = {
"META.INSTRUMENT.NAME" : "SYSTEM",
"META.CALIBRATION_SOFTWARE_VERSION": cal_ver
}
pmap = crds.get_symbolic_mapping(context)
imap = pmap.get_imap("system")
rmapping = imap.get_rmap("crdscfg")
ref = rmapping.get_best_ref(header)
refpath = rmapping.locate_file(ref)
api.dump_references(context, [ref])
except Exception:
log.verbose_warning(
"Failed locating SYSTEM CRDSCFG reference",
"under context", repr(context),
"and cal_ver", repr(cal_ver) + ". Using built-in references.")
log.verbose("Using", srepr(os.path.basename(refpath)),
"to determine applicable default reftypes for", srepr(cal_ver))
return refpath
def organize_references(self, new_mode):
"""Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`.
This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument.
"""
old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True)
old_mode = config.get_crds_ref_subdir_mode(self.observatory)
log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode))
config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory)
new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change.
for refpath in old_refpaths:
with log.error_on_exception("Failed relocating:", repr(refpath)):
desired_loc = config.locate_file(os.path.basename(refpath), observatory=self.observatory)
if desired_loc != refpath:
if os.path.exists(desired_loc):
if not self.args.organize_delete_junk:
log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath))
continue
utils.remove(desired_loc, observatory=self.observatory)
if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)):
log.info("Relocating", repr(refpath), "to", repr(desired_loc))
shutil.move(refpath, desired_loc)
else:
if old_mode != new_mode:
log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode))
else:
log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath))
if new_mode == "flat" and old_mode == "instrument":
log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.")
for instrument in self.locator.INSTRUMENTS:
self.remove_dir(instrument)
def _reduce_header(self, old_header, needed_keys=()):
"""Limit `header` to `needed_keys`, converting all keys to upper case
and making note of any significant duplicates, and adding any missing
`needed_keys` as UNDEFINED.
To detect duplicates, use an item list for `old_header`, not a dict.
"""
needed_keys = tuple(key.upper() for key in needed_keys)
header = {}
if isinstance(old_header, dict):
old_header = old_header.items()
for (key, value) in old_header:
key = str(key).upper()
value = str(value)
if (not needed_keys) or key in needed_keys:
if key in header and header[key] != value:
if key not in DUPLICATES_OK:
log.verbose_warning("Duplicate key", repr(key), "in", repr(self.filepath),
"using", repr(header[key]), "not", repr(value), verbosity=70)
continue
elif key in APPEND_KEYS:
header[key] += "\n" + value
else:
header[key] = value
return ensure_keys_defined(header, needed_keys)
def check_header(self, filename, header):
"""Evalutate the header expression associated with this validator (as its sole value)
with respect to the given `header`.
Note that array-based checkers are not automatically loaded during a classic header
fetch and expressions can involve operations on multiple keywords or arrays.
"""
log.verbose("File=" + repr(os.path.basename(filename)), "Checking",
repr(self.name), "condition", str(self._expr))
for keyword in expr_identifiers(self._expr):
if header.get(keyword, "UNDEFINED") == "UNDEFINED":
log.verbose_warning("Keyword or Array", repr(keyword),
"is 'UNDEFINED'. Skipping ",
repr(self._expr))
return True # fake satisfied
try:
satisfied = eval(self._expr_code, header, self._eval_namespace)
except Exception as exc:
raise RequiredConditionError("Failed checking constraint",
repr(self._expr), ":", str(exc))
if not satisfied:
raise RequiredConditionError("Constraint", str(self._expr),
"is not satisfied.")
elif satisfied == "W": # from warn_only() helper
log.warning("Constraint", str(self._expr), "is not satisfied.")
satisfied = True
return satisfied
def cat_files(self):
"""Print out the files listed after --cat or implied by a combination of
explicitly specified contexts and --mappings or --references.
--files is not allowed.
"""
# --cat files... specifying *no* files still triggers --cat logic
# --contexts context-specifiers [including --all --last --range...]
# context specifiers can be symbolic and will be resolved.
# --cat @file is allowed
catted_files = self.get_words(self.args.cat) + self.implied_files
try:
self._file_info = api.get_file_info_map(
self.observatory,
files=[
os.path.basename(filename) for filename in catted_files
])
except Exception:
log.verbose_warning(
"Failed retrieving CRDS server catalog information. May need to set CRDS_SERVER_URL."
)
# This could be expanded to include the closure of mappings or references
for name in catted_files:
with log.error_on_exception("Failed dumping:", repr(name)):
path = self.locate_file(name)
self._cat_file(path)
def wrapper(*args, **keys):
"""Reassign warnings to CRDS warnings prior to executing `func`, restore
warnings state afterwards and return result of `func`.
"""
# warnings.resetwarnings()
from astropy.utils.exceptions import AstropyUserWarning
with warnings.catch_warnings():
old_showwarning = warnings.showwarning
warnings.showwarning = hijacked_showwarning
warnings.simplefilter("always", AstropyUserWarning)
try:
from stdatamodels.validate import ValidationWarning
except:
log.verbose_warning(
"stdatamodels ValidationWarning import failed. "
"Not a problem for HST.",
verbosity=70)
else:
warnings.filterwarnings("always", r".*", ValidationWarning, r".*jwst.*")
if not config.ALLOW_SCHEMA_VIOLATIONS:
warnings.filterwarnings("error", r".*is not one of.*", ValidationWarning, r".*jwst.*")
try:
result = func(*args, **keys)
finally:
warnings.showwarning = old_showwarning
return result
def _get_config_refpath(context, cal_ver):
"""Given CRDS `context` and calibration s/w version `cal_ver`, identify the applicable
SYSTEM CRDSCFG reference file, cache it, and return the file path.
"""
context = _get_missing_context(context)
cal_ver = _get_missing_calver(cal_ver)
i = 0
while (i < len(REFPATHS) - 1
and not _versions_lt(cal_ver, REFPATHS[i + 1][0])):
i += 1
refpath = os.path.join(HERE, REFPATHS[i][1])
try: # Use a normal try/except because exceptions are expected.
header = {
"META.INSTRUMENT.NAME": "SYSTEM",
"META.CALIBRATION_SOFTWARE_VERSION": cal_ver
}
pmap = crds.get_symbolic_mapping(context)
imap = pmap.get_imap("system")
rmapping = imap.get_rmap("crdscfg")
ref = rmapping.get_best_ref(header)
refpath = rmapping.locate_file(ref)
api.dump_references(context, [ref])
except Exception:
log.verbose_warning("Failed locating SYSTEM CRDSCFG reference",
"under context", repr(context), "and cal_ver",
repr(cal_ver) + ". Using built-in references.")
log.verbose("Using", srepr(os.path.basename(refpath)),
"to determine applicable default reftypes for", srepr(cal_ver))
return refpath
def organize_references(self, new_mode):
"""Find all references in the CRDS cache and relink them to the paths which are implied by `new_mode`.
This is used to reroganize existing file caches into new layouts, e.g. flat --> by instrument.
"""
old_refpaths = rmap.list_references("*", observatory=self.observatory, full_path=True)
old_mode = config.get_crds_ref_subdir_mode(self.observatory)
log.info("Reorganizing", len(old_refpaths), "references from", repr(old_mode), "to", repr(new_mode))
config.set_crds_ref_subdir_mode(new_mode, observatory=self.observatory)
new_mode = config.get_crds_ref_subdir_mode(self.observatory) # did it really change.
for refpath in old_refpaths:
with log.error_on_exception("Failed relocating:", repr(refpath)):
desired_loc = rmap.locate_file(os.path.basename(refpath), observatory=self.observatory)
if desired_loc != refpath:
if os.path.exists(desired_loc):
if not self.args.organize_delete_junk:
log.warning("Link or directory already exists at", repr(desired_loc), "Skipping", repr(refpath))
continue
utils.remove(desired_loc, observatory=self.observatory)
if config.writable_cache_or_info("Skipping file relocation from", repr(refpath), "to", repr(desired_loc)):
log.info("Relocating", repr(refpath), "to", repr(desired_loc))
shutil.move(refpath, desired_loc)
else:
if old_mode != new_mode:
log.verbose_warning("Keeping existing cached file", repr(desired_loc), "already in target mode", repr(new_mode))
else:
log.verbose_warning("No change in subdirectory mode", repr(old_mode), "skipping reorganization of", repr(refpath))
if new_mode == "flat" and old_mode == "instrument":
log.info("Reorganizing from 'instrument' to 'flat' cache, removing instrument directories.")
for instrument in self.locator.INSTRUMENTS:
self.remove_dir(instrument)
def append_tpn_level(results, instrument, filekind):
"""Append the validator key for one level of the `instrument`
and `filekind` mutating list `results`.
"""
try:
tpnfile = self.unified_defs[instrument][filekind][field]
log.verbose("Adding validator key", repr(tpnfile), verbosity=70)
results.append(tpnfile)
except Exception as exc:
log.verbose_warning("Can't find TPN key for",
(instrument, filekind, field), ":", str(exc),
verbosity=75)
def append_tpn_level(results, instrument, filekind):
"""Append the validator key for one level of the `instrument`
and `filekind` mutating list `results`.
"""
try:
tpnfile = self.unified_defs[instrument][filekind][field]
log.verbose("Adding validator key", repr(tpnfile), verbosity=70)
results.append(tpnfile)
except Exception as exc:
log.verbose_warning("Can't find TPN key for",
(instrument, filekind, field), ":", str(exc),
verbosity=75)
def apply_with_retries(func, *pars, **keys):
"""Apply function func() as f(*pargs, **keys) and return the result. Retry on any exception as defined in config.py"""
retries = config.get_client_retry_count()
delay = config.get_client_retry_delay_seconds()
for retry in range(retries):
try:
return func(*pars, **keys)
except Exception as exc:
log.verbose_warning("FAILED: Attempt", str(retry+1), "of", retries, "with:", str(exc))
log.verbose_warning("FAILED: Waiting for", delay, "seconds before retrying") # waits after total fail...
time.sleep(delay)
exc2 = exc
raise exc2
def apply_with_retries(func, *pars, **keys):
"""Apply function func() as f(*pargs, **keys) and return the result. Retry on any exception as defined in config.py"""
retries = config.get_client_retry_count()
delay = config.get_client_retry_delay_seconds()
for retry in range(retries):
try:
return func(*pars, **keys)
except Exception as exc:
log.verbose_warning("FAILED: Attempt", str(retry+1), "of", retries, "with:", str(exc))
log.verbose_warning("FAILED: Waiting for", delay, "seconds before retrying") # waits after total fail...
time.sleep(delay)
exc2 = exc
raise exc2
def main(self):
"""Synchronize files."""
# clear any mutliprocessing locks associated with the CRDS cache.
if self.args.clear_locks:
crds_cache_locking.clear_cache_locks()
return log.errors()
self.handle_misc_switches() # simple side effects only
# if explicitly requested, or the cache is suspect or being ignored, clear
# cached context pickles.
if self.args.clear_pickles or self.args.ignore_cache or self.args.repair_files:
self.clear_pickles()
# utility to change cache structure, e.g. add instrument subdirs.
# do this before syncing anything under the current mode.
if self.args.organize:
self.organize_references(self.args.organize)
# fetching and verifying files both require a server connection.
self.require_server_connection()
# primary sync'ing occurs here, both mappings and references as well as odd-ball
# server sqlite3 database download.
verify_file_list = self.file_transfers()
# verification is relative to sync'ed files, and can include file replacement if
# defects are found.
if (self.args.check_files or self.args.check_sha1sum or self.args.repair_files or
self.args.purge_blacklisted or self.args.purge_rejected):
self.verify_files(verify_file_list)
# context pickles should only be (re)generated after mappings are fully sync'ed and verified
if self.args.save_pickles:
self.pickle_contexts(self.contexts)
# update CRDS cache config area, including stored version of operational context.
# implement pipeline support functions of context update verify and echo
# If --output-dir was specified, do not update cache config.
if self.args.output_dir:
log.verbose_warning("Used --output-dir, skipping cache server_config update including default context and bad files.")
if config.writable_cache_or_verbose("skipping removal of ref_cache_subdir_mode file."):
os.remove(config.get_crds_ref_subdir_file(self.observatory))
else:
self.update_context()
self.report_stats()
log.standard_status()
return log.errors()
def verify_file(self, file, info, bytes_so_far, total_bytes, nth_file, total_files):
"""Check one `file` against the provided CRDS database `info` dictionary."""
path = config.locate_file(file, observatory=self.observatory)
base = os.path.basename(file)
n_bytes = int(info["size"])
# Only output verification info for slow sha1sum checks by default
log.verbose(
api.file_progress(
"Verifying", base, path, n_bytes, bytes_so_far, total_bytes, nth_file, total_files),
verbosity=10 if self.args.check_sha1sum else 60)
if not os.path.exists(path):
if base not in self.bad_files:
log.error("File", repr(base), "doesn't exist at", repr(path))
return
# Checks which force repairs should do if/else to avoid repeat repair
size = os.stat(path).st_size
if int(info["size"]) != size:
self.error_and_repair(path, "File", repr(base), "length mismatch LOCAL size=" + srepr(size),
"CRDS size=" + srepr(info["size"]))
elif self.args.check_sha1sum or config.is_mapping(base):
log.verbose("Computing checksum for", repr(base), "of size", repr(size), verbosity=60)
sha1sum = utils.checksum(path)
if info["sha1sum"] == "none":
log.warning("CRDS doesn't know the checksum for", repr(base))
elif info["sha1sum"] != sha1sum:
self.error_and_repair(path, "File", repr(base), "checksum mismatch CRDS=" + repr(info["sha1sum"]),
"LOCAL=" + repr(sha1sum))
if info["state"] not in ["archived", "operational"]:
log.warning("File", repr(base), "has an unusual CRDS file state", repr(info["state"]))
if info["rejected"] != "false":
log.verbose_warning("File", repr(base), "has been explicitly rejected.", verbosity=60)
if self.args.purge_rejected:
self.remove_files([path], "file")
return
if info["blacklisted"] != "false":
log.verbose_warning("File", repr(base), "has been blacklisted or is dependent on a blacklisted file.",
verbosity=60)
if self.args.purge_blacklisted:
self.remove_files([path], "file")
return
return
def main(self):
"""Synchronize files."""
# clear any mutliprocessing locks associated with the CRDS cache.
if self.args.clear_locks:
crds_cache_locking.clear_cache_locks()
return log.errors()
self.handle_misc_switches() # simple side effects only
# if explicitly requested, or the cache is suspect or being ignored, clear
# cached context pickles.
if self.args.clear_pickles or self.args.ignore_cache or self.args.repair_files:
self.clear_pickles()
# utility to change cache structure, e.g. add instrument subdirs.
# do this before syncing anything under the current mode.
if self.args.organize:
self.organize_references(self.args.organize)
# fetching and verifying files both require a server connection.
self.require_server_connection()
# primary sync'ing occurs here, both mappings and references as well as odd-ball
# server sqlite3 database download.
verify_file_list = self.file_transfers()
# verification is relative to sync'ed files, and can include file replacement if
# defects are found.
if self.args.check_files or self.args.check_sha1sum or self.args.repair_files:
self.verify_files(verify_file_list)
# context pickles should only be (re)generated after mappings are fully sync'ed and verified
if self.args.save_pickles:
self.pickle_contexts(self.contexts)
# update CRDS cache config area, including stored version of operational context.
# implement pipeline support functions of context update verify and echo
# If explicit files were specified, do not update cache config.
if self.args.files and self.args.output_dir:
log.verbose_warning("Used explicit --files list and --output-dir, skipping cache server_config update including default context and bad files.")
else:
self.update_context()
self.report_stats()
log.standard_status()
return log.errors()
def verify_file(self, file, info, bytes_so_far, total_bytes, nth_file, total_files):
"""Check one `file` against the provided CRDS database `info` dictionary."""
path = rmap.locate_file(file, observatory=self.observatory)
base = os.path.basename(file)
n_bytes = int(info["size"])
# Only output verification info for slow sha1sum checks by default
log.verbose(
api.file_progress(
"Verifying", base, path, n_bytes, bytes_so_far, total_bytes, nth_file, total_files),
verbosity=10 if self.args.check_sha1sum else 60)
if not os.path.exists(path):
log.error("File", repr(base), "doesn't exist at", repr(path))
return
# Checks which force repairs should do if/else to avoid repeat repair
size = os.stat(path).st_size
if int(info["size"]) != size:
self.error_and_repair(path, "File", repr(base), "length mismatch LOCAL size=" + srepr(size),
"CRDS size=" + srepr(info["size"]))
elif self.args.check_sha1sum or config.is_mapping(base):
log.verbose("Computing checksum for", repr(base), "of size", repr(size), verbosity=60)
sha1sum = utils.checksum(path)
if info["sha1sum"] == "none":
log.warning("CRDS doesn't know the checksum for", repr(base))
elif info["sha1sum"] != sha1sum:
self.error_and_repair(path, "File", repr(base), "checksum mismatch CRDS=" + repr(info["sha1sum"]),
"LOCAL=" + repr(sha1sum))
if info["state"] not in ["archived", "operational"]:
log.warning("File", repr(base), "has an unusual CRDS file state", repr(info["state"]))
if info["rejected"] != "false":
log.verbose_warning("File", repr(base), "has been explicitly rejected.", verbosity=60)
if self.args.purge_rejected:
self.remove_files([path], "files")
return
if info["blacklisted"] != "false":
log.verbose_warning("File", repr(base), "has been blacklisted or is dependent on a blacklisted file.",
verbosity=60)
if self.args.purge_blacklisted:
self.remove_files([path], "files")
return
return
def cross_strap_header(header):
"""Set up keyword equivalencies in a copy of `header`. Ensure both FITS
and datamodel dotted path variants are defined for each keyword.
Also add variations defined by observatory locator module
CROSS_STRAPPED_KEYWORDS.
"""
crossed = dict(header)
try:
locator = utils.header_to_locator(header)
except Exception:
log.verbose_warning(
"Cannot identify observatory from header. Skipping keyword aliasing")
return crossed
equivalency_pairs = locator.get_cross_strapped_pairs(header)
for pair in equivalency_pairs:
_cross_strap_pair(crossed, pair)
return crossed
def _x_schema_to_flat(schema):
"""Recursively flatten `schema` without addressing case issues."""
results = {}
for feature in ["oneOf", "allOf", "$ref"]:
if feature in schema:
log.verbose_warning("Schema item has unhandled feature {}.",
verbosity=80)
return None
if "anyOf" in schema and "type" in schema["anyOf"]:
schema_type = schema["anyOf"]["type"]
else:
schema_type = schema.get("type", "null")
if schema_type == "object":
subprops = schema["properties"]
for prop in subprops:
with log.augment_exception("In schema property", repr(prop)):
sub_tree = _schema_to_flat(subprops[prop])
if sub_tree is None:
continue
if isinstance(sub_tree, dict):
for subprop, val in list(sub_tree.items()):
results[prop + "." + subprop] = val
else:
results[prop] = sub_tree
elif schema_type in BASIC_TYPES:
return schema
elif schema_type in OPTIONAL_TYPES:
return schema
elif schema_type == "array":
return None
elif schema_type in ["any", "null"]:
return None
else:
log.verbose_warning("Schema item has unhandled type",
repr(schema_type),
verbosity=80)
return results
def get_history(self, handle):
"""Given and ASDF file object `handle`, return the history collected into a
single string.
"""
history = "UNDEFINED" # or BAD FORMAT
with log.error_on_exception(
"Failed reading ASDF history, see ASDF docs on adding history"):
histall = []
hist = handle.tree["history"]
try:
entries = handle.get_history_entries()
except Exception:
log.verbose_warning(
"Using inlined CRDS ASDF history entry reading interface.")
entries = hist["entries"] if "entries" in hist else hist
for entry in entries:
time = timestamp.format_date(entry["time"]).split(".")[0]
description = entry["description"]
histall.append(time + " :: " + description)
if histall:
history = "\n".join(histall)
return history
def _x_schema_to_flat(schema):
"""Recursively flatten `schema` without addressing case issues."""
results = {}
for feature in ["oneOf","allOf","$ref"]:
if feature in schema:
log.verbose_warning("Schema item has unhandled feature {}.", verbosity=80)
return None
if "anyOf" in schema and "type" in schema["anyOf"]:
schema_type = schema["anyOf"]["type"]
else:
schema_type = schema.get("type", "null")
if schema_type == "object":
subprops = schema["properties"]
for prop in subprops:
with log.augment_exception("In schema property", repr(prop)):
sub_tree = _schema_to_flat(subprops[prop])
if sub_tree is None:
continue
if isinstance(sub_tree, dict):
for subprop, val in list(sub_tree.items()):
results[prop + "." + subprop] = val
else:
results[prop] = sub_tree
elif schema_type in BASIC_TYPES:
return schema
elif schema_type in OPTIONAL_TYPES:
return schema
elif schema_type == "array":
return None
elif schema_type in ["any", "null"]:
return None
else:
log.verbose_warning("Schema item has unhandled type", repr(schema_type), verbosity=80)
return results
def cat_files(self):
"""Print out the files listed after --cat or implied by a combination of
explicitly specified contexts and --mappings or --references.
--files is not allowed.
"""
# --cat files... specifying *no* files still triggers --cat logic
# --contexts context-specifiers [including --all --last --range...]
# context specifiers can be symbolic and will be resolved.
# --cat @file is allowed
catted_files = self.get_words(self.args.cat) + self.implied_files
try:
self._file_info = api.get_file_info_map(
self.observatory, files=[os.path.basename(filename) for filename in catted_files])
except Exception:
log.verbose_warning("Failed retrieving CRDS server catalog information. May need to set CRDS_SERVER_URL.")
# This could be expanded to include the closure of mappings or references
for name in catted_files:
with log.error_on_exception("Failed dumping:", repr(name)):
path = self.locate_file(name)
self._cat_file(path)
def check_header(self, filename, header):
"""Evalutate the header expression associated with this validator (as its sole value)
with respect to the given `header`.
Note that array-based checkers are not automatically loaded during a classic header
fetch and expressions can involve operations on multiple keywords or arrays.
"""
log.verbose("File=" + repr(os.path.basename(filename)), "Checking",
repr(self.name), "condition", str(self._expr))
for keyword in expr_identifiers(self._expr):
if header.get(keyword, "UNDEFINED") == "UNDEFINED":
log.verbose_warning("Keyword or Array", repr(keyword),
"is 'UNDEFINED'. Skipping ", repr(self._expr))
return True # fake satisfied
try:
satisfied = eval(self._expr_code, header, self._eval_namespace)
except Exception as exc:
raise RequiredConditionError("Failed checking constraint", repr(self._expr), ":", str(exc))
if not satisfied:
raise RequiredConditionError("Constraint", str(self._expr), "is not satisfied.")
elif satisfied == "W": # from warn_only() helper
log.warning("Constraint", str(self._expr), "is not satisfied.")
satisfied = True
return satisfied
"""This module codifies standard practices for scripted interactions with the
web server file submission system.
"""
from crds.core import log, utils
from . import background
# from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
try:
import requests
DISABLED = []
except (ImportError, RuntimeError):
log.verbose_warning("Import of 'requests' failed. submit disabled.")
DISABLED.append("requests")
try:
from lxml import html
except (ImportError, RuntimeError):
log.verbose_warning("Import of 'lxml' failed. submit disabled.")
DISABLED.append("lxml")
# ==================================================================================================
def log_section(section_name, section_value, verbosity=50, log_function=log.verbose,
divider_name=None):
"""Issue log divider bar followed by a corresponding log message."""
log.divider(name=divider_name, verbosity=verbosity, func=log.verbose)
log_function(section_name, section_value, verbosity=verbosity+5)
# ==================================================================================================
class CrdsDjangoConnection:
def is_reprocessing_required(dataset, dataset_parameters, old_context, new_context, update):
"""This is the top level interface to crds.bestrefs running in "Affected Datasets" mode.
It determines if reprocessing `dataset` with parameters `dataset_parameters` should be performed as
a consequence of switching from `old_reference` to `new_reference`. old_reference is assigned to dataset
by old_context, and new_reference is assigned to dataset by new_context.
Parameters
----------
dataset:
id of dataset being reprocessed, <assoc>:<member> or <unassoc>:<unassoc> format
dataset_parameters:
{ parameter : value, ...} for all matching parameters and row selection parameters
XXX row selection parameters not used in file selection may not be present until
XXX explicitly added to the CRDS interface to the DADSOPS parameter database...
XXX and possibly even to DADSOPS itself. Normally the row selections have only been
XXX done with direct access to dataset .fits files.
old_context: loaded pmap or name of old context, possibly for metadata or None
new_context: loaded pmap or name of new context, possibly for metadata
update: Update object
Returns
-------
True IFF reprocessing should be done as a consequence of the table change.
"""
log.verbose('is_reprocessing_required: Called with:\n',
dataset, '\n',
dataset_parameters, '\n',
old_context, '\n',
new_context, '\n',
update,
verbosity=100)
# no old_context means "single context" mode, always reprocess.
if old_context is None:
return True
# NOTE: non-tables are treated in DeepLook as filekinds which aren't (or maybe someday are) handled,
# hence reprocessed for now.
# Reprocess for non-file special values. Other code will decide what to do with the updates,
# the point here is that table comparison isn't possible so filtering shouldn't be done.
old_ref = update.old_reference.lower()
new_ref = update.new_reference.lower()
incomparable = ('n/a', 'undefined', 'not found')
if old_ref.startswith(incomparable) or new_ref.startswith(incomparable):
return True
# mostly debug wrappers here, allows simple string parameters to work and resolves cache paths.
old_context = rmap.asmapping(old_context, cached=True)
new_context = rmap.asmapping(new_context, cached=True)
old_reference = old_context.locate_file(old_ref)
new_reference = new_context.locate_file(new_ref)
# Log that deep examination is occuring.
log.verbose('Deep Reference examination between {} and {} initiated.'.format(old_reference, new_reference),
verbosity=25)
with log.error_on_exception("Failed fetching comparison reference tables:", repr([old_ref, new_ref])):
api.dump_files(new_context.name, [old_ref, new_ref])
# See if deep checking into the reference is possible.
try:
deep_look = DeepLook.from_filekind(update.instrument, update.filekind)
dataset_id = dataset.split(':')[0]
# **DEBUG**
# ** Since we are not getting full headers, if this is a test
# ** dataset, replace the headers.
#log.verbose_warning('Forcing use of LBYX01010, regardless...', verbosity=25)
#dataset_id = 'LBYX01010' #***DEBUG: force headers regardless of actua data
if dataset_id in deep_look.stub_input:
log.verbose_warning('Substituting header for dataset "{}"'.format(dataset))
dataset_parameters = deep_look.stub_input[dataset_id]['headers']
log.verbose_warning('headers = ', dataset_parameters, verbosity=25)
log.verbose(deep_look.preamble, 'Dataset headers = {}'.format(dataset_parameters), verbosity=75)
log.verbose(deep_look.preamble, 'Comparing references {} and {}.'.format(old_reference, new_reference), verbosity=75)
deep_look.are_different(dataset_parameters, old_reference, new_reference)
log.verbose(deep_look.preamble, 'Reprocessing is {}required.'.format('' if deep_look.is_different else 'not '), verbosity=25)
log.verbose(deep_look.preamble, deep_look.message, verbosity=25)
return deep_look.is_different
except DeepLookError as error:
# Could not determine difference, therefore presume so.
log.verbose_warning('Deep examination error: {}'.format(error.message), verbosity=25)
log.verbose_warning('Deep examination failed, presuming reprocessing.', verbosity=25)
return True
"""This module codifies standard practices for scripted interactions with the
web server file submission system.
"""
from crds.core import log, utils
from . import background
# from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
try:
import requests
DISABLED = []
except (ImportError, RuntimeError):
log.verbose_warning("Import of 'requests' failed. submit disabled.")
DISABLED.append("requests")
try:
from lxml import html
except (ImportError, RuntimeError):
log.verbose_warning("Import of 'lxml' failed. submit disabled.")
DISABLED.append("lxml")
# ==================================================================================================
def log_section(section_name,
section_value,
verbosity=50,
log_function=log.verbose,
divider_name=None):
"""Issue log divider bar followed by a corresponding log message."""
log.divider(name=divider_name, verbosity=verbosity, func=log.verbose)
log_function(section_name, section_value, verbosity=verbosity + 5)
"""This module is used to verify the availability of a list of CRDS files
at the archive web server.
"""
import os.path
import sys
from crds.core import log, config, utils, cmdline
from crds.client import api
DISABLED = []
try:
import requests
except (ImportError, RuntimeError):
log.verbose_warning("Failed to import 'requests' module. check_archive disabled.")
DISABLED.append("requests")
class CheckArchiveScript(cmdline.Script):
"""Command line script for for checking archive file availability."""
description = """Command line script for for checking archive file availability."""
epilog = """
Checking out the archive with respect to CRDS file availability
and setting up a pipeline cache might have a few stages:
1. Run crds.list to generate a list of files to check, for example for
all contexts, do this:
% setenv CRDS_SERVER_URL https://jwst-crds.stsci.edu
% setenv CRDS_PATH $HOME/crds_cache_ops
% crds list --all --mappings --references >files.b6
def is_reprocessing_required(dataset, dataset_parameters, old_context, new_context, update):
"""This is the top level interface to crds.bestrefs running in "Affected Datasets" mode.
It determines if reprocessing `dataset` with parameters `dataset_parameters` should be performed as
a consequence of switching from `old_reference` to `new_reference`. old_reference is assigned to dataset
by old_context, and new_reference is assigned to dataset by new_context.
Parameters
----------
dataset:
id of dataset being reprocessed, <assoc>:<member> or <unassoc>:<unassoc> format
dataset_parameters:
{ parameter : value, ...} for all matching parameters and row selection parameters
XXX row selection parameters not used in file selection may not be present until
XXX explicitly added to the CRDS interface to the DADSOPS parameter database...
XXX and possibly even to DADSOPS itself. Normally the row selections have only been
XXX done with direct access to dataset .fits files.
old_context: loaded pmap or name of old context, possibly for metadata or None
new_context: loaded pmap or name of new context, possibly for metadata
update: Update object
Returns
-------
True IFF reprocessing should be done as a consequence of the table change.
"""
log.verbose('is_reprocessing_required: Called with:\n',
dataset, '\n',
dataset_parameters, '\n',
old_context, '\n',
new_context, '\n',
update,
verbosity=100)
# no old_context means "single context" mode, always reprocess.
if old_context is None:
return True
# NOTE: non-tables are treated in DeepLook as filekinds which aren't (or maybe someday are) handled,
# hence reprocessed for now.
# Reprocess for non-file special values. Other code will decide what to do with the updates,
# the point here is that table comparison isn't possible so filtering shouldn't be done.
old_ref = update.old_reference.lower()
new_ref = update.new_reference.lower()
incomparable = ('n/a', 'undefined', 'not found')
if old_ref.startswith(incomparable) or new_ref.startswith(incomparable):
return True
# mostly debug wrappers here, allows simple string parameters to work and resolves cache paths.
old_context = rmap.asmapping(old_context, cached=True)
new_context = rmap.asmapping(new_context, cached=True)
old_reference = old_context.locate_file(old_ref)
new_reference = new_context.locate_file(new_ref)
# Log that deep examination is occuring.
log.verbose('Deep Reference examination between {} and {} initiated.'.format(old_reference, new_reference),
verbosity=25)
with log.error_on_exception("Failed fetching comparison reference tables:", repr([old_ref, new_ref])):
api.dump_files(new_context.name, [old_ref, new_ref])
# See if deep checking into the reference is possible.
try:
deep_look = DeepLook.from_filekind(update.instrument, update.filekind)
dataset_id = dataset.split(':')[0]
# **DEBUG**
# ** Since we are not getting full headers, if this is a test
# ** dataset, replace the headers.
#log.verbose_warning('Forcing use of LBYX01010, regardless...', verbosity=25)
#dataset_id = 'LBYX01010' #***DEBUG: force headers regardless of actua data
if dataset_id in deep_look.stub_input:
log.verbose_warning('Substituting header for dataset "{}"'.format(dataset))
dataset_parameters = deep_look.stub_input[dataset_id]['headers']
log.verbose_warning('headers = ', dataset_parameters, verbosity=25)
log.verbose(deep_look.preamble, 'Dataset headers = {}'.format(dataset_parameters), verbosity=75)
log.verbose(deep_look.preamble, 'Comparing references {} and {}.'.format(old_reference, new_reference), verbosity=75)
deep_look.are_different(dataset_parameters, old_reference, new_reference)
log.verbose(deep_look.preamble, 'Reprocessing is {}required.'.format('' if deep_look.is_different else 'not '), verbosity=25)
log.verbose(deep_look.preamble, deep_look.message, verbosity=25)
return deep_look.is_different
except DeepLookError as error:
# Could not determine difference, therefore presume so.
log.verbose_warning('Deep examination error: {}'.format(error.message), verbosity=25)
log.verbose_warning('Deep examination failed, presuming reprocessing.', verbosity=25)
return True
