def certify_local_files(self):
"""Run CRDS certify on submitted files and fail/exit if errors are
found.
"""
if log.errors():
raise CrdsError("Errors encountered before CRDS certify run.")
certify.certify_files(
self.files, context=self.pmap_name, dump_provenance=True,
compare_old_reference=True, observatory=self.observatory,
run_fitsverify=True, check_rmap=False, check_sha1sums=True)
if log.errors():
raise CrdsError("Certify errors found. Aborting submission.")
def header(self, source):
"""Return the full header corresponding to `source`. If header is a string, raise an exception."""
header = self._header(source)
if isinstance(header, str):
raise CrdsError("Failed to fetch header for " + repr(source) + ": " + repr(header))
else:
return dict(header)
def reference_uri_to_cache_path(reference_uri, observatory):
"""Convert an abstract CRDS reference file URI into an absolute path for the
file as located in the CRDS cache.
Parameters
----------
reference_uri : string identifying an abstract CRDS reference file,
.e.g. 'crds://jwst_miri_flat_0177.fits'
observatory : string naming the telescope/project for CRDS, e.g. 'jwst', 'roman'
Returns
-------
reference_path : absolute file path to reference_uri in the CRDS cache,
.e.g. '/grp/crds/cache/references/jwst/jwst_miri_flat_0177.fits'
Standard CRDS cache paths are typically defined relative to the CRDS_PATH
environment variable. See https://jwst-crds.stsci.edu guide and top level
page for more info on configuring CRDS.
The default CRDS_PATH value is /grp/crds/cache, currently on the Central Store.
"""
if not reference_uri.startswith("crds://"):
raise CrdsError(
"CRDS reference URI's should start with 'crds://' but got", repr(reference_uri))
basename = config.pop_crds_uri(reference_uri)
return crds.locate_file(basename, observatory)
def rewrite(self, filename, uniqname):
"""Add a FITS checksum to `filename.`"""
with data_file.fits_open(filename,
mode="readonly",
checksum=self.args.verify_file,
do_not_scale_image_data=True) as hdus:
verify_mode = "fix+warn" if not self.args.fits_errors else "fix+exception"
if self.args.verify_file:
hdus.verify(verify_mode)
basefile = os.path.basename(filename)
baseuniq = os.path.basename(uniqname)
if self.args.add_keywords:
now = datetime.datetime.utcnow()
hdus[0].header["FILENAME"] = baseuniq
hdus[0].header["ROOTNAME"] = os.path.splitext(
baseuniq)[0].upper()
hdus[0].header[
"HISTORY"] = "{0} renamed to {1} on {2} {3} {4}".format(
basefile, baseuniq, MONTHS[now.month - 1], now.day,
now.year)
if self.args.output_path:
uniqname = os.path.join(self.args.output_path, baseuniq)
try:
log.info("Rewriting", self.format_file(filename), "-->",
self.format_file(uniqname))
hdus.writeto(uniqname,
output_verify=verify_mode,
checksum=self.args.add_checksum)
except Exception as exc:
if os.path.exists(uniqname):
os.remove(uniqname)
if "buffer is too small" in str(exc):
raise CrdsError("Failed to rename/rewrite", repr(basefile),
"as", repr(baseuniq),
":", "probable file truncation", ":",
str(exc)) from exc
else:
raise CrdsError("Failed to rename/rewrite",
repr(basefile), "as", repr(baseuniq), ":",
str(exc)) from exc
def __init__(self,
locked_instrument="none",
username=None,
password=None,
base_url=None):
if DISABLED:
raise CrdsError("Missing or broken depenencies:", DISABLED)
self.locked_instrument = locked_instrument
self.username = username
self.password = password
self.base_url = base_url
self.session = requests.session()
self.session.headers.update({'referer': self.base_url})
def fetch_source_segment(self, source):
"""Return the segment of dataset ids which surrounds id `source`."""
try:
index = self.sources.index(source) // self.segment_size
except ValueError as exc:
raise CrdsError("Unknown dataset id " + repr(source)) from exc
lower = index * self.segment_size
upper = (index + 1) * self.segment_size
segment_ids = self.sources[lower:upper]
log.verbose("Dumping", len(segment_ids), "datasets from indices", lower, "to",
lower + len(segment_ids), verbosity=20)
dumped_headers = api.get_dataset_headers_by_id(self.context, segment_ids)
log.verbose("Dumped", len(dumped_headers), "datasets", verbosity=20)
if self.save_pickles: # keep all headers, causes memory problems with multiple instruments on ~8G ram.
self.headers.update(dumped_headers)
else: # conserve memory by keeping only the last N headers
self.headers = dumped_headers