def certify_local_files(self): """Run CRDS certify on submitted files and fail/exit if errors are found. """ if log.errors(): raise CrdsError("Errors encountered before CRDS certify run.") certify.certify_files( self.files, context=self.pmap_name, dump_provenance=True, compare_old_reference=True, observatory=self.observatory, run_fitsverify=True, check_rmap=False, check_sha1sums=True) if log.errors(): raise CrdsError("Certify errors found. Aborting submission.")
def header(self, source): """Return the full header corresponding to `source`. If header is a string, raise an exception.""" header = self._header(source) if isinstance(header, str): raise CrdsError("Failed to fetch header for " + repr(source) + ": " + repr(header)) else: return dict(header)
def reference_uri_to_cache_path(reference_uri, observatory): """Convert an abstract CRDS reference file URI into an absolute path for the file as located in the CRDS cache. Parameters ---------- reference_uri : string identifying an abstract CRDS reference file, .e.g. 'crds://jwst_miri_flat_0177.fits' observatory : string naming the telescope/project for CRDS, e.g. 'jwst', 'roman' Returns ------- reference_path : absolute file path to reference_uri in the CRDS cache, .e.g. '/grp/crds/cache/references/jwst/jwst_miri_flat_0177.fits' Standard CRDS cache paths are typically defined relative to the CRDS_PATH environment variable. See https://jwst-crds.stsci.edu guide and top level page for more info on configuring CRDS. The default CRDS_PATH value is /grp/crds/cache, currently on the Central Store. """ if not reference_uri.startswith("crds://"): raise CrdsError( "CRDS reference URI's should start with 'crds://' but got", repr(reference_uri)) basename = config.pop_crds_uri(reference_uri) return crds.locate_file(basename, observatory)
def rewrite(self, filename, uniqname): """Add a FITS checksum to `filename.`""" with data_file.fits_open(filename, mode="readonly", checksum=self.args.verify_file, do_not_scale_image_data=True) as hdus: verify_mode = "fix+warn" if not self.args.fits_errors else "fix+exception" if self.args.verify_file: hdus.verify(verify_mode) basefile = os.path.basename(filename) baseuniq = os.path.basename(uniqname) if self.args.add_keywords: now = datetime.datetime.utcnow() hdus[0].header["FILENAME"] = baseuniq hdus[0].header["ROOTNAME"] = os.path.splitext( baseuniq)[0].upper() hdus[0].header[ "HISTORY"] = "{0} renamed to {1} on {2} {3} {4}".format( basefile, baseuniq, MONTHS[now.month - 1], now.day, now.year) if self.args.output_path: uniqname = os.path.join(self.args.output_path, baseuniq) try: log.info("Rewriting", self.format_file(filename), "-->", self.format_file(uniqname)) hdus.writeto(uniqname, output_verify=verify_mode, checksum=self.args.add_checksum) except Exception as exc: if os.path.exists(uniqname): os.remove(uniqname) if "buffer is too small" in str(exc): raise CrdsError("Failed to rename/rewrite", repr(basefile), "as", repr(baseuniq), ":", "probable file truncation", ":", str(exc)) from exc else: raise CrdsError("Failed to rename/rewrite", repr(basefile), "as", repr(baseuniq), ":", str(exc)) from exc
def __init__(self, locked_instrument="none", username=None, password=None, base_url=None): if DISABLED: raise CrdsError("Missing or broken depenencies:", DISABLED) self.locked_instrument = locked_instrument self.username = username self.password = password self.base_url = base_url self.session = requests.session() self.session.headers.update({'referer': self.base_url})
def fetch_source_segment(self, source): """Return the segment of dataset ids which surrounds id `source`.""" try: index = self.sources.index(source) // self.segment_size except ValueError as exc: raise CrdsError("Unknown dataset id " + repr(source)) from exc lower = index * self.segment_size upper = (index + 1) * self.segment_size segment_ids = self.sources[lower:upper] log.verbose("Dumping", len(segment_ids), "datasets from indices", lower, "to", lower + len(segment_ids), verbosity=20) dumped_headers = api.get_dataset_headers_by_id(self.context, segment_ids) log.verbose("Dumped", len(dumped_headers), "datasets", verbosity=20) if self.save_pickles: # keep all headers, causes memory problems with multiple instruments on ~8G ram. self.headers.update(dumped_headers) else: # conserve memory by keeping only the last N headers self.headers = dumped_headers