def get_deleted_references(old_pmap, new_pmap, cached=True): """Return the list of references from `new_pmap` which were not in `old_pmap`.""" old_pmap = rmap.asmapping(old_pmap, cached=cached) new_pmap = rmap.asmapping(new_pmap, cached=cached) return sorted( list( set(old_pmap.reference_names()) - set(new_pmap.reference_names())))
def mapping_check_references(mapping, derived_from): """Regardless of matching criteria, do a simple check listing added or deleted references as appropritate. """ mapping = rmap.asmapping(mapping, cached="readonly") derived_from = rmap.asmapping(derived_from, cached="readonly") old_refs = set(derived_from.reference_names()) new_refs = set(mapping.reference_names()) if old_refs - new_refs: log.warning("Deleted references for", repr(derived_from.filename), "and", repr(mapping.filename), "=", list(old_refs - new_refs)) if new_refs - old_refs: log.warning("Added references for", repr(derived_from.filename), "and", repr(mapping.filename), "=", list(new_refs - old_refs))
def mapping_check_references(mapping, derived_from): """Regardless of matching criteria, do a simple check listing added or deleted references as appropritate. """ mapping = rmap.asmapping(mapping, cached="readonly") derived_from = rmap.asmapping(derived_from, cached="readonly") old_refs = set(derived_from.reference_names()) new_refs = set(mapping.reference_names()) if old_refs - new_refs: log.warning("Deleted references for", repr(derived_from.filename), "and", repr(mapping.filename), "=", list(old_refs - new_refs)) if new_refs - old_refs: log.warning("Added references for", repr(derived_from.filename), "and", repr(mapping.filename), "=", list(new_refs - old_refs))
def mapping_check_diffs(mapping, derived_from): """Issue warnings for *deletions* in self relative to parent derived_from mapping. Issue warnings for *reversions*, defined as replacements which where the replacement is older than the original, as defined by the names. This is intended to check for missing modes and for inadvertent reversions to earlier versions of files. For speed and simplicity, file time order is currently determined by the names themselves, not file contents, file system, or database info. """ mapping = rmap.asmapping(mapping, cached="readonly") derived_from = rmap.asmapping(derived_from, cached="readonly") log.info("Checking diffs from", repr(derived_from.basename), "to", repr(mapping.basename)) diffs = derived_from.difference(mapping) mapping_check_diffs_core(diffs)
def mapping_check_diffs(mapping, derived_from): """Issue warnings for *deletions* in self relative to parent derived_from mapping. Issue warnings for *reversions*, defined as replacements which where the replacement is older than the original, as defined by the names. This is intended to check for missing modes and for inadvertent reversions to earlier versions of files. For speed and simplicity, file time order is currently determined by the names themselves, not file contents, file system, or database info. """ mapping = rmap.asmapping(mapping, cached="readonly") derived_from = rmap.asmapping(derived_from, cached="readonly") log.info("Checking diffs from", repr(derived_from.basename), "to", repr(mapping.basename)) diffs = derived_from.difference(mapping) mapping_check_diffs_core(diffs)
def get_refactoring_header(rmapping, refname, required_keywords): """Create a composite header which is derived from the file contents overidden by any values as they appear in the rmap. """ rmapping = rmap.asmapping(rmapping) # A fallback source of information is the reference file headers header = rmapping.get_refactor_header( rmap.locate_file(refname, rmapping.observatory), extra_keys=("META.OBSERVATION.DATE", "META.OBSERVATION.TIME", "DATE-OBS","TIME-OBS") + required_keywords) # The primary source of information is the original rmap and the matching values defined there headers2 = matches.find_match_paths_as_dict(rmapping.filename, refname) # Combine the two, using the rmap values to override anything duplicated in the reffile header assert len(headers2) == 1, "Can't refactor file with more than one match: " + srepr(refname) header.update(headers2[0]) return header
def __init__( self, context, graph_file=None, throughput_lookup_file=None, thermal_lookup_file=None, delivered_throughput_files=None, delivered_thermal_files=None ): self.context = context if graph_file is None: self.graph_file = utils.get_cache_path(context, utils.GRAPH_REFTYPE) else: self.graph_file = graph_file if throughput_lookup_file is None: self.throughput_lookup_file = utils.get_cache_path(context, utils.THROUGHPUT_LOOKUP_REFTYPE) else: self.throughput_lookup_file = throughput_lookup_file if thermal_lookup_file is None: self.thermal_lookup_file = utils.get_cache_path(context, utils.THERMAL_LOOKUP_REFTYPE) else: self.thermal_lookup_file = thermal_lookup_file if delivered_throughput_files is None: self.delivered_throughput_files = [] else: self.delivered_throughput_files = delivered_throughput_files if delivered_thermal_files is None: self.delivered_thermal_files = [] else: self.delivered_thermal_files = delivered_thermal_files self.imap = rmap.asmapping(context).get_imap(utils.SYNPHOT_INSTRUMENT) self.throughput_compname_to_path = self._build_compname_to_path( utils.THROUGHPUT_REFTYPE, self.delivered_throughput_files ) self.thermal_compname_to_path = self._build_compname_to_path( utils.THERMAL_REFTYPE, self.delivered_thermal_files )
def insert_references(self): """Insert files specified by --references into the appropriate rmaps identified by --source-context.""" self._setup_source_context() categorized = self.categorize_files(self.args.references) pmap = crds.get_pickled_mapping(self.source_context) # reviewed self.args.rmaps = [] for (instrument, filekind) in categorized: try: self.args.rmaps.append(pmap.get_imap(instrument).get_rmap(filekind).filename) except crexc.CrdsError: log.info("Existing rmap for", (instrument, filekind), "not found. Trying empty spec.") spec_file = os.path.join( os.path.dirname(self.obs_pkg.__file__), "specs", instrument + "_" + filekind + ".rmap") rmapping = rmap.asmapping(spec_file) log.info("Loaded spec file from", repr(spec_file)) self.args.rmaps.append(spec_file) self.rmap_apply(insert_rmap_references, categorized=categorized)
def get_refactoring_header(rmapping, refname, required_keywords): """Create a composite header which is derived from the file contents overidden by any values as they appear in the rmap. """ rmapping = rmap.asmapping(rmapping) # A fallback source of information is the reference file headers header = rmapping.get_refactor_header( rmap.locate_file(refname, rmapping.observatory), extra_keys=("META.OBSERVATION.DATE", "META.OBSERVATION.TIME", "DATE-OBS", "TIME-OBS") + required_keywords) # The primary source of information is the original rmap and the matching values defined there headers2 = matches.find_match_paths_as_dict(rmapping.filename, refname) # Combine the two, using the rmap values to override anything duplicated in the reffile header assert len( headers2 ) == 1, "Can't refactor file with more than one match: " + srepr(refname) header.update(headers2[0]) return header
def get_cache_path(context, reftype, error_on_missing=True): """ Fetch the full path to the single reference file of reftype. """ context = rmap.asmapping(context) imap = context.get_imap(SYNPHOT_INSTRUMENT) mapping = imap.get_rmap(reftype) filenames = mapping.reference_names() if len(filenames) == 0 and not error_on_missing: return None if len(filenames) != 1: raise RuntimeError( "Expected '{}' rmap to contain one reference file".format(reftype)) filename = filenames[0] return config.locate_file(filename, observatory="hst")
def insert_references(self): """Insert files specified by --references into the appropriate rmaps identified by --source-context.""" self._setup_source_context() categorized = self.categorize_files(self.args.references) pmap = crds.get_pickled_mapping(self.source_context) # reviewed self.args.rmaps = [] for (instrument, filekind) in categorized: try: self.args.rmaps.append( pmap.get_imap(instrument).get_rmap(filekind).filename) except crexc.CrdsError: log.info("Existing rmap for", (instrument, filekind), "not found. Trying empty spec.") spec_file = os.path.join( os.path.dirname(self.obs_pkg.__file__), "specs", instrument + "_" + filekind + ".rmap") rmapping = rmap.asmapping(spec_file) log.info("Loaded spec file from", repr(spec_file)) self.args.rmaps.append(spec_file) self.rmap_apply(insert_rmap_references, categorized=categorized)
def insert_mapping(context, mapping): """Replace the filename in file `context` with the same generic name as `mapping` with `mapping`. Re-write `context` in place. If mapping is of the form <instrument>_<type>_"n/a", then it specifies that <type> of <instrument> should be set to "N/A". """ # 'ACS' : 'hst_acs.imap', where = config.locate_mapping(context) # readonly caching is ok because this call is always made on a newly named # copy of the original rmap; the only thing mutated is the uncached new mapping. loaded = rmap.asmapping(context, cache="readonly") if mapping.endswith(("_n/a", "_N/A")): instrument, key, special = mapping.split("_") replacement = special.upper() else: key = loaded.get_item_key(mapping) replacement = mapping key, replaced = loaded.set_item(key, replacement) loaded.write(where) return key, replaced, replacement
def insert_mapping(context, mapping): """Replace the filename in file `context` with the same generic name as `mapping` with `mapping`. Re-write `context` in place. If mapping is of the form <instrument>_<type>_"n/a", then it specifies that <type> of <instrument> should be set to "N/A". """ # 'ACS' : 'hst_acs.imap', where = rmap.locate_mapping(context) # readonly caching is ok because this call is always made on a newly named # copy of the original rmap; the only thing mutated is the uncached new mapping. loaded = rmap.asmapping(context, cache="readonly") if mapping.endswith(("_n/a", "_N/A")): instrument, key, special = mapping.split("_") replacement = special.upper() else: key = loaded.get_item_key(mapping) replacement = mapping key, replaced = loaded.set_item(key, replacement) loaded.write(where) return key, replaced, replacement
def is_reprocessing_required(dataset, dataset_parameters, old_context, new_context, update): """This is the top level interface to crds.bestrefs running in "Affected Datasets" mode. It determines if reprocessing `dataset` with parameters `dataset_parameters` should be performed as a consequence of switching from `old_reference` to `new_reference`. old_reference is assigned to dataset by old_context, and new_reference is assigned to dataset by new_context. Parameters ---------- dataset: id of dataset being reprocessed, <assoc>:<member> or <unassoc>:<unassoc> format dataset_parameters: { parameter : value, ...} for all matching parameters and row selection parameters XXX row selection parameters not used in file selection may not be present until XXX explicitly added to the CRDS interface to the DADSOPS parameter database... XXX and possibly even to DADSOPS itself. Normally the row selections have only been XXX done with direct access to dataset .fits files. old_context: loaded pmap or name of old context, possibly for metadata or None new_context: loaded pmap or name of new context, possibly for metadata update: Update object Returns ------- True IFF reprocessing should be done as a consequence of the table change. """ log.verbose('is_reprocessing_required: Called with:\n', dataset, '\n', dataset_parameters, '\n', old_context, '\n', new_context, '\n', update, verbosity=100) # no old_context means "single context" mode, always reprocess. if old_context is None: return True # NOTE: non-tables are treated in DeepLook as filekinds which aren't (or maybe someday are) handled, # hence reprocessed for now. # Reprocess for non-file special values. Other code will decide what to do with the updates, # the point here is that table comparison isn't possible so filtering shouldn't be done. old_ref = update.old_reference.lower() new_ref = update.new_reference.lower() incomparable = ('n/a', 'undefined', 'not found') if old_ref.startswith(incomparable) or new_ref.startswith(incomparable): return True # mostly debug wrappers here, allows simple string parameters to work and resolves cache paths. old_context = rmap.asmapping(old_context, cached=True) new_context = rmap.asmapping(new_context, cached=True) old_reference = old_context.locate_file(old_ref) new_reference = new_context.locate_file(new_ref) # Log that deep examination is occuring. log.verbose('Deep Reference examination between {} and {} initiated.'.format(old_reference, new_reference), verbosity=25) with log.error_on_exception("Failed fetching comparison reference tables:", repr([old_ref, new_ref])): api.dump_files(new_context.name, [old_ref, new_ref]) # See if deep checking into the reference is possible. try: deep_look = DeepLook.from_filekind(update.instrument, update.filekind) dataset_id = dataset.split(':')[0] # **DEBUG** # ** Since we are not getting full headers, if this is a test # ** dataset, replace the headers. #log.verbose_warning('Forcing use of LBYX01010, regardless...', verbosity=25) #dataset_id = 'LBYX01010' #***DEBUG: force headers regardless of actua data if dataset_id in deep_look.stub_input: log.verbose_warning('Substituting header for dataset "{}"'.format(dataset)) dataset_parameters = deep_look.stub_input[dataset_id]['headers'] log.verbose_warning('headers = ', dataset_parameters, verbosity=25) log.verbose(deep_look.preamble, 'Dataset headers = {}'.format(dataset_parameters), verbosity=75) log.verbose(deep_look.preamble, 'Comparing references {} and {}.'.format(old_reference, new_reference), verbosity=75) deep_look.are_different(dataset_parameters, old_reference, new_reference) log.verbose(deep_look.preamble, 'Reprocessing is {}required.'.format('' if deep_look.is_different else 'not '), verbosity=25) log.verbose(deep_look.preamble, deep_look.message, verbosity=25) return deep_look.is_different except DeepLookError as error: # Could not determine difference, therefore presume so. log.verbose_warning('Deep examination error: {}'.format(error.message), verbosity=25) log.verbose_warning('Deep examination failed, presuming reprocessing.', verbosity=25) return True
def get_deleted_references(old_pmap, new_pmap, cached=True): """Return the list of references from `new_pmap` which were not in `old_pmap`.""" old_pmap = rmap.asmapping(old_pmap, cached=cached) new_pmap = rmap.asmapping(new_pmap, cached=cached) return sorted(list(set(old_pmap.reference_names()) - set(new_pmap.reference_names())))
def is_reprocessing_required(dataset, dataset_parameters, old_context, new_context, update): """This is the top level interface to crds.bestrefs running in "Affected Datasets" mode. It determines if reprocessing `dataset` with parameters `dataset_parameters` should be performed as a consequence of switching from `old_reference` to `new_reference`. old_reference is assigned to dataset by old_context, and new_reference is assigned to dataset by new_context. Parameters ---------- dataset: id of dataset being reprocessed, <assoc>:<member> or <unassoc>:<unassoc> format dataset_parameters: { parameter : value, ...} for all matching parameters and row selection parameters XXX row selection parameters not used in file selection may not be present until XXX explicitly added to the CRDS interface to the DADSOPS parameter database... XXX and possibly even to DADSOPS itself. Normally the row selections have only been XXX done with direct access to dataset .fits files. old_context: loaded pmap or name of old context, possibly for metadata or None new_context: loaded pmap or name of new context, possibly for metadata update: Update object Returns ------- True IFF reprocessing should be done as a consequence of the table change. """ log.verbose('is_reprocessing_required: Called with:\n', dataset, '\n', dataset_parameters, '\n', old_context, '\n', new_context, '\n', update, verbosity=100) # no old_context means "single context" mode, always reprocess. if old_context is None: return True # NOTE: non-tables are treated in DeepLook as filekinds which aren't (or maybe someday are) handled, # hence reprocessed for now. # Reprocess for non-file special values. Other code will decide what to do with the updates, # the point here is that table comparison isn't possible so filtering shouldn't be done. old_ref = update.old_reference.lower() new_ref = update.new_reference.lower() incomparable = ('n/a', 'undefined', 'not found') if old_ref.startswith(incomparable) or new_ref.startswith(incomparable): return True # mostly debug wrappers here, allows simple string parameters to work and resolves cache paths. old_context = rmap.asmapping(old_context, cached=True) new_context = rmap.asmapping(new_context, cached=True) old_reference = old_context.locate_file(old_ref) new_reference = new_context.locate_file(new_ref) # Log that deep examination is occuring. log.verbose('Deep Reference examination between {} and {} initiated.'.format(old_reference, new_reference), verbosity=25) with log.error_on_exception("Failed fetching comparison reference tables:", repr([old_ref, new_ref])): api.dump_files(new_context.name, [old_ref, new_ref]) # See if deep checking into the reference is possible. try: deep_look = DeepLook.from_filekind(update.instrument, update.filekind) dataset_id = dataset.split(':')[0] # **DEBUG** # ** Since we are not getting full headers, if this is a test # ** dataset, replace the headers. #log.verbose_warning('Forcing use of LBYX01010, regardless...', verbosity=25) #dataset_id = 'LBYX01010' #***DEBUG: force headers regardless of actua data if dataset_id in deep_look.stub_input: log.verbose_warning('Substituting header for dataset "{}"'.format(dataset)) dataset_parameters = deep_look.stub_input[dataset_id]['headers'] log.verbose_warning('headers = ', dataset_parameters, verbosity=25) log.verbose(deep_look.preamble, 'Dataset headers = {}'.format(dataset_parameters), verbosity=75) log.verbose(deep_look.preamble, 'Comparing references {} and {}.'.format(old_reference, new_reference), verbosity=75) deep_look.are_different(dataset_parameters, old_reference, new_reference) log.verbose(deep_look.preamble, 'Reprocessing is {}required.'.format('' if deep_look.is_different else 'not '), verbosity=25) log.verbose(deep_look.preamble, deep_look.message, verbosity=25) return deep_look.is_different except DeepLookError as error: # Could not determine difference, therefore presume so. log.verbose_warning('Deep examination error: {}'.format(error.message), verbosity=25) log.verbose_warning('Deep examination failed, presuming reprocessing.', verbosity=25) return True