def check_column(self, filename, header):
"""Extract a column of new_values from `filename` and check them all against
the legal values for this Validator. This checks a single column, not a row/mode.
"""
column_seen = False
for tab in tables.tables(filename):
if self.name in tab.colnames:
column_seen = True
# new_values must not be None, check all, waiting to fail later
for i, value in enumerate(tab.columns[self.name]): # compare to TPN values
self.check_value(filename + "[" + str(i) +"]", value)
if not column_seen:
self.handle_missing(header)
return True
def check_column(self, filename, header):
"""Extract a column of new_values from `filename` and check them all against
the legal values for this Validator. This checks a single column, not a row/mode.
"""
column_seen = False
for tab in tables.tables(filename):
if self.name in tab.colnames:
column_seen = True
# new_values must not be None, check all, waiting to fail later
for i, value in enumerate(
tab.columns[self.name]): # compare to TPN values
self.check_value(filename + "[" + str(i) + "]", value)
if not column_seen:
self.handle_missing(header)
return True
def get_comptab_info(self, synname):
"""Dump the FILENAME column of the component table implied by `synname`
(e.g. "tmc" --> something_tmc.fits) and use pysynphot to interpret the
embedded iraf$-style path env var into a normal filepath. This is used
to locate files within appropriate sub-directories of
<synphot_dir>/comp.
Return the mapping from a component file basename as defined in the CRDS
rmap to the absolute path in a CDBS-style synphot file repo being created.
Returns { component_basename : abs_pysyn_path, ...}
"""
for msg in SYNPHOT_IGNORE:
warnings.filterwarnings("ignore", msg)
from pysynphot import locations
filekind = synname + "tab"
rmap = self.imap.get_rmap(filekind)
references = rmap.reference_names()
assert len(references) == 1, \
"More than one '%s' reference name mentioned in '%s'." % \
(synname, rmap.name)
tab_name = references[0]
# rmap object locate() not module function.
tab_path = rmap.locate_file(tab_name)
# CRDS abstract table object nominally from HDU 1
table = tables.tables(tab_path)[0]
fileinfo = {}
for syn_name in table.columns["FILENAME"]:
iraf_path, basename = syn_name.split("$")
name = basename.split("[")[0] # remove parameterization
dollar_syn_name = syn_name.split("[")[0]
# Use pysynphot to interpret iraf_path
cdbs_filepath = os.path.abspath(
locations.irafconvert(dollar_syn_name))
fileinfo[name] = cdbs_filepath
return fileinfo
def get_comptab_info(self, synname):
"""Dump the FILENAME column of the component table implied by `synname`
(e.g. "tmc" --> something_tmc.fits) and use pysynphot to interpret the
embedded iraf$-style path env var into a normal filepath. This is used
to locate files within appropriate sub-directories of
<synphot_dir>/comp.
Return the mapping from a component file basename as defined in the CRDS
rmap to the absolute path in a CDBS-style synphot file repo being created.
Returns { component_basename : abs_pysyn_path, ...}
"""
for msg in SYNPHOT_IGNORE:
warnings.filterwarnings("ignore",msg)
from pysynphot import locations
filekind = synname + "tab"
rmap = self.imap.get_rmap(filekind)
references = rmap.reference_names()
assert len(references) == 1, \
"More than one '%s' reference name mentioned in '%s'." % \
(synname, rmap.name)
tab_name = references[0]
# rmap object locate() not module function.
tab_path = rmap.locate_file(tab_name)
# CRDS abstract table object nominally from HDU 1
table = tables.tables(tab_path)[0]
fileinfo = {}
for syn_name in table.columns["FILENAME"]:
iraf_path, basename = syn_name.split("$")
name = basename.split("[")[0] # remove parameterization
dollar_syn_name = syn_name.split("[")[0]
# Use pysynphot to interpret iraf_path
cdbs_filepath = os.path.abspath(
locations.irafconvert(dollar_syn_name))
fileinfo[name] = cdbs_filepath
return fileinfo
def are_different(self, headers, old_reference, new_reference):
"""Do the deep examination of the reference files with-respect-to the given dataset headers
Affects
=======
self.is_different: Sets True or False whether the references are different.
self.message: Reason for current state of is_different
"""
# Convert header keys to lowercase for consistency
headers_low = dict((k.lower(), v) for k, v in headers.items())
# Start off that the references are different.
self.is_different = True
self.message = 'Comparision started but not completed.'
# Get values for the mode fields
constraint_values = {}
for field in self.mode_fields:
constraint_values[field] = str_to_number(headers_low[field]) if field in headers_low else None
if None in constraint_values.values():
self.message = 'Not all mode fields are defined in the dataset.'
return
# Modify the constraint values if any "meta" values are
# present.
for key in constraint_values:
if key in self.metavalues:
if constraint_values[key] in self.metavalues[key]:
constraint_values[key] = self.metavalues[key][constraint_values[key]]
# Read the references
data_old = tables.tables(old_reference)[0] # XXXX currently limited to FITS extension 1
data_new = tables.tables(new_reference)[0]
# Columns must be the same between tables.
if sorted(data_old.colnames) != sorted(data_new.colnames):
self.message = 'Columns are different between references.'
return
# Now that values are in hand, produce the full constraint
# dictionary
constraints = {}
for field in self.mode_fields:
constraints[field] = (constraint_values[field],) + self.mode_fields[field]
log.verbose(self.preamble, 'Constraints are:\n', constraints, verbosity=75)
# Reduce the tables to just those rows that match the mode
# specifications.
mode_rows_old = [repr(row) for row in mode_select(data_old, constraints)]
mode_rows_new = [repr(row) for row in mode_select(data_new, constraints)]
# Sort the rows
mode_rows_old.sort()
mode_rows_new.sort()
log.verbose(self.preamble, 'Old reference matching rows:\n', mode_rows_old, verbosity=75)
log.verbose(self.preamble, 'New reference matching rows:\n', mode_rows_new, verbosity=75)
# Check on equality.
# That's all folks.
self.is_different = not mode_equality(mode_rows_old, mode_rows_new)
if self.is_different:
self.message = 'Selection rules have executed and the selected rows are different.'
else:
self.message = 'Selection rules have executed and the selected rows are the same.'
def are_different(self, headers, old_reference, new_reference):
"""Do the deep examination of the reference files with-respect-to the given dataset headers
Affects
=======
self.is_different: Sets True or False whether the references are different.
self.message: Reason for current state of is_different
"""
# Convert header keys to lowercase for consistency
headers_low = dict((k.lower(), v) for k, v in headers.items())
# Start off that the references are different.
self.is_different = True
self.message = 'Comparision started but not completed.'
# Get values for the mode fields
constraint_values = {}
for field in self.mode_fields:
constraint_values[field] = str_to_number(headers_low[field]) if field in headers_low else None
if None in constraint_values.values():
self.message = 'Not all mode fields are defined in the dataset.'
return
# Modify the constraint values if any "meta" values are
# present.
for key in constraint_values:
if key in self.metavalues:
if constraint_values[key] in self.metavalues[key]:
constraint_values[key] = self.metavalues[key][constraint_values[key]]
# Read the references
data_old = tables.tables(old_reference)[0] # XXXX currently limited to FITS extension 1
data_new = tables.tables(new_reference)[0]
# Columns must be the same between tables.
if sorted(data_old.colnames) != sorted(data_new.colnames):
self.message = 'Columns are different between references.'
return
# Now that values are in hand, produce the full constraint
# dictionary
constraints = {}
for field in self.mode_fields:
constraints[field] = (constraint_values[field],) + self.mode_fields[field]
log.verbose(self.preamble, 'Constraints are:\n', constraints, verbosity=75)
# Reduce the tables to just those rows that match the mode
# specifications.
mode_rows_old = [repr(row) for row in mode_select(data_old, constraints)]
mode_rows_new = [repr(row) for row in mode_select(data_new, constraints)]
# Sort the rows
mode_rows_old.sort()
mode_rows_new.sort()
log.verbose(self.preamble, 'Old reference matching rows:\n', mode_rows_old, verbosity=75)
log.verbose(self.preamble, 'New reference matching rows:\n', mode_rows_new, verbosity=75)
# Check on equality.
# That's all folks.
self.is_different = not mode_equality(mode_rows_old, mode_rows_new)
if self.is_different:
self.message = 'Selection rules have executed and the selected rows are different.'
else:
self.message = 'Selection rules have executed and the selected rows are the same.'