def update_file_bestrefs(context, dataset, updates):
"""Update the header of `dataset` with best reference recommendations
`bestrefs` determined by context named `pmap`.
"""
if not updates:
return
version_info = heavy_client.version_info()
instrument = updates[0].instrument
locator = utils.instrument_to_locator(instrument)
prefix = locator.get_env_prefix(instrument)
with data_file.fits_open(dataset, mode="update", do_not_scale_image_data=True, checksum=False) as hdulist:
def set_key(keyword, value):
"""Set a single keyword value with logging, bound to outer-scope hdulist."""
log.verbose("Setting", repr(dataset), keyword, "=", value)
hdulist[0].header[keyword] = value
set_key("CRDS_CTX", context)
set_key("CRDS_VER", version_info)
for update in sorted(updates):
new_ref = update.new_reference.upper()
if new_ref != "N/A":
new_ref = (prefix + new_ref).lower()
keyword = locator.filekind_to_keyword(update.filekind)
set_key(keyword, new_ref)
# This is a workaround for a bug in astropy.io.fits handling of
# FITS updates that are header-only and extend the header.
# This statement appears to do nothing but *is not* pointless.
for hdu in hdulist:
hdu.data
def rewrite(self, filename, uniqname):
"""Add a FITS checksum to `filename.`"""
with data_file.fits_open(filename, mode="readonly", checksum=self.args.verify_file, do_not_scale_image_data=True) as hdus:
verify_mode = "fix+warn" if not self.args.fits_errors else "fix+exception"
if self.args.verify_file:
hdus.verify(verify_mode)
basefile = os.path.basename(filename)
baseuniq = os.path.basename(uniqname)
if self.args.add_keywords:
now = datetime.datetime.utcnow()
hdus[0].header["FILENAME"] = baseuniq
hdus[0].header["ROOTNAME"] = os.path.splitext(baseuniq)[0].upper()
hdus[0].header["HISTORY"] = "{0} renamed to {1} on {2} {3} {4}".format(
basefile, baseuniq, MONTHS[now.month - 1], now.day, now.year)
if self.args.output_path:
uniqname = os.path.join(self.args.output_path, baseuniq)
try:
log.info("Rewriting", self.format_file(filename), "-->", self.format_file(uniqname))
hdus.writeto(uniqname, output_verify=verify_mode, checksum=self.args.add_checksum)
except Exception as exc:
if os.path.exists(uniqname):
os.remove(uniqname)
if "buffer is too small" in str(exc):
raise CrdsError(
"Failed to rename/rewrite", repr(basefile),
"as", repr(baseuniq), ":",
"probable file truncation", ":", str(exc)) from exc
else:
raise CrdsError("Failed to rename/rewrite", repr(basefile),
"as", repr(baseuniq), ":",
str(exc)) from exc
def rewrite(self, filename, uniqname):
"""Add a FITS checksum to `filename.`"""
with data_file.fits_open(filename, mode="readonly", checksum=self.args.verify_file, do_not_scale_image_data=True) as hdus:
verify_mode = "fix+warn" if not self.args.fits_errors else "fix+exception"
if self.args.verify_file:
hdus.verify(verify_mode)
basefile = os.path.basename(filename)
baseuniq = os.path.basename(uniqname)
if self.args.add_keywords:
now = datetime.datetime.utcnow()
hdus[0].header["FILENAME"] = baseuniq
hdus[0].header["ROOTNAME"] = os.path.splitext(baseuniq)[0].upper()
hdus[0].header["HISTORY"] = "{0} renamed to {1} on {2} {3} {4}".format(
basefile, baseuniq, MONTHS[now.month - 1], now.day, now.year)
if self.args.output_path:
uniqname = os.path.join(self.args.outpath, baseuniq)
try:
log.info("Rewriting", self.format_file(filename), "-->", self.format_file(uniqname))
hdus.writeto(uniqname, output_verify=verify_mode, checksum=self.args.add_checksum)
except Exception as exc:
if os.path.exists(uniqname):
os.remove(uniqname)
if "buffer is too small" in str(exc):
raise CrdsError("Failed to rename/rewrite", repr(basefile), "as", repr(baseuniq), ":",
"probable file truncation", ":", str(exc))
else:
raise CrdsError("Failed to rename/rewrite", repr(basefile), "as", repr(baseuniq), ":", str(exc))
def ntables(filename):
"""Return the number of segments / hdus in `filename`."""
if filename.endswith(".fits"):
with data_file.fits_open(filename) as hdus:
return len(hdus) - 1
else:
return 1
def ntables(filename):
"""Return the number of segments / hdus in `filename`."""
if filename.endswith(".fits"):
with data_file.fits_open(filename) as hdus:
return len(hdus) - 1
else:
return 1
def check_header(self, filename, header):
with data_file.fits_open(filename) as hdul:
result = self._check_throughput_first_and_last(hdul)
result = self._check_parametrization(hdul) and result
result = _check_component_filename(
self.context, utils.THROUGHPUT_REFTYPE,
utils.THROUGHPUT_FILENAME_SUFFIX, filename, header) and result
return result
def tables(filename):
"""Return [ SimpleTable(filename, segment), ... ] for each table segment in filename.
This function is self-cached. Clear the cache using clear_cache().
"""
if filename.endswith(".fits"):
with data_file.fits_open(filename) as hdus:
return [SimpleTable(filename, i + 1) for i in range(len(hdus) - 1)]
else:
return [SimpleTable(filename, segment=1)]
def tables(filename):
"""Return [ SimpleTable(filename, segment), ... ] for each table segment in filename.
This function is self-cached. Clear the cache using clear_cache().
"""
if filename.endswith(".fits"):
with data_file.fits_open(filename) as hdus:
return [ SimpleTable(filename, i+1) for i in range(len(hdus)-1) ]
else:
return [ SimpleTable(filename, segment=1) ]
def ntables(filename):
"""Return the number of segments / hdus in `filename`."""
if filename.endswith(".fits"):
tables = 0
with data_file.fits_open(filename) as hdus:
for i,hdu in enumerate(hdus):
if "TABLEHDU" in hdu.__class__.__name__.upper():
tables += 1
return tables
else:
return 1
def ntables(filename):
"""Return the number of segments / hdus in `filename`."""
if filename.endswith(".fits"):
tables = 0
with data_file.fits_open(filename) as hdus:
for i, hdu in enumerate(hdus):
if "TABLEHDU" in hdu.__class__.__name__.upper():
tables += 1
return tables
else:
return 1
def _cat_array_properties(self, path):
"""Print out the CRDS interpretation of every array in `path`, currently FITS only."""
with data_file.fits_open(path) as hdulist:
for i, hdu in enumerate(hdulist):
with log.warn_on_exception("Can't load array properties for HDU[" + str(i) +"]"):
if i > 0:
extname = hdu.header.get("EXTNAME",str(i))
self._cat_banner("CRDS Array Info [" + repr(extname) + "]:",
delim="-", bottom_delim=None)
props = data_file.get_array_properties(path, extname)
props = { prop:value for (prop,value) in props.items() if value is not None }
self._print_lines(path, _pp_lines(props))
def checksum_exists(filename):
"""Return True IFF `path` names a file which already has FITS checksums. As a first guess,
existing checksums should be maintained across file content updates required by the renaming.
"""
with data_file.fits_open(filename, mode="readonly", do_not_scale_image_data=True) as hdus:
for hdu in hdus:
if "CHECKSUM" in hdu.header or "DATASUM" in hdu.header:
add_checksum = True
break
else:
add_checksum = False
return add_checksum
def has_checksum(filename):
"""Return True IFF `path` names a file which already has FITS checksums. As a first guess,
existing checksums should be maintained across file content updates required by the renaming.
"""
with data_file.fits_open(filename, mode="readonly", do_not_scale_image_data=True) as hdus:
for hdu in hdus:
if "CHECKSUM" in hdu.header or "DATASUM" in hdu.header:
add_checksum = True
break
else:
add_checksum = False
return add_checksum
def __init__(self, filename, segment=1):
self.filename = filename
self.segment = segment
self.basename = os.path.basename(filename)
self._columns = None # dynamic, independent of astropy
if filename.endswith(".fits"):
with data_file.fits_open(filename) as hdus:
tab = hdus[segment].data
self.colnames = tuple(name.upper() for name in tab.columns.names)
self.rows = tuple(tuple(row) for row in tab) # readonly
else:
tab = table.Table.read(filename)
self.colnames = tuple(name.upper() for name in tab.columns)
self.rows = tuple(tuple(row) for row in tab) # readonly
log.verbose("Creating", repr(self), verbosity=60)
def __init__(self, filename, segment=1):
self.filename = filename
self.segment = segment
self.basename = os.path.basename(filename)
self._columns = None # dynamic, independent of astropy
if filename.endswith(".fits"):
with data_file.fits_open(filename) as hdus:
tab = hdus[segment].data
self.colnames = tuple(name.upper()
for name in tab.columns.names)
self.rows = tuple(tuple(row) for row in tab) # readonly
else:
tab = table.Table.read(filename)
self.colnames = tuple(name.upper() for name in tab.columns)
self.rows = tuple(tuple(row) for row in tab) # readonly
log.verbose("Creating", repr(self), verbosity=60)
def tables(filename):
"""Return [ SimpleTable(filename, segment), ... ] for each table segment in filename.
This function is self-cached. Clear the cache using clear_cache().
"""
if filename.endswith(".fits"):
tables = []
with data_file.fits_open(filename) as hdus:
for i,hdu in enumerate(hdus):
classname = hdu.__class__.__name__.upper()
if "TABLEHDU" in classname:
tables.append(SimpleTable(filename, i))
if classname == "TABLEHDU":
log.warning("ASCII Table detected in HDU#", str(i) +
". Particularly for HST, verify that it should not be a BIN Table HDU.")
return tables
else:
return [ SimpleTable(filename, segment=1) ]
def _cat_array_properties(self, path):
"""Print out the CRDS interpretation of every array in `path`, currently FITS only."""
with data_file.fits_open(path) as hdulist:
for i, hdu in enumerate(hdulist):
with log.warn_on_exception(
"Can't load array properties for HDU[" + str(i) + "]"):
if i > 0:
extname = hdu.header.get("EXTNAME", str(i))
self._cat_banner("CRDS Array Info [" + repr(extname) +
"]:",
delim="-",
bottom_delim=None)
props = data_file.get_array_properties(path, extname)
props = {
prop: value
for (prop, value) in props.items()
if value is not None
}
self._print_lines(path, _pp_lines(props))
def generate(self, reftype, delivered_files):
"""
Generate and return an HDUList containing an updated lookup table
for the specified reftype.
"""
# Historically ReDCaT has always set this to the current
# local time and not UTC.
timestamp = datetime.datetime.today().strftime(_TIMESTAMP_FORMAT)
original_ref_path = utils.get_cache_path(self._context,
reftype,
error_on_missing=False)
if original_ref_path is None:
return self._generate_hdul(reftype, delivered_files, timestamp)
else:
with data_file.fits_open(original_ref_path) as original_ref:
return self._generate_hdul(reftype,
delivered_files,
timestamp,
original_ref=original_ref)
def tables(filename):
"""Return [ SimpleTable(filename, segment), ... ] for each table segment in filename.
This function is self-cached. Clear the cache using clear_cache().
"""
if filename.endswith(".fits"):
tables = []
with data_file.fits_open(filename) as hdus:
for i, hdu in enumerate(hdus):
classname = hdu.__class__.__name__.upper()
if "TABLEHDU" in classname:
tables.append(SimpleTable(filename, i))
if classname == "TABLEHDU":
log.warning(
"ASCII Table detected in HDU#",
str(i) +
". Particularly for HST, verify that it should not be a BIN Table HDU."
)
return tables
else:
return [SimpleTable(filename, segment=1)]
def check_header(self, filename, header):
"""Evalutate the header expression associated with this validator (as its sole value)
with respect to the given `header`. Read `header` from `filename` if `header` is None.
"""
array_name = self.complex_name
max_ver = 0
with data_file.fits_open(filename) as hdus:
first = dict()
for hdu in hdus:
if hdu.name != self.name:
continue
self.verbose(filename, "ver=" + str(hdu.ver),
"Array has shape=" + str(hdu.data.shape),
"and dtype=" + repr(str(hdu.data.dtype)) + ".")
if hdu.name not in first:
first[hdu.name] = (hdu.data.shape, hdu.data.dtype)
else:
expected = first[hdu.name][0]
got = hdu.data.shape
assert expected == got, \
"Shape mismtatch for " + repr((hdu.name, hdu.ver)) + \
"relative to" + repr((self.name,1)) + ". Expected " + \
str(expected) + " but got " + str(got) + "."
expected = first[hdu.name][1]
got = hdu.data.dtype
assert expected == got, \
"Data type mismtatch for " + \
repr((hdu.name,hdu.ver)) + \
" relative to " + repr((self.name,1)) + \
". Expected " + str(expected) + \
" but got " + str(got) + "."
max_ver = hdu.ver
if self.max_ver is not None:
assert self.max_ver == max_ver, \
"Bad maximum HDU ver for " + repr(self.name) + \
". Expected " + str(self.max_ver) + \
", got " + str(max_ver) + "."
def check_header(self, filename, header):
"""Evalutate the header expression associated with this validator (as its sole value)
with respect to the given `header`. Read `header` from `filename` if `header` is None.
"""
array_name = self.complex_name
max_ver = 0
with data_file.fits_open(filename) as hdus:
first = dict()
for hdu in hdus:
if hdu.name != self.name:
continue
self.verbose(filename, "ver=" + str(hdu.ver),
"Array has shape=" + str(hdu.data.shape),
"and dtype=" + repr(str(hdu.data.dtype)) + ".")
if hdu.name not in first:
first[hdu.name] = (hdu.data.shape, hdu.data.dtype)
else:
expected = first[hdu.name][0]
got = hdu.data.shape
assert expected == got, \
"Shape mismtatch for " + repr((hdu.name, hdu.ver)) + \
"relative to" + repr((self.name,1)) + ". Expected " + \
str(expected) + " but got " + str(got) + "."
expected = first[hdu.name][1]
got = hdu.data.dtype
assert expected == got, \
"Data type mismtatch for " + \
repr((hdu.name,hdu.ver)) + \
" relative to " + repr((self.name,1)) + \
". Expected " + str(expected) + \
" but got " + str(got) + "."
max_ver = hdu.ver
if self.max_ver is not None:
assert self.max_ver == max_ver, \
"Bad maximum HDU ver for " + repr(self.name) + \
". Expected " + str(self.max_ver) + \
", got " + str(max_ver) + "."
def _generate_hdul(self,
reftype,
delivered_files,
timestamp,
original_ref=None):
if original_ref is not None:
original_table = original_ref[-1].data
new_time = original_table["TIME"]
new_compname = original_table["COMPNAME"]
new_filename = original_table["FILENAME"]
new_comment = original_table["COMMENT"]
else:
new_time = np.chararray((0, ), unicode=True)
new_compname = np.chararray((0, ), unicode=True)
new_filename = np.chararray((0, ), unicode=True)
new_comment = np.chararray((0, ), unicode=True)
updated_instruments = set()
created_instruments = set()
for file in delivered_files:
with data_file.fits_open(file) as hdul:
component = hdul[0].header["COMPNAME"]
lookup_filename = utils.get_lookup_filename(
component, os.path.basename(file))
description = hdul[0].header.get("DESCRIP", "")
if component in new_compname:
idx = np.argwhere(new_compname == component)[0][0]
new_time[idx] = timestamp
new_filename[idx] = lookup_filename
new_comment[idx] = description
updated_instruments.add(utils.get_instrument(component))
else:
new_time = np.append(new_time, timestamp)
new_compname = np.append(new_compname, component)
new_filename = np.append(new_filename, lookup_filename)
new_comment = np.append(new_comment, description)
created_instruments.add(utils.get_instrument(component))
ind = np.argsort(new_compname)
columns = [
fits.Column(name="TIME",
format="A26",
array=new_time[ind],
disp="A26"),
fits.Column(name="COMPNAME",
format="A18",
array=new_compname[ind],
disp="A18"),
fits.Column(name="FILENAME",
format="A56",
array=new_filename[ind],
disp="A56"),
fits.Column(name="COMMENT",
format="A68",
array=new_comment[ind],
disp="A68")
]
new_table_hdu = fits.BinTableHDU.from_columns(columns)
new_primary_hdu = fits.PrimaryHDU()
header = new_primary_hdu.header
header["USEAFTER"] = timestamp
header["INSTRUME"] = "HST"
header["COMMENT"] = "Reference file automatically generated by CRDS"
header["DBTABLE"] = _DBTABLE_BY_REFTYPE[reftype]
header["DESCRIP"] = _DESCRIP_BY_REFTYPE[reftype]
header["PEDIGREE"] = self._make_pedigree(new_table_hdu)
if original_ref:
for item in original_ref[0].header["HISTORY"]:
header.add_history(item)
header.add_history(" ")
header.add_history("Updated on {}".format(timestamp))
else:
header.add_history("Created on {}".format(timestamp))
header.add_history(" ")
self._add_instrument_history(reftype, header, updated_instruments,
created_instruments)
return fits.HDUList([new_primary_hdu, new_table_hdu])
def check_header(self, filename, header):
with data_file.fits_open(filename) as hdul:
return self._check_filenames(hdul)
def check_header(self, filename, header):
with data_file.fits_open(filename) as hdul:
result = self._check_connectivity(hdul)
result = self._check_direction(hdul) and result
return result
