def reference_keys_to_dataset_keys(rmapping, header):
"""Given a header dictionary for a reference file, map the header back to
keys relevant to datasets.
"""
result = dict(header)
# XXXXX TODO Add/consolidate logic to handle P_ pattern keywords
# If USEAFTER is defined, or we're configured to fake it...
# don't invent one if its missing and we're not faking it.
if "USEAFTER" in header or config.ALLOW_BAD_USEAFTER:
# Identify reference involved as best as possible
filename = header.get("FILENAME", None) or rmapping.filename
reformatted = timestamp.reformat_useafter(filename, header).split()
result["DATE-OBS"] = reformatted[0]
result["DATE_OBS"] = reformatted[0]
result["TIME-OBS"] = reformatted[1]
result["TIME_OBS"] = reformatted[1]
return result
def reference_keys_to_dataset_keys(rmapping, header):
"""Given a header dictionary for a reference file, map the header back to
keys relevant to datasets.
"""
result = dict(header)
# XXXXX TODO Add/consolidate logic to handle P_ pattern keywords
# If USEAFTER is defined, or we're configured to fake it...
# don't invent one if its missing and we're not faking it.
if "USEAFTER" in header or config.ALLOW_BAD_USEAFTER:
# Identify reference involved as best as possible
filename = header.get("FILENAME", None) or rmapping.filename
reformatted = timestamp.reformat_useafter(filename, header).split()
result["DATE-OBS"] = reformatted[0]
result["DATE_OBS"] = reformatted[0]
result["TIME-OBS"] = reformatted[1]
result["TIME_OBS"] = reformatted[1]
return result
def reference_keys_to_dataset_keys(rmapping, header):
"""Given a header dictionary for a reference file, map the header back to keys
relevant to datasets. So for ACS biasfile the reference says BINAXIS1 but
the dataset says NUMCOLS. This would convert { "BINAXIS1": 1024 } to {
"NUMCOLS" : 1024 }.
In general, rmap parkeys are matched against datset values and are defined
as dataset header keywords. For refactoring though, what's initially
available are reference file keywords... which need to be mapped into the
terms rmaps know: dataset keywords.
"""
header = dict(header)
# Basic common pattern translations
translations = {
"META.EXPOSURE.P_EXPTYPE" : "META.EXPOSURE.TYPE",
"P_EXP_TY" : "META.EXPOSURE.TYPE",
"META.INSTRUMENT.P_BAND" : "META.INSTRUMENT.BAND",
"P_BAND" : "META.INSTRUMENT.BAND",
"META.INSTRUMENT.P_DETECTOR" : "META.INSTRUMENT.DETECTOR",
"P_DETECT" : "META.INSTRUMENT.DETECTOR",
"META.INSTRUMENT.P_CHANNEL" : "META.INSTRUMENT.CHANNEL",
"P_CHANNE" : "META.INSTRUMENT.CHANNEL",
"META.INSTRUMENT.P_FILTER" : "META.INSTRUMENT.FILTER",
"P_FILTER" : "META.INSTRUMENT.FILTER",
"META.INSTRUMENT.P_PUPIL" : "META.INSTRUMENT.PUPIL",
"P_PUPIL" : "META.INSTRUMENT.PUPIL",
"META.INSTRUMENT.P_MODULE" : "META.INSTRUMENT.MODULE",
"P_MODULE" : "META.INSTRUMENT.MODULE",
"META.SUBARRAY.P_SUBARRAY" : "META.SUBARRAY.NAME",
"P_SUBARR" : "META.SUBARRAY.NAME",
"META.INSTRUMENT.P_GRATING" : "META.INSTRUMENT.GRATING",
"P_GRATIN" : "META.INSTRUMENT.GRATING",
"META.EXPOSURE.PREADPATT" : "META.EXPOSURE.READPATT",
"META.EXPOSURE.P_READPATT" : "META.EXPOSURE.READPATT",
"P_READPA" : "META.EXPOSURE.READPATT",
# vvvv Speculative, not currently defined or required by CAL vvvvv
"META.INSTRUMENT.PCORONAGRAPH" : "META.INSTRUMENT.CORONAGRAPH",
"P_CORONM" : "META.INSTRUMENT.CORONAGRAPH",
}
# Rmap header reference_to_dataset field tranlations, can override basic!
try:
translations.update(rmapping.reference_to_dataset)
except AttributeError:
pass
log.verbose("reference_to_dataset translations:\n", log.PP(translations), verbosity=60)
log.verbose("reference_to_dataset input header:\n", log.PP(header), verbosity=80)
for key in header:
# Match META.X.P_SOMETHING or P_SOMETH
if (key.split(".")[-1].startswith("P_")) and key not in translations:
log.warning("CRDS-pattern-like keyword", repr(key),
"w/o CRDS translation to corresponding dataset keyword.")
log.info("Pattern-like keyword", repr(key),
"may be misspelled or missing its translation in CRDS. Pattern will not be used.")
log.info("The translation for", repr(key),
"can be defined in crds.jwst.locate or rmap header reference_to_dataset field.")
log.info("If this is not a pattern keyword, adding a translation to 'not-a-pattern'",
"will suppress this warning.")
# Add replacements for translations *if* the existing untranslated value
# is poor and the translated value is better defined. This is to do
# translations w/o replacing valid/concrete DM values with something
# like guessed values of "UNDEFINED" or "N/A".
for rkey in sorted(translations):
if rkey in header:
dkey = translations[rkey]
dval = header.get(translations[rkey], None)
rval = header[rkey]
if rval not in [None, "UNDEFINED"] and rval != dval:
log.info("Setting", repr(dkey), "=", repr(dval),
"to value of", repr(rkey), "=", repr(rval))
header[dkey] = rval
header = abstract.cross_strap_header(header)
# NOTE: the hacks below happen after cross-strapping and pattern handling
# so if the keywords are still undefined they're undefined. They have to
# be explicitly defined as UNDEFINED somehow since they're nearly universally
# used in constraints as condition variables even if they're not used in rmaps.
# Unlike the targets of constraints, CRDS is nominally unaware of condition
# variables so they need to be incidentally defined. This currently doesn't
# work out if the rmap doesn't use them. Condition variables are eval'ed in
# expressions.
if "SUBARRAY" not in header:
header["SUBARRAY"] = header["META.SUBARRAY.NAME"] = "UNDEFINED"
if "EXP_TYPE" not in header:
header["EXP_TYPE"] = header["META.EXPOSURE.TYPE"] = "UNDEFINED"
if "USEAFTER" not in header and "META.USEAFTER" in header:
header["USEAFTER"] = header["META.USEAFTER"]
if "USEAFTER" not in header and "META.USEAFTER" in header:
header["USEAFTER"] = header["META.USEAFTER"]
# If USEAFTER is defined, or we're configured to fake it...
# don't invent one if its missing and we're not faking it.
if "USEAFTER" in header or config.ALLOW_BAD_USEAFTER:
# Identify this as best as possible,
filename = header.get("FILENAME", None) or rmapping.filename
reformatted = timestamp.reformat_useafter(filename, header).split()
header["DATE-OBS"] = header["META.OBSERVATION.DATE"] = reformatted[0]
header["TIME-OBS"] = header["META.OBSERVATION.TIME"] = reformatted[1]
log.verbose("reference_to_dataset output header:\n", log.PP(header), verbosity=80)
return header
def reference_keys_to_dataset_keys(rmapping, header):
"""Given a header dictionary for a reference file, map the header back to keys
relevant to datasets. So for ACS biasfile the reference says BINAXIS1 but
the dataset says NUMCOLS. This would convert { "BINAXIS1": 1024 } to {
"NUMCOLS" : 1024 }.
In general, rmap parkeys are matched against datset values and are defined
as dataset header keywords. For refactoring though, what's initially
available are reference file keywords... which need to be mapped into the
terms rmaps know: dataset keywords.
Another aspect of this translation is handling reference file "pattern"
keywords which typically define or-barred sets of values rather than
discrete values, any of which the reference is defined to support:
e.g. 'DETECTOR1|DETECTOR2' vs. 'DETECTOR1'. In this case, the reference
file will define a pattern keyword used to define the match pattern in the
rmap, while a dataset will define a discrete valued keyword which is
matched on. e.g. reference file keyword "META.EXPOSURE.P_EXPTYPE" is
translated back to dataset keyword "META.EXPOSURE.TYPE". Reference files
can specify parameters in either form and the P_ pattern variant is given
preference if both values are defined. For CRDS purposes, only the P_
version is checked and used since it will be used to replace the discrete
valued keyword in the header which is certified and used to define the rmap
updates.
Note, can't test unrecognized "P_" keywords because the logging appeares to go to stderr which doctests don't check.
==================================================
Test adding a translation.
>>> reference_keys_to_dataset_keys( \
namedtuple('x', ['reference_to_dataset', 'filename'])({'MOUSE' : 'RAT'}, ''), \
{"MOUSE" : "MICKEY", "RAT" : "MORTIMER"})
{'MOUSE': 'MICKEY', 'RAT': 'MICKEY', 'ROMAN.META.SUBARRAY.NAME': 'UNDEFINED', 'ROMAN.META.EXPOSURE.TYPE': 'UNDEFINED'}
==================================================
Test replacing translated values with untranslated values.
>>> reference_keys_to_dataset_keys( \
namedtuple('x', ['reference_to_dataset', 'filename'])({'MOUSE' : 'RAT'}, ''), \
{"ROMAN.META.EXPOSURE.P_EXPTYPE" : None, \
"ROMAN.META.INSTRUMENT.P_BAND" : "UNDEFINED", \
"ROMAN.META.INSTRUMENT.P_DETECTOR" : "RADAR", \
"ROMAN.META.INSTRUMENT.P_CHANNEL" : None, \
"ROMAN.META.INSTRUMENT.CHANNEL" : None, \
"ROMAN.META.INSTRUMENT.P_FILTER" : "UNDEFINED", \
"ROMAN.META.INSTRUMENT.FILTER" : None, \
"ROMAN.META.INSTRUMENT.P_MODULE" : "LUNAR", \
"ROMAN.META.INSTRUMENT.MODULE" : None, \
"ROMAN.META.SUBARRAY.P_SUBARRAY" : None, \
"ROMAN.META.SUBARRAY.NAME" : "YELLOW", \
"ROMAN.META.INSTRUMENT.P_GRATING" : "UNDEFINED", \
"ROMAN.META.INSTRUMENT.GRATING" : "MOZZARELLA", \
"ROMAN.META.EXPOSURE.PREADPATT" : "CHECKERBOARD", \
"ROMAN.META.EXPOSURE.READPATT" : "CHESSBOARD"})
{'ROMAN.META.EXPOSURE.P_EXPTYPE': None, 'ROMAN.META.INSTRUMENT.P_BAND': 'UNDEFINED', 'ROMAN.META.INSTRUMENT.P_DETECTOR': 'RADAR', 'ROMAN.META.INSTRUMENT.P_CHANNEL': None, 'ROMAN.META.INSTRUMENT.CHANNEL': None, 'ROMAN.META.INSTRUMENT.P_FILTER': 'UNDEFINED', 'ROMAN.META.INSTRUMENT.FILTER': None, 'ROMAN.META.INSTRUMENT.P_MODULE': 'LUNAR', 'ROMAN.META.INSTRUMENT.MODULE': 'LUNAR', 'ROMAN.META.SUBARRAY.P_SUBARRAY': None, 'ROMAN.META.SUBARRAY.NAME': 'YELLOW', 'ROMAN.META.INSTRUMENT.P_GRATING': 'UNDEFINED', 'ROMAN.META.INSTRUMENT.GRATING': 'MOZZARELLA', 'ROMAN.META.EXPOSURE.PREADPATT': 'CHECKERBOARD', 'ROMAN.META.EXPOSURE.READPATT': 'CHECKERBOARD', 'ROMAN.META.INSTRUMENT.DETECTOR': 'RADAR', 'ROMAN.META.EXPOSURE.TYPE': 'UNDEFINED'}
==================================================
Test setting missing subarray and exposure type values.
>>> reference_keys_to_dataset_keys( \
namedtuple('x', ['reference_to_dataset', 'filename'])({}, ''), \
{})
{'ROMAN.META.SUBARRAY.NAME': 'UNDEFINED', 'ROMAN.META.EXPOSURE.TYPE': 'UNDEFINED'}
>>> reference_keys_to_dataset_keys( \
namedtuple('x', ['reference_to_dataset', 'filename'])({}, ''), \
{'ROMAN.META.SUBARRAY.NAME' : 'REDOCTOBER', \
'ROMAN.META.EXPOSURE.TYPE' : 'NORTHFACE'})
{'ROMAN.META.SUBARRAY.NAME': 'REDOCTOBER', 'ROMAN.META.EXPOSURE.TYPE': 'NORTHFACE'}
==================================================
Test preserving existing subarry adn exposure type values.
>>> reference_keys_to_dataset_keys( \
namedtuple('x', ['reference_to_dataset', 'filename'])({}, ''), \
{'ROMAN.META.SUBARRAY.NAME' : 'REDOCTOBER', \
'ROMAN.META.EXPOSURE.TYPE' : 'NORTHFACE'})
{'ROMAN.META.SUBARRAY.NAME': 'REDOCTOBER', 'ROMAN.META.EXPOSURE.TYPE': 'NORTHFACE'}
==================================================
Test preseverving existing DATE/TIME if no USEAFTER value.
>>> config.ALLOW_BAD_USEAFTER.reset()
>>> reference_keys_to_dataset_keys( \
namedtuple('x', ['reference_to_dataset', 'filename'])({}, 'secret_code_file.txt'), \
{'ROMAN.META.OBSERVATION.DATE' : '1879-03-14', \
'ROMAN.META.OBSERVATION.TIME' : '12:34:56'})
{'ROMAN.META.OBSERVATION.DATE': '1879-03-14', 'ROMAN.META.OBSERVATION.TIME': '12:34:56', 'ROMAN.META.SUBARRAY.NAME': 'UNDEFINED', 'ROMAN.META.EXPOSURE.TYPE': 'UNDEFINED'}
==================================================
Test setting DATE/TIME with no USEAFTER, but allowed "bad use after".
>>> config.ALLOW_BAD_USEAFTER.reset()
>>> config.ALLOW_BAD_USEAFTER.set("1")
False
>>> reference_keys_to_dataset_keys(namedtuple('x', ['reference_to_dataset', 'filename'])({}, 'secret_code_file.txt'), {})
{'ROMAN.META.SUBARRAY.NAME': 'UNDEFINED', 'ROMAN.META.EXPOSURE.TYPE': 'UNDEFINED', 'ROMAN.META.OBSERVATION.DATE': '1900-01-01', 'ROMAN.META.OBSERVATION.TIME': '00:00:00'}
==================================================
Test setting DATE/TIME from USEAFTER.
>>> config.ALLOW_BAD_USEAFTER.reset()
>>> config.ALLOW_BAD_USEAFTER.set("1")
False
>>> reference_keys_to_dataset_keys(namedtuple('x', ['reference_to_dataset', 'filename'])({}, 'secret_code_file.txt'), \
{'ROMAN.META.USEAFTER' : '1770-12-01T01:23:45', \
'ROMAN.META.OBSERVATION.DATE' : '1879-03-14', \
'ROMAN.META.OBSERVATION.TIME' : '12:34:56'})
{'ROMAN.META.USEAFTER': '1770-12-01T01:23:45', 'ROMAN.META.OBSERVATION.DATE': '1770-12-01', 'ROMAN.META.OBSERVATION.TIME': '01:23:45', 'ROMAN.META.SUBARRAY.NAME': 'UNDEFINED', 'ROMAN.META.EXPOSURE.TYPE': 'UNDEFINED'}
==================================================
Test bad formatted USEAFTER.
>>> config.ALLOW_BAD_USEAFTER.reset()
>>> reference_keys_to_dataset_keys(namedtuple('x', ['reference_to_dataset', 'filename'])({}, 'secret_code_file.txt'), \
{'ROMAN.META.USEAFTER' : 'bad user after', \
'ROMAN.META.OBSERVATION.DATE' : '1879-03-14', \
'ROMAN.META.OBSERVATION.TIME' : '12:34:56'})
Traceback (most recent call last):
...
crds.core.exceptions.InvalidUseAfterFormat: Bad USEAFTER time format = 'bad user after'
"""
header = dict(header)
# Basic common pattern translations
translations = {
"ROMAN.META.EXPOSURE.P_EXPTYPE": "ROMAN.META.EXPOSURE.TYPE",
"ROMAN.META.INSTRUMENT.P_BAND": "ROMAN.META.INSTRUMENT.BAND",
"ROMAN.META.INSTRUMENT.P_DETECTOR": "ROMAN.META.INSTRUMENT.DETECTOR",
"ROMAN.META.INSTRUMENT.P_CHANNEL": "ROMAN.META.INSTRUMENT.CHANNEL",
"ROMAN.META.INSTRUMENT.P_FILTER": "ROMAN.META.INSTRUMENT.FILTER",
"ROMAN.META.INSTRUMENT.P_MODULE": "ROMAN.META.INSTRUMENT.MODULE",
"ROMAN.META.SUBARRAY.P_SUBARRAY": "ROMAN.META.SUBARRAY.NAME",
"ROMAN.META.INSTRUMENT.P_GRATING": "ROMAN.META.INSTRUMENT.GRATING",
"ROMAN.META.EXPOSURE.PREADPATT": "ROMAN.META.EXPOSURE.READPATT",
"ROMAN.META.EXPOSURE.P_READPATT": "ROMAN.META.EXPOSURE.READPATT",
# vvvv Speculative, not currently defined or required by CAL vvvvv
"ROMAN.META.INSTRUMENT.PCORONAGRAPH":
"ROMAN.META.INSTRUMENT.CORONAGRAPH",
}
# Rmap header reference_to_dataset field tranlations, can override basic!
try:
translations.update(rmapping.reference_to_dataset)
except AttributeError:
pass
log.verbose("reference_to_dataset translations:\n",
log.PP(translations),
verbosity=60)
log.verbose("reference_to_dataset input header:\n",
log.PP(header),
verbosity=80)
for key in header:
# Match META.X.P_SOMETHING or P_SOMETH
if (key.split(".")[-1].startswith("P_")) and key not in translations:
log.warning(
"CRDS-pattern-like keyword", repr(key),
"w/o CRDS translation to corresponding dataset keyword.")
log.info(
"Pattern-like keyword", repr(key),
"may be misspelled or missing its translation in CRDS. Pattern will not be used."
)
log.info(
"The translation for", repr(key),
"can be defined in crds.roman.locate or rmap header reference_to_dataset field."
)
log.info(
"If this is not a pattern keyword, adding a translation to 'not-a-pattern'",
"will suppress this warning.")
# Add replacements for translations *if* the existing untranslated value
# is poor and the translated value is better defined. This is to do
# translations w/o replacing valid/concrete DM values with something
# like guessed values of "UNDEFINED" or "N/A".
for rkey in sorted(translations):
if rkey in header:
dkey = translations[rkey]
dval = header.get(translations[rkey], None)
rval = header[rkey]
if rval not in [None, "UNDEFINED"] and rval != dval:
log.info("Setting", repr(dkey), "=", repr(dval), "to value of",
repr(rkey), "=", repr(rval))
header[dkey] = rval
if "ROMAN.META.SUBARRAY.NAME" not in header:
header["ROMAN.META.SUBARRAY.NAME"] = "UNDEFINED"
if "ROMAN.META.EXPOSURE.TYPE" not in header:
header["ROMAN.META.EXPOSURE.TYPE"] = "UNDEFINED"
# If USEAFTER is defined, or we're configured to fake it...
# don't invent one if its missing and we're not faking it.
if "ROMAN.META.USEAFTER" in header or config.ALLOW_BAD_USEAFTER:
# Identify this as best as possible,
filename = header.get("ROMAN.META.FILENAME", None) or rmapping.filename
reformatted = timestamp.reformat_useafter(filename, header).split()
header["ROMAN.META.OBSERVATION.DATE"] = reformatted[0]
header["ROMAN.META.OBSERVATION.TIME"] = reformatted[1]
log.verbose("reference_to_dataset output header:\n",
log.PP(header),
verbosity=80)
return header
def reference_keys_to_dataset_keys(rmapping, header):
"""Given a header dictionary for a reference file, map the header back to keys
relevant to datasets. So for ACS biasfile the reference says BINAXIS1 but
the dataset says NUMCOLS. This would convert { "BINAXIS1": 1024 } to {
"NUMCOLS" : 1024 }.
In general, rmap parkeys are matched against datset values and are defined
as dataset header keywords. For refactoring though, what's initially
available are reference file keywords... which need to be mapped into the
terms rmaps know: dataset keywords.
Another aspect of this translation is handling reference file "pattern"
keywords which typically define or-barred sets of values rather than
discrete values, any of which the reference is defined to support:
e.g. 'DETECTOR1|DETECTOR2' vs. 'DETECTOR1'. In this case, the reference
file will define a pattern keyword used to define the match pattern in the
rmap, while a dataset will define a discrete valued keyword which is
matched on. e.g. reference file keyword "META.EXPOSURE.P_EXPTYPE" is
translated back to dataset keyword "META.EXPOSURE.TYPE". Reference files
can specify parameters in either form and the P_ pattern variant is given
preference if both values are defined. For CRDS purposes, only the P_
version is checked and used since it will be used to replace the discrete
valued keyword in the header which is certified and used to define the rmap
updates.
"""
header = dict(header)
# Basic common pattern translations
translations = {
"META.EXPOSURE.P_EXPTYPE" : "META.EXPOSURE.TYPE",
"META.INSTRUMENT.P_BAND" : "META.INSTRUMENT.BAND",
"META.INSTRUMENT.P_DETECTOR" : "META.INSTRUMENT.DETECTOR",
"META.INSTRUMENT.P_CHANNEL" : "META.INSTRUMENT.CHANNEL",
"META.INSTRUMENT.P_FILTER" : "META.INSTRUMENT.FILTER",
"META.INSTRUMENT.P_MODULE" : "META.INSTRUMENT.MODULE",
"META.SUBARRAY.P_SUBARRAY" : "META.SUBARRAY.NAME",
"META.INSTRUMENT.P_GRATING" : "META.INSTRUMENT.GRATING",
"META.EXPOSURE.PREADPATT" : "META.EXPOSURE.READPATT",
"META.EXPOSURE.P_READPATT" : "META.EXPOSURE.READPATT",
# vvvv Speculative, not currently defined or required by CAL vvvvv
"META.INSTRUMENT.PCORONAGRAPH" : "META.INSTRUMENT.CORONAGRAPH",
}
# Rmap header reference_to_dataset field tranlations, can override basic!
try:
translations.update(rmapping.reference_to_dataset)
except AttributeError:
pass
log.verbose("reference_to_dataset translations:\n", log.PP(translations), verbosity=60)
log.verbose("reference_to_dataset input header:\n", log.PP(header), verbosity=80)
for key in header:
# Match META.X.P_SOMETHING or P_SOMETH
if (key.split(".")[-1].startswith("P_")) and key not in translations:
log.warning("CRDS-pattern-like keyword", repr(key),
"w/o CRDS translation to corresponding dataset keyword.")
log.info("Pattern-like keyword", repr(key),
"may be misspelled or missing its translation in CRDS. Pattern will not be used.")
log.info("The translation for", repr(key),
"can be defined in crds.roman.locate or rmap header reference_to_dataset field.")
log.info("If this is not a pattern keyword, adding a translation to 'not-a-pattern'",
"will suppress this warning.")
# Add replacements for translations *if* the existing untranslated value
# is poor and the translated value is better defined. This is to do
# translations w/o replacing valid/concrete DM values with something
# like guessed values of "UNDEFINED" or "N/A".
for rkey in sorted(translations):
if rkey in header:
dkey = translations[rkey]
dval = header.get(translations[rkey], None)
rval = header[rkey]
if rval not in [None, "UNDEFINED"] and rval != dval:
log.info("Setting", repr(dkey), "=", repr(dval),
"to value of", repr(rkey), "=", repr(rval))
header[dkey] = rval
if "META.SUBARRAY.NAME" not in header:
header["META.SUBARRAY.NAME"] = "UNDEFINED"
if "META.EXPOSURE.TYPE" not in header:
header["META.EXPOSURE.TYPE"] = "UNDEFINED"
# If USEAFTER is defined, or we're configured to fake it...
# don't invent one if its missing and we're not faking it.
if "META.USEAFTER" in header or config.ALLOW_BAD_USEAFTER:
# Identify this as best as possible,
filename = header.get("META.FILENAME", None) or rmapping.filename
reformatted = timestamp.reformat_useafter(filename, header).split()
header["META.OBSERVATION.DATE"] = reformatted[0]
header["META.OBSERVATION.TIME"] = reformatted[1]
log.verbose("reference_to_dataset output header:\n", log.PP(header), verbosity=80)
return header
