def get_all_reffiles(param_dict):
"""
"""
# Place import statement here in order to avoid circular import
# with crd_tools
from mirage.reference_files import crds_tools
mapping = crds_tools.get_reffiles(param_dict, list(CRDS_FILE_TYPES.values()))
return mapping
def distortion_file_value(self):
"""Make sure the distortion file name is a valid file. If it has a
value of 'crds', then query CRDS and retrieve the appropriate file
"""
if self.distortion_file == 'crds':
if self.paramfile is None:
self.reference_dictionary = {}
self.reference_dictionary['INSTRUME'] = self.instrument
self.reference_dictionary['DETECTOR'] = self.detector
if '5' in self.detector:
self.reference_dictionary[
'DETECTOR'] = self.detector.replace('5', 'LONG')
if self.instrument.upper() == 'NIRCAM':
self.reference_dictionary['CHANNEL'] = self.channel
if 'FGS' in self.reference_dictionary['DETECTOR']:
self.reference_dictionary['DETECTOR'] = 'GUIDER{}'.format(
self.reference_dictionary['DETECTOR'][-1])
# Use the current date and time in order to get the most recent
# reference file
self.reference_dictionary['DATE-OBS'] = datetime.date.today(
).isoformat()
current_date = datetime.datetime.now()
self.reference_dictionary['TIME-OBS'] = current_date.time(
).isoformat()
# For the purposes of choosing reference files, the exposure type should
# always be set to imaging, since it is used to locate sources in the
# seed image, prior to any dispersion.
self.reference_dictionary['EXP_TYPE'] = EXPTYPES[
self.instrument.lower()]["imaging"]
# This assumes that filter and pupil names match up with reality,
# as opposed to the more user-friendly scheme of allowing any
# filter to be in the filter field.
self.reference_dictionary['FILTER'] = self.filter
self.reference_dictionary['PUPIL'] = self.pupil
mapping = crds_tools.get_reffiles(self.reference_dictionary,
['distortion'],
download=True)
self.distortion_file = mapping['distortion']
def full_paths(params, module_path, crds_dictionary, offline=False):
"""Expand the relevant input paths from the input yaml file to be full
paths.
Parameters
----------
params : dict
Nested dictionary read in from an input yaml file
module_path : str
Path to the Mirage module
crds_dictionary : dict
Dictionary of basic observation metadata that is used by CRDS to
select appropriate reference files.
offline : bool
Set to True in order to skip downloading reference files from CRDS
and instead only return the dictionary of reference file names.
Returns
-------
params : dict
Modified nested dictionary including full paths
"""
# Place import statement here in order to avoid circular import
# with crd_tools
from mirage.reference_files import crds_tools
logger = logging.getLogger('mirage.utils.utils.full_paths')
pathdict = {
'Reffiles': [
'dark', 'linearized_darkfile', 'badpixmask', 'superbias',
'linearity', 'saturation', 'gain', 'pixelflat', 'illumflat', 'ipc',
'astrometric', 'crosstalk', 'occult', 'pixelAreaMap',
'transmission', 'subarray_defs', 'filtpupilcombo', 'flux_cal',
'readpattdefs', 'filter_throughput', 'filter_wheel_positions',
'photom'
],
'simSignals': [
'pointsource', 'psfpath', 'galaxyListFile', 'extended',
'movingTargetList', 'movingTargetSersic', 'movingTargetExtended',
'movingTargetToTrack', 'psf_wing_threshold_file', 'zodiacal',
'scattered'
],
'cosmicRay': ['path'],
'newRamp': [
'dq_configfile', 'sat_configfile', 'superbias_configfile',
'refpix_configfile', 'linear_configfile'
],
'Output': ['file', 'directory']
}
all_config_files = {
'nircam': {
'Reffiles-subarray_defs': 'NIRCam_subarray_definitions.list',
'Reffiles-flux_cal': 'NIRCam_zeropoints.list',
'Reffiles-crosstalk': 'xtalk20150303g0.errorcut.txt',
'Reffiles-readpattdefs': 'nircam_read_pattern_definitions.list',
'Reffiles-filter_throughput': 'placeholder.txt',
'Reffiles-filtpupilcombo': 'nircam_filter_pupil_pairings.list',
'Reffiles-filter_wheel_positions':
'nircam_filter_and_pupil_wheel_positions.txt',
'simSignals-psf_wing_threshold_file':
'nircam_psf_wing_rate_thresholds.txt',
'newRamp-dq_configfile': 'dq_init.cfg',
'newRamp-sat_configfile': 'saturation.cfg',
'newRamp-superbias_configfile': 'superbias.cfg',
'newRamp-refpix_configfile': 'refpix.cfg',
'newRamp-linear_configfile': 'linearity.cfg'
},
'niriss': {
'Reffiles-subarray_defs': 'niriss_subarrays.list',
'Reffiles-flux_cal': 'niriss_zeropoints.list',
'Reffiles-crosstalk': 'niriss_xtalk_zeros.txt',
'Reffiles-readpattdefs': 'niriss_readout_pattern.txt',
'Reffiles-filter_throughput': 'placeholder.txt',
'Reffiles-filtpupilcombo': 'niriss_dual_wheel_list.txt',
'Reffiles-filter_wheel_positions':
'niriss_filter_and_pupil_wheel_positions.txt',
'simSignals-psf_wing_threshold_file':
'niriss_psf_wing_rate_thresholds.txt',
'newRamp-dq_configfile': 'dq_init.cfg',
'newRamp-sat_configfile': 'saturation.cfg',
'newRamp-superbias_configfile': 'superbias.cfg',
'newRamp-refpix_configfile': 'refpix.cfg',
'newRamp-linear_configfile': 'linearity.cfg'
},
'fgs': {
'Reffiles-subarray_defs': 'guider_subarrays.list',
'Reffiles-flux_cal': 'guider_zeropoints.list',
'Reffiles-crosstalk': 'guider_xtalk_zeros.txt',
'Reffiles-readpattdefs': 'guider_readout_pattern.txt',
'Reffiles-filter_throughput': 'placeholder.txt',
'Reffiles-filtpupilcombo': 'guider_filter_dummy.list',
'Reffiles-filter_wheel_positions': 'dummy.txt',
'simSignals-psf_wing_threshold_file':
'fgs_psf_wing_rate_thresholds.txt',
'newRamp-dq_configfile': 'dq_init.cfg',
'newRamp-sat_configfile': 'saturation.cfg',
'newRamp-superbias_configfile': 'superbias.cfg',
'newRamp-refpix_configfile': 'refpix.cfg',
'newRamp-linear_configfile': 'linearity.cfg'
}
}
config_files = all_config_files[params['Inst']['instrument'].lower()]
for key1 in pathdict:
for key2 in pathdict[key1]:
# For those using an old or incomplete yaml file, if any of the
# CRDS keywords are missing, create them and set equal to 'crds'.
if key2 not in params[key1].keys():
if key2 in CRDS_FILE_TYPES.keys():
logger.info(
'{}:{} field not present in input. Setting equal to "crds"'
.format(key1, key2))
params[key1][key2] = 'crds'
if params[key1][key2].lower() not in ['none', 'config', 'crds']:
params[key1][key2] = os.path.abspath(
os.path.expandvars(params[key1][key2]))
elif params[key1][key2].lower() == 'config':
cfile = config_files['{}-{}'.format(key1, key2)]
fpath = os.path.join(module_path, 'config', cfile)
params[key1][key2] = fpath
logger.info(
"'config' specified: Using {} for {}:{} input file".format(
fpath, key1, key2))
elif key2 in CRDS_FILE_TYPES.keys() and params[key1][key2].lower(
) == 'crds':
# 'crds' set for one of the reference files means to
# search for the best reference file currently in CRDS
# and download if it is not already present in
# self.crds_datadir
query_dictionary = crds_dictionary
# For NIRCam WFSS and grism time series simluations we need to get the
# imaging mode flat for the crossing filter. This is because the grism-
# specific flat in CRDS is all 1.0's, because most people apply a wavelength-
# dependent flat to the data after they have a 1D sepctrum (according to Pirzkal).
# So when introducing flat field effects into the data, we use the imaging mode
# flat. This assumes no wavelength-dependence to the flat field
if ((params['Inst']['instrument'].lower() == 'nircam')
and (params['Inst']['mode'] in ['wfss', 'ts_grism'])
and (key2 == 'pixelflat')):
img_dict = copy.deepcopy(crds_dictionary)
if img_dict['PUPIL'] in ['GRISMR', 'GRISMC']:
img_dict['PUPIL'] = 'CLEAR'
else:
raise ValueError(
'WFSS or GrismTSO sim, but Pupil is not set to one of the grisms.'
)
query_dictionary = img_dict
if key2 != 'transmission':
mapping = crds_tools.get_reffiles(query_dictionary,
[CRDS_FILE_TYPES[key2]],
download=not offline)
params[key1][key2] = mapping[CRDS_FILE_TYPES[key2]]
logger.info(
"From CRDS, found {} as the {} reference file.".format(
mapping[CRDS_FILE_TYPES[key2]], key2))
else:
# For the moment, the transmission files are stored in the GRISM_NIRCAM and
# GRISM_NIRISS repos. Longer-term they will become CRDS files and this else
# statement can be deleted.
params[key1][key2] = get_transmission_file(
query_dictionary)
logger.info(
"From grism library, found {} as the POM transmission file."
.format(params[key1][key2]))
# If we grab the IPC reference file from CRDS, then we need to invert it prior to use
if key2 == 'ipc':
ipc_path, ipc_file = os.path.split(mapping['ipc'])
inverted_kernel_file = os.path.join(
ipc_path,
'Kernel_to_add_IPC_effects_from_{}'.format(ipc_file))
# Check to see if the version of the IPC file with an
# inverted kernel already exists.
if os.path.isfile(inverted_kernel_file):
logger.info(
"Found an existing inverted kernel for this IPC file: {}"
.format(inverted_kernel_file))
params[key1][key2] = inverted_kernel_file
params['Reffiles']['invertIPC'] = False
else:
logger.info(
"No inverted IPC kernel file found. Kernel will be inverted prior to use."
)
params['Reffiles']['invertIPC'] = True
return params
def full_paths(params, module_path, crds_dictionary, offline=False):
"""Expand the relevant input paths from the input yaml file to be full
paths.
Parameters
----------
params : dict
Nested dictionary read in from an input yaml file
module_path : str
Path to the Mirage module
crds_dictionary : dict
Dictionary of basic observation metadata that is used by CRDS to
select appropriate reference files.
offline : bool
Set to True in order to skip downloading reference files from CRDS
and instead only return the dictionary of reference file names.
Returns
-------
params : dict
Modified nested dictionary including full paths
"""
# Place import statement here in order to avoid circular import
# with crd_tools
from mirage.reference_files import crds_tools
pathdict = {
'Reffiles': [
'dark', 'linearized_darkfile', 'badpixmask', 'superbias',
'linearity', 'saturation', 'gain', 'pixelflat', 'illumflat', 'ipc',
'astrometric', 'crosstalk', 'occult', 'pixelAreaMap',
'subarray_defs', 'filtpupilcombo', 'flux_cal', 'readpattdefs',
'filter_throughput'
],
'simSignals': [
'pointsource', 'psfpath', 'galaxyListFile', 'extended',
'movingTargetList', 'movingTargetSersic', 'movingTargetExtended',
'movingTargetToTrack', 'psf_wing_threshold_file', 'zodiacal',
'scattered'
],
'cosmicRay': ['path'],
'newRamp': [
'dq_configfile', 'sat_configfile', 'superbias_configfile',
'refpix_configfile', 'linear_configfile'
],
'Output': ['file', 'directory']
}
all_config_files = {
'nircam': {
'Reffiles-subarray_defs': 'NIRCam_subarray_definitions.list',
'Reffiles-flux_cal': 'NIRCam_zeropoints.list',
'Reffiles-crosstalk': 'xtalk20150303g0.errorcut.txt',
'Reffiles-readpattdefs': 'nircam_read_pattern_definitions.list',
'Reffiles-filter_throughput': 'placeholder.txt',
'Reffiles-filtpupilcombo': 'nircam_filter_pupil_pairings.list',
'simSignals-psf_wing_threshold_file':
'nircam_psf_wing_rate_thresholds.txt',
'newRamp-dq_configfile': 'dq_init.cfg',
'newRamp-sat_configfile': 'saturation.cfg',
'newRamp-superbias_configfile': 'superbias.cfg',
'newRamp-refpix_configfile': 'refpix.cfg',
'newRamp-linear_configfile': 'linearity.cfg'
},
'niriss': {
'Reffiles-subarray_defs': 'niriss_subarrays.list',
'Reffiles-flux_cal': 'niriss_zeropoints.list',
'Reffiles-crosstalk': 'niriss_xtalk_zeros.txt',
'Reffiles-readpattdefs': 'niriss_readout_pattern.txt',
'Reffiles-filter_throughput': 'placeholder.txt',
'Reffiles-filtpupilcombo': 'niriss_dual_wheel_list.txt',
'simSignals-psf_wing_threshold_file':
'niriss_psf_wing_rate_thresholds.txt',
'newRamp-dq_configfile': 'dq_init.cfg',
'newRamp-sat_configfile': 'saturation.cfg',
'newRamp-superbias_configfile': 'superbias.cfg',
'newRamp-refpix_configfile': 'refpix.cfg',
'newRamp-linear_configfile': 'linearity.cfg'
},
'fgs': {
'Reffiles-subarray_defs': 'guider_subarrays.list',
'Reffiles-flux_cal': 'guider_zeropoints.list',
'Reffiles-crosstalk': 'guider_xtalk_zeros.txt',
'Reffiles-readpattdefs': 'guider_readout_pattern.txt',
'Reffiles-filter_throughput': 'placeholder.txt',
'Reffiles-filtpupilcombo': 'guider_filter_dummy.list',
'simSignals-psf_wing_threshold_file':
'fgs_psf_wing_rate_thresholds.txt',
'newRamp-dq_configfile': 'dq_init.cfg',
'newRamp-sat_configfile': 'saturation.cfg',
'newRamp-superbias_configfile': 'superbias.cfg',
'newRamp-refpix_configfile': 'refpix.cfg',
'newRamp-linear_configfile': 'linearity.cfg'
}
}
config_files = all_config_files[params['Inst']['instrument'].lower()]
for key1 in pathdict:
for key2 in pathdict[key1]:
if params[key1][key2].lower() not in ['none', 'config', 'crds']:
params[key1][key2] = os.path.abspath(
os.path.expandvars(params[key1][key2]))
elif params[key1][key2].lower() == 'config':
cfile = config_files['{}-{}'.format(key1, key2)]
fpath = os.path.join(module_path, 'config', cfile)
params[key1][key2] = fpath
print(
"'config' specified: Using {} for {}:{} input file".format(
fpath, key1, key2))
elif key2 in CRDS_FILE_TYPES.keys() and params[key1][key2].lower(
) == 'crds':
# 'crds' set for one of the reference files means to
# search for the best reference file currently in CRDS
# and download if it is not already present in
# self.crds_datadir
mapping = crds_tools.get_reffiles(crds_dictionary,
[CRDS_FILE_TYPES[key2]],
download=not offline)
params[key1][key2] = mapping[CRDS_FILE_TYPES[key2]]
print("From CRDS, found {} as the {} reference file.".format(
mapping[CRDS_FILE_TYPES[key2]], key2))
# If we grab the IPC reference file from CRDS, then we need to invert it prior to use
if key2 == 'ipc':
ipc_path, ipc_file = os.path.split(mapping['ipc'])
inverted_kernel_file = os.path.join(
ipc_path,
'Kernel_to_add_IPC_effects_from_{}'.format(ipc_file))
# Check to see if the version of the IPC file with an
# inverted kernel already exists.
if os.path.isfile(inverted_kernel_file):
print(
"Found an existing inverted kernel for this IPC file: {}"
.format(inverted_kernel_file))
params[key1][key2] = inverted_kernel_file
params['Reffiles']['invertIPC'] = False
else:
print(
"No inverted IPC kernel file found. Kernel will be inverted prior to use."
)
params['Reffiles']['invertIPC'] = True
return params