def get_all_reffiles(param_dict): """ """ # Place import statement here in order to avoid circular import # with crd_tools from mirage.reference_files import crds_tools mapping = crds_tools.get_reffiles(param_dict, list(CRDS_FILE_TYPES.values())) return mapping
def distortion_file_value(self): """Make sure the distortion file name is a valid file. If it has a value of 'crds', then query CRDS and retrieve the appropriate file """ if self.distortion_file == 'crds': if self.paramfile is None: self.reference_dictionary = {} self.reference_dictionary['INSTRUME'] = self.instrument self.reference_dictionary['DETECTOR'] = self.detector if '5' in self.detector: self.reference_dictionary[ 'DETECTOR'] = self.detector.replace('5', 'LONG') if self.instrument.upper() == 'NIRCAM': self.reference_dictionary['CHANNEL'] = self.channel if 'FGS' in self.reference_dictionary['DETECTOR']: self.reference_dictionary['DETECTOR'] = 'GUIDER{}'.format( self.reference_dictionary['DETECTOR'][-1]) # Use the current date and time in order to get the most recent # reference file self.reference_dictionary['DATE-OBS'] = datetime.date.today( ).isoformat() current_date = datetime.datetime.now() self.reference_dictionary['TIME-OBS'] = current_date.time( ).isoformat() # For the purposes of choosing reference files, the exposure type should # always be set to imaging, since it is used to locate sources in the # seed image, prior to any dispersion. self.reference_dictionary['EXP_TYPE'] = EXPTYPES[ self.instrument.lower()]["imaging"] # This assumes that filter and pupil names match up with reality, # as opposed to the more user-friendly scheme of allowing any # filter to be in the filter field. self.reference_dictionary['FILTER'] = self.filter self.reference_dictionary['PUPIL'] = self.pupil mapping = crds_tools.get_reffiles(self.reference_dictionary, ['distortion'], download=True) self.distortion_file = mapping['distortion']
def full_paths(params, module_path, crds_dictionary, offline=False): """Expand the relevant input paths from the input yaml file to be full paths. Parameters ---------- params : dict Nested dictionary read in from an input yaml file module_path : str Path to the Mirage module crds_dictionary : dict Dictionary of basic observation metadata that is used by CRDS to select appropriate reference files. offline : bool Set to True in order to skip downloading reference files from CRDS and instead only return the dictionary of reference file names. Returns ------- params : dict Modified nested dictionary including full paths """ # Place import statement here in order to avoid circular import # with crd_tools from mirage.reference_files import crds_tools logger = logging.getLogger('mirage.utils.utils.full_paths') pathdict = { 'Reffiles': [ 'dark', 'linearized_darkfile', 'badpixmask', 'superbias', 'linearity', 'saturation', 'gain', 'pixelflat', 'illumflat', 'ipc', 'astrometric', 'crosstalk', 'occult', 'pixelAreaMap', 'transmission', 'subarray_defs', 'filtpupilcombo', 'flux_cal', 'readpattdefs', 'filter_throughput', 'filter_wheel_positions', 'photom' ], 'simSignals': [ 'pointsource', 'psfpath', 'galaxyListFile', 'extended', 'movingTargetList', 'movingTargetSersic', 'movingTargetExtended', 'movingTargetToTrack', 'psf_wing_threshold_file', 'zodiacal', 'scattered' ], 'cosmicRay': ['path'], 'newRamp': [ 'dq_configfile', 'sat_configfile', 'superbias_configfile', 'refpix_configfile', 'linear_configfile' ], 'Output': ['file', 'directory'] } all_config_files = { 'nircam': { 'Reffiles-subarray_defs': 'NIRCam_subarray_definitions.list', 'Reffiles-flux_cal': 'NIRCam_zeropoints.list', 'Reffiles-crosstalk': 'xtalk20150303g0.errorcut.txt', 'Reffiles-readpattdefs': 'nircam_read_pattern_definitions.list', 'Reffiles-filter_throughput': 'placeholder.txt', 'Reffiles-filtpupilcombo': 'nircam_filter_pupil_pairings.list', 'Reffiles-filter_wheel_positions': 'nircam_filter_and_pupil_wheel_positions.txt', 'simSignals-psf_wing_threshold_file': 'nircam_psf_wing_rate_thresholds.txt', 'newRamp-dq_configfile': 'dq_init.cfg', 'newRamp-sat_configfile': 'saturation.cfg', 'newRamp-superbias_configfile': 'superbias.cfg', 'newRamp-refpix_configfile': 'refpix.cfg', 'newRamp-linear_configfile': 'linearity.cfg' }, 'niriss': { 'Reffiles-subarray_defs': 'niriss_subarrays.list', 'Reffiles-flux_cal': 'niriss_zeropoints.list', 'Reffiles-crosstalk': 'niriss_xtalk_zeros.txt', 'Reffiles-readpattdefs': 'niriss_readout_pattern.txt', 'Reffiles-filter_throughput': 'placeholder.txt', 'Reffiles-filtpupilcombo': 'niriss_dual_wheel_list.txt', 'Reffiles-filter_wheel_positions': 'niriss_filter_and_pupil_wheel_positions.txt', 'simSignals-psf_wing_threshold_file': 'niriss_psf_wing_rate_thresholds.txt', 'newRamp-dq_configfile': 'dq_init.cfg', 'newRamp-sat_configfile': 'saturation.cfg', 'newRamp-superbias_configfile': 'superbias.cfg', 'newRamp-refpix_configfile': 'refpix.cfg', 'newRamp-linear_configfile': 'linearity.cfg' }, 'fgs': { 'Reffiles-subarray_defs': 'guider_subarrays.list', 'Reffiles-flux_cal': 'guider_zeropoints.list', 'Reffiles-crosstalk': 'guider_xtalk_zeros.txt', 'Reffiles-readpattdefs': 'guider_readout_pattern.txt', 'Reffiles-filter_throughput': 'placeholder.txt', 'Reffiles-filtpupilcombo': 'guider_filter_dummy.list', 'Reffiles-filter_wheel_positions': 'dummy.txt', 'simSignals-psf_wing_threshold_file': 'fgs_psf_wing_rate_thresholds.txt', 'newRamp-dq_configfile': 'dq_init.cfg', 'newRamp-sat_configfile': 'saturation.cfg', 'newRamp-superbias_configfile': 'superbias.cfg', 'newRamp-refpix_configfile': 'refpix.cfg', 'newRamp-linear_configfile': 'linearity.cfg' } } config_files = all_config_files[params['Inst']['instrument'].lower()] for key1 in pathdict: for key2 in pathdict[key1]: # For those using an old or incomplete yaml file, if any of the # CRDS keywords are missing, create them and set equal to 'crds'. if key2 not in params[key1].keys(): if key2 in CRDS_FILE_TYPES.keys(): logger.info( '{}:{} field not present in input. Setting equal to "crds"' .format(key1, key2)) params[key1][key2] = 'crds' if params[key1][key2].lower() not in ['none', 'config', 'crds']: params[key1][key2] = os.path.abspath( os.path.expandvars(params[key1][key2])) elif params[key1][key2].lower() == 'config': cfile = config_files['{}-{}'.format(key1, key2)] fpath = os.path.join(module_path, 'config', cfile) params[key1][key2] = fpath logger.info( "'config' specified: Using {} for {}:{} input file".format( fpath, key1, key2)) elif key2 in CRDS_FILE_TYPES.keys() and params[key1][key2].lower( ) == 'crds': # 'crds' set for one of the reference files means to # search for the best reference file currently in CRDS # and download if it is not already present in # self.crds_datadir query_dictionary = crds_dictionary # For NIRCam WFSS and grism time series simluations we need to get the # imaging mode flat for the crossing filter. This is because the grism- # specific flat in CRDS is all 1.0's, because most people apply a wavelength- # dependent flat to the data after they have a 1D sepctrum (according to Pirzkal). # So when introducing flat field effects into the data, we use the imaging mode # flat. This assumes no wavelength-dependence to the flat field if ((params['Inst']['instrument'].lower() == 'nircam') and (params['Inst']['mode'] in ['wfss', 'ts_grism']) and (key2 == 'pixelflat')): img_dict = copy.deepcopy(crds_dictionary) if img_dict['PUPIL'] in ['GRISMR', 'GRISMC']: img_dict['PUPIL'] = 'CLEAR' else: raise ValueError( 'WFSS or GrismTSO sim, but Pupil is not set to one of the grisms.' ) query_dictionary = img_dict if key2 != 'transmission': mapping = crds_tools.get_reffiles(query_dictionary, [CRDS_FILE_TYPES[key2]], download=not offline) params[key1][key2] = mapping[CRDS_FILE_TYPES[key2]] logger.info( "From CRDS, found {} as the {} reference file.".format( mapping[CRDS_FILE_TYPES[key2]], key2)) else: # For the moment, the transmission files are stored in the GRISM_NIRCAM and # GRISM_NIRISS repos. Longer-term they will become CRDS files and this else # statement can be deleted. params[key1][key2] = get_transmission_file( query_dictionary) logger.info( "From grism library, found {} as the POM transmission file." .format(params[key1][key2])) # If we grab the IPC reference file from CRDS, then we need to invert it prior to use if key2 == 'ipc': ipc_path, ipc_file = os.path.split(mapping['ipc']) inverted_kernel_file = os.path.join( ipc_path, 'Kernel_to_add_IPC_effects_from_{}'.format(ipc_file)) # Check to see if the version of the IPC file with an # inverted kernel already exists. if os.path.isfile(inverted_kernel_file): logger.info( "Found an existing inverted kernel for this IPC file: {}" .format(inverted_kernel_file)) params[key1][key2] = inverted_kernel_file params['Reffiles']['invertIPC'] = False else: logger.info( "No inverted IPC kernel file found. Kernel will be inverted prior to use." ) params['Reffiles']['invertIPC'] = True return params
def full_paths(params, module_path, crds_dictionary, offline=False): """Expand the relevant input paths from the input yaml file to be full paths. Parameters ---------- params : dict Nested dictionary read in from an input yaml file module_path : str Path to the Mirage module crds_dictionary : dict Dictionary of basic observation metadata that is used by CRDS to select appropriate reference files. offline : bool Set to True in order to skip downloading reference files from CRDS and instead only return the dictionary of reference file names. Returns ------- params : dict Modified nested dictionary including full paths """ # Place import statement here in order to avoid circular import # with crd_tools from mirage.reference_files import crds_tools pathdict = { 'Reffiles': [ 'dark', 'linearized_darkfile', 'badpixmask', 'superbias', 'linearity', 'saturation', 'gain', 'pixelflat', 'illumflat', 'ipc', 'astrometric', 'crosstalk', 'occult', 'pixelAreaMap', 'subarray_defs', 'filtpupilcombo', 'flux_cal', 'readpattdefs', 'filter_throughput' ], 'simSignals': [ 'pointsource', 'psfpath', 'galaxyListFile', 'extended', 'movingTargetList', 'movingTargetSersic', 'movingTargetExtended', 'movingTargetToTrack', 'psf_wing_threshold_file', 'zodiacal', 'scattered' ], 'cosmicRay': ['path'], 'newRamp': [ 'dq_configfile', 'sat_configfile', 'superbias_configfile', 'refpix_configfile', 'linear_configfile' ], 'Output': ['file', 'directory'] } all_config_files = { 'nircam': { 'Reffiles-subarray_defs': 'NIRCam_subarray_definitions.list', 'Reffiles-flux_cal': 'NIRCam_zeropoints.list', 'Reffiles-crosstalk': 'xtalk20150303g0.errorcut.txt', 'Reffiles-readpattdefs': 'nircam_read_pattern_definitions.list', 'Reffiles-filter_throughput': 'placeholder.txt', 'Reffiles-filtpupilcombo': 'nircam_filter_pupil_pairings.list', 'simSignals-psf_wing_threshold_file': 'nircam_psf_wing_rate_thresholds.txt', 'newRamp-dq_configfile': 'dq_init.cfg', 'newRamp-sat_configfile': 'saturation.cfg', 'newRamp-superbias_configfile': 'superbias.cfg', 'newRamp-refpix_configfile': 'refpix.cfg', 'newRamp-linear_configfile': 'linearity.cfg' }, 'niriss': { 'Reffiles-subarray_defs': 'niriss_subarrays.list', 'Reffiles-flux_cal': 'niriss_zeropoints.list', 'Reffiles-crosstalk': 'niriss_xtalk_zeros.txt', 'Reffiles-readpattdefs': 'niriss_readout_pattern.txt', 'Reffiles-filter_throughput': 'placeholder.txt', 'Reffiles-filtpupilcombo': 'niriss_dual_wheel_list.txt', 'simSignals-psf_wing_threshold_file': 'niriss_psf_wing_rate_thresholds.txt', 'newRamp-dq_configfile': 'dq_init.cfg', 'newRamp-sat_configfile': 'saturation.cfg', 'newRamp-superbias_configfile': 'superbias.cfg', 'newRamp-refpix_configfile': 'refpix.cfg', 'newRamp-linear_configfile': 'linearity.cfg' }, 'fgs': { 'Reffiles-subarray_defs': 'guider_subarrays.list', 'Reffiles-flux_cal': 'guider_zeropoints.list', 'Reffiles-crosstalk': 'guider_xtalk_zeros.txt', 'Reffiles-readpattdefs': 'guider_readout_pattern.txt', 'Reffiles-filter_throughput': 'placeholder.txt', 'Reffiles-filtpupilcombo': 'guider_filter_dummy.list', 'simSignals-psf_wing_threshold_file': 'fgs_psf_wing_rate_thresholds.txt', 'newRamp-dq_configfile': 'dq_init.cfg', 'newRamp-sat_configfile': 'saturation.cfg', 'newRamp-superbias_configfile': 'superbias.cfg', 'newRamp-refpix_configfile': 'refpix.cfg', 'newRamp-linear_configfile': 'linearity.cfg' } } config_files = all_config_files[params['Inst']['instrument'].lower()] for key1 in pathdict: for key2 in pathdict[key1]: if params[key1][key2].lower() not in ['none', 'config', 'crds']: params[key1][key2] = os.path.abspath( os.path.expandvars(params[key1][key2])) elif params[key1][key2].lower() == 'config': cfile = config_files['{}-{}'.format(key1, key2)] fpath = os.path.join(module_path, 'config', cfile) params[key1][key2] = fpath print( "'config' specified: Using {} for {}:{} input file".format( fpath, key1, key2)) elif key2 in CRDS_FILE_TYPES.keys() and params[key1][key2].lower( ) == 'crds': # 'crds' set for one of the reference files means to # search for the best reference file currently in CRDS # and download if it is not already present in # self.crds_datadir mapping = crds_tools.get_reffiles(crds_dictionary, [CRDS_FILE_TYPES[key2]], download=not offline) params[key1][key2] = mapping[CRDS_FILE_TYPES[key2]] print("From CRDS, found {} as the {} reference file.".format( mapping[CRDS_FILE_TYPES[key2]], key2)) # If we grab the IPC reference file from CRDS, then we need to invert it prior to use if key2 == 'ipc': ipc_path, ipc_file = os.path.split(mapping['ipc']) inverted_kernel_file = os.path.join( ipc_path, 'Kernel_to_add_IPC_effects_from_{}'.format(ipc_file)) # Check to see if the version of the IPC file with an # inverted kernel already exists. if os.path.isfile(inverted_kernel_file): print( "Found an existing inverted kernel for this IPC file: {}" .format(inverted_kernel_file)) params[key1][key2] = inverted_kernel_file params['Reffiles']['invertIPC'] = False else: print( "No inverted IPC kernel file found. Kernel will be inverted prior to use." ) params['Reffiles']['invertIPC'] = True return params