def test_core_schema(tmp_path): # Set temporary asdf file file_path = tmp_path / "test.asdf" wfi_image = utils.mk_level2_image(arrays=(10, 10)) with asdf.AsdfFile() as af: af.tree = {'roman': wfi_image} with pytest.raises(ValidationError): af.tree['roman'].meta.telescope = 'NOTROMAN' af.tree['roman'].meta['telescope'] = 'NOTROMAN' with pytest.raises(ValidationError): af.write_to(file_path) af.tree['roman'].meta.telescope = 'ROMAN' af.write_to(file_path) # Now mangle the file with open(file_path, 'rb') as fp: fcontents = fp.read() romanloc = fcontents.find(bytes('ROMAN', 'utf-8')) newcontents = fcontents[:romanloc] + \ bytes('X', 'utf-8') + fcontents[romanloc + 1:] with open(file_path, 'wb') as fp: fp.write(newcontents) with pytest.raises(ValidationError): with datamodels.open(file_path) as model: pass asdf.get_config().validate_on_read = False with datamodels.open(file_path) as model: assert model.meta.telescope == 'XOMAN' asdf.get_config().validate_on_read = True
def process(self, input): # Open the input data model with rdd.open(input) as input_model: # Get the name of the dark reference file to use self.dark_name = self.get_reference_file(input_model, 'dark') self.log.info('Using DARK reference file %s', self.dark_name) # Check for a valid reference file if self.dark_name == 'N/A': self.log.warning('No DARK reference file found') self.log.warning('Dark current step will be skipped') result = input_model.copy() result.meta.cal_step.dark = 'SKIPPED' return result # Create name for the intermediate dark, if desired. dark_output = self.dark_output if dark_output is not None: dark_output = self.make_output_path(None, basepath=dark_output, ignore_use_model=True) dark_model = rdd.open(self.dark_name) # Do the dark correction result = dark_sub.do_correction(input_model, dark_model, dark_output) dark_model.close() return result
def test_path_input(tmp_path): file_path = tmp_path / "test.asdf" with asdf.AsdfFile() as af: tree = utils.mk_level2_image() af.tree = {'roman': tree} af.write_to(file_path) # Test with PurePath input: with datamodels.open(file_path) as model: assert model.meta.telescope == "ROMAN" af = model._asdf # When open creates the file pointer, it should be # closed when the model is closed: assert af._closed # Test with string input: with datamodels.open(str(file_path)) as model: assert model.meta.telescope == "ROMAN" af = model._asdf assert af._closed # Appropriate error when file is missing: with pytest.raises(FileNotFoundError): with datamodels.open(tmp_path / "missing.asdf"): pass
def test_opening_linearity_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testlinearity.asdf' utils.mk_linearity(filepath=file_path) linearity = datamodels.open(file_path) assert linearity.meta.instrument.optical_element == 'F158' assert isinstance(linearity, datamodels.LinearityRefModel)
def test_opening_flat_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testflat.asdf' utils.mk_flat(file_path) flat = datamodels.open(file_path) assert flat.meta.instrument.optical_element == 'F158' assert isinstance(flat, datamodels.FlatRefModel)
def test_opening_superbias_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testsuperbias.asdf' utils.mk_superbias(filepath=file_path) superbias = datamodels.open(file_path) assert superbias.meta.instrument.optical_element == 'F158' assert isinstance(superbias, datamodels.SuperbiasRefModel)
def test_opening_pixelarea_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testpixelarea.asdf' utils.mk_pixelarea(filepath=file_path) pixelarea = datamodels.open(file_path) assert pixelarea.meta.instrument.optical_element == 'F158' assert isinstance(pixelarea, datamodels.PixelareaRefModel)
def test_opening_saturation_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testsaturation.asdf' utils.mk_saturation(filepath=file_path) saturation = datamodels.open(file_path) assert saturation.meta.instrument.optical_element == 'F158' assert isinstance(saturation, datamodels.SaturationRefModel)
def test_asdf_file_input(): tree = utils.mk_level2_image() with asdf.AsdfFile() as af: af.tree = {'roman': tree} model = datamodels.open(af) assert model.meta.telescope == 'ROMAN' model.close()
def test_opening_mask_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testmask.asdf' utils.mk_mask(filepath=file_path) mask = datamodels.open(file_path) assert mask.meta.instrument.optical_element == 'F158' assert isinstance(mask, datamodels.MaskRefModel)
def test_opening_dark_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testdark.asdf' utils.mk_dark(filepath=file_path) dark = datamodels.open(file_path) assert dark.meta.instrument.optical_element == 'F158' assert isinstance(dark, datamodels.DarkRefModel)
def test_opening_gain_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testgain.asdf' utils.mk_gain(filepath=file_path) gain = datamodels.open(file_path) assert gain.meta.instrument.optical_element == 'F158' assert isinstance(gain, datamodels.GainRefModel)
def test_opening_ramp_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testramp.asdf' utils.mk_ramp(filepath=file_path) ramp = datamodels.open(file_path) assert ramp.meta.instrument.optical_element == 'F062' assert isinstance(ramp, datamodels.RampModel)
def test_opening_rampfitoutput_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testrampfitoutput.asdf' utils.mk_rampfitoutput(filepath=file_path) rampfitoutput = datamodels.open(file_path) assert rampfitoutput.meta.instrument.optical_element == 'F062' assert isinstance(rampfitoutput, datamodels.RampFitOutputModel)
def test_flat_model(tmp_path): # Set temporary asdf file file_path = tmp_path / "test.asdf" meta = {} utils.add_ref_common(meta) meta['reftype'] = "FLAT" flatref = stnode.FlatRef() flatref['meta'] = meta flatref.meta.instrument['optical_element'] = 'F062' shape = (4096, 4096) flatref['data'] = np.zeros(shape, dtype=np.float32) flatref['dq'] = np.zeros(shape, dtype=np.uint16) flatref['dq_def'] = np.zeros(10, dtype=table_definitions.DQ_DEF_DTYPE) flatref['err'] = np.zeros(shape, dtype=np.float32) # Testing flat file asdf file with asdf.AsdfFile(meta) as af: af.tree = {'roman': flatref} af.write_to(file_path) # Test that asdf file opens properly with datamodels.open(file_path) as model: with pytest.warns(None): model.validate() # Confirm that asdf file is opened as flat file model assert isinstance(model, datamodels.FlatRefModel)
def test_opening_readnoise_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testreadnoise.asdf' utils.mk_readnoise(filepath=file_path) readnoise = datamodels.open(file_path) assert readnoise.meta.instrument.optical_element == 'F158' assert isinstance(readnoise, datamodels.ReadnoiseRefModel)
def test_add_model_attribute(tmp_path): # First make test reference file file_path = tmp_path / 'testreadnoise.asdf' utils.mk_readnoise(filepath=file_path) readnoise = datamodels.open(file_path) readnoise['new_attribute'] = 77 assert readnoise.new_attribute == 77 with pytest.raises(ValueError): readnoise['_underscore'] = 'bad' file_path2 = tmp_path / 'testreadnoise2.asdf' readnoise.save(file_path2) readnoise2 = datamodels.open(file_path2) assert readnoise2.new_attribute == 77 readnoise2.new_attribute = 88 assert readnoise2.new_attribute == 88 with pytest.raises(ValidationError): readnoise['data'] = 'bad_data_value'
def test_opening_level1_science_raw(tmp_path): # First make test reference file file_path = tmp_path / 'testwfi_science_raw.asdf' utils.mk_level1_science_raw(filepath=file_path) wfi_science_raw = datamodels.open(file_path) assert wfi_science_raw.meta.instrument.optical_element == 'F062' assert isinstance(wfi_science_raw, datamodels.ScienceRawModel)
def test_opening_level2_image(tmp_path): # First make test reference file file_path = tmp_path / 'testwfi_image.asdf' utils.mk_level2_image(filepath=file_path) wfi_image = datamodels.open(file_path) assert wfi_image.meta.instrument.optical_element == 'F062' assert isinstance(wfi_image, datamodels.ImageModel)
def process(self, input): # Open the input data model with rdd.open(input) as input_model: # Get the name of the dark reference file to use self.dark_name = self.get_reference_file(input_model, 'dark') self.log.info('Using DARK reference file %s', self.dark_name) # Open dark model dark_model = rdd.open(self.dark_name) # Temporary patch to utilize stcal dark step until MA table support is fully implemented if 'ngroups' not in dark_model.meta.exposure: dark_model.meta.exposure['ngroups'] = dark_model.data.shape[0] if 'nframes' not in dark_model.meta.exposure: dark_model.meta.exposure['nframes'] = input_model.meta.exposure.nframes if 'groupgap' not in dark_model.meta.exposure: dark_model.meta.exposure['groupgap'] = input_model.meta.exposure.groupgap # Reshaping data variables for stcal compatibility input_model.data = input_model.data.astype(np.float32)[np.newaxis, :] input_model.groupdq = input_model.groupdq[np.newaxis, :] input_model.err = input_model.err[np.newaxis, :] # Do the dark correction out_data, dark_data = dark_sub.do_correction( input_model, dark_model, self.dark_output ) # Save dark data to file if dark_data is not None and dark_data.save: save_dark_data_as_dark_model(dark_data, dark_model) dark_model.close() # Reshaping data variables back from stcal input_model.data = input_model.data[0] input_model.groupdq = input_model.groupdq[0] input_model.err = input_model.err[0] # Convert data to RampModel out_ramp = dark_output_data_as_ramp_model(out_data, input_model) dark_model.close() return out_ramp
def test_opening_wfi_img_photom_ref(tmp_path): # First make test reference file file_path = tmp_path / 'testwfi_img_photom.asdf' utils.mk_wfi_img_photom(filepath=file_path) wfi_img_photom = datamodels.open(file_path) assert wfi_img_photom.meta.instrument.optical_element == 'F158' assert isinstance(wfi_img_photom, datamodels.WfiImgPhotomRefModel)
def test_memmap(tmp_path): file_path = tmp_path / "test.asdf" with asdf.AsdfFile() as af: af.tree = {'roman': utils.mk_level2_image()} af.tree['roman'].data = np.zeros(( 400, 400, ), dtype=np.float32) af.write_to(file_path) with datamodels.open(file_path, memmap=True) as model: assert isinstance(model.data.base, np.memmap) with datamodels.open(file_path, memmap=False) as model: assert not isinstance(model.data.base, np.memmap) # Default should be false: with datamodels.open(file_path) as model: assert not isinstance(model.data.base, np.memmap)
def test_model_input(tmp_path): file_path = tmp_path / "test.asdf" data = np.random.uniform(size=(1024, 1024)).astype(np.float32) with asdf.AsdfFile() as af: af.tree = {'roman': utils.mk_level2_image()} af.tree['roman'].meta['bozo'] = 'clown' af.tree['roman'].data = data af.write_to(file_path) original_model = datamodels.open(file_path) reopened_model = datamodels.open(original_model) # It's essential that we get a new instance so that the original # model can be closed without impacting the new model. assert reopened_model is not original_model assert_array_equal(original_model.data, data) original_model.close() assert_array_equal(reopened_model.data, data) reopened_model.close()
def process(self, input): reference_file_names = {} with rdm.open(input) as input_model: for reftype in self.reference_file_types: log.info(f'reftype, {reftype}') reffile = self.get_reference_file(input_model, reftype) reference_file_names[reftype] = reffile if reffile else "" log.debug( f'reference files used in assign_wcs: {reference_file_names}') result = load_wcs(input_model, reference_file_names) return result
def test_datamodel_info_search(capsys): wfi_science_raw = utils.mk_level1_science_raw() af = asdf.AsdfFile() af.tree = {'roman': wfi_science_raw} dm = datamodels.open(af) dm.info(max_rows=200) captured = capsys.readouterr() assert "optical_element" in captured.out result = dm.search('optical_element') assert 'F062' in repr(result) assert result.node == 'F062'
def process(self, input): with rdd.open(input, mode='rw') as input_model: max_cores = self.maximum_cores readnoise_filename = self.get_reference_file( input_model, 'readnoise') gain_filename = self.get_reference_file(input_model, 'gain') input_model.data = input_model.data[np.newaxis, :] input_model.data.dtype = np.float32 input_model.groupdq = input_model.groupdq[np.newaxis, :] input_model.err = input_model.err[np.newaxis, :] log.info('Using READNOISE reference file: %s', readnoise_filename) readnoise_model = rdd.open(readnoise_filename, mode='rw') log.info('Using GAIN reference file: %s', gain_filename) gain_model = rdd.open(gain_filename, mode='rw') log.info('Using algorithm = %s' % self.algorithm) log.info('Using weighting = %s' % self.weighting) buffsize = ramp_fit.BUFSIZE image_info, integ_info, opt_info, gls_opt_model = ramp_fit.ramp_fit( input_model, buffsize, self.save_opt, readnoise_model.data, gain_model.data, self.algorithm, self.weighting, max_cores, dqflags.pixel) readnoise_model.close() gain_model.close() # Save the OLS optional fit product, if it exists if opt_info is not None: opt_model = create_optional_results_model(input_model, opt_info) self.save_model(opt_model, 'fitopt', output_file=self.opt_name) if image_info is not None: out_model = create_image_model(input_model, image_info) out_model.meta.cal_step.ramp_fit = 'COMPLETE' return out_model
def process(self, input): """ Process the Roman WFI data """ log.info('Starting Roman exposure calibration pipeline ...') if isinstance(input, str): input_filename = basename(input) # open the input file input = rdd.open(input) # propagate output_dir to steps that might need it # self.dark_current.output_dir = self.output_dir # self.ramp_fit.output_dir = self.output_dir log.debug('Exposure Processing a WFI exposure') self.dq_init.suffix = 'dq_init' result = self.dq_init(input) if input_filename: result.meta.filename = input_filename result = self.saturation(result) result = self.linearity(result) result = self.dark_current(result) result = self.jump(result) result = self.rampfit(result) result = self.assign_wcs(result) if result.meta.exposure.type == 'WFI_IMAGE': result = self.flatfield(result) else: log.info('Flat Field step is being SKIPPED') result.meta.cal_step.flat_field = 'SKIPPED' result = self.photom(result) # setup output_file for saving self.setup_output(result) log.info('Roman exposure calibration pipeline ending...') return result
def test_invalid_input(): with pytest.raises(TypeError): datamodels.open(fits.HDUList())
def process(self, input): # Open input as a Roman DataModel (single integration; 3D arrays) with rdd.open(input) as input_model: # Extract the needed info from the Roman Data Model meta = input_model.meta r_data = input_model.data r_gdq = input_model.groupdq r_pdq = input_model.pixeldq r_err = input_model.err result = input_model frames_per_group = meta.exposure.nframes # Modify the arrays for input into the 'common' jump (4D) data = np.copy(r_data[np.newaxis, :]) gdq = r_gdq[np.newaxis, :] pdq = r_pdq[np.newaxis, :] err = np.copy(r_err[np.newaxis, :]) tstart = time.time() # Check for an input model with NGROUPS<=2 ngroups = data.shape[1] if ngroups <= 2: self.log.warning('Cannot apply jump detection as NGROUPS<=2;') self.log.warning('Jump step will be skipped') result = input_model result.meta.cal_step.jump = 'SKIPPED' return result # Retrieve the parameter values rej_thresh = self.rejection_threshold three_grp_rej_thresh = self.three_group_rejection_threshold four_grp_rej_thresh = self.four_group_rejection_threshold max_cores = self.maximum_cores max_jump_to_flag_neighbors = self.max_jump_to_flag_neighbors min_jump_to_flag_neighbors = self.min_jump_to_flag_neighbors flag_4_neighbors = self.flag_4_neighbors self.log.info('CR rejection threshold = %g sigma', rej_thresh) if self.maximum_cores != 'none': self.log.info('Maximum cores to use = %s', max_cores) # Get the gain and readnoise reference files gain_filename = self.get_reference_file(input_model, 'gain') self.log.info('Using GAIN reference file: %s', gain_filename) gain_model = rdd.GainRefModel(gain_filename) gain_2d = gain_model.data readnoise_filename = self.get_reference_file(input_model, 'readnoise') self.log.info('Using READNOISE reference file: %s', readnoise_filename) readnoise_model = rdd.ReadnoiseRefModel(readnoise_filename) # This is to clear the WRITEABLE=False flag? readnoise_2d = np.copy(readnoise_model.data) # DG 0810/21: leave for now; make dqflags changes in a later, # separate PR dqflags_d = {} # Dict of DQ flags dqflags_d = { "GOOD": dqflags.group["GOOD"], "DO_NOT_USE": dqflags.group["DO_NOT_USE"], "SATURATED": dqflags.group["SATURATED"], "JUMP_DET": dqflags.group["JUMP_DET"], "NO_GAIN_VALUE": dqflags.pixel["NO_GAIN_VALUE"] } gdq, pdq = detect_jumps(frames_per_group, data, gdq, pdq, err, gain_2d, readnoise_2d, rej_thresh, three_grp_rej_thresh, four_grp_rej_thresh, max_cores, max_jump_to_flag_neighbors, min_jump_to_flag_neighbors, flag_4_neighbors, dqflags_d) gdq = gdq[0, :, :, :] pdq = pdq[0, :, :] result.groupdq = gdq result.pixeldq = pdq gain_model.close() readnoise_model.close() tstop = time.time() self.log.info('The execution time in seconds: %f', tstop - tstart) result.meta.cal_step.jump = 'COMPLETE' return result