def test_miri_lrs_slit_wcs(run_pipeline, rtdata_module, fitsdiff_default_kwargs): rtdata = rtdata_module # get input assign_wcs and truth file output = "jw00623032001_03102_00001_mirimage_assign_wcs.fits" rtdata.output = output rtdata.get_truth(f"truth/test_miri_lrs_slit_spec2/{output}") # Compare the output and truth file with datamodels.open(output) as im, datamodels.open( rtdata.truth) as im_truth: x, y = grid_from_bounding_box(im.meta.wcs.bounding_box) ra, dec, lam = im.meta.wcs(x, y) ratruth, dectruth, lamtruth = im_truth.meta.wcs(x, y) assert_allclose(ra, ratruth) assert_allclose(dec, dectruth) assert_allclose(lam, lamtruth) # Test the inverse transform xtest, ytest = im.meta.wcs.backward_transform(ra, dec, lam) xtruth, ytruth = im_truth.meta.wcs.backward_transform( ratruth, dectruth, lamtruth) assert_allclose(xtest, xtruth) assert_allclose(ytest, ytruth)
def test_create_combined(_jail, wfs_association, data1, data2, dq1, dq2, result_data, result_dq): path_asn, path1, path2 = wfs_association # Populate data and DQ array at [5, 5] with what is to be tested with datamodels.open(path1) as im1, datamodels.open(path2) as im2: # Populate some data and DQ flags for the test im1.data[5, 5] = data1 im2.data[5, 5] = data2 im1.dq[5, 5] = dq1 im2.dq[5, 5] = dq2 im1.save(path1) im2.save(path2) result = WfsCombineStep.call(path_asn, do_refine=False, flip_dithers=False, psf_size=50, blur_size=10, n_size=2) # Check that results are as expected assert result[0].data[5, 5] == result_data assert result[0].dq[5, 5] == result_dq
def test_open(): with datamodels.open() as dm: pass with datamodels.open((50, 50)) as dm: pass warnings.simplefilter("ignore") with datamodels.open(FITS_FILE) as dm: assert isinstance(dm, QuadModel)
def test_open(): with datamodels.open(): pass with datamodels.open((50, 50)): pass with pytest.warns(datamodels.util.NoTypeWarning): with datamodels.open(FITS_FILE) as dm: assert isinstance(dm, QuadModel)
def __init__(self, fitopt_filename, ramp_filename, rate_filename, xlim=[500, 530], ylim=[500, 530]): self.fitopt_file = os.path.basename(fitopt_filename) self.ramp_file = os.path.basename(ramp_filename) self.rate_file = os.path.basename(rate_filename) self.fitopt_dm = datamodels.RampFitOutputModel(fitopt_filename) self.ramp_dm = datamodels.open(ramp_filename) self.rate_dm = datamodels.open(rate_filename) self.xlim = xlim self.ylim = ylim
def test_nirspec_ifu_user_supplied_flat(rtdata, fitsdiff_default_kwargs): """Test using predefined interpolated flat""" data = dm.open(rtdata.get_data('nirspec/ifu/nrs_ifu_nrs1_assign_wcs.fits')) user_supplied_flat = dm.open(rtdata.get_data('nirspec/ifu/nrs_ifu_nrs1_interpolated_flat.fits')) nirspec_ifu(data, None, None, None, None, user_supplied_flat=user_supplied_flat) rtdata.output = 'ff_using_interpolated.fits' data.write(rtdata.output) rtdata.get_truth(TRUTH_PATH + '/' + 'ff_using_interpolated.fits') diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_modelcontainer_error_from_asn(tmpdir): from jwst.associations.asn_from_list import asn_from_list asn = asn_from_list(["foo.fits"], product_name="foo_out") name, serialized = asn.dump(format="json") path = str(tmpdir.join(name)) with open(path, "w") as f: f.write(serialized) # The foo.fits file doesn't exist with pytest.raises(FileNotFoundError): datamodels.open(path)
def test_refine_no_error(wfs_association, xshift, yshift, xerror, yerror, flip_dithers): data_size = 201 path_asn, path1, path2 = wfs_association nircam_pixel_size = 0.031 / 3600. delta_x_pixel = xshift # positive y pixel changes are negative declination changes delta_y_pixel = -1.0 * yshift with datamodels.open(path1) as im1: im1.data = np.zeros(shape=(data_size, data_size), dtype=np.float32) im1.dq = np.zeros(shape=(data_size, data_size), dtype=np.int32) im1.data = add_point_source(im1.data, 200, 100, 100, 4, 4) im1.save(path1) with datamodels.open(path2) as im2: im2.data = np.zeros(shape=(data_size, data_size), dtype=np.float32) im2.dq = np.zeros(shape=(data_size, data_size), dtype=np.int32) im2.meta.wcsinfo = { 'dec_ref': 11.99875540218638 + delta_y_pixel * nircam_pixel_size, 'ra_ref': 22.02351763251896 + delta_x_pixel * nircam_pixel_size, 'roll_ref': 0.005076934167039675, 'v2_ref': 86.039011, 'v3_ref': -493.385704, 'v3yangle': -0.07385127, 'vparity': -1, 'wcsaxes': 2 } im2 = AssignWcsStep.call(im2, sip_approx=False) # shift the actual location of the 2nd image including the error in the WCS position im2.data = add_point_source(im2.data, 200, 100 + delta_x_pixel + xerror, 100 + delta_y_pixel - yerror, 4, 4) im2.save(path2) wfs = wfs_combine.DataSet(path1, path2, 'outfile.fits', do_refine=True, flip_dithers=flip_dithers, psf_size=50, blur_size=10, n_size=2) wfs.do_all() assert np.max(abs(wfs.diff)) < 0.001 if delta_x_pixel + xerror >= 0 or not flip_dithers: assert wfs.off_x == delta_x_pixel + xerror # Positive y pixel errors are negative in declination assert wfs.off_y == delta_y_pixel - yerror else: # Order of exposures and sign of shifts have switched to always align the dithers assert wfs.off_x == abs(delta_x_pixel + xerror) # Positive y pixel errors are negative in declination assert wfs.off_y == -1.0 * (delta_y_pixel - yerror) wfs.input_1.close() wfs.input_2.close()
def test_modelcontainer_error_from_asn(tmp_path): asn = asn_from_list(["foo.fits"], product_name="foo_out") name, serialized = asn.dump(format="json") # The following Path object needs to be stringified because # datamodels.open() doesn't deal with pathlib objects when they are # .json files path = str(tmp_path / name) with open(path, "w") as f: f.write(serialized) # The foo.fits file doesn't exist with pytest.raises(FileNotFoundError): datamodels.open(path)
def process(self, *args): science = datamodels.open(self.science_filename) if self.flat_filename is None: self.flat_filename = join(dirname(__file__), "data/flat.fits") flat = datamodels.open(self.flat_filename) calibrated = [] calibrated.append(self.flat_field(science, flat)) combined = self.combine(calibrated) self.display(combined) dm = datamodels.ImageModel(combined) dm.save(self.output_filename) science.close() flat.close() return dm
def test_guess(guess): """Test the guess parameter to the open func""" path = Path(t_path('test.fits')) with warnings.catch_warnings(): warnings.filterwarnings("ignore", "model_type not found") if guess is None or guess: # Default is to guess at the model type. with datamodels.open(path, guess=guess) as model: assert isinstance(model, JwstDataModel) else: # Without guessing, the call should fail. with pytest.raises(TypeError): with datamodels.open(path, guess=guess) as model: pass
def test_pre_1_2_2_schemas(truth_path, rtdata): rtdata.env = "1.2.1" rtdata.get_truth(truth_path) # Confirm that the file doesn't initially validate # (open with fits first so that the failed call to open doesn't leave behind an open file) with fits.open(rtdata.truth, memmap=False) as hdu: with pytest.raises(ValueError, match="Column names don't match schema"): with datamodels.open(hdu): pass subprocess.check_call(["migrate_data", rtdata.truth, "--in-place"]) # Now the model should validate with datamodels.open(rtdata.truth): pass
def test_nircam_image_stage2_wcs(run_image2pipeline, rtdata_module): """Test that WCS object works as expected""" rtdata = rtdata_module rtdata.input = "jw42424001001_01101_00001_nrca5_uncal.fits" output = "jw42424001001_01101_00001_nrca5_assign_wcs.fits" rtdata.output = output rtdata.get_truth(f"truth/test_nircam_image_stages/{output}") with datamodels.open(rtdata.output) as model, datamodels.open(rtdata.truth) as model_truth: grid = grid_from_bounding_box(model.meta.wcs.bounding_box) ra, dec = model.meta.wcs(*grid) ra_truth, dec_truth = model_truth.meta.wcs(*grid) assert_allclose(ra, ra_truth) assert_allclose(dec, dec_truth)
def test_shift_order_no_refine_with_flip(wfs_association): # change the sign of the the dither and check that we get the same delta pix but with switched # order of images. path_asn, path1, path2 = wfs_association nircam_pixel_size = 0.031 / 3600. delta_pixel = -5 with datamodels.open(path2) as im2: im2.meta.wcsinfo = { 'dec_ref': 11.99875540218638, 'ra_ref': 22.02351763251896 + delta_pixel * nircam_pixel_size, 'roll_ref': 0.005076934167039675, 'v2_ref': 86.039011, 'v3_ref': -493.385704, 'v3yangle': -0.07385127, 'vparity': -1, 'wcsaxes': 2 } im2 = AssignWcsStep.call(im2, sip_approx=False) im2.save(path2) wfs = wfs_combine.DataSet(path1, path2, 'outfile.fits', do_refine=False, flip_dithers=True, psf_size=50, blur_size=10, n_size=2) wfs.do_all() wfs.input_1.close() wfs.input_2.close() # assert wfs.input_1.meta.observation.exposure_number == '2' # assert wfs.input_2.meta.observation.exposure_number == '1' assert wfs.off_x == -1 * delta_pixel
def _dm_setval(filepath, key, value): """Set metadata `key` in file `filepath` to `value` using jwst datamodel. """ from jwst import datamodels with datamodels.open(filepath) as d_model: d_model[key.lower()] = value d_model.save(filepath)
def test_step_neg_shift_no_refine_no_flip(wfs_association): path_asn, path1, path2 = wfs_association nircam_pixel_size = 0.031 / 3600. delta_pixel = -5 with datamodels.open(path2) as im2: im2.meta.wcsinfo = { 'dec_ref': 11.99875540218638, 'ra_ref': 22.02351763251896 + delta_pixel * nircam_pixel_size, 'roll_ref': 0.005076934167039675, 'v2_ref': 86.039011, 'v3_ref': -493.385704, 'v3yangle': -0.07385127, 'vparity': -1, 'wcsaxes': 2 } im2 = AssignWcsStep.call(im2, sip_approx=False) im2.save(path2) wfs = WfsCombineStep.call(path_asn, do_refine=False, flip_dithers=False, psf_size=50, blur_size=10, n_size=2) assert wfs[0].meta.wcsinfo.ra_ref == 22.02351763251896
def test_add_wcs_default(data_file, tmp_path): """Handle when no pointing exists and the default is used.""" expected_name = 'add_wcs_default.fits' try: stp.add_wcs(data_file, siaf_path=siaf_path, tolerance=0, allow_default=True) except ValueError: pass # This is what we want for the test. except Exception as e: pytest.skip('Live ENGDB service is not accessible.' '\nException={}'.format(e)) # Tests with datamodels.Level1bModel(data_file) as model: # Save for post-test comparison and update model.save(tmp_path / expected_name) with datamodels.open(DATA_PATH / expected_name) as expected: for meta in METAS_EQUALITY: assert model[meta] == expected[meta], f'{meta} has changed' for meta in METAS_ISCLOSE: assert np.isclose(model[meta], expected[meta]), f'{meta} has changed' assert word_precision_check(model.meta.wcsinfo.s_region, expected.meta.wcsinfo.s_region)
def get_reference_file(dataset, reference_file_type): """ Gets a reference file from CRDS as a readable file-like object. The actual file may be optionally overridden. Parameters ---------- input_file : jwst.datamodels.ModelBase instance A model of the input file. Metadata on this input file will be used by the CRDS "bestref" algorithm to obtain a reference file. reference_file_type : string The type of reference file to retrieve. For example, to retrieve a flat field reference file, this would be 'flat'. Returns ------- reference_filepath : string The path of the reference in the CRDS file cache. """ if isinstance(dataset, str): from jwst import datamodels with datamodels.open(dataset) as model: return get_multiple_reference_paths(model, [reference_file_type])[reference_file_type] else: return get_multiple_reference_paths(dataset, [reference_file_type])[reference_file_type]
def test_from_hdulist(): from astropy.io import fits warnings.simplefilter("ignore") with fits.open(FITS_FILE, memmap=False) as hdulist: with datamodels.open(hdulist) as dm: dm.data assert not hdulist.fileinfo(0)['file'].closed
def test_input_dir_with_model(mk_tmp_dirs): """Use with an already opened DataModel""" with datamodels.open(t_path('data/flat.fits')) as model: step = StepWithModel() step.run(model) assert step.input_dir == ''
def process(self, input, bkg_list): """ Subtract the background signal from target exposures by subtracting designated background images from them. Parameters ---------- input: data model input target data model to which background subtraction is applied bkg_list: filename list list of background exposure file names Returns ------- result: data model the background-subtracted target data model """ # Load the input data model with datamodels.open(input) as input_model: # Do the background subtraction result = background_sub.background_sub(input_model, bkg_list, self.sigma, self.maxiters) result.meta.cal_step.back_sub = "COMPLETE" return result
def __init__(self, input_DM): """ Short Summary ------------- Set file name of persistence file Parameters ---------- input_DM: data model object input Data Model object """ try: self.input_file = input_DM model = datamodels.open(input_DM) # If model comes back as generic DataModel, reopen as MultiSlit if isinstance(model, datamodels.CubeModel) or isinstance( model, datamodels.ImageModel): pass elif isinstance(model, datamodels.DataModel): model.close() model = datamodels.MultiSlitModel(input_DM) self.input = model except Exception as errmess: log.error('Error opening %s', input_DM) self.input = None
def test_add_wcs_method_full_siafdb(eng_db_ngas, data_file, tmp_path): """Test using the database and a specified siaf db""" expected_name = 'add_wcs_method_full_siafdb.fits' # Calculate stp.add_wcs(data_file, siaf_path=siaf_path, method=stp.Methods.OPS_TR_202111, engdb_url='http://localhost') # Test with datamodels.Level1bModel(data_file) as model: # Save for post-test comparison and update model.save(tmp_path / expected_name) with datamodels.open(DATA_PATH / expected_name) as expected: for meta in METAS_EQUALITY: if isinstance(model[meta], str): assert model[meta] == expected[meta] else: assert np.isclose(model[meta], expected[meta], atol=1e-13) for meta in METAS_ISCLOSE: assert np.isclose(model[meta], expected[meta]) assert word_precision_check(model.meta.wcsinfo.s_region, expected.meta.wcsinfo.s_region)
def test_add_wcs_method_full_nosiafdb(eng_db_ngas, data_file, tmp_path): """Test using the database""" # Only run if `pysiaf` is installed. pytest.importorskip('pysiaf') expected_name = 'add_wcs_method_full_nosiafdb.fits' # Calculate stp.add_wcs(data_file, method=stp.Methods.OPS_TR_202111, engdb_url='http://localhost') # Tests with datamodels.Level1bModel(data_file) as model: # Save for post-test comparison and update model.save(tmp_path / expected_name) with datamodels.open(DATA_PATH / expected_name) as expected: for meta in METAS_EQUALITY: assert model[meta] == expected[meta] for meta in METAS_ISCLOSE: assert np.isclose(model[meta], expected[meta]) assert word_precision_check(model.meta.wcsinfo.s_region, expected.meta.wcsinfo.s_region)
def process(self, input): """ Step for Nonlinearity Correction """ # Load the input data model with datamodels.open(input) as input_model: nonlin_coeff_file = self.get_reference_file( input_model, "nonlincoeff") nonlin_coeff_data = fits.getdata(nonlin_coeff_file).astype( np.float32) c0 = nonlin_coeff_data[0] c1 = nonlin_coeff_data[1] c2 = nonlin_coeff_data[2] c3 = nonlin_coeff_data[3] c4 = nonlin_coeff_data[4] # Get the reference file names input_data = input_model.data[:, :, :, :].astype(np.int32) ramp_list = [] for it in range(len(input_data)): ramp_data = input_data[it] num_reads = len(ramp_data) time_arr = np.arange(0, num_reads, 1).astype(np.int32) result = nonlin_c(ramp_data, time_arr, c0, c1, c2, c3, c4) result = result.astype(np.int32) ramp_list.append(result) #result = uptheramp_c(result,time_arr) output_data = np.array(ramp_list).astype(np.int32) result = iris_pipeline.datamodels.TMTRampModel(data=output_data) result.update(input_model) return result
def _collect_wcs_keywords(f): # Get keywords to go in SCI header from the datamodels schema dm = datamodels.open(f, pass_invalid_values=True) stsci_wcs_kw = [] for k, v in _build_schema2fits_dict(dm.meta._schema).items(): if v[1] == 'SCI': if v[0] != 'WCSAXES': stsci_wcs_kw.append(v[0]) dm.close() # Convert the WCS to a header try: w = wcs.WCS(f[0].header) except InvalidTransformError: if f[0].header['cunit3'] and f[0].header['cunit3'] == 'micron': f[0].header['cunit3'] = 'um' w = wcs.WCS(f[0].header) new_hdr = w.to_header() # Remove kw added by wcslib from the new header for kw in wcslib_kw_to_remove: try: del new_hdr[kw] except KeyError: continue # Add STScI specific kw to new_hdr for kw in stsci_wcs_kw: try: new_hdr[kw] = f[0].header[kw] except KeyError: pass new_hdr = add_default_keywords(new_hdr) return new_hdr
def get_data_model_flat_dict(filepath, needed_keys=()): """Get the header from `filepath` using the jwst data model.""" from jwst import datamodels with log.augment_exception("JWST Data Model (jwst.datamodels)"): with datamodels.open(filepath) as d_model: flat_dict = d_model.to_flat_dict(include_arrays=False) return flat_dict
def test_saturation_step(fits_input): """Make sure the DQInitStep runs without error.""" fname = fits_input[0].header['filename'].replace('.fits', '_saturationstep.fits') SaturationStep.call(datamodels.open(fits_input), output_file=fname, save_results=True)
def test_nirspec_gwa(_jail, background, science_image): """Verify NIRSPEC GWA logic for in the science and background""" # open the background to read in the GWA values back_image = datamodels.open(background) science_image.meta.instrument.gwa_xtilt = back_image.meta.instrument.gwa_xtilt science_image.meta.instrument.gwa_ytilt = back_image.meta.instrument.gwa_ytilt science_image.meta.instrument.gwa_tilt = back_image.meta.instrument.gwa_tilt bkg = [background] # Test Run with GWA values the same - confirm it runs # And gives the predicted result collect_pipeline_cfgs('./config') result = BackgroundStep.call( science_image, bkg, config_file='config/background.cfg', ) test = science_image.data - back_image.data assert_allclose(result.data, test) assert type(result) is type(science_image) assert result.meta.cal_step.back_sub == 'COMPLETE' back_image.close()
def test_lastframe_step(fits_input): """Make sure the LastFrameStep runs without error.""" fname = fits_input[0].header['filename'].split('.fits', '_lastframestep.fits') LastFrameStep.call(datamodels.open(fits_input), output_file=fname, save_results=True)
def _collect_wcs_keywords(f): # Get keywords to go in SCI header from the datamodels schema dm = datamodels.open(f, pass_invalid_values=True) stsci_wcs_kw = [] for k,v in datamodels.schema.build_schema2fits_dict(dm.meta._schema).items(): if v[1] == 'SCI': if v[0] != 'WCSAXES': stsci_wcs_kw.append(v[0]) dm.close() # Convert the WCS to a header try: w = wcs.WCS(f[0].header) except InvalidTransformError: if f[0].header['cunit3'] and f[0].header['cunit3'] == 'micron': f[0].header['cunit3'] = 'um' w = wcs.WCS(f[0].header) new_hdr = w.to_header() # Remove kw added by wcslib from the new header for kw in wcslib_kw_to_remove: try: del new_hdr[kw] except KeyError: continue # Add STScI specific kw to new_hdr for kw in stsci_wcs_kw: try: new_hdr[kw] = f[0].header[kw] except KeyError: pass new_hdr = add_default_keywords(new_hdr) return new_hdr
def replace_ints(self, dir, int_dirs): """ replace the integrations in the full array with the int arrays """ my_output_files = glob.glob(os.path.join(dir, 'det_images', '*.fits')) for f in my_output_files: out_dm = datamodels.open(f) for n, i in enumerate(int_dirs): my_int_file = glob.glob(os.path.join(i, 'det_images', '*.fits'))[0] int_dm = datamodels.open(my_int_file) out_dm.data[n] = int_dm.data out_dm.save(f)
def test_ff_inv(rtdata, fitsdiff_default_kwargs): """Test flat field inversion""" with dm.open(rtdata.get_data('nirspec/imaging/usf_assign_wcs.fits')) as data: flatted = FlatFieldStep.call(data) unflatted = FlatFieldStep.call(flatted, inverse=True) assert np.allclose(data.data, unflatted.data), 'Inversion failed'
def test_open_fits(): """Test opening a model from a FITS file""" with warnings.catch_warnings(): warnings.filterwarnings("ignore", "model_type not found") fits_file = t_path('test.fits') with datamodels.open(fits_file) as model: assert isinstance(model, JwstDataModel)
def create_dms(base_file, level="1b", parfile=None, subarray=None, exp_type='UNKNOWN'): # # Create a Level 1b file from a full-frame base file # base_file is the full-format image the subarray is created from # subarray is a tuple describing the subarray: # (nxstart, nxstop, nystart, nystop, name) # base_root = base_file.split('.')[0] base_hdulist = fits.open(base_file) # # Add the exp_type to the input hdulist base_hdulist[0].header['EXP_TYPE'] = exp_type # # If we've already rotated/flipped the file, it will have a keyword set # and we skip this step # if not rotated_and_flipped(base_hdulist): print('Rotating and flipping data %s' % (base_file)) flip_rotate(base_hdulist) instrument = base_hdulist[0].header['INSTRUME'] if subarray is not None: subarray_file = create_single_subarray(base_hdulist, subarray) print('Parfile = %s' % parfile) if parfile is None: parfile = ''.join((base_root, '_', subarray[4].lower(), '_observation_identifiers.dat')) print(parfile) hdulist = main.generate([subarray_file, parfile], level=level) filename = main.guess_filename(hdulist) hdulist[0].header['FILENAME'] = filename hdulist.writeto(filename, output_verify='silentfix') os.remove(subarray_file) else: if instrument == 'MIRI': split_data_and_refout(base_hdulist) hdulist = main.generate([base_hdulist, parfile], level=level) filename = main.guess_filename(hdulist) hdulist[0].header['FILENAME'] = filename hdulist.writeto(filename, output_verify='silentfix') base_hdulist.close() # # Try and open the file as a JWST datamodel a = datamodels.open(filename) print("%s opened as a %s" % (filename, a.__module__)) a.close() return
def test_miri_lrs_masterbg_user(self): """ Regression test of masterbackgound subtraction with lrs, with user provided 1-D background """ # input file has the background added input_file = self.get_data(*self.test_dir, 'miri_lrs_sci+bkg_cal.fits') # user provided 1-D background user_background = self.get_data(*self.test_dir, 'miri_lrs_bkg_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=user_background, save_results=True) # Compare result (background subtracted image) to science image with no # background. Subtract these images, smooth the subtracted image and # the mean should be close to zero. input_sci_cal_file = self.get_data(*self.test_dir, 'miri_lrs_sci_cal.fits') input_sci = datamodels.open(input_sci_cal_file) # find the LRS region bb = result.meta.wcs.bounding_box x, y = grid_from_bounding_box(bb) result_lrs_region = result.data[y.astype(int), x.astype(int)] sci_lrs_region = input_sci.data[y.astype(int), x.astype(int)] # do a 5 sigma clip on the science image sci_mean = np.nanmean(sci_lrs_region) sci_std = np.nanstd(sci_lrs_region) upper = sci_mean + sci_std*5.0 lower = sci_mean - sci_std*5.0 mask_clean = np.logical_and(sci_lrs_region < upper, sci_lrs_region > lower) sub = result_lrs_region - sci_lrs_region mean_sub = np.absolute(np.mean(sub[mask_clean])) atol = 0.1 rtol = 0.001 assert_allclose(mean_sub, 0, atol=atol, rtol=rtol) # Test 3 Compare background subtracted science data (results) # to a truth file. truth_file = self.get_data(*self.ref_loc, 'miri_lrs_sci+bkg_masterbackgroundstep.fits') result_file = result.meta.filename outputs = [(result_file, truth_file)] self.compare_outputs(outputs) result.close() input_sci.close()
def test_open_asdf_readonly(tmpdir): tmpfile = str(tmpdir.join('readonly.asdf')) with DistortionModel() as model: model.meta.telescope = 'JWST' model.meta.instrument.name = 'NIRCAM' model.meta.instrument.detector = 'NRCA4' model.meta.instrument.channel = 'SHORT' model.save(tmpfile) os.chmod(tmpfile, 0o440) assert os.access(tmpfile, os.W_OK) == False with datamodels.open(tmpfile) as model: assert model.meta.telescope == 'JWST'
def test_open_fits_readonly(tmpdir): """Test opening a FITS-format datamodel that is read-only on disk""" tmpfile = str(tmpdir.join('readonly.fits')) data = np.arange(100, dtype=np.float).reshape(10, 10) with ImageModel(data=data) as model: model.meta.telescope = 'JWST' model.meta.instrument.name = 'NIRCAM' model.meta.instrument.detector = 'NRCA4' model.meta.instrument.channel = 'SHORT' model.save(tmpfile) os.chmod(tmpfile, 0o440) assert os.access(tmpfile, os.W_OK) == False with datamodels.open(tmpfile) as model: assert model.meta.telescope == 'JWST'
def process(self, input): # Open the input data model with datamodels.open(input) as input_model: # check the data is MIRI data detector = input_model.meta.instrument.detector if detector.startswith('MIR'): # Get the name of the rscd reference file to use self.rscd_name = self.get_reference_file(input_model, 'rscd') self.log.info('Using RSCD reference file %s', self.rscd_name) # Check for a valid reference file if self.rscd_name == 'N/A': self.log.warning('No RSCD reference file found') self.log.warning('RSCD step will be skipped') result = input_model.copy() result.meta.cal_step.rscd = 'SKIPPED' return result # Open the rscd ref file data model print('RSCD reference file name',self.rscd_name) #rscd_model = datamodels.RSCDModel(self.rscd_name) rscd_model = rscd_mod.RSCD_Model(self.rscd_name) # Do the rscd correction subtraction result = rscd_sub.do_correction(input_model, rscd_model) # Close the reference file and update the step status rscd_model.close() result.meta.cal_step.rscd = 'COMPLETE' else: self.log.warning('RSCD Correction is only for MIRI data') self.log.warning('RSCD step will be skipped') result = input_model.copy() result.meta.cal_step.rscd = 'SKIPPED' return result
def process(self, input): # Open the input data model with datamodels.open(input) as input_model: # check the data is MIRI data detector = input_model.meta.instrument.detector if detector[:3] == 'MIR': # Get the name of the reset reference file to use self.filename = self.get_reference_file(input_model, 'lastframe') self.log.info('Using Last Frame reference file %s', self.filename) # Check for a valid reference file if self.filename == 'N/A': self.log.warning('No Last Frame reference file found') self.log.warning('Last frame step will be skipped') result = input_model.copy() result.meta.cal_step.lastframe = 'SKIPPED' return result # Open the last frame ref file data model lastframe_model = datamodels.LastFrameModel(self.filename) # Do the lastframe correction subtraction result = lastframe_sub.do_correction(input_model, lastframe_model) # Close the reference file and update the step status lastframe_model.close() result.meta.cal_step.lastframe = 'COMPLETE' else: self.log.warning('Last Frame Correction is only for MIRI data') self.log.warning('Last frame step will be skipped') result = input_model.copy() result.meta.cal_step.lastframe = 'SKIPPED' return result
def test_nirspec_masterbackground_mos_user1d(self): """ Regression test of master background subtraction for NRS MOS when a user 1-D spectrum is provided. """ # input file has 2-D background image added to it input_file = self.get_data(*self.test_dir, 'nrs_mos_sci+bkg_cal.fits') # user provide 1-D background was created from the 2-D background image input_1dbkg_file = self.get_data(*self.test_dir, 'nrs_mos_bkg_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=input_1dbkg_file, save_results=True) # _________________________________________________________________________ # One of out tests is to compare the 1-D extracted spectra from # the science image (no background added) and the masterbackground subtracted # data. # run resample_spec on results from MasterBackground step result_2d = ResampleSpecStep.call(result) # run 1-D extract on results from MasterBackground step result_1d = Extract1dStep.call(result_2d) # get input science data with background added input_sci_cal_file = self.get_data(*self.test_dir, 'nrs_mos_sci_cal.fits') # get 1-D extract on original science data without background # this reference data was also run through ResampleSpec input_sci_1d_file = self.get_data(*self.ref_loc, 'nrs_mos_sci_extract1dstep.fits') input_sci = datamodels.open(input_sci_cal_file) sci_cal_1d = datamodels.open(input_sci_1d_file) num_spec = len(sci_cal_1d.spec) # the user 1D spectrum may not cover the entire wavelength range of the # science data. Find the wavelength range of user 1-D spectra input_1dbkg_1d = datamodels.open(input_1dbkg_file) user_wave = input_1dbkg_1d.spec[0].spec_table['wavelength'] user_flux = input_1dbkg_1d.spec[0].spec_table['flux'] user_wave_valid = np.where(user_flux > 0) min_user_wave = np.amin(user_wave[user_wave_valid]) max_user_wave = np.amax(user_wave[user_wave_valid]) input_1dbkg_1d.close() # loop over each slit and perform 2 tests on each slit for i in range(num_spec): # ______________________________________________________________________ # Test 1 compare extracted spectra data from the science data # to extracted spectra from the output # from MasterBackground subtraction. sci_spec_1d = sci_cal_1d.spec[i].spec_table['flux'] sci_wave = sci_cal_1d.spec[i].spec_table['wavelength'] result_spec_1d = result_1d.spec[i].spec_table['flux'] # find the waverange covered by both user 1-D and science slit sci_wave_valid = np.where(sci_spec_1d > 0) min_wave = np.amin(sci_wave[sci_wave_valid]) max_wave = np.amax(sci_wave[sci_wave_valid]) if min_user_wave > min_wave: min_wave = min_user_wave if max_user_wave < max_wave: max_wave = max_user_wave sub_spec = sci_spec_1d - result_spec_1d valid = np.where(np.logical_and(sci_wave > min_wave, sci_wave < max_wave)) sub_spec = sub_spec[valid] mean_sub = np.nanmean(sub_spec) atol = 1.5 assert_allclose(mean_sub, 0, atol=atol) # ______________________________________________________________________ # Test 2 compare the science data with no background # to the output from the masterBackground Subtraction step # background subtracted science image. bb = input_sci.slits[i].meta.wcs.bounding_box x, y = grid_from_bounding_box(bb) ra, dec, lam = input_sci.slits[i].meta.wcs(x, y) valid = np.isfinite(lam) sci = input_sci.slits[i].data result_slit = result.slits[i].data # check for outliers in the science image that could cause test # to fail. These could be cosmic ray hits or other yeck that # messes up the science data - ignores these cases sci_mean = np.nanmean(sci[valid]) sci_std = np.nanstd(sci[valid]) upper = sci_mean + sci_std*5.0 lower = sci_mean - sci_std*5.0 mask_clean = np.logical_and(sci[valid] < upper, sci[valid] > lower) # for this slit subtract the background subtracted data from # the science data with no background added sub = result_slit - sci # do not look at outliers - they confuse things sub_valid = sub[valid] mean_sub = np.mean(sub_valid[mask_clean]) atol = 0.1 assert_allclose(np.absolute(mean_sub), 0, atol=atol) # ______________________________________________________________________ # Test 3 Compare background sutracted science data (results) # to a truth file. This data is MultiSlit data result_file = result.meta.filename ref_file = self.get_data(*self.ref_loc, 'nrs_mos_sci+bkg_masterbackgroundstep.fits') outputs = [(result_file, ref_file)] self.compare_outputs(outputs) input_sci.close() result.close()
def test_nirspec_ifu_masterbg_user(self): """ Regression test of master background subtraction for NRS IFU when a user 1-D spectrum is provided. """ # input file has 2-D background image added to it input_file = self.get_data(*self.test_dir, 'prism_sci_bkg_cal.fits') # user-provided 1-D background was created from the 2-D background image user_background = self.get_data(*self.test_dir, 'prism_bkg_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=user_background, save_results=True) # Test 1 compare extracted spectra data with # no background added to extracted spectra from the output # from MasterBackground subtraction. First cube_build has to be run # on the data. result_s3d = CubeBuildStep.call(result) # run 1-D extract on results from MasterBackground step result_1d = Extract1dStep.call(result_s3d, subtract_background=False) # get the 1-D extracted spectrum from the science data in truth directory input_sci_1d_file = self.get_data(*self.ref_loc, 'prism_sci_extract1d.fits') sci_1d = datamodels.open(input_sci_1d_file) # read in the valid wavelengths of the user-1d user_background_model = datamodels.open(user_background) user_wave = user_background_model.spec[0].spec_table['wavelength'] user_flux = user_background_model.spec[0].spec_table['net'] user_wave_valid = np.where(user_flux > 0) min_user_wave = np.amin(user_wave[user_wave_valid]) max_user_wave = np.amax(user_wave[user_wave_valid]) user_background_model.close() # find the waverange covered by both user and science sci_spec_1d = sci_1d.spec[0].spec_table['net'] sci_spec_wave = sci_1d.spec[0].spec_table['wavelength'] result_spec_1d = result_1d.spec[0].spec_table['net'] sci_wave_valid = np.where(sci_spec_1d > 0) min_wave = np.amin(sci_spec_wave[sci_wave_valid]) max_wave = np.amax(sci_spec_wave[sci_wave_valid]) if min_user_wave > min_wave: min_wave = min_user_wave if max_user_wave < max_wave: max_wave = max_user_wave sub_spec = sci_spec_1d - result_spec_1d valid = np.where(np.logical_and(sci_spec_wave > min_wave, sci_spec_wave < max_wave)) sub_spec = sub_spec[valid] sub_spec = sub_spec[1:-2] # endpoints are wacky mean_sub = np.absolute(np.nanmean(sub_spec)) atol = 5.0 assert_allclose(mean_sub, 0, atol=atol) # Test 2 compare the science data with no background # to the output from the masterBackground Subtraction step # background subtracted science image. input_sci_cal_file = self.get_data(*self.test_dir, 'prism_sci_cal.fits') input_sci_model = datamodels.open(input_sci_cal_file) # We don't want the slices gaps to impact the statisitic # loop over the 30 Slices for i in range(30): slice_wcs = nirspec.nrs_wcs_set_input(input_sci_model, i) x, y = grid_from_bounding_box(slice_wcs.bounding_box) ra, dec, lam = slice_wcs(x, y) valid = np.isfinite(lam) result_slice_region = result.data[y.astype(int), x.astype(int)] sci_slice_region = input_sci_model.data[y.astype(int), x.astype(int)] sci_slice = sci_slice_region[valid] result_slice = result_slice_region[valid] sub = result_slice - sci_slice # check for outliers in the science image sci_mean = np.nanmean(sci_slice) sci_std = np.nanstd(sci_slice) upper = sci_mean + sci_std*5.0 lower = sci_mean - sci_std*5.0 mask_clean = np.logical_and(sci_slice < upper, sci_slice > lower) sub_mean = np.absolute(np.nanmean(sub[mask_clean])) atol = 2.0 assert_allclose(sub_mean, 0, atol=atol) # Test 3 Compare background sutracted science data (results) # to a truth file. This data is MultiSlit data input_sci_model.close() result_file = result.meta.filename truth_file = self.get_data(*self.ref_loc, 'prism_sci_bkg_masterbackgroundstep.fits') outputs = [(result_file, truth_file)] self.compare_outputs(outputs) input_sci_model.close() result.close()