def test_master_background_userbg(_jail, user_background): """Verify data can run through the step with a user-supplied background""" image = datamodels.ImageModel((10, 10)) image.meta.instrument.name = 'MIRI' image.meta.instrument.detector = 'MIRIMAGE' image.meta.exposure.type = 'MIR_LRS-FIXEDSLIT' image.meta.observation.date = '2018-01-01' image.meta.observation.time = '00:00:00' image.meta.subarray.xstart = 1 image.meta.subarray.ystart = 1 image.meta.wcsinfo.v2_ref = 0 image.meta.wcsinfo.v3_ref = 0 image.meta.wcsinfo.roll_ref = 0 image.meta.wcsinfo.ra_ref = 0 image.meta.wcsinfo.dec_ref = 0 image = AssignWcsStep.call(image) # Run with a user-supplied background and verify this is recorded in header result = MasterBackgroundStep.call(image, user_background=user_background) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( image, config_file='config/master_background.cfg', user_background=user_background, ) # For inputs that are not files, the following should be true assert type(image) is type(result) assert result is not image assert result.meta.cal_step.master_background == 'COMPLETE' assert result.meta.background.master_background_file == 'user_background.fits'
def test_master_background_logic(_jail, user_background, science_image): """Verify if calspec 2 background step was run the master background step will be skipped""" # the background step in calspec2 was done science_image.meta.cal_step.back_sub = 'COMPLETE' # Run with a user-supplied background collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( science_image, config_file='config/master_background.cfg', user_background=user_background, ) assert result.meta.cal_step.master_background == 'SKIPPED' assert type(science_image) is type(result) # Now force it result = MasterBackgroundStep.call( science_image, config_file='config/master_background.cfg', user_background=user_background, force_subtract=True) assert result.meta.cal_step.master_background == 'COMPLETE' assert type(science_image) is type(result)
def test_master_background_init(input_data, status, _jail, user_background): """Verify data can run through the step""" result = MasterBackgroundStep.call(input_data) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( input_data, config_file='config/master_background.cfg') # For inputs that are not files, the following should be true assert type(input_data) is type(result) assert result is not input_data if isinstance(result, datamodels.ModelContainer): for model in result: assert model.meta.cal_step.master_background == status else: assert result.meta.cal_step.master_background == status # Run with a user-supplied background and verify this is recorded in header result = MasterBackgroundStep.call(input_data, user_background=user_background) if isinstance(result, datamodels.ModelContainer): for model in result: if model.meta.cal_step.master_background == 'COMPLETE': assert model.meta.background.master_background_file == 'user_background.fits' else: if result.meta.cal_step.master_background == 'COMPLETE': assert result.meta.background.master_background_file == 'user_background.fits' # Make sure saving the computed background works result = MasterBackgroundStep.call(input_data, save_background=True)
def test_master_background_userbg(_jail, user_background): """Verify data can run through the step with a user-supplied background""" image = datamodels.ImageModel((10, 10)) image.meta.instrument.name = 'MIRI' image.meta.instrument.detector = 'MIRIMAGE' image.meta.exposure.type = 'MIR_LRS-FIXEDSLIT' image.meta.observation.date = '2018-01-01' image.meta.observation.time = '00:00:00' image.meta.subarray.xstart = 1 image.meta.subarray.ystart = 1 image.meta.wcsinfo.v2_ref = 0 image.meta.wcsinfo.v3_ref = 0 image.meta.wcsinfo.roll_ref = 0 image.meta.wcsinfo.ra_ref = 0 image.meta.wcsinfo.dec_ref = 0 image = AssignWcsStep.call(image) # Run with a user-supplied background and verify this is recorded in header result = MasterBackgroundStep.call(image, user_background=user_background) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( image, config_file='config/master_background.cfg', user_background=user_background, ) # For inputs that are not files, the following should be true assert type(image) is type(result) assert result is not image assert result.meta.cal_step.master_background == 'COMPLETE' assert result.meta.background.master_background_file == 'user_background.fits'
def run_pipeline(jail, rtdata_module): rtdata = rtdata_module rtdata.get_asn("miri/mrs/miri_mrs_mbkg_nodded_spec3_asn.json") MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background') return rtdata
def test_nirspec_fs_masterbg_user(self): """ Regression test of master background subtraction for NRS FS when a user 1-D spectrum is provided. """ # input file has 2-D background image added to it input_file = self.get_data(*self.test_dir, 'nrs_sci+bkg_cal.fits') # user provided 1-D background was created from the 2-D background image input_1dbkg_file = self.get_data(*self.test_dir, 'nrs_bkg_user_clean_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=input_1dbkg_file, save_results=True) # Compare background-subtracted science data (results) # to a truth file. These data are MultiSlitModel data result_file = result.meta.filename truth_file = self.get_data(*self.ref_loc, 'nrs_sci+bkg_masterbackgroundstep.fits') outputs = [(result_file, truth_file)] self.compare_outputs(outputs) result.close()
def test_miri_masterbg_mrs_nodded(self): """Run masterbackground step on MIRI MRS association""" asn_file = self.get_data(*self.test_dir, 'miri_mrs_mbkg_spec3_asn.json') for file in raw_from_asn(asn_file): self.get_data(*self.test_dir, file) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( asn_file, config_file='config/master_background.cfg', save_background=True, save_results=True, ) # test 1 # loop over the background subtracted data and compare to truth files # check that the cal_step master_background ran to complete for model in result: assert model.meta.cal_step.master_background == 'COMPLETE' truth_file = self.get_data(*self.ref_loc, model.meta.filename) outputs = [(model.meta.filename, truth_file)] self.compare_outputs(outputs) # test 2 # compare the master background combined file to truth file master_combined_bkg_file = 'MIRI_MRS_nod_seq1_MIRIFUSHORT_12SHORTexp1_o001_masterbg.fits' truth_background = self.get_data(*self.ref_loc, master_combined_bkg_file) outputs = [(master_combined_bkg_file, truth_background)] self.compare_outputs(outputs)
def test_nirspec_masterbg_nodded(self): """Run masterbackground step on NIRSpec association""" asn_file = self.get_data(*self.test_dir, 'nirspec_spec3_asn.json') for file in raw_from_asn(asn_file): self.get_data(*self.test_dir, file) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( asn_file, config_file='config/master_background.cfg', save_background=True, save_results=True) # test 1 # compare background subtracted data to truth files # check that the cal_step master_background ran to complete outputs = [] for model in result: assert model.meta.cal_step.master_background == 'COMPLETE' result_file = model.meta.filename.replace('cal', 'master_background') truth_file = self.get_data(*self.ref_loc, result_file) outputs.append((result_file, truth_file)) self.compare_outputs(outputs) # test 2 # compare the master background combined file to truth file master_combined_bkg_file = 'ifu_prism_source_off_fix_NRS1_o001_masterbg.fits' truth_background = self.get_data(*self.ref_loc, master_combined_bkg_file) outputs = [(master_combined_bkg_file, truth_background)] self.compare_outputs(outputs)
def test_miri_masterbg_lrs_dedicated(self): """Run masterbackground step on MIRI LRS association""" asn_file = self.get_data(*self.test_dir, 'miri_lrs_mbkg_dedicated_spec3_asn.json') for file in raw_from_asn(asn_file): self.get_data(*self.test_dir, file) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( asn_file, config_file='config/master_background.cfg', save_background = True, save_results = True) # test 1 # loop over the background subtracted data and compare to truth files for model in result: assert model.meta.cal_step.master_background == 'COMPLETE' result_file = model.meta.filename.replace('cal', 'master_background') truth_file = self.get_data(*self.ref_loc, result_file) outputs = [(result_file, truth_file)] self.compare_outputs(outputs) # test 2 # compare the master background combined file to truth file master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits' truth_background = self.get_data(*self.ref_loc, master_combined_bkg_file) outputs = [(master_combined_bkg_file, truth_background)] self.compare_outputs(outputs)
def test_nirspec_fs_masterbg_user(self): """ Regression test of master background subtraction for NRS FS when a user 1-D spectrum is provided. """ # input file has 2-D background image added to it input_file = self.get_data(*self.test_dir, 'nrs_sci+bkg_cal.fits') # user provided 1-D background was created from the 2-D background image input_1dbkg_file = self.get_data(*self.test_dir, 'nrs_bkg_user_clean_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=input_1dbkg_file, save_results=True) # Compare background-subtracted science data (results) # to a truth file. These data are MultiSlitModel data result_file = result.meta.filename truth_file = self.get_data(*self.ref_loc, 'nrs_sci+bkg_masterbackgroundstep.fits') outputs = [(result_file, truth_file)] self.compare_outputs(outputs) result.close()
def test_miri_masterbg_lrs_dedicated(self): """Run masterbackground step on MIRI LRS association""" asn_file = self.get_data(*self.test_dir, 'miri_lrs_mbkg_dedicated_spec3_asn.json') for file in raw_from_asn(asn_file): self.get_data(*self.test_dir, file) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( asn_file, config_file='config/master_background.cfg', save_background=True, save_results=True) # test 1 # loop over the background subtracted data and compare to truth files for model in result: assert model.meta.cal_step.master_background == 'COMPLETE' result_file = model.meta.filename.replace('cal', 'master_background') truth_file = self.get_data(*self.ref_loc, result_file) outputs = [(result_file, truth_file)] self.compare_outputs(outputs) # test 2 # compare the master background combined file to truth file master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits' truth_background = self.get_data(*self.ref_loc, master_combined_bkg_file) outputs = [(master_combined_bkg_file, truth_background)] self.compare_outputs(outputs)
def test_nirspec_ifu_mbkg_nod(rtdata, fitsdiff_default_kwargs, output_file): """Test NIRSpec IFU prism nodded data.""" # Get input data rtdata.get_asn("nirspec/ifu/nirspec_spec3_asn.json") MasterBackgroundStep.call(rtdata.input, save_background=True, save_results=True, suffix='master_background') rtdata.output = output_file # Get the truth file rtdata.get_truth(f"truth/test_nirspec_ifu_mbkg_nod/{output_file}") # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_nirspec_ifu_masterbg_user(self): """ Regression test of master background subtraction for NRS IFU when a user 1-D spectrum is provided. """ # input file has 2-D background image added to it input_file = self.get_data(*self.test_dir, 'prism_sci_bkg_cal.fits') # user-provided 1-D background was created from the 2-D background image user_background = self.get_data(*self.test_dir, 'prism_bkg_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=user_background, save_results=True) # Test 2 compare the science data with no background # to the output from the masterBackground Subtraction step # background subtracted science image. input_sci_cal_file = self.get_data(*self.test_dir, 'prism_sci_cal.fits') input_sci_model = datamodels.open(input_sci_cal_file) # We don't want the slices gaps to impact the statisitic # loop over the 30 Slices for i in range(30): slice_wcs = nirspec.nrs_wcs_set_input(input_sci_model, i) x, y = grid_from_bounding_box(slice_wcs.bounding_box) ra, dec, lam = slice_wcs(x, y) valid = np.isfinite(lam) result_slice_region = result.data[y.astype(int), x.astype(int)] sci_slice_region = input_sci_model.data[y.astype(int), x.astype(int)] sci_slice = sci_slice_region[valid] result_slice = result_slice_region[valid] sub = result_slice - sci_slice # check for outliers in the science image sci_mean = np.nanmean(sci_slice) sci_std = np.nanstd(sci_slice) upper = sci_mean + sci_std * 5.0 lower = sci_mean - sci_std * 5.0 mask_clean = np.logical_and(sci_slice < upper, sci_slice > lower) sub_mean = np.absolute(np.nanmean(sub[mask_clean])) atol = 2.0 assert_allclose(sub_mean, 0, atol=atol) # Test 3 Compare background sutracted science data (results) # to a truth file. This data is MultiSlit data input_sci_model.close() result_file = result.meta.filename truth_file = self.get_data(*self.ref_loc, 'prism_sci_bkg_masterbackgroundstep.fits') outputs = [(result_file, truth_file)] self.compare_outputs(outputs) input_sci_model.close() result.close()
def run_pipeline(jail, rtdata_module): rtdata = rtdata_module # This is the user-supplied background file. rtdata.get_data("miri/lrs/miri_lrs_bkg_x1d.fits") user_bkg = rtdata.input # This is the input file for the master_background step. rtdata.get_data("miri/lrs/miri_lrs_sci+bkg_cal.fits") MasterBackgroundStep.call(rtdata.input, user_background=user_bkg, save_results=True, suffix='master_background') return rtdata
def test_miri_lrs_masterbg_user(self): """ Regression test of masterbackgound subtraction with lrs, with user provided 1-D background """ # input file has the background added input_file = self.get_data(*self.test_dir, 'miri_lrs_sci+bkg_cal.fits') # user provided 1-D background user_background = self.get_data(*self.test_dir, 'miri_lrs_bkg_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=user_background, save_results=True) # Compare result (background subtracted image) to science image with no # background. Subtract these images, smooth the subtracted image and # the mean should be close to zero. input_sci_cal_file = self.get_data(*self.test_dir, 'miri_lrs_sci_cal.fits') input_sci = datamodels.open(input_sci_cal_file) # find the LRS region bb = result.meta.wcs.bounding_box x, y = grid_from_bounding_box(bb) result_lrs_region = result.data[y.astype(int), x.astype(int)] sci_lrs_region = input_sci.data[y.astype(int), x.astype(int)] # do a 5 sigma clip on the science image sci_mean = np.nanmean(sci_lrs_region) sci_std = np.nanstd(sci_lrs_region) upper = sci_mean + sci_std * 5.0 lower = sci_mean - sci_std * 5.0 mask_clean = np.logical_and(sci_lrs_region < upper, sci_lrs_region > lower) sub = result_lrs_region - sci_lrs_region mean_sub = np.absolute(np.mean(sub[mask_clean])) atol = 0.1 rtol = 0.001 assert_allclose(mean_sub, 0, atol=atol, rtol=rtol) # Test 3 Compare background subtracted science data (results) # to a truth file. truth_file = self.get_data( *self.ref_loc, 'miri_lrs_sci+bkg_masterbackgroundstep.fits') result_file = result.meta.filename outputs = [(result_file, truth_file)] self.compare_outputs(outputs) result.close() input_sci.close()
def test_nirspec_ifu_mbkg_user(rtdata, fitsdiff_default_kwargs): """Test NIRSpec IFU data with a user-supplied background file.""" # Get user-supplied background user_background = "prism_bkg_x1d.fits" rtdata.get_data(f"nirspec/ifu/{user_background}") # Get input data rtdata.get_data("nirspec/ifu/prism_sci_bkg_cal.fits") MasterBackgroundStep.call(rtdata.input, user_background=user_background, save_results=True, suffix='master_background') output = "prism_sci_bkg_master_background.fits" rtdata.output = output # Get the truth file rtdata.get_truth(f"truth/test_nirspec_ifu_mbkg_user/{output}") # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_nirspec_fs_mbkg_user(rtdata, fitsdiff_default_kwargs): """Run a test for NIRSpec FS data with a user-supplied background file.""" # Get user-supplied background user_background = "v2_nrs_bkg_user_clean_x1d.fits" rtdata.get_data(f"nirspec/fs/{user_background}") # Get input data rtdata.get_data("nirspec/fs/nrs_sci+bkg_cal.fits") MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background', user_background=user_background) output = "nrs_sci+bkg_master_background.fits" rtdata.output = output # Get the truth file rtdata.get_truth(f"truth/test_nirspec_fs_mbkg_user/{output}") # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_miri_lrs_masterbg_user(self): """ Regression test of masterbackgound subtraction with lrs, with user provided 1-D background """ # input file has the background added input_file = self.get_data(*self.test_dir, 'miri_lrs_sci+bkg_cal.fits') # user provided 1-D background user_background = self.get_data(*self.test_dir, 'miri_lrs_bkg_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=user_background, save_results=True) # Compare result (background subtracted image) to science image with no # background. Subtract these images, smooth the subtracted image and # the mean should be close to zero. input_sci_cal_file = self.get_data(*self.test_dir, 'miri_lrs_sci_cal.fits') input_sci = datamodels.open(input_sci_cal_file) # find the LRS region bb = result.meta.wcs.bounding_box x, y = grid_from_bounding_box(bb) result_lrs_region = result.data[y.astype(int), x.astype(int)] sci_lrs_region = input_sci.data[y.astype(int), x.astype(int)] # do a 5 sigma clip on the science image sci_mean = np.nanmean(sci_lrs_region) sci_std = np.nanstd(sci_lrs_region) upper = sci_mean + sci_std*5.0 lower = sci_mean - sci_std*5.0 mask_clean = np.logical_and(sci_lrs_region < upper, sci_lrs_region > lower) sub = result_lrs_region - sci_lrs_region mean_sub = np.absolute(np.mean(sub[mask_clean])) atol = 0.1 rtol = 0.001 assert_allclose(mean_sub, 0, atol=atol, rtol=rtol) # Test 3 Compare background subtracted science data (results) # to a truth file. truth_file = self.get_data(*self.ref_loc, 'miri_lrs_sci+bkg_masterbackgroundstep.fits') result_file = result.meta.filename outputs = [(result_file, truth_file)] self.compare_outputs(outputs) result.close() input_sci.close()
def test_master_background_userbg(_jail, user_background, science_image): """Verify data can run through the step with a user-supplied background""" # Run with a user-supplied background and verify this is recorded in header collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( science_image, config_file='config/master_background.cfg', user_background=user_background, ) assert type(science_image) is type(result) assert result is not science_image assert result.meta.cal_step.master_background == 'COMPLETE' assert result.meta.background.master_background_file == 'user_background.fits'
def test_miri_masterbackground_mrs(self): """Run masterbackground step on MIRI LRS association""" asn_file = self.get_data(*self.test_dir, 'miri_mrs_mbkg_0304_spec3_asn.json') for file in raw_from_asn(asn_file): self.get_data(*self.test_dir, file) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( asn_file, config_file='config/master_background.cfg', save_background=True ) for model in result: assert model.meta.cal_step.master_background == 'COMPLETE'
def test_nirspec_masterbackground_mos_user1d(self): """ Regression test of master background subtraction for NRS MOS when a user 1-D spectrum is provided. """ # input file has 2-D background image added to it input_file = self.get_data(*self.test_dir, 'nrs_mos_sci+bkg_cal.fits') # user provide 1-D background was created from the 2-D background image input_1dbkg_file = self.get_data(*self.test_dir, 'nrs_mos_bkg_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=input_1dbkg_file, save_results=True) # _________________________________________________________________________ # One of out tests is to compare the 1-D extracted spectra from # the science image (no background added) and the masterbackground subtracted # data. # run resample_spec on results from MasterBackground step result_2d = ResampleSpecStep.call(result) # run 1-D extract on results from MasterBackground step result_1d = Extract1dStep.call(result_2d) # get input science data with background added input_sci_cal_file = self.get_data(*self.test_dir, 'nrs_mos_sci_cal.fits') # get 1-D extract on original science data without background # this reference data was also run through ResampleSpec input_sci_1d_file = self.get_data(*self.ref_loc, 'nrs_mos_sci_extract1dstep.fits') input_sci = datamodels.open(input_sci_cal_file) sci_cal_1d = datamodels.open(input_sci_1d_file) num_spec = len(sci_cal_1d.spec) # the user 1D spectrum may not cover the entire wavelength range of the # science data. Find the wavelength range of user 1-D spectra input_1dbkg_1d = datamodels.open(input_1dbkg_file) user_wave = input_1dbkg_1d.spec[0].spec_table['wavelength'] user_flux = input_1dbkg_1d.spec[0].spec_table['flux'] user_wave_valid = np.where(user_flux > 0) min_user_wave = np.amin(user_wave[user_wave_valid]) max_user_wave = np.amax(user_wave[user_wave_valid]) input_1dbkg_1d.close() # loop over each slit and perform 2 tests on each slit for i in range(num_spec): # ______________________________________________________________________ # Test 1 compare extracted spectra data from the science data # to extracted spectra from the output # from MasterBackground subtraction. sci_spec_1d = sci_cal_1d.spec[i].spec_table['flux'] sci_wave = sci_cal_1d.spec[i].spec_table['wavelength'] result_spec_1d = result_1d.spec[i].spec_table['flux'] # find the waverange covered by both user 1-D and science slit sci_wave_valid = np.where(sci_spec_1d > 0) min_wave = np.amin(sci_wave[sci_wave_valid]) max_wave = np.amax(sci_wave[sci_wave_valid]) if min_user_wave > min_wave: min_wave = min_user_wave if max_user_wave < max_wave: max_wave = max_user_wave sub_spec = sci_spec_1d - result_spec_1d valid = np.where(np.logical_and(sci_wave > min_wave, sci_wave < max_wave)) sub_spec = sub_spec[valid] mean_sub = np.nanmean(sub_spec) atol = 1.5 assert_allclose(mean_sub, 0, atol=atol) # ______________________________________________________________________ # Test 2 compare the science data with no background # to the output from the masterBackground Subtraction step # background subtracted science image. bb = input_sci.slits[i].meta.wcs.bounding_box x, y = grid_from_bounding_box(bb) ra, dec, lam = input_sci.slits[i].meta.wcs(x, y) valid = np.isfinite(lam) sci = input_sci.slits[i].data result_slit = result.slits[i].data # check for outliers in the science image that could cause test # to fail. These could be cosmic ray hits or other yeck that # messes up the science data - ignores these cases sci_mean = np.nanmean(sci[valid]) sci_std = np.nanstd(sci[valid]) upper = sci_mean + sci_std*5.0 lower = sci_mean - sci_std*5.0 mask_clean = np.logical_and(sci[valid] < upper, sci[valid] > lower) # for this slit subtract the background subtracted data from # the science data with no background added sub = result_slit - sci # do not look at outliers - they confuse things sub_valid = sub[valid] mean_sub = np.mean(sub_valid[mask_clean]) atol = 0.1 assert_allclose(np.absolute(mean_sub), 0, atol=atol) # ______________________________________________________________________ # Test 3 Compare background sutracted science data (results) # to a truth file. This data is MultiSlit data result_file = result.meta.filename ref_file = self.get_data(*self.ref_loc, 'nrs_mos_sci+bkg_masterbackgroundstep.fits') outputs = [(result_file, ref_file)] self.compare_outputs(outputs) input_sci.close() result.close()
def test_nirspec_ifu_masterbg_user(self): """ Regression test of master background subtraction for NRS IFU when a user 1-D spectrum is provided. """ # input file has 2-D background image added to it input_file = self.get_data(*self.test_dir, 'prism_sci_bkg_cal.fits') # user-provided 1-D background was created from the 2-D background image user_background = self.get_data(*self.test_dir, 'prism_bkg_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=user_background, save_results=True) # Test 1 compare extracted spectra data with # no background added to extracted spectra from the output # from MasterBackground subtraction. First cube_build has to be run # on the data. result_s3d = CubeBuildStep.call(result) # run 1-D extract on results from MasterBackground step result_1d = Extract1dStep.call(result_s3d, subtract_background=False) # get the 1-D extracted spectrum from the science data in truth directory input_sci_1d_file = self.get_data(*self.ref_loc, 'prism_sci_extract1d.fits') sci_1d = datamodels.open(input_sci_1d_file) # read in the valid wavelengths of the user-1d user_background_model = datamodels.open(user_background) user_wave = user_background_model.spec[0].spec_table['wavelength'] user_flux = user_background_model.spec[0].spec_table['flux'] user_wave_valid = np.where(user_flux > 0) min_user_wave = np.amin(user_wave[user_wave_valid]) max_user_wave = np.amax(user_wave[user_wave_valid]) user_background_model.close() # find the waverange covered by both user and science sci_spec_1d = sci_1d.spec[0].spec_table['flux'] sci_spec_wave = sci_1d.spec[0].spec_table['wavelength'] result_spec_1d = result_1d.spec[0].spec_table['flux'] sci_wave_valid = np.where(sci_spec_1d > 0) min_wave = np.amin(sci_spec_wave[sci_wave_valid]) max_wave = np.amax(sci_spec_wave[sci_wave_valid]) if min_user_wave > min_wave: min_wave = min_user_wave if max_user_wave < max_wave: max_wave = max_user_wave sub_spec = sci_spec_1d - result_spec_1d valid = np.where(np.logical_and(sci_spec_wave > min_wave, sci_spec_wave < max_wave)) sub_spec = sub_spec[valid] sub_spec = sub_spec[1:-2] # endpoints are wacky mean_sub = np.absolute(np.nanmean(sub_spec)) atol = 5.0 assert_allclose(mean_sub, 0, atol=atol) # Test 2 compare the science data with no background # to the output from the masterBackground Subtraction step # background subtracted science image. input_sci_cal_file = self.get_data(*self.test_dir, 'prism_sci_cal.fits') input_sci_model = datamodels.open(input_sci_cal_file) # We don't want the slices gaps to impact the statisitic # loop over the 30 Slices for i in range(30): slice_wcs = nirspec.nrs_wcs_set_input(input_sci_model, i) x, y = grid_from_bounding_box(slice_wcs.bounding_box) ra, dec, lam = slice_wcs(x, y) valid = np.isfinite(lam) result_slice_region = result.data[y.astype(int), x.astype(int)] sci_slice_region = input_sci_model.data[y.astype(int), x.astype(int)] sci_slice = sci_slice_region[valid] result_slice = result_slice_region[valid] sub = result_slice - sci_slice # check for outliers in the science image sci_mean = np.nanmean(sci_slice) sci_std = np.nanstd(sci_slice) upper = sci_mean + sci_std*5.0 lower = sci_mean - sci_std*5.0 mask_clean = np.logical_and(sci_slice < upper, sci_slice > lower) sub_mean = np.absolute(np.nanmean(sub[mask_clean])) atol = 2.0 assert_allclose(sub_mean, 0, atol=atol) # Test 3 Compare background sutracted science data (results) # to a truth file. This data is MultiSlit data input_sci_model.close() result_file = result.meta.filename truth_file = self.get_data(*self.ref_loc, 'prism_sci_bkg_masterbackgroundstep.fits') outputs = [(result_file, truth_file)] self.compare_outputs(outputs) input_sci_model.close() result.close()
def test_nirspec_masterbackground_mos_user1d(self): """ Regression test of master background subtraction for NRS MOS when a user 1-D spectrum is provided. """ # input file has 2-D background image added to it input_file = self.get_data(*self.test_dir, 'nrs_mos_sci+bkg_cal.fits') # user provide 1-D background was created from the 2-D background image input_1dbkg_file = self.get_data(*self.test_dir, 'nrs_mos_bkg_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=input_1dbkg_file, save_results=True) # _________________________________________________________________________ # One of out tests is to compare the 1-D extracted spectra from # the science image (no background added) and the masterbackground subtracted # data. # run resample_spec on results from MasterBackground step result_2d = ResampleSpecStep.call(result) # run 1-D extract on results from MasterBackground step result_1d = Extract1dStep.call(result_2d) # get input science data with background added input_sci_cal_file = self.get_data(*self.test_dir, 'nrs_mos_sci_cal.fits') # get 1-D extract on original science data without background # this reference data was also run through ResampleSpec input_sci_1d_file = self.get_data(*self.ref_loc, 'nrs_mos_sci_extract1dstep.fits') input_sci = datamodels.open(input_sci_cal_file) sci_cal_1d = datamodels.open(input_sci_1d_file) num_spec = len(sci_cal_1d.spec) # the user 1D spectrum may not cover the entire wavelength range of the # science data. Find the wavelength range of user 1-D spectra input_1dbkg_1d = datamodels.open(input_1dbkg_file) user_wave = input_1dbkg_1d.spec[0].spec_table['wavelength'] user_flux = input_1dbkg_1d.spec[0].spec_table['flux'] user_wave_valid = np.where(user_flux > 0) min_user_wave = np.amin(user_wave[user_wave_valid]) max_user_wave = np.amax(user_wave[user_wave_valid]) input_1dbkg_1d.close() # loop over each slit and perform 2 tests on each slit for i in range(num_spec): # ______________________________________________________________________ # Test 1 compare extracted spectra data from the science data # to extracted spectra from the output # from MasterBackground subtraction. sci_spec_1d = sci_cal_1d.spec[i].spec_table['flux'] sci_wave = sci_cal_1d.spec[i].spec_table['wavelength'] result_spec_1d = result_1d.spec[i].spec_table['flux'] # find the waverange covered by both user 1-D and science slit sci_wave_valid = np.where(sci_spec_1d > 0) min_wave = np.amin(sci_wave[sci_wave_valid]) max_wave = np.amax(sci_wave[sci_wave_valid]) if min_user_wave > min_wave: min_wave = min_user_wave if max_user_wave < max_wave: max_wave = max_user_wave sub_spec = sci_spec_1d - result_spec_1d valid = np.where( np.logical_and(sci_wave > min_wave, sci_wave < max_wave)) sub_spec = sub_spec[valid] mean_sub = np.nanmean(sub_spec) atol = 1.5 assert_allclose(mean_sub, 0, atol=atol) # ______________________________________________________________________ # Test 2 compare the science data with no background # to the output from the masterBackground Subtraction step # background subtracted science image. bb = input_sci.slits[i].meta.wcs.bounding_box x, y = grid_from_bounding_box(bb) ra, dec, lam = input_sci.slits[i].meta.wcs(x, y) valid = np.isfinite(lam) sci = input_sci.slits[i].data result_slit = result.slits[i].data # check for outliers in the science image that could cause test # to fail. These could be cosmic ray hits or other yeck that # messes up the science data - ignores these cases sci_mean = np.nanmean(sci[valid]) sci_std = np.nanstd(sci[valid]) upper = sci_mean + sci_std * 5.0 lower = sci_mean - sci_std * 5.0 mask_clean = np.logical_and(sci[valid] < upper, sci[valid] > lower) # for this slit subtract the background subtracted data from # the science data with no background added sub = result_slit - sci # do not look at outliers - they confuse things sub_valid = sub[valid] mean_sub = np.mean(sub_valid[mask_clean]) atol = 0.1 assert_allclose(np.absolute(mean_sub), 0, atol=atol) # ______________________________________________________________________ # Test 3 Compare background sutracted science data (results) # to a truth file. This data is MultiSlit data result_file = result.meta.filename ref_file = self.get_data(*self.ref_loc, 'nrs_mos_sci+bkg_masterbackgroundstep.fits') outputs = [(result_file, ref_file)] self.compare_outputs(outputs) input_sci.close() result.close()
def test_miri_masterbackground_lrs_user1d(self): """ Regression test of masterbackgound subtraction with lrs, with user provided 1-D background """ # input file has the background added # Copy original version of file to test file, which will get overwritten by test input_file = self.get_data(*self.test_dir, 'miri_lrs_sci+bkg_cal.fits') # user provided 1-D background input_1d_bkg_file = self.get_data(*self.test_dir, 'miri_lrs_bkg_x1d.fits') input_1d_bkg_model = datamodels.open(input_1d_bkg_file) result = MasterBackgroundStep.call(input_file, user_background=input_1d_bkg_file, save_results=True) # Test 1 # Run extract1D on the master background subtracted data (result) and # the science data with no background added # run 1-D extract on results from MasterBackground step result_1d = Extract1dStep.call(result) # get the 1-D extracted Spectrum from the science data with # no background added to test against sci_cal_1d_file = self.get_data(*self.ref_loc, 'miri_lrs_sci_extract1d.fits') sci_cal_1d = datamodels.open(sci_cal_1d_file) sci_cal_1d_data = sci_cal_1d.spec[0].spec_table['flux'] result_1d_data = result_1d.spec[0].spec_table['flux'] sci_wave = sci_cal_1d.spec[0].spec_table['wavelength'] # find the valid wavelenth of the user provided spectrun user_wave = input_1d_bkg_model.spec[0].spec_table['wavelength'] user_flux = input_1d_bkg_model.spec[0].spec_table['flux'] user_wave_valid = np.where(user_flux > 0) min_user_wave = np.amin(user_wave[user_wave_valid]) max_user_wave = np.amax(user_wave[user_wave_valid]) input_1d_bkg_model.close() # find the waverange covered by both user and science sci_wave_valid = np.where(sci_cal_1d_data > 0) min_wave = np.amin(sci_wave[sci_wave_valid]) max_wave = np.amax(sci_wave[sci_wave_valid]) if min_user_wave > min_wave: min_wave = min_user_wave if max_user_wave < max_wave: max_wave = max_user_wave # Compare the data sub_spec = sci_cal_1d_data - result_1d_data valid = np.where(np.logical_and(sci_wave > min_wave, sci_wave < max_wave)) sub_spec = sub_spec[valid] mean_sub = np.absolute(np.nanmean(sub_spec)) atol = 0.00005 rtol = 0.000001 assert_allclose(mean_sub, 0, atol=atol, rtol=rtol) # Test 2 # Compare result (background subtracted image) to science image with no # background. Subtract these images, smooth the subtracted image and # the mean should be close to zero. input_sci_cal_file = self.get_data(*self.test_dir, 'miri_lrs_sci_cal.fits') input_sci = datamodels.open(input_sci_cal_file) # find the LRS region bb = result.meta.wcs.bounding_box x, y = grid_from_bounding_box(bb) result_lrs_region = result.data[y.astype(int), x.astype(int)] sci_lrs_region = input_sci.data[y.astype(int), x.astype(int)] # do a 5 sigma clip on the science image sci_mean = np.nanmean(sci_lrs_region) sci_std = np.nanstd(sci_lrs_region) upper = sci_mean + sci_std*5.0 lower = sci_mean - sci_std*5.0 mask_clean = np.logical_and(sci_lrs_region < upper, sci_lrs_region > lower) sub = result_lrs_region - sci_lrs_region mean_sub = np.absolute(np.mean(sub[mask_clean])) atol = 0.5 rtol = 0.001 assert_allclose(mean_sub, 0, atol=atol, rtol=rtol) # Test 3 Compare background subtracted science data (results) # to a truth file. truth_file = self.get_data(*self.ref_loc, 'miri_lrs_sci+bkg_masterbackgroundstep.fits') result_file = result.meta.filename outputs = [(result_file, truth_file)] self.compare_outputs(outputs) result.close() input_sci.close()
def test_nirspec_ifu_masterbg_user(self): """ Regression test of master background subtraction for NRS IFU when a user 1-D spectrum is provided. """ # input file has 2-D background image added to it input_file = self.get_data(*self.test_dir, 'prism_sci_bkg_cal.fits') # user-provided 1-D background was created from the 2-D background image user_background = self.get_data(*self.test_dir, 'prism_bkg_x1d.fits') result = MasterBackgroundStep.call(input_file, user_background=user_background, save_results=True) # Test 1 compare extracted spectra data with # no background added to extracted spectra from the output # from MasterBackground subtraction. First cube_build has to be run # on the data. result_s3d = CubeBuildStep.call(result) # run 1-D extract on results from MasterBackground step result_1d = Extract1dStep.call(result_s3d, subtract_background=False) # get the 1-D extracted spectrum from the science data in truth directory input_sci_1d_file = self.get_data(*self.ref_loc, 'prism_sci_extract1d.fits') sci_1d = datamodels.open(input_sci_1d_file) # read in the valid wavelengths of the user-1d user_background_model = datamodels.open(user_background) user_wave = user_background_model.spec[0].spec_table['wavelength'] user_flux = user_background_model.spec[0].spec_table['net'] user_wave_valid = np.where(user_flux > 0) min_user_wave = np.amin(user_wave[user_wave_valid]) max_user_wave = np.amax(user_wave[user_wave_valid]) user_background_model.close() # find the waverange covered by both user and science sci_spec_1d = sci_1d.spec[0].spec_table['net'] sci_spec_wave = sci_1d.spec[0].spec_table['wavelength'] result_spec_1d = result_1d.spec[0].spec_table['net'] sci_wave_valid = np.where(sci_spec_1d > 0) min_wave = np.amin(sci_spec_wave[sci_wave_valid]) max_wave = np.amax(sci_spec_wave[sci_wave_valid]) if min_user_wave > min_wave: min_wave = min_user_wave if max_user_wave < max_wave: max_wave = max_user_wave sub_spec = sci_spec_1d - result_spec_1d valid = np.where(np.logical_and(sci_spec_wave > min_wave, sci_spec_wave < max_wave)) sub_spec = sub_spec[valid] sub_spec = sub_spec[1:-2] # endpoints are wacky mean_sub = np.absolute(np.nanmean(sub_spec)) atol = 5.0 assert_allclose(mean_sub, 0, atol=atol) # Test 2 compare the science data with no background # to the output from the masterBackground Subtraction step # background subtracted science image. input_sci_cal_file = self.get_data(*self.test_dir, 'prism_sci_cal.fits') input_sci_model = datamodels.open(input_sci_cal_file) # We don't want the slices gaps to impact the statisitic # loop over the 30 Slices for i in range(30): slice_wcs = nirspec.nrs_wcs_set_input(input_sci_model, i) x, y = grid_from_bounding_box(slice_wcs.bounding_box) ra, dec, lam = slice_wcs(x, y) valid = np.isfinite(lam) result_slice_region = result.data[y.astype(int), x.astype(int)] sci_slice_region = input_sci_model.data[y.astype(int), x.astype(int)] sci_slice = sci_slice_region[valid] result_slice = result_slice_region[valid] sub = result_slice - sci_slice # check for outliers in the science image sci_mean = np.nanmean(sci_slice) sci_std = np.nanstd(sci_slice) upper = sci_mean + sci_std*5.0 lower = sci_mean - sci_std*5.0 mask_clean = np.logical_and(sci_slice < upper, sci_slice > lower) sub_mean = np.absolute(np.nanmean(sub[mask_clean])) atol = 2.0 assert_allclose(sub_mean, 0, atol=atol) # Test 3 Compare background sutracted science data (results) # to a truth file. This data is MultiSlit data input_sci_model.close() result_file = result.meta.filename truth_file = self.get_data(*self.ref_loc, 'prism_sci_bkg_masterbackgroundstep.fits') outputs = [(result_file, truth_file)] self.compare_outputs(outputs) input_sci_model.close() result.close()