def test_master_background_userbg(_jail, user_background):
"""Verify data can run through the step with a user-supplied background"""
image = datamodels.ImageModel((10, 10))
image.meta.instrument.name = 'MIRI'
image.meta.instrument.detector = 'MIRIMAGE'
image.meta.exposure.type = 'MIR_LRS-FIXEDSLIT'
image.meta.observation.date = '2018-01-01'
image.meta.observation.time = '00:00:00'
image.meta.subarray.xstart = 1
image.meta.subarray.ystart = 1
image.meta.wcsinfo.v2_ref = 0
image.meta.wcsinfo.v3_ref = 0
image.meta.wcsinfo.roll_ref = 0
image.meta.wcsinfo.ra_ref = 0
image.meta.wcsinfo.dec_ref = 0
image = AssignWcsStep.call(image)
# Run with a user-supplied background and verify this is recorded in header
result = MasterBackgroundStep.call(image, user_background=user_background)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
image,
config_file='config/master_background.cfg',
user_background=user_background,
)
# For inputs that are not files, the following should be true
assert type(image) is type(result)
assert result is not image
assert result.meta.cal_step.master_background == 'COMPLETE'
assert result.meta.background.master_background_file == 'user_background.fits'
def test_master_background_logic(_jail, user_background, science_image):
"""Verify if calspec 2 background step was run the master background step will be skipped"""
# the background step in calspec2 was done
science_image.meta.cal_step.back_sub = 'COMPLETE'
# Run with a user-supplied background
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
science_image,
config_file='config/master_background.cfg',
user_background=user_background,
)
assert result.meta.cal_step.master_background == 'SKIPPED'
assert type(science_image) is type(result)
# Now force it
result = MasterBackgroundStep.call(
science_image,
config_file='config/master_background.cfg',
user_background=user_background,
force_subtract=True)
assert result.meta.cal_step.master_background == 'COMPLETE'
assert type(science_image) is type(result)
def test_master_background_init(input_data, status, _jail, user_background):
"""Verify data can run through the step"""
result = MasterBackgroundStep.call(input_data)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
input_data, config_file='config/master_background.cfg')
# For inputs that are not files, the following should be true
assert type(input_data) is type(result)
assert result is not input_data
if isinstance(result, datamodels.ModelContainer):
for model in result:
assert model.meta.cal_step.master_background == status
else:
assert result.meta.cal_step.master_background == status
# Run with a user-supplied background and verify this is recorded in header
result = MasterBackgroundStep.call(input_data,
user_background=user_background)
if isinstance(result, datamodels.ModelContainer):
for model in result:
if model.meta.cal_step.master_background == 'COMPLETE':
assert model.meta.background.master_background_file == 'user_background.fits'
else:
if result.meta.cal_step.master_background == 'COMPLETE':
assert result.meta.background.master_background_file == 'user_background.fits'
# Make sure saving the computed background works
result = MasterBackgroundStep.call(input_data, save_background=True)
def test_master_background_userbg(_jail, user_background):
"""Verify data can run through the step with a user-supplied background"""
image = datamodels.ImageModel((10, 10))
image.meta.instrument.name = 'MIRI'
image.meta.instrument.detector = 'MIRIMAGE'
image.meta.exposure.type = 'MIR_LRS-FIXEDSLIT'
image.meta.observation.date = '2018-01-01'
image.meta.observation.time = '00:00:00'
image.meta.subarray.xstart = 1
image.meta.subarray.ystart = 1
image.meta.wcsinfo.v2_ref = 0
image.meta.wcsinfo.v3_ref = 0
image.meta.wcsinfo.roll_ref = 0
image.meta.wcsinfo.ra_ref = 0
image.meta.wcsinfo.dec_ref = 0
image = AssignWcsStep.call(image)
# Run with a user-supplied background and verify this is recorded in header
result = MasterBackgroundStep.call(image, user_background=user_background)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
image,
config_file='config/master_background.cfg',
user_background=user_background,
)
# For inputs that are not files, the following should be true
assert type(image) is type(result)
assert result is not image
assert result.meta.cal_step.master_background == 'COMPLETE'
assert result.meta.background.master_background_file == 'user_background.fits'
def run_pipeline(jail, rtdata_module):
rtdata = rtdata_module
rtdata.get_asn("miri/mrs/miri_mrs_mbkg_nodded_spec3_asn.json")
MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background')
return rtdata
def test_nirspec_fs_masterbg_user(self):
"""
Regression test of master background subtraction for NRS FS when a
user 1-D spectrum is provided.
"""
# input file has 2-D background image added to it
input_file = self.get_data(*self.test_dir, 'nrs_sci+bkg_cal.fits')
# user provided 1-D background was created from the 2-D background image
input_1dbkg_file = self.get_data(*self.test_dir, 'nrs_bkg_user_clean_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=input_1dbkg_file,
save_results=True)
# Compare background-subtracted science data (results)
# to a truth file. These data are MultiSlitModel data
result_file = result.meta.filename
truth_file = self.get_data(*self.ref_loc,
'nrs_sci+bkg_masterbackgroundstep.fits')
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
result.close()
def test_miri_masterbg_mrs_nodded(self):
"""Run masterbackground step on MIRI MRS association"""
asn_file = self.get_data(*self.test_dir,
'miri_mrs_mbkg_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background=True,
save_results=True,
)
# test 1
# loop over the background subtracted data and compare to truth files
# check that the cal_step master_background ran to complete
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
truth_file = self.get_data(*self.ref_loc, model.meta.filename)
outputs = [(model.meta.filename, truth_file)]
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'MIRI_MRS_nod_seq1_MIRIFUSHORT_12SHORTexp1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def test_nirspec_masterbg_nodded(self):
"""Run masterbackground step on NIRSpec association"""
asn_file = self.get_data(*self.test_dir, 'nirspec_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background=True,
save_results=True)
# test 1
# compare background subtracted data to truth files
# check that the cal_step master_background ran to complete
outputs = []
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
result_file = model.meta.filename.replace('cal',
'master_background')
truth_file = self.get_data(*self.ref_loc, result_file)
outputs.append((result_file, truth_file))
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'ifu_prism_source_off_fix_NRS1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def test_miri_masterbg_lrs_dedicated(self):
"""Run masterbackground step on MIRI LRS association"""
asn_file = self.get_data(*self.test_dir,
'miri_lrs_mbkg_dedicated_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background = True,
save_results = True)
# test 1
# loop over the background subtracted data and compare to truth files
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
result_file = model.meta.filename.replace('cal', 'master_background')
truth_file = self.get_data(*self.ref_loc,
result_file)
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def test_nirspec_fs_masterbg_user(self):
"""
Regression test of master background subtraction for NRS FS when a
user 1-D spectrum is provided.
"""
# input file has 2-D background image added to it
input_file = self.get_data(*self.test_dir, 'nrs_sci+bkg_cal.fits')
# user provided 1-D background was created from the 2-D background image
input_1dbkg_file = self.get_data(*self.test_dir, 'nrs_bkg_user_clean_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=input_1dbkg_file,
save_results=True)
# Compare background-subtracted science data (results)
# to a truth file. These data are MultiSlitModel data
result_file = result.meta.filename
truth_file = self.get_data(*self.ref_loc,
'nrs_sci+bkg_masterbackgroundstep.fits')
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
result.close()
def test_miri_masterbg_lrs_dedicated(self):
"""Run masterbackground step on MIRI LRS association"""
asn_file = self.get_data(*self.test_dir,
'miri_lrs_mbkg_dedicated_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background=True,
save_results=True)
# test 1
# loop over the background subtracted data and compare to truth files
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
result_file = model.meta.filename.replace('cal',
'master_background')
truth_file = self.get_data(*self.ref_loc, result_file)
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def test_nirspec_ifu_mbkg_nod(rtdata, fitsdiff_default_kwargs, output_file):
"""Test NIRSpec IFU prism nodded data."""
# Get input data
rtdata.get_asn("nirspec/ifu/nirspec_spec3_asn.json")
MasterBackgroundStep.call(rtdata.input, save_background=True, save_results=True,
suffix='master_background')
rtdata.output = output_file
# Get the truth file
rtdata.get_truth(f"truth/test_nirspec_ifu_mbkg_nod/{output_file}")
# Compare the results
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
assert diff.identical, diff.report()
def test_nirspec_ifu_masterbg_user(self):
"""
Regression test of master background subtraction for NRS IFU when a
user 1-D spectrum is provided.
"""
# input file has 2-D background image added to it
input_file = self.get_data(*self.test_dir, 'prism_sci_bkg_cal.fits')
# user-provided 1-D background was created from the 2-D background image
user_background = self.get_data(*self.test_dir, 'prism_bkg_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=user_background,
save_results=True)
# Test 2 compare the science data with no background
# to the output from the masterBackground Subtraction step
# background subtracted science image.
input_sci_cal_file = self.get_data(*self.test_dir,
'prism_sci_cal.fits')
input_sci_model = datamodels.open(input_sci_cal_file)
# We don't want the slices gaps to impact the statisitic
# loop over the 30 Slices
for i in range(30):
slice_wcs = nirspec.nrs_wcs_set_input(input_sci_model, i)
x, y = grid_from_bounding_box(slice_wcs.bounding_box)
ra, dec, lam = slice_wcs(x, y)
valid = np.isfinite(lam)
result_slice_region = result.data[y.astype(int), x.astype(int)]
sci_slice_region = input_sci_model.data[y.astype(int),
x.astype(int)]
sci_slice = sci_slice_region[valid]
result_slice = result_slice_region[valid]
sub = result_slice - sci_slice
# check for outliers in the science image
sci_mean = np.nanmean(sci_slice)
sci_std = np.nanstd(sci_slice)
upper = sci_mean + sci_std * 5.0
lower = sci_mean - sci_std * 5.0
mask_clean = np.logical_and(sci_slice < upper, sci_slice > lower)
sub_mean = np.absolute(np.nanmean(sub[mask_clean]))
atol = 2.0
assert_allclose(sub_mean, 0, atol=atol)
# Test 3 Compare background sutracted science data (results)
# to a truth file. This data is MultiSlit data
input_sci_model.close()
result_file = result.meta.filename
truth_file = self.get_data(*self.ref_loc,
'prism_sci_bkg_masterbackgroundstep.fits')
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
input_sci_model.close()
result.close()
def run_pipeline(jail, rtdata_module):
rtdata = rtdata_module
# This is the user-supplied background file.
rtdata.get_data("miri/lrs/miri_lrs_bkg_x1d.fits")
user_bkg = rtdata.input
# This is the input file for the master_background step.
rtdata.get_data("miri/lrs/miri_lrs_sci+bkg_cal.fits")
MasterBackgroundStep.call(rtdata.input,
user_background=user_bkg,
save_results=True,
suffix='master_background')
return rtdata
def test_miri_lrs_masterbg_user(self):
"""
Regression test of masterbackgound subtraction with lrs, with user provided 1-D background
"""
# input file has the background added
input_file = self.get_data(*self.test_dir, 'miri_lrs_sci+bkg_cal.fits')
# user provided 1-D background
user_background = self.get_data(*self.test_dir,
'miri_lrs_bkg_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=user_background,
save_results=True)
# Compare result (background subtracted image) to science image with no
# background. Subtract these images, smooth the subtracted image and
# the mean should be close to zero.
input_sci_cal_file = self.get_data(*self.test_dir,
'miri_lrs_sci_cal.fits')
input_sci = datamodels.open(input_sci_cal_file)
# find the LRS region
bb = result.meta.wcs.bounding_box
x, y = grid_from_bounding_box(bb)
result_lrs_region = result.data[y.astype(int), x.astype(int)]
sci_lrs_region = input_sci.data[y.astype(int), x.astype(int)]
# do a 5 sigma clip on the science image
sci_mean = np.nanmean(sci_lrs_region)
sci_std = np.nanstd(sci_lrs_region)
upper = sci_mean + sci_std * 5.0
lower = sci_mean - sci_std * 5.0
mask_clean = np.logical_and(sci_lrs_region < upper,
sci_lrs_region > lower)
sub = result_lrs_region - sci_lrs_region
mean_sub = np.absolute(np.mean(sub[mask_clean]))
atol = 0.1
rtol = 0.001
assert_allclose(mean_sub, 0, atol=atol, rtol=rtol)
# Test 3 Compare background subtracted science data (results)
# to a truth file.
truth_file = self.get_data(
*self.ref_loc, 'miri_lrs_sci+bkg_masterbackgroundstep.fits')
result_file = result.meta.filename
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
result.close()
input_sci.close()
def test_nirspec_ifu_mbkg_user(rtdata, fitsdiff_default_kwargs):
"""Test NIRSpec IFU data with a user-supplied background file."""
# Get user-supplied background
user_background = "prism_bkg_x1d.fits"
rtdata.get_data(f"nirspec/ifu/{user_background}")
# Get input data
rtdata.get_data("nirspec/ifu/prism_sci_bkg_cal.fits")
MasterBackgroundStep.call(rtdata.input, user_background=user_background,
save_results=True, suffix='master_background')
output = "prism_sci_bkg_master_background.fits"
rtdata.output = output
# Get the truth file
rtdata.get_truth(f"truth/test_nirspec_ifu_mbkg_user/{output}")
# Compare the results
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
assert diff.identical, diff.report()
def test_nirspec_fs_mbkg_user(rtdata, fitsdiff_default_kwargs):
"""Run a test for NIRSpec FS data with a user-supplied background file."""
# Get user-supplied background
user_background = "v2_nrs_bkg_user_clean_x1d.fits"
rtdata.get_data(f"nirspec/fs/{user_background}")
# Get input data
rtdata.get_data("nirspec/fs/nrs_sci+bkg_cal.fits")
MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background',
user_background=user_background)
output = "nrs_sci+bkg_master_background.fits"
rtdata.output = output
# Get the truth file
rtdata.get_truth(f"truth/test_nirspec_fs_mbkg_user/{output}")
# Compare the results
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
assert diff.identical, diff.report()
def test_miri_lrs_masterbg_user(self):
"""
Regression test of masterbackgound subtraction with lrs, with user provided 1-D background
"""
# input file has the background added
input_file = self.get_data(*self.test_dir, 'miri_lrs_sci+bkg_cal.fits')
# user provided 1-D background
user_background = self.get_data(*self.test_dir, 'miri_lrs_bkg_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=user_background,
save_results=True)
# Compare result (background subtracted image) to science image with no
# background. Subtract these images, smooth the subtracted image and
# the mean should be close to zero.
input_sci_cal_file = self.get_data(*self.test_dir,
'miri_lrs_sci_cal.fits')
input_sci = datamodels.open(input_sci_cal_file)
# find the LRS region
bb = result.meta.wcs.bounding_box
x, y = grid_from_bounding_box(bb)
result_lrs_region = result.data[y.astype(int), x.astype(int)]
sci_lrs_region = input_sci.data[y.astype(int), x.astype(int)]
# do a 5 sigma clip on the science image
sci_mean = np.nanmean(sci_lrs_region)
sci_std = np.nanstd(sci_lrs_region)
upper = sci_mean + sci_std*5.0
lower = sci_mean - sci_std*5.0
mask_clean = np.logical_and(sci_lrs_region < upper, sci_lrs_region > lower)
sub = result_lrs_region - sci_lrs_region
mean_sub = np.absolute(np.mean(sub[mask_clean]))
atol = 0.1
rtol = 0.001
assert_allclose(mean_sub, 0, atol=atol, rtol=rtol)
# Test 3 Compare background subtracted science data (results)
# to a truth file.
truth_file = self.get_data(*self.ref_loc,
'miri_lrs_sci+bkg_masterbackgroundstep.fits')
result_file = result.meta.filename
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
result.close()
input_sci.close()
def test_master_background_userbg(_jail, user_background, science_image):
"""Verify data can run through the step with a user-supplied background"""
# Run with a user-supplied background and verify this is recorded in header
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
science_image,
config_file='config/master_background.cfg',
user_background=user_background,
)
assert type(science_image) is type(result)
assert result is not science_image
assert result.meta.cal_step.master_background == 'COMPLETE'
assert result.meta.background.master_background_file == 'user_background.fits'
def test_miri_masterbackground_mrs(self):
"""Run masterbackground step on MIRI LRS association"""
asn_file = self.get_data(*self.test_dir,
'miri_mrs_mbkg_0304_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background=True
)
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
def test_nirspec_masterbackground_mos_user1d(self):
"""
Regression test of master background subtraction for NRS MOS when a user 1-D spectrum is provided.
"""
# input file has 2-D background image added to it
input_file = self.get_data(*self.test_dir,
'nrs_mos_sci+bkg_cal.fits')
# user provide 1-D background was created from the 2-D background image
input_1dbkg_file = self.get_data(*self.test_dir,
'nrs_mos_bkg_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=input_1dbkg_file,
save_results=True)
# _________________________________________________________________________
# One of out tests is to compare the 1-D extracted spectra from
# the science image (no background added) and the masterbackground subtracted
# data.
# run resample_spec on results from MasterBackground step
result_2d = ResampleSpecStep.call(result)
# run 1-D extract on results from MasterBackground step
result_1d = Extract1dStep.call(result_2d)
# get input science data with background added
input_sci_cal_file = self.get_data(*self.test_dir,
'nrs_mos_sci_cal.fits')
# get 1-D extract on original science data without background
# this reference data was also run through ResampleSpec
input_sci_1d_file = self.get_data(*self.ref_loc,
'nrs_mos_sci_extract1dstep.fits')
input_sci = datamodels.open(input_sci_cal_file)
sci_cal_1d = datamodels.open(input_sci_1d_file)
num_spec = len(sci_cal_1d.spec)
# the user 1D spectrum may not cover the entire wavelength range of the
# science data. Find the wavelength range of user 1-D spectra
input_1dbkg_1d = datamodels.open(input_1dbkg_file)
user_wave = input_1dbkg_1d.spec[0].spec_table['wavelength']
user_flux = input_1dbkg_1d.spec[0].spec_table['flux']
user_wave_valid = np.where(user_flux > 0)
min_user_wave = np.amin(user_wave[user_wave_valid])
max_user_wave = np.amax(user_wave[user_wave_valid])
input_1dbkg_1d.close()
# loop over each slit and perform 2 tests on each slit
for i in range(num_spec):
# ______________________________________________________________________
# Test 1 compare extracted spectra data from the science data
# to extracted spectra from the output
# from MasterBackground subtraction.
sci_spec_1d = sci_cal_1d.spec[i].spec_table['flux']
sci_wave = sci_cal_1d.spec[i].spec_table['wavelength']
result_spec_1d = result_1d.spec[i].spec_table['flux']
# find the waverange covered by both user 1-D and science slit
sci_wave_valid = np.where(sci_spec_1d > 0)
min_wave = np.amin(sci_wave[sci_wave_valid])
max_wave = np.amax(sci_wave[sci_wave_valid])
if min_user_wave > min_wave:
min_wave = min_user_wave
if max_user_wave < max_wave:
max_wave = max_user_wave
sub_spec = sci_spec_1d - result_spec_1d
valid = np.where(np.logical_and(sci_wave > min_wave, sci_wave < max_wave))
sub_spec = sub_spec[valid]
mean_sub = np.nanmean(sub_spec)
atol = 1.5
assert_allclose(mean_sub, 0, atol=atol)
# ______________________________________________________________________
# Test 2 compare the science data with no background
# to the output from the masterBackground Subtraction step
# background subtracted science image.
bb = input_sci.slits[i].meta.wcs.bounding_box
x, y = grid_from_bounding_box(bb)
ra, dec, lam = input_sci.slits[i].meta.wcs(x, y)
valid = np.isfinite(lam)
sci = input_sci.slits[i].data
result_slit = result.slits[i].data
# check for outliers in the science image that could cause test
# to fail. These could be cosmic ray hits or other yeck that
# messes up the science data - ignores these cases
sci_mean = np.nanmean(sci[valid])
sci_std = np.nanstd(sci[valid])
upper = sci_mean + sci_std*5.0
lower = sci_mean - sci_std*5.0
mask_clean = np.logical_and(sci[valid] < upper, sci[valid] > lower)
# for this slit subtract the background subtracted data from
# the science data with no background added
sub = result_slit - sci
# do not look at outliers - they confuse things
sub_valid = sub[valid]
mean_sub = np.mean(sub_valid[mask_clean])
atol = 0.1
assert_allclose(np.absolute(mean_sub), 0, atol=atol)
# ______________________________________________________________________
# Test 3 Compare background sutracted science data (results)
# to a truth file. This data is MultiSlit data
result_file = result.meta.filename
ref_file = self.get_data(*self.ref_loc,
'nrs_mos_sci+bkg_masterbackgroundstep.fits')
outputs = [(result_file, ref_file)]
self.compare_outputs(outputs)
input_sci.close()
result.close()
def test_nirspec_ifu_masterbg_user(self):
"""
Regression test of master background subtraction for NRS IFU when a
user 1-D spectrum is provided.
"""
# input file has 2-D background image added to it
input_file = self.get_data(*self.test_dir, 'prism_sci_bkg_cal.fits')
# user-provided 1-D background was created from the 2-D background image
user_background = self.get_data(*self.test_dir, 'prism_bkg_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=user_background,
save_results=True)
# Test 1 compare extracted spectra data with
# no background added to extracted spectra from the output
# from MasterBackground subtraction. First cube_build has to be run
# on the data.
result_s3d = CubeBuildStep.call(result)
# run 1-D extract on results from MasterBackground step
result_1d = Extract1dStep.call(result_s3d, subtract_background=False)
# get the 1-D extracted spectrum from the science data in truth directory
input_sci_1d_file = self.get_data(*self.ref_loc, 'prism_sci_extract1d.fits')
sci_1d = datamodels.open(input_sci_1d_file)
# read in the valid wavelengths of the user-1d
user_background_model = datamodels.open(user_background)
user_wave = user_background_model.spec[0].spec_table['wavelength']
user_flux = user_background_model.spec[0].spec_table['flux']
user_wave_valid = np.where(user_flux > 0)
min_user_wave = np.amin(user_wave[user_wave_valid])
max_user_wave = np.amax(user_wave[user_wave_valid])
user_background_model.close()
# find the waverange covered by both user and science
sci_spec_1d = sci_1d.spec[0].spec_table['flux']
sci_spec_wave = sci_1d.spec[0].spec_table['wavelength']
result_spec_1d = result_1d.spec[0].spec_table['flux']
sci_wave_valid = np.where(sci_spec_1d > 0)
min_wave = np.amin(sci_spec_wave[sci_wave_valid])
max_wave = np.amax(sci_spec_wave[sci_wave_valid])
if min_user_wave > min_wave:
min_wave = min_user_wave
if max_user_wave < max_wave:
max_wave = max_user_wave
sub_spec = sci_spec_1d - result_spec_1d
valid = np.where(np.logical_and(sci_spec_wave > min_wave, sci_spec_wave < max_wave))
sub_spec = sub_spec[valid]
sub_spec = sub_spec[1:-2] # endpoints are wacky
mean_sub = np.absolute(np.nanmean(sub_spec))
atol = 5.0
assert_allclose(mean_sub, 0, atol=atol)
# Test 2 compare the science data with no background
# to the output from the masterBackground Subtraction step
# background subtracted science image.
input_sci_cal_file = self.get_data(*self.test_dir,
'prism_sci_cal.fits')
input_sci_model = datamodels.open(input_sci_cal_file)
# We don't want the slices gaps to impact the statisitic
# loop over the 30 Slices
for i in range(30):
slice_wcs = nirspec.nrs_wcs_set_input(input_sci_model, i)
x, y = grid_from_bounding_box(slice_wcs.bounding_box)
ra, dec, lam = slice_wcs(x, y)
valid = np.isfinite(lam)
result_slice_region = result.data[y.astype(int), x.astype(int)]
sci_slice_region = input_sci_model.data[y.astype(int),
x.astype(int)]
sci_slice = sci_slice_region[valid]
result_slice = result_slice_region[valid]
sub = result_slice - sci_slice
# check for outliers in the science image
sci_mean = np.nanmean(sci_slice)
sci_std = np.nanstd(sci_slice)
upper = sci_mean + sci_std*5.0
lower = sci_mean - sci_std*5.0
mask_clean = np.logical_and(sci_slice < upper, sci_slice > lower)
sub_mean = np.absolute(np.nanmean(sub[mask_clean]))
atol = 2.0
assert_allclose(sub_mean, 0, atol=atol)
# Test 3 Compare background sutracted science data (results)
# to a truth file. This data is MultiSlit data
input_sci_model.close()
result_file = result.meta.filename
truth_file = self.get_data(*self.ref_loc,
'prism_sci_bkg_masterbackgroundstep.fits')
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
input_sci_model.close()
result.close()
def test_nirspec_masterbackground_mos_user1d(self):
"""
Regression test of master background subtraction for NRS MOS when a user 1-D spectrum is provided.
"""
# input file has 2-D background image added to it
input_file = self.get_data(*self.test_dir, 'nrs_mos_sci+bkg_cal.fits')
# user provide 1-D background was created from the 2-D background image
input_1dbkg_file = self.get_data(*self.test_dir,
'nrs_mos_bkg_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=input_1dbkg_file,
save_results=True)
# _________________________________________________________________________
# One of out tests is to compare the 1-D extracted spectra from
# the science image (no background added) and the masterbackground subtracted
# data.
# run resample_spec on results from MasterBackground step
result_2d = ResampleSpecStep.call(result)
# run 1-D extract on results from MasterBackground step
result_1d = Extract1dStep.call(result_2d)
# get input science data with background added
input_sci_cal_file = self.get_data(*self.test_dir,
'nrs_mos_sci_cal.fits')
# get 1-D extract on original science data without background
# this reference data was also run through ResampleSpec
input_sci_1d_file = self.get_data(*self.ref_loc,
'nrs_mos_sci_extract1dstep.fits')
input_sci = datamodels.open(input_sci_cal_file)
sci_cal_1d = datamodels.open(input_sci_1d_file)
num_spec = len(sci_cal_1d.spec)
# the user 1D spectrum may not cover the entire wavelength range of the
# science data. Find the wavelength range of user 1-D spectra
input_1dbkg_1d = datamodels.open(input_1dbkg_file)
user_wave = input_1dbkg_1d.spec[0].spec_table['wavelength']
user_flux = input_1dbkg_1d.spec[0].spec_table['flux']
user_wave_valid = np.where(user_flux > 0)
min_user_wave = np.amin(user_wave[user_wave_valid])
max_user_wave = np.amax(user_wave[user_wave_valid])
input_1dbkg_1d.close()
# loop over each slit and perform 2 tests on each slit
for i in range(num_spec):
# ______________________________________________________________________
# Test 1 compare extracted spectra data from the science data
# to extracted spectra from the output
# from MasterBackground subtraction.
sci_spec_1d = sci_cal_1d.spec[i].spec_table['flux']
sci_wave = sci_cal_1d.spec[i].spec_table['wavelength']
result_spec_1d = result_1d.spec[i].spec_table['flux']
# find the waverange covered by both user 1-D and science slit
sci_wave_valid = np.where(sci_spec_1d > 0)
min_wave = np.amin(sci_wave[sci_wave_valid])
max_wave = np.amax(sci_wave[sci_wave_valid])
if min_user_wave > min_wave:
min_wave = min_user_wave
if max_user_wave < max_wave:
max_wave = max_user_wave
sub_spec = sci_spec_1d - result_spec_1d
valid = np.where(
np.logical_and(sci_wave > min_wave, sci_wave < max_wave))
sub_spec = sub_spec[valid]
mean_sub = np.nanmean(sub_spec)
atol = 1.5
assert_allclose(mean_sub, 0, atol=atol)
# ______________________________________________________________________
# Test 2 compare the science data with no background
# to the output from the masterBackground Subtraction step
# background subtracted science image.
bb = input_sci.slits[i].meta.wcs.bounding_box
x, y = grid_from_bounding_box(bb)
ra, dec, lam = input_sci.slits[i].meta.wcs(x, y)
valid = np.isfinite(lam)
sci = input_sci.slits[i].data
result_slit = result.slits[i].data
# check for outliers in the science image that could cause test
# to fail. These could be cosmic ray hits or other yeck that
# messes up the science data - ignores these cases
sci_mean = np.nanmean(sci[valid])
sci_std = np.nanstd(sci[valid])
upper = sci_mean + sci_std * 5.0
lower = sci_mean - sci_std * 5.0
mask_clean = np.logical_and(sci[valid] < upper, sci[valid] > lower)
# for this slit subtract the background subtracted data from
# the science data with no background added
sub = result_slit - sci
# do not look at outliers - they confuse things
sub_valid = sub[valid]
mean_sub = np.mean(sub_valid[mask_clean])
atol = 0.1
assert_allclose(np.absolute(mean_sub), 0, atol=atol)
# ______________________________________________________________________
# Test 3 Compare background sutracted science data (results)
# to a truth file. This data is MultiSlit data
result_file = result.meta.filename
ref_file = self.get_data(*self.ref_loc,
'nrs_mos_sci+bkg_masterbackgroundstep.fits')
outputs = [(result_file, ref_file)]
self.compare_outputs(outputs)
input_sci.close()
result.close()
def test_miri_masterbackground_lrs_user1d(self):
"""
Regression test of masterbackgound subtraction with lrs, with user provided 1-D background
"""
# input file has the background added
# Copy original version of file to test file, which will get overwritten by test
input_file = self.get_data(*self.test_dir,
'miri_lrs_sci+bkg_cal.fits')
# user provided 1-D background
input_1d_bkg_file = self.get_data(*self.test_dir,
'miri_lrs_bkg_x1d.fits')
input_1d_bkg_model = datamodels.open(input_1d_bkg_file)
result = MasterBackgroundStep.call(input_file,
user_background=input_1d_bkg_file,
save_results=True)
# Test 1
# Run extract1D on the master background subtracted data (result) and
# the science data with no background added
# run 1-D extract on results from MasterBackground step
result_1d = Extract1dStep.call(result)
# get the 1-D extracted Spectrum from the science data with
# no background added to test against
sci_cal_1d_file = self.get_data(*self.ref_loc,
'miri_lrs_sci_extract1d.fits')
sci_cal_1d = datamodels.open(sci_cal_1d_file)
sci_cal_1d_data = sci_cal_1d.spec[0].spec_table['flux']
result_1d_data = result_1d.spec[0].spec_table['flux']
sci_wave = sci_cal_1d.spec[0].spec_table['wavelength']
# find the valid wavelenth of the user provided spectrun
user_wave = input_1d_bkg_model.spec[0].spec_table['wavelength']
user_flux = input_1d_bkg_model.spec[0].spec_table['flux']
user_wave_valid = np.where(user_flux > 0)
min_user_wave = np.amin(user_wave[user_wave_valid])
max_user_wave = np.amax(user_wave[user_wave_valid])
input_1d_bkg_model.close()
# find the waverange covered by both user and science
sci_wave_valid = np.where(sci_cal_1d_data > 0)
min_wave = np.amin(sci_wave[sci_wave_valid])
max_wave = np.amax(sci_wave[sci_wave_valid])
if min_user_wave > min_wave:
min_wave = min_user_wave
if max_user_wave < max_wave:
max_wave = max_user_wave
# Compare the data
sub_spec = sci_cal_1d_data - result_1d_data
valid = np.where(np.logical_and(sci_wave > min_wave, sci_wave < max_wave))
sub_spec = sub_spec[valid]
mean_sub = np.absolute(np.nanmean(sub_spec))
atol = 0.00005
rtol = 0.000001
assert_allclose(mean_sub, 0, atol=atol, rtol=rtol)
# Test 2
# Compare result (background subtracted image) to science image with no
# background. Subtract these images, smooth the subtracted image and
# the mean should be close to zero.
input_sci_cal_file = self.get_data(*self.test_dir,
'miri_lrs_sci_cal.fits')
input_sci = datamodels.open(input_sci_cal_file)
# find the LRS region
bb = result.meta.wcs.bounding_box
x, y = grid_from_bounding_box(bb)
result_lrs_region = result.data[y.astype(int), x.astype(int)]
sci_lrs_region = input_sci.data[y.astype(int), x.astype(int)]
# do a 5 sigma clip on the science image
sci_mean = np.nanmean(sci_lrs_region)
sci_std = np.nanstd(sci_lrs_region)
upper = sci_mean + sci_std*5.0
lower = sci_mean - sci_std*5.0
mask_clean = np.logical_and(sci_lrs_region < upper, sci_lrs_region > lower)
sub = result_lrs_region - sci_lrs_region
mean_sub = np.absolute(np.mean(sub[mask_clean]))
atol = 0.5
rtol = 0.001
assert_allclose(mean_sub, 0, atol=atol, rtol=rtol)
# Test 3 Compare background subtracted science data (results)
# to a truth file.
truth_file = self.get_data(*self.ref_loc,
'miri_lrs_sci+bkg_masterbackgroundstep.fits')
result_file = result.meta.filename
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
result.close()
input_sci.close()
def test_nirspec_ifu_masterbg_user(self):
"""
Regression test of master background subtraction for NRS IFU when a
user 1-D spectrum is provided.
"""
# input file has 2-D background image added to it
input_file = self.get_data(*self.test_dir, 'prism_sci_bkg_cal.fits')
# user-provided 1-D background was created from the 2-D background image
user_background = self.get_data(*self.test_dir, 'prism_bkg_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=user_background,
save_results=True)
# Test 1 compare extracted spectra data with
# no background added to extracted spectra from the output
# from MasterBackground subtraction. First cube_build has to be run
# on the data.
result_s3d = CubeBuildStep.call(result)
# run 1-D extract on results from MasterBackground step
result_1d = Extract1dStep.call(result_s3d, subtract_background=False)
# get the 1-D extracted spectrum from the science data in truth directory
input_sci_1d_file = self.get_data(*self.ref_loc, 'prism_sci_extract1d.fits')
sci_1d = datamodels.open(input_sci_1d_file)
# read in the valid wavelengths of the user-1d
user_background_model = datamodels.open(user_background)
user_wave = user_background_model.spec[0].spec_table['wavelength']
user_flux = user_background_model.spec[0].spec_table['net']
user_wave_valid = np.where(user_flux > 0)
min_user_wave = np.amin(user_wave[user_wave_valid])
max_user_wave = np.amax(user_wave[user_wave_valid])
user_background_model.close()
# find the waverange covered by both user and science
sci_spec_1d = sci_1d.spec[0].spec_table['net']
sci_spec_wave = sci_1d.spec[0].spec_table['wavelength']
result_spec_1d = result_1d.spec[0].spec_table['net']
sci_wave_valid = np.where(sci_spec_1d > 0)
min_wave = np.amin(sci_spec_wave[sci_wave_valid])
max_wave = np.amax(sci_spec_wave[sci_wave_valid])
if min_user_wave > min_wave:
min_wave = min_user_wave
if max_user_wave < max_wave:
max_wave = max_user_wave
sub_spec = sci_spec_1d - result_spec_1d
valid = np.where(np.logical_and(sci_spec_wave > min_wave, sci_spec_wave < max_wave))
sub_spec = sub_spec[valid]
sub_spec = sub_spec[1:-2] # endpoints are wacky
mean_sub = np.absolute(np.nanmean(sub_spec))
atol = 5.0
assert_allclose(mean_sub, 0, atol=atol)
# Test 2 compare the science data with no background
# to the output from the masterBackground Subtraction step
# background subtracted science image.
input_sci_cal_file = self.get_data(*self.test_dir,
'prism_sci_cal.fits')
input_sci_model = datamodels.open(input_sci_cal_file)
# We don't want the slices gaps to impact the statisitic
# loop over the 30 Slices
for i in range(30):
slice_wcs = nirspec.nrs_wcs_set_input(input_sci_model, i)
x, y = grid_from_bounding_box(slice_wcs.bounding_box)
ra, dec, lam = slice_wcs(x, y)
valid = np.isfinite(lam)
result_slice_region = result.data[y.astype(int), x.astype(int)]
sci_slice_region = input_sci_model.data[y.astype(int),
x.astype(int)]
sci_slice = sci_slice_region[valid]
result_slice = result_slice_region[valid]
sub = result_slice - sci_slice
# check for outliers in the science image
sci_mean = np.nanmean(sci_slice)
sci_std = np.nanstd(sci_slice)
upper = sci_mean + sci_std*5.0
lower = sci_mean - sci_std*5.0
mask_clean = np.logical_and(sci_slice < upper, sci_slice > lower)
sub_mean = np.absolute(np.nanmean(sub[mask_clean]))
atol = 2.0
assert_allclose(sub_mean, 0, atol=atol)
# Test 3 Compare background sutracted science data (results)
# to a truth file. This data is MultiSlit data
input_sci_model.close()
result_file = result.meta.filename
truth_file = self.get_data(*self.ref_loc,
'prism_sci_bkg_masterbackgroundstep.fits')
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
input_sci_model.close()
result.close()