def test_miri_lrs_slit_wcs(run_pipeline, rtdata_module,
fitsdiff_default_kwargs):
rtdata = rtdata_module
# get input assign_wcs and truth file
output = "jw00623032001_03102_00001_mirimage_assign_wcs.fits"
rtdata.output = output
rtdata.get_truth(f"truth/test_miri_lrs_slit_spec2/{output}")
# Compare the output and truth file
with datamodels.open(output) as im, datamodels.open(
rtdata.truth) as im_truth:
x, y = grid_from_bounding_box(im.meta.wcs.bounding_box)
ra, dec, lam = im.meta.wcs(x, y)
ratruth, dectruth, lamtruth = im_truth.meta.wcs(x, y)
assert_allclose(ra, ratruth)
assert_allclose(dec, dectruth)
assert_allclose(lam, lamtruth)
# Test the inverse transform
xtest, ytest = im.meta.wcs.backward_transform(ra, dec, lam)
xtruth, ytruth = im_truth.meta.wcs.backward_transform(
ratruth, dectruth, lamtruth)
assert_allclose(xtest, xtruth)
assert_allclose(ytest, ytruth)
def test_create_combined(_jail, wfs_association, data1, data2, dq1, dq2,
result_data, result_dq):
path_asn, path1, path2 = wfs_association
# Populate data and DQ array at [5, 5] with what is to be tested
with datamodels.open(path1) as im1, datamodels.open(path2) as im2:
# Populate some data and DQ flags for the test
im1.data[5, 5] = data1
im2.data[5, 5] = data2
im1.dq[5, 5] = dq1
im2.dq[5, 5] = dq2
im1.save(path1)
im2.save(path2)
result = WfsCombineStep.call(path_asn,
do_refine=False,
flip_dithers=False,
psf_size=50,
blur_size=10,
n_size=2)
# Check that results are as expected
assert result[0].data[5, 5] == result_data
assert result[0].dq[5, 5] == result_dq
def test_open():
with datamodels.open() as dm:
pass
with datamodels.open((50, 50)) as dm:
pass
warnings.simplefilter("ignore")
with datamodels.open(FITS_FILE) as dm:
assert isinstance(dm, QuadModel)
def test_open():
with datamodels.open():
pass
with datamodels.open((50, 50)):
pass
with pytest.warns(datamodels.util.NoTypeWarning):
with datamodels.open(FITS_FILE) as dm:
assert isinstance(dm, QuadModel)
def __init__(self, fitopt_filename, ramp_filename, rate_filename, xlim=[500, 530], ylim=[500, 530]):
self.fitopt_file = os.path.basename(fitopt_filename)
self.ramp_file = os.path.basename(ramp_filename)
self.rate_file = os.path.basename(rate_filename)
self.fitopt_dm = datamodels.RampFitOutputModel(fitopt_filename)
self.ramp_dm = datamodels.open(ramp_filename)
self.rate_dm = datamodels.open(rate_filename)
self.xlim = xlim
self.ylim = ylim
def test_nirspec_ifu_user_supplied_flat(rtdata, fitsdiff_default_kwargs):
"""Test using predefined interpolated flat"""
data = dm.open(rtdata.get_data('nirspec/ifu/nrs_ifu_nrs1_assign_wcs.fits'))
user_supplied_flat = dm.open(rtdata.get_data('nirspec/ifu/nrs_ifu_nrs1_interpolated_flat.fits'))
nirspec_ifu(data, None, None, None, None, user_supplied_flat=user_supplied_flat)
rtdata.output = 'ff_using_interpolated.fits'
data.write(rtdata.output)
rtdata.get_truth(TRUTH_PATH + '/' + 'ff_using_interpolated.fits')
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
assert diff.identical, diff.report()
def test_modelcontainer_error_from_asn(tmpdir):
from jwst.associations.asn_from_list import asn_from_list
asn = asn_from_list(["foo.fits"], product_name="foo_out")
name, serialized = asn.dump(format="json")
path = str(tmpdir.join(name))
with open(path, "w") as f:
f.write(serialized)
# The foo.fits file doesn't exist
with pytest.raises(FileNotFoundError):
datamodels.open(path)
def test_refine_no_error(wfs_association, xshift, yshift, xerror, yerror,
flip_dithers):
data_size = 201
path_asn, path1, path2 = wfs_association
nircam_pixel_size = 0.031 / 3600.
delta_x_pixel = xshift
# positive y pixel changes are negative declination changes
delta_y_pixel = -1.0 * yshift
with datamodels.open(path1) as im1:
im1.data = np.zeros(shape=(data_size, data_size), dtype=np.float32)
im1.dq = np.zeros(shape=(data_size, data_size), dtype=np.int32)
im1.data = add_point_source(im1.data, 200, 100, 100, 4, 4)
im1.save(path1)
with datamodels.open(path2) as im2:
im2.data = np.zeros(shape=(data_size, data_size), dtype=np.float32)
im2.dq = np.zeros(shape=(data_size, data_size), dtype=np.int32)
im2.meta.wcsinfo = {
'dec_ref': 11.99875540218638 + delta_y_pixel * nircam_pixel_size,
'ra_ref': 22.02351763251896 + delta_x_pixel * nircam_pixel_size,
'roll_ref': 0.005076934167039675,
'v2_ref': 86.039011,
'v3_ref': -493.385704,
'v3yangle': -0.07385127,
'vparity': -1,
'wcsaxes': 2
}
im2 = AssignWcsStep.call(im2, sip_approx=False)
# shift the actual location of the 2nd image including the error in the WCS position
im2.data = add_point_source(im2.data, 200,
100 + delta_x_pixel + xerror,
100 + delta_y_pixel - yerror, 4, 4)
im2.save(path2)
wfs = wfs_combine.DataSet(path1,
path2,
'outfile.fits',
do_refine=True,
flip_dithers=flip_dithers,
psf_size=50,
blur_size=10,
n_size=2)
wfs.do_all()
assert np.max(abs(wfs.diff)) < 0.001
if delta_x_pixel + xerror >= 0 or not flip_dithers:
assert wfs.off_x == delta_x_pixel + xerror
# Positive y pixel errors are negative in declination
assert wfs.off_y == delta_y_pixel - yerror
else: # Order of exposures and sign of shifts have switched to always align the dithers
assert wfs.off_x == abs(delta_x_pixel + xerror)
# Positive y pixel errors are negative in declination
assert wfs.off_y == -1.0 * (delta_y_pixel - yerror)
wfs.input_1.close()
wfs.input_2.close()
def test_modelcontainer_error_from_asn(tmp_path):
asn = asn_from_list(["foo.fits"], product_name="foo_out")
name, serialized = asn.dump(format="json")
# The following Path object needs to be stringified because
# datamodels.open() doesn't deal with pathlib objects when they are
# .json files
path = str(tmp_path / name)
with open(path, "w") as f:
f.write(serialized)
# The foo.fits file doesn't exist
with pytest.raises(FileNotFoundError):
datamodels.open(path)
def process(self, *args):
science = datamodels.open(self.science_filename)
if self.flat_filename is None:
self.flat_filename = join(dirname(__file__), "data/flat.fits")
flat = datamodels.open(self.flat_filename)
calibrated = []
calibrated.append(self.flat_field(science, flat))
combined = self.combine(calibrated)
self.display(combined)
dm = datamodels.ImageModel(combined)
dm.save(self.output_filename)
science.close()
flat.close()
return dm
def test_guess(guess):
"""Test the guess parameter to the open func"""
path = Path(t_path('test.fits'))
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "model_type not found")
if guess is None or guess:
# Default is to guess at the model type.
with datamodels.open(path, guess=guess) as model:
assert isinstance(model, JwstDataModel)
else:
# Without guessing, the call should fail.
with pytest.raises(TypeError):
with datamodels.open(path, guess=guess) as model:
pass
def test_pre_1_2_2_schemas(truth_path, rtdata):
rtdata.env = "1.2.1"
rtdata.get_truth(truth_path)
# Confirm that the file doesn't initially validate
# (open with fits first so that the failed call to open doesn't leave behind an open file)
with fits.open(rtdata.truth, memmap=False) as hdu:
with pytest.raises(ValueError, match="Column names don't match schema"):
with datamodels.open(hdu):
pass
subprocess.check_call(["migrate_data", rtdata.truth, "--in-place"])
# Now the model should validate
with datamodels.open(rtdata.truth):
pass
def test_nircam_image_stage2_wcs(run_image2pipeline, rtdata_module):
"""Test that WCS object works as expected"""
rtdata = rtdata_module
rtdata.input = "jw42424001001_01101_00001_nrca5_uncal.fits"
output = "jw42424001001_01101_00001_nrca5_assign_wcs.fits"
rtdata.output = output
rtdata.get_truth(f"truth/test_nircam_image_stages/{output}")
with datamodels.open(rtdata.output) as model, datamodels.open(rtdata.truth) as model_truth:
grid = grid_from_bounding_box(model.meta.wcs.bounding_box)
ra, dec = model.meta.wcs(*grid)
ra_truth, dec_truth = model_truth.meta.wcs(*grid)
assert_allclose(ra, ra_truth)
assert_allclose(dec, dec_truth)
def test_shift_order_no_refine_with_flip(wfs_association):
# change the sign of the the dither and check that we get the same delta pix but with switched
# order of images.
path_asn, path1, path2 = wfs_association
nircam_pixel_size = 0.031 / 3600.
delta_pixel = -5
with datamodels.open(path2) as im2:
im2.meta.wcsinfo = {
'dec_ref': 11.99875540218638,
'ra_ref': 22.02351763251896 + delta_pixel * nircam_pixel_size,
'roll_ref': 0.005076934167039675,
'v2_ref': 86.039011,
'v3_ref': -493.385704,
'v3yangle': -0.07385127,
'vparity': -1,
'wcsaxes': 2
}
im2 = AssignWcsStep.call(im2, sip_approx=False)
im2.save(path2)
wfs = wfs_combine.DataSet(path1,
path2,
'outfile.fits',
do_refine=False,
flip_dithers=True,
psf_size=50,
blur_size=10,
n_size=2)
wfs.do_all()
wfs.input_1.close()
wfs.input_2.close()
# assert wfs.input_1.meta.observation.exposure_number == '2'
# assert wfs.input_2.meta.observation.exposure_number == '1'
assert wfs.off_x == -1 * delta_pixel
def _dm_setval(filepath, key, value):
"""Set metadata `key` in file `filepath` to `value` using jwst datamodel.
"""
from jwst import datamodels
with datamodels.open(filepath) as d_model:
d_model[key.lower()] = value
d_model.save(filepath)
def test_step_neg_shift_no_refine_no_flip(wfs_association):
path_asn, path1, path2 = wfs_association
nircam_pixel_size = 0.031 / 3600.
delta_pixel = -5
with datamodels.open(path2) as im2:
im2.meta.wcsinfo = {
'dec_ref': 11.99875540218638,
'ra_ref': 22.02351763251896 + delta_pixel * nircam_pixel_size,
'roll_ref': 0.005076934167039675,
'v2_ref': 86.039011,
'v3_ref': -493.385704,
'v3yangle': -0.07385127,
'vparity': -1,
'wcsaxes': 2
}
im2 = AssignWcsStep.call(im2, sip_approx=False)
im2.save(path2)
wfs = WfsCombineStep.call(path_asn,
do_refine=False,
flip_dithers=False,
psf_size=50,
blur_size=10,
n_size=2)
assert wfs[0].meta.wcsinfo.ra_ref == 22.02351763251896
def test_add_wcs_default(data_file, tmp_path):
"""Handle when no pointing exists and the default is used."""
expected_name = 'add_wcs_default.fits'
try:
stp.add_wcs(data_file,
siaf_path=siaf_path,
tolerance=0,
allow_default=True)
except ValueError:
pass # This is what we want for the test.
except Exception as e:
pytest.skip('Live ENGDB service is not accessible.'
'\nException={}'.format(e))
# Tests
with datamodels.Level1bModel(data_file) as model:
# Save for post-test comparison and update
model.save(tmp_path / expected_name)
with datamodels.open(DATA_PATH / expected_name) as expected:
for meta in METAS_EQUALITY:
assert model[meta] == expected[meta], f'{meta} has changed'
for meta in METAS_ISCLOSE:
assert np.isclose(model[meta],
expected[meta]), f'{meta} has changed'
assert word_precision_check(model.meta.wcsinfo.s_region,
expected.meta.wcsinfo.s_region)
def get_reference_file(dataset, reference_file_type):
"""
Gets a reference file from CRDS as a readable file-like object.
The actual file may be optionally overridden.
Parameters
----------
input_file : jwst.datamodels.ModelBase instance
A model of the input file. Metadata on this input file will
be used by the CRDS "bestref" algorithm to obtain a reference
file.
reference_file_type : string
The type of reference file to retrieve. For example, to
retrieve a flat field reference file, this would be 'flat'.
Returns
-------
reference_filepath : string
The path of the reference in the CRDS file cache.
"""
if isinstance(dataset, str):
from jwst import datamodels
with datamodels.open(dataset) as model:
return get_multiple_reference_paths(model, [reference_file_type])[reference_file_type]
else:
return get_multiple_reference_paths(dataset, [reference_file_type])[reference_file_type]
def test_from_hdulist():
from astropy.io import fits
warnings.simplefilter("ignore")
with fits.open(FITS_FILE, memmap=False) as hdulist:
with datamodels.open(hdulist) as dm:
dm.data
assert not hdulist.fileinfo(0)['file'].closed
def test_input_dir_with_model(mk_tmp_dirs):
"""Use with an already opened DataModel"""
with datamodels.open(t_path('data/flat.fits')) as model:
step = StepWithModel()
step.run(model)
assert step.input_dir == ''
def process(self, input, bkg_list):
"""
Subtract the background signal from target exposures by subtracting
designated background images from them.
Parameters
----------
input: data model
input target data model to which background subtraction is applied
bkg_list: filename list
list of background exposure file names
Returns
-------
result: data model
the background-subtracted target data model
"""
# Load the input data model
with datamodels.open(input) as input_model:
# Do the background subtraction
result = background_sub.background_sub(input_model, bkg_list,
self.sigma, self.maxiters)
result.meta.cal_step.back_sub = "COMPLETE"
return result
def __init__(self, input_DM):
"""
Short Summary
-------------
Set file name of persistence file
Parameters
----------
input_DM: data model object
input Data Model object
"""
try:
self.input_file = input_DM
model = datamodels.open(input_DM)
# If model comes back as generic DataModel, reopen as MultiSlit
if isinstance(model, datamodels.CubeModel) or isinstance(
model, datamodels.ImageModel):
pass
elif isinstance(model, datamodels.DataModel):
model.close()
model = datamodels.MultiSlitModel(input_DM)
self.input = model
except Exception as errmess:
log.error('Error opening %s', input_DM)
self.input = None
def test_add_wcs_method_full_siafdb(eng_db_ngas, data_file, tmp_path):
"""Test using the database and a specified siaf db"""
expected_name = 'add_wcs_method_full_siafdb.fits'
# Calculate
stp.add_wcs(data_file,
siaf_path=siaf_path,
method=stp.Methods.OPS_TR_202111,
engdb_url='http://localhost')
# Test
with datamodels.Level1bModel(data_file) as model:
# Save for post-test comparison and update
model.save(tmp_path / expected_name)
with datamodels.open(DATA_PATH / expected_name) as expected:
for meta in METAS_EQUALITY:
if isinstance(model[meta], str):
assert model[meta] == expected[meta]
else:
assert np.isclose(model[meta], expected[meta], atol=1e-13)
for meta in METAS_ISCLOSE:
assert np.isclose(model[meta], expected[meta])
assert word_precision_check(model.meta.wcsinfo.s_region,
expected.meta.wcsinfo.s_region)
def test_add_wcs_method_full_nosiafdb(eng_db_ngas, data_file, tmp_path):
"""Test using the database"""
# Only run if `pysiaf` is installed.
pytest.importorskip('pysiaf')
expected_name = 'add_wcs_method_full_nosiafdb.fits'
# Calculate
stp.add_wcs(data_file,
method=stp.Methods.OPS_TR_202111,
engdb_url='http://localhost')
# Tests
with datamodels.Level1bModel(data_file) as model:
# Save for post-test comparison and update
model.save(tmp_path / expected_name)
with datamodels.open(DATA_PATH / expected_name) as expected:
for meta in METAS_EQUALITY:
assert model[meta] == expected[meta]
for meta in METAS_ISCLOSE:
assert np.isclose(model[meta], expected[meta])
assert word_precision_check(model.meta.wcsinfo.s_region,
expected.meta.wcsinfo.s_region)
def process(self, input):
"""
Step for Nonlinearity Correction
"""
# Load the input data model
with datamodels.open(input) as input_model:
nonlin_coeff_file = self.get_reference_file(
input_model, "nonlincoeff")
nonlin_coeff_data = fits.getdata(nonlin_coeff_file).astype(
np.float32)
c0 = nonlin_coeff_data[0]
c1 = nonlin_coeff_data[1]
c2 = nonlin_coeff_data[2]
c3 = nonlin_coeff_data[3]
c4 = nonlin_coeff_data[4]
# Get the reference file names
input_data = input_model.data[:, :, :, :].astype(np.int32)
ramp_list = []
for it in range(len(input_data)):
ramp_data = input_data[it]
num_reads = len(ramp_data)
time_arr = np.arange(0, num_reads, 1).astype(np.int32)
result = nonlin_c(ramp_data, time_arr, c0, c1, c2, c3, c4)
result = result.astype(np.int32)
ramp_list.append(result)
#result = uptheramp_c(result,time_arr)
output_data = np.array(ramp_list).astype(np.int32)
result = iris_pipeline.datamodels.TMTRampModel(data=output_data)
result.update(input_model)
return result
def _collect_wcs_keywords(f):
# Get keywords to go in SCI header from the datamodels schema
dm = datamodels.open(f, pass_invalid_values=True)
stsci_wcs_kw = []
for k, v in _build_schema2fits_dict(dm.meta._schema).items():
if v[1] == 'SCI':
if v[0] != 'WCSAXES':
stsci_wcs_kw.append(v[0])
dm.close()
# Convert the WCS to a header
try:
w = wcs.WCS(f[0].header)
except InvalidTransformError:
if f[0].header['cunit3'] and f[0].header['cunit3'] == 'micron':
f[0].header['cunit3'] = 'um'
w = wcs.WCS(f[0].header)
new_hdr = w.to_header()
# Remove kw added by wcslib from the new header
for kw in wcslib_kw_to_remove:
try:
del new_hdr[kw]
except KeyError:
continue
# Add STScI specific kw to new_hdr
for kw in stsci_wcs_kw:
try:
new_hdr[kw] = f[0].header[kw]
except KeyError:
pass
new_hdr = add_default_keywords(new_hdr)
return new_hdr
def get_data_model_flat_dict(filepath, needed_keys=()):
"""Get the header from `filepath` using the jwst data model."""
from jwst import datamodels
with log.augment_exception("JWST Data Model (jwst.datamodels)"):
with datamodels.open(filepath) as d_model:
flat_dict = d_model.to_flat_dict(include_arrays=False)
return flat_dict
def test_saturation_step(fits_input):
"""Make sure the DQInitStep runs without error."""
fname = fits_input[0].header['filename'].replace('.fits',
'_saturationstep.fits')
SaturationStep.call(datamodels.open(fits_input),
output_file=fname,
save_results=True)
def test_nirspec_gwa(_jail, background, science_image):
"""Verify NIRSPEC GWA logic for in the science and background"""
# open the background to read in the GWA values
back_image = datamodels.open(background)
science_image.meta.instrument.gwa_xtilt = back_image.meta.instrument.gwa_xtilt
science_image.meta.instrument.gwa_ytilt = back_image.meta.instrument.gwa_ytilt
science_image.meta.instrument.gwa_tilt = back_image.meta.instrument.gwa_tilt
bkg = [background]
# Test Run with GWA values the same - confirm it runs
# And gives the predicted result
collect_pipeline_cfgs('./config')
result = BackgroundStep.call(
science_image,
bkg,
config_file='config/background.cfg',
)
test = science_image.data - back_image.data
assert_allclose(result.data, test)
assert type(result) is type(science_image)
assert result.meta.cal_step.back_sub == 'COMPLETE'
back_image.close()
def test_lastframe_step(fits_input):
"""Make sure the LastFrameStep runs without error."""
fname = fits_input[0].header['filename'].split('.fits',
'_lastframestep.fits')
LastFrameStep.call(datamodels.open(fits_input),
output_file=fname,
save_results=True)
def _collect_wcs_keywords(f):
# Get keywords to go in SCI header from the datamodels schema
dm = datamodels.open(f, pass_invalid_values=True)
stsci_wcs_kw = []
for k,v in datamodels.schema.build_schema2fits_dict(dm.meta._schema).items():
if v[1] == 'SCI':
if v[0] != 'WCSAXES':
stsci_wcs_kw.append(v[0])
dm.close()
# Convert the WCS to a header
try:
w = wcs.WCS(f[0].header)
except InvalidTransformError:
if f[0].header['cunit3'] and f[0].header['cunit3'] == 'micron':
f[0].header['cunit3'] = 'um'
w = wcs.WCS(f[0].header)
new_hdr = w.to_header()
# Remove kw added by wcslib from the new header
for kw in wcslib_kw_to_remove:
try:
del new_hdr[kw]
except KeyError:
continue
# Add STScI specific kw to new_hdr
for kw in stsci_wcs_kw:
try:
new_hdr[kw] = f[0].header[kw]
except KeyError:
pass
new_hdr = add_default_keywords(new_hdr)
return new_hdr
def replace_ints(self, dir, int_dirs):
"""
replace the integrations in the full array with the int arrays
"""
my_output_files = glob.glob(os.path.join(dir, 'det_images', '*.fits'))
for f in my_output_files:
out_dm = datamodels.open(f)
for n, i in enumerate(int_dirs):
my_int_file = glob.glob(os.path.join(i, 'det_images', '*.fits'))[0]
int_dm = datamodels.open(my_int_file)
out_dm.data[n] = int_dm.data
out_dm.save(f)
def test_ff_inv(rtdata, fitsdiff_default_kwargs):
"""Test flat field inversion"""
with dm.open(rtdata.get_data('nirspec/imaging/usf_assign_wcs.fits')) as data:
flatted = FlatFieldStep.call(data)
unflatted = FlatFieldStep.call(flatted, inverse=True)
assert np.allclose(data.data, unflatted.data), 'Inversion failed'
def test_open_fits():
"""Test opening a model from a FITS file"""
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "model_type not found")
fits_file = t_path('test.fits')
with datamodels.open(fits_file) as model:
assert isinstance(model, JwstDataModel)
def create_dms(base_file, level="1b", parfile=None, subarray=None, exp_type='UNKNOWN'):
#
# Create a Level 1b file from a full-frame base file
# base_file is the full-format image the subarray is created from
# subarray is a tuple describing the subarray:
# (nxstart, nxstop, nystart, nystop, name)
#
base_root = base_file.split('.')[0]
base_hdulist = fits.open(base_file)
#
# Add the exp_type to the input hdulist
base_hdulist[0].header['EXP_TYPE'] = exp_type
#
# If we've already rotated/flipped the file, it will have a keyword set
# and we skip this step
#
if not rotated_and_flipped(base_hdulist):
print('Rotating and flipping data %s' % (base_file))
flip_rotate(base_hdulist)
instrument = base_hdulist[0].header['INSTRUME']
if subarray is not None:
subarray_file = create_single_subarray(base_hdulist,
subarray)
print('Parfile = %s' % parfile)
if parfile is None:
parfile = ''.join((base_root, '_', subarray[4].lower(),
'_observation_identifiers.dat'))
print(parfile)
hdulist = main.generate([subarray_file, parfile], level=level)
filename = main.guess_filename(hdulist)
hdulist[0].header['FILENAME'] = filename
hdulist.writeto(filename, output_verify='silentfix')
os.remove(subarray_file)
else:
if instrument == 'MIRI':
split_data_and_refout(base_hdulist)
hdulist = main.generate([base_hdulist, parfile],
level=level)
filename = main.guess_filename(hdulist)
hdulist[0].header['FILENAME'] = filename
hdulist.writeto(filename, output_verify='silentfix')
base_hdulist.close()
#
# Try and open the file as a JWST datamodel
a = datamodels.open(filename)
print("%s opened as a %s" % (filename, a.__module__))
a.close()
return
def test_miri_lrs_masterbg_user(self):
"""
Regression test of masterbackgound subtraction with lrs, with user provided 1-D background
"""
# input file has the background added
input_file = self.get_data(*self.test_dir, 'miri_lrs_sci+bkg_cal.fits')
# user provided 1-D background
user_background = self.get_data(*self.test_dir, 'miri_lrs_bkg_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=user_background,
save_results=True)
# Compare result (background subtracted image) to science image with no
# background. Subtract these images, smooth the subtracted image and
# the mean should be close to zero.
input_sci_cal_file = self.get_data(*self.test_dir,
'miri_lrs_sci_cal.fits')
input_sci = datamodels.open(input_sci_cal_file)
# find the LRS region
bb = result.meta.wcs.bounding_box
x, y = grid_from_bounding_box(bb)
result_lrs_region = result.data[y.astype(int), x.astype(int)]
sci_lrs_region = input_sci.data[y.astype(int), x.astype(int)]
# do a 5 sigma clip on the science image
sci_mean = np.nanmean(sci_lrs_region)
sci_std = np.nanstd(sci_lrs_region)
upper = sci_mean + sci_std*5.0
lower = sci_mean - sci_std*5.0
mask_clean = np.logical_and(sci_lrs_region < upper, sci_lrs_region > lower)
sub = result_lrs_region - sci_lrs_region
mean_sub = np.absolute(np.mean(sub[mask_clean]))
atol = 0.1
rtol = 0.001
assert_allclose(mean_sub, 0, atol=atol, rtol=rtol)
# Test 3 Compare background subtracted science data (results)
# to a truth file.
truth_file = self.get_data(*self.ref_loc,
'miri_lrs_sci+bkg_masterbackgroundstep.fits')
result_file = result.meta.filename
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
result.close()
input_sci.close()
def test_open_asdf_readonly(tmpdir):
tmpfile = str(tmpdir.join('readonly.asdf'))
with DistortionModel() as model:
model.meta.telescope = 'JWST'
model.meta.instrument.name = 'NIRCAM'
model.meta.instrument.detector = 'NRCA4'
model.meta.instrument.channel = 'SHORT'
model.save(tmpfile)
os.chmod(tmpfile, 0o440)
assert os.access(tmpfile, os.W_OK) == False
with datamodels.open(tmpfile) as model:
assert model.meta.telescope == 'JWST'
def test_open_fits_readonly(tmpdir):
"""Test opening a FITS-format datamodel that is read-only on disk"""
tmpfile = str(tmpdir.join('readonly.fits'))
data = np.arange(100, dtype=np.float).reshape(10, 10)
with ImageModel(data=data) as model:
model.meta.telescope = 'JWST'
model.meta.instrument.name = 'NIRCAM'
model.meta.instrument.detector = 'NRCA4'
model.meta.instrument.channel = 'SHORT'
model.save(tmpfile)
os.chmod(tmpfile, 0o440)
assert os.access(tmpfile, os.W_OK) == False
with datamodels.open(tmpfile) as model:
assert model.meta.telescope == 'JWST'
def process(self, input):
# Open the input data model
with datamodels.open(input) as input_model:
# check the data is MIRI data
detector = input_model.meta.instrument.detector
if detector.startswith('MIR'):
# Get the name of the rscd reference file to use
self.rscd_name = self.get_reference_file(input_model, 'rscd')
self.log.info('Using RSCD reference file %s', self.rscd_name)
# Check for a valid reference file
if self.rscd_name == 'N/A':
self.log.warning('No RSCD reference file found')
self.log.warning('RSCD step will be skipped')
result = input_model.copy()
result.meta.cal_step.rscd = 'SKIPPED'
return result
# Open the rscd ref file data model
print('RSCD reference file name',self.rscd_name)
#rscd_model = datamodels.RSCDModel(self.rscd_name)
rscd_model = rscd_mod.RSCD_Model(self.rscd_name)
# Do the rscd correction subtraction
result = rscd_sub.do_correction(input_model, rscd_model)
# Close the reference file and update the step status
rscd_model.close()
result.meta.cal_step.rscd = 'COMPLETE'
else:
self.log.warning('RSCD Correction is only for MIRI data')
self.log.warning('RSCD step will be skipped')
result = input_model.copy()
result.meta.cal_step.rscd = 'SKIPPED'
return result
def process(self, input):
# Open the input data model
with datamodels.open(input) as input_model:
# check the data is MIRI data
detector = input_model.meta.instrument.detector
if detector[:3] == 'MIR':
# Get the name of the reset reference file to use
self.filename = self.get_reference_file(input_model, 'lastframe')
self.log.info('Using Last Frame reference file %s', self.filename)
# Check for a valid reference file
if self.filename == 'N/A':
self.log.warning('No Last Frame reference file found')
self.log.warning('Last frame step will be skipped')
result = input_model.copy()
result.meta.cal_step.lastframe = 'SKIPPED'
return result
# Open the last frame ref file data model
lastframe_model = datamodels.LastFrameModel(self.filename)
# Do the lastframe correction subtraction
result = lastframe_sub.do_correction(input_model, lastframe_model)
# Close the reference file and update the step status
lastframe_model.close()
result.meta.cal_step.lastframe = 'COMPLETE'
else:
self.log.warning('Last Frame Correction is only for MIRI data')
self.log.warning('Last frame step will be skipped')
result = input_model.copy()
result.meta.cal_step.lastframe = 'SKIPPED'
return result
def test_nirspec_masterbackground_mos_user1d(self):
"""
Regression test of master background subtraction for NRS MOS when a user 1-D spectrum is provided.
"""
# input file has 2-D background image added to it
input_file = self.get_data(*self.test_dir,
'nrs_mos_sci+bkg_cal.fits')
# user provide 1-D background was created from the 2-D background image
input_1dbkg_file = self.get_data(*self.test_dir,
'nrs_mos_bkg_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=input_1dbkg_file,
save_results=True)
# _________________________________________________________________________
# One of out tests is to compare the 1-D extracted spectra from
# the science image (no background added) and the masterbackground subtracted
# data.
# run resample_spec on results from MasterBackground step
result_2d = ResampleSpecStep.call(result)
# run 1-D extract on results from MasterBackground step
result_1d = Extract1dStep.call(result_2d)
# get input science data with background added
input_sci_cal_file = self.get_data(*self.test_dir,
'nrs_mos_sci_cal.fits')
# get 1-D extract on original science data without background
# this reference data was also run through ResampleSpec
input_sci_1d_file = self.get_data(*self.ref_loc,
'nrs_mos_sci_extract1dstep.fits')
input_sci = datamodels.open(input_sci_cal_file)
sci_cal_1d = datamodels.open(input_sci_1d_file)
num_spec = len(sci_cal_1d.spec)
# the user 1D spectrum may not cover the entire wavelength range of the
# science data. Find the wavelength range of user 1-D spectra
input_1dbkg_1d = datamodels.open(input_1dbkg_file)
user_wave = input_1dbkg_1d.spec[0].spec_table['wavelength']
user_flux = input_1dbkg_1d.spec[0].spec_table['flux']
user_wave_valid = np.where(user_flux > 0)
min_user_wave = np.amin(user_wave[user_wave_valid])
max_user_wave = np.amax(user_wave[user_wave_valid])
input_1dbkg_1d.close()
# loop over each slit and perform 2 tests on each slit
for i in range(num_spec):
# ______________________________________________________________________
# Test 1 compare extracted spectra data from the science data
# to extracted spectra from the output
# from MasterBackground subtraction.
sci_spec_1d = sci_cal_1d.spec[i].spec_table['flux']
sci_wave = sci_cal_1d.spec[i].spec_table['wavelength']
result_spec_1d = result_1d.spec[i].spec_table['flux']
# find the waverange covered by both user 1-D and science slit
sci_wave_valid = np.where(sci_spec_1d > 0)
min_wave = np.amin(sci_wave[sci_wave_valid])
max_wave = np.amax(sci_wave[sci_wave_valid])
if min_user_wave > min_wave:
min_wave = min_user_wave
if max_user_wave < max_wave:
max_wave = max_user_wave
sub_spec = sci_spec_1d - result_spec_1d
valid = np.where(np.logical_and(sci_wave > min_wave, sci_wave < max_wave))
sub_spec = sub_spec[valid]
mean_sub = np.nanmean(sub_spec)
atol = 1.5
assert_allclose(mean_sub, 0, atol=atol)
# ______________________________________________________________________
# Test 2 compare the science data with no background
# to the output from the masterBackground Subtraction step
# background subtracted science image.
bb = input_sci.slits[i].meta.wcs.bounding_box
x, y = grid_from_bounding_box(bb)
ra, dec, lam = input_sci.slits[i].meta.wcs(x, y)
valid = np.isfinite(lam)
sci = input_sci.slits[i].data
result_slit = result.slits[i].data
# check for outliers in the science image that could cause test
# to fail. These could be cosmic ray hits or other yeck that
# messes up the science data - ignores these cases
sci_mean = np.nanmean(sci[valid])
sci_std = np.nanstd(sci[valid])
upper = sci_mean + sci_std*5.0
lower = sci_mean - sci_std*5.0
mask_clean = np.logical_and(sci[valid] < upper, sci[valid] > lower)
# for this slit subtract the background subtracted data from
# the science data with no background added
sub = result_slit - sci
# do not look at outliers - they confuse things
sub_valid = sub[valid]
mean_sub = np.mean(sub_valid[mask_clean])
atol = 0.1
assert_allclose(np.absolute(mean_sub), 0, atol=atol)
# ______________________________________________________________________
# Test 3 Compare background sutracted science data (results)
# to a truth file. This data is MultiSlit data
result_file = result.meta.filename
ref_file = self.get_data(*self.ref_loc,
'nrs_mos_sci+bkg_masterbackgroundstep.fits')
outputs = [(result_file, ref_file)]
self.compare_outputs(outputs)
input_sci.close()
result.close()
def test_nirspec_ifu_masterbg_user(self):
"""
Regression test of master background subtraction for NRS IFU when a
user 1-D spectrum is provided.
"""
# input file has 2-D background image added to it
input_file = self.get_data(*self.test_dir, 'prism_sci_bkg_cal.fits')
# user-provided 1-D background was created from the 2-D background image
user_background = self.get_data(*self.test_dir, 'prism_bkg_x1d.fits')
result = MasterBackgroundStep.call(input_file,
user_background=user_background,
save_results=True)
# Test 1 compare extracted spectra data with
# no background added to extracted spectra from the output
# from MasterBackground subtraction. First cube_build has to be run
# on the data.
result_s3d = CubeBuildStep.call(result)
# run 1-D extract on results from MasterBackground step
result_1d = Extract1dStep.call(result_s3d, subtract_background=False)
# get the 1-D extracted spectrum from the science data in truth directory
input_sci_1d_file = self.get_data(*self.ref_loc, 'prism_sci_extract1d.fits')
sci_1d = datamodels.open(input_sci_1d_file)
# read in the valid wavelengths of the user-1d
user_background_model = datamodels.open(user_background)
user_wave = user_background_model.spec[0].spec_table['wavelength']
user_flux = user_background_model.spec[0].spec_table['net']
user_wave_valid = np.where(user_flux > 0)
min_user_wave = np.amin(user_wave[user_wave_valid])
max_user_wave = np.amax(user_wave[user_wave_valid])
user_background_model.close()
# find the waverange covered by both user and science
sci_spec_1d = sci_1d.spec[0].spec_table['net']
sci_spec_wave = sci_1d.spec[0].spec_table['wavelength']
result_spec_1d = result_1d.spec[0].spec_table['net']
sci_wave_valid = np.where(sci_spec_1d > 0)
min_wave = np.amin(sci_spec_wave[sci_wave_valid])
max_wave = np.amax(sci_spec_wave[sci_wave_valid])
if min_user_wave > min_wave:
min_wave = min_user_wave
if max_user_wave < max_wave:
max_wave = max_user_wave
sub_spec = sci_spec_1d - result_spec_1d
valid = np.where(np.logical_and(sci_spec_wave > min_wave, sci_spec_wave < max_wave))
sub_spec = sub_spec[valid]
sub_spec = sub_spec[1:-2] # endpoints are wacky
mean_sub = np.absolute(np.nanmean(sub_spec))
atol = 5.0
assert_allclose(mean_sub, 0, atol=atol)
# Test 2 compare the science data with no background
# to the output from the masterBackground Subtraction step
# background subtracted science image.
input_sci_cal_file = self.get_data(*self.test_dir,
'prism_sci_cal.fits')
input_sci_model = datamodels.open(input_sci_cal_file)
# We don't want the slices gaps to impact the statisitic
# loop over the 30 Slices
for i in range(30):
slice_wcs = nirspec.nrs_wcs_set_input(input_sci_model, i)
x, y = grid_from_bounding_box(slice_wcs.bounding_box)
ra, dec, lam = slice_wcs(x, y)
valid = np.isfinite(lam)
result_slice_region = result.data[y.astype(int), x.astype(int)]
sci_slice_region = input_sci_model.data[y.astype(int),
x.astype(int)]
sci_slice = sci_slice_region[valid]
result_slice = result_slice_region[valid]
sub = result_slice - sci_slice
# check for outliers in the science image
sci_mean = np.nanmean(sci_slice)
sci_std = np.nanstd(sci_slice)
upper = sci_mean + sci_std*5.0
lower = sci_mean - sci_std*5.0
mask_clean = np.logical_and(sci_slice < upper, sci_slice > lower)
sub_mean = np.absolute(np.nanmean(sub[mask_clean]))
atol = 2.0
assert_allclose(sub_mean, 0, atol=atol)
# Test 3 Compare background sutracted science data (results)
# to a truth file. This data is MultiSlit data
input_sci_model.close()
result_file = result.meta.filename
truth_file = self.get_data(*self.ref_loc,
'prism_sci_bkg_masterbackgroundstep.fits')
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
input_sci_model.close()
result.close()
