def test_miri_image_wcs(_bigdata):
"""
Regression test of creating a WCS object and doing pixel to sky transformation.
"""
suffix = 'assignwcsstep'
output_file_base, output_file = add_suffix('miri_image_wcs_output.fits',
suffix)
try:
os.remove(output_file)
except:
pass
input_file = os.path.join(
_bigdata, 'miri', 'test_wcs', 'image',
'jw00001001001_01101_00001_MIRIMAGE_ramp_fit.fits')
ref_file = os.path.join(
_bigdata, 'miri', 'test_wcs', 'image',
'jw00001001001_01101_00001_MIRIMAGE_assign_wcs.fits')
AssignWcsStep.call(input_file, output_file=output_file_base, suffix=suffix)
im = ImageModel(output_file)
imref = ImageModel(ref_file)
x, y = np.mgrid[:1031, :1024]
ra, dec = im.meta.wcs(x, y)
raref, decref = imref.meta.wcs(x, y)
utils.assert_allclose(ra, raref)
utils.assert_allclose(dec, decref)
def test_miri_fixed_slit_wcs(_bigdata):
"""
Regression test of creating a WCS object and doing pixel to sky transformation.
"""
output_file_base, output_file = add_suffix('miri_fixed_wcs_output.fits',
'assignwcsstep')
try:
os.remove(output_file)
except:
pass
input_file = os.path.join(_bigdata, 'miri', 'test_wcs', 'fixed',
'jw00035001001_01101_00001_mirimage_rate.fits')
ref_file = os.path.join(
_bigdata, 'miri', 'test_wcs', 'fixed',
'jw00035001001_01101_00001_mirimage_assign_wcs.fits')
AssignWcsStep.call(input_file, output_file=output_file_base)
im = ImageModel(output_file)
imref = ImageModel(ref_file)
y, x = np.mgrid[:1031, :1024]
ra, dec, lam = im.meta.wcs(x, y)
raref, decref, lamref = imref.meta.wcs(x, y)
utils.assert_allclose(ra, raref)
utils.assert_allclose(dec, decref)
utils.assert_allclose(lam, lamref)
def test_nirspec_ifu_wcs(_bigdata, test_id, input_file, truth_file):
"""
Regression test of creating a WCS object and doing pixel to sky transformation.
"""
del test_id
input_file = os.path.join(_bigdata, 'nirspec', 'test_wcs', 'nrs1-ifu',
input_file)
truth_file = os.path.join(_bigdata, 'nirspec', 'test_wcs', 'nrs1-ifu',
truth_file)
result = AssignWcsStep.call(input_file,
save_results=True,
suffix='assign_wcs')
result.close()
im = ImageModel(result.meta.filename)
imref = ImageModel(truth_file)
w = nirspec.nrs_wcs_set_input(im, 0)
grid = grid_from_bounding_box(w.bounding_box)
ra, dec, lam = w(*grid)
wref = nirspec.nrs_wcs_set_input(imref, 0)
raref, decref, lamref = wref(*grid)
# equal_nan is used here as many of the entries are nan.
# The domain is defined but it is only a few entries in there that are valid
# as it is a curved narrow slit.
assert_allclose(ra, raref, equal_nan=True)
assert_allclose(dec, decref, equal_nan=True)
assert_allclose(lam, lamref, equal_nan=True)
def test_nirspec_nrs1_wcs(self):
"""
Regression test of creating a WCS object and doing pixel to sky transformation.
"""
input_file = self.get_data(*self.test_dir,
'jw00023001001_01101_00001_NRS1_ramp_fit.fits')
ref_file = self.get_data(*self.ref_loc,
'jw00023001001_01101_00001_NRS1_ramp_fit_assign_wcs.fits')
result = AssignWcsStep.call(input_file, save_results=True, suffix='assign_wcs')
result.close()
im = ImageModel(result.meta.filename)
imref = ImageModel(ref_file)
for slit in ['S200A1', 'S200A2', 'S400A1', 'S1600A1']:
w = nirspec.nrs_wcs_set_input(im, slit)
grid = grid_from_bounding_box(w.bounding_box)
ra, dec, lam = w(*grid)
wref = nirspec.nrs_wcs_set_input(imref, slit)
raref, decref, lamref = wref(*grid)
assert_allclose(ra, raref, equal_nan=True)
assert_allclose(dec, decref, equal_nan=True)
assert_allclose(lam, lamref, equal_nan=True)
def blot_segmap(filename, files):
"""
Blots the segmap values from the drizzled image onto the individual
files than went into the drizzle, and write these out as new segmaps.
Parameters
----------
filename : str
The drizzled file.
files : list
The files that went into creating the drizzled file.
Outputs
-------
{file}_seg.fits
A corresponding segmap for each file in files.
"""
# Make an image model using the drizzled image WCS and the corresponding
# segmentation maps data.
drizzle_model = ImageModel(filename)
drizzle_segmap = fits.getdata(filename.replace('.fits', '_seg.fits'))
drizzle_segmap[drizzle_segmap != 0] = 1
drizzle_model.data = drizzle_segmap
# Blot the segmap data from the drizzled image onto each individual file,
# and write out the corresponding segmap for each.
for f in files:
outfile = f.replace('.fits', '_seg.fits')
model = ImageModel(f)
blotted_data = gwcs_blot(drizzle_model, model, interp='nearest')
fits.writeto(outfile, blotted_data, overwrite=True)
def test_nirspec_nrs1_wcs(_bigdata):
"""
Regression test of creating a WCS object and doing pixel to sky transformation.
"""
output_file_base, output_file = add_suffix('nrs1_ifu_wcs_output.fits', 'assignwcsstep')
try:
os.remove(output_file)
except:
pass
input_file = os.path.join(_bigdata, 'nirspec', 'test_wcs', 'nrs1-ifu', 'jw00011001001_01120_00001_NRS1_rate.fits')
ref_file = os.path.join(_bigdata, 'nirspec', 'test_wcs', 'nrs1-ifu', 'jw00011001001_01120_00001_NRS1_rate_assign_wcs.fits')
AssignWcsStep.call(input_file,
output_file=output_file_base, name='assignwcsstep'
)
im = ImageModel(output_file)
imref = ImageModel(ref_file)
a_wcs = nirspec.nrs_wcs_set_input(im, 0)
w = a_wcs
y, x = np.mgrid[w.bounding_box[1][0]:w.bounding_box[1][1], w.bounding_box[0][0]: w.bounding_box[0][1]]
ra, dec, lam = w(x, y)
a_wcs_ref = nirspec.nrs_wcs_set_input(im, 0)
wref = a_wcs_ref
raref, decref, lamref = wref(x, y)
# equal_nan is used here as many of the entries are nan.
# The domain is defined but it is only a few entries in there that are valid
# as it is a curved narrow slit.
utils.assert_allclose(ra, raref, equal_nan=True)
utils.assert_allclose(dec, decref, equal_nan=True)
utils.assert_allclose(lam, lamref, equal_nan=True)
def test_validate_on_read():
schema = ImageModel((10, 10))._schema.copy()
schema['properties']['meta']['properties']['calibration_software_version'][
'fits_required'] = True
with pytest.raises(jsonschema.ValidationError):
with ImageModel(FITS_FILE, schema=schema, strict_validation=True):
pass
def test_nirspec_wcs(_jail, rtdata, test_id, input_file, truth_file):
"""
Test of the AssignWcs step on 4 different NIRSpec exposures:
1) IFU NRS1 exposure,
2) IFU NRS1 exposure with FILTER=OPAQUE,
3) IFU NRS2 exposure, and
4) FS NRS1 exposure with 4 slits.
"""
# Get the input and truth files
rtdata.get_data('nirspec/test_wcs/' + input_file)
rtdata.get_truth('truth/test_nirspec_wcs/' + truth_file)
# Run the AssignWcs step
result = AssignWcsStep.call(input_file, save_results=True, suffix='assign_wcs')
result.close()
# Open the output and truth files
im = ImageModel(result.meta.filename)
im_ref = ImageModel(truth_file)
if result.meta.exposure.type == 'NRS_FIXEDSLIT':
# Loop over the 4 slit instances
for slit in ['S200A1', 'S200A2', 'S400A1', 'S1600A1']:
# Create WCS objects for each image
wcs = nirspec.nrs_wcs_set_input(im, slit)
wcs_ref = nirspec.nrs_wcs_set_input(im_ref, slit)
# Compute RA, Dec, lambda values for each image array
grid = grid_from_bounding_box(wcs.bounding_box)
ra, dec, lam = wcs(*grid)
ra_ref, dec_ref, lam_ref = wcs_ref(*grid)
# Compare the sky coordinates
assert_allclose(ra, ra_ref, equal_nan=True)
assert_allclose(dec, dec_ref, equal_nan=True)
assert_allclose(lam, lam_ref, equal_nan=True)
else:
# Create WCS objects for each image
wcs = nirspec.nrs_wcs_set_input(im, 0)
wcs_ref = nirspec.nrs_wcs_set_input(im_ref, 0)
# Compute RA, Dec, lambda values for each image array
grid = grid_from_bounding_box(wcs.bounding_box)
ra, dec, lam = wcs(*grid)
ra_ref, dec_ref, lam_ref = wcs_ref(*grid)
# Compare the sky coordinates
# equal_nan is used, because many of the entries are NaN,
# due to the bounding_box being rectilinear while the
# defined spectral traces are curved
assert_allclose(ra, ra_ref, equal_nan=True)
assert_allclose(dec, dec_ref, equal_nan=True)
assert_allclose(lam, lam_ref, equal_nan=True)
def test_getarray_noinit_noinit():
"""Test that calling on a non-existant array does not initialize that array"""
arr = np.ones((5, 5))
model = ImageModel(data=arr)
try:
model.getarray_noinit('area')
except AttributeError:
pass
assert 'area' not in model.instance
def test_boundingbox_from_indices():
dm = ImageModel()
dm.data = np.ones((10,10))
bbox = ((1,2), (3,4))
result = boundingbox_to_indices(dm, bbox)
assert (result == (1, 3, 3, 5))
def test_ifuimage():
data = np.arange(24).reshape((6, 4))
im = ImageModel(data=data, err=data / 2, dq=data)
ifuimage = IFUImageModel(im)
assert_array_equal(im.data, ifuimage.data)
assert_array_equal(im.err, ifuimage.err)
assert_array_equal(im.dq, ifuimage.dq)
im = ImageModel(ifuimage)
assert type(im) == ImageModel
def test_ifuimage():
data = np.arange(24, dtype=np.float32).reshape((6, 4))
im = ImageModel(data=data, err=data / 2, dq=data)
ifuimage = IFUImageModel(im)
assert_allclose(im.data, ifuimage.data)
assert_allclose(im.err, ifuimage.err)
assert_allclose(im.dq, ifuimage.dq)
im = ImageModel(ifuimage)
assert type(im) == ImageModel
def process(self, *args):
container = ModelContainer()
model1 = ImageModel(args[0]).copy()
model2 = ImageModel(args[0]).copy()
model1.meta.filename = 'swc_model1.fits'
model2.meta.filename = 'swc_model2.fits'
container.append(model1)
container.append(model2)
return container
def test_miri_ifu_wcs(_bigdata):
"""
Regression test of creating a WCS object and doing pixel to sky transformation.
"""
try:
os.remove("miri_ifu_wcs_output.fits")
except:
pass
input_file = os.path.join(
_bigdata, 'miri', 'test_wcs', 'ifu',
'jw00024001001_01101_00001_MIRIFUSHORT_uncal_MiriSloperPipeline.fits')
ref_file = os.path.join(
_bigdata, 'miri', 'test_wcs', 'ifu',
'jw00024001001_01101_00001_MIRIFUSHORT_assign_wcs.fits')
AssignWcsStep.call(input_file, output_file='miri_ifu_wcs', suffix='output')
im = ImageModel('miri_ifu_wcs_output.fits')
imref = ImageModel(ref_file)
# Get the region file
region = RegionsModel(crds_client.get_reference_file(im, 'regions'))
# inputs
shape = region.regions.shape
y, x = np.mgrid[:shape[0], :shape[1]]
# Get indices where pixels == 0. These should be NaNs in the output.
ind_zeros = region.regions == 0
ra, dec, lam = im.meta.wcs(x, y)
raref, decref, lamref = imref.meta.wcs(x, y)
utils.assert_allclose(ra, raref, equal_nan=True)
utils.assert_allclose(dec, decref, equal_nan=True)
utils.assert_allclose(lam, lamref, equal_nan=True)
# Test that we got NaNs at ind_zero
assert (np.isnan(ra).nonzero()[0] == ind_zeros.nonzero()[0]).all()
assert (np.isnan(ra).nonzero()[1] == ind_zeros.nonzero()[1]).all()
# Test the inverse transform
x1, y1 = im.meta.wcs.backward_transform(ra, dec, lam)
assert (np.isnan(x1).nonzero()[0] == ind_zeros.nonzero()[0]).all()
assert (np.isnan(x1).nonzero()[1] == ind_zeros.nonzero()[1]).all()
# Also run a smoke test with values outside the region.
dec[100][200] = -80
ra[100][200] = 7
lam[100][200] = 15
x2, y2 = im.meta.wcs.backward_transform(ra, dec, lam)
assert np.isnan(x2[100][200])
assert np.isnan(x2[100][200])
def data_path(jail):
"""Create data file with needed header parameters"""
model = ImageModel()
model.meta.target.ra = 90.75541666666666
model.meta.target.dec = -66.56055555555554
model.meta.pointing.ra_v1 = 91.08142004561715
model.meta.pointing.dec_v1 = -66.60547868904696
model.meta.wcsinfo.ra_ref = 90.70377653291781
model.meta.wcsinfo.dec_ref = -66.59540223936895
model.save('data.fits')
return model.meta.filename
def test_validate_on_read():
""" Test for proper validation error
Note: The FITS file is opened separately in order to properly close
the file.
"""
schema = ImageModel((10, 10))._schema.copy()
schema['properties']['meta']['properties']['calibration_software_version'][
'fits_required'] = True
with fits.open(FITS_FILE) as hduls:
with pytest.raises(jsonschema.ValidationError):
ImageModel(hduls, schema=schema, strict_validation=True)
def test_build_driz_weight(weight_type):
"""Check that correct weight map is returned of different weight types"""
model = ImageModel((10, 10))
model.dq[0] = DO_NOT_USE
model.meta.exposure.exposure_time = 10.0
model.var_rnoise += 0.1
weight_map = build_driz_weight(model,
weight_type=weight_type,
good_bits="GOOD")
assert_array_equal(weight_map[0], 0)
assert_array_equal(weight_map[1:], 10.0)
assert weight_map.dtype == np.float32
def miri_rate_zero_crossing():
xsize = 1032
ysize = 1024
shape = (ysize, xsize)
im = ImageModel(shape)
im.var_rnoise = np.random.random(shape)
im.meta.wcsinfo = {
'dec_ref': 2.16444343946559e-05,
'ra_ref': -0.00026031780056776,
'roll_ref': 0.0,
'v2_ref': -415.0690466121227,
'v3_ref': -400.575920398547,
'v3yangle': 0.0,
'vparity': -1
}
im.meta.instrument = {
'detector': 'MIRIMAGE',
'filter': 'P750L',
'name': 'MIRI'
}
im.meta.observation = {'date': '2019-01-01', 'time': '17:00:00'}
im.meta.subarray = {
'fastaxis': 1,
'name': 'FULL',
'slowaxis': 2,
'xsize': xsize,
'xstart': 1,
'ysize': ysize,
'ystart': 1
}
im.meta.exposure = {
'duration': 11.805952,
'end_time': 58119.85416,
'exposure_time': 11.776,
'frame_time': 0.11776,
'group_time': 0.11776,
'groupgap': 0,
'integration_time': 11.776,
'nframes': 1,
'ngroups': 100,
'nints': 1,
'nresets_between_ints': 0,
'nsamples': 1,
'readpatt': 'FAST',
'sample_time': 10.0,
'start_time': 58119.8333,
'type': 'MIR_LRS-FIXEDSLIT',
'zero_frame': False
}
return im
def test_update_from_datamodel(tmpdir, datamodel_for_update, only, extra_fits):
"""Test update method does not update from extra_fits unless asked"""
tmpfile = datamodel_for_update
newtmpfile = str(tmpdir.join("new.fits"))
with ImageModel((5, 5)) as newim:
with ImageModel(tmpfile) as oldim:
# Verify the fixture returns keywords we expect
assert oldim.meta.telescope == "JWST"
assert oldim.meta.wcsinfo.crval1 == 5
assert oldim.extra_fits.PRIMARY.header == [['FOO', 'BAR', '']]
assert oldim.extra_fits.SCI.header == [['BAZ', 'BUZ', '']]
newim.update(oldim, only=only, extra_fits=extra_fits)
newim.save(newtmpfile)
with fits.open(newtmpfile) as hdulist:
if extra_fits:
if only == "PRIMARY":
assert "TELESCOP" in hdulist["PRIMARY"].header
assert "CRVAL1" not in hdulist["SCI"].header
assert "FOO" in hdulist["PRIMARY"].header
assert "BAZ" not in hdulist["SCI"].header
elif only == "SCI":
assert "TELESCOP" not in hdulist["PRIMARY"].header
assert "CRVAL1" in hdulist["SCI"].header
assert "FOO" not in hdulist["PRIMARY"].header
assert "BAZ" in hdulist["SCI"].header
else:
assert "TELESCOP" in hdulist["PRIMARY"].header
assert "CRVAL1" in hdulist["SCI"].header
assert "FOO" in hdulist["PRIMARY"].header
assert "BAZ" in hdulist["SCI"].header
else:
assert "FOO" not in hdulist["PRIMARY"].header
assert "BAZ" not in hdulist["SCI"].header
if only == "PRIMARY":
assert "TELESCOP" in hdulist["PRIMARY"].header
assert "CRVAL1" not in hdulist["SCI"].header
elif only == "SCI":
assert "TELESCOP" not in hdulist["PRIMARY"].header
assert "CRVAL1" in hdulist["SCI"].header
else:
assert "TELESCOP" in hdulist["PRIMARY"].header
assert "CRVAL1" in hdulist["SCI"].header
def setup():
global TMP_DIR, TMP_NAMES, TMP_FILES, INPUT_VALUES, OUTPUT_VALUES
TMP_DIR = tempfile.mkdtemp()
TMP_FILES = [ImageModel() for i in range(3)]
INPUT_VALUES = {
'meta.exposure.start_time': start_times,
'meta.exposure.exposure_time': exp_times,
'meta.exposure.end_time': end_times,
'meta.filename': filenames,
'meta.instrument.name': instrument_names,
'meta.date': datetimes,
'meta.observation.date': dates,
'meta.observation.date_beg': datetimes
}
OUTPUT_VALUES = {
'meta.exposure.start_time': start_times[0],
'meta.exposure.exposure_time': np.sum(exp_times),
'meta.exposure.end_time': end_times[-1],
'meta.filename': filenames[0],
'meta.instrument.name': instrument_names[0],
'meta.date': datetimes[0],
'meta.observation.date': dates[1],
'meta.observation.date_beg': datetimes[1]
}
for i, tfile in enumerate(TMP_FILES):
for attr in INPUT_VALUES:
tfile[attr] = INPUT_VALUES[attr][i]
def test_calc_deltas(engdb, data_path):
"""Test `calc_deltas` basic running"""
with ImageModel(data_path) as model:
deltas = ps.calc_deltas([model])
truth = Table.read(DATA_PATH / 'calc_deltas_truth.ecsv')
assert report_diff_values(truth, deltas, fileobj=sys.stderr)
def test_init_from_pathlib(make_models):
"""Test initializing model from a PurePath object"""
path = Path(make_models['model'])
model = ImageModel(path)
# Test is basically, did we open the model?
assert isinstance(model, ImageModel)
def create_datamodel(hdul):
im = ImageModel(hdul)
ref = create_reference_files(im)
pipeline = miri.create_pipeline(im, ref)
wcsobj = wcs.WCS(pipeline)
im.meta.wcs = wcsobj
return im
def test_broadcast2():
with ImageModel() as dm:
data = np.empty((52, 50))
dm.data = data
dq = np.empty((50, ))
dm.dq = dq
def setup_class(self):
self.nrs = ImageModel()
self.nrs.meta.instrument.name = "NIRSPEC"
self.nrs.meta.instrument.filter = "CLEAR"
self.nrs.meta.instrument.grating = "MIRROR"
self.nrs.meta.exposure.type = 'NRS_FIXEDSLIT'
self.nrs.meta.instrument.detector = 'NRS1'
self.slits = {
'S200A1': 'SLIT_A_200_1',
'S200A2': 'SLIT_A_200_2',
'S400A1': 'SLIT_A_400',
'S1600A1': 'SLIT_A_1600',
#'S200B1': 'SLIT_B_200',
#'IFU': 'IFU_window'
}
self.reference = fits.open(
get_pkg_data_filename(
"data/onSkyAndOnDetectorProjectionSLIT.fits.gz"))
self.field_names = list(self.reference[1].data.field('name'))
print(self.field_names)
step = assign_wcs_step.AssignWcsStep()
self.model = step.process(self.nrs)
self.lam = 2e-6
self.wcs_S200A1 = self.model.meta.wcs_S200A1
self.wcs_S200A2 = self.model.meta.wcs_S200A2
self.wcs_S400A1 = self.model.meta.wcs_S400A1
self.wcs_S1600A1 = self.model.meta.wcs_S1600A1
def test_from_models_mast(tmp_path):
"""Test v1_calculate_from_models for basic running"""
model = ImageModel()
model.meta.exposure.start_time = MAST_GOOD_STARTTIME.mjd
model.meta.exposure.end_time = MAST_GOOD_ENDTIME.mjd
try:
v1_table = v1c.v1_calculate_from_models(
[model],
method=stp.Methods.COARSE_TR_202111,
engdb_url=engdb_mast.MAST_BASE_URL)
except ValueError as exception:
pytest.xfail(
f'MAST engineering database not available, possibly no token specified: {exception}'
)
v1_formatted = v1c.simplify_table(v1_table)
# Save for post-test examination
v1_formatted.write(tmp_path / 'test_from_models_mast.ecsv',
format='ascii.ecsv')
truth = Table.read(DATA_PATH / 'test_from_models_mast.ecsv')
errors = v1_compare_simplified_tables(v1_formatted, truth)
errors_str = '\n'.join(errors)
assert len(errors) == 0, f'V1 tables are different: {errors_str}'
def process(self, *args):
with ImageModel(args[0]) as model:
self.log.info('Saving model as "processed"')
self.save_model(model, 'processed')
return model
def process(self, *args):
model = ImageModel(args[0])
r = self.stepwithmodel(model)
r = self.savestep(r)
return r
def test_get_fits_wcs(tmpdir):
with ImageModel(FITS_FILE) as dm:
# Refer to the data array to initialize it.
dm.data = np.zeros((5, 5))
# Now continue with the test.
wcs1 = dm.get_fits_wcs()
dm2 = dm.copy()
wcs2 = dm2.get_fits_wcs()
x = np.random.rand(2**16, wcs1.wcs.naxis)
world1 = wcs1.all_pix2world(x, 1)
world2 = wcs2.all_pix2world(x, 1)
assert_array_almost_equal(world1, world2)
wcs1.wcs.crpix[0] = 42.0
dm2.set_fits_wcs(wcs1)
assert dm2.meta.wcsinfo.crpix1 == 42.0
with warnings.catch_warnings():
# Filter out warnings generated by WCSLIB>=7.1
warnings.simplefilter("ignore")
wcs2 = dm2.get_fits_wcs()
assert wcs2.wcs.crpix[0] == 42.0
dm2_tmp_fits = str(tmpdir.join("tmp_dm2.fits"))
dm2.to_fits(dm2_tmp_fits)
with ImageModel(dm2_tmp_fits) as dm3:
wcs3 = dm3.get_fits_wcs()
assert wcs3.wcs.crpix[0] == 42.0
x = np.random.rand(2**16, wcs1.wcs.naxis)
world1 = wcs1.all_pix2world(x, 1)
world2 = wcs3.all_pix2world(x, 1)
dm4 = ImageModel((10, 10))
dm4.set_fits_wcs(wcs3)
dm4_tmp_fits = str(tmpdir.join("tmp_dm4.fits"))
dm4.to_fits(dm4_tmp_fits, overwrite=True)
with ImageModel(dm4_tmp_fits) as dm5:
with warnings.catch_warnings():
# Filter out warnings generated by WCSLIB>=7.1
warnings.simplefilter("ignore")
wcs5 = dm5.get_fits_wcs()
assert wcs5.wcs.crpix[0] == 42.0
def test_object_node_iterator():
im = ImageModel()
items = []
for i in im.meta.items():
items.append(i[0])
assert 'date' in items
assert 'model_type' in items
