def setUp(self):
# Create a 64x64 simple MiriMeasuredModel object, with no error
# or quality arrays.
self.data = np.linspace(0.0, 100000.0, 64 * 64)
self.data.shape = [64, 64]
self.simpleproduct = MiriMeasuredModel(data=self.data)
# Add some example metadata.
self.simpleproduct.set_housekeeping_metadata('MIRI EC', 'Joe Bloggs',
'V1.0')
self.simpleproduct.set_instrument_metadata(detector='MIRIMAGE',
filt='F560W',
ccc_pos='OPEN',
deck_temperature=10.0,
detector_temperature=7.0)
self.simpleproduct.set_exposure_metadata(readpatt='SLOW',
nints=1,
ngroups=10,
frame_time=30.0,
integration_time=30.0,
group_time=300.0,
reset_time=0,
frame_resets=3)
# Create a more complex MiriMeasuredModel object from primary,
# error and quality arrays.
self.primary = [[10, 20, 30, 40], [50, 60, 70, 80],
[90, 100, 110, 120]]
self.error = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
self.quality = [[1, 0, 0, 0], [0, 1, 0, 1], [1, 0, 1, 0]]
self.dataproduct = MiriMeasuredModel(data=self.primary,
err=self.error,
dq=self.quality,
dq_def=master_flags)
# Add some example metadata.
self.dataproduct.set_instrument_metadata(detector='MIRIFUSHORT',
channel='1',
ccc_pos='OPEN',
deck_temperature=11.0,
detector_temperature=6.0)
self.dataproduct.set_exposure_metadata(readpatt='FAST',
nints=1,
ngroups=1,
frame_time=1.0,
integration_time=10.0,
group_time=10.0,
reset_time=0,
frame_resets=3)
self.testfile1 = "MiriMeasuredModel_test1.fits"
self.testfile2 = "MiriMeasuredModel_test2.fits"
self.tempfiles = [self.testfile1, self.testfile2]
def setUp(self):
# Create an instance of Class1 and prepare it to be tested.
# The setUp function is automatically executed before each test.
a = [[10, 10, 20, 40, 20, 10, 10], [10, 10, 20, 50, 20, 10, 10],
[10, 10, 20, 150, 20, 10, 10], [10, 10, 20, 70, 50, 25, 10],
[10, 10, 20, 50, 20, 10, 10], [10, 10, 40, 70, 30, 10, 10],
[10, 10, 20, 170, 20, 10, 10], [10, 10, 20, 60, 20, 10, 10],
[10, 10, 20, 40, 20, 10, 10]]
err = [[0.5, 0.5, 0.2, 0.05, 0.2, 0.1, 0.5],
[0.6, 0.5, 0.2, 0.05, 0.2, 0.5, 0.6],
[0.6, 0.5, 0.2, 0.05, 0.2, 0.5, 0.6],
[0.5, 0.4, 0.2, 0.05, 0.2, 0.4, 0.5],
[0.4, 0.5, 0.2, 0.02, 0.2, 0.5, 0.5],
[0.1, 0.4, 0.2, 0.02, 0.2, 0.6, 0.4],
[0.6, 0.5, 0.2, 0.03, 0.2, 0.5, 0.6],
[0.6, 0.5, 0.2, 0.02, 0.2, 0.5, 0.5],
[0.5, 0.5, 0.2, 0.05, 0.2, 0.5, 0.4]]
self.dataproduct = MiriMeasuredModel(data=a, err=err)
self.assertTrue(hasattr(self.dataproduct, 'data'))
def flatten_to_image(self, weighted=True, perfect=False, masked=False):
"""
Flatten a data cube in the wavelength direction to generate one
combined image.
The output signal is the (weighted) average of the input signal.
The output error is the RMS of the input error.
:Parameters:
weighted: bool, optional
Set to True to use the error array (if present) to make a
weighted average.
Set to False for an unweighted average.
perfect: bool, optional
Set to True to regard as bad any pixel in the image
which has contributing bad pixel.
Set to False to regard a pixel in the image as bad only
if all of the contributing pixels are bad (and use the
error array to mark the reduction in quality).
masked: bool, optional
Set to True to use the masked versions of the data arrays.
By default, the non-masked versions are used. This is more
efficient but may generate runtime warning messages.
:Returns:
image: MiriMeasuredModel
A 2-D image object containing the averaged image.
"""
if masked:
(flat_data, flat_error, flat_quality) = \
spec3d_to_image(self.data_masked,
self.err_masked, self.dq,
waveaxis=self.waveaxis,
weighted=weighted, perfect=perfect)
else:
(flat_data, flat_error, flat_quality) = \
spec3d_to_image(self.data, self.err, self.dq,
waveaxis=self.waveaxis,
weighted=weighted, perfect=perfect)
title = "(flattened to image)"
# NOTE: The STScI data model doesn't not support masked arrays,
# so they need to be converted back to contiguous arrays before
# they are used to create a new data product.
image = MiriMeasuredModel(data=np.ascontiguousarray(flat_data),
err=np.ascontiguousarray(flat_error),
dq=np.ascontiguousarray(flat_quality),
title=title)
return image
def test_fitsio(self):
# Suppress metadata warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Check that the data products can be written to a FITS
# file and read back again without changing the data.
self.simpleproduct.save(self.testfile1, overwrite=True)
with MiriMeasuredModel(self.testfile1) as readback:
self.assertTrue(
np.allclose(self.simpleproduct.data, readback.data))
del readback
self.dataproduct.save(self.testfile2, overwrite=True)
with MiriMeasuredModel(self.testfile2) as readback:
assert_products_equal(self,
self.dataproduct,
readback,
arrays=['data', 'err', 'dq'])
# FIXME: removed dq_def until data corruption bug fixed. Bug 589
# tables='dq_def' )
del readback
def test_masking(self):
# The DQ array must mask off bad values in the SCI and ERR arrays.
a2 = [[10, 999, 10, 999], [999, 10, 10, 999], [10, 10, 999, 10]]
b2 = [[1, 99, 1, 99], [99, 1, 1, 99], [1, 1, 99, 1]]
c2 = [[0, 1, 0, 1], [1, 0, 0, 1], [0, 0, 1, 0]]
# Without a DQ array (assuming the default quality value is 0)
# the SCI and ERR arrays are not masked, so their averages
# include the 999s and are greater than they ought to be.
newdp = MiriMeasuredModel(data=a2, err=b2)
meandata = np.mean(newdp.data_masked)
self.assertGreater(meandata, 10)
meanerr = np.mean(newdp.err_masked)
self.assertGreater(meanerr, 1)
# The addition of the quality data should cause the SCI and ERR
# arrays to be masked off and give the correct average.
newdp2 = MiriMeasuredModel(data=a2, err=b2, dq=c2)
meandata2 = np.mean(newdp2.data_masked)
self.assertAlmostEqual(meandata2, 10)
meanerr2 = np.mean(newdp2.err_masked)
self.assertAlmostEqual(meanerr2, 1)
del newdp, newdp2
[10, 10, 20, 150, 20, 10, 10], [10, 10, 20, 70, 50, 25, 10],
[10, 10, 20, 50, 20, 10, 10], [10, 10, 40, 70, 30, 10, 10],
[10, 10, 20, 170, 20, 10, 10], [10, 10, 20, 60, 20, 10, 10],
[10, 10, 20, 40, 20, 10, 10]]
b = [[0.5, 0.5, 0.2, 0.05, 0.2, 0.1, 0.5],
[0.6, 0.5, 0.2, 0.05, 0.2, 0.5, 0.6],
[0.6, 0.5, 0.2, 0.05, 0.2, 0.5, 0.6],
[0.5, 0.4, 0.2, 0.05, 0.2, 0.4, 0.5],
[0.4, 0.5, 0.2, 0.02, 0.2, 0.5, 0.5],
[0.1, 0.4, 0.2, 0.02, 0.2, 0.6, 0.4],
[0.6, 0.5, 0.2, 0.03, 0.2, 0.5, 0.6],
[0.6, 0.5, 0.2, 0.02, 0.2, 0.5, 0.5],
[0.5, 0.5, 0.2, 0.05, 0.2, 0.5, 0.4]]
print("Creating slope product from arrays")
product = MiriMeasuredModel(data=a, err=b)
print(product)
if PLOTTING:
product.plot()
print("lrs2D_spextract")
lrs2d = lrs2D_spextract(product, xmin=3, xmax=6, ymin=3, ymax=6)
print(lrs2d)
if PLOTTING:
lrs2d.plot()
print("subtractBackground")
lrs2d = subtractBackground(lrs2d, lrs2d)
print(lrs2d)
if PLOTTING:
lrs2d.plot()
class TestMiriMeasuredModel(unittest.TestCase):
# Test the MiriMeasuredModel class
def setUp(self):
# Create a 64x64 simple MiriMeasuredModel object, with no error
# or quality arrays.
self.data = np.linspace(0.0, 100000.0, 64 * 64)
self.data.shape = [64, 64]
self.simpleproduct = MiriMeasuredModel(data=self.data)
# Add some example metadata.
self.simpleproduct.set_housekeeping_metadata('MIRI EC', 'Joe Bloggs',
'V1.0')
self.simpleproduct.set_instrument_metadata(detector='MIRIMAGE',
filt='F560W',
ccc_pos='OPEN',
deck_temperature=10.0,
detector_temperature=7.0)
self.simpleproduct.set_exposure_metadata(readpatt='SLOW',
nints=1,
ngroups=10,
frame_time=30.0,
integration_time=30.0,
group_time=300.0,
reset_time=0,
frame_resets=3)
# Create a more complex MiriMeasuredModel object from primary,
# error and quality arrays.
self.primary = [[10, 20, 30, 40], [50, 60, 70, 80],
[90, 100, 110, 120]]
self.error = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
self.quality = [[1, 0, 0, 0], [0, 1, 0, 1], [1, 0, 1, 0]]
self.dataproduct = MiriMeasuredModel(data=self.primary,
err=self.error,
dq=self.quality,
dq_def=master_flags)
# Add some example metadata.
self.dataproduct.set_instrument_metadata(detector='MIRIFUSHORT',
channel='1',
ccc_pos='OPEN',
deck_temperature=11.0,
detector_temperature=6.0)
self.dataproduct.set_exposure_metadata(readpatt='FAST',
nints=1,
ngroups=1,
frame_time=1.0,
integration_time=10.0,
group_time=10.0,
reset_time=0,
frame_resets=3)
self.testfile1 = "MiriMeasuredModel_test1.fits"
self.testfile2 = "MiriMeasuredModel_test2.fits"
self.tempfiles = [self.testfile1, self.testfile2]
def tearDown(self):
# Tidy up
del self.dataproduct
del self.primary, self.error, self.quality
del self.simpleproduct
del self.data
# Remove temporary files, if they exist and if able to.
for tempfile in self.tempfiles:
if os.path.isfile(tempfile):
try:
os.remove(tempfile)
except Exception as e:
strg = "Could not remove temporary file, " + tempfile + \
"\n " + str(e)
warnings.warn(strg)
del self.tempfiles
def test_creation(self):
# Check that the DQ_DEF field names in the class variable are the same
# as the ones declared in the schema.
dq_def_names = list(MiriMeasuredModel.dq_def_names)
schema_names = list(self.dataproduct.get_field_names('dq_def'))
self.assertEqual(
dq_def_names, schema_names,
"'dq_def_names' class variable does not match schema")
# Test that the error and quality arrays are optional.
a2 = [[10, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120]]
b2 = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
c2 = [[1, 0, 0, 0], [0, 1, 0, 1], [1, 0, 1, 0]]
# 1) Data array only. Data array must exist and be non-empty.
# Other arrays should exist and be the same size and shape as the
# data array. They should be full of default values.
newdp1 = MiriMeasuredModel(data=a2)
self.assertIsNotNone(newdp1.data)
self.assertGreater(len(newdp1.data), 0)
self.assertIsNotNone(newdp1.err)
self.assertEqual(newdp1.err.shape, newdp1.data.shape)
# Assumes default is 0.0 - see schema
self.assertAlmostEqual(np.mean(newdp1.err), 0.0)
self.assertIsNotNone(newdp1.dq)
self.assertEqual(newdp1.dq.shape, newdp1.dq.shape)
# Assumes default is 0 - see schema
self.assertEqual(np.mean(newdp1.dq), 0)
descr1 = str(newdp1)
self.assertIsNotNone(descr1)
del newdp1, descr1
# 2) Data and error arrays only. Data and error arrays must exist
# and be non-empty. Quality array should exist but be the same
# size and shape as the data array. It should be full of default
# values.
newdp2 = MiriMeasuredModel(data=a2, err=b2)
self.assertIsNotNone(newdp2.data)
self.assertGreater(len(newdp2.data), 0)
self.assertIsNotNone(newdp2.err)
self.assertEqual(newdp2.err.shape, newdp2.data.shape)
# The error array must not be full of default values.
self.assertNotAlmostEqual(np.mean(newdp2.err), 0.0)
self.assertIsNotNone(newdp2.dq)
self.assertEqual(newdp2.dq.shape, newdp2.dq.shape)
# Assumes default is 0 - see schema
self.assertEqual(np.mean(newdp2.dq), 0)
descr2 = str(newdp2)
self.assertIsNotNone(descr2)
del newdp2, descr2
# 3) Data, error and quality arrays. All arrays must exist,
# be non-empty and be the same size and shape.
newdp3 = MiriMeasuredModel(data=a2, err=b2, dq=c2)
self.assertIsNotNone(newdp3.data)
self.assertGreater(len(newdp3.data), 0)
self.assertIsNotNone(newdp3.err)
self.assertEqual(newdp3.err.shape, newdp3.data.shape)
# The error array must not be full of default values.
self.assertNotAlmostEqual(np.mean(newdp3.err), 0.0)
self.assertIsNotNone(newdp3.dq)
self.assertEqual(newdp3.dq.shape, newdp3.dq.shape)
# The quality array must not be full of default values.
self.assertNotEqual(np.mean(newdp3.dq), 0)
descr3 = str(newdp3)
self.assertIsNotNone(descr3)
del newdp3, descr3
# It should be possible to set up an empty data product with
# a specified shape. All three arrays should be initialised to
# the same shape.
emptydp = MiriMeasuredModel((4, 4))
self.assertIsNotNone(emptydp.data)
self.assertEqual(emptydp.data.shape, (4, 4))
self.assertIsNotNone(emptydp.err)
self.assertEqual(emptydp.err.shape, (4, 4))
self.assertIsNotNone(emptydp.dq)
self.assertEqual(emptydp.dq.shape, (4, 4))
descr = str(emptydp)
self.assertIsNotNone(descr)
del emptydp, descr
# A null data product can also be created and populated
# with data later.
nulldp = MiriMeasuredModel()
descr1 = str(nulldp)
self.assertIsNotNone(descr1)
nulldp.data = np.asarray(a2)
self.assertIsNotNone(nulldp.err)
self.assertIsNotNone(nulldp.dq)
descr2 = str(nulldp)
self.assertIsNotNone(descr2)
del nulldp, descr1, descr2
# A scalar data product is possible, even if of little use.
scalardp = MiriMeasuredModel(data=42)
self.assertEqual(scalardp.data, 42)
self.assertIsNotNone(scalardp.err)
self.assertIsNotNone(scalardp.dq)
descr = str(scalardp)
self.assertIsNotNone(descr)
del scalardp, descr
# Attempts to create a data product from invalid data types
# and stupid values must be detected.
# NOTE: A bug in the JWST data model might cause an AttributeError
# to be raised instead of a ValueError. If this happens, try a newer
# version of the JWST data model library.
self.assertRaises(ValueError, MiriMeasuredModel, init=[])
self.assertRaises(ValueError, MiriMeasuredModel, init=42)
self.assertRaises(ValueError,
MiriMeasuredModel,
init='not a file name')
self.assertRaises(IOError, MiriMeasuredModel, init='nosuchfile.fits')
#self.assertRaises(ValueError, MiriMeasuredModel, init='')
self.assertRaises(ValueError, MiriMeasuredModel, data='badstring')
def test_metadata(self):
# Check the dataproducts contain metadata
# First test the basic STScI FITS keyword lookup method.
kwstrg = self.simpleproduct.find_fits_keyword('TELESCOP',
return_result=True)
self.assertIsNotNone(kwstrg)
# kwstrg is a list - assume the first entry is what we want.
telname = self.simpleproduct[kwstrg[0]]
self.assertEqual(telname, 'JWST')
# Accessing the tree structure directly should also work.
telname = self.simpleproduct.meta.telescope
self.assertEqual(telname, 'JWST')
# An alternative lookup provided by the MIRI data model.
telname = self.simpleproduct.get_fits_keyword('TELESCOP')
self.assertEqual(telname, 'JWST')
kwstrg = self.simpleproduct.find_fits_keyword('INSTRUME',
return_result=True)
self.assertIsNotNone(kwstrg)
insname = self.simpleproduct[kwstrg[0]]
self.assertEqual(insname, 'MIRI')
insname = self.simpleproduct.meta.instrument.name
self.assertEqual(insname, 'MIRI')
insname = self.simpleproduct.get_fits_keyword('INSTRUME')
self.assertEqual(insname, 'MIRI')
# Add some history records and check they exist.
self.simpleproduct.add_history('History 1')
self.simpleproduct.add_history('History 2')
self.simpleproduct.add_history('History 3')
self.assertGreaterEqual(len(self.simpleproduct.get_history()), 3)
strg = self.simpleproduct.get_history_str()
self.assertIsNotNone(strg)
self.assertGreater(len(strg), 0)
def test_content(self):
# The data, err and dq attributes are aliases for the primary,
# error and quality arrays
self.assertTrue(np.allclose(self.primary, self.dataproduct.data))
self.assertTrue(np.allclose(self.error, self.dataproduct.err))
self.assertTrue(np.allclose(self.quality, self.dataproduct.dq))
def test_copy(self):
# Test that a copy can be made of the data product.
datacopy = self.dataproduct.copy()
self.assertIsNotNone(datacopy)
assert_products_equal(self,
self.dataproduct,
datacopy,
arrays=['data', 'err', 'dq'],
tables='dq_def')
del datacopy
def test_fitsio(self):
# Suppress metadata warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Check that the data products can be written to a FITS
# file and read back again without changing the data.
self.simpleproduct.save(self.testfile1, overwrite=True)
with MiriMeasuredModel(self.testfile1) as readback:
self.assertTrue(
np.allclose(self.simpleproduct.data, readback.data))
del readback
self.dataproduct.save(self.testfile2, overwrite=True)
with MiriMeasuredModel(self.testfile2) as readback:
assert_products_equal(self,
self.dataproduct,
readback,
arrays=['data', 'err', 'dq'])
# FIXME: removed dq_def until data corruption bug fixed. Bug 589
# tables='dq_def' )
del readback
def test_asciiio(self):
# Check that the data products can be written to an ASCII
# file and read back again without changing the data.
# TODO: At the moment jwst_lib only supports FITS I/O
pass
# # Suppress metadata warnings
# with warnings.catch_warnings():
# warnings.simplefilter("ignore")
# self.simpleproduct.save(self.testfile_ascii, overwrite=True)
# with MiriMeasuredModel(self.testfile_ascii) as readback:
# self.assertTrue( np.allclose(self.simpleproduct.data,
# readback.data) )
# del readback
def test_masking(self):
# The DQ array must mask off bad values in the SCI and ERR arrays.
a2 = [[10, 999, 10, 999], [999, 10, 10, 999], [10, 10, 999, 10]]
b2 = [[1, 99, 1, 99], [99, 1, 1, 99], [1, 1, 99, 1]]
c2 = [[0, 1, 0, 1], [1, 0, 0, 1], [0, 0, 1, 0]]
# Without a DQ array (assuming the default quality value is 0)
# the SCI and ERR arrays are not masked, so their averages
# include the 999s and are greater than they ought to be.
newdp = MiriMeasuredModel(data=a2, err=b2)
meandata = np.mean(newdp.data_masked)
self.assertGreater(meandata, 10)
meanerr = np.mean(newdp.err_masked)
self.assertGreater(meanerr, 1)
# The addition of the quality data should cause the SCI and ERR
# arrays to be masked off and give the correct average.
newdp2 = MiriMeasuredModel(data=a2, err=b2, dq=c2)
meandata2 = np.mean(newdp2.data_masked)
self.assertAlmostEqual(meandata2, 10)
meanerr2 = np.mean(newdp2.err_masked)
self.assertAlmostEqual(meanerr2, 1)
del newdp, newdp2
def test_arithmetic(self):
a2 = [[90, 80, 70, 60], [50, 40, 30, 20], [10, 0, -10, -20]]
b2 = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
c2 = [[0, 1, 1, 0], [0, 2, 0, 2], [1, 0, 1, 0]]
newdp = MiriMeasuredModel(data=a2, err=b2, dq=c2)
# Self-subtraction of the simple product. The result
# should be zero.
newsimple = self.simpleproduct - self.simpleproduct
self.assertAlmostEqual(newsimple.data.all(), 0.0)
del newsimple
# Scalar addition
result = self.dataproduct + 42
test1 = self.dataproduct.data + 42
test2 = result.data
self.assertEqual(test1.all(), test2.all())
del result
# Data product addition
result = self.dataproduct + newdp
test1 = self.dataproduct.data + newdp.data
test2 = result.data
self.assertEqual(test1.all(), test2.all())
# Test that error arrays are combined properly - at least for
# a couple of unmasked points.
expectedsq = self.error[1][0] * self.error[1][0] + b2[1][0] * b2[1][0]
actualsq = result.err[1, 0] * result.err[1, 0]
self.assertAlmostEqual(expectedsq, actualsq)
expectedsq = self.error[2][1] * self.error[2][1] + b2[2][1] * b2[2][1]
actualsq = result.err[2, 1] * result.err[2, 1]
self.assertAlmostEqual(expectedsq, actualsq)
del result
# Scalar subtraction
result = self.dataproduct - 42
test1 = self.dataproduct.data - 42
test2 = result.data
self.assertEqual(test1.all(), test2.all())
del result
# Data product subtraction
result = self.dataproduct - newdp
test1 = self.dataproduct.data - newdp.data
test2 = result.data
self.assertEqual(test1.all(), test2.all())
# Test that error arrays are combined properly - at least for
# a couple of unmasked points.
expectedsq = self.error[1][0] * self.error[1][0] + b2[1][0] * b2[1][0]
actualsq = result.err[1, 0] * result.err[1, 0]
self.assertAlmostEqual(expectedsq, actualsq)
expectedsq = self.error[2][1] * self.error[2][1] + b2[2][1] * b2[2][1]
actualsq = result.err[2, 1] * result.err[2, 1]
self.assertAlmostEqual(expectedsq, actualsq)
del result
# Addition and subtraction should cancel each other out
result = self.dataproduct + newdp - newdp
test1 = self.dataproduct.data
test2 = result.data
self.assertEqual(test1.all(), test2.all())
del result
# Scalar multiplication
result = self.dataproduct * 3
test1 = self.dataproduct.data * 3
test2 = result.data
self.assertEqual(test1.all(), test2.all())
del result
# Data product multiplication
result = self.dataproduct * newdp
test1 = self.dataproduct.data * newdp.data
test2 = result.data
self.assertEqual(test1.all(), test2.all())
err1 = self.dataproduct.err
da1 = self.dataproduct.data
err2 = newdp.err
da2 = newdp.data
expectedErr = np.sqrt(err1 * err1 * da2 * da2 +
err2 * err2 * da1 * da1)
self.assertTrue(np.array_equal(expectedErr, result.err))
del result, da1, da2, err1, err2, expectedErr
# Scalar division
result = self.dataproduct / 3.0
test1 = self.dataproduct.data / 3.0
test2 = result.data
self.assertAlmostEqual(test1.all(), test2.all())
del test1, test2, result
# Division by zero
self.assertRaises(ValueError, self.dataproduct.__truediv__, 0.0)
# Data product division
#print("NOTE: The following test is expected to generate run time warnings.")
with warnings.catch_warnings():
warnings.simplefilter("ignore")
result = self.dataproduct / newdp
test1 = self.dataproduct.data / newdp.data
test2 = result.data
self.assertEqual(test1.all(), test2.all())
# Test Juergen Schreiber error propagation
dat = self.dataproduct.data[1][1]
newdat = newdp.data[1][1]
resultErr = result.err[1][1]
dpErr = self.dataproduct.err[1][1]
newdpErr = newdp.err[1][1]
expectErr = np.sqrt( dpErr * dpErr/(newdat * newdat) + \
newdpErr * newdpErr * dat * dat / \
(newdat * newdat * newdat * newdat))
self.assertEqual(expectErr, resultErr)
del test1, test2, result
# More complex arithmetic should be possible.
newdp2 = newdp * 2
newdp3 = newdp * 3
newdp4 = newdp2 + newdp3
result = ((self.dataproduct - newdp) * newdp2 / newdp3) + newdp4
del newdp, newdp2, newdp3, newdp4
del result
def test_broadcasting(self):
# Test that operations where the broadcasting of one array
# onto a similar shaped array work.
a4x3 = [[90, 80, 70, 60], [50, 40, 30, 20], [10, 0, -10, -20]]
b4x3 = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
#c4x3 = [[0,1,0,0],[0,0,1,0],[1,0,0,1]]
a4x1 = [4, 3, 2, 1]
b4x1 = [1, 2, 1, 2]
c4x1 = [0, 1, 0, 0]
a5x1 = [5, 4, 3, 2, 1]
b5x1 = [1, 2, 3, 2, 1]
c5x1 = [0, 1, 0, 0, 1]
# Create an object with 4x3 primary and error arrays but a 4x1
# quality array. This should succeed because the quality array
# is broadcastable.
newdp1 = MiriMeasuredModel(data=a4x3, err=b4x3, dq=c4x1)
self.assertTrue(np.allclose(a4x3, newdp1.data))
self.assertTrue(np.allclose(b4x3, newdp1.err))
self.assertTrue(np.allclose(c4x1, newdp1.dq))
# 5x1 is not broadcastable onto 4x3 and this statement should fail.
self.assertRaises(TypeError,
MiriMeasuredModel,
data=a4x3,
err=b5x1,
dq=c5x1)
# Combine two broadcastable object mathematically.
# The + and - operations should be commutative and the result
# should be saveable to a FITS file.
newdp2 = MiriMeasuredModel(data=a4x1, err=b4x1, dq=c4x1)
result1 = newdp1 + newdp2
result2 = newdp2 + newdp1
self.assertEqual(result1.data.shape, result2.data.shape)
self.assertTrue(np.allclose(result1.data, result2.data))
self.assertTrue(np.allclose(result1.err, result2.err))
self.assertTrue(np.allclose(result1.dq, result2.dq))
result1.save(self.testfile1, overwrite=True)
result2.save(self.testfile2, overwrite=True)
del result1, result2
result1 = newdp1 * newdp2
result2 = newdp2 * newdp1
self.assertEqual(result1.data.shape, result2.data.shape)
self.assertTrue(np.allclose(result1.data, result2.data))
self.assertTrue(np.allclose(result1.err, result2.err))
self.assertTrue(np.allclose(result1.dq, result2.dq))
result1.save(self.testfile1, overwrite=True)
result2.save(self.testfile2, overwrite=True)
del result1, result2
# The - and / operations are not commutative, but the data shape
# should be consistent and the quality arrays should be combined
# in the same way.
result1 = newdp1 - newdp2
result2 = newdp2 - newdp1
self.assertEqual(result1.data.shape, result2.data.shape)
self.assertTrue(np.allclose(result1.err, result2.err))
self.assertTrue(np.allclose(result1.dq, result2.dq))
result1.save(self.testfile1, overwrite=True)
result2.save(self.testfile2, overwrite=True)
del result1, result2
result1 = newdp1 / newdp2
result2 = newdp2 / newdp1
self.assertEqual(result1.data.shape, result2.data.shape)
# The errors resulting from division depend on the order
# of the operation.
self.assertTrue(np.allclose(result1.dq, result2.dq))
result1.save(self.testfile1, overwrite=True)
result2.save(self.testfile2, overwrite=True)
del result1, result2
def test_description(self):
# Test that the querying and description functions work.
# For the test to pass these need to run without error
# and generate non-null strings.
descr = str(self.simpleproduct)
self.assertIsNotNone(descr)
del descr
descr = repr(self.simpleproduct)
self.assertIsNotNone(descr)
del descr
descr = self.simpleproduct.stats()
self.assertIsNotNone(descr)
del descr
descr = str(self.dataproduct)
self.assertIsNotNone(descr)
del descr
descr = str(self.dataproduct)
self.assertIsNotNone(descr)
del descr
descr = self.dataproduct.stats()
self.assertIsNotNone(descr)
del descr
# Attempt to access the SCI, ERROR and DQ arrays through attributes.
descr = str(self.dataproduct.data)
self.assertIsNotNone(descr)
del descr
descr = str(self.dataproduct.err)
self.assertIsNotNone(descr)
del descr
descr = str(self.dataproduct.dq)
self.assertIsNotNone(descr)
del descr
def test_broadcasting(self):
# Test that operations where the broadcasting of one array
# onto a similar shaped array work.
a4x3 = [[90, 80, 70, 60], [50, 40, 30, 20], [10, 0, -10, -20]]
b4x3 = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
#c4x3 = [[0,1,0,0],[0,0,1,0],[1,0,0,1]]
a4x1 = [4, 3, 2, 1]
b4x1 = [1, 2, 1, 2]
c4x1 = [0, 1, 0, 0]
a5x1 = [5, 4, 3, 2, 1]
b5x1 = [1, 2, 3, 2, 1]
c5x1 = [0, 1, 0, 0, 1]
# Create an object with 4x3 primary and error arrays but a 4x1
# quality array. This should succeed because the quality array
# is broadcastable.
newdp1 = MiriMeasuredModel(data=a4x3, err=b4x3, dq=c4x1)
self.assertTrue(np.allclose(a4x3, newdp1.data))
self.assertTrue(np.allclose(b4x3, newdp1.err))
self.assertTrue(np.allclose(c4x1, newdp1.dq))
# 5x1 is not broadcastable onto 4x3 and this statement should fail.
self.assertRaises(TypeError,
MiriMeasuredModel,
data=a4x3,
err=b5x1,
dq=c5x1)
# Combine two broadcastable object mathematically.
# The + and - operations should be commutative and the result
# should be saveable to a FITS file.
newdp2 = MiriMeasuredModel(data=a4x1, err=b4x1, dq=c4x1)
result1 = newdp1 + newdp2
result2 = newdp2 + newdp1
self.assertEqual(result1.data.shape, result2.data.shape)
self.assertTrue(np.allclose(result1.data, result2.data))
self.assertTrue(np.allclose(result1.err, result2.err))
self.assertTrue(np.allclose(result1.dq, result2.dq))
result1.save(self.testfile1, overwrite=True)
result2.save(self.testfile2, overwrite=True)
del result1, result2
result1 = newdp1 * newdp2
result2 = newdp2 * newdp1
self.assertEqual(result1.data.shape, result2.data.shape)
self.assertTrue(np.allclose(result1.data, result2.data))
self.assertTrue(np.allclose(result1.err, result2.err))
self.assertTrue(np.allclose(result1.dq, result2.dq))
result1.save(self.testfile1, overwrite=True)
result2.save(self.testfile2, overwrite=True)
del result1, result2
# The - and / operations are not commutative, but the data shape
# should be consistent and the quality arrays should be combined
# in the same way.
result1 = newdp1 - newdp2
result2 = newdp2 - newdp1
self.assertEqual(result1.data.shape, result2.data.shape)
self.assertTrue(np.allclose(result1.err, result2.err))
self.assertTrue(np.allclose(result1.dq, result2.dq))
result1.save(self.testfile1, overwrite=True)
result2.save(self.testfile2, overwrite=True)
del result1, result2
result1 = newdp1 / newdp2
result2 = newdp2 / newdp1
self.assertEqual(result1.data.shape, result2.data.shape)
# The errors resulting from division depend on the order
# of the operation.
self.assertTrue(np.allclose(result1.dq, result2.dq))
result1.save(self.testfile1, overwrite=True)
result2.save(self.testfile2, overwrite=True)
del result1, result2
def test_arithmetic(self):
a2 = [[90, 80, 70, 60], [50, 40, 30, 20], [10, 0, -10, -20]]
b2 = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
c2 = [[0, 1, 1, 0], [0, 2, 0, 2], [1, 0, 1, 0]]
newdp = MiriMeasuredModel(data=a2, err=b2, dq=c2)
# Self-subtraction of the simple product. The result
# should be zero.
newsimple = self.simpleproduct - self.simpleproduct
self.assertAlmostEqual(newsimple.data.all(), 0.0)
del newsimple
# Scalar addition
result = self.dataproduct + 42
test1 = self.dataproduct.data + 42
test2 = result.data
self.assertEqual(test1.all(), test2.all())
del result
# Data product addition
result = self.dataproduct + newdp
test1 = self.dataproduct.data + newdp.data
test2 = result.data
self.assertEqual(test1.all(), test2.all())
# Test that error arrays are combined properly - at least for
# a couple of unmasked points.
expectedsq = self.error[1][0] * self.error[1][0] + b2[1][0] * b2[1][0]
actualsq = result.err[1, 0] * result.err[1, 0]
self.assertAlmostEqual(expectedsq, actualsq)
expectedsq = self.error[2][1] * self.error[2][1] + b2[2][1] * b2[2][1]
actualsq = result.err[2, 1] * result.err[2, 1]
self.assertAlmostEqual(expectedsq, actualsq)
del result
# Scalar subtraction
result = self.dataproduct - 42
test1 = self.dataproduct.data - 42
test2 = result.data
self.assertEqual(test1.all(), test2.all())
del result
# Data product subtraction
result = self.dataproduct - newdp
test1 = self.dataproduct.data - newdp.data
test2 = result.data
self.assertEqual(test1.all(), test2.all())
# Test that error arrays are combined properly - at least for
# a couple of unmasked points.
expectedsq = self.error[1][0] * self.error[1][0] + b2[1][0] * b2[1][0]
actualsq = result.err[1, 0] * result.err[1, 0]
self.assertAlmostEqual(expectedsq, actualsq)
expectedsq = self.error[2][1] * self.error[2][1] + b2[2][1] * b2[2][1]
actualsq = result.err[2, 1] * result.err[2, 1]
self.assertAlmostEqual(expectedsq, actualsq)
del result
# Addition and subtraction should cancel each other out
result = self.dataproduct + newdp - newdp
test1 = self.dataproduct.data
test2 = result.data
self.assertEqual(test1.all(), test2.all())
del result
# Scalar multiplication
result = self.dataproduct * 3
test1 = self.dataproduct.data * 3
test2 = result.data
self.assertEqual(test1.all(), test2.all())
del result
# Data product multiplication
result = self.dataproduct * newdp
test1 = self.dataproduct.data * newdp.data
test2 = result.data
self.assertEqual(test1.all(), test2.all())
err1 = self.dataproduct.err
da1 = self.dataproduct.data
err2 = newdp.err
da2 = newdp.data
expectedErr = np.sqrt(err1 * err1 * da2 * da2 +
err2 * err2 * da1 * da1)
self.assertTrue(np.array_equal(expectedErr, result.err))
del result, da1, da2, err1, err2, expectedErr
# Scalar division
result = self.dataproduct / 3.0
test1 = self.dataproduct.data / 3.0
test2 = result.data
self.assertAlmostEqual(test1.all(), test2.all())
del test1, test2, result
# Division by zero
self.assertRaises(ValueError, self.dataproduct.__truediv__, 0.0)
# Data product division
#print("NOTE: The following test is expected to generate run time warnings.")
with warnings.catch_warnings():
warnings.simplefilter("ignore")
result = self.dataproduct / newdp
test1 = self.dataproduct.data / newdp.data
test2 = result.data
self.assertEqual(test1.all(), test2.all())
# Test Juergen Schreiber error propagation
dat = self.dataproduct.data[1][1]
newdat = newdp.data[1][1]
resultErr = result.err[1][1]
dpErr = self.dataproduct.err[1][1]
newdpErr = newdp.err[1][1]
expectErr = np.sqrt( dpErr * dpErr/(newdat * newdat) + \
newdpErr * newdpErr * dat * dat / \
(newdat * newdat * newdat * newdat))
self.assertEqual(expectErr, resultErr)
del test1, test2, result
# More complex arithmetic should be possible.
newdp2 = newdp * 2
newdp3 = newdp * 3
newdp4 = newdp2 + newdp3
result = ((self.dataproduct - newdp) * newdp2 / newdp3) + newdp4
del newdp, newdp2, newdp3, newdp4
del result
def test_creation(self):
# Check that the DQ_DEF field names in the class variable are the same
# as the ones declared in the schema.
dq_def_names = list(MiriMeasuredModel.dq_def_names)
schema_names = list(self.dataproduct.get_field_names('dq_def'))
self.assertEqual(
dq_def_names, schema_names,
"'dq_def_names' class variable does not match schema")
# Test that the error and quality arrays are optional.
a2 = [[10, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120]]
b2 = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
c2 = [[1, 0, 0, 0], [0, 1, 0, 1], [1, 0, 1, 0]]
# 1) Data array only. Data array must exist and be non-empty.
# Other arrays should exist and be the same size and shape as the
# data array. They should be full of default values.
newdp1 = MiriMeasuredModel(data=a2)
self.assertIsNotNone(newdp1.data)
self.assertGreater(len(newdp1.data), 0)
self.assertIsNotNone(newdp1.err)
self.assertEqual(newdp1.err.shape, newdp1.data.shape)
# Assumes default is 0.0 - see schema
self.assertAlmostEqual(np.mean(newdp1.err), 0.0)
self.assertIsNotNone(newdp1.dq)
self.assertEqual(newdp1.dq.shape, newdp1.dq.shape)
# Assumes default is 0 - see schema
self.assertEqual(np.mean(newdp1.dq), 0)
descr1 = str(newdp1)
self.assertIsNotNone(descr1)
del newdp1, descr1
# 2) Data and error arrays only. Data and error arrays must exist
# and be non-empty. Quality array should exist but be the same
# size and shape as the data array. It should be full of default
# values.
newdp2 = MiriMeasuredModel(data=a2, err=b2)
self.assertIsNotNone(newdp2.data)
self.assertGreater(len(newdp2.data), 0)
self.assertIsNotNone(newdp2.err)
self.assertEqual(newdp2.err.shape, newdp2.data.shape)
# The error array must not be full of default values.
self.assertNotAlmostEqual(np.mean(newdp2.err), 0.0)
self.assertIsNotNone(newdp2.dq)
self.assertEqual(newdp2.dq.shape, newdp2.dq.shape)
# Assumes default is 0 - see schema
self.assertEqual(np.mean(newdp2.dq), 0)
descr2 = str(newdp2)
self.assertIsNotNone(descr2)
del newdp2, descr2
# 3) Data, error and quality arrays. All arrays must exist,
# be non-empty and be the same size and shape.
newdp3 = MiriMeasuredModel(data=a2, err=b2, dq=c2)
self.assertIsNotNone(newdp3.data)
self.assertGreater(len(newdp3.data), 0)
self.assertIsNotNone(newdp3.err)
self.assertEqual(newdp3.err.shape, newdp3.data.shape)
# The error array must not be full of default values.
self.assertNotAlmostEqual(np.mean(newdp3.err), 0.0)
self.assertIsNotNone(newdp3.dq)
self.assertEqual(newdp3.dq.shape, newdp3.dq.shape)
# The quality array must not be full of default values.
self.assertNotEqual(np.mean(newdp3.dq), 0)
descr3 = str(newdp3)
self.assertIsNotNone(descr3)
del newdp3, descr3
# It should be possible to set up an empty data product with
# a specified shape. All three arrays should be initialised to
# the same shape.
emptydp = MiriMeasuredModel((4, 4))
self.assertIsNotNone(emptydp.data)
self.assertEqual(emptydp.data.shape, (4, 4))
self.assertIsNotNone(emptydp.err)
self.assertEqual(emptydp.err.shape, (4, 4))
self.assertIsNotNone(emptydp.dq)
self.assertEqual(emptydp.dq.shape, (4, 4))
descr = str(emptydp)
self.assertIsNotNone(descr)
del emptydp, descr
# A null data product can also be created and populated
# with data later.
nulldp = MiriMeasuredModel()
descr1 = str(nulldp)
self.assertIsNotNone(descr1)
nulldp.data = np.asarray(a2)
self.assertIsNotNone(nulldp.err)
self.assertIsNotNone(nulldp.dq)
descr2 = str(nulldp)
self.assertIsNotNone(descr2)
del nulldp, descr1, descr2
# A scalar data product is possible, even if of little use.
scalardp = MiriMeasuredModel(data=42)
self.assertEqual(scalardp.data, 42)
self.assertIsNotNone(scalardp.err)
self.assertIsNotNone(scalardp.dq)
descr = str(scalardp)
self.assertIsNotNone(descr)
del scalardp, descr
# Attempts to create a data product from invalid data types
# and stupid values must be detected.
# NOTE: A bug in the JWST data model might cause an AttributeError
# to be raised instead of a ValueError. If this happens, try a newer
# version of the JWST data model library.
self.assertRaises(ValueError, MiriMeasuredModel, init=[])
self.assertRaises(ValueError, MiriMeasuredModel, init=42)
self.assertRaises(ValueError,
MiriMeasuredModel,
init='not a file name')
self.assertRaises(IOError, MiriMeasuredModel, init='nosuchfile.fits')
#self.assertRaises(ValueError, MiriMeasuredModel, init='')
self.assertRaises(ValueError, MiriMeasuredModel, data='badstring')
class TestLrsExtract(unittest.TestCase):
def setUp(self):
# Create an instance of Class1 and prepare it to be tested.
# The setUp function is automatically executed before each test.
a = [[10, 10, 20, 40, 20, 10, 10], [10, 10, 20, 50, 20, 10, 10],
[10, 10, 20, 150, 20, 10, 10], [10, 10, 20, 70, 50, 25, 10],
[10, 10, 20, 50, 20, 10, 10], [10, 10, 40, 70, 30, 10, 10],
[10, 10, 20, 170, 20, 10, 10], [10, 10, 20, 60, 20, 10, 10],
[10, 10, 20, 40, 20, 10, 10]]
err = [[0.5, 0.5, 0.2, 0.05, 0.2, 0.1, 0.5],
[0.6, 0.5, 0.2, 0.05, 0.2, 0.5, 0.6],
[0.6, 0.5, 0.2, 0.05, 0.2, 0.5, 0.6],
[0.5, 0.4, 0.2, 0.05, 0.2, 0.4, 0.5],
[0.4, 0.5, 0.2, 0.02, 0.2, 0.5, 0.5],
[0.1, 0.4, 0.2, 0.02, 0.2, 0.6, 0.4],
[0.6, 0.5, 0.2, 0.03, 0.2, 0.5, 0.6],
[0.6, 0.5, 0.2, 0.02, 0.2, 0.5, 0.5],
[0.5, 0.5, 0.2, 0.05, 0.2, 0.5, 0.4]]
self.dataproduct = MiriMeasuredModel(data=a, err=err)
self.assertTrue(hasattr(self.dataproduct, 'data'))
def tearDown(self):
# Clean the files and objects created.
# The tearDown function is automatically executed after each test.
del self.dataproduct
def test_lrs_extract(self):
# Test the method Class1.method1
# See http://docs.python.org/library/unittest.html#test-cases for further
# information
prod = self.dataproduct.copy()
result1 = lrs2D_spextract(prod, xmin=0, xmax=2, ymin=0, ymax=2)
# All the assert* methods available in the module unittest.TestCase for
# python >=2.5 are listed below with a usage example
self.assertTrue(np.shape(result1.data)[0] == 2)
self.assertTrue(np.shape(result1.data)[1] == 2)
sub = subtractBackground(result1, result1)
self.assertTrue(sub.data[0, 0] == 0.)
spec = lrs_extract_spec(sub)
self.assertTrue(len(spec.data) == 2)
self.assertTrue(len(spec.err) == 2)
spec1 = lrs_extract_spec_with_fit(prod.copy())
self.assertTrue(len(spec1) == 3)
spec1 = optimalSpecExtraction(prod.copy())
self.assertTrue(len(spec1.data) == 9)
self.assertTrue(len(spec1.err) == 9)
spec1 = get_psf_fit(prod.copy())
self.assertTrue(len(spec1) == 4)
wave = np.arange(5, 10, 0.5)
pos = np.arange(0, 5, 0.5)
rows, new_wave = interpolWaveOnRows(pos, wave)
self.assertTrue(len(rows) == 5)
self.assertTrue(len(new_wave) == 5)
self.assertTrue(np.alltrue(np.equal(np.mod(new_wave, 1), 0)))
spec = np.arange(100, 105, 0.5)
new_spec = interpolLin(wave, spec, new_wave)
self.assertTrue(len(new_spec) == 5)
self.assertTrue(np.alltrue(np.equal(np.mod(new_spec, 1), 0)))
spec = np.arange(100, 105, 0.5)
new_spec = interpol_lin(wave, spec, new_wave)
self.assertTrue(len(new_spec) == 5)
self.assertTrue(np.alltrue(np.equal(np.mod(new_spec, 1), 0)))
spec = np.arange(100, 105, 0.5)
new_spec = interpolSpline(wave, spec, new_wave)
self.assertTrue(len(new_spec) == 5)
del result1, spec, spec1, sub, prod, new_spec