def test_masking(self):
# Ramp data must have a dq array which gives a view of one
# or both of the pixeldq and groupdq masks
self.assertIsNotNone(self.dataproduct.dq)
# Create a data product masked by the pixeldq array.
# The dq and pixeldq arrays must be the same
mask1 = MiriRampModel(data=self.ahyper,
pixeldq=self.c1,
groupdq=self.chyper,
maskwith='pixeldq')
self.assertIsNotNone(mask1.pixeldq)
self.assertGreater(len(mask1.pixeldq), 0)
self.assertIsNotNone(mask1.dq)
self.assertGreater(len(mask1.dq), 0)
self.assertEqual(mask1.dq.shape, mask1.pixeldq.shape)
self.assertTrue(np.all(mask1.dq == mask1.pixeldq))
del mask1
# Create a data product masked by the groupdq array.
# The dq and groupdq arrays must be the same
mask2 = MiriRampModel(data=self.ahyper,
pixeldq=self.c1,
groupdq=self.chyper,
maskwith='groupdq')
self.assertIsNotNone(mask2.groupdq)
self.assertGreater(len(mask2.groupdq), 0)
self.assertIsNotNone(mask2.dq)
self.assertGreater(len(mask2.dq), 0)
self.assertEqual(mask2.dq.shape, mask2.groupdq.shape)
self.assertTrue(np.all(mask2.dq == mask2.groupdq))
del mask2
# Create a data product masked by both pixeldq and groupdq arrays.
# The result must have the same shape as the groupdq array but be
# a combination of both masks.
mask3 = MiriRampModel(data=self.ahyper,
pixeldq=self.c1,
groupdq=self.chyper,
maskwith='both')
self.assertIsNotNone(mask3.pixeldq)
self.assertGreater(len(mask3.pixeldq), 0)
self.assertIsNotNone(mask3.groupdq)
self.assertGreater(len(mask3.groupdq), 0)
self.assertIsNotNone(mask3.dq)
self.assertGreater(len(mask3.dq), 0)
self.assertEqual(mask3.dq.shape, mask3.groupdq.shape)
expected = mask3.groupdq | mask3.pixeldq
self.assertTrue(np.all(mask3.dq == expected))
del mask3
def setUp(self):
# Create a ramp data product.
# NOTE: A ramp product does not contain an ERR array.
self.a1 = [[10, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120]]
self.c1 = [[1, 0, 0, 0], [0, 1, 0, 1], [1, 0, 1, 0]]
self.c2 = [[0, 1, 1, 0], [1, 0, 0, 1], [1, 0, 1, 0]]
self.acube = [self.a1, self.a1, self.a1]
self.ccube = [self.c1, self.c2, self.c1]
self.ahyper = [self.acube, self.acube]
self.chyper = [self.ccube, self.ccube]
self.refout = np.ones_like(self.chyper)
self.dataproduct = MiriRampModel(data=self.ahyper,
refout=self.refout,
pixeldq=self.c1,
dq_def=pixeldq_flags,
groupdq=self.chyper,
groupdq_def=groupdq_flags)
# Add some example metadata.
self.dataproduct.set_housekeeping_metadata('MIRI EC', 'Joe Bloggs',
'V1.0')
self.dataproduct.set_observation_metadata()
self.dataproduct.set_target_metadata(0.0, 0.0)
self.dataproduct.set_instrument_metadata(detector='MIRIFULONG',
channel='1',
ccc_pos='OPEN',
deck_temperature=11.0,
detector_temperature=6.0)
self.dataproduct.set_exposure_metadata(readpatt='FAST',
nints=1,
ngroups=1,
frame_time=1.0,
integration_time=10.0,
group_time=10.0,
reset_time=0,
frame_resets=3)
self.testfile = "MiriRampModel_test.fits"
def test_fitsio(self):
# Suppress metadata warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Check that the data product can be written to a FITS
# file and read back again without changing the data.
self.dataproduct.save(self.testfile, overwrite=True)
with MiriRampModel(self.testfile) as readback:
assert_products_equal(
self,
self.dataproduct,
readback,
arrays=['data', 'refout', 'pixeldq', 'groupdq'],
tables=['group'])
del readback
def test_arithmetic(self):
# The ramp data model supports all the arithmetic operations
# supported by the MiriMeasuredModel. The following are exceptions
# specific to the ramp model.
# Create a data model in which the DATA and DQ arrays have different
# shapes.
testdp = MiriRampModel(data=self.ahyper,
pixeldq=self.c1,
groupdq=self.chyper,
maskwith='both')
descr = str(testdp)
self.assertIsNotNone(descr)
del descr
# Suppress warning about the DQ array being propagated only from GROUPDQ
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Check the product can be combined with itself
double = testdp * 2.0
self.assertIsNotNone(double.data)
self.assertGreater(len(double.data), 0)
expected = double.data * 2.0
self.assertTrue(np.all((double.data - expected) < 0.001))
descr = str(double)
self.assertIsNotNone(descr)
del descr
# When this is combined with another data product, the DATA
# array is masked with both the pixeldq and groupdq arrays.
warnings.simplefilter("ignore")
result = self.dataproduct + testdp
self.assertIsNotNone(result.data)
self.assertGreater(len(result.data), 0)
self.assertIsNotNone(result.dq)
self.assertGreater(len(result.dq), 0)
descr = str(result)
self.assertIsNotNone(descr)
del descr
# Boolean flags.
verb = options.verb
debug = options.debug
silent = options.silent
# Set the verbosity level according to the --verbose and --silent
# options. (Note that --debug wins over --verbose and --silent wins
# over all the other options if they are provided together.)
verbose = 1
if verb: verbose = 2
if debug: verbose = 4
if silent: verbose = 0
# Create a new exposure data object from the file.
exposure = MiriRampModel(inputfile)
if verbose > 1:
print(exposure)
exposure.statistics()
# Plot the exposure data.
if plottype == 'RAMP':
exposure.plot_ramp(row, column, description=description)
elif plottype == 'AVERAGED':
exposure.plot_ramp(row, column, averaged=True, description=description)
else:
exposure.plot(description=description)
if verbose > 0:
print("Plot completed.")
def test_creation(self):
# Test that any of the quality arrays are optional.
b1 = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
bcube = [b1, b1, b1]
bhyper = [bcube, bcube]
# 1) Data array only. Data array must exist and be non-empty.
# The quality arrays must be 2-D and 4-D.
# Unspecified arrays must be filled with default values.
newdp1 = MiriRampModel(data=self.ahyper)
self.assertIsNotNone(newdp1.data)
self.assertGreater(len(newdp1.data), 0)
# Assumes default is 0.0 - see schema
self.assertIsNotNone(newdp1.pixeldq)
self.assertTrue(newdp1.pixeldq.ndim == 2)
# Assumes default is 0 - see schema
# FIXME: The pixeldq array ends up containing null values.
#self.assertEqual(np.mean(newdp1.pixeldq), 0)
self.assertIsNotNone(newdp1.groupdq)
self.assertTrue(newdp1.groupdq.ndim == 4)
# Assumes default is 0 - see schema
self.assertEqual(np.mean(newdp1.groupdq), 0)
descr1 = str(newdp1)
del newdp1, descr1
# 2) Data and both quality arrays. All arrays must exist,
# be non-empty and be the shape specified.
newdp3 = MiriRampModel(data=self.ahyper,
pixeldq=self.c1,
groupdq=self.chyper)
self.assertIsNotNone(newdp3.data)
self.assertGreater(len(newdp3.data), 0)
# The pixeldq array must not be full of default values.
self.assertIsNotNone(newdp3.pixeldq)
self.assertTrue(newdp3.pixeldq.ndim == 2)
self.assertNotEqual(np.mean(newdp3.pixeldq), 0)
self.assertIsNotNone(newdp3.groupdq)
self.assertTrue(newdp3.groupdq.ndim == 4)
# The groupdq array must not be full of default values.
self.assertNotEqual(np.mean(newdp3.groupdq), 0)
descr3 = str(newdp3)
del newdp3, descr3
# 3) Data and pixeldq array only. All arrays must exist,
# be non-empty and be the shape specified.
newdp4 = MiriRampModel(data=self.ahyper, pixeldq=self.c1)
self.assertIsNotNone(newdp4.data)
self.assertGreater(len(newdp4.data), 0)
# The pixeldq array must not be full of default values.
self.assertIsNotNone(newdp4.pixeldq)
self.assertTrue(newdp4.pixeldq.ndim == 2)
self.assertNotEqual(np.mean(newdp4.pixeldq), 0)
self.assertIsNotNone(newdp4.groupdq)
self.assertTrue(newdp4.groupdq.ndim == 4)
descr4 = str(newdp4)
del newdp4, descr4
# 4) Data and groupdq array only. All arrays must exist,
# be non-empty and be the shape specified.
newdp5 = MiriRampModel(data=self.ahyper, groupdq=self.chyper)
self.assertIsNotNone(newdp5.data)
self.assertGreater(len(newdp5.data), 0)
self.assertIsNotNone(newdp5.pixeldq)
self.assertTrue(newdp5.pixeldq.ndim == 2)
# The groupdq array must not be full of default values.
self.assertIsNotNone(newdp5.groupdq)
self.assertTrue(newdp5.groupdq.ndim == 4)
# The groupdq array must not be full of default values.
self.assertNotEqual(np.mean(newdp5.groupdq), 0)
descr5 = str(newdp5)
del newdp5, descr5
# It should be possible to set up an empty data product with
# a specified 4-D shape. Data array should be
# initialised to the same shape.
emptydp = MiriRampModel((2, 2, 2, 2))
self.assertIsNotNone(emptydp.data)
self.assertEqual(emptydp.data.shape, (2, 2, 2, 2))
self.assertIsNotNone(emptydp.pixeldq)
#self.assertEqual(emptydp.pixeldq.shape, (2,2))
self.assertIsNotNone(emptydp.groupdq)
self.assertEqual(emptydp.groupdq.shape, (2, 2, 2, 2))
descr = str(emptydp)
self.assertIsNotNone(descr)
del emptydp, descr
# A null data product can also be created and populated
# with data later.
nulldp = MiriRampModel()
descr1 = str(nulldp)
self.assertIsNotNone(descr1)
nulldp.data = np.asarray(self.ahyper)
self.assertIsNotNone(nulldp.pixeldq)
self.assertIsNotNone(nulldp.groupdq)
descr2 = str(nulldp)
self.assertIsNotNone(descr2)
del nulldp, descr1, descr2
# Creating an object with other than 4 dimensions must fail.
a1d = [10, 20, 30, 40]
c1d = [1, 0, 0, 0]
self.assertRaises(ValueError, MiriRampModel, data=a1d, pixeldq=c1d)
a2d = [[10, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120]]
c2d = [[1, 0, 0, 0], [0, 1, 0, 1], [1, 0, 1, 0]]
self.assertRaises(ValueError, MiriRampModel, data=a2d, groupdq=c2d)
a3d = [a2d, a2d, a2d]
c3d = [c2d, c2d, c2d]
self.assertRaises(ValueError, MiriRampModel, data=a3d, pixeldq=c3d)
self.assertRaises(ValueError, MiriRampModel, data=a3d, groupdq=c3d)
# The pixeldq array must be 2-D.
self.assertRaises(ValueError,
MiriRampModel,
data=self.ahyper,
pixeldq=self.ccube)
# The groupdq array must be 4-D.
self.assertRaises(ValueError,
MiriRampModel,
data=self.ahyper,
groupdq=self.c1)
class TestMiriRampModel(unittest.TestCase):
# Most of the necessary tests are already carried out by
# the TestMiriMeasuredModel class.
def setUp(self):
# Create a ramp data product.
# NOTE: A ramp product does not contain an ERR array.
self.a1 = [[10, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120]]
self.c1 = [[1, 0, 0, 0], [0, 1, 0, 1], [1, 0, 1, 0]]
self.c2 = [[0, 1, 1, 0], [1, 0, 0, 1], [1, 0, 1, 0]]
self.acube = [self.a1, self.a1, self.a1]
self.ccube = [self.c1, self.c2, self.c1]
self.ahyper = [self.acube, self.acube]
self.chyper = [self.ccube, self.ccube]
self.refout = np.ones_like(self.chyper)
self.dataproduct = MiriRampModel(data=self.ahyper,
refout=self.refout,
pixeldq=self.c1,
dq_def=pixeldq_flags,
groupdq=self.chyper,
groupdq_def=groupdq_flags)
# Add some example metadata.
self.dataproduct.set_housekeeping_metadata('MIRI EC', 'Joe Bloggs',
'V1.0')
self.dataproduct.set_observation_metadata()
self.dataproduct.set_target_metadata(0.0, 0.0)
self.dataproduct.set_instrument_metadata(detector='MIRIFULONG',
channel='1',
ccc_pos='OPEN',
deck_temperature=11.0,
detector_temperature=6.0)
self.dataproduct.set_exposure_metadata(readpatt='FAST',
nints=1,
ngroups=1,
frame_time=1.0,
integration_time=10.0,
group_time=10.0,
reset_time=0,
frame_resets=3)
self.testfile = "MiriRampModel_test.fits"
def tearDown(self):
# Tidy up
del self.a1, self.c1, self.c2
del self.acube, self.ccube
del self.ahyper, self.chyper
del self.dataproduct
# Remove temporary file, if able to.
if os.path.isfile(self.testfile):
try:
os.remove(self.testfile)
except Exception as e:
strg = "Could not remove temporary file, " + self.testfile + \
"\n " + str(e)
warnings.warn(strg)
def test_creation(self):
# Test that any of the quality arrays are optional.
b1 = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
bcube = [b1, b1, b1]
bhyper = [bcube, bcube]
# 1) Data array only. Data array must exist and be non-empty.
# The quality arrays must be 2-D and 4-D.
# Unspecified arrays must be filled with default values.
newdp1 = MiriRampModel(data=self.ahyper)
self.assertIsNotNone(newdp1.data)
self.assertGreater(len(newdp1.data), 0)
# Assumes default is 0.0 - see schema
self.assertIsNotNone(newdp1.pixeldq)
self.assertTrue(newdp1.pixeldq.ndim == 2)
# Assumes default is 0 - see schema
# FIXME: The pixeldq array ends up containing null values.
#self.assertEqual(np.mean(newdp1.pixeldq), 0)
self.assertIsNotNone(newdp1.groupdq)
self.assertTrue(newdp1.groupdq.ndim == 4)
# Assumes default is 0 - see schema
self.assertEqual(np.mean(newdp1.groupdq), 0)
descr1 = str(newdp1)
del newdp1, descr1
# 2) Data and both quality arrays. All arrays must exist,
# be non-empty and be the shape specified.
newdp3 = MiriRampModel(data=self.ahyper,
pixeldq=self.c1,
groupdq=self.chyper)
self.assertIsNotNone(newdp3.data)
self.assertGreater(len(newdp3.data), 0)
# The pixeldq array must not be full of default values.
self.assertIsNotNone(newdp3.pixeldq)
self.assertTrue(newdp3.pixeldq.ndim == 2)
self.assertNotEqual(np.mean(newdp3.pixeldq), 0)
self.assertIsNotNone(newdp3.groupdq)
self.assertTrue(newdp3.groupdq.ndim == 4)
# The groupdq array must not be full of default values.
self.assertNotEqual(np.mean(newdp3.groupdq), 0)
descr3 = str(newdp3)
del newdp3, descr3
# 3) Data and pixeldq array only. All arrays must exist,
# be non-empty and be the shape specified.
newdp4 = MiriRampModel(data=self.ahyper, pixeldq=self.c1)
self.assertIsNotNone(newdp4.data)
self.assertGreater(len(newdp4.data), 0)
# The pixeldq array must not be full of default values.
self.assertIsNotNone(newdp4.pixeldq)
self.assertTrue(newdp4.pixeldq.ndim == 2)
self.assertNotEqual(np.mean(newdp4.pixeldq), 0)
self.assertIsNotNone(newdp4.groupdq)
self.assertTrue(newdp4.groupdq.ndim == 4)
descr4 = str(newdp4)
del newdp4, descr4
# 4) Data and groupdq array only. All arrays must exist,
# be non-empty and be the shape specified.
newdp5 = MiriRampModel(data=self.ahyper, groupdq=self.chyper)
self.assertIsNotNone(newdp5.data)
self.assertGreater(len(newdp5.data), 0)
self.assertIsNotNone(newdp5.pixeldq)
self.assertTrue(newdp5.pixeldq.ndim == 2)
# The groupdq array must not be full of default values.
self.assertIsNotNone(newdp5.groupdq)
self.assertTrue(newdp5.groupdq.ndim == 4)
# The groupdq array must not be full of default values.
self.assertNotEqual(np.mean(newdp5.groupdq), 0)
descr5 = str(newdp5)
del newdp5, descr5
# It should be possible to set up an empty data product with
# a specified 4-D shape. Data array should be
# initialised to the same shape.
emptydp = MiriRampModel((2, 2, 2, 2))
self.assertIsNotNone(emptydp.data)
self.assertEqual(emptydp.data.shape, (2, 2, 2, 2))
self.assertIsNotNone(emptydp.pixeldq)
#self.assertEqual(emptydp.pixeldq.shape, (2,2))
self.assertIsNotNone(emptydp.groupdq)
self.assertEqual(emptydp.groupdq.shape, (2, 2, 2, 2))
descr = str(emptydp)
self.assertIsNotNone(descr)
del emptydp, descr
# A null data product can also be created and populated
# with data later.
nulldp = MiriRampModel()
descr1 = str(nulldp)
self.assertIsNotNone(descr1)
nulldp.data = np.asarray(self.ahyper)
self.assertIsNotNone(nulldp.pixeldq)
self.assertIsNotNone(nulldp.groupdq)
descr2 = str(nulldp)
self.assertIsNotNone(descr2)
del nulldp, descr1, descr2
# Creating an object with other than 4 dimensions must fail.
a1d = [10, 20, 30, 40]
c1d = [1, 0, 0, 0]
self.assertRaises(ValueError, MiriRampModel, data=a1d, pixeldq=c1d)
a2d = [[10, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120]]
c2d = [[1, 0, 0, 0], [0, 1, 0, 1], [1, 0, 1, 0]]
self.assertRaises(ValueError, MiriRampModel, data=a2d, groupdq=c2d)
a3d = [a2d, a2d, a2d]
c3d = [c2d, c2d, c2d]
self.assertRaises(ValueError, MiriRampModel, data=a3d, pixeldq=c3d)
self.assertRaises(ValueError, MiriRampModel, data=a3d, groupdq=c3d)
# The pixeldq array must be 2-D.
self.assertRaises(ValueError,
MiriRampModel,
data=self.ahyper,
pixeldq=self.ccube)
# The groupdq array must be 4-D.
self.assertRaises(ValueError,
MiriRampModel,
data=self.ahyper,
groupdq=self.c1)
def test_masking(self):
# Ramp data must have a dq array which gives a view of one
# or both of the pixeldq and groupdq masks
self.assertIsNotNone(self.dataproduct.dq)
# Create a data product masked by the pixeldq array.
# The dq and pixeldq arrays must be the same
mask1 = MiriRampModel(data=self.ahyper,
pixeldq=self.c1,
groupdq=self.chyper,
maskwith='pixeldq')
self.assertIsNotNone(mask1.pixeldq)
self.assertGreater(len(mask1.pixeldq), 0)
self.assertIsNotNone(mask1.dq)
self.assertGreater(len(mask1.dq), 0)
self.assertEqual(mask1.dq.shape, mask1.pixeldq.shape)
self.assertTrue(np.all(mask1.dq == mask1.pixeldq))
del mask1
# Create a data product masked by the groupdq array.
# The dq and groupdq arrays must be the same
mask2 = MiriRampModel(data=self.ahyper,
pixeldq=self.c1,
groupdq=self.chyper,
maskwith='groupdq')
self.assertIsNotNone(mask2.groupdq)
self.assertGreater(len(mask2.groupdq), 0)
self.assertIsNotNone(mask2.dq)
self.assertGreater(len(mask2.dq), 0)
self.assertEqual(mask2.dq.shape, mask2.groupdq.shape)
self.assertTrue(np.all(mask2.dq == mask2.groupdq))
del mask2
# Create a data product masked by both pixeldq and groupdq arrays.
# The result must have the same shape as the groupdq array but be
# a combination of both masks.
mask3 = MiriRampModel(data=self.ahyper,
pixeldq=self.c1,
groupdq=self.chyper,
maskwith='both')
self.assertIsNotNone(mask3.pixeldq)
self.assertGreater(len(mask3.pixeldq), 0)
self.assertIsNotNone(mask3.groupdq)
self.assertGreater(len(mask3.groupdq), 0)
self.assertIsNotNone(mask3.dq)
self.assertGreater(len(mask3.dq), 0)
self.assertEqual(mask3.dq.shape, mask3.groupdq.shape)
expected = mask3.groupdq | mask3.pixeldq
self.assertTrue(np.all(mask3.dq == expected))
del mask3
def test_arithmetic(self):
# The ramp data model supports all the arithmetic operations
# supported by the MiriMeasuredModel. The following are exceptions
# specific to the ramp model.
# Create a data model in which the DATA and DQ arrays have different
# shapes.
testdp = MiriRampModel(data=self.ahyper,
pixeldq=self.c1,
groupdq=self.chyper,
maskwith='both')
descr = str(testdp)
self.assertIsNotNone(descr)
del descr
# Suppress warning about the DQ array being propagated only from GROUPDQ
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Check the product can be combined with itself
double = testdp * 2.0
self.assertIsNotNone(double.data)
self.assertGreater(len(double.data), 0)
expected = double.data * 2.0
self.assertTrue(np.all((double.data - expected) < 0.001))
descr = str(double)
self.assertIsNotNone(descr)
del descr
# When this is combined with another data product, the DATA
# array is masked with both the pixeldq and groupdq arrays.
warnings.simplefilter("ignore")
result = self.dataproduct + testdp
self.assertIsNotNone(result.data)
self.assertGreater(len(result.data), 0)
self.assertIsNotNone(result.dq)
self.assertGreater(len(result.dq), 0)
descr = str(result)
self.assertIsNotNone(descr)
del descr
def test_fitsio(self):
# Suppress metadata warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Check that the data product can be written to a FITS
# file and read back again without changing the data.
self.dataproduct.save(self.testfile, overwrite=True)
with MiriRampModel(self.testfile) as readback:
assert_products_equal(
self,
self.dataproduct,
readback,
arrays=['data', 'refout', 'pixeldq', 'groupdq'],
tables=['group'])
del readback
def test_description(self):
# Test that the querying and description functions work.
# For the test to pass these need to run without error
# and generate non-null strings.
descr = str(self.dataproduct)
self.assertIsNotNone(descr)
del descr
descr = repr(self.dataproduct)
self.assertIsNotNone(descr)
del descr
descr = self.dataproduct.stats()
self.assertIsNotNone(descr)
del descr
# Attempt to access the SCI, REFOUR and DQ arrays through attributes.
descr = str(self.dataproduct.data)
self.assertIsNotNone(descr)
del descr
descr = str(self.dataproduct.refout)
self.assertIsNotNone(descr)
del descr
descr = str(self.dataproduct.dq)
self.assertIsNotNone(descr)
del descr