def test_simple_flat(inplace):
expect = np.ones((20, 20))*3
expect[0:5, 0:5] = 3/0.5
# Checking flat division:
frame1 = FrameData(np.ones((20, 20))*3, unit=u.adu)
master_flat_dimless = FrameData(np.ones((20, 20)), unit=None)
master_flat_dimless.data[0:5, 0:5] = 0.5
res1 = flat_correct(frame1, master_flat_dimless, inplace=inplace)
check.is_true(isinstance(res1, FrameData))
npt.assert_array_equal(res1.data, expect)
check.equal(res1.header['hierarch astropop flat_corrected'], True)
# # Checking flat-corrected frame unit:
# check.equal(res1.unit, u.Unit('adu'))
# Check inplace statement:
if inplace:
check.is_true(res1.data is frame1.data)
else:
check.is_false(res1.data is frame1.data)
def gen_frame(v):
# Gen frames with {'v', 'u'} dict
shape = (10, 10)
if v['u'] is None:
frame = FrameData(np.ones(shape, dtype='f8'), unit='adu')
else:
frame = FrameData(np.ones(shape, dtype='f8'), unit='adu',
uncertainty=v['u'])
frame.data[:] = v['v']
return frame
def test_register_framedata(self, inplace, cval, fill):
im1 = [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 1, 2, 2, 2, 1, 1, 1, 1, 1], [1, 2, 4, 6, 4, 2, 1, 1, 1, 1],
[1, 2, 6, 8, 6, 2, 1, 1, 1, 1], [1, 2, 4, 6, 4, 2, 1, 1, 1, 1],
[1, 1, 2, 2, 2, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
im2 = [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 2, 2, 2, 1, 1, 1],
[1, 1, 1, 2, 4, 6, 4, 2, 1, 1], [1, 1, 1, 2, 6, 8, 6, 2, 1, 1],
[1, 1, 1, 2, 4, 6, 4, 2, 1, 1], [1, 1, 1, 1, 2, 2, 2, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
expect = np.array(im1, dtype='f8')
expect[0, :] = fill
expect[:, -2:] = fill
mask = np.zeros_like(im2, dtype=bool)
mask[0, :] = 1
mask[:, -2:] = 1
expect_unct = np.ones_like(im2, dtype='f8')
expect_unct[0, :] = np.nan
expect_unct[:, -2:] = np.nan
frame1 = FrameData(im1, dtype='f8')
frame1.meta['moving'] = False
frame1.uncertainty = np.ones_like(im1)
frame2 = FrameData(im2, dtype='f8')
frame2.meta['moving'] = True
frame2.uncertainty = np.ones_like(im2)
ar = CrossCorrelationRegister()
frame_reg = ar.register_framedata(frame1,
frame2,
cval=cval,
inplace=inplace)
assert_equal(frame_reg.data, expect)
assert_equal(frame_reg.mask, mask)
assert_equal(frame_reg.uncertainty, expect_unct)
assert_equal(frame_reg.meta['astropop registration'],
'cross-correlation')
assert_equal(frame_reg.meta['astropop registration_shift'], [2, -1])
assert_equal(frame_reg.meta['astropop registration_rot'], 0)
assert_equal(frame_reg.meta['moving'], True)
if inplace:
assert_is(frame_reg, frame2)
else:
assert_is_not(frame_reg, frame2)
def gen_frame(value, uncertainty=None, unit='adu'):
"""Gen frames with {'v', 'u'} dict"""
shape = SHAPE
frame = FrameData(np.ones(shape, dtype='f8'),
unit=unit,
uncertainty=uncertainty)
frame.data[:] = value
return frame
def test_invalid_method(self):
n = 10
d = np.ones((10, 10))
l = [FrameData(d, unit='adu') for i in range(n)]
comb = ImCombiner()
with pytest.raises(ValueError,
match='hulk-smash is not a valid '
'combining method.'):
comb.combine(l, method='hulk-smash')
def test_simple_bias(self, inplace):
expected = np.ones((20, 20)) * 2
expected[0:5, 0:5] = 2.5
frame4bias = FrameData(np.ones((20, 20)) * 3, unit='adu')
master_bias = FrameData(np.ones((20, 20)), unit='adu')
master_bias.data[0:5, 0:5] = 0.5
res4 = subtract_bias(frame4bias, master_bias, inplace=inplace)
assert_is_instance(res4, FrameData)
assert_equal(res4.data, expected)
assert_equal(res4.header['hierarch astropop bias_corrected'], True)
if inplace:
assert_is(res4.data, frame4bias.data)
else:
assert_is_not(res4.data, frame4bias.data)
def test_simple_flat(self, inplace):
expect = np.ones((20, 20)) * 3
expect[0:5, 0:5] = 3 / 0.5
# Checking flat division:
frame1 = FrameData(np.ones((20, 20)) * 3, unit=u.adu)
master_flat_dimless = FrameData(np.ones((20, 20)), unit=None)
master_flat_dimless.data[0:5, 0:5] = 0.5
res1 = flat_correct(frame1, master_flat_dimless, inplace=inplace)
assert_is_instance(res1, FrameData)
assert_equal(res1.data, expect)
assert_equal(res1.header['hierarch astropop flat_corrected'], True)
if inplace:
assert_is(res1.data, frame1.data)
else:
assert_is_not(res1.data, frame1.data)
def create_framedata_array(self, values, size=(100, 100), unct=None):
# values is an array containing the scale factor for each image
with NumpyRNGContext(123):
data = np.random.normal(loc=100, scale=20, size=size)
arr = [None] * len(values)
for i, k in enumerate(values):
u = unct * data * k if unct is not None else unct
arr[i] = FrameData(data * k, uncertainty=u, unit='adu')
return arr, data
def test_register_frame_equal(self, inplace):
im = gen_image((50, 50), [25], [25], [10000], 10, 0, sigma=3)
im = FrameData(im)
ar = CrossCorrelationRegister()
im_reg = ar.register_framedata(im, im, inplace=inplace)
if inplace:
assert_is(im_reg, im)
else:
assert_is_not(im_reg, im)
assert_equal(im_reg.data, im.data)
assert_equal(im_reg.mask, np.zeros_like(im))
assert_equal(im_reg.meta['astropop registration_shift'], [0, 0])
def create_images(self):
images = []
for i in range(30):
meta = {
'first_equal': 1,
'second_equal': 2,
'first_differ': i,
'second_differ': i // 2,
'third_differ': i % 3
}
images.append(FrameData(np.ones((10, 10)), unit='adu', meta=meta))
return images
def test_simple_bias(inplace):
expected = np.ones((20, 20))*2
expected[0:5, 0:5] = 2.5
frame4bias = FrameData(np.ones((20, 20))*3, unit='adu')
master_bias = FrameData(np.ones((20,20)),unit='adu')
master_bias.data[0:5, 0:5] = 0.5
res4 = subtract_bias(frame4bias, master_bias, inplace=inplace)
check.is_true(isinstance(res4, FrameData))
npt.assert_array_equal(res4.data, expected)
check.equal(res4.header['hierarch astropop bias_corrected'], True)
# # Checking bias-subtracted frame unit:
# check.equal(res1.unit, u.Unit('adu'))
# Check inplace statement:
if inplace:
check.is_true(res4.data is frame4bias.data)
else:
check.is_false(res4.data is frame4bias.data)
def gen_frame_list(self, size):
sky = 800
rdnoise = 10
n = 100
x, y, f = gen_position_flux(np.array(size) + 80, n, 1e4, 4e6)
x -= 40
y -= 40
frame_list = []
for shift in self._shifts:
x1, y1, flux1 = gen_positions_transformed(x, y, f, *shift, size)
im1 = gen_image(size, x1, y1, flux1, sky, rdnoise, sigma=2)
frame = FrameData(im1, meta={'test expect_shift': list(shift)})
frame_list.append(frame)
return frame_list
def test_combine_mask_mean(self):
comb = ImCombiner()
images = [None] * 10
shape = (10, 10)
for i in range(10):
mask = np.zeros(shape)
# all (2, 5) elements are masked, so the result must be
mask[2, 5] = 1
images[i] = FrameData(np.ones(shape), unit='adu', mask=mask)
# these points in result must no be masked
images[5].mask[7, 7] = 1
images[2].mask[7, 7] = 1
images[8].mask[8, 8] = 1
expect = np.zeros(shape)
expect[2, 5] = 1
res = comb.combine(images, 'mean')
assert_equal(res.mask, expect)
def test_chunk_yielder_uncertainty(self):
n = 100
d = np.random.random((100, 100)).astype(np.float64)
u = np.random.random((100, 100)).astype(np.float64)
l = [FrameData(d, uncertainty=u, unit='adu') for i in range(n)]
# simple sum with uncertainties
comb = ImCombiner(max_memory=2e6, dtype=np.float64)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='sum'):
i += 1
for k, un in zip(chunk, unct):
assert_in(k.shape, ((7, 100), (2, 100)))
assert_almost_equal(k, d[slc])
assert_almost_equal(un, u[slc])
assert_is_instance(un, np.ma.MaskedArray)
assert_equal(i, 15)
# if a single uncertainty is empty, disable it
logs = []
lh = log_to_list(logger, logs, False)
level = logger.getEffectiveLevel()
logger.setLevel('DEBUG')
l[5].uncertainty = None
comb = ImCombiner(max_memory=2e6, dtype=np.float64)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='sum'):
i += 1
for k in chunk:
assert_in(k.shape, ((7, 100), (2, 100)))
assert_almost_equal(k, d[slc])
assert_equal(unct, None)
assert_equal(i, 15)
assert_in(
'One or more frames have empty uncertainty. '
'Some features are disabled.', logs)
logs.clear()
logger.setLevel(level)
logger.removeHandler(lh)
def test_image_loading_framedata(self, tmpdir, disk_cache):
tmp = tmpdir.strpath
n = 10
d = np.ones((10, 10))
l = [
FrameData(d,
unit='adu',
uncertainty=d,
cache_folder=tmp,
cache_filename=f'test{i}') for i in range(n)
]
comb = ImCombiner(use_disk_cache=disk_cache)
# must start empty
assert_equal(len(comb._images), 0)
assert_is_none(comb._buffer)
comb._load_images(l)
assert_equal(len(comb._images), n)
assert_is_none(comb._buffer)
for i, v in enumerate(comb._images):
fil = os.path.join(tmp, f'test{i}')
assert_is_instance(v, FrameData)
if disk_cache:
assert_true(v._memmapping)
# assert_path_exists(fil+'_copy_copy.data')
# assert_path_exists(fil+'_copy_copy.unct')
# assert_path_exists(fil+'_copy_copy.mask')
comb._clear()
# must start empty
assert_equal(len(comb._images), 0)
assert_is_none(comb._buffer)
# ensure tmp files cleaned
if disk_cache:
for i in range(n):
fil = os.path.join(tmp, f'test{i}')
assert_path_not_exists(fil + '_copy_copy.data')
assert_path_not_exists(fil + '_copy_copy.unct')
assert_path_not_exists(fil + '_copy_copy.mask')
def test_check_consistency(self):
n = 10
d = np.ones((10, 10))
l = [FrameData(d, unit='adu') for i in range(n)]
comb = ImCombiner()
# empty should raise
with pytest.raises(ValueError, match='Combiner have no images.'):
comb._check_consistency()
comb._load_images(l)
# nothing should raise
comb._check_consistency()
# incompatible unit should raise
comb._images[3].unit = 'm'
with pytest.raises(ValueError, match='.* unit incompatible .*'):
comb._check_consistency()
comb._images[3].unit = 'adu'
# incompatible shape should raise
comb._images[4].data = np.ones((2, 2))
with pytest.raises(ValueError, match='.* shape incompatible .*'):
comb._check_consistency()
def test_error_incompatible_shapes(self):
frame_list = [FrameData(np.zeros((i + 1, i + 1))) for i in range(10)]
with pytest.raises(ValueError, match='incompatible shapes'):
register_framedata_list(frame_list)
def test_invalid_shapes():
frame1 = FrameData(np.zeros((10, 10)), unit='')
frame2 = FrameData(np.zeros((5, 5)), unit='')
with pytest.raises(ValueError):
imarith(frame1, frame2, '+')
def test_invalid_op():
frame1 = FrameData(np.zeros((10, 10)), unit='')
frame2 = FrameData(np.zeros((10, 10)), unit='')
with pytest.raises(ValueError) as exc:
imarith(frame1, frame2, 'not an op')
check.is_in('not supported', str(exc.value))
def test_chunk_yielder_f32(self):
# using float32, the number of chunks are almost halved
n = 100
d = np.random.random((100, 100)).astype(np.float64)
l = [FrameData(d, unit='adu') for i in range(n)]
# data size = 4 000 000 = 4 bytes * 100 * 100 * 100
# mask size = 1 000 000 = 1 bytes * 100 * 100 * 100
# total size = 5 000 000
comb = ImCombiner(max_memory=1e6, dtype=np.float32)
comb._load_images(l)
logs = []
lh = log_to_list(logger, logs, False)
level = logger.getEffectiveLevel()
logger.setLevel('DEBUG')
# for median, tot_size=5*4.5=22.5
# xstep = 4, so n_chuks=25
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_equal(k.shape, (4, 100))
assert_almost_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 25)
assert_in('Splitting the images into 25 chunks.', logs)
logs.clear()
# for mean and sum, tot_size=5*3=15
# xstep = 6, so n_chunks=16+1
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='mean'):
i += 1
for k in chunk:
assert_in(k.shape, [(6, 100), (4, 100)])
assert_almost_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 17)
assert_in('Splitting the images into 17 chunks.', logs)
logs.clear()
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='sum'):
i += 1
for k in chunk:
assert_in(k.shape, [(6, 100), (4, 100)])
assert_almost_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 17)
assert_in('Splitting the images into 17 chunks.', logs)
logs.clear()
# this should not split into chunks
comb = ImCombiner(max_memory=1e8, dtype=np.float32)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_equal(k.shape, (100, 100))
assert_almost_equal(k, d)
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 1)
assert_equal(len(logs), 0)
logs.clear()
# this should split in 300 chunks!
# total_size = 4.5*5e6=22.5e6 = 225 chunks
# x_step = 1
# y_step = 45
comb = ImCombiner(max_memory=1e5, dtype=np.float32)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_in(k.shape, ((1, 45), (1, 10)))
assert_almost_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 300)
assert_in('Splitting the images into 300 chunks.', logs)
logs.clear()
logger.setLevel(level)
logger.removeHandler(lh)
def test_error_unkown_algorithm(self):
with pytest.raises(ValueError, match='Algorithm noexisting unknown.'):
register_framedata_list([FrameData(None) for i in range(10)],
algorithm='noexisting')
def test_simple_dark(inplace):
frame4dark = FrameData(np.ones((20, 20))*3, unit='adu')
master_dark0 = FrameData(np.zeros((20,20)),unit='adu')
master_dark0.data[0:5, 0:4] = 0.5
master_dark1 = FrameData(np.ones((20,20)),unit=None)
master_dark1.data[0:5, 0:4] = 1.5
master_dark5 = FrameData(np.ones((20,20))*5,unit='adu')
master_dark5.data[0:5, 0:4] = 5.5
exposure1 = 1.0
exposure2 = 1.2
res5_zeros = subtract_dark(frame4dark, master_dark0, exposure1, exposure1,
inplace = inplace)
res5_ones = subtract_dark(frame4dark, master_dark1, exposure1, exposure1,
inplace = inplace)
res5_fives = subtract_dark(frame4dark, master_dark5, exposure1, exposure1,
inplace = inplace)
res5_scaled = subtract_dark(frame4dark, master_dark1, exposure1, exposure2,
inplace = inplace)
exptd4zeros = np.ones((20, 20))*(3-0)
exptd4zeros[0:5, 0:4] = 3.0 - 0.5
exptd4ones = np.ones((20, 20))*(3-1)
exptd4ones[0:5, 0:4] = 3.0 - 1.5
exptd4fives = np.ones((20, 20))*(3-5)
exptd4fives[0:5, 0:4] = 3.0 - 5.5
exptd4scaled = np.ones((20, 20))*(3-1*exposure2/exposure1)
exptd4scaled[0:5, 0:4] = 3.0 - 1.5*exposure2/exposure1
## Checking with the Ones matrix: ------------------------------------------
check.is_true(isinstance(res5_ones, FrameData))
npt.assert_array_equal(res5_ones.data, exptd4ones)
check.equal(res5_ones.header['hierarch astropop dark_corrected'], True)
check.equal(res5_ones.header['hierarch astropop dark_corrected_scale'],
1.0)
# # Checking dark-subtracted frame unit:
# check.equal(res5_ones.unit, u.Unit('adu'))
# Check inplace statement:
if inplace:
check.is_true(res5_ones.data is frame4dark.data)
else:
check.is_false(res5_ones.data is frame4dark.data)
## Checking with the Zeros matrix: -----------------------------------------
check.is_true(isinstance(res5_zeros, FrameData))
npt.assert_array_equal(res5_zeros.data, exptd4zeros)
check.equal(res5_zeros.header['hierarch astropop dark_corrected'], True)
check.equal(res5_zeros.header['hierarch astropop dark_corrected_scale'],
1.0)
# # Checking dark-subtracted frame unit:
# check.equal(res5_zeros.unit, u.Unit('adu'))
## Checking with the Fives matrix (master dark masking the signal): -------
check.is_true(isinstance(res5_fives, FrameData))
npt.assert_array_equal(res5_fives.data, exptd4fives)
check.equal(res5_fives.header['hierarch astropop dark_corrected'], True)
check.equal(res5_fives.header['hierarch astropop dark_corrected_scale'],
1.0)
# # Checking dark-subtracted frame unit:
# check.equal(res5_fives.unit, u.Unit('adu'))
# ## Checking with Scaled Exposure: -----------------------------------------
# check.is_true(isinstance(res5_scaled, FrameData))
# npt.assert_array_equal(res5_scaled.data, exptd4scaled)
# check.equal(res5_scaled.header['hierarch astropop dark_corrected'], True)
# check.equal(res5_scaled.header['hierarch astropop dark_corrected_scale'],
# exposure2/exposure1)
def test_chunk_yielder_f64(self):
n = 100
d = np.random.random((100, 100)).astype(np.float64)
l = [FrameData(d, unit='adu') for i in range(n)]
# data size = 8 000 000 = 8 bytes * 100 * 100 * 100
# mask size = 1 000 000 = 1 bytes * 100 * 100 * 100
# total size = 9 000 000
comb = ImCombiner(max_memory=1e6, dtype=np.float64)
comb._load_images(l)
logs = []
lh = log_to_list(logger, logs, False)
level = logger.getEffectiveLevel()
logger.setLevel('DEBUG')
# for median, tot_size=9*4.5=41
# xstep = 2, so n_chuks=50
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_equal(k.shape, (2, 100))
assert_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 50)
assert_in('Splitting the images into 50 chunks.', logs)
logs.clear()
# for mean and sum, tot_size=9*3=27
# xstep = 3, so n_chunks=33+1
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='mean'):
i += 1
for k in chunk:
assert_in(k.shape, [(3, 100), (1, 100)])
assert_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 34)
assert_in('Splitting the images into 34 chunks.', logs)
logs.clear()
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='sum'):
i += 1
for k in chunk:
assert_in(k.shape, [(3, 100), (1, 100)])
assert_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 34)
assert_in('Splitting the images into 34 chunks.', logs)
logs.clear()
# this should not split into chunks
comb = ImCombiner(max_memory=1e8)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_equal(k.shape, (100, 100))
assert_equal(k, d)
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 1)
assert_equal(len(logs), 0)
logs.clear()
# this should split in 400 chunks!
comb = ImCombiner(max_memory=1e5)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_equal(k.shape, (1, 25))
assert_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 400)
assert_in('Splitting the images into 400 chunks.', logs)
logs.clear()
logger.setLevel(level)
logger.removeHandler(lh)