def test_chunk_yielder_uncertainty(self): n = 100 d = np.random.random((100, 100)).astype(np.float64) u = np.random.random((100, 100)).astype(np.float64) l = [FrameData(d, uncertainty=u, unit='adu') for i in range(n)] # simple sum with uncertainties comb = ImCombiner(max_memory=2e6, dtype=np.float64) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='sum'): i += 1 for k, un in zip(chunk, unct): assert_in(k.shape, ((7, 100), (2, 100))) assert_almost_equal(k, d[slc]) assert_almost_equal(un, u[slc]) assert_is_instance(un, np.ma.MaskedArray) assert_equal(i, 15) # if a single uncertainty is empty, disable it logs = [] lh = log_to_list(logger, logs, False) level = logger.getEffectiveLevel() logger.setLevel('DEBUG') l[5].uncertainty = None comb = ImCombiner(max_memory=2e6, dtype=np.float64) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='sum'): i += 1 for k in chunk: assert_in(k.shape, ((7, 100), (2, 100))) assert_almost_equal(k, d[slc]) assert_equal(unct, None) assert_equal(i, 15) assert_in( 'One or more frames have empty uncertainty. ' 'Some features are disabled.', logs) logs.clear() logger.setLevel(level) logger.removeHandler(lh)
def test_chunk_yielder_f32(self): # using float32, the number of chunks are almost halved n = 100 d = np.random.random((100, 100)).astype(np.float64) l = [FrameData(d, unit='adu') for i in range(n)] # data size = 4 000 000 = 4 bytes * 100 * 100 * 100 # mask size = 1 000 000 = 1 bytes * 100 * 100 * 100 # total size = 5 000 000 comb = ImCombiner(max_memory=1e6, dtype=np.float32) comb._load_images(l) logs = [] lh = log_to_list(logger, logs, False) level = logger.getEffectiveLevel() logger.setLevel('DEBUG') # for median, tot_size=5*4.5=22.5 # xstep = 4, so n_chuks=25 i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_equal(k.shape, (4, 100)) assert_almost_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 25) assert_in('Splitting the images into 25 chunks.', logs) logs.clear() # for mean and sum, tot_size=5*3=15 # xstep = 6, so n_chunks=16+1 i = 0 for chunk, unct, slc in comb._chunk_yielder(method='mean'): i += 1 for k in chunk: assert_in(k.shape, [(6, 100), (4, 100)]) assert_almost_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 17) assert_in('Splitting the images into 17 chunks.', logs) logs.clear() i = 0 for chunk, unct, slc in comb._chunk_yielder(method='sum'): i += 1 for k in chunk: assert_in(k.shape, [(6, 100), (4, 100)]) assert_almost_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 17) assert_in('Splitting the images into 17 chunks.', logs) logs.clear() # this should not split into chunks comb = ImCombiner(max_memory=1e8, dtype=np.float32) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_equal(k.shape, (100, 100)) assert_almost_equal(k, d) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 1) assert_equal(len(logs), 0) logs.clear() # this should split in 300 chunks! # total_size = 4.5*5e6=22.5e6 = 225 chunks # x_step = 1 # y_step = 45 comb = ImCombiner(max_memory=1e5, dtype=np.float32) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_in(k.shape, ((1, 45), (1, 10))) assert_almost_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 300) assert_in('Splitting the images into 300 chunks.', logs) logs.clear() logger.setLevel(level) logger.removeHandler(lh)
def test_chunk_yielder_f64(self): n = 100 d = np.random.random((100, 100)).astype(np.float64) l = [FrameData(d, unit='adu') for i in range(n)] # data size = 8 000 000 = 8 bytes * 100 * 100 * 100 # mask size = 1 000 000 = 1 bytes * 100 * 100 * 100 # total size = 9 000 000 comb = ImCombiner(max_memory=1e6, dtype=np.float64) comb._load_images(l) logs = [] lh = log_to_list(logger, logs, False) level = logger.getEffectiveLevel() logger.setLevel('DEBUG') # for median, tot_size=9*4.5=41 # xstep = 2, so n_chuks=50 i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_equal(k.shape, (2, 100)) assert_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 50) assert_in('Splitting the images into 50 chunks.', logs) logs.clear() # for mean and sum, tot_size=9*3=27 # xstep = 3, so n_chunks=33+1 i = 0 for chunk, unct, slc in comb._chunk_yielder(method='mean'): i += 1 for k in chunk: assert_in(k.shape, [(3, 100), (1, 100)]) assert_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 34) assert_in('Splitting the images into 34 chunks.', logs) logs.clear() i = 0 for chunk, unct, slc in comb._chunk_yielder(method='sum'): i += 1 for k in chunk: assert_in(k.shape, [(3, 100), (1, 100)]) assert_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 34) assert_in('Splitting the images into 34 chunks.', logs) logs.clear() # this should not split into chunks comb = ImCombiner(max_memory=1e8) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_equal(k.shape, (100, 100)) assert_equal(k, d) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 1) assert_equal(len(logs), 0) logs.clear() # this should split in 400 chunks! comb = ImCombiner(max_memory=1e5) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_equal(k.shape, (1, 25)) assert_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 400) assert_in('Splitting the images into 400 chunks.', logs) logs.clear() logger.setLevel(level) logger.removeHandler(lh)