def test_logger_list_debug(): mylog = logger.getChild('testing') logs = [] log_to_list(mylog, logs) mylog.setLevel('DEBUG') mylog.warn('Warning test') mylog.error('Error test') mylog.info('Info test') mylog.debug('Debug test') check.equal(mylog.name, 'astropop.testing') check.equal(len(logs), 4)
def test_logger_list_only_messagens(level, expected): mylog = logger.getChild('testing') logs = [] log_to_list(mylog, logs, full_record=False) mylog.setLevel(level) mylog.error('Error test') mylog.warn('Warning test') mylog.info('Info test') mylog.debug('Debug test') check.equal(mylog.name, 'astropop.testing') check.equal(len(logs), expected) for i, k in zip(['Error test', 'Warning test', 'Info test', 'Debug test'][0:expected], logs): check.equal(i, k)
def test_logger_list_only_messagens(self, level, expected): mylog = logger.getChild('testing') logs = [] log_to_list(mylog, logs, full_record=False) mylog.setLevel(level) mylog.error('Error test %i', 40) mylog.warning('Warning test %i', 30) mylog.info('Info test %i', 20) mylog.debug('Debug test %i', 10) assert_equal(mylog.name, 'astropop.testing') assert_equal(len(logs), expected) for i, k in zip(['Error test 40', 'Warning test 30', 'Info test 20', 'Debug test 10'][0:expected], logs): assert_equal(i, k)
def test_logger_list_defaults(self, level, expected): mylog = logger.getChild('testing') logs = [] log_to_list(mylog, logs) mylog.setLevel(level) mylog.error('Error test') mylog.warning('Warning test') mylog.info('Info test') mylog.debug('Debug test') assert_equal(mylog.name, 'astropop.testing') assert_equal(len(logs), expected) for i, k in zip(['Error test', 'Warning test', 'Info test', 'Debug test'][0:expected], logs): assert_equal(i, k)
def test_image_loading_fitshdu(self, disk_cache): n = 10 d = np.ones((10, 10)) l = [fits.PrimaryHDU(d) for i in range(n)] logs = [] lh = log_to_list(logger, logs, full_record=True) comb = ImCombiner(use_disk_cache=disk_cache) comb._load_images(l) # check if the logging is properly being emitted. log = [ i for i in logs if i.msg == 'The images to combine are not ' 'FrameData. Some features may be disabled.' ] assert_equal(len(log), 1) assert_equal(log[0].levelname, 'WARNING') logger.removeHandler(lh) assert_equal(len(comb._images), n) assert_is_none(comb._buffer) for i, v in enumerate(comb._images): assert_is_instance(v, FrameData) if disk_cache: assert_true(v._memmapping) comb._clear() # must start empty assert_equal(len(comb._images), 0) assert_is_none(comb._buffer)
def test_image_loading_fitsfile(self, tmpdir, disk_cache): tmp = tmpdir.strpath n = 10 d = np.ones((10, 10)) l = [os.path.join(tmp, f'fits_test{i}') for i in range(n)] for f in l: fits.PrimaryHDU(d).writeto(f) logs = [] lh = log_to_list(logger, logs, full_record=True) comb = ImCombiner(use_disk_cache=disk_cache) comb._load_images(l) # check if the logging is properly being emitted. log = [ i for i in logs if i.msg == 'The images to combine are not ' 'FrameData. Some features may be disabled.' ] assert_equal(len(log), 1) assert_equal(log[0].levelname, 'WARNING') assert_equal(len(comb._images), n) assert_is_none(comb._buffer) for i, v in enumerate(comb._images): assert_is_instance(v, FrameData) if disk_cache: assert_true(v._memmapping) comb._clear() # must start empty assert_equal(len(comb._images), 0) assert_is_none(comb._buffer)
def test_logger_list_full_record(level, expected): mylog = logger.getChild('testing') logs = [] log_to_list(mylog, logs, full_record=True) mylog.setLevel(level) mylog.error('Error test') mylog.warn('Warning test') mylog.info('Info test') mylog.debug('Debug test') check.equal(mylog.name, 'astropop.testing') check.equal(len(logs), expected) for i, k, n in zip(['Error', 'Warning', 'Info', 'Debug'][0:expected], logs, [40, 30, 20, 10][0:expected]): check.equal(f'{i} test', k.msg) check.equal(k.name, 'astropop.testing') check.equal(k.levelno, n) check.equal(k.levelname, i.upper())
def test_logger_remove_handler(): mylog = logger.getChild('testing') msg = 'Some error happend here.' logs = [] lh = log_to_list(mylog, logs) mylog.setLevel('DEBUG') mylog.error(msg) check.is_instance(lh, ListHandler) check.is_in(lh, mylog.handlers) mylog.removeHandler(lh) check.is_not_in(lh, mylog.handlers) check.equal(logs[0], msg) check.equal(lh.log_list[0], msg) check.equal(lh.log_list, logs)
def test_logging_err(self, com): logl = [] # stdout messages must not appear due to loglevel expect_log = ['this is an error'] lh = log_to_list(logger, logl) stdout = [] stderr = [] _, out, err = run_command(com, stdout=stdout, stderr=stderr, stdout_loglevel='DEBUG', stderr_loglevel='ERROR') assert_is(out, stdout) assert_is(err, stderr) assert_equal(stdout, []) assert_equal(stderr, ['this is an error']) assert_equal(logl, expect_log) logger.removeHandler(lh)
def test_logging(self, com): logl = [] logcmd = com if isinstance(com, list) else shlex.split(com) logcmd = " ".join(logcmd) expect_log = [f"Runing: {logcmd}"] expect_log += list(range(1, 11)) expect_log += [f"Done with process: {logcmd}"] lh = log_to_list(logger, logl) stdout = [] stderr = [] _, out, err = run_command(com, stdout=stdout, stderr=stderr, stdout_loglevel='WARN') assert_is(out, stdout) assert_is(err, stderr) assert_equal(stdout, [str(i) for i in range(1, 11)]) assert_equal(stderr, []) assert_equal(logl, expect_log) logger.removeHandler(lh)
def test_chunk_yielder_uncertainty(self): n = 100 d = np.random.random((100, 100)).astype(np.float64) u = np.random.random((100, 100)).astype(np.float64) l = [FrameData(d, uncertainty=u, unit='adu') for i in range(n)] # simple sum with uncertainties comb = ImCombiner(max_memory=2e6, dtype=np.float64) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='sum'): i += 1 for k, un in zip(chunk, unct): assert_in(k.shape, ((7, 100), (2, 100))) assert_almost_equal(k, d[slc]) assert_almost_equal(un, u[slc]) assert_is_instance(un, np.ma.MaskedArray) assert_equal(i, 15) # if a single uncertainty is empty, disable it logs = [] lh = log_to_list(logger, logs, False) level = logger.getEffectiveLevel() logger.setLevel('DEBUG') l[5].uncertainty = None comb = ImCombiner(max_memory=2e6, dtype=np.float64) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='sum'): i += 1 for k in chunk: assert_in(k.shape, ((7, 100), (2, 100))) assert_almost_equal(k, d[slc]) assert_equal(unct, None) assert_equal(i, 15) assert_in( 'One or more frames have empty uncertainty. ' 'Some features are disabled.', logs) logs.clear() logger.setLevel(level) logger.removeHandler(lh)
def test_data_in_other_hdu(self, tmpdir): tbl = Table(np.ones(10).reshape(5, 2)) data = 100 * np.ones(self.shape) hdul = fits.HDUList(hdus=[ fits.PrimaryHDU(), fits.TableHDU(tbl.as_array()), fits.ImageHDU(data) ]) fname = tmpdir.join('test_table.fits').strpath hdul.writeto(fname) logs = [] lh = log_to_list(logger, logs, full_record=True) f = _extract_fits(fname) assert_equal(f['data'], 100 * np.ones(self.shape)) assert_equal(f['unit'], None) # ensure log emitting logs = [i for i in logs if i.message == 'First hdu with image data: 2'] assert_equal(len(logs), 1) assert_equal(logs[0].levelname, 'INFO') logger.removeHandler(lh)
def test_chunk_yielder_f32(self): # using float32, the number of chunks are almost halved n = 100 d = np.random.random((100, 100)).astype(np.float64) l = [FrameData(d, unit='adu') for i in range(n)] # data size = 4 000 000 = 4 bytes * 100 * 100 * 100 # mask size = 1 000 000 = 1 bytes * 100 * 100 * 100 # total size = 5 000 000 comb = ImCombiner(max_memory=1e6, dtype=np.float32) comb._load_images(l) logs = [] lh = log_to_list(logger, logs, False) level = logger.getEffectiveLevel() logger.setLevel('DEBUG') # for median, tot_size=5*4.5=22.5 # xstep = 4, so n_chuks=25 i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_equal(k.shape, (4, 100)) assert_almost_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 25) assert_in('Splitting the images into 25 chunks.', logs) logs.clear() # for mean and sum, tot_size=5*3=15 # xstep = 6, so n_chunks=16+1 i = 0 for chunk, unct, slc in comb._chunk_yielder(method='mean'): i += 1 for k in chunk: assert_in(k.shape, [(6, 100), (4, 100)]) assert_almost_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 17) assert_in('Splitting the images into 17 chunks.', logs) logs.clear() i = 0 for chunk, unct, slc in comb._chunk_yielder(method='sum'): i += 1 for k in chunk: assert_in(k.shape, [(6, 100), (4, 100)]) assert_almost_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 17) assert_in('Splitting the images into 17 chunks.', logs) logs.clear() # this should not split into chunks comb = ImCombiner(max_memory=1e8, dtype=np.float32) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_equal(k.shape, (100, 100)) assert_almost_equal(k, d) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 1) assert_equal(len(logs), 0) logs.clear() # this should split in 300 chunks! # total_size = 4.5*5e6=22.5e6 = 225 chunks # x_step = 1 # y_step = 45 comb = ImCombiner(max_memory=1e5, dtype=np.float32) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_in(k.shape, ((1, 45), (1, 10))) assert_almost_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 300) assert_in('Splitting the images into 300 chunks.', logs) logs.clear() logger.setLevel(level) logger.removeHandler(lh)
def test_chunk_yielder_f64(self): n = 100 d = np.random.random((100, 100)).astype(np.float64) l = [FrameData(d, unit='adu') for i in range(n)] # data size = 8 000 000 = 8 bytes * 100 * 100 * 100 # mask size = 1 000 000 = 1 bytes * 100 * 100 * 100 # total size = 9 000 000 comb = ImCombiner(max_memory=1e6, dtype=np.float64) comb._load_images(l) logs = [] lh = log_to_list(logger, logs, False) level = logger.getEffectiveLevel() logger.setLevel('DEBUG') # for median, tot_size=9*4.5=41 # xstep = 2, so n_chuks=50 i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_equal(k.shape, (2, 100)) assert_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 50) assert_in('Splitting the images into 50 chunks.', logs) logs.clear() # for mean and sum, tot_size=9*3=27 # xstep = 3, so n_chunks=33+1 i = 0 for chunk, unct, slc in comb._chunk_yielder(method='mean'): i += 1 for k in chunk: assert_in(k.shape, [(3, 100), (1, 100)]) assert_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 34) assert_in('Splitting the images into 34 chunks.', logs) logs.clear() i = 0 for chunk, unct, slc in comb._chunk_yielder(method='sum'): i += 1 for k in chunk: assert_in(k.shape, [(3, 100), (1, 100)]) assert_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 34) assert_in('Splitting the images into 34 chunks.', logs) logs.clear() # this should not split into chunks comb = ImCombiner(max_memory=1e8) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_equal(k.shape, (100, 100)) assert_equal(k, d) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 1) assert_equal(len(logs), 0) logs.clear() # this should split in 400 chunks! comb = ImCombiner(max_memory=1e5) comb._load_images(l) i = 0 for chunk, unct, slc in comb._chunk_yielder(method='median'): i += 1 for k in chunk: assert_equal(k.shape, (1, 25)) assert_equal(k, d[slc]) assert_is_none(unct) assert_is_instance(k, np.ma.MaskedArray) assert_equal(i, 400) assert_in('Splitting the images into 400 chunks.', logs) logs.clear() logger.setLevel(level) logger.removeHandler(lh)