def test_chunk_yielder_uncertainty(self):
n = 100
d = np.random.random((100, 100)).astype(np.float64)
u = np.random.random((100, 100)).astype(np.float64)
l = [FrameData(d, uncertainty=u, unit='adu') for i in range(n)]
# simple sum with uncertainties
comb = ImCombiner(max_memory=2e6, dtype=np.float64)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='sum'):
i += 1
for k, un in zip(chunk, unct):
assert_in(k.shape, ((7, 100), (2, 100)))
assert_almost_equal(k, d[slc])
assert_almost_equal(un, u[slc])
assert_is_instance(un, np.ma.MaskedArray)
assert_equal(i, 15)
# if a single uncertainty is empty, disable it
logs = []
lh = log_to_list(logger, logs, False)
level = logger.getEffectiveLevel()
logger.setLevel('DEBUG')
l[5].uncertainty = None
comb = ImCombiner(max_memory=2e6, dtype=np.float64)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='sum'):
i += 1
for k in chunk:
assert_in(k.shape, ((7, 100), (2, 100)))
assert_almost_equal(k, d[slc])
assert_equal(unct, None)
assert_equal(i, 15)
assert_in(
'One or more frames have empty uncertainty. '
'Some features are disabled.', logs)
logs.clear()
logger.setLevel(level)
logger.removeHandler(lh)
def main():
parser = OptionParser("usage: %prog key value image1.fits [image2 ...]")
parser.add_option("-v",
"--verbose",
dest="verbose",
action="count",
help="Enable 'DEBUG' output in python log")
parser.add_option("-e",
"--ext",
dest="hdu",
default=0,
metavar="HDU",
help="HDU where key will be setted.")
(options, args) = parser.parse_args()
if options.verbose is None:
logger.setLevel('WARN')
elif options.verbose == 1:
logger.setLevel('INFO')
else:
logger.setLevel('DEBUG')
key = args[0]
value = args[1]
files = args[2:]
for i in files:
logger.debug("setting key:{} with value:{} on {}".format(
key, value, i))
fits.setval(i, key, value=value, ext=options.hdu)
def main():
parser = OptionParser("usage: %prog [options] raw_dir [raw_dir2, ...]")
parser.add_option("-s",
"--stack",
action="store_true",
dest="stack_images",
default=False,
help="Stack all science images in one (sum)")
parser.add_option("-i",
"--individual",
action="store_true",
dest="save_calibed",
default=True,
help="Save individual calibed science images "
"in 'calib_images' subfolder")
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="Enable 'DEBUG' output in python log")
parser.add_option("-a",
"--astrometry",
dest="astrometry",
action="store_true",
default=False,
help="Enable astrometry solving of stacked images "
"with astrometry.net")
parser.add_option("-n",
"--science-catalog",
dest="science_catalog",
default=None,
metavar="FILE",
help="ASCII catalog to identify science stars. "
"Has to be astropy's table readable with columns "
"ID, RA, DEC")
parser.add_option("-l",
"--save-log",
dest="save_log",
default=None,
metavar="FILE",
help="Save log to FILE. If '%date' value, automatic name"
" based on date will be created.")
parser.add_option("-d",
"--dest",
dest="reduced_folder",
default='~/astropop_reduced',
metavar="FOLDER",
help="Reduced images (and created calib frames) will "
"be saved at inside FOLDER")
parser.add_option("-c",
"--calib",
dest="calib_folder",
default=None,
metavar="FOLDER",
help="Load/save calibration frames from/in FOLDER. "
"If not set, reduced_folder/calib will be used"
" instead.")
(options, args) = parser.parse_args()
if len(args) < 1:
raise ValueError('No raw folder passed!')
raw_dirs = args
if options.verbose:
logger.setLevel('DEBUG')
else:
logger.setLevel('INFO')
stack_images = options.stack_images
individual = options.save_calibed
astrometry = options.astrometry
reduced_folder = os.path.expanduser(options.reduced_folder)
reduced_folder = os.path.abspath(reduced_folder)
mkdir_p(reduced_folder)
if options.calib_folder is not None:
calib_folder = os.path.expanduser(options.calib_folder)
calib_folder = os.path.abspath(calib_folder)
else:
calib_folder = os.path.join(reduced_folder, 'calib')
sci_cat = options.science_catalog
if sci_cat is not None:
sci_cat = ASCIICatalogClass(sci_cat,
id_key='ID',
ra_key='RA',
dec_key='DEC',
format='ascii')
mkdir_p(reduced_folder)
pipe = ImpactonCalib(product_dir=reduced_folder,
calib_dir=calib_folder,
ext=0,
fits_extensions=['.fit'],
compression=True)
if stack_images:
pipe_phot = ImpactonStackedPhotometry(product_dir=reduced_folder,
image_ext=0)
else:
pipe_lc = ImpactonLightCurve(product_dir=reduced_folder, image_ext=0)
def _process():
for fold in raw_dirs:
prods = pipe.run(fold,
stack_images=stack_images,
save_calibed=individual,
astrometry=astrometry)
if stack_images:
pipe_phot.process_products(prods, sci_cat)
else:
pipe_lc.process_products(prods)
if options.save_log is not None:
name = options.save_log
if name == '%date':
d = datetime.datetime.now()
d = d.isoformat(timespec='seconds')
name = "astropop_{}.log".format(d)
name = os.path.join(reduced_folder, name)
with logger.log_to_file(name):
_process()
else:
_process()
def main():
parser = OptionParser("usage: %prog [options] config_file.json "
"[config_file2.json, ...]"
"\n\nAtention:\ncalib, raw and product folder must"
"be set in the config files!"
"\n\nUse -vvv for DEBUG.")
parser.add_option("-v", "--verbose", dest="verbose",
action="count",
help="Enable 'DEBUG' output in python log")
parser.add_option("-e", "--skip-exist", dest="check_exist",
action="store_true",
default=False,
help="Skip if the product file already exists.")
parser.add_option("-r", "--raise-error", dest="raise_error",
action="store_true",
default=False,
help="Raise errors instead of just log them.")
parser.add_option("-i", "--include", dest="include_file",
default=None, metavar="FILE",
help="Preload FILE config before run the code.")
parser.add_option("-p", "--product", dest="product",
default=None, metavar="NAME",
help="Product ID inside the configure files.")
parser.add_option("-l", "--save-log", dest="save_log",
default=None, metavar="FILE",
help="Save log to FILE. If '%date' value, automatic name"
" based on date will be created.")
(options, args) = parser.parse_args()
if len(args) < 1:
raise ValueError('No raw folder passed!')
files = args
if options.verbose is None:
logger.setLevel('WARN')
elif options.verbose == 1:
logger.setLevel('INFO')
else:
logger.setLevel('DEBUG')
pipe = MasterPipeline()
if options.include_file is not None:
pipe.load_default_file(options.include_file)
def _process():
for f in files:
if options.product is not None:
pipe.process_product(f, dataset=options.product,
check_exist=options.check_exist,
raise_error=options.raise_error)
else:
pipe.process_product(f, check_exist=options.check_exist,
raise_error=options.raise_error)
if options.save_log is not None:
name = options.save_log
if name == '%date':
d = datetime.datetime.now()
d = d.isoformat(timespec='seconds')
name = "astropop_{}.log".format(d)
with logger.log_to_file(name):
_process()
else:
_process()
def test_chunk_yielder_f32(self):
# using float32, the number of chunks are almost halved
n = 100
d = np.random.random((100, 100)).astype(np.float64)
l = [FrameData(d, unit='adu') for i in range(n)]
# data size = 4 000 000 = 4 bytes * 100 * 100 * 100
# mask size = 1 000 000 = 1 bytes * 100 * 100 * 100
# total size = 5 000 000
comb = ImCombiner(max_memory=1e6, dtype=np.float32)
comb._load_images(l)
logs = []
lh = log_to_list(logger, logs, False)
level = logger.getEffectiveLevel()
logger.setLevel('DEBUG')
# for median, tot_size=5*4.5=22.5
# xstep = 4, so n_chuks=25
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_equal(k.shape, (4, 100))
assert_almost_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 25)
assert_in('Splitting the images into 25 chunks.', logs)
logs.clear()
# for mean and sum, tot_size=5*3=15
# xstep = 6, so n_chunks=16+1
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='mean'):
i += 1
for k in chunk:
assert_in(k.shape, [(6, 100), (4, 100)])
assert_almost_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 17)
assert_in('Splitting the images into 17 chunks.', logs)
logs.clear()
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='sum'):
i += 1
for k in chunk:
assert_in(k.shape, [(6, 100), (4, 100)])
assert_almost_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 17)
assert_in('Splitting the images into 17 chunks.', logs)
logs.clear()
# this should not split into chunks
comb = ImCombiner(max_memory=1e8, dtype=np.float32)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_equal(k.shape, (100, 100))
assert_almost_equal(k, d)
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 1)
assert_equal(len(logs), 0)
logs.clear()
# this should split in 300 chunks!
# total_size = 4.5*5e6=22.5e6 = 225 chunks
# x_step = 1
# y_step = 45
comb = ImCombiner(max_memory=1e5, dtype=np.float32)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_in(k.shape, ((1, 45), (1, 10)))
assert_almost_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 300)
assert_in('Splitting the images into 300 chunks.', logs)
logs.clear()
logger.setLevel(level)
logger.removeHandler(lh)
def test_chunk_yielder_f64(self):
n = 100
d = np.random.random((100, 100)).astype(np.float64)
l = [FrameData(d, unit='adu') for i in range(n)]
# data size = 8 000 000 = 8 bytes * 100 * 100 * 100
# mask size = 1 000 000 = 1 bytes * 100 * 100 * 100
# total size = 9 000 000
comb = ImCombiner(max_memory=1e6, dtype=np.float64)
comb._load_images(l)
logs = []
lh = log_to_list(logger, logs, False)
level = logger.getEffectiveLevel()
logger.setLevel('DEBUG')
# for median, tot_size=9*4.5=41
# xstep = 2, so n_chuks=50
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_equal(k.shape, (2, 100))
assert_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 50)
assert_in('Splitting the images into 50 chunks.', logs)
logs.clear()
# for mean and sum, tot_size=9*3=27
# xstep = 3, so n_chunks=33+1
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='mean'):
i += 1
for k in chunk:
assert_in(k.shape, [(3, 100), (1, 100)])
assert_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 34)
assert_in('Splitting the images into 34 chunks.', logs)
logs.clear()
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='sum'):
i += 1
for k in chunk:
assert_in(k.shape, [(3, 100), (1, 100)])
assert_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 34)
assert_in('Splitting the images into 34 chunks.', logs)
logs.clear()
# this should not split into chunks
comb = ImCombiner(max_memory=1e8)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_equal(k.shape, (100, 100))
assert_equal(k, d)
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 1)
assert_equal(len(logs), 0)
logs.clear()
# this should split in 400 chunks!
comb = ImCombiner(max_memory=1e5)
comb._load_images(l)
i = 0
for chunk, unct, slc in comb._chunk_yielder(method='median'):
i += 1
for k in chunk:
assert_equal(k.shape, (1, 25))
assert_equal(k, d[slc])
assert_is_none(unct)
assert_is_instance(k, np.ma.MaskedArray)
assert_equal(i, 400)
assert_in('Splitting the images into 400 chunks.', logs)
logs.clear()
logger.setLevel(level)
logger.removeHandler(lh)
def main():
parser = OptionParser("usage: %prog [options] raw_dir [raw_dir2, ...]")
parser.add_option("-v", "--verbose", dest="verbose",
action="store_true",
default=False,
help="Enable 'DEBUG' output in python log")
parser.add_option("-a", "--astrometry", dest="astrometry",
action="store_true",
default=False,
help="Enable astrometry solving of stacked images "
"with astrometry.net")
parser.add_option("-n", "--science-catalog", dest="science_catalog",
default=None, metavar="FILE",
help="ASCII catalog to identify science stars. "
"Has to be astropy's table readable with columns "
"ID, RA, DEC")
parser.add_option("-l", "--save-log", dest="save_log",
default=None, metavar="FILE",
help="Save log to FILE. If '%date' value, automatic name"
" based on date will be created.")
parser.add_option("-d", "--dest", dest="reduced_folder",
default='~/astropop_reduced', metavar="FOLDER",
help="Reduced images (and created calib frames) will "
"be saved at inside FOLDER")
(options, args) = parser.parse_args()
if len(args) < 1:
raise ValueError('No raw folder passed!')
raw_dirs = args
if options.verbose:
logger.setLevel('DEBUG')
else:
logger.setLevel('INFO')
reduced_folder = os.path.expanduser(options.reduced_folder)
reduced_folder = os.path.abspath(reduced_folder)
mkdir_p(reduced_folder)
sci_cat = options.science_catalog
if sci_cat is not None:
sci_cat = ASCIICatalogClass(sci_cat, id_key='ID', ra_key='RA',
dec_key='DEC', format='ascii')
mkdir_p(reduced_folder)
pipe_phot = BSMStackedPhotometry(product_dir=reduced_folder, image_ext=0)
def _process():
for fold in raw_dirs:
pipe_phot.run(fold, sci_cat, options.astrometry)
if options.save_log is not None:
name = options.save_log
if name == '%date':
d = datetime.datetime.now()
d = d.isoformat(timespec='seconds')
name = "astropop_{}.log".format(d)
name = os.path.join(reduced_folder, name)
with logger.log_to_file(name):
_process()
else:
_process()