def cli(ctx, optcr, drzonly, usehlet, ctype, itype, ofile, ptask):
"""
Drizzles final mosaic for each filter
"""
dsn = ctx.dataset_name
ctx.log('Running task %s for dataset %s', task, dsn)
procdir = os.path.join(ctx.rundir, dsn)
os.chdir(procdir)
cfgf = '%s_cfg.json' % dsn
cfg = hutils.rConfig(cfgf)
tcfg = cfg['tasks'][task] = {}
tcfg['ptask'] = ptask
tcfg['itype'] = itype
tcfg['ofile'] = ofile
tcfg['stime'] = ctx.dt()
tcfg['completed'] = False
for instdet, data in cfg['images'].items():
for fltr, images in data.items():
outfile = '%s_%s_%s' % (dsn, instdet, fltr.lower())
infiles = [str('%s%s' % (i, itype)) for i in images]
ctx.vlog('Drizzling mosaic %s', outfile)
try:
if cfg['refimg']:
refimg = str(cfg['refimg'])
drizzle_mosaic.drzMosaic(infiles,
outfile,
refimg=refimg,
optcr=optcr,
drzonly=drzonly,
usehlet=usehlet,
ctype=ctype)
else:
drizzle_mosaic.drzMosaic(infiles,
outfile,
optcr=optcr,
drzonly=drzonly,
usehlet=usehlet,
ctype=ctype)
except Exception as e:
hutils.wConfig(cfg, cfgf)
print(e)
raise
tcfg['etime'] = ctx.dt()
tcfg['completed'] = True
ctx.vlog('Writing configuration file %s for %s task', cfgf, task)
hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, otype, ptask):
"""
Drizzles and individual images to be used for alignment
"""
dsn = ctx.dataset_name
ctx.log('Running task %s for dataset %s', task, dsn)
procdir = os.path.join(ctx.rundir, dsn)
os.chdir(procdir)
cfgf = '%s_cfg.json' % dsn
cfg = hutils.rConfig(cfgf)
tcfg = cfg['tasks'][task] = {}
tcfg['ptask'] = ptask
tcfg['itype'] = itype
tcfg['otype'] = otype
tcfg['stime'] = ctx.dt()
tcfg['completed'] = False
if ctx.refimg:
refimg = str(ctx.refimg)
refwcs = HSTWCS(refimg)
cfg['refimg'] = refimg
pscale = refwcs.pscale
orientat = refwcs.orientat
else:
# use native pixel scale
pscale = None
orientat = 0
images = hutils.imgList(cfg['images'])
infiles = [str('%s%s' % (i, itype)) for i in images]
n = len(infiles)
with click.progressbar(infiles,
label='Generating single drizzled images') as pbar:
for i, f in enumerate(pbar):
ctx.vlog('\n\nDrizzling image %s - %s of %s', f, i + 1, n)
try:
drizzle_image.drzImage(f, pscale, orientat)
except Exception as e:
hutils.wConfig(cfg, cfgf)
print(e)
raise
tcfg['etime'] = ctx.dt()
tcfg['completed'] = True
ctx.vlog('Writing configuration file %s for %s task', cfgf, task)
hutils.wConfig(cfg, cfgf)
def cli(ctx, restore, hlet, itype, otype, ptask):
"""
Applies offsets to images computed by superalign
"""
dsn = ctx.dataset_name
ctx.log('Running task %s for dataset %s', task, dsn)
procdir = os.path.join(ctx.rundir, dsn)
os.chdir(procdir)
cfgf = '%s_cfg.json' % dsn
cfg = hutils.rConfig(cfgf)
tcfg = cfg['tasks'][task] = {}
tcfg['ptask'] = ptask
tcfg['itype'] = itype
tcfg['otype'] = otype
tcfg['stime'] = ctx.dt()
tcfg['completed'] = False
originf = cfg['infiles']
images = hutils.imgList(cfg['images'])
infiles = [str('%s%s' % (i, itype)) for i in images]
n = len(infiles)
with click.progressbar(infiles,
label='Generating catalogs for images') as pbar:
for i, inf in enumerate(pbar):
outf = inf.replace('_drz_sci.fits', '_flt.fits')
ctx.vlog('\nApplying offsets image %s - %s of %s', inf, i + 1, n)
try:
if restore:
apply_shift.restoreWCSdrz(inf, 0)
for ext in hutils.sciexts[hutils.getInstDet(outf)]:
origflt = [s for s in originf if outf in s][0]
restoreWCSflt(outf, origflt, ext)
ctx.vlog('Applying offsets')
apply_shift.applyOffset(inf, outf, hlet=hlet)
except Exception as e:
hutils.wConfig(cfg, cfgf)
print(e)
raise
tcfg['etime'] = ctx.dt()
tcfg['completed'] = True
ctx.vlog('Writing configuration file %s for %s task', cfgf, task)
hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, otype, ptask):
"""
Create IMV weight maps for WFC3IR images
"""
dsn = ctx.dataset_name
ctx.log('Running task %s for dataset %s', task, dsn)
procdir = os.path.join(ctx.rundir, dsn)
os.chdir(procdir)
cfgf = '%s_cfg.json' % dsn
cfg = hutils.rConfig(cfgf)
tcfg = cfg['tasks'][task] = {}
tcfg['ptask'] = ptask
tcfg['itype'] = itype
tcfg['otype'] = otype
tcfg['stime'] = ctx.dt()
tcfg['completed'] = False
iref = ctx.iref
images = hutils.imgList(cfg['images'])
infiles = [str('%s%s' % (i, itype)) for i in images]
n = len(infiles)
with click.progressbar(infiles,
label='Generating sky subtracted image') as pbar:
for i, f in enumerate(pbar):
ctx.vlog('\n\nCreating weight map for image %s - %s of %s', f,
i + 1, n)
instdet = hutils.getInstDet(f)
if instdet == 'wfc3ir':
try:
make_ivm.makeweight(f, iref)
except Exception as e:
hutils.wConfig(cfg, cfgf)
print(e)
raise
tcfg['etime'] = ctx.dt()
tcfg['completed'] = True
ctx.vlog('Writing configuration file %s for %s task', cfgf, task)
hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, otype, ptask):
"""
Correct for ACS amplifier discontinuities
"""
dsn = ctx.dataset_name
ctx.log('Running task %s for dataset %s', task, dsn)
procdir = os.path.join(ctx.rundir, dsn)
os.chdir(procdir)
cfgf = '%s_cfg.json' % dsn
cfg = hutils.rConfig(cfgf)
tcfg = cfg['tasks'][task] = {}
tcfg['ptask'] = ptask
tcfg['itype'] = itype
tcfg['otype'] = otype
tcfg['stime'] = ctx.dt()
tcfg['completed'] = False
images = hutils.imgList(cfg['images'])
infiles = [str('%s%s' % (i, itype)) for i in images]
n = len(infiles)
with click.progressbar(
infiles, label='Generating amplifier corrected image') as pbar:
for i, f in enumerate(pbar):
ctx.vlog('\n\nCorrecting image %s - %s of %s', f, i + 1, n)
instdet = hutils.getInstDet(f)
if instdet == 'acswfc':
try:
amp_correct.ampcorr(f)
except Exception as e:
hutils.wConfig(cfg, cfgf)
print(e)
raise
tcfg['etime'] = ctx.dt()
tcfg['completed'] = True
ctx.vlog('Writing configuration file %s for %s task', cfgf, task)
hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, otype, ptask):
"""
Applies mask to images
"""
dsn = ctx.dataset_name
pmaskdir = ctx.pmaskdir
ctx.log('Running task %s for dataset %s', task, dsn)
procdir = os.path.join(ctx.rundir, dsn)
os.chdir(procdir)
cfgf = '%s_cfg.json' % dsn
cfg = hutils.rConfig(cfgf)
tcfg = cfg['tasks'][task] = {}
tcfg['ptask'] = ptask
tcfg['itype'] = itype
tcfg['otype'] = otype
tcfg['stime'] = ctx.dt()
tcfg['completed'] = False
images = hutils.imgList(cfg['images'])
infiles = [str('%s%s' % (i, itype)) for i in images]
n = len(infiles)
with click.progressbar(infiles,
label='Generating masks for images') as pbar:
for i, f in enumerate(pbar):
ctx.vlog('\n\nChecking masks for image %s - %s of %s', f, i + 1, n)
masks = [
fp for fp in glob.glob('%s*.reg' % f[:9])
if 'footprint' not in fp
]
if masks:
try:
for m in masks:
if 'SCI' in m:
# ACSWFC and WFC3UV
chip = int(m.split('.reg')[0][-1])
if chip == 1:
dq_ext = 3
if chip == 2:
dq_ext = 6
else:
# WFC3IR
dq_ext = 3
ctx.vlog('Applying mask to %s - DQ extention %s', f,
dq_ext)
apply_mask.applymask(f, m, dq_ext)
except Exception as e:
hutils.wConfig(cfg, cfgf)
print(e)
raise
else:
ctx.vlog('No masks found for %s', f)
if hutils.getInstDet(f) == 'wfc3ir':
if pmaskdir:
ctx.vlog('Checking for persistance mask for image %s', f)
pmaskfile = os.path.join(
pmaskdir, f.replace('flt.fits', 'pmask.fits'))
if os.path.exists(pmaskfile):
ctx.vlog('Found persistance mask image %s',
os.path.basename(pmaskfile))
apply_mask.applypersist(f, pmaskfile)
tcfg['etime'] = ctx.dt()
tcfg['completed'] = True
ctx.vlog('Writing configuration file %s for %s task', cfgf, task)
hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, ofile, ptask):
"""
Runs superalign on catalogs for alignment
"""
dsn = ctx.dataset_name
ctx.log('Running task %s for dataset %s', task, dsn)
procdir = os.path.join(ctx.rundir, dsn)
os.chdir(procdir)
cfgf = '%s_cfg.json' % dsn
cfg = hutils.rConfig(cfgf)
tcfg = cfg['tasks'][task] = {}
tcfg['ptask'] = ptask
tcfg['itype'] = itype
tcfg['ofile'] = cfg['sfile'] = ofile
tcfg['stime'] = ctx.dt()
tcfg['completed'] = False
images = hutils.imgList(cfg['images'])
infiles = [str('%s%s' % (i, itype)) for i in images]
refimg = cfg['refimg']
refcat = cfg['refcat']
refcat_sa = cfg['refcat_sa']
refwcs = HSTWCS(fits.open(refimg))
mkcat = make_catalog.MakeCat(refimg)
sa_hlf = find_executable('superalign_hlfred')
if not sa_hlf:
ctx.elog(
'Unable to find "superalign_hlfred" executable. Make sure it is in your PATH.'
)
sys.exit(1)
ctx.vlog('Grouping images by visit for alignment')
visits = set([i[:6] for i in infiles])
vdata = {}
for v in visits:
vdata[v] = []
for e in infiles:
if e[:6] == v:
vdata[v].append(e)
# Run superalign on all visits
ctx.vlog('Generating the superalign input')
with open('superalign_failed_visits.txt', 'w') as sfv:
for visit, inf in vdata.items():
super_align.makeSAin(visit, inf, refwcs, refcat_sa)
try:
sa_cmd = '%s %s_superalign.in %s_sources.cat %s_offsets.cat' % (
sa_hlf, visit, visit, visit)
ctx.vlog('Running: %s', sa_cmd)
ecode = super_align.runSuperAlign(sa_cmd)
if ecode != 0:
print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
print('Superalign FAILED on %s' % visit)
print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
sfv.write('%s\n' % visit)
else:
print('----------------------------------')
print('Completed superalign on %s' % visit)
print('----------------------------------')
super_align.makeSourceCat(visit, refwcs)
ctx.vlog('Running simplematch')
super_align.runSimpleMatch(visit)
ctx.vlog('Running applyshifts')
super_align.applyShiftsSA(visit)
except Exception as e:
sfv.close()
hutils.wConfig(cfg, cfgf)
print(e)
raise
# Refine the shifts with MCMC
ctx.vlog('Refining shifts...')
with open('mcmc_shifts.txt', 'a') as ms:
for drz in infiles:
super_align.stars2cat(drz, refwcs)
for visit, inf in vdata.items():
ctx.vlog('Generating matched catalogs for visit %s', visit)
for c1 in glob.glob('%s???_drz_sci_sa.cat.stars.cat' % visit):
c2 = refcat
c1m = c1.replace('.cat.stars.cat', '_match.cat')
c2m = c1.replace('.cat.stars.cat', '_ref_match.cat')
super_align.nn_match(c1, c2, refwcs, c1m, c2m)
for drz in infiles:
with fits.open(drz) as hdu:
wn = hdu[0].header['wcsname']
if wn == 'DRZWCS_1':
ctx.vlog('Refining shifts for %s' % drz)
offset = super_align.refineShiftMCMC(drz)
dxp, dyp, dtp = offset
ms.write('%s %.3f %.3f %.3f\n' % (drz, dxp, dyp, dtp))
ctx.vlog('Generating catalogs for checking alignment for %s',
drz)
whtf = drz.replace('sci', 'wht')
instdet = hutils.getInstDet(drz)
micat = drz.replace('.fits', '_all.cat')
os.rename(micat, micat.replace('_all.cat', '_all_orig.cat'))
omrcat = drz.replace('.fits', '_ref.cat')
os.rename(omrcat, omrcat.replace('_ref.cat', '_ref_orig.cat'))
os.rename(drz.replace('.fits', '.cat'),
drz.replace('.fits', '_orig.cat'))
mkcat.makeCat(drz, instdet, weightfile=whtf)
omicat = drz.replace('.fits', '.cat')
# create new catalogs with only maching pairs
hutils.nn_match_radec(micat, refcat, omicat, omrcat)
ctx.vlog('Calculating total shifts from shift files...')
sasd = {}
with open('sa_shifts.txt') as sas:
sasl = [i.split() for i in sas.read().splitlines()]
for sasi in sasl:
ffn, sdx, sdy, sdt = sasi
sasd[ffn] = [float(sdx), float(sdy), float(sdt)]
mcsd = {}
with open('mcmc_shifts.txt') as mcs:
mcsl = [i.split() for i in mcs.read().splitlines()]
for mcsi in mcsl:
ffn, mdx, mdy, mdt = mcsi
mcsd[ffn] = [float(mdx), float(mdy), float(mdt)]
with open('{}_total_shifts.txt'.format(dsn), 'w') as tsf:
for ff, sas in sasd.items():
mcs = mcsd[ff]
tsh = [sum(i) for i in zip(sas, mcs)]
ash = sas + mcs + tsh
tshifts = [ff] + ash
tsf.write(
'{} {: .3f} {: .3f} {: .3f} {: .3f} {: .3f} {: .3f} {: .3f} {: .3f} {: .3f}\n'
.format(*tshifts))
tcfg['etime'] = ctx.dt()
tcfg['completed'] = True
ctx.vlog('Writing configuration file %s for %s task', cfgf, task)
hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, otype, ptask):
"""
Generates object catalogs for each input image sutable for alignment
"""
dsn = ctx.dataset_name
ctx.log('Running task %s for dataset %s', task, dsn)
procdir = os.path.join(ctx.rundir, dsn)
os.chdir(procdir)
cfgf = '%s_cfg.json' % dsn
cfg = hutils.rConfig(cfgf)
refimg = ctx.refimg
refcat = ctx.refcat
tcfg = cfg['tasks'][task] = {}
tcfg['ptask'] = ptask
tcfg['itype'] = itype
tcfg['otype'] = otype
tcfg['stime'] = ctx.dt()
tcfg['completed'] = False
images = hutils.imgList(cfg['images'])
infiles = [str('%s%s' % (i, itype)) for i in images]
refwht = None
extref = False
if refimg:
extref = True
if not refimg:
# For now if a reference image is not given just use one of the input images
# TODO Need to determine best image to use for the reference image from the input list if no refimg is given
refimg = infiles[0]
refwht = refimg.replace('drz_sci.fits', 'drz_wht.fits')
cfg['refimg'] = refimg
cfg['refcat'] = refimg.replace('.fits', '.cat')
mkcat = make_catalog.MakeCat(refimg)
if refcat:
cfg['refcat'] = refcat
refcat_sa = '%s_refcat_sa.cat' % dsn
cfg['refcat_sa'] = refcat_sa
ctx.vlog('Generating catalog from external reference catalog %s',
refcat)
mkcat.makeSACatExtRef(refcat, refcat_sa)
else:
cfg['refcat'] = refimg.replace('.fits', '.cat')
cfg['refcat_sa'] = refimg.replace('.fits', '_sa.cat')
instdet = hutils.getInstDet(refimg)
if extref:
ctx.vlog('Generating catalog for external reference image %s',
refimg)
mkcat.makeCat(refimg, instdet, weightfile=refwht)
mkcat.makeSACat(refimg, extref=True)
else:
ctx.vlog('Generating catalog for internal reference image %s',
refimg)
mkcat.makeCat(refimg, instdet, weightfile=refwht)
mkcat.makeSACat(refimg)
n = len(infiles)
with click.progressbar(infiles,
label='Generating catalogs for images') as pbar:
for i, inf in enumerate(pbar):
ctx.vlog('\nGenerating catalogs for image %s - %s of %s', inf,
i + 1, n)
whtf = inf.replace('sci', 'wht')
instdet = hutils.getInstDet(inf)
mkcat.makeCat(inf, instdet, weightfile=whtf)
micat = inf.replace('.fits', '_all.cat')
omicat = inf.replace('.fits', '.cat')
omrcat = inf.replace('.fits', '_ref.cat')
# create new catalogs with only maching pairs
hutils.nn_match_radec(micat, refcat, omicat, omrcat)
mkcat.makeSACat(inf)
tcfg['etime'] = ctx.dt()
tcfg['completed'] = True
ctx.vlog('Writing configuration file %s for %s task', cfgf, task)
hutils.wConfig(cfg, cfgf)
def cli(ctx):
"""Initializes a pipeline run. The init task copies the fits files from the input dataset directory to the rundir/dataset_name.
All input fits image should be of type flt.fits or flc.fits. For convenience the init task will copy all fits files to the dataset_name directory as type flt.fits.
"""
cfg = {}
dsn = cfg['dsname'] = ctx.dataset_name
ctx.log('Initializing pipeline for dataset %s', dsn)
cfgf = '%s_cfg.json' % dsn
cfg['tasks'] = {}
tcfg = cfg['tasks'][task] = {}
tcfg['ptask'] = None
tcfg['otype'] = '_flt.fits'
tcfg['stime'] = ctx.dt()
tcfg['completed'] = False
dsdir = os.path.join(ctx.dsdir, dsn)
if not os.path.isdir(dsdir):
ctx.elog('Input dataset %s not found! Bye.', dsdir)
sys.exit(1)
procdir = cfg['procdir'] = os.path.join(ctx.rundir, dsn)
infiles = cfg['infiles'] = glob.glob(os.path.join(
dsdir, '*_flt.fits')) + glob.glob(os.path.join(dsdir, '*_flc.fits'))
cfg['refimg'] = ctx.refimg
cfg['refcat'] = ctx.refcat
try:
ctx.vlog('Creating run directory %s', dsn)
os.makedirs(procdir)
except OSError:
if not os.path.isdir(procdir):
raise
os.chdir(procdir)
outfiles = []
n = len(infiles)
with click.progressbar(infiles,
label='Copying images to run directory') as pbar:
for i, f in enumerate(pbar):
fn = str(os.path.basename(f))
if '_flc.fits' in fn:
fn = fn.replace('_flc.fits', '_flt.fits')
if not os.path.exists(fn):
try:
ctx.vlog('\nCopying image %s of %s to %s', i + 1, n, dsn)
shutil.copy(f, fn)
outfiles.append(fn)
masks = glob.glob(os.path.join(dsdir, '*.reg'))
if masks:
for m in masks:
shutil.copy(m, os.path.basename(m))
except shutil.Error as e:
ctx.elog('Error: %s', e)
# eg. source or destination doesn't exist
raise
except IOError as e:
ctx.elog('Error: %s', e.strerror)
raise
else:
outfiles.append(fn)
images = {}
for f in outfiles:
ctx.vlog('Preping fits file %s for pipeline run', f)
id, ftr = init_image.initImage(f)
ctx.vlog('Creating footprint for fits file %s', f)
hutils.getFootprint(f)
if id not in list(images.keys()):
images[id] = {}
fid = f.split('_flt.fits')[0]
if ftr not in list(images[id].keys()):
images[id][ftr] = []
images[id][ftr].append(fid)
else:
images[id][ftr].append(fid)
cfg['images'] = images
tcfg['etime'] = ctx.dt()
tcfg['completed'] = True
ctx.vlog('Writing configuration file %s for %s task', cfgf, task)
hutils.wConfig(cfg, cfgf)
ctx.vlog('Writing image lists')
hutils.wImgLists(cfg)