def cli(ctx, optcr, drzonly, usehlet, ctype, itype, ofile, ptask): """ Drizzles final mosaic for each filter """ dsn = ctx.dataset_name ctx.log('Running task %s for dataset %s', task, dsn) procdir = os.path.join(ctx.rundir, dsn) os.chdir(procdir) cfgf = '%s_cfg.json' % dsn cfg = hutils.rConfig(cfgf) tcfg = cfg['tasks'][task] = {} tcfg['ptask'] = ptask tcfg['itype'] = itype tcfg['ofile'] = ofile tcfg['stime'] = ctx.dt() tcfg['completed'] = False for instdet, data in cfg['images'].items(): for fltr, images in data.items(): outfile = '%s_%s_%s' % (dsn, instdet, fltr.lower()) infiles = [str('%s%s' % (i, itype)) for i in images] ctx.vlog('Drizzling mosaic %s', outfile) try: if cfg['refimg']: refimg = str(cfg['refimg']) drizzle_mosaic.drzMosaic(infiles, outfile, refimg=refimg, optcr=optcr, drzonly=drzonly, usehlet=usehlet, ctype=ctype) else: drizzle_mosaic.drzMosaic(infiles, outfile, optcr=optcr, drzonly=drzonly, usehlet=usehlet, ctype=ctype) except Exception as e: hutils.wConfig(cfg, cfgf) print(e) raise tcfg['etime'] = ctx.dt() tcfg['completed'] = True ctx.vlog('Writing configuration file %s for %s task', cfgf, task) hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, otype, ptask): """ Drizzles and individual images to be used for alignment """ dsn = ctx.dataset_name ctx.log('Running task %s for dataset %s', task, dsn) procdir = os.path.join(ctx.rundir, dsn) os.chdir(procdir) cfgf = '%s_cfg.json' % dsn cfg = hutils.rConfig(cfgf) tcfg = cfg['tasks'][task] = {} tcfg['ptask'] = ptask tcfg['itype'] = itype tcfg['otype'] = otype tcfg['stime'] = ctx.dt() tcfg['completed'] = False if ctx.refimg: refimg = str(ctx.refimg) refwcs = HSTWCS(refimg) cfg['refimg'] = refimg pscale = refwcs.pscale orientat = refwcs.orientat else: # use native pixel scale pscale = None orientat = 0 images = hutils.imgList(cfg['images']) infiles = [str('%s%s' % (i, itype)) for i in images] n = len(infiles) with click.progressbar(infiles, label='Generating single drizzled images') as pbar: for i, f in enumerate(pbar): ctx.vlog('\n\nDrizzling image %s - %s of %s', f, i + 1, n) try: drizzle_image.drzImage(f, pscale, orientat) except Exception as e: hutils.wConfig(cfg, cfgf) print(e) raise tcfg['etime'] = ctx.dt() tcfg['completed'] = True ctx.vlog('Writing configuration file %s for %s task', cfgf, task) hutils.wConfig(cfg, cfgf)
def cli(ctx, restore, hlet, itype, otype, ptask): """ Applies offsets to images computed by superalign """ dsn = ctx.dataset_name ctx.log('Running task %s for dataset %s', task, dsn) procdir = os.path.join(ctx.rundir, dsn) os.chdir(procdir) cfgf = '%s_cfg.json' % dsn cfg = hutils.rConfig(cfgf) tcfg = cfg['tasks'][task] = {} tcfg['ptask'] = ptask tcfg['itype'] = itype tcfg['otype'] = otype tcfg['stime'] = ctx.dt() tcfg['completed'] = False originf = cfg['infiles'] images = hutils.imgList(cfg['images']) infiles = [str('%s%s' % (i, itype)) for i in images] n = len(infiles) with click.progressbar(infiles, label='Generating catalogs for images') as pbar: for i, inf in enumerate(pbar): outf = inf.replace('_drz_sci.fits', '_flt.fits') ctx.vlog('\nApplying offsets image %s - %s of %s', inf, i + 1, n) try: if restore: apply_shift.restoreWCSdrz(inf, 0) for ext in hutils.sciexts[hutils.getInstDet(outf)]: origflt = [s for s in originf if outf in s][0] restoreWCSflt(outf, origflt, ext) ctx.vlog('Applying offsets') apply_shift.applyOffset(inf, outf, hlet=hlet) except Exception as e: hutils.wConfig(cfg, cfgf) print(e) raise tcfg['etime'] = ctx.dt() tcfg['completed'] = True ctx.vlog('Writing configuration file %s for %s task', cfgf, task) hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, otype, ptask): """ Create IMV weight maps for WFC3IR images """ dsn = ctx.dataset_name ctx.log('Running task %s for dataset %s', task, dsn) procdir = os.path.join(ctx.rundir, dsn) os.chdir(procdir) cfgf = '%s_cfg.json' % dsn cfg = hutils.rConfig(cfgf) tcfg = cfg['tasks'][task] = {} tcfg['ptask'] = ptask tcfg['itype'] = itype tcfg['otype'] = otype tcfg['stime'] = ctx.dt() tcfg['completed'] = False iref = ctx.iref images = hutils.imgList(cfg['images']) infiles = [str('%s%s' % (i, itype)) for i in images] n = len(infiles) with click.progressbar(infiles, label='Generating sky subtracted image') as pbar: for i, f in enumerate(pbar): ctx.vlog('\n\nCreating weight map for image %s - %s of %s', f, i + 1, n) instdet = hutils.getInstDet(f) if instdet == 'wfc3ir': try: make_ivm.makeweight(f, iref) except Exception as e: hutils.wConfig(cfg, cfgf) print(e) raise tcfg['etime'] = ctx.dt() tcfg['completed'] = True ctx.vlog('Writing configuration file %s for %s task', cfgf, task) hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, otype, ptask): """ Correct for ACS amplifier discontinuities """ dsn = ctx.dataset_name ctx.log('Running task %s for dataset %s', task, dsn) procdir = os.path.join(ctx.rundir, dsn) os.chdir(procdir) cfgf = '%s_cfg.json' % dsn cfg = hutils.rConfig(cfgf) tcfg = cfg['tasks'][task] = {} tcfg['ptask'] = ptask tcfg['itype'] = itype tcfg['otype'] = otype tcfg['stime'] = ctx.dt() tcfg['completed'] = False images = hutils.imgList(cfg['images']) infiles = [str('%s%s' % (i, itype)) for i in images] n = len(infiles) with click.progressbar( infiles, label='Generating amplifier corrected image') as pbar: for i, f in enumerate(pbar): ctx.vlog('\n\nCorrecting image %s - %s of %s', f, i + 1, n) instdet = hutils.getInstDet(f) if instdet == 'acswfc': try: amp_correct.ampcorr(f) except Exception as e: hutils.wConfig(cfg, cfgf) print(e) raise tcfg['etime'] = ctx.dt() tcfg['completed'] = True ctx.vlog('Writing configuration file %s for %s task', cfgf, task) hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, otype, ptask): """ Applies mask to images """ dsn = ctx.dataset_name pmaskdir = ctx.pmaskdir ctx.log('Running task %s for dataset %s', task, dsn) procdir = os.path.join(ctx.rundir, dsn) os.chdir(procdir) cfgf = '%s_cfg.json' % dsn cfg = hutils.rConfig(cfgf) tcfg = cfg['tasks'][task] = {} tcfg['ptask'] = ptask tcfg['itype'] = itype tcfg['otype'] = otype tcfg['stime'] = ctx.dt() tcfg['completed'] = False images = hutils.imgList(cfg['images']) infiles = [str('%s%s' % (i, itype)) for i in images] n = len(infiles) with click.progressbar(infiles, label='Generating masks for images') as pbar: for i, f in enumerate(pbar): ctx.vlog('\n\nChecking masks for image %s - %s of %s', f, i + 1, n) masks = [ fp for fp in glob.glob('%s*.reg' % f[:9]) if 'footprint' not in fp ] if masks: try: for m in masks: if 'SCI' in m: # ACSWFC and WFC3UV chip = int(m.split('.reg')[0][-1]) if chip == 1: dq_ext = 3 if chip == 2: dq_ext = 6 else: # WFC3IR dq_ext = 3 ctx.vlog('Applying mask to %s - DQ extention %s', f, dq_ext) apply_mask.applymask(f, m, dq_ext) except Exception as e: hutils.wConfig(cfg, cfgf) print(e) raise else: ctx.vlog('No masks found for %s', f) if hutils.getInstDet(f) == 'wfc3ir': if pmaskdir: ctx.vlog('Checking for persistance mask for image %s', f) pmaskfile = os.path.join( pmaskdir, f.replace('flt.fits', 'pmask.fits')) if os.path.exists(pmaskfile): ctx.vlog('Found persistance mask image %s', os.path.basename(pmaskfile)) apply_mask.applypersist(f, pmaskfile) tcfg['etime'] = ctx.dt() tcfg['completed'] = True ctx.vlog('Writing configuration file %s for %s task', cfgf, task) hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, ofile, ptask): """ Runs superalign on catalogs for alignment """ dsn = ctx.dataset_name ctx.log('Running task %s for dataset %s', task, dsn) procdir = os.path.join(ctx.rundir, dsn) os.chdir(procdir) cfgf = '%s_cfg.json' % dsn cfg = hutils.rConfig(cfgf) tcfg = cfg['tasks'][task] = {} tcfg['ptask'] = ptask tcfg['itype'] = itype tcfg['ofile'] = cfg['sfile'] = ofile tcfg['stime'] = ctx.dt() tcfg['completed'] = False images = hutils.imgList(cfg['images']) infiles = [str('%s%s' % (i, itype)) for i in images] refimg = cfg['refimg'] refcat = cfg['refcat'] refcat_sa = cfg['refcat_sa'] refwcs = HSTWCS(fits.open(refimg)) mkcat = make_catalog.MakeCat(refimg) sa_hlf = find_executable('superalign_hlfred') if not sa_hlf: ctx.elog( 'Unable to find "superalign_hlfred" executable. Make sure it is in your PATH.' ) sys.exit(1) ctx.vlog('Grouping images by visit for alignment') visits = set([i[:6] for i in infiles]) vdata = {} for v in visits: vdata[v] = [] for e in infiles: if e[:6] == v: vdata[v].append(e) # Run superalign on all visits ctx.vlog('Generating the superalign input') with open('superalign_failed_visits.txt', 'w') as sfv: for visit, inf in vdata.items(): super_align.makeSAin(visit, inf, refwcs, refcat_sa) try: sa_cmd = '%s %s_superalign.in %s_sources.cat %s_offsets.cat' % ( sa_hlf, visit, visit, visit) ctx.vlog('Running: %s', sa_cmd) ecode = super_align.runSuperAlign(sa_cmd) if ecode != 0: print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') print('Superalign FAILED on %s' % visit) print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') sfv.write('%s\n' % visit) else: print('----------------------------------') print('Completed superalign on %s' % visit) print('----------------------------------') super_align.makeSourceCat(visit, refwcs) ctx.vlog('Running simplematch') super_align.runSimpleMatch(visit) ctx.vlog('Running applyshifts') super_align.applyShiftsSA(visit) except Exception as e: sfv.close() hutils.wConfig(cfg, cfgf) print(e) raise # Refine the shifts with MCMC ctx.vlog('Refining shifts...') with open('mcmc_shifts.txt', 'a') as ms: for drz in infiles: super_align.stars2cat(drz, refwcs) for visit, inf in vdata.items(): ctx.vlog('Generating matched catalogs for visit %s', visit) for c1 in glob.glob('%s???_drz_sci_sa.cat.stars.cat' % visit): c2 = refcat c1m = c1.replace('.cat.stars.cat', '_match.cat') c2m = c1.replace('.cat.stars.cat', '_ref_match.cat') super_align.nn_match(c1, c2, refwcs, c1m, c2m) for drz in infiles: with fits.open(drz) as hdu: wn = hdu[0].header['wcsname'] if wn == 'DRZWCS_1': ctx.vlog('Refining shifts for %s' % drz) offset = super_align.refineShiftMCMC(drz) dxp, dyp, dtp = offset ms.write('%s %.3f %.3f %.3f\n' % (drz, dxp, dyp, dtp)) ctx.vlog('Generating catalogs for checking alignment for %s', drz) whtf = drz.replace('sci', 'wht') instdet = hutils.getInstDet(drz) micat = drz.replace('.fits', '_all.cat') os.rename(micat, micat.replace('_all.cat', '_all_orig.cat')) omrcat = drz.replace('.fits', '_ref.cat') os.rename(omrcat, omrcat.replace('_ref.cat', '_ref_orig.cat')) os.rename(drz.replace('.fits', '.cat'), drz.replace('.fits', '_orig.cat')) mkcat.makeCat(drz, instdet, weightfile=whtf) omicat = drz.replace('.fits', '.cat') # create new catalogs with only maching pairs hutils.nn_match_radec(micat, refcat, omicat, omrcat) ctx.vlog('Calculating total shifts from shift files...') sasd = {} with open('sa_shifts.txt') as sas: sasl = [i.split() for i in sas.read().splitlines()] for sasi in sasl: ffn, sdx, sdy, sdt = sasi sasd[ffn] = [float(sdx), float(sdy), float(sdt)] mcsd = {} with open('mcmc_shifts.txt') as mcs: mcsl = [i.split() for i in mcs.read().splitlines()] for mcsi in mcsl: ffn, mdx, mdy, mdt = mcsi mcsd[ffn] = [float(mdx), float(mdy), float(mdt)] with open('{}_total_shifts.txt'.format(dsn), 'w') as tsf: for ff, sas in sasd.items(): mcs = mcsd[ff] tsh = [sum(i) for i in zip(sas, mcs)] ash = sas + mcs + tsh tshifts = [ff] + ash tsf.write( '{} {: .3f} {: .3f} {: .3f} {: .3f} {: .3f} {: .3f} {: .3f} {: .3f} {: .3f}\n' .format(*tshifts)) tcfg['etime'] = ctx.dt() tcfg['completed'] = True ctx.vlog('Writing configuration file %s for %s task', cfgf, task) hutils.wConfig(cfg, cfgf)
def cli(ctx, itype, otype, ptask): """ Generates object catalogs for each input image sutable for alignment """ dsn = ctx.dataset_name ctx.log('Running task %s for dataset %s', task, dsn) procdir = os.path.join(ctx.rundir, dsn) os.chdir(procdir) cfgf = '%s_cfg.json' % dsn cfg = hutils.rConfig(cfgf) refimg = ctx.refimg refcat = ctx.refcat tcfg = cfg['tasks'][task] = {} tcfg['ptask'] = ptask tcfg['itype'] = itype tcfg['otype'] = otype tcfg['stime'] = ctx.dt() tcfg['completed'] = False images = hutils.imgList(cfg['images']) infiles = [str('%s%s' % (i, itype)) for i in images] refwht = None extref = False if refimg: extref = True if not refimg: # For now if a reference image is not given just use one of the input images # TODO Need to determine best image to use for the reference image from the input list if no refimg is given refimg = infiles[0] refwht = refimg.replace('drz_sci.fits', 'drz_wht.fits') cfg['refimg'] = refimg cfg['refcat'] = refimg.replace('.fits', '.cat') mkcat = make_catalog.MakeCat(refimg) if refcat: cfg['refcat'] = refcat refcat_sa = '%s_refcat_sa.cat' % dsn cfg['refcat_sa'] = refcat_sa ctx.vlog('Generating catalog from external reference catalog %s', refcat) mkcat.makeSACatExtRef(refcat, refcat_sa) else: cfg['refcat'] = refimg.replace('.fits', '.cat') cfg['refcat_sa'] = refimg.replace('.fits', '_sa.cat') instdet = hutils.getInstDet(refimg) if extref: ctx.vlog('Generating catalog for external reference image %s', refimg) mkcat.makeCat(refimg, instdet, weightfile=refwht) mkcat.makeSACat(refimg, extref=True) else: ctx.vlog('Generating catalog for internal reference image %s', refimg) mkcat.makeCat(refimg, instdet, weightfile=refwht) mkcat.makeSACat(refimg) n = len(infiles) with click.progressbar(infiles, label='Generating catalogs for images') as pbar: for i, inf in enumerate(pbar): ctx.vlog('\nGenerating catalogs for image %s - %s of %s', inf, i + 1, n) whtf = inf.replace('sci', 'wht') instdet = hutils.getInstDet(inf) mkcat.makeCat(inf, instdet, weightfile=whtf) micat = inf.replace('.fits', '_all.cat') omicat = inf.replace('.fits', '.cat') omrcat = inf.replace('.fits', '_ref.cat') # create new catalogs with only maching pairs hutils.nn_match_radec(micat, refcat, omicat, omrcat) mkcat.makeSACat(inf) tcfg['etime'] = ctx.dt() tcfg['completed'] = True ctx.vlog('Writing configuration file %s for %s task', cfgf, task) hutils.wConfig(cfg, cfgf)
def cli(ctx): """Initializes a pipeline run. The init task copies the fits files from the input dataset directory to the rundir/dataset_name. All input fits image should be of type flt.fits or flc.fits. For convenience the init task will copy all fits files to the dataset_name directory as type flt.fits. """ cfg = {} dsn = cfg['dsname'] = ctx.dataset_name ctx.log('Initializing pipeline for dataset %s', dsn) cfgf = '%s_cfg.json' % dsn cfg['tasks'] = {} tcfg = cfg['tasks'][task] = {} tcfg['ptask'] = None tcfg['otype'] = '_flt.fits' tcfg['stime'] = ctx.dt() tcfg['completed'] = False dsdir = os.path.join(ctx.dsdir, dsn) if not os.path.isdir(dsdir): ctx.elog('Input dataset %s not found! Bye.', dsdir) sys.exit(1) procdir = cfg['procdir'] = os.path.join(ctx.rundir, dsn) infiles = cfg['infiles'] = glob.glob(os.path.join( dsdir, '*_flt.fits')) + glob.glob(os.path.join(dsdir, '*_flc.fits')) cfg['refimg'] = ctx.refimg cfg['refcat'] = ctx.refcat try: ctx.vlog('Creating run directory %s', dsn) os.makedirs(procdir) except OSError: if not os.path.isdir(procdir): raise os.chdir(procdir) outfiles = [] n = len(infiles) with click.progressbar(infiles, label='Copying images to run directory') as pbar: for i, f in enumerate(pbar): fn = str(os.path.basename(f)) if '_flc.fits' in fn: fn = fn.replace('_flc.fits', '_flt.fits') if not os.path.exists(fn): try: ctx.vlog('\nCopying image %s of %s to %s', i + 1, n, dsn) shutil.copy(f, fn) outfiles.append(fn) masks = glob.glob(os.path.join(dsdir, '*.reg')) if masks: for m in masks: shutil.copy(m, os.path.basename(m)) except shutil.Error as e: ctx.elog('Error: %s', e) # eg. source or destination doesn't exist raise except IOError as e: ctx.elog('Error: %s', e.strerror) raise else: outfiles.append(fn) images = {} for f in outfiles: ctx.vlog('Preping fits file %s for pipeline run', f) id, ftr = init_image.initImage(f) ctx.vlog('Creating footprint for fits file %s', f) hutils.getFootprint(f) if id not in list(images.keys()): images[id] = {} fid = f.split('_flt.fits')[0] if ftr not in list(images[id].keys()): images[id][ftr] = [] images[id][ftr].append(fid) else: images[id][ftr].append(fid) cfg['images'] = images tcfg['etime'] = ctx.dt() tcfg['completed'] = True ctx.vlog('Writing configuration file %s for %s task', cfgf, task) hutils.wConfig(cfg, cfgf) ctx.vlog('Writing image lists') hutils.wImgLists(cfg)