def mkscaledtemplate( targetfilter, imfile1, imfile2=None, outfile=None, scalefactor=1, filtdir='HSTFILTERS', verbose=True, clobber=False ): """ Combine and/or scale imfile1 (and imfile2) to match the filter transmission function for targetfilter (e.g. for making a template image for F814W from two images in F775W and F850LP). If an outfile is specified and the wht and bpx files associated with the input image files are available, then they will also be combined and written to appropriately named output files The argument targetfilter must have the form 'INSTRUMENT-DETECTOR-FILTER'. For example: 'ACS-WFC-F606W' Returns the names of all files generated (sci, wht, bpx) """ from astropy.io import fits as pyfits import imarith import shutil import badpix sourcefilter1 = camfiltername( imfile1 ) if imfile2 : sourcefilter2 = camfiltername( imfile2 ) if verbose : print('Scaling %s + %s to match %s'%(sourcefilter1,sourcefilter2,targetfilter)) else : sourcefilter2 = None if verbose : print('Scaling %s to match %s'%(sourcefilter1,targetfilter)) scalefactor = scalefactor * computeFilterScaling( targetfilter, sourcefilter1, source2=sourcefilter2, filtdir=filtdir) if imfile2 : imdat = imarith.imsum( imfile1, imfile2 ) else : imdat = pyfits.getdata( imfile1 ) imdatscaled = imdat * scalefactor outhdr = pyfits.getheader( imfile1 ) outhdr['FILTER'] = '~' + targetfilter.split('-')[-1] if 'FILTER1' in outhdr : outhdr.remove('FILTER1') if 'FILTER2' in outhdr : outhdr.remove('FILTER2') if outfile is None : return( imdatscaled ) outdir = os.path.split( outfile )[0] if outdir : if not os.path.isdir(outdir): os.makedirs( outdir ) pyfits.writeto( outfile, imdatscaled, header=outhdr, clobber=clobber ) if not outfile.endswith('sci.fits') or not imfile1.endswith('sci.fits'): return( outfile ) outwht = outfile.replace('sci.fits','wht.fits') outbpx = outfile.replace('sci.fits','bpx.fits') inwht1 = imfile1.replace('sci.fits','wht.fits') inbpx1 = imfile1.replace('sci.fits','bpx.fits') if imfile2 is None : if os.path.isfile( inwht1 ) and os.path.isfile( inbpx1 ) : shutil.copy( inwht1, outwht ) shutil.copy( inbpx1, outbpx ) return( outfile, outwht, outbpx) else : return( outfile ) inwht2 = imfile2.replace('sci.fits','wht.fits') inbpx2 = imfile2.replace('sci.fits','bpx.fits') if os.path.isfile( inwht2 ) : outwht = imarith.combine_ivm_maps( inwht1, inwht2, outwht, clobber=clobber, verbose=verbose ) if os.path.isfile( inbpx2 ) : outbpx = badpix.unionmask( inbpx1, inbpx2, outbpx, clobber=clobber, verbose=verbose ) assert os.path.isfile( outbpx ), "Scaled badpix image %s not created."%outbpx else: outbpx = None assert os.path.isfile( outfile ), "Scaled template %s not created."%outfile assert os.path.isfile( outwht ), "Scaled weight image %s not created."%outwht return( outfile, outwht, outbpx)
def doScaleSubMask( targname, targfilter, targepoch, tempfilter, tempepoch, tempfilter2=None, tempepoch2=None, scalefactor=1, filtdir='HSTFILTERS', clean=True, verbose=True, clobber=False ): """ Primary function for making pseudo-filter diff images : * make a scaled template image, scaling the template filter to match the target filter * subtract scaled template from target filter+epoch * make a union badpix mask * apply the union badpix mask * make a composite weight mask """ import os import imarith import badpix from astropy.io import fits as pyfits targdir = '%s.e%02i'%(targname, targepoch) targsci = '%s_%s_e%02i_reg_drc_sci.fits'%(targname, targfilter, targepoch) targsci = os.path.join( targdir, targsci ) if not os.path.isfile( targsci ) : targsci = targsci.replace('_drc','_drz') targwht = targsci.replace( '_sci.fits','_wht.fits') targbpx = targsci.replace( '_sci.fits','_bpx.fits') assert os.path.isfile( targsci ) assert os.path.isfile( targwht ) # assert os.path.isfile( targbpx ) hdr = pyfits.getheader( targsci ) if 'CAMERA' in hdr : instrument = hdr['CAMERA'] detector = '' elif 'INSTRUME' in hdr : instrument = hdr['INSTRUME'] if 'DETECTOR' in hdr : detector = hdr['DETECTOR'] targetcamfilter = '-'.join([instrument,detector,targfilter.upper()]).rstrip('-') if targetcamfilter.endswith('L') : targetcamfilter += 'P' tempdir = '%s.e%02i'%(targname, tempepoch) tempfile = '%s_%s_e%02i_reg_drc_sci.fits'%(targname, tempfilter, tempepoch) tempfile = os.path.join( tempdir, tempfile ) if not os.path.isfile( tempfile ) : tempfile = tempfile.replace('_drc','_drz') assert os.path.isfile( tempfile ) if tempfilter2 is None : tempfile2 = None else : if tempepoch2 is None : tempepoch2 = tempepoch tempdir2 = '%s.e%02i'%(targname, tempepoch2) tempfile2 = '%s_%s_e%02i_reg_drc_sci.fits'%(targname, tempfilter2, tempepoch2) tempfile2 = os.path.join( tempdir2, tempfile2 ) if not os.path.isfile( tempfile2 ) : tempfile2 = tempfile2.replace('_drc','_drz') assert os.path.isfile( tempfile2 ) outfile = os.path.join( tempdir, '%s_~%s_e%02i_reg_drc_sci.fits'%(targname, targfilter, tempepoch) ) if os.path.exists( outfile ) and not clobber : print("Template file %s already exists, not clobbering."%outfile) tempsci = outfile tempwht = tempsci.replace('sci.fits','wht.fits') tempbpx = tempsci.replace('sci.fits','bpx.fits') else : tempsci, tempwht, tempbpx = mkscaledtemplate( targetcamfilter, tempfile, imfile2=tempfile2, outfile=outfile, scalefactor=scalefactor, filtdir=filtdir, verbose=verbose, clobber=clobber) subsci = '%s_%s_e%02i-e%02i_sub_sci.fits'%(targname, targfilter, targepoch, tempepoch) subsci = os.path.join( targdir, subsci ) if verbose : print( "Making pseudo diff image %s"%subsci ) diffim = imarith.imsubtract( tempsci, targsci, outfile=subsci, clobber=clobber, verbose=verbose ) diffwht = imarith.combine_ivm_maps( targwht, tempwht, diffim.replace('sci.fits','wht.fits'), clobber=clobber, verbose=verbose ) if os.path.exists(targbpx): diffbpx = badpix.unionmask( tempbpx, targbpx, diffim.replace('sci.fits','bpx.fits'), clobber=clobber, verbose=verbose) diffim_masked = badpix.applymask( diffim, diffbpx, clobber=clobber, verbose=verbose) if clean: # delete the sub_sci.fits, b/c it is superseded os.remove( diffim ) else: diffim_masked = diffim.replace('sub_sci.fits', 'sub_masked.fits') os.rename(diffim, diffim_masked) diffbpx = None if verbose : print("Created diff image %s, wht map %s, and bpx mask %s"%( diffim_masked, diffwht, diffbpx ) ) return( diffim_masked, diffwht, diffbpx )
def runpipe( outroot, onlyfilters=[], onlyepochs=[], # Run all the processing steps doall=False, # Setup : copy flts into sub-directories dosetup=False, epochlistfile=None, # construct a ref image dorefim=False, # Single Visit tweakreg/drizzle pass : dodriz1=False, drizcr=False, intravisitreg=False, # Register to a given image or epoch, visit and filter doreg=False, refim=None, refepoch=None, refvisit=None, reffilter=None, # Drizzle registered flts by epoch and filter dodriz2=False, # make diff images dodiff=False, tempepoch=0, refcat=None, interactive=False, threshold=4, peakmin=None, peakmax=None, rfluxmax=27, rfluxmin=14, searchrad=1.5, mjdmin=0, mjdmax=0, epochspan=5, ra=None, dec=None, rot=0, imsize_arcsec=None, pixscale=None, pixfrac=None, wht_type='ERR', clobber=False, verbose=True, debug=False ): """ Primary pipeline function. Executes all the intermediate steps: register, sort, drizzle, subtract, mask """ import pyfits import shutil if debug : import pdb; pdb.set_trace() from drizzlepac.tweakback import tweakback topdir = os.path.abspath( '.' ) fltdir = outroot + '.flt' if doall : dosetup=True dorefim=True dodriz1=True doreg=True dodriz2=True dodiff=True if onlyfilters : if type(onlyfilters)==str : onlyfilters = onlyfilters.lower().split(',') onlyfilters = [ filt[:5].lower() for filt in onlyfilters ] if type(onlyepochs) in [str,int,float] : onlyepochs = [ int(ep) for ep in str(onlyepochs).split(',') ] # STAGE 0 : (always runs) # get a list of exposures and epochs, sorting flt files into epochs fltlist = glob.glob( "%s/*fl?.fits"%fltdir ) if not len( fltlist ) : raise( exceptions.RuntimeError( "There are no flt/flc files in %s !!"%fltdir) ) explist = exposures.get_explist( fltlist, outroot=outroot ) if not epochlistfile : epochlistfile = "%s_epochs.txt"%explist[0].outroot if os.path.exists( epochlistfile ) and not clobber : print( "%s exists. Adopting existing epoch sorting."%epochlistfile ) exposures.read_epochs( explist, epochlistfile, checkradec=[ra,dec], onlyfilters=onlyfilters ) else : exposures.define_epochs( explist, epochspan=epochspan, mjdmin=mjdmin, mjdmax=mjdmax ) exposures.print_epochs( explist, outfile=epochlistfile, verbose=verbose, clobber=clobber, checkradec=[ra,dec], onlyfilters=onlyfilters, onlyepochs=onlyepochs ) if refim and not os.path.exists( refim ) : raise exceptions.RuntimeError( 'Ref image %s does not exist.'%refim ) if not refim : # No refimage has been specified, so set the default refimage name refdrzdir = '%s.refim'%outroot refimbasename = '%s_wcsref_sci.fits'%(outroot) refim = os.path.abspath( os.path.join( refdrzdir, refimbasename ) ) new_explist = [] for exp in explist: if onlyfilters and exp.filter not in onlyfilters : continue elif onlyepochs and exp.epoch not in onlyepochs : continue else: new_explist.append( exp ) explist = new_explist # STAGE 1 : # copy pristine flt files into epoch sub-directories if dosetup : if verbose : print("SNDRIZZLE : (1) SETUP : copying flt files into subdirs") exposures.copy_to_epochdirs( explist, checkradec=[ra,dec], onlyfilters=onlyfilters, onlyepochs=onlyepochs, verbose=verbose, clobber=clobber ) FEVgrouplist = sorted( np.unique( [ exp.FEVgroup for exp in explist ] ) ) FEgrouplist = sorted( np.unique( [ exp.FEgroup for exp in explist ] ) ) filterlist = sorted( np.unique( [ exp.filter for exp in explist ] ) ) epochlist = sorted( np.unique([ exp.epoch for exp in explist ] ) ) # STAGE 2 : # Construct the WCS reference image if dorefim : if verbose : print("SNDRIZZLE : (2) REFIM : Constructing WCS ref image.") if os.path.exists( refim ) and clobber : os.remove( refim ) if os.path.exists( refim ) : print("%s already exists. Not clobbering."%refim) else : refdrzdir = os.path.dirname( refim ) if verbose : print( " Constructing reference image %s in %s"%( os.path.basename(refim), refdrzdir ) ) # Collect the necessary flt files for constructing the ref image if not refepoch : refepoch = np.min( epochlist ) if not reffilter : reffilter = sorted( [ exp.filter for exp in explist if exp.epoch==refepoch ] )[0] reffilter = reffilter.lower() # if refvisit is not set, then find the maximum depth visit and use that if not refvisit : visits, exp_times, visit_depth = np.array( ([], [], []) ) for exp in explist: if (exp.epoch != refepoch) and (exp.filter != reffilter): continue visits = np.append( visits, exp.visit ) exp_times = np.append( exp_times, exp.exposure_time ) unique_visits = np.unique( visits ) for u in unique_visits: ind = np.where( visits == u ) visit_depth = np.append( visit_depth, np.sum(exp_times[ind]) ) max_ind = np.argmax( visit_depth ) refvisit = unique_visits[max_ind] refvisit = refvisit.upper() explistRI = sorted( [ exp for exp in explist if exp.epoch==refepoch and exp.filter==reffilter and exp.visit==refvisit ] ) refdrzdir = os.path.dirname( refim ) if not os.path.isdir( refdrzdir ) : os.makedirs( refdrzdir ) for exp in explistRI : fltfile = os.path.basename( exp.filename ) refsrcdir = os.path.abspath( exp.epochdir ) shutil.copy( os.path.join( refsrcdir, fltfile ), refdrzdir ) fltlistRI = [ exp.filename for exp in explistRI ] refimroot = '%s_wcsref'%outroot os.chdir( refdrzdir ) refimsci, refimwht = drizzle.secondDrizzle( fltlistRI, refimroot, refimage=None, ra=ra, dec=dec, rot=rot, imsize_arcsec=imsize_arcsec, wht_type=wht_type, pixscale=pixscale, pixfrac=pixfrac, clobber=clobber, verbose=verbose, debug=debug ) os.rename(refimsci,refim) os.chdir(topdir) print refvisit # STAGE 3 : # Drizzle together each drizzle group (same epoch, visit and # filter), using almost-default parameters, doing CR rejection # and applying intravisit registrations if requested. The output # is a drz_sci.fits file in the native rotation. if dodriz1 : if verbose : print("SNDRIZZLE : (3) DRIZ1 : first astrodrizzle pass.") for FEVgroup in FEVgrouplist : explistFEV = [ exp for exp in explist if exp.FEVgroup == FEVgroup ] thisepoch = explistFEV[0].epoch thisfilter = explistFEV[0].filter if onlyepochs and thisepoch not in onlyepochs : continue if onlyfilters and thisfilter not in onlyfilters : continue epochdir = explistFEV[0].epochdir fltlistFEV = [ exp.filename for exp in explistFEV ] outrootFEV = '%s_%s_nat'%(outroot,FEVgroup) os.chdir( epochdir ) outsciFEV = '%s_%s_sci.fits'%(outrootFEV,explistFEV[0].drzsuffix) if os.path.exists( outsciFEV ) and clobber : os.remove( outsciFEV ) if os.path.exists( outsciFEV ) : if verbose: print("%s exists. Not clobbering."%outsciFEV ) os.chdir( topdir ) continue if intravisitreg : # run tweakreg for intravisit registration tweaks register.intraVisit( fltlistFEV, peakmin=peakmin, peakmax=peakmax, threshold=threshold, interactive=interactive, debug=debug ) drizzle.firstDrizzle( fltlistFEV, outrootFEV, driz_cr=drizcr, wcskey=((intravisitreg and 'INTRAVIS') or '') ) # TODO : Update the WCS of the refim so that it matches the reference catalog #if refcat : # if verbose : print( " Registering reference image %s to ref catalog %s"%(refim,refcat)) # register.toCatalog( refim, refcat, refim, rfluxmax=rfluxmax, rfluxmin=rfluxmin, # searchrad=searchrad, peakmin=peakmin, peakmax=peakmax, threshold=threshold, # interactive=interactive, debug=debug ) os.chdir( topdir ) if doreg or dodriz2 : if not os.path.exists( refim ): raise exceptions.RuntimeError("No refim file %s! Maybe you should re-run with dorefim=True."%refim) else : refim = os.path.abspath( refim ) # Fix the output ra and dec center point if not provided by the user. if not ra and not dec : ra = pyfits.getval( refim, "CRVAL1" ) dec = pyfits.getval( refim, "CRVAL2" ) # STAGE 4 : # Run tweakreg to register all the single-visit drz images # to a common WCS defined by the refcat/refim, updating the drz file headers. # Then use tweakback to propagate that back into the flt files if doreg : if verbose : print("SNDRIZZLE : (4) REG : running tweakreg.") for FEVgroup in FEVgrouplist : explistFEV = [ exp for exp in explist if exp.FEVgroup == FEVgroup ] thisepoch = explistFEV[0].epoch thisfilter = explistFEV[0].filter if onlyepochs and thisepoch not in onlyepochs : continue if onlyfilters and thisfilter not in onlyfilters : continue epochdir = explistFEV[0].epochdir fltlistFEV = [ exp.filename for exp in explistFEV ] outrootFEV = '%s_%s_nat'%(outroot,FEVgroup) outsciFEV = '%s_%s_sci.fits'%(outrootFEV,explistFEV[0].drzsuffix) refimpath = os.path.abspath(refim) if refcat : refcatpath = os.path.abspath(refcat) else : refcatpath = None os.chdir( epochdir ) if not os.path.exists( outsciFEV ) : exceptions.RuntimeError( "Missing %s."%outsciFEV ) # register to the ref image and ref catalog origwcs = pyfits.getval( outsciFEV,'WCSNAME').strip() wcsname = register.toRefim( outsciFEV, refim=refimpath, refcat=refcatpath, searchrad=searchrad, peakmin=peakmin, peakmax=peakmax, threshold=threshold, interactive=interactive, clobber=clobber, debug=debug ) # Run tweakback to update the constituent flts tweakback( outsciFEV, input=fltlistFEV, origwcs=origwcs, wcsname=wcsname, verbose=verbose, force=clobber ) os.chdir(topdir) # STAGE 5 # Second and final astrodrizzle pass, wherein we rerun # astrodrizzle to get wcs- and pixel-registered drz images # combining all flt files with the same filter and epoch. if dodriz2 : if verbose : print("SNDRIZZLE : (5) DRIZ2 : second astrodrizzle pass.") for FEgroup in FEgrouplist : explistFE = [ exp for exp in explist if exp.FEgroup == FEgroup ] thisepoch = explistFE[0].epoch thisfilter = explistFE[0].filter if onlyepochs and thisepoch not in onlyepochs : continue if onlyfilters and thisfilter not in onlyfilters : continue epochdir = explistFE[0].epochdir fltlistFE = [ exp.filename for exp in explistFE ] outrootFE = '%s_%s_reg'%(outroot,FEgroup) err = None os.chdir( epochdir ) outsciFE = '%s_%s_sci.fits'%(outrootFE,explistFE[0].drzsuffix) if os.path.exists( outsciFE ) and clobber : os.remove( outsciFE ) if os.path.exists( outsciFE ) : if verbose: print("%s exists. Not clobbering."%outsciFE ) os.chdir( topdir ) continue outsciFE, outwhtFE = drizzle.secondDrizzle( fltlistFE, outrootFE, refimage=refim, ra=ra, dec=dec, rot=rot, imsize_arcsec=imsize_arcsec, wht_type=wht_type, pixscale=pixscale, pixfrac=pixfrac, clobber=clobber, verbose=verbose, debug=debug ) outbpxFE = outwhtFE.replace('_wht','_bpx') outbpxFE = badpix.zerowht2badpix( outwhtFE, outbpxFE, verbose=verbose, clobber=clobber ) os.chdir( topdir ) # STAGE 6 # Define a template epoch for each filter and subtract it from the other epochs if dodiff : if verbose : print("SNDRIZZLE : (6) DIFF : subtracting template images.") for filter in filterlist : if onlyfilters and filter not in onlyfilters : continue template = None if tempepoch == None : # User did not define a template epoch, so we default # to use the first available epoch for epoch in epochlist : explistFE = [ exp for exp in explist if exp.filter==filter and exp.epoch==epoch ] if len(explistFE)==0: continue drzsuffix = explistFE[0].drzsuffix epochdir = explistFE[0].epochdir FEgroup = explistFE[0].FEgroup outrootFE = '%s_%s'%(outroot, FEgroup ) if os.path.isfile( os.path.join( epochdir, outrootFE+"_reg_%s_sci.fits"%drzsuffix )) : tempepoch = epoch break else : explistF = [ exp for exp in explist if exp.filter==filter ] drzsuffix = explistF[0].drzsuffix # now do the subtractions tempdir = outroot+'.e%02i'%tempepoch tempsci = os.path.join( os.path.abspath(tempdir), '%s_%s_e%02i_reg_%s_sci.fits'%(outroot,filter,tempepoch,drzsuffix)) tempwht = tempsci.replace('sci.fits','wht.fits') tempbpx = tempsci.replace('sci.fits','bpx.fits') topdir = os.path.abspath( '.' ) for epoch in epochlist : if onlyepochs and epoch not in onlyepochs : continue if epoch == tempepoch : continue explistFE = [ exp for exp in explist if exp.filter==filter and exp.epoch==epoch ] if len(explistFE)==0 : continue epochdirFE = explistFE[0].epochdir outrootFE = "%s_%s"%(outroot,explistFE[0].FEgroup) drzsuffix = explistFE[0].drzsuffix thisregsci = "%s_reg_%s_sci.fits"%(outrootFE,drzsuffix) thisregwht = "%s_reg_%s_wht.fits"%(outrootFE,drzsuffix) thisregbpx = "%s_reg_%s_bpx.fits"%(outrootFE,drzsuffix) os.chdir( epochdirFE ) if not os.path.isfile( thisregsci ) : os.chdir( topdir ) continue thisdiffim = outrootFE + "-e%02i_sub_sci.fits"%tempepoch if not os.path.isfile( tempsci ) or not os.path.isfile( thisregsci) : print( "Can't create diff image %s"%thisdiffim ) print( " missing either %s"%tempsci ) print( " or %s"%thisregsci ) os.chdir( topdir ) continue diffim = imarith.imsubtract( tempsci, thisregsci, outfile=thisdiffim, clobber=clobber, verbose=verbose, debug=debug) diffwht = badpix.combine_ivm_maps( thisregwht, tempwht, diffim.replace('sci.fits','wht.fits'), clobber=clobber, verbose=verbose ) diffbpx = badpix.unionmask( tempbpx, thisregbpx, diffim.replace('sci.fits','bpx.fits'), clobber=clobber, verbose=verbose) diffim_masked = badpix.applymask( diffim, diffbpx, clobber=clobber, verbose=verbose) print("Created diff image %s, wht map %s, and bpx mask %s"%( diffim_masked, diffwht, diffbpx ) ) os.chdir( topdir ) return 0