def mkscaledtemplate( targetfilter, imfile1, imfile2=None, outfile=None,
scalefactor=1, filtdir='HSTFILTERS', verbose=True,
clobber=False ):
""" Combine and/or scale imfile1 (and imfile2) to match the
filter transmission function for targetfilter (e.g. for making a
template image for F814W from two images in F775W and F850LP).
If an outfile is specified and the wht and bpx files associated with the
input image files are available, then they will also be combined and
written to appropriately named output files
The argument targetfilter must have the form 'INSTRUMENT-DETECTOR-FILTER'.
For example: 'ACS-WFC-F606W'
Returns the names of all files generated (sci, wht, bpx)
"""
from astropy.io import fits as pyfits
import imarith
import shutil
import badpix
sourcefilter1 = camfiltername( imfile1 )
if imfile2 :
sourcefilter2 = camfiltername( imfile2 )
if verbose :
print('Scaling %s + %s to match %s'%(sourcefilter1,sourcefilter2,targetfilter))
else :
sourcefilter2 = None
if verbose :
print('Scaling %s to match %s'%(sourcefilter1,targetfilter))
scalefactor = scalefactor * computeFilterScaling(
targetfilter, sourcefilter1, source2=sourcefilter2, filtdir=filtdir)
if imfile2 :
imdat = imarith.imsum( imfile1, imfile2 )
else :
imdat = pyfits.getdata( imfile1 )
imdatscaled = imdat * scalefactor
outhdr = pyfits.getheader( imfile1 )
outhdr['FILTER'] = '~' + targetfilter.split('-')[-1]
if 'FILTER1' in outhdr : outhdr.remove('FILTER1')
if 'FILTER2' in outhdr : outhdr.remove('FILTER2')
if outfile is None :
return( imdatscaled )
outdir = os.path.split( outfile )[0]
if outdir :
if not os.path.isdir(outdir):
os.makedirs( outdir )
pyfits.writeto( outfile, imdatscaled, header=outhdr, clobber=clobber )
if not outfile.endswith('sci.fits') or not imfile1.endswith('sci.fits'):
return( outfile )
outwht = outfile.replace('sci.fits','wht.fits')
outbpx = outfile.replace('sci.fits','bpx.fits')
inwht1 = imfile1.replace('sci.fits','wht.fits')
inbpx1 = imfile1.replace('sci.fits','bpx.fits')
if imfile2 is None :
if os.path.isfile( inwht1 ) and os.path.isfile( inbpx1 ) :
shutil.copy( inwht1, outwht )
shutil.copy( inbpx1, outbpx )
return( outfile, outwht, outbpx)
else :
return( outfile )
inwht2 = imfile2.replace('sci.fits','wht.fits')
inbpx2 = imfile2.replace('sci.fits','bpx.fits')
if os.path.isfile( inwht2 ) :
outwht = imarith.combine_ivm_maps( inwht1, inwht2, outwht,
clobber=clobber, verbose=verbose )
if os.path.isfile( inbpx2 ) :
outbpx = badpix.unionmask( inbpx1, inbpx2, outbpx,
clobber=clobber, verbose=verbose )
assert os.path.isfile( outbpx ), "Scaled badpix image %s not created."%outbpx
else:
outbpx = None
assert os.path.isfile( outfile ), "Scaled template %s not created."%outfile
assert os.path.isfile( outwht ), "Scaled weight image %s not created."%outwht
return( outfile, outwht, outbpx)
def doScaleSubMask( targname, targfilter, targepoch, tempfilter, tempepoch,
tempfilter2=None, tempepoch2=None, scalefactor=1,
filtdir='HSTFILTERS',
clean=True, verbose=True, clobber=False ):
""" Primary function for making pseudo-filter diff images :
* make a scaled template image, scaling the template filter to match the target filter
* subtract scaled template from target filter+epoch
* make a union badpix mask
* apply the union badpix mask
* make a composite weight mask
"""
import os
import imarith
import badpix
from astropy.io import fits as pyfits
targdir = '%s.e%02i'%(targname, targepoch)
targsci = '%s_%s_e%02i_reg_drc_sci.fits'%(targname, targfilter, targepoch)
targsci = os.path.join( targdir, targsci )
if not os.path.isfile( targsci ) :
targsci = targsci.replace('_drc','_drz')
targwht = targsci.replace( '_sci.fits','_wht.fits')
targbpx = targsci.replace( '_sci.fits','_bpx.fits')
assert os.path.isfile( targsci )
assert os.path.isfile( targwht )
# assert os.path.isfile( targbpx )
hdr = pyfits.getheader( targsci )
if 'CAMERA' in hdr :
instrument = hdr['CAMERA']
detector = ''
elif 'INSTRUME' in hdr :
instrument = hdr['INSTRUME']
if 'DETECTOR' in hdr :
detector = hdr['DETECTOR']
targetcamfilter = '-'.join([instrument,detector,targfilter.upper()]).rstrip('-')
if targetcamfilter.endswith('L') :
targetcamfilter += 'P'
tempdir = '%s.e%02i'%(targname, tempepoch)
tempfile = '%s_%s_e%02i_reg_drc_sci.fits'%(targname, tempfilter, tempepoch)
tempfile = os.path.join( tempdir, tempfile )
if not os.path.isfile( tempfile ) :
tempfile = tempfile.replace('_drc','_drz')
assert os.path.isfile( tempfile )
if tempfilter2 is None :
tempfile2 = None
else :
if tempepoch2 is None :
tempepoch2 = tempepoch
tempdir2 = '%s.e%02i'%(targname, tempepoch2)
tempfile2 = '%s_%s_e%02i_reg_drc_sci.fits'%(targname, tempfilter2, tempepoch2)
tempfile2 = os.path.join( tempdir2, tempfile2 )
if not os.path.isfile( tempfile2 ) :
tempfile2 = tempfile2.replace('_drc','_drz')
assert os.path.isfile( tempfile2 )
outfile = os.path.join( tempdir, '%s_~%s_e%02i_reg_drc_sci.fits'%(targname, targfilter, tempepoch) )
if os.path.exists( outfile ) and not clobber :
print("Template file %s already exists, not clobbering."%outfile)
tempsci = outfile
tempwht = tempsci.replace('sci.fits','wht.fits')
tempbpx = tempsci.replace('sci.fits','bpx.fits')
else :
tempsci, tempwht, tempbpx = mkscaledtemplate(
targetcamfilter, tempfile, imfile2=tempfile2, outfile=outfile,
scalefactor=scalefactor, filtdir=filtdir, verbose=verbose, clobber=clobber)
subsci = '%s_%s_e%02i-e%02i_sub_sci.fits'%(targname, targfilter, targepoch, tempepoch)
subsci = os.path.join( targdir, subsci )
if verbose : print( "Making pseudo diff image %s"%subsci )
diffim = imarith.imsubtract( tempsci, targsci, outfile=subsci,
clobber=clobber, verbose=verbose )
diffwht = imarith.combine_ivm_maps( targwht, tempwht,
diffim.replace('sci.fits','wht.fits'),
clobber=clobber, verbose=verbose )
if os.path.exists(targbpx):
diffbpx = badpix.unionmask( tempbpx, targbpx,
diffim.replace('sci.fits','bpx.fits'),
clobber=clobber, verbose=verbose)
diffim_masked = badpix.applymask( diffim, diffbpx,
clobber=clobber, verbose=verbose)
if clean:
# delete the sub_sci.fits, b/c it is superseded
os.remove( diffim )
else:
diffim_masked = diffim.replace('sub_sci.fits', 'sub_masked.fits')
os.rename(diffim, diffim_masked)
diffbpx = None
if verbose :
print("Created diff image %s, wht map %s, and bpx mask %s"%(
diffim_masked, diffwht, diffbpx ) )
return( diffim_masked, diffwht, diffbpx )
def runpipe( outroot, onlyfilters=[], onlyepochs=[],
# Run all the processing steps
doall=False,
# Setup : copy flts into sub-directories
dosetup=False, epochlistfile=None,
# construct a ref image
dorefim=False,
# Single Visit tweakreg/drizzle pass :
dodriz1=False, drizcr=False, intravisitreg=False,
# Register to a given image or epoch, visit and filter
doreg=False, refim=None, refepoch=None, refvisit=None, reffilter=None,
# Drizzle registered flts by epoch and filter
dodriz2=False,
# make diff images
dodiff=False, tempepoch=0,
refcat=None,
interactive=False, threshold=4, peakmin=None, peakmax=None,
rfluxmax=27, rfluxmin=14, searchrad=1.5,
mjdmin=0, mjdmax=0, epochspan=5,
ra=None, dec=None, rot=0, imsize_arcsec=None,
pixscale=None, pixfrac=None, wht_type='ERR',
clobber=False, verbose=True, debug=False ):
"""
Primary pipeline function. Executes all the intermediate steps:
register, sort, drizzle, subtract, mask
"""
import pyfits
import shutil
if debug : import pdb; pdb.set_trace()
from drizzlepac.tweakback import tweakback
topdir = os.path.abspath( '.' )
fltdir = outroot + '.flt'
if doall :
dosetup=True
dorefim=True
dodriz1=True
doreg=True
dodriz2=True
dodiff=True
if onlyfilters :
if type(onlyfilters)==str :
onlyfilters = onlyfilters.lower().split(',')
onlyfilters = [ filt[:5].lower() for filt in onlyfilters ]
if type(onlyepochs) in [str,int,float] :
onlyepochs = [ int(ep) for ep in str(onlyepochs).split(',') ]
# STAGE 0 : (always runs)
# get a list of exposures and epochs, sorting flt files into epochs
fltlist = glob.glob( "%s/*fl?.fits"%fltdir )
if not len( fltlist ) :
raise( exceptions.RuntimeError( "There are no flt/flc files in %s !!"%fltdir) )
explist = exposures.get_explist( fltlist, outroot=outroot )
if not epochlistfile :
epochlistfile = "%s_epochs.txt"%explist[0].outroot
if os.path.exists( epochlistfile ) and not clobber :
print( "%s exists. Adopting existing epoch sorting."%epochlistfile )
exposures.read_epochs( explist, epochlistfile, checkradec=[ra,dec],
onlyfilters=onlyfilters )
else :
exposures.define_epochs( explist, epochspan=epochspan,
mjdmin=mjdmin, mjdmax=mjdmax )
exposures.print_epochs( explist, outfile=epochlistfile,
verbose=verbose, clobber=clobber, checkradec=[ra,dec],
onlyfilters=onlyfilters, onlyepochs=onlyepochs )
if refim and not os.path.exists( refim ) :
raise exceptions.RuntimeError( 'Ref image %s does not exist.'%refim )
if not refim :
# No refimage has been specified, so set the default refimage name
refdrzdir = '%s.refim'%outroot
refimbasename = '%s_wcsref_sci.fits'%(outroot)
refim = os.path.abspath( os.path.join( refdrzdir, refimbasename ) )
new_explist = []
for exp in explist:
if onlyfilters and exp.filter not in onlyfilters :
continue
elif onlyepochs and exp.epoch not in onlyepochs :
continue
else:
new_explist.append( exp )
explist = new_explist
# STAGE 1 :
# copy pristine flt files into epoch sub-directories
if dosetup :
if verbose :
print("SNDRIZZLE : (1) SETUP : copying flt files into subdirs")
exposures.copy_to_epochdirs( explist, checkradec=[ra,dec],
onlyfilters=onlyfilters, onlyepochs=onlyepochs,
verbose=verbose, clobber=clobber )
FEVgrouplist = sorted( np.unique( [ exp.FEVgroup for exp in explist ] ) )
FEgrouplist = sorted( np.unique( [ exp.FEgroup for exp in explist ] ) )
filterlist = sorted( np.unique( [ exp.filter for exp in explist ] ) )
epochlist = sorted( np.unique([ exp.epoch for exp in explist ] ) )
# STAGE 2 :
# Construct the WCS reference image
if dorefim :
if verbose :
print("SNDRIZZLE : (2) REFIM : Constructing WCS ref image.")
if os.path.exists( refim ) and clobber :
os.remove( refim )
if os.path.exists( refim ) :
print("%s already exists. Not clobbering."%refim)
else :
refdrzdir = os.path.dirname( refim )
if verbose :
print( " Constructing reference image %s in %s"%(
os.path.basename(refim), refdrzdir ) )
# Collect the necessary flt files for constructing the ref image
if not refepoch :
refepoch = np.min( epochlist )
if not reffilter :
reffilter = sorted( [ exp.filter for exp in explist
if exp.epoch==refepoch ] )[0]
reffilter = reffilter.lower()
# if refvisit is not set, then find the maximum depth visit and use that
if not refvisit :
visits, exp_times, visit_depth = np.array( ([], [], []) )
for exp in explist:
if (exp.epoch != refepoch) and (exp.filter != reffilter):
continue
visits = np.append( visits, exp.visit )
exp_times = np.append( exp_times, exp.exposure_time )
unique_visits = np.unique( visits )
for u in unique_visits:
ind = np.where( visits == u )
visit_depth = np.append( visit_depth, np.sum(exp_times[ind]) )
max_ind = np.argmax( visit_depth )
refvisit = unique_visits[max_ind]
refvisit = refvisit.upper()
explistRI = sorted( [ exp for exp in explist if exp.epoch==refepoch
and exp.filter==reffilter and exp.visit==refvisit ] )
refdrzdir = os.path.dirname( refim )
if not os.path.isdir( refdrzdir ) :
os.makedirs( refdrzdir )
for exp in explistRI :
fltfile = os.path.basename( exp.filename )
refsrcdir = os.path.abspath( exp.epochdir )
shutil.copy( os.path.join( refsrcdir, fltfile ), refdrzdir )
fltlistRI = [ exp.filename for exp in explistRI ]
refimroot = '%s_wcsref'%outroot
os.chdir( refdrzdir )
refimsci, refimwht = drizzle.secondDrizzle(
fltlistRI, refimroot, refimage=None, ra=ra, dec=dec, rot=rot,
imsize_arcsec=imsize_arcsec, wht_type=wht_type,
pixscale=pixscale, pixfrac=pixfrac,
clobber=clobber, verbose=verbose, debug=debug )
os.rename(refimsci,refim)
os.chdir(topdir)
print refvisit
# STAGE 3 :
# Drizzle together each drizzle group (same epoch, visit and
# filter), using almost-default parameters, doing CR rejection
# and applying intravisit registrations if requested. The output
# is a drz_sci.fits file in the native rotation.
if dodriz1 :
if verbose :
print("SNDRIZZLE : (3) DRIZ1 : first astrodrizzle pass.")
for FEVgroup in FEVgrouplist :
explistFEV = [ exp for exp in explist if exp.FEVgroup == FEVgroup ]
thisepoch = explistFEV[0].epoch
thisfilter = explistFEV[0].filter
if onlyepochs and thisepoch not in onlyepochs : continue
if onlyfilters and thisfilter not in onlyfilters : continue
epochdir = explistFEV[0].epochdir
fltlistFEV = [ exp.filename for exp in explistFEV ]
outrootFEV = '%s_%s_nat'%(outroot,FEVgroup)
os.chdir( epochdir )
outsciFEV = '%s_%s_sci.fits'%(outrootFEV,explistFEV[0].drzsuffix)
if os.path.exists( outsciFEV ) and clobber :
os.remove( outsciFEV )
if os.path.exists( outsciFEV ) :
if verbose: print("%s exists. Not clobbering."%outsciFEV )
os.chdir( topdir )
continue
if intravisitreg :
# run tweakreg for intravisit registration tweaks
register.intraVisit(
fltlistFEV, peakmin=peakmin, peakmax=peakmax,
threshold=threshold, interactive=interactive, debug=debug )
drizzle.firstDrizzle(
fltlistFEV, outrootFEV, driz_cr=drizcr,
wcskey=((intravisitreg and 'INTRAVIS') or '') )
# TODO : Update the WCS of the refim so that it matches the reference catalog
#if refcat :
# if verbose : print( " Registering reference image %s to ref catalog %s"%(refim,refcat))
# register.toCatalog( refim, refcat, refim, rfluxmax=rfluxmax, rfluxmin=rfluxmin,
# searchrad=searchrad, peakmin=peakmin, peakmax=peakmax, threshold=threshold,
# interactive=interactive, debug=debug )
os.chdir( topdir )
if doreg or dodriz2 :
if not os.path.exists( refim ):
raise exceptions.RuntimeError("No refim file %s! Maybe you should re-run with dorefim=True."%refim)
else :
refim = os.path.abspath( refim )
# Fix the output ra and dec center point if not provided by the user.
if not ra and not dec :
ra = pyfits.getval( refim, "CRVAL1" )
dec = pyfits.getval( refim, "CRVAL2" )
# STAGE 4 :
# Run tweakreg to register all the single-visit drz images
# to a common WCS defined by the refcat/refim, updating the drz file headers.
# Then use tweakback to propagate that back into the flt files
if doreg :
if verbose :
print("SNDRIZZLE : (4) REG : running tweakreg.")
for FEVgroup in FEVgrouplist :
explistFEV = [ exp for exp in explist if exp.FEVgroup == FEVgroup ]
thisepoch = explistFEV[0].epoch
thisfilter = explistFEV[0].filter
if onlyepochs and thisepoch not in onlyepochs : continue
if onlyfilters and thisfilter not in onlyfilters : continue
epochdir = explistFEV[0].epochdir
fltlistFEV = [ exp.filename for exp in explistFEV ]
outrootFEV = '%s_%s_nat'%(outroot,FEVgroup)
outsciFEV = '%s_%s_sci.fits'%(outrootFEV,explistFEV[0].drzsuffix)
refimpath = os.path.abspath(refim)
if refcat : refcatpath = os.path.abspath(refcat)
else : refcatpath = None
os.chdir( epochdir )
if not os.path.exists( outsciFEV ) :
exceptions.RuntimeError( "Missing %s."%outsciFEV )
# register to the ref image and ref catalog
origwcs = pyfits.getval( outsciFEV,'WCSNAME').strip()
wcsname = register.toRefim(
outsciFEV, refim=refimpath, refcat=refcatpath,
searchrad=searchrad, peakmin=peakmin, peakmax=peakmax,
threshold=threshold,
interactive=interactive, clobber=clobber, debug=debug )
# Run tweakback to update the constituent flts
tweakback( outsciFEV, input=fltlistFEV, origwcs=origwcs,
wcsname=wcsname, verbose=verbose, force=clobber )
os.chdir(topdir)
# STAGE 5
# Second and final astrodrizzle pass, wherein we rerun
# astrodrizzle to get wcs- and pixel-registered drz images
# combining all flt files with the same filter and epoch.
if dodriz2 :
if verbose :
print("SNDRIZZLE : (5) DRIZ2 : second astrodrizzle pass.")
for FEgroup in FEgrouplist :
explistFE = [ exp for exp in explist if exp.FEgroup == FEgroup ]
thisepoch = explistFE[0].epoch
thisfilter = explistFE[0].filter
if onlyepochs and thisepoch not in onlyepochs : continue
if onlyfilters and thisfilter not in onlyfilters : continue
epochdir = explistFE[0].epochdir
fltlistFE = [ exp.filename for exp in explistFE ]
outrootFE = '%s_%s_reg'%(outroot,FEgroup)
err = None
os.chdir( epochdir )
outsciFE = '%s_%s_sci.fits'%(outrootFE,explistFE[0].drzsuffix)
if os.path.exists( outsciFE ) and clobber :
os.remove( outsciFE )
if os.path.exists( outsciFE ) :
if verbose: print("%s exists. Not clobbering."%outsciFE )
os.chdir( topdir )
continue
outsciFE, outwhtFE = drizzle.secondDrizzle(
fltlistFE, outrootFE, refimage=refim, ra=ra, dec=dec, rot=rot,
imsize_arcsec=imsize_arcsec, wht_type=wht_type,
pixscale=pixscale, pixfrac=pixfrac,
clobber=clobber, verbose=verbose, debug=debug )
outbpxFE = outwhtFE.replace('_wht','_bpx')
outbpxFE = badpix.zerowht2badpix(
outwhtFE, outbpxFE, verbose=verbose, clobber=clobber )
os.chdir( topdir )
# STAGE 6
# Define a template epoch for each filter and subtract it from the other epochs
if dodiff :
if verbose :
print("SNDRIZZLE : (6) DIFF : subtracting template images.")
for filter in filterlist :
if onlyfilters and filter not in onlyfilters : continue
template = None
if tempepoch == None :
# User did not define a template epoch, so we default
# to use the first available epoch
for epoch in epochlist :
explistFE = [ exp for exp in explist
if exp.filter==filter and exp.epoch==epoch ]
if len(explistFE)==0: continue
drzsuffix = explistFE[0].drzsuffix
epochdir = explistFE[0].epochdir
FEgroup = explistFE[0].FEgroup
outrootFE = '%s_%s'%(outroot, FEgroup )
if os.path.isfile( os.path.join( epochdir, outrootFE+"_reg_%s_sci.fits"%drzsuffix )) :
tempepoch = epoch
break
else :
explistF = [ exp for exp in explist if exp.filter==filter ]
drzsuffix = explistF[0].drzsuffix
# now do the subtractions
tempdir = outroot+'.e%02i'%tempepoch
tempsci = os.path.join( os.path.abspath(tempdir), '%s_%s_e%02i_reg_%s_sci.fits'%(outroot,filter,tempepoch,drzsuffix))
tempwht = tempsci.replace('sci.fits','wht.fits')
tempbpx = tempsci.replace('sci.fits','bpx.fits')
topdir = os.path.abspath( '.' )
for epoch in epochlist :
if onlyepochs and epoch not in onlyepochs : continue
if epoch == tempepoch : continue
explistFE = [ exp for exp in explist if exp.filter==filter and exp.epoch==epoch ]
if len(explistFE)==0 : continue
epochdirFE = explistFE[0].epochdir
outrootFE = "%s_%s"%(outroot,explistFE[0].FEgroup)
drzsuffix = explistFE[0].drzsuffix
thisregsci = "%s_reg_%s_sci.fits"%(outrootFE,drzsuffix)
thisregwht = "%s_reg_%s_wht.fits"%(outrootFE,drzsuffix)
thisregbpx = "%s_reg_%s_bpx.fits"%(outrootFE,drzsuffix)
os.chdir( epochdirFE )
if not os.path.isfile( thisregsci ) :
os.chdir( topdir )
continue
thisdiffim = outrootFE + "-e%02i_sub_sci.fits"%tempepoch
if not os.path.isfile( tempsci ) or not os.path.isfile( thisregsci) :
print( "Can't create diff image %s"%thisdiffim )
print( " missing either %s"%tempsci )
print( " or %s"%thisregsci )
os.chdir( topdir )
continue
diffim = imarith.imsubtract( tempsci, thisregsci, outfile=thisdiffim,
clobber=clobber, verbose=verbose, debug=debug)
diffwht = badpix.combine_ivm_maps( thisregwht, tempwht,
diffim.replace('sci.fits','wht.fits'),
clobber=clobber, verbose=verbose )
diffbpx = badpix.unionmask( tempbpx, thisregbpx,
diffim.replace('sci.fits','bpx.fits'),
clobber=clobber, verbose=verbose)
diffim_masked = badpix.applymask( diffim, diffbpx,
clobber=clobber, verbose=verbose)
print("Created diff image %s, wht map %s, and bpx mask %s"%(
diffim_masked, diffwht, diffbpx ) )
os.chdir( topdir )
return 0
