def prep():
import glob
import os
import threedhst
import threedhst.prep_flt_astrodrizzle as init
from threedhst import catIO
import unicorn
import research.hawkiff
### Make ACS associations
unicorn.candels.make_asn_files(uniquename=True, translate={'-ROT': ''})
threedhst.options['FLT_PERSISTENCE_PATH'] = '../Persistence/'
#### IR
files = glob.glob('*-F1*asn.fits')
radec = None
for asn_file in files:
init.prep_direct_grism_pair(direct_asn=asn_file,
grism_asn=None,
radec=radec,
ACS=False,
align_threshold=5,
raw_path='../RAW/',
order=0)
###
for asn_file in files:
drizzlepac.astrodrizzle.AstroDrizzle(
asn_file,
static=False,
skysub=False,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
final_wht_type='IVM',
clean=True,
final_wcs=True,
final_refimage='IRAS23436+5257-87-304-F814W_drc_sci.fits',
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=576,
preserve=False)
### Match WCS of NB observations
init.copy_adriz_headerlets(glob.glob('*-F1*[MW]*asn.fits')[0],
glob.glob('*-F1*[N]*asn.fits')[0],
order=[1, 0])
### Flag negative pixels
for asn_file in files:
asn = threedhst.utils.ASNFile(asn_file)
for exp in asn.exposures:
im = pyfits.open('%s_flt.fits' % (exp), mode='update')
neg = im['SCI'].data < -3 * im['ERR'].data
print '%s #negative = %d' % (exp, neg.sum())
im['DQ'].data[neg] |= 4096
im.flush()
#
drizzlepac.astrodrizzle.AstroDrizzle(asn_file,
skysub=True,
static=True,
driz_separate=True,
driz_sep_wcs=True,
median=True,
blot=True,
driz_cr=True,
driz_combine=True,
clean=True,
final_wcs=True,
final_scale=0.1,
final_rot=0,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=64,
preserve=False)
sci = pyfits.open(asn_file.replace('_asn', '_drz_sci'), mode='update')
wht = pyfits.open(asn_file.replace('_asn', '_drz_wht'))
sci[0].data[wht[0].data == 0] = 0
sci.flush()
#### ACS / UVIS
files = glob.glob('*-F6*asn.fits')
files = glob.glob('*-F[48]*asn.fits')
files.sort()
for asn_file in files[::-1]:
asn = threedhst.utils.ASNFile(asn_file)
for i in range(len(asn.exposures)):
asn.exposures[i] = asn.exposures[i].split('_flc')[0]
print asn_file, asn.exposures
asn.write(asn_file)
if ('814' in asn_file) | ('625' in asn_file):
radec = None
else:
root = glob.glob('*-F[86]*W*sci.fits')[0].split('_sci')[0]
if not os.path.exists('%s_sci.cat' % (root)):
os.system(
'cp /user/brammer/software/share/gauss_5.0_9x9.conv .')
research.hawkiff.make_catalog(root=root,
sci_ext='sci',
threshold=3,
use_rms=False,
subtract_background=True)
cat = catIO.Table('%s_sci.cat' % (root))
cat['X_WORLD',
'Y_WORLD'].write('ref_opt.radec',
format='ascii.commented_header')
radec = 'ref_opt.radec'
radec = None
init.prep_direct_grism_pair(direct_asn=asn_file,
grism_asn=None,
radec=radec,
ACS=True,
align_threshold=5,
raw_path='../RAW/')
#
for asn_file in files:
radec = None
init.prep_direct_grism_pair(direct_asn=asn_file,
grism_asn=None,
radec=radec,
ACS=True,
align_threshold=5,
raw_path='../RAW/')
files = glob.glob('*sci.fits')
ims = {}
for file in files:
im = pyfits.open(file)
filter = file.split('-')[-1][:5]
pix = np.sqrt(im[0].header['CD1_1']**2 +
im[0].header['CD1_2']**2) * 3600
scl = im[0].header['PHOTFLAM'] / 1.e-19 * (0.1 / pix)**2
im[0].data *= scl
ims[filter] = im
### Try Multidrizzle CR with combined image
import drizzlepac
asn_files = glob.glob('*-F[4-8]*asn.fits')
exposures = []
for file in asn_files:
asn = threedhst.utils.ASNFile(file)
exposures.extend(asn.exposures)
target = '-'.join(file.split('-')[:-1])
asn.exposures = exposures
asn.product = target
asn_file = '%s_asn.fits' % (asn.product)
asn.write(asn_file, clobber=True)
### modify: make copy of FLCs, make new mosaic and then copy FLCs back
##xxx
drizzlepac.astrodrizzle.AstroDrizzle(asn_file,
skysub=False,
clean=True,
final_wcs=True,
final_scale=0.05,
final_pixfrac=0.8,
context=False,
resetbits=4096,
final_bits=576,
preserve=False)
#
bits = 576
drizzlepac.astrodrizzle.AstroDrizzle(asn_file,
static=False,
skysub=False,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
final_wht_type='IVM',
clean=True,
final_wcs=True,
final_rot=0,
final_scale=0.1,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=bits,
preserve=False)
#### rerun final mosaics
refimage = '%s_drz_sci.fits' % (target)
for file in asn_files:
drizzlepac.astrodrizzle.AstroDrizzle(file,
static=False,
skysub=False,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
final_wht_type='IVM',
clean=True,
final_wcs=True,
final_refimage=refimage,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=bits,
preserve=False)
for file in asn_files:
print file
sci = pyfits.open(glob.glob(file.replace('_asn', '_dr?_sci'))[0],
mode='update')
wht = pyfits.open(sci.filename().replace('_sci', '_wht'))
sci[0].data[wht[0].data == 0] = 0
sci.flush()
#### Try own driz CR
import stwcs
from drizzlepac import astrodrizzle, quickDeriv
import scipy.ndimage as nd
ref_file = 'ESO550-IG02-13-083-F435W_drc_sci.fits'
ref_file = 'ESO550-IG02-13-083-F814W_drc_sci.fits'
ref_file = 'ESO550-IG02-13-083_drc_sci.fits'
ref_ext = 0
ref = pyfits.open(ref_file)
ref_wcs = stwcs.wcsutil.HSTWCS(ref, ext=ref_ext)
asn_file = 'ESO550-IG02-13-083-F814W_asn.fits'
asn = threedhst.utils.ASNFile(asn_file)
for exp in asn.exposures:
flt = pyfits.open('%s_flc.fits' % (exp)) #, mode='update')
for ext in [1, 2]:
flt_wcs = stwcs.wcsutil.HSTWCS(flt, ext=('sci', ext))
blotted_ref = astrodrizzle.ablot.do_blot(ref[ref_ext].data,
ref_wcs,
flt_wcs,
1,
coeffs=True,
interp='nearest',
sinscl=1.0,
stepsize=10,
wcsmap=None)
#
blotDeriv = quickDeriv.qderiv(blotted_ref)
scl = flt['sci', ext].header['PHOTFLAM'] / ref[0].header[
'PHOTFLAM'] / flt[0].header['EXPTIME']
noise = flt['err', ext].data
driz_scale = 1.2
driz_snr = 3.5
driz_scale = 1.0
driz_snr = 6
driz_f = 2
# |data_image - blotted_image| > scale x deriv + SNR x noise
abs = np.abs(flt['sci', ext].data * scl - blotted_ref * driz_f)
xderiv = driz_scale * blotDeriv
xsnr = driz_snr * scl * noise
mask = abs > (xderiv + xsnr)
new_dq = mask & ((flt['dq', ext].data & 4096) == 0)
### Try "cosmics" LA Cosmic
im = pyfits.open('ESO550-IG02-13-083-F814W_drc_sci.fits')
slx, sly = slice(1500, 2600), slice(1800, 2800)
ext = 0
subim = im[ext].data[sly, slx]
h = im[ext].header
subim *= im[0].header['EXPTIME']
#### FLT
im = pyfits.open('j9cv13pcq_flc.fits')
ext = 0
slx, sly = slice(1500, 2600), slice(0, 400)
subim = im[4].data[sly, slx]
h = im[0].header
import cosmics
c = cosmics.cosmicsimage(subim,
pssl=20.0,
gain=1,
readnoise=h['READNSEA'],
sigclip=4.0,
sigfrac=0.3,
objlim=3.8,
satlevel=84700.0,
verbose=True)
c.run(maxiter=4)
crflux = subim * c.mask
plt.hist(np.log10(crflux[c.mask]), range=[1, 6], bins=100, alpha=0.5)
####
import research.pab.pab
files = glob.glob('*-F*asn.fits')
#files=glob.glob('IC*-F814*asn.fits')
for file in files:
asn = threedhst.utils.ASNFile(file)
for exp in asn.exposures:
flc = pyfits.open('%s_flc.fits' % (exp))
research.pab.pab.run_lacosmic('%s_flc.fits' % (exp),
split=2048,
sigclip=5,
pssl=flc[1].header['MDRIZSK0'])
#
drizzlepac.updatehdr.update_from_shiftfile(
file.replace('asn.fits', 'shifts.txt'))
drizzlepac.astrodrizzle.AstroDrizzle(file,
skysub=True,
clean=True,
final_wcs=True,
final_scale=0.05,
final_pixfrac=0.8,
context=False,
resetbits=0,
final_bits=576,
preserve=False)
drizzlepac.astrodrizzle.AstroDrizzle(file,
output='sub_' +
file.split('_asn')[0],
skysub=True,
static=True,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
clean=True,
final_wcs=True,
final_scale=0.08,
final_ra=60.635233,
final_dec=60.344597,
final_outnx=800,
final_outny=800,
final_rot=0,
final_pixfrac=0.8,
context=False,
resetbits=0,
final_bits=576,
preserve=False)
### Final mosaics
f110w = glob.glob('*110W*asn.fits')[0]
drizzlepac.astrodrizzle.AstroDrizzle(f110w,
output='sub_' +
f110w.split('_asn')[0],
skysub=True,
static=True,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
clean=True,
final_wcs=True,
final_scale=0.1,
final_rot=0,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=576 - 512,
preserve=False)
files = glob.glob('*-F*asn.fits')
for file in files:
if 'F110W' in file:
continue
#
drizzlepac.astrodrizzle.AstroDrizzle(
file,
output='sub_' + file.split('_asn')[0],
skysub=True,
static=True,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
clean=True,
final_wcs=True,
final_refimage='sub_' + f110w.replace('_asn', '_drz_sci'),
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=576 - 512 * ('-F1' in file),
preserve=False)
for sci_im in glob.glob('sub*sci.fits'):
print sci_im
sci = pyfits.open(sci_im, mode='update')
wht = pyfits.open(sci_im.replace('_sci', '_wht'))
sci[0].data[wht[0].data == 0] = 0
sci.flush()
#### Test alignment
import align
ra_list, de_list = np.loadtxt('f814w.radec', unpack=True)
for geom in ['shift', 'shift', 'rxyscale', 'shift', 'shift']:
align.get_align_to_subaru(
sci='sub_ESO550-IG025-41-115-F673N_drc_sci.fits',
wht='sub_ESO550-IG025-41-115-F673N_drc_wht.fits',
verbose=False,
fitgeometry=geom,
align_data=(ra_list, de_list, ra_list * 0),
THRESH=1.2)
#drizzlepac.tweakback.tweakback('ESO550-IG025-41-115-F673N_drc_sci.fits', force=True, verbose=True) #, origwcs='DRZWCS')
ra_list, de_list = np.loadtxt('f110w.radec', unpack=True)
for geom in ['shift', 'shift', 'rxyscale', 'shift', 'shift']:
align.get_align_to_subaru(
sci='sub_ESO550-IG025-41-115-F132N_drz_sci.fits',
wht='sub_ESO550-IG025-41-115-F132N_drz_wht.fits',
verbose=False,
fitgeometry=geom,
align_data=(ra_list, de_list, ra_list * 0),
THRESH=1.2)
def prep_direct_grism_pair(direct_asn='goodss-34-F140W_asn.fits', grism_asn='goodss-34-G141_asn.fits', radec=None, raw_path='../RAW/', mask_grow=18, scattered_light=False, final_scale=None, skip_direct=False, ACS=False, jump=False, order=2, get_shift=True, align_threshold=20, column_average=True, sky_iter=3, run_acs_lacosmic=False):
"""
Process both the direct and grism observations of a given visit
"""
import threedhst.prep_flt_astrodrizzle as prep
import drizzlepac
from stwcs import updatewcs
import time
t0 = time.time()
#direct_asn='goodss-34-F140W_asn.fits'; grism_asn='goodss-34-G141_asn.fits'; radec=None; raw_path='../RAW/'
#radec = os.getenv('THREEDHST') + '/ASTRODRIZZLE_FLT/Catalog/goodss_radec.dat'
################################
#### Direct image processing
################################
#### xx add astroquery 2MASS/SDSS workaround for radec=None
if not skip_direct:
#### Get fresh FLTS from ../RAW/
asn = threedhst.utils.ASNFile(direct_asn)
if ACS:
for exp in asn.exposures:
print 'cp %s/%s_flc.fits.gz .' %(raw_path, exp)
os.system('cp %s/%s_flc.fits.gz .' %(raw_path, exp))
os.system('gunzip -f %s_flc.fits.gz' %(exp))
if run_acs_lacosmic:
try:
import lacosmicx
status = True
except:
print 'import lacosmicx failed!'
status = False
if status:
im = pyfits.open('%s_flc.fits' %(exp), mode='update')
for ext in [1,2]:
indata = im['SCI',ext].data
#inmask = im['DQ',ext].data > 0
if im['SCI',ext].header['BUNIT'] == 'ELECTRONS':
gain = 1
else:
gain = 1./im[0].header['EXPTIME']
if 'MDRIZSK0' in im['SCI',ext].header:
pssl = im['SCI',ext].header['MDRIZSK0']
else:
pssl = 0.
if 'FLASHLVL' in im[0].header:
pssl += im[0].header['FLASHLVL']
sig_scale = 1.8
else:
sig_scale = 1.
out = lacosmicx.lacosmicx(indata, inmask=None,
sigclip=3.5*sig_scale, sigfrac=0.2,
objlim=7.0, gain=gain,
readnoise=im[0].header['READNSEA'],
satlevel=np.inf, pssl=pssl, niter=5,
sepmed=True, cleantype='meanmask',
fsmode='median', psfmodel='gauss',
psffwhm=2.5,psfsize=7, psfk=None,
psfbeta=4.765, verbose=True)
crmask, cleanarr = out
im['DQ',ext].data |= 16*crmask
### Low pixels
if im[0].header['INSTRUME'] == 'WFC3':
bad = im['SCI',ext].data < -4*im['ERR',ext].data
im['DQ',ext].data |= 16*bad
im.flush()
else:
threedhst.process_grism.fresh_flt_files(direct_asn, from_path=raw_path)
if (not ACS):
#### Subtract WFC3/IR direct backgrounds
prep.subtract_flt_background(root=direct_asn.split('_asn')[0], scattered_light=scattered_light, order=order)
#### Flag IR CRs again within runTweakReg
#### Run TweakReg
if (radec is None) & (not ACS):
print len(asn.exposures)
if len(asn.exposures) > 1:
drizzlepac.astrodrizzle.AstroDrizzle(direct_asn, clean=True, final_scale=None, final_pixfrac=0.8, context=False, final_bits=576, preserve=False, driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7')
else:
drizzlepac.astrodrizzle.AstroDrizzle(direct_asn, clean=True, final_scale=None, final_pixfrac=1, context=False, final_bits=576, preserve=False, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_cr_corr=False, driz_combine=True)
else:
if get_shift:
prep.runTweakReg(asn_file=direct_asn, master_catalog=radec, final_scale=None, ACS=ACS, threshold=align_threshold)
#### Subtract background of direct ACS images
if ACS:
for exp in asn.exposures:
flc = pyfits.open('%s_flc.fits' %(exp), mode='update')
if 'SUB' in flc[0].header['APERTURE']:
extensions = [1]
else:
extensions = [1,4]
for ext in extensions:
threedhst.showMessage('Subtract background from %s_flc.fits[%d] : %.4f' %(exp, ext, flc[ext].header['MDRIZSKY']))
flc[ext].data -= flc[ext].header['MDRIZSKY']
flc[ext].header['MDRIZSK0'] = flc[ext].header['MDRIZSKY']
flc[ext].header['MDRIZSKY'] = 0.
#
flc.flush()
else:
pass
#### Do this later, gives segfaults here???
#prep.subtract_flt_background(root=direct_asn.split('_asn')[0], scattered_light=scattered_light)
#### Flag CRs again on BG-subtracted image
#drizzlepac.astrodrizzle.AstroDrizzle(direct_asn, clean=True, final_scale=None, final_pixfrac=0.8, context=False, final_bits=576, preserve=False, driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7') # ,
################################
#### Grism image processing
################################
if grism_asn:
asn = threedhst.utils.ASNFile(grism_asn)
if ACS:
for exp in asn.exposures:
print 'cp %s/%s_flc.fits.gz .' %(raw_path, exp)
os.system('cp %s/%s_flc.fits.gz .' %(raw_path, exp))
os.system('gunzip -f %s_flc.fits.gz' %(exp))
updatewcs.updatewcs('%s_flc.fits' %(exp))
prep.copy_adriz_headerlets(direct_asn=direct_asn, grism_asn=grism_asn, ACS=True)
prep.subtract_acs_grism_background(asn_file=grism_asn, final_scale=None)
else:
#### Remove the sky and flag CRs
## with mask from rough zodi-only subtraction
prep.subtract_grism_background(asn_file=grism_asn, PATH_TO_RAW='../RAW/', final_scale=None, visit_sky=True, column_average=False, mask_grow=mask_grow, first_run=True)
## Redo making mask from better combined image
prep.subtract_grism_background(asn_file=grism_asn, PATH_TO_RAW='../RAW/', final_scale=final_scale, visit_sky=True, column_average=column_average, mask_grow=mask_grow, first_run=False, sky_iter=sky_iter)
#### Copy headers from direct images
if radec is not None:
prep.copy_adriz_headerlets(direct_asn=direct_asn, grism_asn=grism_asn, ACS=False)
#### Run CR rejection with final shifts
drizzlepac.astrodrizzle.AstroDrizzle(grism_asn, clean=True, skysub=False, final_wcs=True, final_scale=final_scale, final_pixfrac=0.8, context=False, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale='2.5 0.7') # driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7')
if not grism_asn:
t1 = time.time()
threedhst.showMessage('direct: %s\n\nDone (%d s).' %(direct_asn, int(t1-t0)))
else:
t1 = time.time()
threedhst.showMessage('direct: %s\ngrism: %s\n\nDone (%d s).' %(direct_asn, grism_asn, int(t1-t0)))
def prep_direct_grism_pair(direct_asn='goodss-34-F140W_asn.fits', grism_asn='goodss-34-G141_asn.fits', radec=None, raw_path='../RAW/', mask_grow=18, scattered_light=False, final_scale=None, skip_direct=False, ACS=False, jump=False, order=2, get_shift=True, align_threshold=20, column_average=True, sky_iter=3):
"""
Process both the direct and grism observations of a given visit
"""
import threedhst.prep_flt_astrodrizzle as prep
import drizzlepac
from stwcs import updatewcs
import time
t0 = time.time()
#direct_asn='goodss-34-F140W_asn.fits'; grism_asn='goodss-34-G141_asn.fits'; radec=None; raw_path='../RAW/'
#radec = os.getenv('THREEDHST') + '/ASTRODRIZZLE_FLT/Catalog/goodss_radec.dat'
################################
#### Direct image processing
################################
#### xx add astroquery 2MASS/SDSS workaround for radec=None
if not skip_direct:
#### Get fresh FLTS from ../RAW/
asn = threedhst.utils.ASNFile(direct_asn)
if ACS:
for exp in asn.exposures:
print 'cp %s/%s_flc.fits.gz .' %(raw_path, exp)
os.system('cp %s/%s_flc.fits.gz .' %(raw_path, exp))
os.system('gunzip %s_flc.fits.gz' %(exp))
else:
threedhst.process_grism.fresh_flt_files(direct_asn, from_path=raw_path)
if (not ACS):
#### Subtract WFC3/IR direct backgrounds
prep.subtract_flt_background(root=direct_asn.split('_asn')[0], scattered_light=scattered_light, order=order)
#### Flag IR CRs again within runTweakReg
#### Run TweakReg
if (radec is None) & (not ACS):
drizzlepac.astrodrizzle.AstroDrizzle(direct_asn, clean=True, final_scale=None, final_pixfrac=0.8, context=False, final_bits=576, preserve=False, driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7') # ,
else:
if get_shift:
prep.runTweakReg(asn_file=direct_asn, master_catalog=radec, final_scale=None, ACS=ACS, threshold=align_threshold)
#### Subtract background of direct ACS images
if ACS:
for exp in asn.exposures:
flc = pyfits.open('%s_flc.fits' %(exp), mode='update')
for ext in [1,4]:
threedhst.showMessage('Subtract background from %s_flc.fits[%d] : %.4f' %(exp, ext, flc[ext].header['MDRIZSKY']))
flc[ext].data -= flc[ext].header['MDRIZSKY']
flc[ext].header['MDRIZSK0'] = flc[ext].header['MDRIZSKY']
flc[ext].header['MDRIZSKY'] = 0.
#
flc.flush()
else:
pass
#### Do this later, gives segfaults here???
#prep.subtract_flt_background(root=direct_asn.split('_asn')[0], scattered_light=scattered_light)
#### Flag CRs again on BG-subtracted image
#drizzlepac.astrodrizzle.AstroDrizzle(direct_asn, clean=True, final_scale=None, final_pixfrac=0.8, context=False, final_bits=576, preserve=False, driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7') # ,
################################
#### Grism image processing
################################
if grism_asn:
asn = threedhst.utils.ASNFile(grism_asn)
if ACS:
for exp in asn.exposures:
print 'cp %s/%s_flc.fits.gz .' %(raw_path, exp)
os.system('cp %s/%s_flc.fits.gz .' %(raw_path, exp))
os.system('gunzip %s_flc.fits.gz' %(exp))
updatewcs.updatewcs('%s_flc.fits' %(exp))
prep.copy_adriz_headerlets(direct_asn=direct_asn, grism_asn=grism_asn, ACS=True)
prep.subtract_acs_grism_background(asn_file=grism_asn, final_scale=None)
else:
#### Remove the sky and flag CRs
## with mask from rough zodi-only subtraction
prep.subtract_grism_background(asn_file=grism_asn, PATH_TO_RAW='../RAW/', final_scale=None, visit_sky=True, column_average=False, mask_grow=mask_grow, first_run=True)
## Redo making mask from better combined image
prep.subtract_grism_background(asn_file=grism_asn, PATH_TO_RAW='../RAW/', final_scale=final_scale, visit_sky=True, column_average=column_average, mask_grow=mask_grow, first_run=False, sky_iter=sky_iter)
#### Copy headers from direct images
if radec is not None:
prep.copy_adriz_headerlets(direct_asn=direct_asn, grism_asn=grism_asn, ACS=False)
#### Run CR rejection with final shifts
drizzlepac.astrodrizzle.AstroDrizzle(grism_asn, clean=True, skysub=False, final_wcs=True, final_scale=final_scale, final_pixfrac=0.8, context=False, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale='2.5 0.7') # driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7')
if not grism_asn:
t1 = time.time()
threedhst.showMessage('direct: %s\n\nDone (%d s).' %(direct_asn, int(t1-t0)))
else:
t1 = time.time()
threedhst.showMessage('direct: %s\ngrism: %s\n\nDone (%d s).' %(direct_asn, grism_asn, int(t1-t0)))
def prep():
import glob
import os
import threedhst
import threedhst.prep_flt_astrodrizzle as init
from threedhst import catIO
import unicorn
import research.hawkiff
### Make ACS associations
unicorn.candels.make_asn_files(uniquename=True, translate={'-ROT': ''})
threedhst.options['FLT_PERSISTENCE_PATH'] = '../Persistence/'
#### IR
files = glob.glob('*-F1*asn.fits')
radec = None
for asn_file in files:
init.prep_direct_grism_pair(
direct_asn=asn_file,
grism_asn=None,
radec=radec,
ACS=False,
align_threshold=5,
raw_path='../RAW/',
order=0)
###
for asn_file in files:
drizzlepac.astrodrizzle.AstroDrizzle(
asn_file,
static=False,
skysub=False,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
final_wht_type='IVM',
clean=True,
final_wcs=True,
final_refimage='IRAS23436+5257-87-304-F814W_drc_sci.fits',
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=576,
preserve=False)
### Match WCS of NB observations
init.copy_adriz_headerlets(
glob.glob('*-F1*[MW]*asn.fits')[0],
glob.glob('*-F1*[N]*asn.fits')[0],
order=[1, 0])
### Flag negative pixels
for asn_file in files:
asn = threedhst.utils.ASNFile(asn_file)
for exp in asn.exposures:
im = pyfits.open('%s_flt.fits' % (exp), mode='update')
neg = im['SCI'].data < -3 * im['ERR'].data
print '%s #negative = %d' % (exp, neg.sum())
im['DQ'].data[neg] |= 4096
im.flush()
#
drizzlepac.astrodrizzle.AstroDrizzle(
asn_file,
skysub=True,
static=True,
driz_separate=True,
driz_sep_wcs=True,
median=True,
blot=True,
driz_cr=True,
driz_combine=True,
clean=True,
final_wcs=True,
final_scale=0.1,
final_rot=0,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=64,
preserve=False)
sci = pyfits.open(asn_file.replace('_asn', '_drz_sci'), mode='update')
wht = pyfits.open(asn_file.replace('_asn', '_drz_wht'))
sci[0].data[wht[0].data == 0] = 0
sci.flush()
#### ACS / UVIS
files = glob.glob('*-F6*asn.fits')
files = glob.glob('*-F[48]*asn.fits')
files.sort()
for asn_file in files[::-1]:
asn = threedhst.utils.ASNFile(asn_file)
for i in range(len(asn.exposures)):
asn.exposures[i] = asn.exposures[i].split('_flc')[0]
print asn_file, asn.exposures
asn.write(asn_file)
if ('814' in asn_file) | ('625' in asn_file):
radec = None
else:
root = glob.glob('*-F[86]*W*sci.fits')[0].split('_sci')[0]
if not os.path.exists('%s_sci.cat' % (root)):
os.system(
'cp /user/brammer/software/share/gauss_5.0_9x9.conv .')
research.hawkiff.make_catalog(
root=root,
sci_ext='sci',
threshold=3,
use_rms=False,
subtract_background=True)
cat = catIO.Table('%s_sci.cat' % (root))
cat['X_WORLD', 'Y_WORLD'].write(
'ref_opt.radec', format='ascii.commented_header')
radec = 'ref_opt.radec'
radec = None
init.prep_direct_grism_pair(
direct_asn=asn_file,
grism_asn=None,
radec=radec,
ACS=True,
align_threshold=5,
raw_path='../RAW/')
#
for asn_file in files:
radec = None
init.prep_direct_grism_pair(
direct_asn=asn_file,
grism_asn=None,
radec=radec,
ACS=True,
align_threshold=5,
raw_path='../RAW/')
files = glob.glob('*sci.fits')
ims = {}
for file in files:
im = pyfits.open(file)
filter = file.split('-')[-1][:5]
pix = np.sqrt(im[0].header['CD1_1']**2 +
im[0].header['CD1_2']**2) * 3600
scl = im[0].header['PHOTFLAM'] / 1.e-19 * (0.1 / pix)**2
im[0].data *= scl
ims[filter] = im
### Try Multidrizzle CR with combined image
import drizzlepac
asn_files = glob.glob('*-F[4-8]*asn.fits')
exposures = []
for file in asn_files:
asn = threedhst.utils.ASNFile(file)
exposures.extend(asn.exposures)
target = '-'.join(file.split('-')[:-1])
asn.exposures = exposures
asn.product = target
asn_file = '%s_asn.fits' % (asn.product)
asn.write(asn_file, clobber=True)
### modify: make copy of FLCs, make new mosaic and then copy FLCs back
##xxx
drizzlepac.astrodrizzle.AstroDrizzle(
asn_file,
skysub=False,
clean=True,
final_wcs=True,
final_scale=0.05,
final_pixfrac=0.8,
context=False,
resetbits=4096,
final_bits=576,
preserve=False)
#
bits = 576
drizzlepac.astrodrizzle.AstroDrizzle(
asn_file,
static=False,
skysub=False,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
final_wht_type='IVM',
clean=True,
final_wcs=True,
final_rot=0,
final_scale=0.1,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=bits,
preserve=False)
#### rerun final mosaics
refimage = '%s_drz_sci.fits' % (target)
for file in asn_files:
drizzlepac.astrodrizzle.AstroDrizzle(
file,
static=False,
skysub=False,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
final_wht_type='IVM',
clean=True,
final_wcs=True,
final_refimage=refimage,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=bits,
preserve=False)
for file in asn_files:
print file
sci = pyfits.open(
glob.glob(file.replace('_asn', '_dr?_sci'))[0], mode='update')
wht = pyfits.open(sci.filename().replace('_sci', '_wht'))
sci[0].data[wht[0].data == 0] = 0
sci.flush()
#### Try own driz CR
import stwcs
from drizzlepac import astrodrizzle, quickDeriv
import scipy.ndimage as nd
ref_file = 'ESO550-IG02-13-083-F435W_drc_sci.fits'
ref_file = 'ESO550-IG02-13-083-F814W_drc_sci.fits'
ref_file = 'ESO550-IG02-13-083_drc_sci.fits'
ref_ext = 0
ref = pyfits.open(ref_file)
ref_wcs = stwcs.wcsutil.HSTWCS(ref, ext=ref_ext)
asn_file = 'ESO550-IG02-13-083-F814W_asn.fits'
asn = threedhst.utils.ASNFile(asn_file)
for exp in asn.exposures:
flt = pyfits.open('%s_flc.fits' % (exp)) #, mode='update')
for ext in [1, 2]:
flt_wcs = stwcs.wcsutil.HSTWCS(flt, ext=('sci', ext))
blotted_ref = astrodrizzle.ablot.do_blot(
ref[ref_ext].data,
ref_wcs,
flt_wcs,
1,
coeffs=True,
interp='nearest',
sinscl=1.0,
stepsize=10,
wcsmap=None)
#
blotDeriv = quickDeriv.qderiv(blotted_ref)
scl = flt['sci', ext].header['PHOTFLAM'] / ref[0].header[
'PHOTFLAM'] / flt[0].header['EXPTIME']
noise = flt['err', ext].data
driz_scale = 1.2
driz_snr = 3.5
driz_scale = 1.0
driz_snr = 6
driz_f = 2
# |data_image - blotted_image| > scale x deriv + SNR x noise
abs = np.abs(flt['sci', ext].data * scl - blotted_ref * driz_f)
xderiv = driz_scale * blotDeriv
xsnr = driz_snr * scl * noise
mask = abs > (xderiv + xsnr)
new_dq = mask & ((flt['dq', ext].data & 4096) == 0)
### Try "cosmics" LA Cosmic
im = pyfits.open('ESO550-IG02-13-083-F814W_drc_sci.fits')
slx, sly = slice(1500, 2600), slice(1800, 2800)
ext = 0
subim = im[ext].data[sly, slx]
h = im[ext].header
subim *= im[0].header['EXPTIME']
#### FLT
im = pyfits.open('j9cv13pcq_flc.fits')
ext = 0
slx, sly = slice(1500, 2600), slice(0, 400)
subim = im[4].data[sly, slx]
h = im[0].header
import cosmics
c = cosmics.cosmicsimage(
subim,
pssl=20.0,
gain=1,
readnoise=h['READNSEA'],
sigclip=4.0,
sigfrac=0.3,
objlim=3.8,
satlevel=84700.0,
verbose=True)
c.run(maxiter=4)
crflux = subim * c.mask
plt.hist(np.log10(crflux[c.mask]), range=[1, 6], bins=100, alpha=0.5)
####
import research.pab.pab
files = glob.glob('*-F*asn.fits')
#files=glob.glob('IC*-F814*asn.fits')
for file in files:
asn = threedhst.utils.ASNFile(file)
for exp in asn.exposures:
flc = pyfits.open('%s_flc.fits' % (exp))
research.pab.pab.run_lacosmic(
'%s_flc.fits' % (exp),
split=2048,
sigclip=5,
pssl=flc[1].header['MDRIZSK0'])
#
drizzlepac.updatehdr.update_from_shiftfile(
file.replace('asn.fits', 'shifts.txt'))
drizzlepac.astrodrizzle.AstroDrizzle(
file,
skysub=True,
clean=True,
final_wcs=True,
final_scale=0.05,
final_pixfrac=0.8,
context=False,
resetbits=0,
final_bits=576,
preserve=False)
drizzlepac.astrodrizzle.AstroDrizzle(
file,
output='sub_' + file.split('_asn')[0],
skysub=True,
static=True,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
clean=True,
final_wcs=True,
final_scale=0.08,
final_ra=60.635233,
final_dec=60.344597,
final_outnx=800,
final_outny=800,
final_rot=0,
final_pixfrac=0.8,
context=False,
resetbits=0,
final_bits=576,
preserve=False)
### Final mosaics
f110w = glob.glob('*110W*asn.fits')[0]
drizzlepac.astrodrizzle.AstroDrizzle(
f110w,
output='sub_' + f110w.split('_asn')[0],
skysub=True,
static=True,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
clean=True,
final_wcs=True,
final_scale=0.1,
final_rot=0,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=576 - 512,
preserve=False)
files = glob.glob('*-F*asn.fits')
for file in files:
if 'F110W' in file:
continue
#
drizzlepac.astrodrizzle.AstroDrizzle(
file,
output='sub_' + file.split('_asn')[0],
skysub=True,
static=True,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
clean=True,
final_wcs=True,
final_refimage='sub_' + f110w.replace('_asn', '_drz_sci'),
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=576 - 512 * ('-F1' in file),
preserve=False)
for sci_im in glob.glob('sub*sci.fits'):
print sci_im
sci = pyfits.open(sci_im, mode='update')
wht = pyfits.open(sci_im.replace('_sci', '_wht'))
sci[0].data[wht[0].data == 0] = 0
sci.flush()
#### Test alignment
import align
ra_list, de_list = np.loadtxt('f814w.radec', unpack=True)
for geom in ['shift', 'shift', 'rxyscale', 'shift', 'shift']:
align.get_align_to_subaru(
sci='sub_ESO550-IG025-41-115-F673N_drc_sci.fits',
wht='sub_ESO550-IG025-41-115-F673N_drc_wht.fits',
verbose=False,
fitgeometry=geom,
align_data=(ra_list, de_list, ra_list * 0),
THRESH=1.2)
#drizzlepac.tweakback.tweakback('ESO550-IG025-41-115-F673N_drc_sci.fits', force=True, verbose=True) #, origwcs='DRZWCS')
ra_list, de_list = np.loadtxt('f110w.radec', unpack=True)
for geom in ['shift', 'shift', 'rxyscale', 'shift', 'shift']:
align.get_align_to_subaru(
sci='sub_ESO550-IG025-41-115-F132N_drz_sci.fits',
wht='sub_ESO550-IG025-41-115-F132N_drz_wht.fits',
verbose=False,
fitgeometry=geom,
align_data=(ra_list, de_list, ra_list * 0),
THRESH=1.2)
