def fetch_nearby(radec=[40.055709, -1.6262032],
r=3,
instruments=['WFC3'],
min_exptime=50):
from threedhst import catIO
dd = r / 60.
dr = dd / np.cos(radec[1] / 180 * np.pi)
instrument_query = ' OR '.join([
'INSTRUMENT.INSTRUMENT_NAME = \'%s\'' % (inst) for inst in instruments
])
url = r"http://archives.esac.esa.int/ehst-sl-server/servlet/metadata-action?RESOURCE_CLASS=SIMPLE_OBSERVATION&SELECTED_FIELDS=SIMPLE_OBSERVATION,INSTRUMENT.INSTRUMENT_NAME,OPTICAL_ELEMENT.OPTICAL_ELEMENT_NAME,PROPOSAL.PROPOSAL_ID,TARGET.TARGET_NAME,&QUERY=(SIMPLE_OBSERVATION.INTENT = 'Science' AND (%s) AND (POSITION.DEC > %.4f AND POSITION.DEC < %.4f AND POSITION.RA > %.4f AND POSITION.RA < %.4f) AND (SIMPLE_OBSERVATION.EXPOSURE_DURATION > %.1f))&ORDER_BY=SIMPLE_OBSERVATION.START_TIME&RETURN_TYPE=CSV" % (
instrument_query, radec[1] - dd, radec[1] + dd, radec[0] - dr,
radec[0] + dr, min_exptime)
os.system('curl -o esa.csv "%s"' % (url.replace(' ', '%20')))
query = catIO.Table('esa.csv')
### Get unique visits
root = {'WFC3': 'I', 'ACS': 'J'}
base_names = np.unique([
'%s%s%s' % (root[l['INSTRUMENT_NAME']], l['PROGRAM_ID'], l['SET_ID'])
for l in query
])
for base in base_names:
for ext in [1, 2, 3, 4, 5, 6]:
for extn in ['ASN', 'JIT', 'JIF']:
file_base = "%s0%d0_%s" % (base, ext, extn)
file_url = r"http://archives.esac.esa.int/ehst-sl-server/servlet/data-action?ARTIFACT_ID=%s" % (
file_base)
os.system('curl -o %s.fits.gz "%s"' %
(file_base.lower(), file_url))
def mp_compute_models():
"""
Testing multiprocessing on multiple FLTs
"""
import multiprocessing as mp
import multiprocessing.queues
import mywfc3.flt
import mywfc3.reprocess_wfc3
import time
global FLTs
FLTs = {}
keys = range(8)
### testing pickling
if False:
self = mywfc3.flt.model.GrismFLT(file='icka01t7q_flt.fits', refimage='F160W_mosaic.fits', segimage='F160W_seg.fits')
csex = catIO.Table('/Users/brammer/3DHST/Spectra/Work/3DHST_Detection/GOODS-N_IR.cat')
cat = self.blot_catalog(csex, sextractor=True)
del(self.refimage)
del(self.segimage)
del(self.im)
np.save('flt.npy', [self])
self2 = np.load('flt.npy')[0]
for key in keys:
if os.path.exists('flt.npy'):
FLTs[key] = np.load('flt.npy')[0]
else:
self = mywfc3.flt.model.GrismFLT(file='icka01t7q_flt.fits', refimage='F160W_mosaic.fits', segimage='F160W_seg.fits')
#xspec = np.arange(1.e4,1.8e4)
#yspec = (xspec/1.6e4)**-0.4
#xsh, ysh, x_rms, y_rms = self.align_bright_objects(xspec=xspec, yspec=yspec, ds9=ds9)
#self.update_wcs_with_shift(xsh=xsh, ysh=ysh)
csex = catIO.Table('/Users/brammer/3DHST/Spectra/Work/3DHST_Detection/GOODS-N_IR.cat')
cat = self.blot_catalog(csex, sextractor=True)
FLTs[key] = self
### For testing
for key in keys:
FLTs[key].flam *= (key+1)
FLTs[key].clip *= (key+1)
######## Queued process, has awkward I/O for getting data out
t0_queue = time.time()
queue = mp.Queue()
processes = [mp.Process(target=_go_compute_model, args=(key,), kwargs={'queue':queue}, name='FLT%d' %(key)) for key in keys]
# Run processes
for p in processes:
p.start()
# Exit the completed processes
for p in processes:
p.join()
# Get the outputs
results = [queue.get(timeout=1) for p in processes]
for i, key in enumerate(keys):
print results[i]
FLTs[key].modelf = np.load(results[i])
FLTs[key].model = FLTs[key].modelf.reshape((1014,1014))
os.remove(results[i])
print FLTs[key].modelf.max()
t1_queue = time.time()
#print t1_queue - t0_queue
######## Slow, test single process execution
t0_raw = time.time()
out = _go_compute_model(1, queue=None)
t1_raw = time.time()
#print t1_raw - t0_raw
######## apply_async for getting data out, works the best
t0_pool = time.time()
pool = mp.Pool(processes=len(keys))
results=[pool.apply_async(_go_compute_model, (key, None)) for key in keys]
pool.close()
pool.join()
for res in results:
key, modelf = res.get(timeout=1)
FLTs[key].modelf = modelf
FLTs[key].model = FLTs[key].modelf.reshape((1014,1014))
print FLTs[key].modelf.max()
t1_pool = time.time()
print ' Raw: %.3f s (x%d: %.3f)' %(t1_raw - t0_raw, len(keys), (t1_raw - t0_raw)*len(keys))
print 'Queue: %.3f s (%.1f speedup)' %(t1_queue - t0_queue, (t1_raw - t0_raw)*len(keys)/(t1_queue - t0_queue))
print ' Pool: %.3f s (%.1f speedup)' %(t1_pool - t0_pool, (t1_raw - t0_raw)*len(keys)/(t1_pool - t0_pool))
import stwcs
import numpy as np
from scipy.interpolate.fitpack import splev, splrep
import threedhst.dq
ds9 = threedhst.dq.myDS9()
import mywfc3.flt.model
import astropy.io.fits as pyfits
from threedhst import catIO
files = glob.glob('ico205l[orux]q*flt.fits')
refimage = 'wfi2026-4536-05-235-f140w_drz_sci.fits'
### Refsdal
info = catIO.Table('info')
pa = np.cast[int](info['PA_V3'])
pas = np.unique(pa[info['FILTER'] == 'G141'])
files = info['FILE'][(info['FILTER'] == 'G141') & (pa == 111)]
files = info['FILE'][(info['FILTER'] == 'G141') & (pa == 119)]
#files=info['FILE'][(info['FILTER'] == 'G141') & (np.abs(pa-115) < 5)]
#files=info['FILE'][(info['FILTER'] == 'G141')]
refimage = '../MACS1149/Catalog/MACS1149-F140W_drz_sci.fits'
#files=files[:1]
FLT = {}
for i, file in enumerate(files):
print '%d/%d %s' % (i + 1, len(files), file)
g = mywfc3.flt.model.GrismFLT(file=file, refimage=refimage)
FLT[file] = g
def prep():
import glob
import os
import threedhst
import threedhst.prep_flt_astrodrizzle as init
from threedhst import catIO
import unicorn
import research.hawkiff
### Make ACS associations
unicorn.candels.make_asn_files(uniquename=True, translate={'-ROT': ''})
threedhst.options['FLT_PERSISTENCE_PATH'] = '../Persistence/'
#### IR
files = glob.glob('*-F1*asn.fits')
radec = None
for asn_file in files:
init.prep_direct_grism_pair(direct_asn=asn_file,
grism_asn=None,
radec=radec,
ACS=False,
align_threshold=5,
raw_path='../RAW/',
order=0)
###
for asn_file in files:
drizzlepac.astrodrizzle.AstroDrizzle(
asn_file,
static=False,
skysub=False,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
final_wht_type='IVM',
clean=True,
final_wcs=True,
final_refimage='IRAS23436+5257-87-304-F814W_drc_sci.fits',
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=576,
preserve=False)
### Match WCS of NB observations
init.copy_adriz_headerlets(glob.glob('*-F1*[MW]*asn.fits')[0],
glob.glob('*-F1*[N]*asn.fits')[0],
order=[1, 0])
### Flag negative pixels
for asn_file in files:
asn = threedhst.utils.ASNFile(asn_file)
for exp in asn.exposures:
im = pyfits.open('%s_flt.fits' % (exp), mode='update')
neg = im['SCI'].data < -3 * im['ERR'].data
print '%s #negative = %d' % (exp, neg.sum())
im['DQ'].data[neg] |= 4096
im.flush()
#
drizzlepac.astrodrizzle.AstroDrizzle(asn_file,
skysub=True,
static=True,
driz_separate=True,
driz_sep_wcs=True,
median=True,
blot=True,
driz_cr=True,
driz_combine=True,
clean=True,
final_wcs=True,
final_scale=0.1,
final_rot=0,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=64,
preserve=False)
sci = pyfits.open(asn_file.replace('_asn', '_drz_sci'), mode='update')
wht = pyfits.open(asn_file.replace('_asn', '_drz_wht'))
sci[0].data[wht[0].data == 0] = 0
sci.flush()
#### ACS / UVIS
files = glob.glob('*-F6*asn.fits')
files = glob.glob('*-F[48]*asn.fits')
files.sort()
for asn_file in files[::-1]:
asn = threedhst.utils.ASNFile(asn_file)
for i in range(len(asn.exposures)):
asn.exposures[i] = asn.exposures[i].split('_flc')[0]
print asn_file, asn.exposures
asn.write(asn_file)
if ('814' in asn_file) | ('625' in asn_file):
radec = None
else:
root = glob.glob('*-F[86]*W*sci.fits')[0].split('_sci')[0]
if not os.path.exists('%s_sci.cat' % (root)):
os.system(
'cp /user/brammer/software/share/gauss_5.0_9x9.conv .')
research.hawkiff.make_catalog(root=root,
sci_ext='sci',
threshold=3,
use_rms=False,
subtract_background=True)
cat = catIO.Table('%s_sci.cat' % (root))
cat['X_WORLD',
'Y_WORLD'].write('ref_opt.radec',
format='ascii.commented_header')
radec = 'ref_opt.radec'
radec = None
init.prep_direct_grism_pair(direct_asn=asn_file,
grism_asn=None,
radec=radec,
ACS=True,
align_threshold=5,
raw_path='../RAW/')
#
for asn_file in files:
radec = None
init.prep_direct_grism_pair(direct_asn=asn_file,
grism_asn=None,
radec=radec,
ACS=True,
align_threshold=5,
raw_path='../RAW/')
files = glob.glob('*sci.fits')
ims = {}
for file in files:
im = pyfits.open(file)
filter = file.split('-')[-1][:5]
pix = np.sqrt(im[0].header['CD1_1']**2 +
im[0].header['CD1_2']**2) * 3600
scl = im[0].header['PHOTFLAM'] / 1.e-19 * (0.1 / pix)**2
im[0].data *= scl
ims[filter] = im
### Try Multidrizzle CR with combined image
import drizzlepac
asn_files = glob.glob('*-F[4-8]*asn.fits')
exposures = []
for file in asn_files:
asn = threedhst.utils.ASNFile(file)
exposures.extend(asn.exposures)
target = '-'.join(file.split('-')[:-1])
asn.exposures = exposures
asn.product = target
asn_file = '%s_asn.fits' % (asn.product)
asn.write(asn_file, clobber=True)
### modify: make copy of FLCs, make new mosaic and then copy FLCs back
##xxx
drizzlepac.astrodrizzle.AstroDrizzle(asn_file,
skysub=False,
clean=True,
final_wcs=True,
final_scale=0.05,
final_pixfrac=0.8,
context=False,
resetbits=4096,
final_bits=576,
preserve=False)
#
bits = 576
drizzlepac.astrodrizzle.AstroDrizzle(asn_file,
static=False,
skysub=False,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
final_wht_type='IVM',
clean=True,
final_wcs=True,
final_rot=0,
final_scale=0.1,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=bits,
preserve=False)
#### rerun final mosaics
refimage = '%s_drz_sci.fits' % (target)
for file in asn_files:
drizzlepac.astrodrizzle.AstroDrizzle(file,
static=False,
skysub=False,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
final_wht_type='IVM',
clean=True,
final_wcs=True,
final_refimage=refimage,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=bits,
preserve=False)
for file in asn_files:
print file
sci = pyfits.open(glob.glob(file.replace('_asn', '_dr?_sci'))[0],
mode='update')
wht = pyfits.open(sci.filename().replace('_sci', '_wht'))
sci[0].data[wht[0].data == 0] = 0
sci.flush()
#### Try own driz CR
import stwcs
from drizzlepac import astrodrizzle, quickDeriv
import scipy.ndimage as nd
ref_file = 'ESO550-IG02-13-083-F435W_drc_sci.fits'
ref_file = 'ESO550-IG02-13-083-F814W_drc_sci.fits'
ref_file = 'ESO550-IG02-13-083_drc_sci.fits'
ref_ext = 0
ref = pyfits.open(ref_file)
ref_wcs = stwcs.wcsutil.HSTWCS(ref, ext=ref_ext)
asn_file = 'ESO550-IG02-13-083-F814W_asn.fits'
asn = threedhst.utils.ASNFile(asn_file)
for exp in asn.exposures:
flt = pyfits.open('%s_flc.fits' % (exp)) #, mode='update')
for ext in [1, 2]:
flt_wcs = stwcs.wcsutil.HSTWCS(flt, ext=('sci', ext))
blotted_ref = astrodrizzle.ablot.do_blot(ref[ref_ext].data,
ref_wcs,
flt_wcs,
1,
coeffs=True,
interp='nearest',
sinscl=1.0,
stepsize=10,
wcsmap=None)
#
blotDeriv = quickDeriv.qderiv(blotted_ref)
scl = flt['sci', ext].header['PHOTFLAM'] / ref[0].header[
'PHOTFLAM'] / flt[0].header['EXPTIME']
noise = flt['err', ext].data
driz_scale = 1.2
driz_snr = 3.5
driz_scale = 1.0
driz_snr = 6
driz_f = 2
# |data_image - blotted_image| > scale x deriv + SNR x noise
abs = np.abs(flt['sci', ext].data * scl - blotted_ref * driz_f)
xderiv = driz_scale * blotDeriv
xsnr = driz_snr * scl * noise
mask = abs > (xderiv + xsnr)
new_dq = mask & ((flt['dq', ext].data & 4096) == 0)
### Try "cosmics" LA Cosmic
im = pyfits.open('ESO550-IG02-13-083-F814W_drc_sci.fits')
slx, sly = slice(1500, 2600), slice(1800, 2800)
ext = 0
subim = im[ext].data[sly, slx]
h = im[ext].header
subim *= im[0].header['EXPTIME']
#### FLT
im = pyfits.open('j9cv13pcq_flc.fits')
ext = 0
slx, sly = slice(1500, 2600), slice(0, 400)
subim = im[4].data[sly, slx]
h = im[0].header
import cosmics
c = cosmics.cosmicsimage(subim,
pssl=20.0,
gain=1,
readnoise=h['READNSEA'],
sigclip=4.0,
sigfrac=0.3,
objlim=3.8,
satlevel=84700.0,
verbose=True)
c.run(maxiter=4)
crflux = subim * c.mask
plt.hist(np.log10(crflux[c.mask]), range=[1, 6], bins=100, alpha=0.5)
####
import research.pab.pab
files = glob.glob('*-F*asn.fits')
#files=glob.glob('IC*-F814*asn.fits')
for file in files:
asn = threedhst.utils.ASNFile(file)
for exp in asn.exposures:
flc = pyfits.open('%s_flc.fits' % (exp))
research.pab.pab.run_lacosmic('%s_flc.fits' % (exp),
split=2048,
sigclip=5,
pssl=flc[1].header['MDRIZSK0'])
#
drizzlepac.updatehdr.update_from_shiftfile(
file.replace('asn.fits', 'shifts.txt'))
drizzlepac.astrodrizzle.AstroDrizzle(file,
skysub=True,
clean=True,
final_wcs=True,
final_scale=0.05,
final_pixfrac=0.8,
context=False,
resetbits=0,
final_bits=576,
preserve=False)
drizzlepac.astrodrizzle.AstroDrizzle(file,
output='sub_' +
file.split('_asn')[0],
skysub=True,
static=True,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
clean=True,
final_wcs=True,
final_scale=0.08,
final_ra=60.635233,
final_dec=60.344597,
final_outnx=800,
final_outny=800,
final_rot=0,
final_pixfrac=0.8,
context=False,
resetbits=0,
final_bits=576,
preserve=False)
### Final mosaics
f110w = glob.glob('*110W*asn.fits')[0]
drizzlepac.astrodrizzle.AstroDrizzle(f110w,
output='sub_' +
f110w.split('_asn')[0],
skysub=True,
static=True,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
clean=True,
final_wcs=True,
final_scale=0.1,
final_rot=0,
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=576 - 512,
preserve=False)
files = glob.glob('*-F*asn.fits')
for file in files:
if 'F110W' in file:
continue
#
drizzlepac.astrodrizzle.AstroDrizzle(
file,
output='sub_' + file.split('_asn')[0],
skysub=True,
static=True,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_combine=True,
clean=True,
final_wcs=True,
final_refimage='sub_' + f110w.replace('_asn', '_drz_sci'),
final_pixfrac=1,
context=False,
resetbits=0,
final_bits=576 - 512 * ('-F1' in file),
preserve=False)
for sci_im in glob.glob('sub*sci.fits'):
print sci_im
sci = pyfits.open(sci_im, mode='update')
wht = pyfits.open(sci_im.replace('_sci', '_wht'))
sci[0].data[wht[0].data == 0] = 0
sci.flush()
#### Test alignment
import align
ra_list, de_list = np.loadtxt('f814w.radec', unpack=True)
for geom in ['shift', 'shift', 'rxyscale', 'shift', 'shift']:
align.get_align_to_subaru(
sci='sub_ESO550-IG025-41-115-F673N_drc_sci.fits',
wht='sub_ESO550-IG025-41-115-F673N_drc_wht.fits',
verbose=False,
fitgeometry=geom,
align_data=(ra_list, de_list, ra_list * 0),
THRESH=1.2)
#drizzlepac.tweakback.tweakback('ESO550-IG025-41-115-F673N_drc_sci.fits', force=True, verbose=True) #, origwcs='DRZWCS')
ra_list, de_list = np.loadtxt('f110w.radec', unpack=True)
for geom in ['shift', 'shift', 'rxyscale', 'shift', 'shift']:
align.get_align_to_subaru(
sci='sub_ESO550-IG025-41-115-F132N_drz_sci.fits',
wht='sub_ESO550-IG025-41-115-F132N_drz_wht.fits',
verbose=False,
fitgeometry=geom,
align_data=(ra_list, de_list, ra_list * 0),
THRESH=1.2)
def drzTweakReg(sci='goodss-34-F140W_drz_sci.fits', master_catalog='goodss_radec.dat', threshold=20, apply=True):
import drizzlepac
from drizzlepac import tweakback
from stwcs import updatewcs
from threedhst import catIO
se = threedhst.sex.SExtractor()
se.options['WEIGHT_IMAGE'] = sci.replace('sci','wht')
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
#
se.params['X_IMAGE'] = True; se.params['Y_IMAGE'] = True
se.params['X_WORLD'] = True; se.params['Y_WORLD'] = True
se.params['MAG_AUTO'] = True
#
se.options['CATALOG_NAME'] = sci+'.align.cat'
se.options['FILTER'] = 'N'
se.options['DETECT_THRESH'] = '%f' %(threshold)
se.options['ANALYSIS_THRESH'] = '%f' %(threshold)
#
se.sextractImage(sci)
threedhst.sex.sexcatRegions(sci+'.align.cat', sci+'.align.reg', format=2)
c = catIO.Table(sci+'.align.cat', format='ascii.sextractor')
#c.ra = c['X_WORLD']
#c.dec = c['Y_WORLD']
m = catIO.CoordinateMatcher(c, ra_column='X_WORLD', dec_column='Y_WORLD')
r0, d0 = np.loadtxt(master_catalog, unpack=True)
### clip list to nearby objects
rmed, dmed = np.median(c['X_WORLD']), np.median(c['Y_WORLD'])
delta = np.sqrt((r0-rmed)**2/np.cos(dmed/180*np.pi)**2+(d0-dmed)**2)*60.
nearby = delta < 8 # arcmin
r0, d0 = r0[nearby], d0[nearby]
dr, idx = m.match_list(r0, d0)
dx = (c['X_WORLD'][idx]-r0)*np.cos(d0/180*np.pi)*3600
dy = (c['Y_WORLD'][idx]-d0)*3600
x0 = (c['X_WORLD'][idx]-np.median(c['X_WORLD']))*np.cos(d0/180*np.pi)*3600
y0 = (c['Y_WORLD'][idx]-np.median(c['Y_WORLD']))*3600
ok = dr < 1.5
if ok.sum() == 0:
threedhst.showMessage('No matches found within 1.5".')
return False
# plt.scatter(x0[ok], y0[ok], color='black')
# for i in np.arange(len(ok))[ok]:
# plt.plot(x0[i]+np.array([0, dx[i]*20]), y0[i]+np.array([0, dy[i]*20]), color='black')
dra = (c['X_WORLD'][idx]-r0)
dde = (c['Y_WORLD'][idx]-d0)
rshift, dshift = np.median(dra[ok]), np.median(dde[ok])
fp = open(sci.split('.fits')[0]+'.align.dat','w')
lines = ['# dx dy xrms yrms N\n# %s %s\n' %(sci, master_catalog), '%f %f %f %f %d\n' %(np.median(dx[ok]), np.median(dy[ok]), np.std(dx[ok]), np.std(dy[ok]), ok.sum())]
fp.writelines(lines)
fp.close()
threedhst.showMessage(''.join(lines))
if not apply:
print 'Not applying shift. Re-run with apply=true to apply them.'
return rshift, dshift
for fits in [sci.replace('sci','wht'), sci]:
print 'Update WCS: %s' %(fits)
im = pyfits.open(fits, mode='update')
im[0].header['CRVAL1'] -= rshift
im[0].header['CRVAL2'] -= dshift
im.flush()
im = pyfits.open(sci)
for i in range(im[0].header['NDRIZIM']):
flt_im = im[0].header['D%03dDATA' %(i+1)].split('[')[0]
print 'Update WCS: %s' %(flt_im)
flt = pyfits.open(flt_im, mode='update')
for ext in [1,2]:
flt[ext].header['CRVAL1'] -= rshift
flt[ext].header['CRVAL2'] -= dshift
flt.flush()
def runTweakReg(asn_file='GOODS-S-15-F140W_asn.fits', master_catalog='goodss_radec.dat', final_scale=0.06, ACS=False, threshold=5, WFPC2=False):
"""
Wrapper around tweakreg, generating source catalogs separately from
`findpars`.
"""
import glob
import shutil
import drizzlepac
from drizzlepac import tweakreg
from stwcs import updatewcs
import threedhst.prep_flt_astrodrizzle
asn = threedhst.utils.ASNFile(asn_file)
if ACS:
NCHIP=2
sci_ext = [1,4]
wht_ext = [2,5]
ext = 'flc'
dext = 'crclean'
else:
NCHIP=1
sci_ext = [1]
wht_ext = [2]
ext = 'flt'
dext = 'flt'
### Generate CRCLEAN images
for exp in asn.exposures:
updatewcs.updatewcs('%s_%s.fits' %(exp, ext))
has_crclean = True
for exp in asn.exposures:
has_crclean &= os.path.exists('%s_crclean.fits' %(exp))
threedhst.showMessage('# exposures: %d' %(len(asn.exposures)))
if not has_crclean:
if len(asn.exposures) == 1:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=False, context=False, preserve=False, skysub=True, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_cr_corr=False, driz_combine=True)
shutil.copy('%s_%s.fits' %(asn.exposures[0], ext), '%s_crclean.fits' %(asn.exposures[0]))
else:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=False, context=False, preserve=False, skysub=True, driz_separate=True, driz_sep_wcs=True, median=True, blot=True, driz_cr=True, driz_cr_corr=True, driz_combine=True)
#### Make SExtractor source catalogs in *each* flt
for exp in asn.exposures:
#updatewcs.updatewcs('%s_%s.fits' %(exp, ext))
for i in range(NCHIP):
se = threedhst.sex.SExtractor()
se.options['WEIGHT_IMAGE'] = '%s_%s.fits[%d]' %(exp, dext, wht_ext[i]-1)
se.options['WEIGHT_TYPE'] = 'MAP_RMS'
#
se.params['X_IMAGE'] = True; se.params['Y_IMAGE'] = True
se.params['MAG_AUTO'] = True
#
se.options['CATALOG_NAME'] = '%s_%s_%d.cat' %(exp, ext, sci_ext[i])
se.options['FILTER'] = 'N'
se.options['DETECT_THRESH'] = '%f' %(threshold)
se.options['ANALYSIS_THRESH'] = '%f' %(threshold)
#
se.sextractImage('%s_%s.fits[%d]' %(exp, dext, sci_ext[i]-1))
threedhst.sex.sexcatRegions('%s_%s_%d.cat' %(exp, ext, sci_ext[i]), '%s_%s_%d.reg' %(exp, ext, sci_ext[i]), format=1)
#### TweakReg catfile
asn_root = asn_file.split('_asn')[0]
catfile = '%s.catfile' %(asn_root)
fp = open(catfile,'w')
ncat = 0
for exp in asn.exposures:
c = catIO.Table('%s_%s_%d.cat' %(exp, ext, sci_ext[i]))
if len(c) < 20:
continue
else:
ncat += 1
line = '%s_%s.fits' %(exp, ext)
for i in range(NCHIP):
line += ' %s_%s_%d.cat' %(exp, ext, sci_ext[i])
fp.write(line + '\n')
fp.close()
#### First run AstroDrizzle mosaic
#drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, context=False, preserve=False, skysub=True, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True)
#### Make room for TWEAK wcsname
for exp in asn.exposures:
threedhst.prep_flt_astrodrizzle.clean_wcsname(flt='%s_%s.fits' %(exp, ext), wcsname='TWEAK', ACS=ACS)
#### Main run of TweakReg
if ACS:
refimage = '%s_drc_sci.fits' %(asn_root)
else:
refimage = '%s_drz_sci.fits' %(asn_root)
if ncat >= 1:
tweakreg.TweakReg(asn_file, refimage=refimage, updatehdr=True, updatewcs=True, catfile=catfile, xcol=2, ycol=3, xyunits='pixels', refcat=master_catalog, refxcol=1, refycol=2, refxyunits='degrees', shiftfile=True, outshifts='%s_shifts.txt' %(asn_root), outwcs='%s_wcs.fits' %(asn_root), searchrad=5, tolerance=12, wcsname='TWEAK', interactive=False, residplot='No plot', see2dplot=False, clean=True, headerlet=True, clobber=True)
#### Run AstroDrizzle again
if ACS:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, preserve=False)
else:
if len(asn.exposures) == 1:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale = '2.5 0.7', driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_cr_corr=False) # , final_wcs=True, final_rot=0)
else:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale = '2.5 0.7') # , final_wcs=True, final_rot=0)
for exp in asn.exposures:
files=glob.glob('%s*coo' %(exp))
files.extend(glob.glob('%s*crclean.fits' %(exp)))
for file in files:
os.remove(file)
def etc_stats():
"""
Produce some numbers for the ETC
e.g., number of excess electrons as a function of exposure time
"""
import glob
from threedhst import catIO
import numpy as np
import matplotlib.pyplot as plt
files = glob.glob('*orbit.dat')
N = len(files)
texp = np.zeros(N)
excess = np.zeros(N)
excess_zodi = np.zeros(N)
min = np.zeros(N)
zodi = np.zeros(N)
for i in range(N):
print i, files[i]
c = catIO.Table(files[i], format='ascii.commented_header')
dt = np.diff(c['Seconds'])
dt = np.append(dt, dt[-1])
tread = c['Seconds'] + dt / 2.
texp[i] = tread[-1]
no_shadow = c['shadow'] == 0
excess[i] = ((c['bg'] - c['minbg']) * dt)[no_shadow].sum()
excess_zodi[i] = ((c['bg'] - c['zodi']) * dt)[no_shadow].sum()
min[i] = c['minbg'][0]
zodi[i] = c['zodi'][0]
#
plt.scatter(texp + np.random.rand(N) * 0.1 * texp,
excess_zodi / texp,
alpha=0.1)
stat_levels = [25, 50, 75, 95]
stats = np.zeros((3, len(stat_levels)))
med_texp = np.zeros(3)
for j in range(3):
sample = (texp > j * 600) & (texp <= (j + 1) * 600)
stats[j, :] = np.percentile((excess_zodi / texp)[sample], stat_levels)
med_texp[j] = np.median(texp[sample])
full_stats = np.percentile(excess_zodi / texp, stat_levels)
for j in range(len(stat_levels))[::-1]:
plt.plot(med_texp,
stats[:, j],
label='%d (%.3f)' % (stat_levels[j], full_stats[j]),
marker='o')
plt.xlim(100, 2000)
plt.ylim(-0.1, 3)
plt.xlabel('Image exposure time (s)')
plt.ylabel('Excess background flux (e-/s)')
plt.legend(loc='best')
plt.title('F105W')
plt.savefig('F105W_excess_stats.pdf')
def test():
import numpy as np
from drizzlepac import astrodrizzle
import stwcs
from threedhst import catIO
import research.hawki_driz
import unicorn
import unicorn.interlace_test
import os
import astropy.io.fits as pyfits
grism, filter = 'G102', 'F105W'
main_root = 'FIGS-GS1-G102'
#flt_files = glob.glob('icoi1*flt.fits')
id=28445 # two knots
id = 29649 # edge-on
id = 30691 # fuzzy
lrest = 6563.
z = None
size=3
pixscale=0.05
pixfrac = 0.4
#twod_files = glob.glob('FIGS-GS1-*-G102_%05d.2D.fits' %(id))
#twod = unicorn.reduce.Interlace2D(twod_files[0])
#wave = 10625.5
gris = unicorn.interlace_test.SimultaneousFit('%s_%05d' %(main_root, id), fast=True)
if z is None:
wave = lrest*(1+gris.z_max_spec)
else:
wave = lrest*(1+z)
print 'Continue? (wave=%.1f, lrest=%.1f)' %(wave, lrest)
var = raw_input()
if var != '':
print breakme
else:
print 'OK!'
ra, dec = gris.twod.im[0].header['RA'], gris.twod.im[0].header['DEC']
#ra, dec = 53.158163486, -27.7810907
hout, wcs_out = research.hawki_driz.get_wcsheader(ra=ra, dec=dec, size=size, pixscale=pixscale, get_hdu=False, theta=0)
sh = (int(hout['NAXIS2']), int(hout['NAXIS1']))
files = catIO.Table('flt_files')
flt_files = files['FILE'][files['FILTER'] == filter]
outsci = np.zeros(sh, dtype=np.float32)
outwht = np.zeros(sh, dtype=np.float32)
outcon = np.zeros(sh, dtype=np.float32)
ds9.frame(1)
for file in flt_files:
print file
flt = pyfits.open(file)
flt_wcs = stwcs.wcsutil.HSTWCS(flt, 1)
# xflt, yflt = flt_wcs.all_world2pix([ra], [dec], 0)
# x0, y0 = int(xflt[0]), int(yflt[0])
# sub_flt = flt[1].data[y0-11:y0+10, x0-30:x0+30]
# ds9.view(sub_flt)
sci = flt[1].data
wht = 1/flt['ERR'].data**2
wht = np.ones((1014,1014), dtype=np.float32)*np.median(wht)
dq = flt['DQ'].data.copy()
for bit in [64,512]:
dq -= dq & bit
msk = (dq > 0) | (~np.isfinite(wht))
wht[msk] = 0.
sci[msk] = 0.
astrodrizzle.adrizzle.do_driz(sci, flt_wcs, wht, wcs_out, outsci, outwht, outcon, 1., 'cps', 1, wcslin_pscale=1.0, uniqid=1, pixfrac=pixfrac, kernel='square', fillval=0, stepsize=10, wcsmap=None)
ds9.view(outsci, header=hout)
## Try drizzling spectra
flt_files = files['FILE'][files['FILTER'] == grism]
import mywfc3.grism
conf = mywfc3.grism.aXeConf('%s/CONF/%s.test27s.gbb.conf' %(os.getenv('THREEDHST'), grism))
goutsci = np.zeros(sh, dtype=np.float32)
goutwht = np.zeros(sh, dtype=np.float32)
goutcon = np.zeros(sh, dtype=np.float32)
xpix = np.arange(0,300)
ds9.frame(2)
for file in flt_files:
flt = pyfits.open(file)
flt_wcs = stwcs.wcsutil.HSTWCS(flt, 1)
sci = flt[1].data
wht = 1/flt['ERR'].data**2
wht = np.ones((1014,1014), dtype=np.float32)*np.median(wht)
dq = flt['DQ'].data.copy()
for bit in [64,512]:
dq -= dq & bit
#
msk = (dq > 0) | (~np.isfinite(wht))
wht[msk] = 0.
sci[msk] = 0.
### offset for wavelength
xflt, yflt = flt_wcs.all_world2pix([ra], [dec], 0)
ytrace, lam = conf.get_beam_trace(x=xflt[0], y=yflt[0], dx=xpix, beam='A')
dx = np.interp(wave, lam, xpix)
dy = np.interp(wave, lam, ytrace)
print '%s (%.1f,%.1f), (%.2f,%.2f)' %(file, xflt[0], yflt[0], dx, dy)
flt_wcs.wcs.crpix += np.array([dx, dy])
try:
flt_wcs.sip.crpix += np.array([dx, dy])
except:
pass
x0, y0 = int(xflt[0]+dx), int(yflt[0]+dy)
sub_flt = flt[1].data[y0-30:y0+30, x0-50:x0+30]
prof = np.median(sub_flt, axis=1)
yprof = np.zeros(1014)
yprof[y0-30:y0+30] += prof
sci = (sci.T-yprof).T
#ds9.view(sub_flt)
#dx = int(np.round(dx))
#dy = int(np.round(dy))
#sci =
astrodrizzle.adrizzle.do_driz(sci, flt_wcs, wht, wcs_out, goutsci, goutwht, goutcon, 1., 'cps', 1, wcslin_pscale=1.0, uniqid=1, pixfrac=pixfrac, kernel='square', fillval=0, stepsize=10, wcsmap=None)
ds9.view(goutsci, header=hout)
### Get continuum extraction from the 2D files
import mywfc3.grism
conf = mywfc3.grism.aXeConf('%s/CONF/%s.test27s.gbb.conf' %(os.getenv('THREEDHST'), grism))
xpix = np.arange(0,300)
ds9.frame(3)
im = pyfits.open(os.getenv('iref')+'ir_wfc3_map.fits')
PAM = im[1].data
im.close()
drizzled_sci = {}
drizzled_wht = {}
drizzled_ctx = {}
drizzled_mod = {}
drizzled_contam = {}
twod_files=glob.glob('*-???-%s_%05d.2D.fits' %(grism, id))
for twod_file in twod_files:
twod = unicorn.reduce.Interlace2D(twod_file)
twod.compute_model()
root = twod_file.split('_')[0]
print 'Interlaced: %s' %(root)
inter = pyfits.open('%s_inter.fits' %(root))
sh2 = twod.model.shape
h = twod.im[0].header
sly = slice(h['YGRISM0'], h['YGRISM0']+sh2[0])
slx = slice(h['XGRISM0'], h['XGRISM0']+sh2[1])
### continuum subtraction
clean = inter[1].data*0
clean[sly, slx] += twod.model #+ twod.im['CONTAM'].data
contam = inter[1].data*0
contam[sly, slx] += twod.im['CONTAM'].data
### Get FLT components and "de-interlace" model
asn = threedhst.utils.ASNFile('%s_asn.fits' %(root))
if root.split('-G10')[0] in grism_ref_exp.keys():
ref_exp = grism_ref_exp[root.split('-G10')[0]]
else:
ref_exp = 0
dxs, dys = unicorn.reduce.wcs_interlace_offsets('%s_asn.fits' %(root), growx=2, growy=2, path_to_flt='./', verbose=False, ref_exp=ref_exp, raw=False, reference_pixel=None)
hi = inter[1].header
x0 = hi['PAD']/2 + hi['NGROWX']*hi['GROWX']
y0 = hi['PAD']/2 + hi['NGROWY']*hi['GROWY']
foutsci = np.zeros(sh, dtype=np.float32)
foutwht = np.zeros(sh, dtype=np.float32)
foutctx = np.zeros(sh, dtype=np.float32)
coutsci = np.zeros(sh, dtype=np.float32)
coutwht = np.zeros(sh, dtype=np.float32)
coutctx = np.zeros(sh, dtype=np.float32)
moutsci = np.zeros(sh, dtype=np.float32)
moutwht = np.zeros(sh, dtype=np.float32)
moutctx = np.zeros(sh, dtype=np.float32)
for i,exp in enumerate(asn.exposures):
#print exp
flt = pyfits.open(exp+'_flt.fits')
islx = slice(x0+dxs[i], x0+dxs[i]+1014*hi['GROWX'], hi['GROWX'])
isly = slice(y0+dys[i], y0+dys[i]+1014*hi['GROWY'], hi['GROWY'])
flt_wcs = stwcs.wcsutil.HSTWCS(flt, 1)
sci = flt[1].data #- cutout
cutout = clean[isly, islx]*(hi['GROWX']*hi['GROWY'])/PAM
contam_cutout = contam[isly, islx]*(hi['GROWX']*hi['GROWY'])
#wht = 1/flt['ERR'].data**2
#wht = np.ones((1014,1014), dtype=np.float32)*np.median(wht)
err = (np.median(flt['ERR'].data)**2 + contam_cutout**2*2)
wht = 1/err
dq = flt['DQ'].data.copy()
for bit in [64,512]:
dq -= dq & bit
#
msk = (dq > 0) | (~np.isfinite(wht))
wht[msk] = 0.
sci[msk] = 0.
### offset for wavelength
xflt, yflt = flt_wcs.all_world2pix([ra], [dec], 0)
ytrace, lam = conf.get_beam_trace(x=xflt[0], y=yflt[0], dx=xpix, beam='A')
dx = np.interp(wave, lam, xpix)
dy = np.interp(wave, lam, ytrace)
print '%s (%.1f,%.1f), (%.2f,%.2f)' %(exp, xflt[0], yflt[0], dx, dy)
flt_wcs.wcs.crpix += np.array([dx, dy])
try:
flt_wcs.sip.crpix += np.array([dx, dy])
except:
pass
astrodrizzle.adrizzle.do_driz(sci, flt_wcs, wht, wcs_out, foutsci, foutwht, foutctx, 1., 'cps', 1, wcslin_pscale=1.0, uniqid=1, pixfrac=pixfrac, kernel='square', fillval=0, stepsize=10, wcsmap=None)
astrodrizzle.adrizzle.do_driz(contam_cutout, flt_wcs, wht, wcs_out, coutsci, coutwht, coutctx, 1., 'cps', 1, wcslin_pscale=1.0, uniqid=1, pixfrac=pixfrac, kernel='square', fillval=0, stepsize=10, wcsmap=None)
astrodrizzle.adrizzle.do_driz(cutout, flt_wcs, wht, wcs_out, moutsci, moutwht, moutctx, 1., 'cps', 1, wcslin_pscale=1.0, uniqid=1, pixfrac=pixfrac, kernel='square', fillval=0, stepsize=10, wcsmap=None)
ds9.frame(3)
ds9.view(foutsci, header=hout)
ds9.frame(4)
ds9.view(foutsci-moutsci-coutsci, header=hout)
drizzled_sci[root] = foutsci
drizzled_wht[root] = foutwht
drizzled_ctx[root] = foutctx
drizzled_mod[root] = moutsci
drizzled_contam[root] = coutsci
sci = foutsci*0
wht = sci*0
csci = sci*0
for root in drizzled_sci.keys():
print root
wht += drizzled_wht[root]
sci += drizzled_sci[root]*drizzled_wht[root]
csci += (drizzled_mod[root]+drizzled_contam[root])*drizzled_wht[root]
sci /= wht
csci /= wht
ds9.frame(3)
ds9.view(sci, header=hout)
ds9.frame(4)
ds9.view(sci-csci, header=hout)
#wave = 10625.5
def extend_sens(sens_file='/Users/brammer/3DHST/Spectra/Work/CONF/WFC3.IR.G141.3rd.sens.2.fits'):
"""
Some of the "v2" sensitivity files are cut off at the wavelength extremes for no apparent reason.
Here, extend them with multi-gaussian component fits
"""
import os
import matplotlib.pyplot as plt
from astroML.sum_of_norms import sum_of_norms, norm
from threedhst import catIO
#sens = catIO.Table('/Users/brammer/3DHST/Spectra/Work/CONF/WFC3.IR.G102.2nd.sens.2.fits')
#sens = catIO.Table('/Users/brammer/3DHST/Spectra/Work/CONF/WFC3.IR.G102.3rd.sens.2.fits')
if 'G102' in sens_file:
xarr = np.arange(7100,1.19e4)
n_gaussians, spacing = 20, 'linear'
else:
xarr = np.arange(9700,1.75e4)
n_gaussians, spacing = 20, 'log'
#sens = catIO.Table('/Users/brammer/3DHST/Spectra/Work/CONF/WFC3.IR.G141.2nd.sens.2.fits')
sens = catIO.Table(sens_file)
ok = sens['SENSITIVITY'] > 0
x = sens['WAVELENGTH'][ok]
y = sens['SENSITIVITY'][ok]
ye = sens['ERROR'][ok]
out = {}
fig = plt.figure()
for i, col in enumerate(['SENSITIVITY', 'ERROR']):
ax = fig.add_subplot(211+i)
y = sens[col][ok]
w_best, rms, locs, widths = sum_of_norms(x, y, n_gaussians,
spacing=spacing,
full_output=True)
norms = w_best * norm(xarr[:, None], locs, widths)
out[col] = norms*1
ax.plot(sens['WAVELENGTH'], sens[col], '-k', linewidth=3, label=col)
ylim = ax.get_ylim()
if col == 'SENSITIVITY':
g = out[col].sum(1)
keep = g > 1.e-4*g.max()
ax.set_title(os.path.basename(sens.filename))
ax.plot(xarr[keep], out[col][keep,:], ls='-', c='#FFAAAA')
ax.plot(xarr[keep], out[col][keep,:].sum(1), '-r', linewidth=3, alpha=0.7)
ax.set_ylim(-0.1 * ylim[1], 1.05*ylim[1])
ax.legend(loc='upper right', fontsize=9)
ax.set_xlabel('wavelength')
fig.tight_layout(pad=1)
t = catIO.table_base()
t.add_column(catIO.Column(name='WAVELENGTH', data=xarr[keep]))
for i, col in enumerate(['SENSITIVITY', 'ERROR']):
t.add_column(catIO.Column(name=col, data=out[col][keep,:].sum(1)))
outfile = sens.filename.replace('sens.2.fits', 'sens.X.fits')
if os.path.exists(outfile):
os.remove(outfile)
t.write(outfile)
print outfile
plt.savefig(outfile.replace('.fits', '.png'))