def applyOffset(drzfile, fltfile, hlet=False):
"""Apply offset to an flt image from an aligned drizzled image"""
tweakback.tweakback(drzfile,
input=fltfile,
origwcs='DRZWCS',
verbose=True,
force=True)
if hlet:
headerlet.write_headerlet(fltfile,
'HLFRED',
output=None,
sciext='SCI',
wcsname='DRZWCS_2',
wcskey='PRIMARY',
destim=None,
sipname=None,
npolfile=None,
d2imfile=None,
author=None,
descrip=None,
history=None,
nmatch=None,
catalog=None,
attach=True,
clobber=False,
logging=False)
def shift_wcs(im):
try:
im_wcs = wcs.WCS(fits.getheader(im, 1))
except:
raise AssertionError('COULDN\'T FIND WCS FOR {}'.format(im))
if im_wcs.wcs.name == 'HSC':
print '{} already aligned to HSC, skipping'.format(im)
return
f = [ln.strip('\n') for ln in open('hsc_shifts.txt').readlines()][-1]
xsh, ysh, rot, scl, xrms, yrms = f.split()[1:]
ref = 'hsc_shifts_wcs.fits'
print im
try:
updatehdr.updatewcs_with_shift(im,
ref,
xsh=xsh,
ysh=ysh,
rot=rot,
scale=scl,
wcsname='HSC',
force=False,
sciext='SCI')
tweakback.tweakback(im,
force=False,
origwcs='DRZWCS',
newname='HSC',
wcsname='HSC')
except:
print 'COULDN\'T UPDATE {}, ERRORING'.format(im)
raise
def tback(drz):
flts = tweakback.extract_input_filenames(drz)
print 'Tweaking back exposures for {} with input ims:'.format(drz)
for f in flts:
print f
try:
tweakback.tweakback(drz)
except:
return
def tback(drz):
"""Runs tweakback to align flt/flcs with the drz/drc astrometry"""
flts = tweakback.extract_input_filenames(drz)
print 'Tweaking back exposures for {} with input ims:'.format(drz)
for f in flts:
print f
try:
tweakback.tweakback(drz)
except:
return
def shift_wcs(im):
try:
im_wcs = wcs.WCS(fits.getheader(im,1))
except:
raise AssertionError('COULDN\'T FIND WCS FOR {}'.format(im))
if im_wcs.wcs.name == 'HSC':
print '{} already aligned to HSC, skipping'.format(im)
return
f = [ln.strip('\n') for ln in open('hsc_shifts.txt').readlines()][-1]
xsh, ysh, rot, scl, xrms, yrms = f.split()[1:]
ref = 'hsc_shifts_wcs.fits'
print im
try:
updatehdr.updatewcs_with_shift(im,ref,xsh=xsh,ysh=ysh,rot=rot,scale=scl,wcsname='HSC',force=False,sciext='SCI')
tweakback.tweakback(im,force=False,origwcs='DRZWCS',newname='HSC',wcsname='HSC')
except:
print 'COULDN\'T UPDATE {}, ERRORING'.format(im)
raise
def align_drizzled(images=['MACS2129-35-F814W_drc_sci.fits', 'MACS2129-36-F814W_drc_sci.fits']):
from astropy.table import Table as table
from drizzlepac import astrodrizzle, tweakreg, tweakback
for image in images:
root = image.split('_sci.fits')[0]
se = threedhst.sex.SExtractor()
se.options['WEIGHT_IMAGE'] = '%s_wht.fits[0]' %(root)
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
#
se.params['X_IMAGE'] = True; se.params['Y_IMAGE'] = True
se.params['MAG_AUTO'] = True
#
se.options['CATALOG_NAME'] = '%s_sci.cat' %(root)
se.options['FILTER'] = 'N'
se.options['DETECT_THRESH'] = '5'
se.options['ANALYSIS_THRESH'] = '5'
#
se.sextractImage('%s_sci.fits[0]' %(root))
threedhst.sex.sexcatRegions('%s_sci.cat' %(root), '%s_sci.reg' %(root), format=1)
#
t = table.read('%s_sci.cat' %(root), format='ascii.sextractor')
np.savetxt('%s_sci.xy' %(root), np.array([t['X_IMAGE'], t['Y_IMAGE']]).T, fmt='%.7f')
fp = open('%s_sci.catfile' %(root), 'w')
fp.write('%s_sci.fits %s_sci.xy\n' %(root, root))
fp.close()
reference = '%s_sci.xy' %(images[0].split('_sci.fits')[0])
for image in images[1:]:
root = image.split('_sci.fits')[0]
tweakreg.TweakReg(image, refimage=images[0], updatehdr=True, updatewcs=True, catfile='%s_sci.catfile' %(root), xcol=1, ycol=2, xyunits='pixels', refcat=reference, refxcol=1, refycol=2, refxyunits='pixels', shiftfile=False, searchrad=5, tolerance=12, wcsname='TWEAK3', interactive=False, residplot='No plot', see2dplot=False, clean=True, headerlet=True, clobber=True)
tweakback.tweakback(image)
pass
USE:
python run_tweakback.py -drz drz.fits --> apply WCS solution from drz.fits to constituent flts (must be located in cwd).
'''
__author__='D.M. HAMMER'
__version__= 0.1
import os, glob, argparse, pyfits, pdb
from pyraf import iraf
import numpy as np
from drizzlepac import tweakback
from stsci.tools import teal
if __name__=='__main__':
# -- Parse input parameters
parser = argparse.ArgumentParser(description='Run tweakback to apply WCS from drz to fl? images.')
parser.add_argument('-drz', '--drzim',default='NONE', type=str, help='Input drizzled image (no wildcards). \
Default is NONE - requires input.')
options = parser.parse_args()
# -- initialize names of drizzled/flt(c) images
if options.drzim == 'NONE': raise Exception('Must input a drizzled image.')
drzim = options.drzim
# -- run tweakback
iraf.unlearn('tweakback')
teal.unlearn('tweakback')
tweakback.tweakback(drzim,verbose=True)
def drizzle_many(inputs_dict, output_dict={},
# darks_dict={},
kernel='lanczos3',
reference_filter=None, updatewcs=False,
clean=True, logfile=None, verbose=False,
wcsname=None, exp_ratio=0.9,
realign_only=False,
# dark_med_factor=3,
align_kwargs={}, drizzle_kwargs={},
realign_kwargs={}, tweakback_kwargs={},
redrizzle_kwargs={}):
"""
Drizzle together multiple sets of images to same aligned pixel scale
Parameters
----------
inputs_dict : dictionary (Filter -> list of input files)
output_dict : dictionary (Filter -> name of output file)
darks_dict : dictionary (Filter -> name of dark file)
"""
logger = RegularPrint() if verbose else PrintRedirect(logfile)
all_filters = list(inputs_dict.keys())
if reference_filter is None:
reference_filter = all_filters[0]
assert reference_filter in all_filters, (
f"{reference_filter} not in provided dictionary")
# put reference filter first
all_filters.remove(reference_filter)
all_filters = [reference_filter] + list(all_filters)
kwargs = {
'updatewcs': updatewcs, 'clean': clean, 'logfile': logfile,
'verbose': verbose}
align_kwargs = {**kwargs, **align_kwargs}
kwargs = {
'kernel': kernel, 'refimage': None, 'logfile': logfile,
'verbose': verbose, 'clean': clean}
drizzle_kwargs = {**kwargs, **drizzle_kwargs}
# Align and drizzle individual filters
if not realign_only:
print('*******************************')
print('Initial Alignment and Drizzling')
for filt in all_filters:
print(f'--{filt}: Aligning')
input_files = inputs_dict[filt]
output_file = output_dict[filt]
align_kwargs['wcsname'] = (f'INITIAL_{filt}')
myalign(input_files, **align_kwargs)
print(f'--{filt}: Drizzling')
mydrizzle(input_files, output_file, **drizzle_kwargs)
# Align drizzled images
kwargs = {
'threshold': 200, 'nclip': 10,
'refimage': output_dict[reference_filter],
'wcsname': f'TWEAK_{reference_filter}',
'clean': clean, 'verbose': verbose,
'logfile': logfile, 'updatewcs': False,
'refimagefindcfg': {'threshold': 200},
}
realign_kwargs = {**kwargs, **realign_kwargs}
kwargs = {
'force': True,
'verbose': verbose
}
tweakback_kwargs = {**kwargs, **tweakback_kwargs}
kwargs = {
'final_refimage': output_dict[reference_filter],
'final_wcs': True,
'final_rot': None,
}
kwargs = {**drizzle_kwargs, **kwargs}
redrizzle_kwargs = {**redrizzle_kwargs, **kwargs}
print('*****************************************************')
print(f'Realign and Redrizzle relative to {reference_filter}')
for filt in all_filters[1:]:
input_files = inputs_dict[filt]
output_file = output_dict[filt]
print(f'--{filt}: Realigning')
myalign(output_file, **realign_kwargs)
print(f'--{filt}: TweakBack')
with logger:
print(tweakback_kwargs)
tweakback.tweakback(output_file, input=input_files,
**tweakback_kwargs)
input_files = inputs_dict[filt]
output_file = output_dict[filt]
print(f'--{filt}: Redrizzling')
mydrizzle(input_files, output_file, **redrizzle_kwargs)
# dark_final = {filt: '{:s}_dark_drz.fits'.format(filt) for filt in all_filters}
# # Create temorary dark files
# print('--Creating temporary dark files')
# dark_inputs = {}
# dark_outputs = {}
# with logger:
# for filt in all_filters:
# if filt in darks_dict:
# dark_inputs[filt] = []
# for f in inputs_dict[filt]:
# newfile = _copy_dark_header(f, darks_dict[filt])
# dark_inputs[filt].append(newfile)
# dark_outputs[filt] = 'temp_dark_{:s}_drz.fits'.format(filt)
# Drizzle the images
# print('--Drizzling Images with {:s} kernel'.format(kernel))
# with logger:
# # Drizzle first filter
# first_filt = all_filters[0]
# if len(inputs_dict[first_filt]) >= 4:
# old_mode = d_kwgs.pop('combine_type', 'minmed')
# d_kwgs['combine_type'] = 'median'
# if len(inputs_dict[first_filt]) == 4:
# old_nhigh = d_kwgs.pop('combine_nhigh', 0)
# d_kwgs['combine_nhigh'] = 1
# astrodrizzle.AstroDrizzle(
# inputs_dict[first_filt], output=output_dict[first_filt], **d_kwgs)
# if len(inputs_dict[first_filt]) >= 4:
# d_kwgs['combine_type'] = old_mode
# if len(inputs_dict[first_filt]) == 4:
# d_kwgs['combine_nhigh'] = old_nhigh
# for filt in all_filters[1:]:
# if len(inputs_dict[filt]) >= 4:
# old_mode = d_kwgs.pop('combine_type', 'minmed')
# d_kwgs['combine_type'] = 'median'
# if len(inputs_dict[filt]) == 4:
# old_nhigh = d_kwgs.pop('combine_nhigh', 0)
# d_kwgs['combine_nhigh'] = 1
# # Align all other filters relative to first
# astrodrizzle.AstroDrizzle(
# inputs_dict[filt], output=output_dict[filt],
# final_refimage=output_dict[first_filt],
# **d_kwgs)
# if len(inputs_dict[filt]) >= 4:
# d_kwgs['combine_type'] = old_mode
# if len(inputs_dict[filt]) == 4:
# d_kwgs['combine_nhigh'] = old_nhigh
# _add_flag(output_dict[filt], exp_ratio=exp_ratio)
# Align and drizzle the dark images
# dark_kwgs = d_kwgs.copy()
# dark_kwgs.update(
# {'driz_separate': False,
# 'median': False,
# 'blot': False,
# 'driz_cr': False}
# )
# for filt in dark_inputs:
# if len(dark_inputs[filt]) < 3:
# old_mode = d_kwgs.pop('combine_type', 'median')
# d_kwgs['combine_type'] = 'minmed'
# astrodrizzle.AstroDrizzle(
# dark_inputs[filt], output=dark_outputs[filt],
# final_refimage=output_dict[first_filt], **dark_kwgs)
# if len(dark_inputs[filt]) < 3:
# d_kwgs['combine_type'] = old_mode
print('*************************')
print('Accounting for Noise and Errors')
# Compute and add dark values
# dark_meds = {}
# for filt in dark_outputs:
# dark_meds[filt] = _add_dark(output_dict[filt], dark_outputs[filt],
# dark_med_factor=dark_med_factor)
# Add flag HDUs
for filt in all_filters:
_add_flag(output_dict[filt], exp_ratio=exp_ratio)
# Compute Sky noise
sky_vals = {}
exposures = {}
for filt in all_filters:
with fits.open(output_dict[filt], mode='update') as h:
sky_vals[filt] = Table(h['HDRTAB'].data).to_pandas()[['MDRIZSKY']].values.sum() / 2.
exposures[filt] = h[0].header['EXPTIME']
h[0].header['SKYNOISE'] = (sky_vals[filt], 'Total sky value (sum of MDRIZSKY)')
# h[0].header['NOISE'] = (sky_vals[filt] + dark_meds.get(filt, 0),
# 'Total noise (SKYNOISE + DRKNOISE)')
# Track later steps
h[0].header['BKGDCOMP'] = ('PENDING', 'Has background been computed?')
h[0].header['SEXTRACT'] = ('PENDING', 'Have SExtractor sources been masked?')
h[0].header['MANUAL'] = ('PENDING', 'Have manual sources been masked?')
h[0].header['REGIONS'] = ('PENDING', 'Have PCMD regions been computed?')
for filt in all_filters:
# noise = dark_meds.get(filt, 0.) + sky_vals[filt]
noise = sky_vals[filt]
exp = exposures[filt]
print(f'--Sky Noise ({filt}): {noise:.1f} counts ({noise/exp:.3f} cps)')
# print('--Sky: {:.1f} counts ({:.3f} cps)'.format(sky_vals[filt], sky_vals[filt]/exp))
# print('--Dark: {:.1f} counts ({:.3f} cps)'.format(dark_meds.get(filt, 0), dark_meds.get(filt, 0)/exp))
# if clean:
# print('--Removing temporary files')
# all_files = []
# for v in dark_inputs.values():
# if isinstance(v, list):
# all_files.extend(v)
# else:
# all_files.append(v)
# for v in dark_outputs.values():
# if isinstance(v, list):
# all_files.extend(v)
# else:
# all_files.append(v)
# for f in all_files:
# try:
# os.remove(f)
# except FileNotFoundError:
# pass
print('--Done')
def offset_position():
"""
Check grism extractions for 13580 program that put Vy2-2 just
off of the left edge of the frame to test extrapolation of wavelength
calibration.
"""
import astropy.io.fits as pyfits
from astropy.table import Table as table
import drizzlepac
from drizzlepac import tweakreg, tweakback
import stwcs
import unicorn
unicorn.candels.make_asn_files(uniquename=True)
info = table.read('files.info', format='ascii.commented_header')
for filter in ['F098M', 'F105W']:
filter_files = list(info['FILE'][info['FILTER'] == filter])
#
files = glob.glob('VY2-2*%s_asn.fits' %(filter))
for file in files:
prep.prep_direct_grism_pair(direct_asn=file, grism_asn=False, radec='2mass.radec', scattered_light=False, skip_direct=False)
#
driz_images = glob.glob('VY2-2*%s_drz_sci.fits' %(filter))
tweakreg.TweakReg(driz_images, refimage=driz_images[0], updatehdr=True, updatewcs=True, catfile=None, xcol=2, ycol=3, xyunits='pixels', refcat=None, refxcol=1, refycol=2, refxyunits='degrees', shiftfile=True, outshifts='%s_shifts.txt' %(filter), outwcs='%s_wcs.fits' %(filter), searchrad=5, tolerance=12, wcsname='TWEAK', interactive=False, residplot='No plot', see2dplot=False, clean=True, headerlet=True, clobber=True)
tweakback.tweakback(driz_images[1])
#
drizzlepac.astrodrizzle.AstroDrizzle(filter_files, output='VY22-%s' %(filter), clean=True, skysub=False, final_scale=None, final_pixfrac=1, context=False, final_bits=576, preserve=False, driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7')
drizzlepac.astrodrizzle.AstroDrizzle(filter_files, output='VY22-%s' %(filter), clean=True, context=False, preserve=False, skysub=True, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True)
### Put WCS from direct F105W images into G102 at same POS-TARG
info = table.read('files.info', format='ascii.commented_header')
idx = np.arange(len(info))[info['FILTER'] == 'F105W']
asn = threedhst.utils.ASNFile('../RAW/ibhj01030_asn.fits')
for i in idx:
direct = info['FILE'][i]
dx, dy = info['POSTARG1'][i], info['POSTARG2'][i]
ix_gris = (info['POSTARG1'] == dx) & (info['POSTARG2'] == dy) & (info['FILTER'] == 'G102')
grism = info['FILE'][ix_gris][0]
sign = {True:'+', False:'-'}
#
asn.product = 'VY22%s%02d%s%02d-F105W' %(sign[dx > 0], np.abs(dx), sign[dy > 0], np.abs(dy))
asn.exposures = [direct.split('_flt')[0]]
asn.write(asn.product + '_asn.fits')
#
asn.product = 'VY22%s%02d%s%02d-G102' %(sign[dx > 0], np.abs(dx), sign[dy > 0], np.abs(dy))
asn.exposures = [grism.split('_flt')[0]]
asn.write(asn.product + '_asn.fits')
#### update WCS header
imd = pyfits.open(direct)
img = pyfits.open(grism)
sci_ext=1
direct_WCS = stwcs.wcsutil.HSTWCS(imd, ext=sci_ext)
drizzlepac.updatehdr.update_wcs(grism, sci_ext, direct_WCS, verbose=True)
#### Make reference catalog
root = 'VY22-F105W'
se = threedhst.sex.SExtractor()
se.aXeParams()
se.copyConvFile()
se.overwrite = True
se.options['CHECKIMAGE_TYPE'] = 'SEGMENTATION, BACKGROUND'
se.options['CHECKIMAGE_NAME'] = '%s_drz_seg.fits, %s_drz_bkg.fits' %(root, root)
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
se.options['WEIGHT_IMAGE'] = '%s_drz_wht.fits' %(root)
se.options['WEIGHT_GAIN'] = 'Y'
se.options['GAIN'] = '0'
se.options['FILTER'] = 'Y'
se.options['DETECT_THRESH'] = '2.'
se.options['ANALYSIS_THRESH'] = '2.'
se.options['DETECT_MINAREA'] = '10'
se.options['MASK_TYPE'] = 'NONE'
se.options['DEBLEND_NTHRESH'] = '64'
se.options['DEBLEND_MINCONT'] = '0.1'
se.options['SEEING_FWHM'] = '0.12'
se.options['BACK_TYPE'] = 'MANUAL'
se.options['BACKPHOTO_TYPE'] = 'LOCAL'
se.options['MAG_ZEROPOINT'] = '%.2f' %(unicorn.reduce.ZPs['F105W'])
se.options['CATALOG_TYPE'] = 'ASCII_HEAD'
se.options['CATALOG_NAME'] = '%s_drz_sci.cat' %(root)
status = se.sextractImage('%s_drz_sci.fits[0]' %(root))
threedhst.sex.sexcatRegions('%s_drz_sci.cat' %(root), '%s_drz_sci.reg' %(root), format=1)
#### Make interlaced images
files = glob.glob('VY22[+-]??[+-]??-F105W_asn.fits')
for file in files:
unicorn.reduce.interlace_combine(file.split('_asn')[0], growx=1, growy=1, NGROW=50, pad=60, view=False)
unicorn.reduce.interlace_combine(file.split('_asn')[0].replace('F105W', 'G102'), growx=1, growy=1, NGROW=50, pad=60, view=False)
red = unicorn.reduce
red.adriz_blot_from_reference(pointing=file.split('_asn')[0], pad=60, NGROW=50, growx=1, growy=1, auto_offsets=False, ref_exp=0, ref_image='VY22-F105W_drz_sci.fits', ref_ext=0, ref_filter='F105W', seg_image='VY22-F105W_drz_seg.fits', cat_file='VY22-F105W_drz_sci.cat')
### extract spectra
id = 798
files = glob.glob('VY22[+-]??[+-]??-F105W_asn.fits')
for file in files:
model = unicorn.reduce.GrismModel(root=file.split('-F10')[0], direct='F105W', grism='G102', growx=1, growy=1, grow_factor=1)
model.twod_spectrum(id, miny=-30, refine=False, CONTAMINATING_MAGLIMIT=0)
files = glob.glob('*2D.fits')
yi, xi = np.indices((30,30))
xs, ys = np.zeros(len(files)), np.zeros(len(files))
xpix = xs*0
for i, file in enumerate(files):
twod = unicorn.reduce.Interlace2D(file)
xs[i] = np.sum(xi*twod.im['DSCI'].data/twod.im['DSCI'].data.sum())
ys[i] = np.sum(xi*twod.im['DSCI'].data/twod.im['DSCI'].data.sum())
xpix[i] = twod.im[0].header['X_PIX']
xs -= np.median(xs) #+ 0.5
ys -= np.median(ys)
#xs -= 0.5
fig = plt.figure(figsize=[16,4])
fig.subplots_adjust(left=0.04, right=0.98, top=0.92)
for i, file in enumerate(files):
twod = unicorn.reduce.Interlace2D(file)
w, f = twod.optimal_extract(twod.im['SCI'].data)
c = {True: 'red', False: 'blue'}
#plt.plot(w-np.diff(w)[0]*xs[i], f/twod.im['SENS'].data, alpha=0.5, marker='o', ms=2, label='%s, %s' %(file[4:7], file[7:10])) # , color=c['2-2' in file]
ff = f*0.
for k in range(ff.shape[0]):
y0 = int(np.round(twod.im['YTRACE'].data[k]))
ff[k] = np.sum(twod.im['SCI'].data[y0-4:y0+4, k])
#
plt.plot(w-np.diff(w)[0]*xs[i], ff/twod.im['SENS'].data, alpha=0.5, marker='o', ms=2, label='%s, %s' %(file[4:7], file[7:10])) # , color=c['2-2' in file]
#plt.plot(twod.oned.data['wave']-np.diff(w)[0]*xs[i], twod.oned.data['flux']/twod.oned.data['sensitivity'], alpha=0.5, marker='o', ms=2, label='%s, %s' %(file[4:7], file[7:10])) # , color=c['2-2' in file]
#
print file, np.diff(w)[0]
#ds9.frame(i+1)
#ds9.view(twod.im['DSCI'].data)
PNe_lines = [9071.403457, 9534.921052, 10049.850283, 10833.000000, 12821.000000, 16112.000000, 16412.000000]
for line in PNe_lines:
plt.plot([line, line], [0.1,1.e5], color='black', linewidth=3, alpha=0.2, zorder=-5)
#plt.plot(w-np.diff(w)[0]*xs[i]-np.diff(w)[0], f/twod.im['SENS'].data, alpha=0.5, color='green', marker='o', ms=2)
plt.legend(loc='upper right', prop={'size':9}, title='POS-TARG')
plt.title('VY2-2, G102, 13580')
plt.xlim(8500, 11500)
plt.ylim(700,14000)
plt.ylim(600,64000)
plt.semilogy()
plt.xlabel(r'$\lambda$')
plt.savefig('vy22-edge_v2.pdf') #, dpi=100)
plt.close()
def hst_button(
galaxies,
skymethod='globalmin+match',
instruments="ACS/WFC",
prop_ids=None,
filters=None,
radius=None,
filepath=None,
download_data=True,
correct_astrometry=True,
create_mosaic=True,
jy_conversion=True,
verbose=False,
log_filename='hst.log',
):
"""Create a HST mosaic, given a galaxy name.
Using a galaxy name and radius, queries around that object,
downloads available HST data and mosaics into a final product. It
will create separate mosaics for each proposal ID, and the file structure
will look like ``/galaxy/HST/proposal_id/galaxy_instrument_filter_proposal_id.fits``.
N.B. I must confess to not being well-versed with HST data, so if
anyone can help improve this please let me know.
This data button uses a number of tools included in the drizzlepac
Python package. This includes alignimages/tweakreg and astrodrizzle,
which correct astrometry and are specifically tailored for the setup
of HST data. This means that 1) creating mosaics with this will likely
take a long time and 2) you will need a beefy computer (especially with
regards to hard drive space).
Args:
galaxies (str or list): Names of galaxies to create mosaics for.
Resolved by NED.
skymethod (str, optional): Method used for AstroDrizzle's background
matching step. In general, this can be left untouched but for
mosaics with little overlap, it may be worth playing around
with this. For instance, I've had some luck when there isn't
much overlap between exposures using 'globalmin'. Options are
'localmin', 'globalmin+match', 'globalmin', and 'match'. Defaults
to 'globalmin+match'.
instruments (str or list, optional): Instrument to download data
for. Can be any combination of 'ACS/WFC', 'WFC3/IR',
'WFC3/UVIS', 'WFPC2/PC', or 'WFPC2/WFC'. If you want all
available data for all these instruments, select 'all', but
this is not recommended! Defaults to 'ACS/WFC'.
prop_ids (str or list, optional): Proposal IDs to download data for.
Defaults to None, which will pull out all proposal IDs for each
instrument.
filters (str or list, optional): Filters to download data for.
The script will look for each filter, for each instrument.
Defaults to None, which will pull out all applicable filters
for each instrument, for each proposal ID.
radius (astropy.units.Quantity, optional): Radius around the
galaxy to search for observations. Defaults to None, where
it will query Ned to get size.
filepath (str, optional): Path to save the working and output
files to. If not specified, saves to current working
directory.
download_data (bool, optional): If True, will download data from
MAST. Defaults to True.
correct_astrometry (bool, optional): If True, will perform astrometric
corrections to the downloaded data using alignimages. Defaults
to True.
create_mosaic (bool, optional): Switching this to True will
mosaic the data using astrodrizzle as appropriate. Defaults
to True.
jy_conversion (bool, optional): Convert the mosaicked file from
raw units to Jy/pix. Defaults to True.
verbose (bool, optional): Can be used to suppress most of the
output messages produced during the process. Mainly useful
for debugging. Defaults to False.
log_filename (str, optional): Will produce a stripped down log
of what data the code is reducing. By default, will save to
galaxy/hst.log.
"""
if isinstance(galaxies, str):
galaxies = [galaxies]
if isinstance(instruments,str):
instruments = [instruments]
if instruments == 'all':
instruments = ['ACS/WFC',
'WFC3/IR','WFC3/UVIS',
'WFPC2/PC','WFPC2/WFC']
if isinstance(filters,str):
filters = [filters]
if isinstance(prop_ids,str):
prop_ids = [prop_ids]
if filepath is not None:
os.chdir(filepath)
orig_dir = os.getcwd()
if radius is not None:
original_radius = radius.copy()
else:
original_radius = None
steps = []
if download_data:
steps.append(1)
if correct_astrometry:
steps.append(2)
if create_mosaic:
steps.append(3)
if jy_conversion:
steps.append(4)
# Set up folders for various corrections
os.environ['CRDS_SERVER_URL'] = 'https://hst-crds.stsci.edu'
os.environ['CRDS_PATH'] = orig_dir+'/reference_files'
os.environ['iref'] = orig_dir+'/reference_files/references/hst/wfc3/'
os.environ['jref'] = orig_dir+'/reference_files/references/hst/acs/'
os.environ['uref'] = orig_dir+'/reference_files/references/hst/wfpc2/'
# For large proposals, astrodrizzle can run into file open
# issues so raise the max file open amount.
_, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE,(hard,hard))
# Change the temp directory -- if this gets filled up it can cause
# problems.
orig_tmpdir = os.environ['TMPDIR']
if not os.path.exists('tmp'):
os.mkdir('tmp')
os.environ['TMPDIR'] = orig_dir+'/tmp'
for galaxy in galaxies:
if not os.path.exists(galaxy):
os.mkdir(galaxy)
if not os.path.exists(galaxy+'/HST'):
os.mkdir(galaxy+'/HST')
if not verbose:
# Various packages used here put out a lot of messages. Silence info messages.
loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]
for logger in loggers:
logger.setLevel(logging.ERROR)
# Even if verbose is not True, still print out some useful messages to the
# console.
hst_logger = logging.getLogger('data_buttons')
handler = logging.FileHandler(galaxy+'/'+log_filename,mode='w')
hst_logger.addHandler(handler)
hst_logger.addHandler(logging.StreamHandler())
hst_logger.setLevel(logging.INFO)
hst_logger.info('Beginning '+galaxy)
hst_logger.info(' ')
hst_logger.info(' ')
if radius is None:
try:
size_query = Ned.get_table(galaxy,table='diameters')
radius = np.max(size_query['NED Major Axis'])/2*u.arcsec
radius = radius.to(u.deg)
except:
hst_logger.warning(galaxy+' not resolved by Ned, using 0.2deg radius.')
radius = 0.2*u.degree
obs_table = Observations.query_criteria(objectname=galaxy,
radius=radius,
obs_type='all',
obs_collection='HST')
# Ignore any calibration observations.
obs_table = obs_table[obs_table['intentType'] == 'science']
for instrument in instruments:
# Pixel sizes for final mosaics selected to match the HLA.
pix_size = {'ACS/HRC':0.025,
'ACS/SBC':0.03,
'ACS/WFC':0.05,
'NICMOS/NIC1':0.025,
'NICMOS/NIC2':0.05,
'NICMOS/NIC3':0.1,
'WFC3/IR':0.09,
'WFC3/UVIS':0.04,
'WFPC2/PC':0.05,
'WFPC2/WFC':0.1}[instrument]
# Bits to consider good for drizzling.
bits = {'ACS/HRC':256,
'ACS/SBC':256,
'ACS/WFC':256,
'NICMOS/NIC1':0,
'NICMOS/NIC2':0,
'NICMOS/NIC3':0,
'WFC3/IR':768,
'WFC3/UVIS':256,
'WFPC2/PC':'8,1024',
'WFPC2/WFC':'8,1024'}[instrument]
# Filename extension, in order of preference.
suffixes = {'ACS/WFC':['FLC','FLT'],
'WFC3/IR':['FLT'],
'WFC3/UVIS':['FLC','FLT'],
'WFPC2/PC':[['C0M','C1M']],
'WFPC2/WFC':[['C0M','C1M']],
}[instrument]
# The instruments often have / in the name, so account for
# this in making folders and files.
hst_logger.info('Beginning '+instrument)
if not os.path.exists(galaxy+'/HST/'+instrument.replace('/','_')):
os.mkdir(galaxy+'/HST/'+instrument.replace('/','_'))
reset_filters = False
instrument_table = obs_table[obs_table['instrument_name'] == instrument]
reset_prop_ids = False
if not prop_ids:
prop_ids = list(np.unique(instrument_table['proposal_id']))
reset_prop_ids = True
hst_logger.info('Available proposal IDs: '+','.join(prop_ids))
hst_logger.info(' ')
for prop_id in prop_ids:
hst_logger.info('Proposal ID: '+str(prop_id))
prop_table = instrument_table[instrument_table['proposal_id'] == prop_id]
if not filters:
filters = list(np.unique(prop_table['filters']))
reset_filters = True
hst_logger.info('Available filters: '+','.join(filters))
for hst_filter in filters:
# If we have a highly illegal filter, just skip.
# TODO: This needs to be sorted for some fringe
# cases, probably.
if not hst_filter[0] == 'F':
continue
hst_logger.info('Filter: '+str(hst_filter))
# Pull out available data and download.
filter_table = prop_table[prop_table['filters'] == hst_filter]
if len(filter_table) == 0:
hst_logger.warning('No available data to download. Skipping...')
continue
data_products_id = Observations.get_product_list(filter_table)
for suffix in suffixes:
download_table = Observations.filter_products(data_products_id,
productSubGroupDescription=suffix,
mrp_only=False)
if len(download_table) > 0:
break
if isinstance(suffix,list):
filename_exts = [ext.lower() for ext in suffix]
else:
filename_exts = [suffix.lower()]
hst_logger.info(instrument+'/'+prop_id+'/'+hst_filter)
if not os.path.exists(galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter):
os.mkdir(galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter)
if not os.path.exists(galaxy+'/HST/'+prop_id):
os.mkdir(galaxy+'/HST/'+prop_id)
full_filepath = (galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter+
'/'
+prop_id)
if not os.path.exists(full_filepath):
os.mkdir(full_filepath)
if 1 in steps:
# Download files
download_mast(download_table,
download_dir="hst_temp/" + galaxy)
if not os.path.exists(full_filepath+'/raw'):
os.mkdir(full_filepath+'/raw')
if not os.path.exists(full_filepath+'/outputs'):
os.mkdir(full_filepath+'/outputs')
# Pull out the relevant files, and move to base folder.
for filename_ext in filename_exts:
matches = []
for root, _, filenames in os.walk("hst_temp/" + galaxy):
for filename in fnmatch.filter(
filenames, "*_"+filename_ext+".fits"
):
matches.append(os.path.join(root, filename))
for match in matches:
filename = match.split('/')
os.rename(match,full_filepath+'/raw/'+filename[-1])
# Clean up any temporary files.
shutil.rmtree("hst_temp/" + galaxy, ignore_errors=True)
filename_ext = filename_exts[0]
hst_files = glob.glob(full_filepath+'/raw/*_'+filename_ext+'.fits')
if 2 in steps:
# First, update the WCS information in case it's
# required.
for filename_ext in filename_exts:
hst_files = glob.glob(full_filepath+'/raw/*_'+filename_ext+'.fits')
crds.assign_bestrefs(hst_files,
sync_references=True)
# For WFPC2, the CRDS doesn't download everything
# needed. Download the GEIS data files and
# rerun the bestrefs assignment.
if 'WFPC2' in instrument:
geis_hdrs = glob.glob(os.environ['uref']+'/*h')
for geis_hdr in geis_hdrs:
geis_data = geis_hdr[:-1]+'d'
if not os.path.exists(geis_data):
geis_data = geis_data.split('/')[-1]
print(geis_data)
print(os.environ['uref'])
wget.download(
os.environ['CRDS_SERVER_URL']+'/unchecked_get/references/hst/'+geis_data,
out=os.environ['uref'])
crds.assign_bestrefs(hst_files,sync_references=True)
for hst_file in hst_files:
stwcs.updatewcs.updatewcs(hst_file,
use_db=False)
os.chdir(full_filepath+'/raw')
filename_ext = filename_exts[0]
hst_files = glob.glob('*_'+filename_ext+'.fits')
# Normalize all files.
photeq.photeq(', '.join(hst_files),readonly=False)
os.rename('photeq.log','../outputs/photeq.log')
if 'WFPC' in instrument:
# Using tweakreg, align each frame to GAIA.
gaia_table = Gaia.query_object_async(coordinate=galaxy,
radius=2*radius)
ras = gaia_table['ra']
decs = gaia_table['dec']
source_table = Table([ras,decs])
source_table.write('gaia.cat',
format='ascii.fast_commented_header')
tweakreg.TweakReg(hst_files,
imagefindcfg={'threshold':5,'conv_width':3},
refcat='gaia.cat',
#expand_refcat=True,
enforce_user_order=False,
shiftfile=True,
outshifts='shifts.txt',
searchrad=10,
minobj=5,
separation=0,
updatehdr=True,
reusename=True,
wcsname='TWEAK',
interactive=False,
fitgeometry='general',
clean=True,
see2dplot=False
)
# Update the c1m files to use the TWEAK
# wcs
for hst_file in hst_files:
dq_file = hst_file.replace('c0','c1')
tweakback.tweakback(hst_file,
dq_file,
newname='TWEAK')
plot_files = glob.glob('*.png')
for plot_file in plot_files:
os.remove(plot_file)
cat_files = glob.glob('*.coo')
for cat_file in cat_files:
os.remove(cat_file)
os.rename('shifts_wcs.fits','../outputs/shifts_wcs.fits')
os.rename('tweakreg.log','../outputs/tweakreg.log')
os.rename('shifts.txt','../outputs/shifts.txt')
elif 'ACS' in instrument or 'WFC3' in instrument:
# Correct astrometry using alignimages. First,
# correct each frame separately.
pool = mp.Pool(mp.cpu_count())
suitable_hst_files = pool.map(astrometric_correction,
hst_files)
pool.close()
suitable_hst_files = [x for x in suitable_hst_files
if x is not None]
if len(suitable_hst_files) == 0:
hst_logger.warning('Failure with astrometry corrections. Skipping')
os.chdir(orig_dir)
continue
# Now, align every suitable frame simultaneously.
output_table = astrometric_correction(suitable_hst_files)
with open('../outputs/astrometry.pkl','wb') as table_file:
pickle.dump(output_table,table_file)
else:
raise Exception('Unknown instrument!')
os.chdir(orig_dir)
os.chdir(full_filepath)
if 3 in steps:
os.chdir('raw')
if 'WFPC2' in instrument:
hst_files = glob.glob('*_c0m.fits')
wcskey = 'TWEAK'
elif 'ACS' in instrument or 'WFC3' in instrument:
with open('../outputs/astrometry.pkl','rb') as table_file:
output_table = pickle.load(table_file)
# We only want fits where an acceptable astrometric
# solution has been found.
suitable_fits = np.where(output_table['fit_qual'] < 5)
# (output_table['fit_qual'] >= 1)
hst_files = list(output_table[suitable_fits]['imageName'])
if len(output_table[suitable_fits]) == 0:
hst_logger.warning('Failure with astrometry corrections. Skipping')
os.chdir(orig_dir)
continue
wcskey = ' '
else:
raise Exception('Unknown instrument!')
# Following Dalcanton+ (2012), group exposures into
# long (>50s) and short (<=50s), and process for cosmic
# rays separately
exp_times = []
for hst_file in hst_files:
hdu = fits.open(hst_file)[0]
exp_time = hdu.header['EXPTIME']
exp_times.append(exp_time)
for exp_group in ['short','long']:
hst_files_group = []
for i in range(len(exp_times)):
if exp_times[i] > 50 and exp_group == 'long':
hst_files_group.append(hst_files[i])
elif exp_times[i] <= 50 and exp_group == 'short':
hst_files_group.append(hst_files[i])
if len(hst_files_group) == len(hst_files):
exp_group = ''
if len(hst_files_group) == 0:
continue
if len(exp_group) > 0:
output_name = '../outputs/'+galaxy+'_'+exp_group
drizzle_log_name = '../outputs/astrodrizzle_'+exp_group+'.log'
else:
output_name = '../outputs/'+galaxy
drizzle_log_name = '../outputs/astrodrizzle.log'
# Perform the mosaicking. Generally, use iminmed.
# However, sometimes iminmed will fail so
# for the other instruments we'll use imedian as
# a fallback.
combine_types = ['iminmed','imedian']
if 'WFPC2' in instrument:
combine_nhigh = 1
else:
combine_nhigh = 0
for combine_type in combine_types:
try:
astrodrizzle.AstroDrizzle(
input=hst_files_group,
output=output_name,
preserve=False,
clean=True,
combine_type=combine_type,
combine_nhigh=combine_nhigh,
skymethod=skymethod,
sky_bits=bits,
driz_sep_bits=bits,
driz_sep_fillval=99999,
combine_hthresh=90000,
final_scale=pix_size,
final_bits=bits,
final_fillval=0,
wcskey=wcskey,
final_rot=0,
)
break
except ValueError:
pass
# Move the AstroDrizzle log.
os.rename('astrodrizzle.log',
drizzle_log_name)
# Move back to the original directory.
os.chdir(orig_dir)
if 4 in steps:
mosaic_outputs = glob.glob(full_filepath+'/outputs/*_sci.fits')
for mosaic_output in mosaic_outputs:
# Replace any fillvals with NaNs.
hdu = fits.open(mosaic_output)[0]
hdu.data[hdu.data == 0] = np.nan
fits.writeto(mosaic_output,
hdu.data,hdu.header,
overwrite=True)
if '_long_' in mosaic_output.split('/')[-1]:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'_long.fits')
elif '_short_' in mosaic_output.split('/')[-1]:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'_short.fits')
else:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'.fits')
convert_to_jy(mosaic_output,
new_filename)
if reset_filters:
filters = None
hst_logger.info(' ')
if reset_prop_ids:
prop_ids = None
hst_logger.info(' ')
if original_radius is None:
radius = None
else:
radius = original_radius.copy()
# Clear out the tmp folder and reset to the original.
shutil.rmtree('tmp/', ignore_errors=True)
os.environ['TMPDIR'] = orig_tmpdir
def runpipe( outroot, onlyfilters=[], onlyepochs=[],
# Run all the processing steps
doall=False,
# Setup : copy flts into sub-directories
dosetup=False, epochlistfile=None,
# construct a ref image
dorefim=False,
# Single Visit tweakreg/drizzle pass :
dodriz1=False, drizcr=False, intravisitreg=False,
# Register to a given image or epoch, visit and filter
doreg=False, refim=None, refepoch=None, refvisit=None, reffilter=None,
# Drizzle registered flts by epoch and filter
dodriz2=False,
# make diff images
dodiff=False, tempepoch=0,
refcat=None,
interactive=False, threshold=4, peakmin=None, peakmax=None,
rfluxmax=27, rfluxmin=14, searchrad=1.5,
mjdmin=0, mjdmax=0, epochspan=5,
ra=None, dec=None, rot=0, imsize_arcsec=None,
pixscale=None, pixfrac=None, wht_type='ERR',
clobber=False, verbose=True, debug=False ):
"""
Primary pipeline function. Executes all the intermediate steps:
register, sort, drizzle, subtract, mask
"""
import pyfits
import shutil
if debug : import pdb; pdb.set_trace()
from drizzlepac.tweakback import tweakback
topdir = os.path.abspath( '.' )
fltdir = outroot + '.flt'
if doall :
dosetup=True
dorefim=True
dodriz1=True
doreg=True
dodriz2=True
dodiff=True
if onlyfilters :
if type(onlyfilters)==str :
onlyfilters = onlyfilters.lower().split(',')
onlyfilters = [ filt[:5].lower() for filt in onlyfilters ]
if type(onlyepochs) in [str,int,float] :
onlyepochs = [ int(ep) for ep in str(onlyepochs).split(',') ]
# STAGE 0 : (always runs)
# get a list of exposures and epochs, sorting flt files into epochs
fltlist = glob.glob( "%s/*fl?.fits"%fltdir )
if not len( fltlist ) :
raise( exceptions.RuntimeError( "There are no flt/flc files in %s !!"%fltdir) )
explist = exposures.get_explist( fltlist, outroot=outroot )
if not epochlistfile :
epochlistfile = "%s_epochs.txt"%explist[0].outroot
if os.path.exists( epochlistfile ) and not clobber :
print( "%s exists. Adopting existing epoch sorting."%epochlistfile )
exposures.read_epochs( explist, epochlistfile, checkradec=[ra,dec],
onlyfilters=onlyfilters )
else :
exposures.define_epochs( explist, epochspan=epochspan,
mjdmin=mjdmin, mjdmax=mjdmax )
exposures.print_epochs( explist, outfile=epochlistfile,
verbose=verbose, clobber=clobber, checkradec=[ra,dec],
onlyfilters=onlyfilters, onlyepochs=onlyepochs )
if refim and not os.path.exists( refim ) :
raise exceptions.RuntimeError( 'Ref image %s does not exist.'%refim )
if not refim :
# No refimage has been specified, so set the default refimage name
refdrzdir = '%s.refim'%outroot
refimbasename = '%s_wcsref_sci.fits'%(outroot)
refim = os.path.abspath( os.path.join( refdrzdir, refimbasename ) )
new_explist = []
for exp in explist:
if onlyfilters and exp.filter not in onlyfilters :
continue
elif onlyepochs and exp.epoch not in onlyepochs :
continue
else:
new_explist.append( exp )
explist = new_explist
# STAGE 1 :
# copy pristine flt files into epoch sub-directories
if dosetup :
if verbose :
print("SNDRIZZLE : (1) SETUP : copying flt files into subdirs")
exposures.copy_to_epochdirs( explist, checkradec=[ra,dec],
onlyfilters=onlyfilters, onlyepochs=onlyepochs,
verbose=verbose, clobber=clobber )
FEVgrouplist = sorted( np.unique( [ exp.FEVgroup for exp in explist ] ) )
FEgrouplist = sorted( np.unique( [ exp.FEgroup for exp in explist ] ) )
filterlist = sorted( np.unique( [ exp.filter for exp in explist ] ) )
epochlist = sorted( np.unique([ exp.epoch for exp in explist ] ) )
# STAGE 2 :
# Construct the WCS reference image
if dorefim :
if verbose :
print("SNDRIZZLE : (2) REFIM : Constructing WCS ref image.")
if os.path.exists( refim ) and clobber :
os.remove( refim )
if os.path.exists( refim ) :
print("%s already exists. Not clobbering."%refim)
else :
refdrzdir = os.path.dirname( refim )
if verbose :
print( " Constructing reference image %s in %s"%(
os.path.basename(refim), refdrzdir ) )
# Collect the necessary flt files for constructing the ref image
if not refepoch :
refepoch = np.min( epochlist )
if not reffilter :
reffilter = sorted( [ exp.filter for exp in explist
if exp.epoch==refepoch ] )[0]
reffilter = reffilter.lower()
# if refvisit is not set, then find the maximum depth visit and use that
if not refvisit :
visits, exp_times, visit_depth = np.array( ([], [], []) )
for exp in explist:
if (exp.epoch != refepoch) and (exp.filter != reffilter):
continue
visits = np.append( visits, exp.visit )
exp_times = np.append( exp_times, exp.exposure_time )
unique_visits = np.unique( visits )
for u in unique_visits:
ind = np.where( visits == u )
visit_depth = np.append( visit_depth, np.sum(exp_times[ind]) )
max_ind = np.argmax( visit_depth )
refvisit = unique_visits[max_ind]
refvisit = refvisit.upper()
explistRI = sorted( [ exp for exp in explist if exp.epoch==refepoch
and exp.filter==reffilter and exp.visit==refvisit ] )
refdrzdir = os.path.dirname( refim )
if not os.path.isdir( refdrzdir ) :
os.makedirs( refdrzdir )
for exp in explistRI :
fltfile = os.path.basename( exp.filename )
refsrcdir = os.path.abspath( exp.epochdir )
shutil.copy( os.path.join( refsrcdir, fltfile ), refdrzdir )
fltlistRI = [ exp.filename for exp in explistRI ]
refimroot = '%s_wcsref'%outroot
os.chdir( refdrzdir )
refimsci, refimwht = drizzle.secondDrizzle(
fltlistRI, refimroot, refimage=None, ra=ra, dec=dec, rot=rot,
imsize_arcsec=imsize_arcsec, wht_type=wht_type,
pixscale=pixscale, pixfrac=pixfrac,
clobber=clobber, verbose=verbose, debug=debug )
os.rename(refimsci,refim)
os.chdir(topdir)
print refvisit
# STAGE 3 :
# Drizzle together each drizzle group (same epoch, visit and
# filter), using almost-default parameters, doing CR rejection
# and applying intravisit registrations if requested. The output
# is a drz_sci.fits file in the native rotation.
if dodriz1 :
if verbose :
print("SNDRIZZLE : (3) DRIZ1 : first astrodrizzle pass.")
for FEVgroup in FEVgrouplist :
explistFEV = [ exp for exp in explist if exp.FEVgroup == FEVgroup ]
thisepoch = explistFEV[0].epoch
thisfilter = explistFEV[0].filter
if onlyepochs and thisepoch not in onlyepochs : continue
if onlyfilters and thisfilter not in onlyfilters : continue
epochdir = explistFEV[0].epochdir
fltlistFEV = [ exp.filename for exp in explistFEV ]
outrootFEV = '%s_%s_nat'%(outroot,FEVgroup)
os.chdir( epochdir )
outsciFEV = '%s_%s_sci.fits'%(outrootFEV,explistFEV[0].drzsuffix)
if os.path.exists( outsciFEV ) and clobber :
os.remove( outsciFEV )
if os.path.exists( outsciFEV ) :
if verbose: print("%s exists. Not clobbering."%outsciFEV )
os.chdir( topdir )
continue
if intravisitreg :
# run tweakreg for intravisit registration tweaks
register.intraVisit(
fltlistFEV, peakmin=peakmin, peakmax=peakmax,
threshold=threshold, interactive=interactive, debug=debug )
drizzle.firstDrizzle(
fltlistFEV, outrootFEV, driz_cr=drizcr,
wcskey=((intravisitreg and 'INTRAVIS') or '') )
# TODO : Update the WCS of the refim so that it matches the reference catalog
#if refcat :
# if verbose : print( " Registering reference image %s to ref catalog %s"%(refim,refcat))
# register.toCatalog( refim, refcat, refim, rfluxmax=rfluxmax, rfluxmin=rfluxmin,
# searchrad=searchrad, peakmin=peakmin, peakmax=peakmax, threshold=threshold,
# interactive=interactive, debug=debug )
os.chdir( topdir )
if doreg or dodriz2 :
if not os.path.exists( refim ):
raise exceptions.RuntimeError("No refim file %s! Maybe you should re-run with dorefim=True."%refim)
else :
refim = os.path.abspath( refim )
# Fix the output ra and dec center point if not provided by the user.
if not ra and not dec :
ra = pyfits.getval( refim, "CRVAL1" )
dec = pyfits.getval( refim, "CRVAL2" )
# STAGE 4 :
# Run tweakreg to register all the single-visit drz images
# to a common WCS defined by the refcat/refim, updating the drz file headers.
# Then use tweakback to propagate that back into the flt files
if doreg :
if verbose :
print("SNDRIZZLE : (4) REG : running tweakreg.")
for FEVgroup in FEVgrouplist :
explistFEV = [ exp for exp in explist if exp.FEVgroup == FEVgroup ]
thisepoch = explistFEV[0].epoch
thisfilter = explistFEV[0].filter
if onlyepochs and thisepoch not in onlyepochs : continue
if onlyfilters and thisfilter not in onlyfilters : continue
epochdir = explistFEV[0].epochdir
fltlistFEV = [ exp.filename for exp in explistFEV ]
outrootFEV = '%s_%s_nat'%(outroot,FEVgroup)
outsciFEV = '%s_%s_sci.fits'%(outrootFEV,explistFEV[0].drzsuffix)
refimpath = os.path.abspath(refim)
if refcat : refcatpath = os.path.abspath(refcat)
else : refcatpath = None
os.chdir( epochdir )
if not os.path.exists( outsciFEV ) :
exceptions.RuntimeError( "Missing %s."%outsciFEV )
# register to the ref image and ref catalog
origwcs = pyfits.getval( outsciFEV,'WCSNAME').strip()
wcsname = register.toRefim(
outsciFEV, refim=refimpath, refcat=refcatpath,
searchrad=searchrad, peakmin=peakmin, peakmax=peakmax,
threshold=threshold,
interactive=interactive, clobber=clobber, debug=debug )
# Run tweakback to update the constituent flts
tweakback( outsciFEV, input=fltlistFEV, origwcs=origwcs,
wcsname=wcsname, verbose=verbose, force=clobber )
os.chdir(topdir)
# STAGE 5
# Second and final astrodrizzle pass, wherein we rerun
# astrodrizzle to get wcs- and pixel-registered drz images
# combining all flt files with the same filter and epoch.
if dodriz2 :
if verbose :
print("SNDRIZZLE : (5) DRIZ2 : second astrodrizzle pass.")
for FEgroup in FEgrouplist :
explistFE = [ exp for exp in explist if exp.FEgroup == FEgroup ]
thisepoch = explistFE[0].epoch
thisfilter = explistFE[0].filter
if onlyepochs and thisepoch not in onlyepochs : continue
if onlyfilters and thisfilter not in onlyfilters : continue
epochdir = explistFE[0].epochdir
fltlistFE = [ exp.filename for exp in explistFE ]
outrootFE = '%s_%s_reg'%(outroot,FEgroup)
err = None
os.chdir( epochdir )
outsciFE = '%s_%s_sci.fits'%(outrootFE,explistFE[0].drzsuffix)
if os.path.exists( outsciFE ) and clobber :
os.remove( outsciFE )
if os.path.exists( outsciFE ) :
if verbose: print("%s exists. Not clobbering."%outsciFE )
os.chdir( topdir )
continue
outsciFE, outwhtFE = drizzle.secondDrizzle(
fltlistFE, outrootFE, refimage=refim, ra=ra, dec=dec, rot=rot,
imsize_arcsec=imsize_arcsec, wht_type=wht_type,
pixscale=pixscale, pixfrac=pixfrac,
clobber=clobber, verbose=verbose, debug=debug )
outbpxFE = outwhtFE.replace('_wht','_bpx')
outbpxFE = badpix.zerowht2badpix(
outwhtFE, outbpxFE, verbose=verbose, clobber=clobber )
os.chdir( topdir )
# STAGE 6
# Define a template epoch for each filter and subtract it from the other epochs
if dodiff :
if verbose :
print("SNDRIZZLE : (6) DIFF : subtracting template images.")
for filter in filterlist :
if onlyfilters and filter not in onlyfilters : continue
template = None
if tempepoch == None :
# User did not define a template epoch, so we default
# to use the first available epoch
for epoch in epochlist :
explistFE = [ exp for exp in explist
if exp.filter==filter and exp.epoch==epoch ]
if len(explistFE)==0: continue
drzsuffix = explistFE[0].drzsuffix
epochdir = explistFE[0].epochdir
FEgroup = explistFE[0].FEgroup
outrootFE = '%s_%s'%(outroot, FEgroup )
if os.path.isfile( os.path.join( epochdir, outrootFE+"_reg_%s_sci.fits"%drzsuffix )) :
tempepoch = epoch
break
else :
explistF = [ exp for exp in explist if exp.filter==filter ]
drzsuffix = explistF[0].drzsuffix
# now do the subtractions
tempdir = outroot+'.e%02i'%tempepoch
tempsci = os.path.join( os.path.abspath(tempdir), '%s_%s_e%02i_reg_%s_sci.fits'%(outroot,filter,tempepoch,drzsuffix))
tempwht = tempsci.replace('sci.fits','wht.fits')
tempbpx = tempsci.replace('sci.fits','bpx.fits')
topdir = os.path.abspath( '.' )
for epoch in epochlist :
if onlyepochs and epoch not in onlyepochs : continue
if epoch == tempepoch : continue
explistFE = [ exp for exp in explist if exp.filter==filter and exp.epoch==epoch ]
if len(explistFE)==0 : continue
epochdirFE = explistFE[0].epochdir
outrootFE = "%s_%s"%(outroot,explistFE[0].FEgroup)
drzsuffix = explistFE[0].drzsuffix
thisregsci = "%s_reg_%s_sci.fits"%(outrootFE,drzsuffix)
thisregwht = "%s_reg_%s_wht.fits"%(outrootFE,drzsuffix)
thisregbpx = "%s_reg_%s_bpx.fits"%(outrootFE,drzsuffix)
os.chdir( epochdirFE )
if not os.path.isfile( thisregsci ) :
os.chdir( topdir )
continue
thisdiffim = outrootFE + "-e%02i_sub_sci.fits"%tempepoch
if not os.path.isfile( tempsci ) or not os.path.isfile( thisregsci) :
print( "Can't create diff image %s"%thisdiffim )
print( " missing either %s"%tempsci )
print( " or %s"%thisregsci )
os.chdir( topdir )
continue
diffim = imarith.imsubtract( tempsci, thisregsci, outfile=thisdiffim,
clobber=clobber, verbose=verbose, debug=debug)
diffwht = badpix.combine_ivm_maps( thisregwht, tempwht,
diffim.replace('sci.fits','wht.fits'),
clobber=clobber, verbose=verbose )
diffbpx = badpix.unionmask( tempbpx, thisregbpx,
diffim.replace('sci.fits','bpx.fits'),
clobber=clobber, verbose=verbose)
diffim_masked = badpix.applymask( diffim, diffbpx,
clobber=clobber, verbose=verbose)
print("Created diff image %s, wht map %s, and bpx mask %s"%(
diffim_masked, diffwht, diffbpx ) )
os.chdir( topdir )
return 0
def drizzle_fields( obsDates, fltList, cluster, filter, search_rad=1, thresh=1):
'''
PURPOSE : IF THERE ARE OBSERVATIONS FROM DIFFERENT DATES / OBSRUNS
THEN DRIZZLE THE DATES THAT ARE THE SAME TOGETHER
THEN TWEAK THEM TO ONE REFERENCE DRIZZLE IMAGE
TWEAK BACK TO THE FLT FITS
INPUTS : OBSDATES : CAN BE ANY UNIQUE IDENTIFIER TO NAME OF THE RUN
OR DATE
thresh=1.0
search_rad=1.0 #Often this parameter is too big if the images hardly overlap. Keep at 1 if limited sources
'''
outputfilename = cluster+'_'+filter
uniqueObs = np.unique(np.array(obsDates))
print 'MORE THAN ONE OBSERVATION DATE/RUN SO DRIZZLING INDIVIDUALLY'
print uniqueObs
call(["mkdir", "keep"])
for iDate in uniqueObs:
obsDrizzle = np.array(fltList)[ np.array(obsDates) == str(iDate) ]
drizzleString = ",".join(obsDrizzle)
print iDate
print obsDrizzle
if not os.path.isfile("keep/"+str(iDate)+"_drz_sci.fits"):
astrodrizzle.AstroDrizzle(drizzleString, \
output=str(iDate), \
final_wcs=True, \
final_scale=0.03, \
final_pixfrac=0.8, \
combine_type='iminmed')
call(["cp",str(iDate)+"_drz_sci.fits","keep"])
else:
call(["cp","keep/"+str(iDate)+"_drz_sci.fits","."])
#3. NOW TWEAK EACH DRIZZLED IMAGE
#-----------------------------------------
#First sextract them using tweak_sextract
drzList=[]
for iDate in uniqueObs:
drzList.append(str(iDate)+"_drz_sci.fits")
try:
drzList.remove(str(outputfilename)+'_drz_sci.fits')
except:
print 'its okay, cluster drizzle doesn exist'
#Run sextractor on the images
tweaksex.tweakreg_sextract(drzList,'catfile')
refDate=obsDates[1]#
refimage=drzList[1]#
drzString= ",".join(drzList)
tweaksex.ref_sex( refimage )
tweakreg.TweakReg(drzString, \
residplot='both', \
see2dplot=True, \
threshold=np.float(thresh), \
searchrad=np.float(search_rad), \
fitgeometry='rscale', updatehdr='True', \
refimage=refimage,
catfile='catfile', \
xcol=1, ycol=2, fluxcol=3, \
minobj=2, \
refcat='reference.cat', \
refxcol=1, refycol=2, rfluxcol=3)
for iDate in uniqueObs:
if iDate != obsDates:
TweakFLT= np.array(fltList)[ np.array(obsDates) == str(iDate) ]
tweakString = ",".join(TweakFLT)
print tweakString
tweakback.tweakback( str(iDate)+'_drz_sci.fits', \
input=tweakString )
def applyOffset(drzfile, fltfile, hlet=False):
"""Apply offset to an flt image from an aligned drizzled image"""
tweakback.tweakback(drzfile, input=fltfile, origwcs='DRZWCS', verbose=True, force=True)
if hlet:
headerlet.write_headerlet(fl, d, output=None, sciext='SCI', wcsname='DRZWCS_1', wcskey='PRIMARY', destim=None, sipname=None, npolfile=None, d2imfile=None, author=None, descrip=None, history=None, nmatch=None, catalog=None, attach=True, clobber=False, logging=False)
