def __init__(self, flt_files=DEMO_LIST, info=None, driz_image=DEMO_IMAGE):
"""
Object for making drizzled PSFs
Parameters
----------
flt_files : list
List of FLT files that were used to create the drizzled image.
driz_image : str
Filename of the drizzled image.
"""
if info is None:
self.wcs, self.footprint = self._get_flt_wcs(flt_files)
self.flt_files = flt_files
else:
self.wcs, self.footprint = info
self.flt_files = list(self.wcs.keys())
self.ePSF = utils.EffectivePSF()
self.driz_image = driz_image
self.driz_header = pyfits.getheader(driz_image)
self.driz_wcs = pywcs.WCS(self.driz_header)
self.driz_pscale = utils.get_wcs_pscale(self.driz_wcs)
def get_driz_cutout(self,
ra=53.06967306,
dec=-27.72333015,
size=15,
get_cutout=False):
xy = self.driz_wcs.all_world2pix(np.array([[ra, dec]]), 0)[0]
xyp = np.cast[int](np.round(xy))
N = int(np.round(size * 0.128254 / self.driz_pscale))
slx = slice(xyp[0] - N, xyp[0] + N)
sly = slice(xyp[1] - N, xyp[1] + N)
wcs_slice = model.ImageData.get_slice_wcs(self.driz_wcs, slx, sly)
wcs_slice.pscale = utils.get_wcs_pscale(wcs_slice)
# outsci = np.zeros((2*N,2*N), dtype=np.float32)
# outwht = np.zeros((2*N,2*N), dtype=np.float32)
# outctx = np.zeros((2*N,2*N), dtype=np.int32)
if get_cutout:
os.system("getfits -o sub.fits {0} {1} {2} {3} {3}".format(
self.driz_image, xyp[0], xyp[1], 2 * N))
hdu = pyfits.open('sub.fits')
return slx, sly, hdu
return slx, sly, wcs_slice
def _get_flt_wcs(flt_files):
"""
TBD
"""
from shapely.geometry import Polygon, Point
from grizli import utils
wcs = OrderedDict()
footprint = OrderedDict()
flt_keys = []
for file in flt_files:
flt_j = pyfits.open(file)
for ext in range(1, 5):
if ('SCI', ext) in flt_j:
key = file, ext
wcs[key] = pywcs.WCS(flt_j['SCI', ext],
relax=True,
fobj=flt_j)
wcs[key].pscale = utils.get_wcs_pscale(wcs[key])
wcs[key].expweight = flt_j[0].header['EXPTIME']
footprint[key] = Polygon(wcs[key].calc_footprint())
flt_keys.append(key)
return flt_keys, wcs, footprint
def _get_wcs_from_beams(beams):
"""
TBD
"""
from shapely.geometry import Polygon, Point
from grizli import utils
wcs = OrderedDict()
footprint = OrderedDict()
flt_keys = []
for beam in beams:
#flt_j = pyfits.open(file)
file = beam.direct.parent_file
ext = beam.direct.sci_extn
key = file, ext
wcs[key] = beam.direct.wcs.copy()
wcs[key].pscale = utils.get_wcs_pscale(wcs[key])
footprint[key] = Polygon(wcs[key].calc_footprint())
flt_keys.append(key)
return flt_keys, wcs, footprint
def _get_wcs_from_beams(beams):
"""
TBD
"""
from shapely.geometry import Polygon, Point
from grizli import utils
wcs = OrderedDict()
footprint = OrderedDict()
flt_keys = []
for beam in beams:
#flt_j = pyfits.open(file)
file = beam.direct.parent_file
ext = beam.direct.sci_extn
key = file, ext
wcs[key] = beam.direct.wcs.copy()
wcs[key].pscale = utils.get_wcs_pscale(wcs[key])
wcs[key].expweight = beam.grism.exptime
footprint[key] = Polygon(wcs[key].calc_footprint())
flt_keys.append(key)
return flt_keys, wcs, footprint
def get_driz_cutout(self, ra=53.06967306, dec=-27.72333015, size=15, get_cutout=False, N=None):
"""
TBD
"""
xy = self.driz_wcs.all_world2pix(np.array([[ra,dec]]), 0)[0]
xyp = np.cast[int](np.round(xy))
if N is None:
N = int(np.round(size*self.wcs[self.flt_keys[0]].pscale/self.driz_pscale))
slx = slice(xyp[0]-N, xyp[0]+N)
sly = slice(xyp[1]-N, xyp[1]+N)
wcs_slice = model.ImageData.get_slice_wcs(self.driz_wcs, slx, sly)
wcs_slice.pscale = utils.get_wcs_pscale(wcs_slice)
# outsci = np.zeros((2*N,2*N), dtype=np.float32)
# outwht = np.zeros((2*N,2*N), dtype=np.float32)
# outctx = np.zeros((2*N,2*N), dtype=np.int32)
if get_cutout > 1:
os.system("getfits -o sub.fits {0} {1} {2} {3} {3}".format(self.driz_image, xyp[0], xyp[1], 2*N))
hdu = pyfits.open('sub.fits')
return slx, sly, hdu
elif get_cutout == 1:
im = pyfits.open(self.driz_image)
data = im[0].data[sly, slx]*1
header = utils.to_header(wcs_slice, relax=True)
hdu = pyfits.PrimaryHDU(data=data, header=header)
return slx, sly, pyfits.HDUList([hdu])
return slx, sly, wcs_slice
def _get_flt_wcs(flt_files):
"""
TBD
"""
from shapely.geometry import Polygon, Point
from grizli import utils
wcs = OrderedDict()
footprint = OrderedDict()
flt_keys = []
for file in flt_files:
flt_j = pyfits.open(file)
for ext in range(1,5):
if ('SCI',ext) in flt_j:
key = file, ext
wcs[key] = pywcs.WCS(flt_j['SCI', ext], relax=True,
fobj=flt_j)
wcs[key].pscale = utils.get_wcs_pscale(wcs[key])
footprint[key] = Polygon(wcs[key].calc_footprint())
flt_keys.append(key)
return flt_keys, wcs, footprint
def get_driz_cutout(self, ra=53.06967306, dec=-27.72333015, size=15, get_cutout=False, N=None):
"""
TBD
"""
xy = self.driz_wcs.all_world2pix(np.array([[ra,dec]]), 0)[0]
xyp = np.cast[int](np.round(xy))
if N is None:
N = int(np.round(size*self.wcs[self.flt_keys[0]].pscale/self.driz_pscale))
slx = slice(xyp[0]-N, xyp[0]+N)
sly = slice(xyp[1]-N, xyp[1]+N)
wcs_slice = model.ImageData.get_slice_wcs(self.driz_wcs, slx, sly)
wcs_slice.pscale = utils.get_wcs_pscale(wcs_slice)
# outsci = np.zeros((2*N,2*N), dtype=np.float32)
# outwht = np.zeros((2*N,2*N), dtype=np.float32)
# outctx = np.zeros((2*N,2*N), dtype=np.int32)
if get_cutout > 1:
os.system("getfits -o sub.fits {0} {1} {2} {3} {3}".format(self.driz_image, xyp[0], xyp[1], 2*N))
hdu = pyfits.open('sub.fits')
return slx, sly, hdu
elif get_cutout == 1:
im = pyfits.open(self.driz_image)
data = im[0].data[sly, slx]*1
header = utils.to_header(wcs_slice, relax=True)
hdu = pyfits.PrimaryHDU(data=data, header=header)
return slx, sly, pyfits.HDUList([hdu])
return slx, sly, wcs_slice
def __init__(self,
flt_files=DEMO_LIST,
info=None,
driz_image=DEMO_IMAGE,
driz_hdu=None,
beams=None):
"""
Object for making drizzled PSFs
Parameters
----------
flt_files : list
List of FLT files that were used to create the drizzled image.
driz_image : str
Filename of the drizzled image.
"""
if info is None:
if beams is not None:
info = self._get_wcs_from_beams(beams)
else:
if flt_files is None:
info = self._get_wcs_from_hdrtab(driz_image)
else:
info = self._get_flt_wcs(flt_files)
self.flt_keys, self.wcs, self.footprint = info
self.flt_files = list(np.unique([key[0] for key in self.flt_keys]))
self.ePSF = utils.EffectivePSF()
if driz_hdu is None:
self.driz_image = driz_image
self.driz_header = pyfits.getheader(driz_image)
self.driz_wcs = pywcs.WCS(self.driz_header)
self.driz_pscale = utils.get_wcs_pscale(self.driz_wcs)
else:
self.driz_image = driz_image
self.driz_header = driz_hdu.header
self.driz_wcs = pywcs.WCS(self.driz_header)
self.driz_pscale = utils.get_wcs_pscale(self.driz_wcs)
self.driz_wcs.pscale = self.driz_pscale
def __init__(self, flt_files=DEMO_LIST, info=None, driz_image=DEMO_IMAGE, driz_hdu=None, beams=None):
"""
Object for making drizzled PSFs
Parameters
----------
flt_files : list
List of FLT files that were used to create the drizzled image.
driz_image : str
Filename of the drizzled image.
"""
if info is None:
if beams is None:
info = self._get_flt_wcs(flt_files)
else:
info = self._get_wcs_from_beams(beams)
self.flt_keys, self.wcs, self.footprint = info
self.flt_files = list(np.unique([key[0] for key in self.flt_keys]))
self.ePSF = utils.EffectivePSF()
if driz_hdu is None:
self.driz_image = driz_image
self.driz_header = pyfits.getheader(driz_image)
self.driz_wcs = pywcs.WCS(self.driz_header)
self.driz_pscale = utils.get_wcs_pscale(self.driz_wcs)
else:
self.driz_image = driz_image
self.driz_header = driz_hdu.header
self.driz_wcs = pywcs.WCS(self.driz_header)
self.driz_pscale = utils.get_wcs_pscale(self.driz_wcs)
self.driz_wcs.pscale = self.driz_pscale
def _get_wcs_from_hdrtab(drz_file):
"""
Read tabulated exposure WCS info from the HDRTAB
extension of an AstroDrizzle output file
"""
from shapely.geometry import Polygon, Point
drz = pyfits.open(drz_file)
if 'HDRTAB' not in drz:
print('No HDRTAB extension found in {0}'.format(file))
return None
hdr = utils.GTable(drz['HDRTAB'].data)
wcs = OrderedDict()
footprint = OrderedDict()
flt_keys = []
N = len(hdr)
if 'CCDCHIP' in hdr.colnames:
ext_key = 'CCDCHIP'
else:
ext_key = 'EXTNAME'
for i in range(N):
h = pyfits.Header()
for c in hdr.colnames:
try:
h[c] = hdr[c][i]
except:
h[c] = 1
key = (h['ROOTNAME'], h[ext_key])
flt_keys.append(key)
wcs[key] = pywcs.WCS(h, relax=True)
wcs[key].pscale = utils.get_wcs_pscale(wcs[key])
if 'EXPTIME' in h:
wcs[key].expweight = h['EXPTIME']
else:
wcs[key].expweight = 1
footprint[key] = Polygon(wcs[key].calc_footprint())
return flt_keys, wcs, footprint
def get_psf(self,
ra=53.06967306,
dec=-27.72333015,
filter='F140W',
pixfrac=0.1,
kernel='point',
verbose=True,
wcs_slice=None,
get_extended=True,
get_weight=False):
from drizzlepac.astrodrizzle import adrizzle
from shapely.geometry import Polygon, Point
pix = np.arange(-13, 14)
#wcs_slice = self.get_driz_cutout(ra=ra, dec=dec)
outsci, outwht, outctx = self._get_empty_driz(wcs_slice)
count = 1
for file in self.flt_files:
if self.footprint[file].contains(Point(ra, dec)):
if verbose:
print(file)
xy = self.wcs[file].all_world2pix(np.array([[ra, dec]]), 0)[0]
xyp = np.cast[int](xy) #+1
dx = xy[0] - int(xy[0])
dy = xy[1] - int(xy[1])
psf_xy = self.ePSF.get_at_position(xy[0], xy[1], filter=filter)
yp, xp = np.meshgrid(pix - dy,
pix - dx,
sparse=False,
indexing='ij')
if get_extended:
extended_data = self.ePSF.extended_epsf[filter]
else:
extended_data = None
psf = self.ePSF.eval_ePSF(psf_xy,
xp,
yp,
extended_data=extended_data)
if get_weight:
fltim = pyfits.open(file)
flt_weight = fltim[0].header['EXPTIME']
else:
flt_weight = 1
N = 13
psf_wcs = model.ImageData.get_slice_wcs(
self.wcs[file], slice(xyp[0] - N, xyp[0] + N + 1),
slice(xyp[1] - N, xyp[1] + N + 1))
psf_wcs.pscale = utils.get_wcs_pscale(self.wcs[file])
adrizzle.do_driz(psf,
psf_wcs,
psf * 0 + flt_weight,
wcs_slice,
outsci,
outwht,
outctx,
1.,
'cps',
1,
wcslin_pscale=1.,
uniqid=1,
pixfrac=pixfrac,
kernel=kernel,
fillval=0,
stepsize=10,
wcsmap=None)
if False:
count += 1
hdu = pyfits.HDUList([
pyfits.PrimaryHDU(),
pyfits.ImageHDU(data=psf * 100,
header=utils.to_header(psf_wcs))
])
ds9.set('frame {0}'.format(count + 1))
ds9.set_pyfits(hdu)
#ss = 1000000/2
ss = 1. / outsci.sum()
hdu = pyfits.HDUList([
pyfits.PrimaryHDU(),
pyfits.ImageHDU(data=outsci * ss,
header=utils.to_header(wcs_slice))
])
if False:
ds9.set('frame 2')
ds9.set_pyfits(hdu)
return hdu
def show_all_thumbnails(label='j022708p4901_00273', filters=['visb', 'visr', 'y', 'j', 'h'], scale_ab=21, close=True, thumb_height=2., rgb_params=RGB_PARAMS, ext='png', xl=0.04, yl=0.98, fs=7):
"""
Show individual filter and RGB thumbnails
"""
import glob
#from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import astropy.io.fits as pyfits
from astropy.visualization import make_lupton_rgb
from grizli.pipeline import auto_script
from grizli import utils
all_files = glob.glob('{0}-f*sci.fits'.format(label))
all_filters = [f.split('_dr')[0].split('-')[-1] for f in all_files]
ims = {}
for filter in filters:
drz_files = glob.glob('{0}-{1}*_dr*sci.fits'.format(label, filter))
if len(drz_files) > 0:
im = pyfits.open(drz_files[0])
ims[filter] = im
rgb_params['scale_ab'] = scale_ab
slx, sly, rgb_filts, fig = auto_script.field_rgb(root=label, HOME_PATH=None, **rgb_params) # xsize=4, output_dpi=None, HOME_PATH=None, show_ir=False, pl=1, pf=1, scl=1, rgb_scl=[1, 1, 1], ds9=None, force_ir=False, filters=all_filters, add_labels=False, output_format='png', rgb_min=-0.01, xyslice=None, pure_sort=False, verbose=True, force_rgb=None, suffix='.rgb', scale_ab=scale_ab)
if close:
plt.close()
#rgb = np.array(Image.open('{0}.rgb.png'.format(label)))
rgb = plt.imread('{0}.rgb.png'.format(label))
NX = (len(filters)+1)
fig = plt.figure(figsize=[thumb_height*NX, thumb_height])
ax = fig.add_subplot(1, NX, NX)
ax.imshow(rgb, origin='upper', interpolation='nearest')
# ax.text(0.05, 0.95, label, ha='left', va='top', transform=ax.transAxes, fontsize=7, color='w', bbox=dict(facecolor='k', edgecolor='None', alpha=0.8))
# ax.text(0.05, 0.05, ' '.join(rgb_filts), ha='left', va='bottom', transform=ax.transAxes, fontsize=6, color='w', bbox=dict(facecolor='k', edgecolor='None', alpha=0.8))
for i, filter in enumerate(filters):
if filter in ims:
zp_i = utils.calc_header_zeropoint(ims[filter], ext=0)
scl = 10**(-0.4*(zp_i-5-scale_ab))
pixscl = utils.get_wcs_pscale(ims[filter][0].header.copy())
scl *= (0.06/pixscl)**2
img = ims[filter][0].data*scl
image = make_lupton_rgb(img, img, img, stretch=0.1, minimum=-0.01)
ax = fig.add_subplot(1, NX, i+1)
ax.imshow(255-image, origin='lower', interpolation='nearest')
for ax in fig.axes:
ax.set_xticklabels([])
ax.set_yticklabels([])
ax.set_xticks([])
ax.set_yticks([])
fig.tight_layout(pad=0.1)
# Add labels
#xl, yl = 0.04, 0.98
for i, filter in enumerate(filters):
if filter in ims:
if filter in ['uv', 'visb', 'visr', 'y', 'j', 'h']:
grouped_filters = []
h_i = ims[filter][0].header
for j in range(h_i['NCOMBINE']):
grouped_filters.append(h_i['CFILT{0}'.format(j+1)])
text_label = '+'.join(grouped_filters)
else:
text_label = filter
fig.text((i+xl)/NX, yl, text_label, fontsize=fs,
ha='left', va='top', transform=fig.transFigure,
bbox=dict(facecolor='w', edgecolor='None', alpha=0.9))
fig.text((i+1+xl)/NX, yl, label, ha='left', va='top', transform=fig.transFigure, fontsize=fs, color='w', bbox=dict(facecolor='k', edgecolor='None', alpha=0.8))
fig.text((i+1+0.04)/NX, 1-yl, ' '.join(rgb_filts), ha='left', va='bottom', transform=fig.transFigure, fontsize=fs, color='w', bbox=dict(facecolor='k', edgecolor='None', alpha=0.8))
fig.savefig('{0}.thumb.{1}'.format(label, ext))
if close:
plt.close()
return fig
def get_psf(self, ra=53.06967306, dec=-27.72333015, filter='F140W', pixfrac=0.1, kernel='point', verbose=True, wcs_slice=None, get_extended=True, get_weight=False):
from drizzlepac import adrizzle
from shapely.geometry import Polygon, Point
pix = np.arange(-13,14)
#wcs_slice = self.get_driz_cutout(ra=ra, dec=dec)
outsci, outwht, outctx = self._get_empty_driz(wcs_slice)
count = 1
for key in self.flt_keys:
if self.footprint[key].contains(Point(ra, dec)):
file, ext = key
if verbose:
print('{0}[SCI,{1}]'.format(file, ext))
xy = self.wcs[key].all_world2pix(np.array([[ra,dec]]), 0)[0]
xyp = np.cast[int](xy)#+1
dx = xy[0]-int(xy[0])
dy = xy[1]-int(xy[1])
if ext == 2:
# UVIS
#print('UVIS1!')
chip_offset = 2051
else:
chip_offset = 0
psf_xy = self.ePSF.get_at_position(xy[0], xy[1]+chip_offset, filter=filter)
yp, xp = np.meshgrid(pix-dy, pix-dx, sparse=False, indexing='ij')
if get_extended:
extended_data = self.ePSF.extended_epsf[filter]
else:
extended_data = None
psf = self.ePSF.eval_ePSF(psf_xy, xp, yp, extended_data=extended_data)
if get_weight:
fltim = pyfits.open(file)
flt_weight = fltim[0].header['EXPTIME']
else:
flt_weight = 1
N = 13
psf_wcs = model.ImageData.get_slice_wcs(self.wcs[key], slice(xyp[0]-N, xyp[0]+N+1), slice(xyp[1]-N, xyp[1]+N+1))
psf_wcs.pscale = utils.get_wcs_pscale(self.wcs[key])
adrizzle.do_driz(psf, psf_wcs, psf*0+flt_weight, wcs_slice,
outsci, outwht, outctx, 1., 'cps', 1,
wcslin_pscale=1., uniqid=1,
pixfrac=pixfrac, kernel=kernel, fillval=0,
stepsize=10, wcsmap=None)
if False:
count += 1
hdu = pyfits.HDUList([pyfits.PrimaryHDU(), pyfits.ImageHDU(data=psf*100, header=utils.to_header(psf_wcs))])
ds9.set('frame {0}'.format(count+1))
ds9.set_pyfits(hdu)
#ss = 1000000/2
ss = 1./outsci.sum()
hdu = pyfits.HDUList([pyfits.PrimaryHDU(), pyfits.ImageHDU(data=outsci*ss, header=utils.to_header(wcs_slice))])
if False:
ds9.set('frame 2')
ds9.set_pyfits(hdu)
return hdu
def irac_mosaics(root='j000308m3303', home='/GrizliImaging/', pixfrac=0.2, kernel='square', initial_pix=1.0, final_pix=0.5, pulldown_mag=15.2, sync_xbcd=True, skip_fetch=False, radec=None, mosaic_pad=2.5, drizzle_ref_file='', run_alignment=True, assume_close=True, bucket='grizli-v1', aor_query='r*', mips_ext='[_e]bcd.fits', channels=['ch1','ch2','ch3','ch4','mips1'], drz_query='r*', sync_results=True, ref_seg=None, min_frame={'irac':5, 'mips':1.0}, med_max_size=500e6, stop_at='', make_psf=True, **kwargs):
"""
stop_at: preprocess, make_compact
"""
from grizli import utils
from . import irac
from .utils import get_wcslist, fetch_irac
PATH = os.path.join(home, root)
try:
os.mkdir(PATH)
except:
pass
os.chdir(PATH)
if not skip_fetch:
# Fetch IRAC bcds
if not os.path.exists(f'{root}_ipac.fits'):
os.system(f'wget https://s3.amazonaws.com/{bucket}/IRAC/{root}_ipac.fits')
res = fetch_irac(root=root, path='./', channels=channels)
if res in [False, None]:
# Nothing to do
make_html(root, bucket=bucket)
print(f'### Done: \n https://s3.amazonaws.com/{bucket}/Pipeline/{root}/IRAC/{root}.irac.html')
utils.log_comment(f'/tmp/{root}.success', 'Done!',
verbose=True, show_date=True)
return True
# Sync CHArGE HST images
os.system(f'aws s3 sync s3://{bucket}/Pipeline/{root}/Prep/ ./ '
f' --exclude "*" --include "{root}*seg.fits*"'
f' --include "{root}-ir_drz*fits*"'
f' --include "{root}*psf.fits*"'
f' --include "{root}-f[01]*_drz*fits.gz"'
f' --include "{root}*phot.fits"')
# Drizzle properties of the preliminary mosaic
#pixfrac, pix, kernel = 0.2, 1.0, 'square'
# Define an output WCS aligned in pixel phase to the HST mosaic ()
if not os.path.exists('ref_hdu.fits'):
wcslist = get_wcslist(skip=-500)
out_hdu = utils.make_maximal_wcs(wcslist, pixel_scale=initial_pix, theta=0, pad=5, get_hdu=True, verbose=True)
# Make sure pixels align
ref_file = glob.glob('{0}-f[01]*_drz_sci.fits*'.format(root))
if len(ref_file) == 0:
os.system(f'aws s3 sync s3://{bucket}/Pipeline/{root}/Prep/ ./ '
f' --exclude "*"'
f' --include "{root}-f[678]*_dr*fits.gz"')
ref_file = glob.glob('{0}-f[678]*_dr*_sci.fits*'.format(root))
ref_file = ref_file[-1]
print(f'\nHST reference image: {ref_file}\n')
ref_hdu = pyfits.open(ref_file)[0].header
ref_filter = utils.get_hst_filter(ref_hdu).lower()
ref_wcs = pywcs.WCS(ref_hdu)
ref_rd = ref_wcs.all_pix2world(np.array([[-0.5, -0.5]]), 0).flatten()
target_phase = np.array([0.5, 0.5])#/(pix/0.1)
for k in ['RADESYS', 'LATPOLE', 'LONPOLE']:
out_hdu.header[k] = ref_hdu[k]
# Shift CRVAL to same tangent point
out_wcs = pywcs.WCS(out_hdu.header)
out_xy = out_wcs.all_world2pix(np.array([ref_wcs.wcs.crval]), 1).flatten()
out_hdu.header['CRVAL1'], out_hdu.header['CRVAL2'] = tuple(ref_wcs.wcs.crval)
out_hdu.header['CRPIX1'], out_hdu.header['CRPIX2'] = tuple(out_xy)
# Align integer pixel phase
out_wcs = pywcs.WCS(out_hdu.header)
out_xy = out_wcs.all_world2pix(np.array([ref_rd]), 0).flatten()
xy_phase = out_xy - np.floor(out_xy)
new_crpix = out_wcs.wcs.crpix - (xy_phase - target_phase)
out_hdu.header['CRPIX1'], out_hdu.header['CRPIX2'] = tuple(new_crpix)
out_wcs = pywcs.WCS(out_hdu.header)
out_hdu.writeto('ref_hdu.fits', output_verify='Fix')
else:
out_hdu = pyfits.open('ref_hdu.fits')[1]
########
files = []
for ch in channels:
if 'mips' in ch:
mc = ch.replace('mips','ch')
files += glob.glob(f'{aor_query}/{mc}/bcd/SPITZER_M*{mips_ext}')
files += glob.glob(f'{aor_query}/{mc}/bcd/SPITZER_M*xbcd.fits.gz')
else:
files += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*cbcd.fits')
files += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*xbcd.fits.gz')
files.sort()
roots = np.array([file.split('/')[0] for file in files])
with_channels = np.array([file.split('_')[1] for file in files])
all_roots = np.array(['{0}-{1}'.format(r, c.replace('I','ch').replace('M', 'mips')) for r, c in zip(roots, with_channels)])
tab = {'aor':[], 'N':[], 'channel':[]}
for r in np.unique(all_roots):
tab['aor'].append(r.split('-')[0])
tab['N'].append((all_roots == r).sum())
tab['channel'].append(r.split('-')[1])
aors = utils.GTable(tab)
print(aors)
########
SKIP = True # Don't regenerate finished files
delete_group = False # Delete intermediate products from memory
zip_outputs = False # GZip intermediate products
aors_ch = {}
########
# Process mosaics by AOR
# Process in groups, helps for fields like HFF with dozens/hundreds of AORs!
for ch in channels:
aor = aors[(aors['channel'] == ch) & (aors['N'] > 5)]
if len(aor) == 0:
continue
#aors_ch[ch] = []
if ch in ['ch1','ch2']:
NPER, instrument = 500, 'irac'
if ch in ['ch3','ch4']:
NPER, instrument = 500, 'irac'
elif ch in ['mips1']:
NPER, instrument = 400, 'mips'
min_frametime = min_frame[instrument]
nsort = np.cumsum(aor['N']/NPER)
NGROUP = int(np.ceil(nsort.max()))
count = 0
for g in range(NGROUP):
root_i = root+'-{0:02d}'.format(g)
gsel = (nsort > g) & (nsort <= g+1)
aor_ids = list(aor['aor'][gsel])
print('{0}-{1} N_AOR = {2:>2d} N_EXP = {3:>4d}'.format(root_i, ch, len(aor_ids), aor['N'][gsel].sum()))
count += gsel.sum()
files = glob.glob('{0}-{1}*'.format(root_i, ch))
if (len(files) > 0) & (SKIP):
print('Skip {0}-{1}'.format(root_i, ch))
continue
with open('{0}-{1}.log'.format(root_i, ch),'w') as fp:
fp.write(time.ctime())
# Do internal alignment to GAIA.
# Otherwise, set `radec` to the name of a file that has two columns with
# reference ra/dec.
#radec = None
# Pipeline
if instrument == 'mips':
aors_ch[ch] = irac.process_all(channel=ch.replace('mips','ch'), output_root=root_i, driz_scale=initial_pix, kernel=kernel, pixfrac=pixfrac, wcslist=None, pad=0, out_hdu=out_hdu, aor_ids=aor_ids, flat_background=False, two_pass=True, min_frametime=min_frametime, instrument=instrument, align_threshold=0.15, radec=radec, run_alignment=False, mips_ext=mips_ext, ref_seg=ref_seg, global_mask=root+'_mask.reg')
else:
aors_ch[ch] = irac.process_all(channel=ch, output_root=root_i, driz_scale=initial_pix, kernel=kernel, pixfrac=pixfrac, wcslist=None, pad=0, out_hdu=out_hdu, aor_ids=aor_ids, flat_background=False, two_pass=True, min_frametime=min_frametime, instrument=instrument, radec=radec, run_alignment=run_alignment, assume_close=assume_close, ref_seg=ref_seg, global_mask=root+'_mask.reg', med_max_size=med_max_size)
if len(aors_ch[ch]) == 0:
continue
# PSFs
plt.ioff()
if (instrument != 'mips') & make_psf:
ch_num = int(ch[-1])
segmask=True
# psf_size=20
# for p in [0.1, final_pix]:
# irac.mosaic_psf(output_root=root_i, target_pix=p, channel=ch_num, aors=aors_ch[ch], kernel=kernel, pixfrac=pixfrac, size=psf_size, native_orientation=False, instrument=instrument, subtract_background=False, segmentation_mask=segmask, max_R=10)
# plt.close('all')
psf_size=30
p = 0.1
irac.mosaic_psf(output_root=root_i, target_pix=p, channel=ch_num, aors=aors_ch[ch], kernel=kernel, pixfrac=pixfrac, size=psf_size, native_orientation=True, subtract_background=False, segmentation_mask=segmask, max_R=10)
plt.close('all')
if delete_group:
del(aors_ch[ch])
print('Done {0}-{1}, gzip products'.format(root_i, ch))
if zip_outputs:
os.system('gzip {0}*-{1}_drz*fits'.format(root_i, ch))
# PSFs
if (instrument != 'mips') & make_psf:
# Average PSF
p = 0.1
files = glob.glob('*{0}-{1:.1f}*psfr.fits'.format(ch, p))
if len(files) == 0:
continue
files.sort()
avg = None
for file in files:
im = pyfits.open(file)
if avg is None:
wht = im[0].data != 0
avg = im[0].data*wht
else:
wht_i = im[0].data != 0
avg += im[0].data*wht_i
wht += wht_i
im.close()
avg = avg/wht
avg[wht == 0] = 0
# Window
from photutils import (HanningWindow, TukeyWindow,
CosineBellWindow,
SplitCosineBellWindow, TopHatWindow)
coswindow = CosineBellWindow(alpha=1)
avg *= coswindow(avg.shape)**0.05
avg /= avg.sum()
pyfits.writeto('{0}-{1}-{2:0.1f}.psfr_avg.fits'.format(root, ch, p), data=avg, header=im[0].header, overwrite=True)
####
## Show the initial product
plt.ioff()
for i in range(10):
files = glob.glob(f'{root}-{i:02d}-ch*sci.fits')
if len(files) > 0:
break
files.sort()
if len(files) == 1:
subs = 1,1
fs = [7,7]
elif len(files) == 2:
subs = 1,2
fs = [14,7]
elif len(files) == 3:
subs = 2,2
fs = [14,14]
else:
subs = 2,2
fs = [14,14]
fig = plt.figure(figsize=fs)
for i, file in enumerate(files[:4]):
im = pyfits.open(file)
print('{0} {1} {2:.1f} s'.format(file, im[0].header['FILTER'], im[0].header['EXPTIME']))
ax = fig.add_subplot(subs[0], subs[1], 1+i)
ax.imshow(im[0].data, vmin=-0.1, vmax=1, cmap='gray_r', origin='lower')
ax.text(0.05, 0.95, file, ha='left', va='top', color='k',
transform=ax.transAxes)
im.close()
if len(files) > 1:
fig.axes[1].set_yticklabels([])
if len(files) > 2:
fig.axes[0].set_xticklabels([])
fig.axes[1].set_xticklabels([])
if len(files) > 3:
fig.axes[3].set_yticklabels([])
fig.tight_layout(pad=0.5)
fig.savefig(f'{root}.init.png')
plt.close('all')
if stop_at == 'preprocess':
return True
#######
# Make more compact individual exposures and clean directories
wfiles = []
for ch in channels:
if 'mips' in ch:
chq = ch.replace('mips','ch')
wfiles += glob.glob(f'{aor_query}/{chq}/bcd/SPITZER_M*wcs.fits')
else:
wfiles += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*wcs.fits')
#wfiles = glob.glob('r*/*/bcd/*_I[1-4]_*wcs.fits')
#wfiles += glob.glob('r*/*/bcd/*_M[1-4]_*wcs.fits')
wfiles.sort()
for wcsfile in wfiles:
outfile = wcsfile.replace('_wcs.fits', '_xbcd.fits.gz')
if os.path.exists(outfile):
print(outfile)
else:
irac.combine_products(wcsfile)
print('Run: ', outfile)
if os.path.exists(outfile):
remove_files = glob.glob('{0}*fits'.format(wcsfile.split('_wcs')[0]))
for f in remove_files:
print(' rm ', f)
os.remove(f)
if stop_at == 'make_compact':
return True
#############
# Drizzle final mosaics
# Make final mosaic a bit bigger than the HST image
pad = mosaic_pad
# Pixel scale of final mosaic.
# Don't make too small if not many dithers available as in this example.
# But for well-sampled mosaics like RELICS / HFF, can push this to perhaps 0.3" / pix
pixscale = final_pix #0.5
# Again, if have many dithers maybe can use more aggressive drizzle parameters,
# like a 'point' kernel or smaller pixfrac (a 'point' kernel is pixfrac=0)
#kernel, pixfrac = 'square', 0.2
# Correction for bad columns near bright stars
#pulldown_mag = 15.2
##############
# Dilation for CR rejection
dil = np.ones((3,3))
driz_cr = [7, 4]
blot_interp = 'poly5'
bright_fmax = 0.5
### Drizzle
for ch in channels: #[:2]:
###########
# Files and reference image for extra CR rejection
if ch == 'mips1':
files = glob.glob('{0}/ch1/bcd/SPITZER_M1_*xbcd.fits*'.format(drz_query, ch))
files.sort()
pulldown_mag = -10
pixscale = 1.
kernel = 'point'
else:
files = glob.glob('{0}/{1}/bcd/*_I?_*xbcd.fits*'.format(drz_query, ch))
files.sort()
#ref = pyfits.open('{0}-00-{1}_drz_sci.fits'.format(root, ch))
#ref_data = ref[0].data.astype(np.float32)
ref_files = glob.glob(f'{root}-??-{ch}*sci.fits')
if len(ref_files) == 0:
continue
num = None
for ref_file in ref_files:
ref = pyfits.open(ref_file)
wht = pyfits.open(ref_file.replace('_sci.fits', '_wht.fits'))
if num is None:
num = ref[0].data*wht[0].data
den = wht[0].data
else:
num += ref[0].data*wht[0].data
den += wht[0].data
ref_data = (num/den).astype(np.float32)
ref_data[den <= 0] = 0
ref_wcs = pywcs.WCS(ref[0].header, relax=True)
ref_wcs.pscale = utils.get_wcs_pscale(ref_wcs)
if (not hasattr(ref_wcs, '_naxis1')) & hasattr(ref_wcs, '_naxis'):
ref_wcs._naxis1, ref_wcs._naxis2 = ref_wcs._naxis
##############
# Output WCS based on HST footprint
if drizzle_ref_file == '':
try:
hst_im = pyfits.open(glob.glob('{0}-f[01]*_drz_sci.fits*'.format(root))[-1])
except:
hst_im = pyfits.open(glob.glob('{0}-f[578]*_dr*sci.fits*'.format(root))[-1])
hst_wcs = pywcs.WCS(hst_im[0])
hst_wcs.pscale = utils.get_wcs_pscale(hst_wcs)
try:
size = (np.round(np.array([hst_wcs._naxis1, hst_wcs._naxis2])*hst_wcs.pscale*pad/pixscale)*pixscale)
except:
size = (np.round(np.array([hst_wcs._naxis[0], hst_wcs._naxis[1]])*hst_wcs.pscale*pad/pixscale)*pixscale)
hst_rd = hst_wcs.calc_footprint().mean(axis=0)
_x = utils.make_wcsheader(ra=hst_rd[0], dec=hst_rd[1],
size=size,
pixscale=pixscale,
get_hdu=False, theta=0)
out_header, out_wcs = _x
else:
driz_ref_im = pyfits.open(drizzle_ref_file)
out_wcs = pywcs.WCS(driz_ref_im[0].header, relax=True)
out_wcs.pscale = utils.get_wcs_pscale(out_wcs)
out_header = utils.to_header(out_wcs)
if (not hasattr(out_wcs, '_naxis1')) & hasattr(out_wcs, '_naxis'):
out_wcs._naxis1, out_wcs._naxis2 = out_wcs._naxis
##############
# Bright stars for pulldown correction
cat_file = glob.glob(f'{root}-[0-9][0-9]-{ch}.cat.fits')[0]
ph = utils.read_catalog(cat_file)
bright = (ph['mag_auto'] < pulldown_mag) # & (ph['flux_radius'] < 3)
ph = ph[bright]
##############
# Now do the drizzling
yp, xp = np.indices((256, 256))
orig_files = []
out_header['DRIZ_CR0'] = driz_cr[0]
out_header['DRIZ_CR1'] = driz_cr[1]
out_header['KERNEL'] = kernel
out_header['PIXFRAC'] = pixfrac
out_header['NDRIZIM'] = 0
out_header['EXPTIME'] = 0
out_header['BUNIT'] = 'microJy'
out_header['FILTER'] = ch
med_root = 'xxx'
N = len(files)
for i, file in enumerate(files):#[:100]):
print('{0}/{1} {2}'.format(i, N, file))
if file in orig_files:
continue
im = pyfits.open(file)
ivar = 1/im['CBUNC'].data**2
msk = (~np.isfinite(ivar)) | (~np.isfinite(im['CBCD'].data))
im['CBCD'].data[msk] = 0
ivar[msk] = 0
wcs = pywcs.WCS(im['WCS'].header, relax=True)
wcs.pscale = utils.get_wcs_pscale(wcs)
if (not hasattr(wcs, '_naxis1')) & hasattr(wcs, '_naxis'):
wcs._naxis1, wcs._naxis2 = wcs._naxis
fp = Path(wcs.calc_footprint())
med_root_i = im.filename().split('/')[0]
if med_root != med_root_i:
print('\n Read {0}-{1}_med.fits \n'.format(med_root_i, ch))
med = pyfits.open('{0}-{1}_med.fits'.format(med_root_i, ch))
med_data = med[0].data.astype(np.float32)
med_root = med_root_i
med.close()
try:
gaia_rd = utils.read_catalog('{0}-{1}_gaia.radec'.format(med_root_i, ch))
ii, rr = gaia_rd.match_to_catalog_sky(ph)
gaia_rd = gaia_rd[ii][rr.value < 2]
gaia_pts = np.array([gaia_rd['ra'].data,
gaia_rd['dec'].data]).T
except:
gaia_rd = []
#data = im['CBCD'].data - aor_med[0].data
# Change output units to uJy / pix
if ch == 'mips1':
# un = 1*u.MJy/u.sr
# #to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(out_wcs.pscale**2)
# to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(native_scale**2)
to_ujy_px = 146.902690
else:
# native_scale = 1.223
# un = 1*u.MJy/u.sr
# #to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(out_wcs.pscale**2)
# to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(native_scale**2)
to_ujy_px = 35.17517196810
blot_data = ablot.do_blot(ref_data, ref_wcs, wcs, 1, coeffs=True,
interp=blot_interp,
sinscl=1.0, stepsize=10,
wcsmap=None)/to_ujy_px
# mask for bright stars
eblot = 1-np.clip(blot_data, 0, bright_fmax)/bright_fmax
# Initial CR
clean = im[0].data - med_data - im['WCS'].header['PEDESTAL']
dq = (clean - blot_data)*np.sqrt(ivar)*eblot > driz_cr[0]
# Adjacent CRs
dq_dil = binary_dilation(dq, selem=dil)
dq |= ((clean - blot_data)*np.sqrt(ivar)*eblot > driz_cr[1]) & (dq_dil)
# Very negative pixels
dq |= clean*np.sqrt(ivar) < -4
original_dq = im['WCS'].data - (im['WCS'].data & 1)
dq |= original_dq > 0
# Pulldown correction for bright stars
if len(gaia_rd) > 0:
mat = fp.contains_points(gaia_pts)
if mat.sum() > 0:
xg, yg = wcs.all_world2pix(gaia_rd['ra'][mat], gaia_rd['dec'][mat], 0)
sh = dq.shape
mat = (xg > 0) & (xg < sh[1]) & (yg > 0) & (yg < sh[0])
if mat.sum() > 0:
for xi, yi in zip(xg[mat], yg[mat]):
dq |= (np.abs(xp-xi) < 2) & (np.abs(yp-yi) > 10)
if i == 0:
res = utils.drizzle_array_groups([clean], [ivar*(dq == 0)], [wcs], outputwcs=out_wcs, kernel=kernel, pixfrac=pixfrac, data=None, verbose=False)
# Copy header keywords
wcs_header = utils.to_header(wcs)
for k in im[0].header:
if (k not in ['', 'HISTORY', 'COMMENT']) & (k not in out_header) & (k not in wcs_header):
out_header[k] = im[0].header[k]
else:
_ = utils.drizzle_array_groups([clean], [ivar*(dq == 0)], [wcs], outputwcs=out_wcs, kernel=kernel, pixfrac=pixfrac, data=res[:3], verbose=False)
out_header['NDRIZIM'] += 1
out_header['EXPTIME'] += im[0].header['EXPTIME']
im.close()
# Pixel scale factor for weights
wht_scale = (out_wcs.pscale/wcs.pscale)**-4
# Write final images
pyfits.writeto('{0}-{1}_drz_sci.fits'.format(root, ch), data=res[0]*to_ujy_px, header=out_header,
output_verify='fix', overwrite=True)
pyfits.writeto('{0}-{1}_drz_wht.fits'.format(root, ch), data=res[1]*wht_scale/to_ujy_px**2,
header=out_header, output_verify='fix', overwrite=True)
##########
## Show the final drizzled images
plt.ioff()
files = glob.glob(f'{root}-ch*sci.fits')
files.sort()
if len(files) == 1:
subs = 1,1
fs = [7,7]
elif len(files) == 2:
subs = 1,2
fs = [14,7]
elif len(files) == 3:
subs = 2,2
fs = [14,14]
else:
subs = 2,2
fs = [14,14]
fig = plt.figure(figsize=fs)
for i, file in enumerate(files[:4]):
im = pyfits.open(file)
print('{0} {1} {2:.1f} s'.format(file, im[0].header['FILTER'], im[0].header['EXPTIME']))
ax = fig.add_subplot(subs[0], subs[1], 1+i)
scl = (final_pix/initial_pix)**2
ax.imshow(im[0].data, vmin=-0.1*scl, vmax=1*scl, cmap='gray_r', origin='lower')
ax.text(0.05, 0.95, file, ha='left', va='top', color='k',
transform=ax.transAxes)
im.close()
if len(files) > 1:
fig.axes[1].set_yticklabels([])
if len(files) > 2:
fig.axes[0].set_xticklabels([])
fig.axes[1].set_xticklabels([])
if len(files) > 3:
fig.axes[3].set_yticklabels([])
fig.tight_layout(pad=0.5)
fig.savefig(f'{root}.final.png')
plt.close('all')
if sync_results:
print('gzip mosaics')
os.system(f'gzip -f {root}-ch*_drz*fits {root}-mips*_drz*fits')
######## Sync
## Sync
print(f's3://{bucket}/Pipeline/{root}/IRAC/')
make_html(root, bucket=bucket)
os.system(f'aws s3 sync ./ s3://{bucket}/Pipeline/{root}/IRAC/'
f' --exclude "*" --include "{root}-ch*drz*fits*"'
f' --include "{root}-mips*drz*fits*"'
f' --include "{root}.*png"'
' --include "*-ch*psf*" --include "*log.fits"'
' --include "*wcs.[lp]*"'
' --include "*html" --include "*fail*"'
' --acl public-read')
if sync_xbcd:
aor_files = glob.glob('r*-ch*med.fits')
for aor_file in aor_files:
aor = aor_file.split('-ch')[0]
os.system(f'aws s3 sync ./{aor}/ s3://{bucket}/IRAC/AORS/{aor}/ --exclude "*" --include "ch*/bcd/*xbcd.fits.gz" --acl public-read')
os.system(f'aws s3 cp {aor_file} s3://{bucket}/IRAC/AORS/ --acl public-read')
msg = f'### Done: \n https://s3.amazonaws.com/{bucket}/Pipeline/{root}/IRAC/{root}.irac.html'
utils.log_comment(f'/tmp/{root}.success', msg, verbose=True, show_date=True)
