def matching_kernel(psf1,
psf2,
window_type='TukeyWindow',
alpha=None,
beta=None):
"""Use photutils to create a matching kernel given two PSFs and
the window type and parameters
Parameters
----------
psf1 : numpy.ndarray
2D array containing the first PSF
psf2 : numpy.ndarray
2D array containing the second PSF
window_type : str
Name of the window function to use when filtering the matching kernel
alpha : float
Optional input for some of the window functions
beta : float
Optional input for some of the window functions
Returns
-------
matched_kernel : numpy.ndarray
2D array containing the matching PSF kernel
"""
# Create the filtering window
orig_window_type = copy.deepcopy(window_type)
window_type = window_type.lower()
if window_type == 'tophatwindow':
window = TopHatWindow(beta=beta)
elif window_type == 'cosinebellwindow':
window = CosineBellWindow(alpha=alpha)
elif window_type == 'splitcosinebellwindow':
window = SplitCosineBellWindow(alpha=alpha, beta=beta)
elif window_type == 'tukeywindow':
window = TukeyWindow(alpha=alpha)
elif window_type == 'hanningwindow':
window = HanningWindow()
else:
raise ValueError(
"ERROR: Unrecognized window_type: {}".format(orig_window_type))
# Create the matching kernel
matched_kernel = create_matching_kernel(psf1, psf2, window=window)
return matched_kernel
import photutils
from photutils import create_matching_kernel
from astropy.io import fits
from matplotlib import pyplot as plt
import numpy as np
import scipy
from photutils import TopHatWindow
from photutils import CosineBellWindow
window = CosineBellWindow(alpha=0.35)
from scipy.signal import convolve as scipy_convolve
window = TopHatWindow(0.35)
dir1 = '/home/sourabh/ULIRG_package/data/OPTICAL_PSF/'
data_ref = fits.getdata(dir1 + 'f165psf.fits')
data_psf = fits.getdata(dir1 + 'PSF_775_gal4_rotate_cut.fits')
kernel = create_matching_kernel(data_psf, data_ref) # , window = window )
fits.writeto('ker.fits', data=kernel, overwrite=True)
plt.imshow(kernel, cmap='Greys_r', origin='lower')
filename = '/home/sourabh/ULIRG_package/data/IRASF10594+3818/gal1_HA.fits'
fileout = '/home/sourabh/ULIRG_package/data/IRASF10594+3818/gal1_HA_psfmatch.fits'
ker = 'ker.fits'
#ker_shift = np.pad(kernel, ((0, 1), (0, 1)), mode='constant')
data1 = scipy_convolve(data_psf, kernel, mode='same')
fits.writeto('test2.fits', data=data1, overwrite=True)
data3 = data1 - data_ref
fits.writeto('test3.fits', data=data3, overwrite=True)
def psf_match(filename, fileout, ker):
hdulist = fits.open(filename)
def effective_psf(log,
rd=None,
size=30,
pixel_scale=0.1,
pixfrac=0.2,
kernel='square',
recenter=False,
use_native_orientation=False):
"""
Drizzle effective PSF model given the oversampled model PRF
** not used, testing **
"""
from photutils import (HanningWindow, TukeyWindow, CosineBellWindow,
SplitCosineBellWindow, TopHatWindow)
import grizli.utils
# r48106752/ch1/bcd/SPITZER_I1_48106752_0001_0000_2_cbcd.fits
ch = int(log['file'][0].split("_")[1][-1])
if __name__ == "__main__":
import golfir
_path = os.path.dirname(golfir.__file__)
else:
_path = os.path.dirname(__file__)
h_file = os.path.join(_path, f'data/bcd_ch{ch}.header')
sip_h = pyfits.Header.fromtextfile(h_file)
N = len(log)
#if rd is None:
rd = np.mean(log['crval'][:N // 2], axis=0)
#ipsf = pyfits.open(f'../AvgPSF/IRAC/apex_sh_IRACPC{ch}_col129_row129_x100.fits', relax=True)
ipsf = pyfits.open(
f'../AvgPSF/Cryo/apex_sh_IRAC{ch}_col129_row129_x100.fits', relax=True)
#ipsf = pyfits.open(f'../AvgPSF/IRAC/IRACPC{ch}_col129_row129.fits')
if 'PRFXRSMP' in ipsf[0].header:
osamp = ipsf[0].header['PRFXRSMP']
else:
osamp = 100
ish = ipsf[0].data.shape
if recenter:
# Centroid
yp, xp = np.indices(ish)
pp = ipsf[0].data
xc = int(np.round((pp * xp).sum() / pp.sum()))
yc = int(np.round((pp * yp).sum() / pp.sum()))
i0 = [xc, yc]
else:
i0 = [s // 2 for s in ish]
# Number of pixels to extract
inp = np.min([xc, yc]) - osamp // 2
# Extra padding
if osamp == 100:
inp -= 20
else:
inp -= 1
wcs_list = []
sci_list = []
wht_list = []
wht_i = np.ones((256, 256), dtype=np.float32)
coords = [rd]
cosd = np.cos(rd[1] / 180 * np.pi)
for dx in np.linspace(-1, 1, 4):
for dy in np.linspace(-1, 1, 4):
if dx == dy == 0:
continue
delta = np.array([dx / cosd / 60, dy / 60])
coords.append(rd + delta)
for k in range(N):
print('Parse file: {0}'.format(log['file'][k]))
if 0:
cd = log['cd'][k]
theta = np.arctan2(cd[1][0], cd[1][1]) / np.pi * 180
print(k, theta)
sip_h['CRPIX1'] = log['crpix'][k][0]
sip_h['CRPIX2'] = log['crpix'][k][1]
sip_h['CRVAL1'] = log['crval'][k][0]
sip_h['CRVAL2'] = log['crval'][k][1]
sip_h['LATPOLE'] = log['crval'][k][1]
for i in range(2):
for j in range(2):
key = f'CD{i+1}_{j+1}'
sip_h[key] = log['cd'][k][i, j]
wcs_i = pywcs.WCS(sip_h, relax=True)
wcs_i.pscale = grizli.utils.get_wcs_pscale(wcs_i)
sci_i = np.zeros((256, 256), dtype=np.float32)
for coo in coords:
try:
xy = wcs_i.all_world2pix([coo], 0).flatten() + 0.5
# if ch == 2:
# xy += 0.25
except:
print('wcs failed')
continue
xyp = np.cast[int](np.floor(xy))
phase = np.cast[int](np.round((xy - xyp) * osamp - osamp / 2.))
oslx = slice(i0[0] - phase[0] - inp, i0[0] - phase[0] + inp + 1,
osamp)
osly = slice(i0[1] - phase[1] - inp, i0[1] - phase[1] + inp + 1,
osamp)
psf_sub = ipsf[0].data[osly, oslx]
osh = psf_sub.shape
nparent = 256
slx0 = slice(xyp[0] - osh[1] // 2, xyp[0] + osh[1] // 2)
sly0 = slice(xyp[1] - osh[0] // 2, xyp[1] + osh[0] // 2)
slpx, slcx = (match_slice_to_shape(slx0, nparent))
slpy, slcy = (match_slice_to_shape(sly0, nparent))
try:
sci_i[slpy, slpx] += psf_sub[slcy, slcx]
except:
print('slice failed')
wcs_list.append(wcs_i)
sci_list.append(sci_i)
wht_list.append((sci_i != 0).astype(np.float32))
if use_native_orientation:
cd = log['cd'][0]
theta = np.arctan2(cd[1][0], cd[1][1]) / np.pi * 180
else:
theta = 0
for k, coo in enumerate(coords):
print('Drizzle coords: {0}'.format(coo))
out_hdu = grizli.utils.make_wcsheader(ra=coo[0],
dec=coo[1],
size=size,
pixscale=pixel_scale,
theta=-theta,
get_hdu=True)
#out_h, out_wcs = _out
out_wcs = pywcs.WCS(out_hdu.header)
out_wcs.pscale = grizli.utils.get_wcs_pscale(out_wcs)
if False:
_drz = grizli.utils.drizzle_array_groups(sci_list,
wht_list,
wcs_list,
outputwcs=out_wcs,
pixfrac=pixfrac,
kernel=kernel,
verbose=False)
drz_psf = _drz[0] / _drz[0].sum()
pyfits.writeto(f'irsa_{pixel_scale}pix_ch{ch}_{k}_psf.fits',
data=drz_psf,
overwrite=True)
else:
if k == 0:
_drz = grizli.utils.drizzle_array_groups(sci_list,
wht_list,
wcs_list,
outputwcs=out_wcs,
pixfrac=pixfrac,
kernel=kernel,
verbose=False)
else:
_ = grizli.utils.drizzle_array_groups(sci_list,
wht_list,
wcs_list,
outputwcs=out_wcs,
pixfrac=pixfrac,
kernel=kernel,
verbose=False,
data=_drz[:3])
sci, wht, ctx, head, w = _drz
coswindow = CosineBellWindow(alpha=1)(_drz[0].shape)**0.05
drz_psf = (_drz[0] * coswindow) / (_drz[0] * coswindow).sum()
pyfits.writeto(f'irsa_{pixel_scale}pix_ch{ch}_psf.fits',
data=drz_psf,
overwrite=True)
def run_model(root=''):
import golfir.irac
import golfir.model
from astropy.modeling.models import Moffat2D, Gaussian2D, Sersic2D
from astropy.modeling.fitting import LevMarLSQFitter
from tractor.psf import GaussianMixtureEllipsePSF, GaussianMixturePSF
from golfir.irac import MixturePSF
from photutils import (HanningWindow, TukeyWindow, CosineBellWindow,
SplitCosineBellWindow, TopHatWindow)
window = CosineBellWindow(1)
fitter = LevMarLSQFitter()
import grizli.ds9
ds9 = grizli.ds9.DS9()
P0 = None
bkg_func = None
kwargs = {
'ds9': ds9,
'mag_limit': [24, 27],
'galfit_flux_limit': np.inf,
'refine_brightest': False,
'run_alignment': True,
'any_limit': 10,
'point_limit': -10,
'bright_sn': 10,
'bkg_kwargs': {
'order_npix': 64
},
'psf_only': False,
'use_saved_components': False,
'window': None,
'use_avg_psf': True,
'align_type': 1
}
files = glob.glob(f'{root}-[kgrz]*sci.fits')
files.sort()
bands = [file.split('_drz')[0].split('-')[-1] for file in files]
orig_pix = 0.262
for band in bands:
psf_im = pyfits.open(f'{root}-{band}_psf.fits')[0]
h = psf_im.header
psf_obj = MixturePSF(N=psf_im.header['PSFN'])
for i in range(psf_obj.N):
psf_obj.coeffs[i] = h[f'PSFC{i}']
pars = [h[f'PSFP{i}_{j}'] for j in range(6)]
psf_obj.mogs[i].setParams(pars)
psf_obj.set_pixelgrid(size=32,
instep=orig_pix,
outstep=orig_pix,
oversample=orig_pix / 0.1)
modeler = golfir.model.ImageModeler(root=root,
prefer_filter='f160w',
lores_filter=band,
psf_obj=psf_obj)
if not os.path.exists(f'{root}_waterseg.fits'):
pyfits.writeto(f'{root}_waterseg.fits', data=modeler.waterseg)
#!/usr/bin/env python
import numpy as np
from astropy.io import fits
# from scipy import signal, ndimage
# from hconvolve import fftdeconvolve, hconvolve
from scipy.signal import convolve2d
import os, subprocess
from scipy import ndimage
from skimage.restoration import richardson_lucy
from photutils import CosineBellWindow, create_matching_kernel
window_default = CosineBellWindow(alpha=0.35)
def psfmatch(psf_sharp, psf_broad, kernelname, method='fft', window=window_default, iterations=30):
"""Derive the kernel that matches psf_sharp to psf_broad"""
psf1 = fits.getdata(psf_sharp)
psf2 = fits.getdata(psf_broad)
assert psf1.shape[0] % 2 == 1
assert psf2.shape[0] % 2 == 1
assert psf1.shape[0] == psf1.shape[1]
assert psf2.shape[0] == psf2.shape[1]
psf1 = psf1 / psf1.sum()
psf2 = psf2 / psf2.sum()
if psf1.shape[0] > psf2.shape[0]:
pad = (psf1.shape[0] - psf2.shape[0]) / 2
psf1 = psf1[pad:-pad, pad:-pad]
elif psf2.shape[0] > psf1.shape[0]:
"""
Created on Tue Jun 12 11:36:12 2018
Code to create a kernel that will match the PSFs of two stacks
@author: ppxee
"""
### Import Modules ###
import numpy as np
from photutils import create_matching_kernel, TopHatWindow, CosineBellWindow
import matplotlib.pyplot as plt
from astropy.io import fits
plt.close('all')
psf11 = fits.open('11B_K_PSF.fits')[0].data
psf12 = fits.open('12B_K_PSF.fits')[0].data
#plt.figure()
#plt.imshow(psf12-psf11)
#plt.figure()
#plt.imshow(psf11-psf12)
kernel = create_matching_kernel(psf11, psf12, window=CosineBellWindow(0.35)) #need to check which is larger
plt.figure()
plt.imshow(kernel)
def irac_mosaics(root='j000308m3303', home='/GrizliImaging/', pixfrac=0.2, kernel='square', initial_pix=1.0, final_pix=0.5, pulldown_mag=15.2, sync_xbcd=True, skip_fetch=False, radec=None, mosaic_pad=2.5, drizzle_ref_file='', run_alignment=True, assume_close=True, bucket='grizli-v1', aor_query='r*', mips_ext='[_e]bcd.fits', channels=['ch1','ch2','ch3','ch4','mips1'], drz_query='r*', sync_results=True, ref_seg=None, min_frame={'irac':5, 'mips':1.0}, med_max_size=500e6, stop_at='', make_psf=True, **kwargs):
"""
stop_at: preprocess, make_compact
"""
from grizli import utils
from . import irac
from .utils import get_wcslist, fetch_irac
PATH = os.path.join(home, root)
try:
os.mkdir(PATH)
except:
pass
os.chdir(PATH)
if not skip_fetch:
# Fetch IRAC bcds
if not os.path.exists(f'{root}_ipac.fits'):
os.system(f'wget https://s3.amazonaws.com/{bucket}/IRAC/{root}_ipac.fits')
res = fetch_irac(root=root, path='./', channels=channels)
if res in [False, None]:
# Nothing to do
make_html(root, bucket=bucket)
print(f'### Done: \n https://s3.amazonaws.com/{bucket}/Pipeline/{root}/IRAC/{root}.irac.html')
utils.log_comment(f'/tmp/{root}.success', 'Done!',
verbose=True, show_date=True)
return True
# Sync CHArGE HST images
os.system(f'aws s3 sync s3://{bucket}/Pipeline/{root}/Prep/ ./ '
f' --exclude "*" --include "{root}*seg.fits*"'
f' --include "{root}-ir_drz*fits*"'
f' --include "{root}*psf.fits*"'
f' --include "{root}-f[01]*_drz*fits.gz"'
f' --include "{root}*phot.fits"')
# Drizzle properties of the preliminary mosaic
#pixfrac, pix, kernel = 0.2, 1.0, 'square'
# Define an output WCS aligned in pixel phase to the HST mosaic ()
if not os.path.exists('ref_hdu.fits'):
wcslist = get_wcslist(skip=-500)
out_hdu = utils.make_maximal_wcs(wcslist, pixel_scale=initial_pix, theta=0, pad=5, get_hdu=True, verbose=True)
# Make sure pixels align
ref_file = glob.glob('{0}-f[01]*_drz_sci.fits*'.format(root))
if len(ref_file) == 0:
os.system(f'aws s3 sync s3://{bucket}/Pipeline/{root}/Prep/ ./ '
f' --exclude "*"'
f' --include "{root}-f[678]*_dr*fits.gz"')
ref_file = glob.glob('{0}-f[678]*_dr*_sci.fits*'.format(root))
ref_file = ref_file[-1]
print(f'\nHST reference image: {ref_file}\n')
ref_hdu = pyfits.open(ref_file)[0].header
ref_filter = utils.get_hst_filter(ref_hdu).lower()
ref_wcs = pywcs.WCS(ref_hdu)
ref_rd = ref_wcs.all_pix2world(np.array([[-0.5, -0.5]]), 0).flatten()
target_phase = np.array([0.5, 0.5])#/(pix/0.1)
for k in ['RADESYS', 'LATPOLE', 'LONPOLE']:
out_hdu.header[k] = ref_hdu[k]
# Shift CRVAL to same tangent point
out_wcs = pywcs.WCS(out_hdu.header)
out_xy = out_wcs.all_world2pix(np.array([ref_wcs.wcs.crval]), 1).flatten()
out_hdu.header['CRVAL1'], out_hdu.header['CRVAL2'] = tuple(ref_wcs.wcs.crval)
out_hdu.header['CRPIX1'], out_hdu.header['CRPIX2'] = tuple(out_xy)
# Align integer pixel phase
out_wcs = pywcs.WCS(out_hdu.header)
out_xy = out_wcs.all_world2pix(np.array([ref_rd]), 0).flatten()
xy_phase = out_xy - np.floor(out_xy)
new_crpix = out_wcs.wcs.crpix - (xy_phase - target_phase)
out_hdu.header['CRPIX1'], out_hdu.header['CRPIX2'] = tuple(new_crpix)
out_wcs = pywcs.WCS(out_hdu.header)
out_hdu.writeto('ref_hdu.fits', output_verify='Fix')
else:
out_hdu = pyfits.open('ref_hdu.fits')[1]
########
files = []
for ch in channels:
if 'mips' in ch:
mc = ch.replace('mips','ch')
files += glob.glob(f'{aor_query}/{mc}/bcd/SPITZER_M*{mips_ext}')
files += glob.glob(f'{aor_query}/{mc}/bcd/SPITZER_M*xbcd.fits.gz')
else:
files += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*cbcd.fits')
files += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*xbcd.fits.gz')
files.sort()
roots = np.array([file.split('/')[0] for file in files])
with_channels = np.array([file.split('_')[1] for file in files])
all_roots = np.array(['{0}-{1}'.format(r, c.replace('I','ch').replace('M', 'mips')) for r, c in zip(roots, with_channels)])
tab = {'aor':[], 'N':[], 'channel':[]}
for r in np.unique(all_roots):
tab['aor'].append(r.split('-')[0])
tab['N'].append((all_roots == r).sum())
tab['channel'].append(r.split('-')[1])
aors = utils.GTable(tab)
print(aors)
########
SKIP = True # Don't regenerate finished files
delete_group = False # Delete intermediate products from memory
zip_outputs = False # GZip intermediate products
aors_ch = {}
########
# Process mosaics by AOR
# Process in groups, helps for fields like HFF with dozens/hundreds of AORs!
for ch in channels:
aor = aors[(aors['channel'] == ch) & (aors['N'] > 5)]
if len(aor) == 0:
continue
#aors_ch[ch] = []
if ch in ['ch1','ch2']:
NPER, instrument = 500, 'irac'
if ch in ['ch3','ch4']:
NPER, instrument = 500, 'irac'
elif ch in ['mips1']:
NPER, instrument = 400, 'mips'
min_frametime = min_frame[instrument]
nsort = np.cumsum(aor['N']/NPER)
NGROUP = int(np.ceil(nsort.max()))
count = 0
for g in range(NGROUP):
root_i = root+'-{0:02d}'.format(g)
gsel = (nsort > g) & (nsort <= g+1)
aor_ids = list(aor['aor'][gsel])
print('{0}-{1} N_AOR = {2:>2d} N_EXP = {3:>4d}'.format(root_i, ch, len(aor_ids), aor['N'][gsel].sum()))
count += gsel.sum()
files = glob.glob('{0}-{1}*'.format(root_i, ch))
if (len(files) > 0) & (SKIP):
print('Skip {0}-{1}'.format(root_i, ch))
continue
with open('{0}-{1}.log'.format(root_i, ch),'w') as fp:
fp.write(time.ctime())
# Do internal alignment to GAIA.
# Otherwise, set `radec` to the name of a file that has two columns with
# reference ra/dec.
#radec = None
# Pipeline
if instrument == 'mips':
aors_ch[ch] = irac.process_all(channel=ch.replace('mips','ch'), output_root=root_i, driz_scale=initial_pix, kernel=kernel, pixfrac=pixfrac, wcslist=None, pad=0, out_hdu=out_hdu, aor_ids=aor_ids, flat_background=False, two_pass=True, min_frametime=min_frametime, instrument=instrument, align_threshold=0.15, radec=radec, run_alignment=False, mips_ext=mips_ext, ref_seg=ref_seg, global_mask=root+'_mask.reg')
else:
aors_ch[ch] = irac.process_all(channel=ch, output_root=root_i, driz_scale=initial_pix, kernel=kernel, pixfrac=pixfrac, wcslist=None, pad=0, out_hdu=out_hdu, aor_ids=aor_ids, flat_background=False, two_pass=True, min_frametime=min_frametime, instrument=instrument, radec=radec, run_alignment=run_alignment, assume_close=assume_close, ref_seg=ref_seg, global_mask=root+'_mask.reg', med_max_size=med_max_size)
if len(aors_ch[ch]) == 0:
continue
# PSFs
plt.ioff()
if (instrument != 'mips') & make_psf:
ch_num = int(ch[-1])
segmask=True
# psf_size=20
# for p in [0.1, final_pix]:
# irac.mosaic_psf(output_root=root_i, target_pix=p, channel=ch_num, aors=aors_ch[ch], kernel=kernel, pixfrac=pixfrac, size=psf_size, native_orientation=False, instrument=instrument, subtract_background=False, segmentation_mask=segmask, max_R=10)
# plt.close('all')
psf_size=30
p = 0.1
irac.mosaic_psf(output_root=root_i, target_pix=p, channel=ch_num, aors=aors_ch[ch], kernel=kernel, pixfrac=pixfrac, size=psf_size, native_orientation=True, subtract_background=False, segmentation_mask=segmask, max_R=10)
plt.close('all')
if delete_group:
del(aors_ch[ch])
print('Done {0}-{1}, gzip products'.format(root_i, ch))
if zip_outputs:
os.system('gzip {0}*-{1}_drz*fits'.format(root_i, ch))
# PSFs
if (instrument != 'mips') & make_psf:
# Average PSF
p = 0.1
files = glob.glob('*{0}-{1:.1f}*psfr.fits'.format(ch, p))
if len(files) == 0:
continue
files.sort()
avg = None
for file in files:
im = pyfits.open(file)
if avg is None:
wht = im[0].data != 0
avg = im[0].data*wht
else:
wht_i = im[0].data != 0
avg += im[0].data*wht_i
wht += wht_i
im.close()
avg = avg/wht
avg[wht == 0] = 0
# Window
from photutils import (HanningWindow, TukeyWindow,
CosineBellWindow,
SplitCosineBellWindow, TopHatWindow)
coswindow = CosineBellWindow(alpha=1)
avg *= coswindow(avg.shape)**0.05
avg /= avg.sum()
pyfits.writeto('{0}-{1}-{2:0.1f}.psfr_avg.fits'.format(root, ch, p), data=avg, header=im[0].header, overwrite=True)
####
## Show the initial product
plt.ioff()
for i in range(10):
files = glob.glob(f'{root}-{i:02d}-ch*sci.fits')
if len(files) > 0:
break
files.sort()
if len(files) == 1:
subs = 1,1
fs = [7,7]
elif len(files) == 2:
subs = 1,2
fs = [14,7]
elif len(files) == 3:
subs = 2,2
fs = [14,14]
else:
subs = 2,2
fs = [14,14]
fig = plt.figure(figsize=fs)
for i, file in enumerate(files[:4]):
im = pyfits.open(file)
print('{0} {1} {2:.1f} s'.format(file, im[0].header['FILTER'], im[0].header['EXPTIME']))
ax = fig.add_subplot(subs[0], subs[1], 1+i)
ax.imshow(im[0].data, vmin=-0.1, vmax=1, cmap='gray_r', origin='lower')
ax.text(0.05, 0.95, file, ha='left', va='top', color='k',
transform=ax.transAxes)
im.close()
if len(files) > 1:
fig.axes[1].set_yticklabels([])
if len(files) > 2:
fig.axes[0].set_xticklabels([])
fig.axes[1].set_xticklabels([])
if len(files) > 3:
fig.axes[3].set_yticklabels([])
fig.tight_layout(pad=0.5)
fig.savefig(f'{root}.init.png')
plt.close('all')
if stop_at == 'preprocess':
return True
#######
# Make more compact individual exposures and clean directories
wfiles = []
for ch in channels:
if 'mips' in ch:
chq = ch.replace('mips','ch')
wfiles += glob.glob(f'{aor_query}/{chq}/bcd/SPITZER_M*wcs.fits')
else:
wfiles += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*wcs.fits')
#wfiles = glob.glob('r*/*/bcd/*_I[1-4]_*wcs.fits')
#wfiles += glob.glob('r*/*/bcd/*_M[1-4]_*wcs.fits')
wfiles.sort()
for wcsfile in wfiles:
outfile = wcsfile.replace('_wcs.fits', '_xbcd.fits.gz')
if os.path.exists(outfile):
print(outfile)
else:
irac.combine_products(wcsfile)
print('Run: ', outfile)
if os.path.exists(outfile):
remove_files = glob.glob('{0}*fits'.format(wcsfile.split('_wcs')[0]))
for f in remove_files:
print(' rm ', f)
os.remove(f)
if stop_at == 'make_compact':
return True
#############
# Drizzle final mosaics
# Make final mosaic a bit bigger than the HST image
pad = mosaic_pad
# Pixel scale of final mosaic.
# Don't make too small if not many dithers available as in this example.
# But for well-sampled mosaics like RELICS / HFF, can push this to perhaps 0.3" / pix
pixscale = final_pix #0.5
# Again, if have many dithers maybe can use more aggressive drizzle parameters,
# like a 'point' kernel or smaller pixfrac (a 'point' kernel is pixfrac=0)
#kernel, pixfrac = 'square', 0.2
# Correction for bad columns near bright stars
#pulldown_mag = 15.2
##############
# Dilation for CR rejection
dil = np.ones((3,3))
driz_cr = [7, 4]
blot_interp = 'poly5'
bright_fmax = 0.5
### Drizzle
for ch in channels: #[:2]:
###########
# Files and reference image for extra CR rejection
if ch == 'mips1':
files = glob.glob('{0}/ch1/bcd/SPITZER_M1_*xbcd.fits*'.format(drz_query, ch))
files.sort()
pulldown_mag = -10
pixscale = 1.
kernel = 'point'
else:
files = glob.glob('{0}/{1}/bcd/*_I?_*xbcd.fits*'.format(drz_query, ch))
files.sort()
#ref = pyfits.open('{0}-00-{1}_drz_sci.fits'.format(root, ch))
#ref_data = ref[0].data.astype(np.float32)
ref_files = glob.glob(f'{root}-??-{ch}*sci.fits')
if len(ref_files) == 0:
continue
num = None
for ref_file in ref_files:
ref = pyfits.open(ref_file)
wht = pyfits.open(ref_file.replace('_sci.fits', '_wht.fits'))
if num is None:
num = ref[0].data*wht[0].data
den = wht[0].data
else:
num += ref[0].data*wht[0].data
den += wht[0].data
ref_data = (num/den).astype(np.float32)
ref_data[den <= 0] = 0
ref_wcs = pywcs.WCS(ref[0].header, relax=True)
ref_wcs.pscale = utils.get_wcs_pscale(ref_wcs)
if (not hasattr(ref_wcs, '_naxis1')) & hasattr(ref_wcs, '_naxis'):
ref_wcs._naxis1, ref_wcs._naxis2 = ref_wcs._naxis
##############
# Output WCS based on HST footprint
if drizzle_ref_file == '':
try:
hst_im = pyfits.open(glob.glob('{0}-f[01]*_drz_sci.fits*'.format(root))[-1])
except:
hst_im = pyfits.open(glob.glob('{0}-f[578]*_dr*sci.fits*'.format(root))[-1])
hst_wcs = pywcs.WCS(hst_im[0])
hst_wcs.pscale = utils.get_wcs_pscale(hst_wcs)
try:
size = (np.round(np.array([hst_wcs._naxis1, hst_wcs._naxis2])*hst_wcs.pscale*pad/pixscale)*pixscale)
except:
size = (np.round(np.array([hst_wcs._naxis[0], hst_wcs._naxis[1]])*hst_wcs.pscale*pad/pixscale)*pixscale)
hst_rd = hst_wcs.calc_footprint().mean(axis=0)
_x = utils.make_wcsheader(ra=hst_rd[0], dec=hst_rd[1],
size=size,
pixscale=pixscale,
get_hdu=False, theta=0)
out_header, out_wcs = _x
else:
driz_ref_im = pyfits.open(drizzle_ref_file)
out_wcs = pywcs.WCS(driz_ref_im[0].header, relax=True)
out_wcs.pscale = utils.get_wcs_pscale(out_wcs)
out_header = utils.to_header(out_wcs)
if (not hasattr(out_wcs, '_naxis1')) & hasattr(out_wcs, '_naxis'):
out_wcs._naxis1, out_wcs._naxis2 = out_wcs._naxis
##############
# Bright stars for pulldown correction
cat_file = glob.glob(f'{root}-[0-9][0-9]-{ch}.cat.fits')[0]
ph = utils.read_catalog(cat_file)
bright = (ph['mag_auto'] < pulldown_mag) # & (ph['flux_radius'] < 3)
ph = ph[bright]
##############
# Now do the drizzling
yp, xp = np.indices((256, 256))
orig_files = []
out_header['DRIZ_CR0'] = driz_cr[0]
out_header['DRIZ_CR1'] = driz_cr[1]
out_header['KERNEL'] = kernel
out_header['PIXFRAC'] = pixfrac
out_header['NDRIZIM'] = 0
out_header['EXPTIME'] = 0
out_header['BUNIT'] = 'microJy'
out_header['FILTER'] = ch
med_root = 'xxx'
N = len(files)
for i, file in enumerate(files):#[:100]):
print('{0}/{1} {2}'.format(i, N, file))
if file in orig_files:
continue
im = pyfits.open(file)
ivar = 1/im['CBUNC'].data**2
msk = (~np.isfinite(ivar)) | (~np.isfinite(im['CBCD'].data))
im['CBCD'].data[msk] = 0
ivar[msk] = 0
wcs = pywcs.WCS(im['WCS'].header, relax=True)
wcs.pscale = utils.get_wcs_pscale(wcs)
if (not hasattr(wcs, '_naxis1')) & hasattr(wcs, '_naxis'):
wcs._naxis1, wcs._naxis2 = wcs._naxis
fp = Path(wcs.calc_footprint())
med_root_i = im.filename().split('/')[0]
if med_root != med_root_i:
print('\n Read {0}-{1}_med.fits \n'.format(med_root_i, ch))
med = pyfits.open('{0}-{1}_med.fits'.format(med_root_i, ch))
med_data = med[0].data.astype(np.float32)
med_root = med_root_i
med.close()
try:
gaia_rd = utils.read_catalog('{0}-{1}_gaia.radec'.format(med_root_i, ch))
ii, rr = gaia_rd.match_to_catalog_sky(ph)
gaia_rd = gaia_rd[ii][rr.value < 2]
gaia_pts = np.array([gaia_rd['ra'].data,
gaia_rd['dec'].data]).T
except:
gaia_rd = []
#data = im['CBCD'].data - aor_med[0].data
# Change output units to uJy / pix
if ch == 'mips1':
# un = 1*u.MJy/u.sr
# #to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(out_wcs.pscale**2)
# to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(native_scale**2)
to_ujy_px = 146.902690
else:
# native_scale = 1.223
# un = 1*u.MJy/u.sr
# #to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(out_wcs.pscale**2)
# to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(native_scale**2)
to_ujy_px = 35.17517196810
blot_data = ablot.do_blot(ref_data, ref_wcs, wcs, 1, coeffs=True,
interp=blot_interp,
sinscl=1.0, stepsize=10,
wcsmap=None)/to_ujy_px
# mask for bright stars
eblot = 1-np.clip(blot_data, 0, bright_fmax)/bright_fmax
# Initial CR
clean = im[0].data - med_data - im['WCS'].header['PEDESTAL']
dq = (clean - blot_data)*np.sqrt(ivar)*eblot > driz_cr[0]
# Adjacent CRs
dq_dil = binary_dilation(dq, selem=dil)
dq |= ((clean - blot_data)*np.sqrt(ivar)*eblot > driz_cr[1]) & (dq_dil)
# Very negative pixels
dq |= clean*np.sqrt(ivar) < -4
original_dq = im['WCS'].data - (im['WCS'].data & 1)
dq |= original_dq > 0
# Pulldown correction for bright stars
if len(gaia_rd) > 0:
mat = fp.contains_points(gaia_pts)
if mat.sum() > 0:
xg, yg = wcs.all_world2pix(gaia_rd['ra'][mat], gaia_rd['dec'][mat], 0)
sh = dq.shape
mat = (xg > 0) & (xg < sh[1]) & (yg > 0) & (yg < sh[0])
if mat.sum() > 0:
for xi, yi in zip(xg[mat], yg[mat]):
dq |= (np.abs(xp-xi) < 2) & (np.abs(yp-yi) > 10)
if i == 0:
res = utils.drizzle_array_groups([clean], [ivar*(dq == 0)], [wcs], outputwcs=out_wcs, kernel=kernel, pixfrac=pixfrac, data=None, verbose=False)
# Copy header keywords
wcs_header = utils.to_header(wcs)
for k in im[0].header:
if (k not in ['', 'HISTORY', 'COMMENT']) & (k not in out_header) & (k not in wcs_header):
out_header[k] = im[0].header[k]
else:
_ = utils.drizzle_array_groups([clean], [ivar*(dq == 0)], [wcs], outputwcs=out_wcs, kernel=kernel, pixfrac=pixfrac, data=res[:3], verbose=False)
out_header['NDRIZIM'] += 1
out_header['EXPTIME'] += im[0].header['EXPTIME']
im.close()
# Pixel scale factor for weights
wht_scale = (out_wcs.pscale/wcs.pscale)**-4
# Write final images
pyfits.writeto('{0}-{1}_drz_sci.fits'.format(root, ch), data=res[0]*to_ujy_px, header=out_header,
output_verify='fix', overwrite=True)
pyfits.writeto('{0}-{1}_drz_wht.fits'.format(root, ch), data=res[1]*wht_scale/to_ujy_px**2,
header=out_header, output_verify='fix', overwrite=True)
##########
## Show the final drizzled images
plt.ioff()
files = glob.glob(f'{root}-ch*sci.fits')
files.sort()
if len(files) == 1:
subs = 1,1
fs = [7,7]
elif len(files) == 2:
subs = 1,2
fs = [14,7]
elif len(files) == 3:
subs = 2,2
fs = [14,14]
else:
subs = 2,2
fs = [14,14]
fig = plt.figure(figsize=fs)
for i, file in enumerate(files[:4]):
im = pyfits.open(file)
print('{0} {1} {2:.1f} s'.format(file, im[0].header['FILTER'], im[0].header['EXPTIME']))
ax = fig.add_subplot(subs[0], subs[1], 1+i)
scl = (final_pix/initial_pix)**2
ax.imshow(im[0].data, vmin=-0.1*scl, vmax=1*scl, cmap='gray_r', origin='lower')
ax.text(0.05, 0.95, file, ha='left', va='top', color='k',
transform=ax.transAxes)
im.close()
if len(files) > 1:
fig.axes[1].set_yticklabels([])
if len(files) > 2:
fig.axes[0].set_xticklabels([])
fig.axes[1].set_xticklabels([])
if len(files) > 3:
fig.axes[3].set_yticklabels([])
fig.tight_layout(pad=0.5)
fig.savefig(f'{root}.final.png')
plt.close('all')
if sync_results:
print('gzip mosaics')
os.system(f'gzip -f {root}-ch*_drz*fits {root}-mips*_drz*fits')
######## Sync
## Sync
print(f's3://{bucket}/Pipeline/{root}/IRAC/')
make_html(root, bucket=bucket)
os.system(f'aws s3 sync ./ s3://{bucket}/Pipeline/{root}/IRAC/'
f' --exclude "*" --include "{root}-ch*drz*fits*"'
f' --include "{root}-mips*drz*fits*"'
f' --include "{root}.*png"'
' --include "*-ch*psf*" --include "*log.fits"'
' --include "*wcs.[lp]*"'
' --include "*html" --include "*fail*"'
' --acl public-read')
if sync_xbcd:
aor_files = glob.glob('r*-ch*med.fits')
for aor_file in aor_files:
aor = aor_file.split('-ch')[0]
os.system(f'aws s3 sync ./{aor}/ s3://{bucket}/IRAC/AORS/{aor}/ --exclude "*" --include "ch*/bcd/*xbcd.fits.gz" --acl public-read')
os.system(f'aws s3 cp {aor_file} s3://{bucket}/IRAC/AORS/ --acl public-read')
msg = f'### Done: \n https://s3.amazonaws.com/{bucket}/Pipeline/{root}/IRAC/{root}.irac.html'
utils.log_comment(f'/tmp/{root}.success', msg, verbose=True, show_date=True)