def redo_query(root='j023507-040202',
instruments=['WFC3/IR', 'WFC3/UVIS', 'ACS/WFC'],
filters=[],
proposal_id=[]):
"""
Redo the MAST query based on the parent table to look for new data
"""
from astropy.io.misc import yaml
from mastquery import query, overlaps
from grizli import utils
# Read the existing catalog
parent = utils.read_catalog('{0}_footprint.fits'.format(root))
# Overlap keywords, if available
kwargs = {}
if 'BUFFER' in parent.meta:
kwargs['buffer_arcmin'] = parent.meta['BUFFER']
if 'FOLAP' in parent.meta:
kwargs['fractional_overlap'] = parent.meta['FOLAP']
if 'MIN_AREA' in parent.meta:
kwargs['min_area'] = parent.meta['MIN_AREA']
# Rerun overlap query to get new data
tabs = overlaps.find_overlaps(parent,
filters=filters,
proposal_id=proposal_id,
instruments=instruments,
close=True,
keep_single_name=True,
**kwargs)
def group_by_filter():
"""
aws s3 sync --exclude "*" --include "cosmos_visits*" s3://grizli-preprocess/CosmosMosaic/ ./
"""
from grizli import prep, utils
import numpy as np
master = 'cosmos'
master = 'grizli-cosmos-v2'
master = 'grizli-jan2019'
master = 'grizli-v1-19.12.04'
master = 'grizli-v1-19.12.05'
tab = utils.read_catalog('{0}_visits.fits'.format(master))
all_visits = np.load('{0}_visits.npy'.format(master), allow_pickle=True)[0]
# By filter
# Exclude DASH
dash = utils.column_string_operation(tab['product'], 'icxe', 'startswith')
dash |= utils.column_string_operation(tab['product'], '_icxe',
'count', 'or')
# Don't exclude DASH
dash = utils.column_string_operation(tab['product'], 'xxxx', 'startswith')
groups = {}
fpstr = {}
for filt in np.unique(tab['filter']):
mat = (tab['filter'] == filt) & (~dash)
groups[filt] = {'filter': filt, 'files': [], 'awspath': [], 'footprints': []}
fpstr[filt] = 'fk5\n'
for ix in np.where(mat)[0]:
fp = all_visits[ix]['footprint']
if hasattr(fp, '__len__'):
fps = fp
else:
fps = [fp]
for fp in fps:
xy = fp.boundary.xy
pstr = 'polygon('+','.join(['{0:.6f}'.format(i) for i in np.array([xy[0].tolist(), xy[1].tolist()]).T.flatten()])+') # text={{{0}}}\n'.format(all_visits[ix]['product'])
fpstr[filt] += pstr
for k in ['files', 'awspath', 'footprints']:
groups[filt][k].extend(all_visits[ix][k])
fp = open('{0}-pointings-{1}.reg'.format(master, filt), 'w')
fp.write(fpstr[filt])
fp.close()
print('{0:6} {1:>3d} {2:>4d} ({3:>4d})'.format(filt, mat.sum(), len(groups[filt]['files']), len(np.unique(groups[filt]['files']))))
np.save('{0}_filter_groups.npy'.format(master), [groups])
def make_masks(files=None, inspect='grizli_inspect.fits', ext=1):
"""
Make satellite trail masks
"""
import os
import astropy.io.fits as pyfits
try:
from .. import utils
from ..ds9 import DS9
except:
from grizli import utils
from grizli.ds9 import DS9
if files is None:
insp = utils.read_catalog(inspect)
flag = (insp['satellite'] > 0) | (insp['earth'] > 0)
files = [
f.replace('_ramp.png', '_flt.fits') for f in insp['images'][flag]
]
ds9 = DS9()
for file in files:
im = pyfits.open(file)
med = np.median(im['SCI', ext].data)
ds9.view(im['SCI', ext].data - med)
reg_file = file.replace('_flt.fits', '.*.mask.reg').replace(
'_flc.fits', '.*.mask.reg').replace('_c0m.fits',
'.*.mask.reg').replace(
'*', '{0:02d}'.format(ext))
if os.path.exists(reg_file):
ds9.set('regions file ' + reg_file)
x = input(file + ': draw region (x to skip, q to abort): ')
if x in ['q']:
print('Abort.')
#continue
return False
elif x in ['x']:
print('Skip {0}.'.format(file))
continue
ds9.set('regions save ' + reg_file)
def make_masks(files=None, inspect='grizli_inspect.fits', ext=1):
"""
Make satellite trail masks
"""
import os
import astropy.io.fits as pyfits
try:
from .. import utils
from ..ds9 import DS9
except:
from grizli import utils
from grizli.ds9 import DS9
if files is None:
insp = utils.read_catalog(inspect)
flag = (insp['satellite'] > 0) | (insp['earth'] > 0)
files=[f.replace('_ramp.png','_flt.fits') for f in insp['images'][flag]]
ds9 = DS9()
for file in files:
im = pyfits.open(file)
med = np.median(im['SCI',ext].data)
ds9.view(im['SCI',ext].data-med)
reg_file = file.replace('_flt.fits','.*.mask.reg').replace('_flc.fits','.*.mask.reg').replace('_c0m.fits','.*.mask.reg').replace('*', '{0:02d}'.format(ext))
if os.path.exists(reg_file):
ds9.set('regions file '+reg_file)
x = input(file+': draw region (x to skip, q to abort): ')
if x in ['q']:
print('Abort.')
#continue
return False
elif x in ['x']:
print('Skip {0}.'.format(file))
continue
ds9.set('regions save '+reg_file)
def eazy_photoz(root,
force=False,
object_only=True,
apply_background=True,
aper_ix=1,
apply_prior=True,
beta_prior=True,
get_external_photometry=True,
external_limits=3,
external_sys_err=0.3,
external_timeout=300,
sys_err=0.05,
z_step=0.01,
z_min=0.01,
z_max=12,
total_flux='flux'):
import os
import eazy
import numpy as np
from grizli import utils
import mastquery.utils
if (os.path.exists('{0}.eazypy.self.npy'.format(root))) & (not force):
self = np.load('{0}.eazypy.self.npy'.format(root))[0]
zout = utils.read_catalog('{0}.eazypy.zout.fits'.format(root))
cat = utils.read_catalog('{0}_phot_apcorr.fits'.format(root))
return self, cat, zout
trans = {
'f098m': 201,
'f105w': 202,
'f110w': 241,
'f125w': 203,
'f140w': 204,
'f160w': 205,
'f435w': 233,
'f438w': 211,
'f606w': 236,
'f625w': 237,
'f814w': 239,
'f702w': 15,
'f555w': 235,
'f350lp': 339
}
cat = utils.read_catalog('{0}_phot.fits'.format(root))
filters = []
for c in cat.meta:
if c.endswith('_ZP'):
filters.append(c.split('_ZP')[0].lower())
if get_external_photometry:
print('Get external photometry from Vizier')
try:
ext = get_external_catalog(cat,
external_limits=external_limits,
timeout=external_timeout,
sys_err=external_sys_err)
for c in ext.colnames:
if c not in cat.colnames:
cat[c] = ext[c]
for k in ext.meta:
cat.meta[k] = ext.meta[k]
except:
print(' - External catalog FAILED')
pass
# Total flux
cat.meta['TOTALCOL'] = total_flux, 'Column for total flux'
apcorr = {}
for i in range(5):
if 'flux_aper_{0}'.format(i) in cat.colnames:
cat['apcorr_{0}'.format(
i)] = cat[total_flux] / cat['flux_aper_{0}'.format(i)]
for f in filters:
bkgc = '{0}_bkg_aper_{1}'.format(f, i)
if (bkgc in cat.colnames) & apply_background:
bkg = cat[bkgc]
else:
bkg = 0.
cat['{0}_corr_{1}'.format(
f, i)] = (cat['{0}_flux_aper_{1}'.format(f, i)] -
bkg) * cat['apcorr_{0}'.format(i)]
cat['{0}_ecorr_{1}'.format(
f, i)] = cat['{0}_fluxerr_aper_{1}'.format(
f, i)] * cat['apcorr_{0}'.format(i)]
# mask_thresh = np.percentile(cat['{0}_mask_aper_{1}'.format(f, i)], 95)
aper_area = np.pi * (cat.meta['APER_{0}'.format(i)] / 2)**2
mask_thresh = aper_area
bad = cat['{0}_mask_aper_{1}'.format(f, i)] > 0.2 * mask_thresh
cat['{0}_corr_{1}'.format(f, i)][bad] = -99
cat['{0}_ecorr_{1}'.format(f, i)][bad] = -99
cat.rename_column('number', 'id')
cat['z_spec'] = cat['id'] * 0. - 1
# Spurious sources, sklearn SVM model trained for a single field
morph_model = os.path.join(os.path.dirname(utils.__file__),
'data/sep_catalog_junk.pkl')
if os.path.exists(morph_model):
from sklearn.externals import joblib
clf = joblib.load(morph_model)
X = np.hstack([[cat['peak'] / cat['flux'],
cat['cpeak'] / cat['peak']]]).T
# Predict labels, which where generated for
# bad_bright, bad_faint, stars, big_galaxies, small_galaxies
pred = clf.predict_proba(X)
# Should be >~ 0.9 for valid sources, stars & galaxies in "ir" image
cat['class_valid'] = pred[:, -3:].sum(axis=1)
cat['class_valid'].format = '.2f'
cat.write('{0}_phot_apcorr.fits'.format(root), overwrite=True)
# Translate
fp = open('zphot.translate', 'w')
for f in filters:
if f in trans:
fp.write('{0}_corr_{1} F{2}\n'.format(f, aper_ix, trans[f]))
fp.write('{0}_ecorr_{1} E{2}\n'.format(f, aper_ix, trans[f]))
fp.close()
params = {}
params['CATALOG_FILE'] = '{0}_phot_apcorr.fits'.format(root)
params['Z_STEP'] = z_step
params['MAIN_OUTPUT_FILE'] = '{0}.eazypy'.format(root)
params['Z_MAX'] = z_max
params['MW_EBV'] = mastquery.utils.get_irsa_dust(cat['ra'].mean(),
cat['dec'].mean())
params['PRIOR_ABZP'] = 23.9
params['SYS_ERR'] = sys_err
params['CAT_HAS_EXTCORR'] = False
# Pick prior filter, starting from reddest
for f in [
'f435w', 'f606w', 'f814w', 'f105w', 'f110w', 'f125w', 'f140w',
'f160w'
][::-1]:
if f in filters:
params['PRIOR_FILTER'] = trans[f]
mag = 23.9 - 2.5 * np.log10(cat['{0}_corr_{1}'.format(f, aper_ix)])
break
#
params['TEMPLATES_FILE'] = 'templates/fsps_full/tweak_fsps_QSF_12_v3.param'
zpfile = None
load_products = False
eazy.symlink_eazy_inputs(
path='/usr/local/share/python/eazy-py/eazy-photoz', path_is_env=False)
self = eazy.photoz.PhotoZ(param_file=None,
translate_file='zphot.translate',
zeropoint_file=zpfile,
params=params,
load_prior=True,
load_products=load_products)
if object_only:
return self
idx = np.arange(self.NOBJ)
#sample = (mag < 27) #& (self.cat['star_flag'] != 1)
#sample |= (self.cat['z_spec'] > 0)
sample = np.isfinite(mag)
for iter in range(2):
self.fit_parallel(idx[sample], n_proc=10)
self.error_residuals()
self.standard_output(prior=apply_prior, beta_prior=beta_prior)
zout = utils.read_catalog('{0}.eazypy.zout.fits'.format(root))
np.save('{0}.eazypy.self.npy'.format(root), [self])
return self, cat, zout
def drizzle_images(label='macs0647-jd1', ra=101.9822125, dec=70.24326667, pixscale=0.1, size=10, wcs=None, pixfrac=0.33, kernel='square', theta=0, half_optical_pixscale=True, filters=['f160w', 'f140w', 'f125w', 'f105w', 'f110w', 'f098m', 'f850lp', 'f814w', 'f775w', 'f606w', 'f475w', 'f555w', 'f600lp', 'f390w', 'f350lp'], skip=None, remove=True, rgb_params=RGB_PARAMS, master='grizli-jan2019', aws_bucket='s3://grizli/CutoutProducts/', scale_ab=21, thumb_height=2.0, sync_fits=True, subtract_median=True, include_saturated=True, include_ir_psf=False, show_filters=['visb', 'visr', 'y', 'j', 'h'], combine_similar_filters=True, single_output=True, aws_prep_dir=None, make_segmentation_figure=False, get_dict=False, dryrun=False, **kwargs):
"""
label='cp561356'; ra=150.208875; dec=1.850241667; size=40; filters=['f160w','f814w', 'f140w','f125w','f105w','f606w','f475w']
master: These are sets of large lists of available exposures
'cosmos': deprecated
'grizli-cosmos-v2': All imaging covering the COSMOS field
'candels-july2019': CANDELS fields other than COSMOS
'grizli-v1': First processing of the Grizli CHArGE dataset
'grizli-v1-19.12.04': Updated CHArGE fields
"""
import glob
import copy
import os
import numpy as np
import astropy.io.fits as pyfits
from astropy.coordinates import SkyCoord
import astropy.units as u
from drizzlepac.adrizzle import do_driz
import boto3
from grizli import prep, utils
from grizli.pipeline import auto_script
# Function arguments
if get_dict:
frame = inspect.currentframe()
args = inspect.getargvalues(frame).locals
pop_args = ['get_dict', 'frame', 'kwargs']
pop_classes = (np.__class__, do_driz.__class__, SkyCoord.__class__)
for k in kwargs:
args[k] = kwargs[k]
for k in args:
if isinstance(args[k], pop_classes):
pop_args.append(k)
for k in pop_args:
if k in args:
args.pop(k)
return args
# Boto objects
s3 = boto3.resource('s3')
s3_client = boto3.client('s3')
if isinstance(ra, str):
coo = SkyCoord('{0} {1}'.format(ra, dec), unit=(u.hour, u.deg))
ra, dec = coo.ra.value, coo.dec.value
if label is None:
try:
import mastquery.utils
label = mastquery.utils.radec_to_targname(ra=ra, dec=dec, round_arcsec=(1/15, 1), targstr='j{rah}{ram}{ras}{sign}{ded}{dem}{des}')
except:
label = 'grizli-cutout'
#master = 'cosmos'
#master = 'grizli-jan2019'
if master == 'grizli-jan2019':
parent = 's3://grizli/MosaicTools/'
bkt = s3.Bucket('grizli')
elif master == 'cosmos':
parent = 's3://grizli-preprocess/CosmosMosaic/'
bkt = s3.Bucket('grizli-preprocess')
elif master == 'grizli-cosmos-v2':
parent = 's3://grizli-cosmos-v2/Mosaics/'
bkt = s3.Bucket('grizli-cosmos-v2')
elif master == 'candels-july2019':
parent = 's3://grizli-v1/Mosaics/'
bkt = s3.Bucket('grizli-v1')
elif master == 'grizli-v1-19.12.04':
parent = 's3://grizli-v1/Mosaics/'
bkt = s3.Bucket('grizli-v1')
elif master == 'grizli-v1-19.12.05':
parent = 's3://grizli-v1/Mosaics/'
bkt = s3.Bucket('grizli-v1')
else:
# Run on local files, e.g., "Prep" directory
parent = None
bkt = None
#remove = False
# Download summary files from S3
for ext in ['_visits.fits', '_visits.npy', '_filter_groups.npy'][-1:]:
newfile = '{0}{1}'.format(master, ext)
if (not os.path.exists(newfile)) & (parent is not None):
s3_path = parent.split('/')[-2]
s3_file = '{0}{1}'.format(master, ext)
print('{0}{1}'.format(parent, s3_file))
bkt.download_file(s3_path+'/'+s3_file, s3_file,
ExtraArgs={"RequestPayer": "requester"})
#os.system('aws s3 cp {0}{1}{2} ./'.format(parent, master, ext))
#tab = utils.read_catalog('{0}_visits.fits'.format(master))
#all_visits = np.load('{0}_visits.npy'.format(master))[0]
if parent is not None:
groups = np.load('{0}_filter_groups.npy'.format(master), allow_pickle=True)[0]
else:
if aws_prep_dir is not None:
spl = aws_prep_dir.replace('s3://', '').split('/')
prep_bucket = spl[0]
prep_root = spl[2]
prep_bkt = s3.Bucket(prep_bucket)
s3_prep_path = 'Pipeline/{0}/Prep/'.format(prep_root)
s3_full_path = '{0}/{1}'.format(prep_bucket, s3_prep_path)
s3_file = '{0}_visits.npy'.format(prep_root)
# Make output path Prep/../Thumbnails/
if aws_bucket is not None:
aws_bucket = ('s3://' +
s3_full_path.replace('/Prep/', '/Thumbnails/'))
print('{0}{1}'.format(s3_prep_path, s3_file))
if not os.path.exists(s3_file):
prep_bkt.download_file(os.path.join(s3_prep_path, s3_file),
s3_file, ExtraArgs={"RequestPayer": "requester"})
groups_files = glob.glob('{0}_filter_groups.npy'.format(prep_root))
visit_query = prep_root+'_'
else:
groups_files = glob.glob('*filter_groups.npy')
visit_query = '*'
# Reformat local visits.npy into a groups file
if (len(groups_files) == 0):
visit_file = glob.glob(visit_query+'visits.npy')[0]
visits, groups, info = np.load(visit_file, allow_pickle=True)
visit_root = visit_file.split('_visits')[0]
visit_filters = np.array([v['product'].split('-')[-1] for v in visits])
groups = {}
for filt in np.unique(visit_filters):
groups[filt] = {}
groups[filt]['filter'] = filt
groups[filt]['files'] = []
groups[filt]['footprints'] = []
groups[filt]['awspath'] = []
ix = np.where(visit_filters == filt)[0]
for i in ix:
groups[filt]['files'].extend(visits[i]['files'])
groups[filt]['footprints'].extend(visits[i]['footprints'])
Nf = len(groups[filt]['files'])
print('{0:>6}: {1:>3} exposures'.format(filt, Nf))
if aws_prep_dir is not None:
groups[filt]['awspath'] = [s3_full_path
for file in range(Nf)]
np.save('{0}_filter_groups.npy'.format(visit_root), [groups])
else:
print('Use groups file: {0}'.format(groups_files[0]))
groups = np.load(groups_files[0], allow_pickle=True)[0]
#filters = ['f160w','f814w', 'f110w', 'f098m', 'f140w','f125w','f105w','f606w', 'f475w']
filt_dict = FilterDict()
filt_dict.meta['label'] = label
filt_dict.meta['ra'] = ra
filt_dict.meta['dec'] = dec
filt_dict.meta['size'] = size
filt_dict.meta['master'] = master
filt_dict.meta['parent'] = parent
if filters is None:
filters = list(groups.keys())
has_filts = []
lower_filters = [f.lower() for f in filters]
for filt in lower_filters:
if filt not in groups:
continue
visits = [copy.deepcopy(groups[filt])]
#visits[0]['reference'] = 'CarlosGG/ak03_j1000p0228/Prep/ak03_j1000p0228-f160w_drz_sci.fits'
visits[0]['product'] = label+'-'+filt
if wcs is None:
hdu = utils.make_wcsheader(ra=ra, dec=dec, size=size, pixscale=pixscale, get_hdu=True, theta=theta)
h = hdu.header
else:
h = utils.to_header(wcs)
if (filt[:2] in ['f0', 'f1', 'g1']) | (not half_optical_pixscale):
#data = hdu.data
pass
else:
for k in ['NAXIS1', 'NAXIS2', 'CRPIX1', 'CRPIX2']:
h[k] *= 2
h['CRPIX1'] -= 0.5
h['CRPIX2'] -= 0.5
for k in ['CD1_1', 'CD1_2', 'CD2_1', 'CD2_2']:
if k in h:
h[k] /= 2
#data = np.zeros((h['NAXIS2'], h['NAXIS1']), dtype=np.int16)
#pyfits.PrimaryHDU(header=h, data=data).writeto('ref.fits', overwrite=True, output_verify='fix')
#visits[0]['reference'] = 'ref.fits'
print('\n\n###\nMake filter: {0}'.format(filt))
if (filt.upper() in ['F105W', 'F110W', 'F125W', 'F140W', 'F160W']) & include_ir_psf:
clean_i = False
else:
clean_i = remove
status = utils.drizzle_from_visit(visits[0], h, pixfrac=pixfrac, kernel=kernel, clean=clean_i, include_saturated=include_saturated, skip=skip, dryrun=dryrun)
if dryrun:
filt_dict[filt] = status
continue
elif status is not None:
sci, wht, outh, filt_dict[filt] = status
if subtract_median:
#med = np.median(sci[sci != 0])
try:
un_data = np.unique(sci[(sci != 0) & np.isfinite(sci)])
med = utils.mode_statistic(un_data)
except:
med = 0.
if not np.isfinite(med):
med = 0.
print('\n\nMedian {0} = {1:.3f}\n\n'.format(filt, med))
outh['IMGMED'] = (med, 'Median subtracted from the image')
else:
med = 0.
outh['IMGMED'] = (0., 'Median subtracted from the image')
pyfits.writeto('{0}-{1}_drz_sci.fits'.format(label, filt),
data=sci, header=outh, overwrite=True,
output_verify='fix')
pyfits.writeto('{0}-{1}_drz_wht.fits'.format(label, filt),
data=wht, header=outh, overwrite=True,
output_verify='fix')
has_filts.append(filt)
if (filt.upper() in ['F105W', 'F110W', 'F125W', 'F140W', 'F160W']) & include_ir_psf:
from grizli.galfit.psf import DrizzlePSF
hdu = pyfits.open('{0}-{1}_drz_sci.fits'.format(label, filt),
mode='update')
flt_files = [] # visits[0]['files']
for i in range(1, 10000):
key = 'FLT{0:05d}'.format(i)
if key not in hdu[0].header:
break
flt_files.append(hdu[0].header[key])
try:
dp = DrizzlePSF(flt_files=flt_files, driz_hdu=hdu[0])
psf = dp.get_psf(ra=dp.driz_wcs.wcs.crval[0],
dec=dp.driz_wcs.wcs.crval[1],
filter=filt.upper(),
pixfrac=dp.driz_header['PIXFRAC'],
kernel=dp.driz_header['KERNEL'],
wcs_slice=dp.driz_wcs, get_extended=True,
verbose=False, get_weight=False)
psf[1].header['EXTNAME'] = 'PSF'
#psf[1].header['EXTVER'] = filt
hdu.append(psf[1])
hdu.flush()
except:
pass
if remove:
os.system('rm *_fl*fits')
# Dry run, just return dictionary of the found exposure files
if dryrun:
return filt_dict
# Nothing found
if len(has_filts) == 0:
return []
if combine_similar_filters:
combine_filters(label=label)
if rgb_params:
#auto_script.field_rgb(root=label, HOME_PATH=None, filters=has_filts, **rgb_params)
show_all_thumbnails(label=label, thumb_height=thumb_height, scale_ab=scale_ab, close=True, rgb_params=rgb_params, filters=show_filters)
if (single_output != 0):
# Concatenate into a single FITS file
files = glob.glob('{0}-f*_dr[cz]_sci.fits'.format(label))
files.sort()
if combine_similar_filters:
comb_files = glob.glob('{0}-[a-eg-z]*_dr[cz]_sci.fits'.format(label))
comb_files.sort()
files += comb_files
hdul = None
for file in files:
hdu_i = pyfits.open(file)
hdu_i[0].header['EXTNAME'] = 'SCI'
if 'NCOMBINE' in hdu_i[0].header:
if hdu_i[0].header['NCOMBINE'] <= single_output:
continue
filt_i = file.split('-')[-1].split('_dr')[0]
else:
filt_i = utils.get_hst_filter(hdu_i[0].header)
for h in hdu_i:
h.header['EXTVER'] = filt_i
if hdul is None:
hdul = pyfits.HDUList([h])
else:
hdul.append(h)
print('Add to {0}.thumb.fits: {1}'.format(label, file))
# Weight
hdu_i = pyfits.open(file.replace('_sci', '_wht'))
hdu_i[0].header['EXTNAME'] = 'WHT'
for h in hdu_i:
h.header['EXTVER'] = filt_i
if hdul is None:
hdul = pyfits.HDUList([h])
else:
hdul.append(h)
hdul.writeto('{0}.thumb.fits'.format(label), overwrite=True,
output_verify='fix')
for file in files:
for f in [file, file.replace('_sci', '_wht')]:
if os.path.exists(f):
print('Remove {0}'.format(f))
os.remove(f)
# Segmentation figure
thumb_file = '{0}.thumb.fits'.format(label)
if (make_segmentation_figure) & (os.path.exists(thumb_file)) & (aws_prep_dir is not None):
print('Make segmentation figure')
# Fetch segmentation image and catalog
s3_prep_path = 'Pipeline/{0}/Prep/'.format(prep_root)
s3_full_path = '{0}/{1}'.format(prep_bucket, s3_prep_path)
s3_file = '{0}_visits.npy'.format(prep_root)
has_seg_files = True
seg_files = ['{0}-ir_seg.fits.gz'.format(prep_root),
'{0}_phot.fits'.format(prep_root)]
for s3_file in seg_files:
if not os.path.exists(s3_file):
remote_file = os.path.join(s3_prep_path, s3_file)
try:
print('Fetch {0}'.format(remote_file))
prep_bkt.download_file(remote_file, s3_file,
ExtraArgs={"RequestPayer": "requester"})
except:
has_seg_files = False
print('Make segmentation figure failed: {0}'.format(remote_file))
break
if has_seg_files:
s3_cat = utils.read_catalog(seg_files[1])
segmentation_figure(label, s3_cat, seg_files[0])
if aws_bucket:
#aws_bucket = 's3://grizli-cosmos/CutoutProducts/'
#aws_bucket = 's3://grizli/CutoutProducts/'
s3 = boto3.resource('s3')
s3_client = boto3.client('s3')
bkt = s3.Bucket(aws_bucket.split("/")[2])
aws_path = '/'.join(aws_bucket.split("/")[3:])
if sync_fits:
files = glob.glob('{0}*'.format(label))
else:
files = glob.glob('{0}*png'.format(label))
for file in files:
print('{0} -> {1}'.format(file, aws_bucket))
bkt.upload_file(file, '{0}/{1}'.format(aws_path, file).replace('//', '/'), ExtraArgs={'ACL': 'public-read'})
#os.system('aws s3 sync --exclude "*" --include "{0}*" ./ {1} --acl public-read'.format(label, aws_bucket))
#os.system("""echo "<pre>" > index.html; aws s3 ls AWSBUCKETX --human-readable | sort -k 1 -k 2 | grep -v index | awk '{printf("%s %s",$1, $2); printf(" %6s %s ", $3, $4); print "<a href="$5">"$5"</a>"}'>> index.html; aws s3 cp index.html AWSBUCKETX --acl public-read""".replace('AWSBUCKETX', aws_bucket))
return has_filts
def apply_catalog_corrections(root,
total_flux='flux_auto',
auto_corr=True,
get_external_photometry=False,
aperture_indices='all',
suffix='_apcorr',
verbose=True,
apply_background=True):
"""
Aperture and background corrections to photometric catalog
"""
import os
import eazy
import numpy as np
from grizli import utils
import mastquery.utils
cat = utils.read_catalog('{0}_phot.fits'.format(root))
filters = []
for c in cat.meta:
if c.endswith('_ZP'):
filters.append(c.split('_ZP')[0].lower())
if get_external_photometry:
print('Get external photometry from Vizier')
try:
ext = get_external_catalog(cat,
external_limits=external_limits,
timeout=external_timeout,
sys_err=external_sys_err)
for c in ext.colnames:
if c not in cat.colnames:
cat[c] = ext[c]
for k in ext.meta:
cat.meta[k] = ext.meta[k]
except:
print(' - External catalog FAILED')
pass
# Fix: Take flux_auto when flag==0, flux otherwise
if (total_flux == 'flux_auto_fix') & (total_flux not in cat.colnames):
flux = cat['flux_auto'] * 1.
flagged = (cat['flag'] > 0)
flux[flagged] = cat['flux'][flagged]
cat['flux_auto_fix'] = flux * 1.
# Additional auto correction
cat.meta['TOTALCOL'] = total_flux, 'Column for total flux'
#cat.meta['HASTOT'] = (auto_corr & ('tot_corr' in cat.colnames), 'Catalog has full total flux')
apcorr = {}
for NAPER in range(100):
if 'APER_{0}'.format(NAPER) not in cat.meta:
break
if aperture_indices == 'all':
aperture_indices = range(NAPER)
for i in aperture_indices:
if verbose:
print('Compute aperture corrections: i={0}, D={1:.2f}" aperture'.
format(i, cat.meta['ASEC_{0}'.format(i)]))
if 'flux_aper_{0}'.format(i) in cat.colnames:
cat['apcorr_{0}'.format(
i)] = cat[total_flux] / cat['flux_aper_{0}'.format(i)]
for f in filters:
bkgc = '{0}_bkg_aper_{1}'.format(f, i)
# if (bkgc in cat.colnames) & apply_background:
# bkg = cat[bkgc]
# else:
# bkg = 0.
# background already subtracted from flux columns
bkg = 0.
cat['{0}_corr_{1}'.format(
f, i)] = (cat['{0}_flux_aper_{1}'.format(f, i)] -
bkg) * cat['apcorr_{0}'.format(i)]
cat['{0}_ecorr_{1}'.format(
f, i)] = cat['{0}_fluxerr_aper_{1}'.format(
f, i)] * cat['apcorr_{0}'.format(i)]
# mask_thresh = np.percentile(cat['{0}_mask_aper_{1}'.format(f, i)], 95)
aper_area = np.pi * (cat.meta['APER_{0}'.format(i)] / 2)**2
mask_thresh = aper_area
bad = cat['{0}_mask_aper_{1}'.format(f, i)] > 0.2 * mask_thresh
cat['{0}_corr_{1}'.format(f, i)][bad] = -99
cat['{0}_ecorr_{1}'.format(f, i)][bad] = -99
tot_col = '{0}_tot_corr'.format(f.lower())
if auto_corr and (tot_col in cat.colnames):
cat['{0}_tot_{1}'.format(
f,
i)] = cat['{0}_corr_{1}'.format(f, i)] * cat[tot_col]
cat['{0}_etot_{1}'.format(
f,
i)] = cat['{0}_ecorr_{1}'.format(f, i)] * cat[tot_col]
cat['{0}_tot_{1}'.format(f, i)][bad] = -99
cat['{0}_etot_{1}'.format(f, i)][bad] = -99
if 'id' not in cat.colnames:
cat.rename_column('number', 'id')
cat['z_spec'] = cat['id'] * 0. - 1
# Spurious sources, sklearn SVM model trained for a single field
morph_model = os.path.join(os.path.dirname(utils.__file__),
'data/sep_catalog_junk.pkl')
# Only apply if detection pixel scale is 0.06"
if 'ASEC_0' in cat.meta:
try:
detection_pscale = cat.meta['ASEC_0'][0] / cat.meta['APER_0'][0]
except:
detection_pscale = cat.meta['ASEC_0'] / cat.meta['APER_0']
run_morph_model = np.isclose(detection_pscale, 0.06, atol=0.005)
else:
run_morph_model = True
if os.path.exists(morph_model) & run_morph_model:
if verbose:
print('Apply morphological validity class')
from sklearn.externals import joblib
clf = joblib.load(morph_model)
X = np.hstack([[cat['peak'] / cat['flux'],
cat['cpeak'] / cat['peak']]]).T
# Predict labels, which where generated for
# bad_bright, bad_faint, stars, big_galaxies, small_galaxies
pred = clf.predict_proba(X)
# Should be >~ 0.9 for valid sources, stars & galaxies in "ir" image
cat['class_valid'] = pred[:, -3:].sum(axis=1)
cat['class_valid'].format = '.2f'
cat['dummy_err'] = 10**(-0.4 * (8 - 23.9))
cat['dummy_flux'] = cat[total_flux] # detection band
if suffix:
if verbose:
print('Write {0}_phot{1}.fits'.format(root, suffix))
cat.write('{0}_phot{1}.fits'.format(root, suffix), overwrite=True)
return cat
def eazy_photoz(root,
force=False,
object_only=True,
apply_background=True,
aper_ix=1,
apply_prior=False,
beta_prior=True,
get_external_photometry=False,
external_limits=3,
external_sys_err=0.3,
external_timeout=300,
sys_err=0.05,
z_step=0.01,
z_min=0.01,
z_max=12,
total_flux='flux_auto',
auto_corr=True,
compute_residuals=False,
dummy_prior=False,
extra_rf_filters=[],
quiet=True):
import os
import eazy
import numpy as np
from grizli import utils
import mastquery.utils
if (os.path.exists('{0}.eazypy.self.npy'.format(root))) & (not force):
self = np.load('{0}.eazypy.self.npy'.format(root),
allow_pickle=True)[0]
zout = utils.read_catalog('{0}.eazypy.zout.fits'.format(root))
cat = utils.read_catalog('{0}_phot_apcorr.fits'.format(root))
return self, cat, zout
trans = {
'f098m': 201,
'f105w': 202,
'f110w': 241,
'f125w': 203,
'f140w': 204,
'f160w': 205,
'f435w': 233,
'f475w': 234,
'f555w': 235,
'f606w': 236,
'f625w': 237,
'f775w': 238,
'f814w': 239,
'f850lp': 240,
'f702w': 15,
'f600lpu': 243,
'f225wu': 207,
'f275wu': 208,
'f336w': 209,
'f350lpu': 339,
'f438wu': 211,
'f475wu': 212,
'f475xu': 242,
'f555wu': 213,
'f606wu': 214,
'f625wu': 215,
'f775wu': 216,
'f814wu': 217,
'f390wu': 210,
'ch1': 18,
'ch2': 19
}
#trans.pop('f814w')
print('Apply catalog corrections')
apply_catalog_corrections(root, suffix='_apcorr')
cat = utils.read_catalog('{0}_phot_apcorr.fits'.format(root))
filters = []
for c in cat.meta:
if c.endswith('_ZP'):
filters.append(c.split('_ZP')[0].lower())
# Translate
fp = open('zphot.translate', 'w')
for f in filters:
if f in trans:
fp.write('{0}_tot_{1} F{2}\n'.format(f, aper_ix, trans[f]))
fp.write('{0}_etot_{1} E{2}\n'.format(f, aper_ix, trans[f]))
fp.write('irac_ch1_flux F18\n')
fp.write('irac_ch1_err E18\n')
fp.write('irac_ch2_flux F19\n')
fp.write('irac_ch2_err E19\n')
# For zeropoint
if dummy_prior:
fp.write('dummy_flux F205x\n')
fp.write('dummy_err E205x\n')
fp.close()
params = {}
params['CATALOG_FILE'] = '{0}_phot_apcorr.fits'.format(root)
params['Z_STEP'] = z_step
params['MAIN_OUTPUT_FILE'] = '{0}.eazypy'.format(root)
params['Z_MAX'] = z_max
params['MW_EBV'] = mastquery.utils.get_irsa_dust(cat['ra'].mean(),
cat['dec'].mean())
params['PRIOR_ABZP'] = 23.9
params['SYS_ERR'] = sys_err
params['CAT_HAS_EXTCORR'] = False
# Pick prior filter, starting from reddest
for f in [
'f435w', 'f606w', 'f814w', 'f105w', 'f110w', 'f125w', 'f140w',
'f160w'
][::-1]:
if f in filters:
if dummy_prior:
params['PRIOR_FILTER'] = 'dummy_flux'
else:
params['PRIOR_FILTER'] = trans[f]
mag = 23.9 - 2.5 * np.log10(cat['{0}_corr_{1}'.format(f, aper_ix)])
break
#
if os.path.exists(
'templates/fsps_full/tweak_fsps_QSF_11_v3_noRed.param.fits'):
params[
'TEMPLATES_FILE'] = 'templates/fsps_full/tweak_fsps_QSF_11_v3_noRed.param'
else:
params[
'TEMPLATES_FILE'] = 'templates/fsps_full/tweak_fsps_QSF_12_v3.param'
zpfile = None
load_products = False
if (not os.path.exists('FILTER.RES.latest')
or not os.path.exists('templates')):
try:
# should work with eazy-py >= 0.2.0
eazy.symlink_eazy_inputs(path=None)
except:
print("""
The filter file `FILTER.RES.latest` and `templates` directory were not
found in the working directory and the automatic command to retrieve them
failed:
>>> import eazy; eazy.symlink_eazy_inputs(path=None)
Run it with `path` pointing to the location of the `eazy-photoz` repository."""
)
return False
self = eazy.photoz.PhotoZ(param_file=None,
translate_file='zphot.translate',
zeropoint_file=zpfile,
params=params,
load_prior=True,
load_products=load_products)
if quiet:
self.param.params['VERBOSITY'] = 1.
if object_only:
return self
idx = np.arange(self.NOBJ)
#sample = (mag < 27) #& (self.cat['star_flag'] != 1)
#sample |= (self.cat['z_spec'] > 0)
sample = np.isfinite(self.cat['id']) #mag)
for iter in range(1 + (get_external_photometry & compute_residuals) * 1):
self.fit_parallel(idx[sample], n_proc=10)
if compute_residuals:
self.error_residuals()
self.standard_output(prior=apply_prior,
beta_prior=beta_prior,
extra_rf_filters=extra_rf_filters)
zout = utils.read_catalog('{0}.eazypy.zout.fits'.format(root))
np.save('{0}.eazypy.self.npy'.format(root), [self])
return self, cat, zout
def irac_mosaics(root='j000308m3303', home='/GrizliImaging/', pixfrac=0.2, kernel='square', initial_pix=1.0, final_pix=0.5, pulldown_mag=15.2, sync_xbcd=True, skip_fetch=False, radec=None, mosaic_pad=2.5, drizzle_ref_file='', run_alignment=True, assume_close=True, bucket='grizli-v1', aor_query='r*', mips_ext='[_e]bcd.fits', channels=['ch1','ch2','ch3','ch4','mips1'], drz_query='r*', sync_results=True, ref_seg=None, min_frame={'irac':5, 'mips':1.0}, med_max_size=500e6, stop_at='', make_psf=True, **kwargs):
"""
stop_at: preprocess, make_compact
"""
from grizli import utils
from . import irac
from .utils import get_wcslist, fetch_irac
PATH = os.path.join(home, root)
try:
os.mkdir(PATH)
except:
pass
os.chdir(PATH)
if not skip_fetch:
# Fetch IRAC bcds
if not os.path.exists(f'{root}_ipac.fits'):
os.system(f'wget https://s3.amazonaws.com/{bucket}/IRAC/{root}_ipac.fits')
res = fetch_irac(root=root, path='./', channels=channels)
if res in [False, None]:
# Nothing to do
make_html(root, bucket=bucket)
print(f'### Done: \n https://s3.amazonaws.com/{bucket}/Pipeline/{root}/IRAC/{root}.irac.html')
utils.log_comment(f'/tmp/{root}.success', 'Done!',
verbose=True, show_date=True)
return True
# Sync CHArGE HST images
os.system(f'aws s3 sync s3://{bucket}/Pipeline/{root}/Prep/ ./ '
f' --exclude "*" --include "{root}*seg.fits*"'
f' --include "{root}-ir_drz*fits*"'
f' --include "{root}*psf.fits*"'
f' --include "{root}-f[01]*_drz*fits.gz"'
f' --include "{root}*phot.fits"')
# Drizzle properties of the preliminary mosaic
#pixfrac, pix, kernel = 0.2, 1.0, 'square'
# Define an output WCS aligned in pixel phase to the HST mosaic ()
if not os.path.exists('ref_hdu.fits'):
wcslist = get_wcslist(skip=-500)
out_hdu = utils.make_maximal_wcs(wcslist, pixel_scale=initial_pix, theta=0, pad=5, get_hdu=True, verbose=True)
# Make sure pixels align
ref_file = glob.glob('{0}-f[01]*_drz_sci.fits*'.format(root))
if len(ref_file) == 0:
os.system(f'aws s3 sync s3://{bucket}/Pipeline/{root}/Prep/ ./ '
f' --exclude "*"'
f' --include "{root}-f[678]*_dr*fits.gz"')
ref_file = glob.glob('{0}-f[678]*_dr*_sci.fits*'.format(root))
ref_file = ref_file[-1]
print(f'\nHST reference image: {ref_file}\n')
ref_hdu = pyfits.open(ref_file)[0].header
ref_filter = utils.get_hst_filter(ref_hdu).lower()
ref_wcs = pywcs.WCS(ref_hdu)
ref_rd = ref_wcs.all_pix2world(np.array([[-0.5, -0.5]]), 0).flatten()
target_phase = np.array([0.5, 0.5])#/(pix/0.1)
for k in ['RADESYS', 'LATPOLE', 'LONPOLE']:
out_hdu.header[k] = ref_hdu[k]
# Shift CRVAL to same tangent point
out_wcs = pywcs.WCS(out_hdu.header)
out_xy = out_wcs.all_world2pix(np.array([ref_wcs.wcs.crval]), 1).flatten()
out_hdu.header['CRVAL1'], out_hdu.header['CRVAL2'] = tuple(ref_wcs.wcs.crval)
out_hdu.header['CRPIX1'], out_hdu.header['CRPIX2'] = tuple(out_xy)
# Align integer pixel phase
out_wcs = pywcs.WCS(out_hdu.header)
out_xy = out_wcs.all_world2pix(np.array([ref_rd]), 0).flatten()
xy_phase = out_xy - np.floor(out_xy)
new_crpix = out_wcs.wcs.crpix - (xy_phase - target_phase)
out_hdu.header['CRPIX1'], out_hdu.header['CRPIX2'] = tuple(new_crpix)
out_wcs = pywcs.WCS(out_hdu.header)
out_hdu.writeto('ref_hdu.fits', output_verify='Fix')
else:
out_hdu = pyfits.open('ref_hdu.fits')[1]
########
files = []
for ch in channels:
if 'mips' in ch:
mc = ch.replace('mips','ch')
files += glob.glob(f'{aor_query}/{mc}/bcd/SPITZER_M*{mips_ext}')
files += glob.glob(f'{aor_query}/{mc}/bcd/SPITZER_M*xbcd.fits.gz')
else:
files += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*cbcd.fits')
files += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*xbcd.fits.gz')
files.sort()
roots = np.array([file.split('/')[0] for file in files])
with_channels = np.array([file.split('_')[1] for file in files])
all_roots = np.array(['{0}-{1}'.format(r, c.replace('I','ch').replace('M', 'mips')) for r, c in zip(roots, with_channels)])
tab = {'aor':[], 'N':[], 'channel':[]}
for r in np.unique(all_roots):
tab['aor'].append(r.split('-')[0])
tab['N'].append((all_roots == r).sum())
tab['channel'].append(r.split('-')[1])
aors = utils.GTable(tab)
print(aors)
########
SKIP = True # Don't regenerate finished files
delete_group = False # Delete intermediate products from memory
zip_outputs = False # GZip intermediate products
aors_ch = {}
########
# Process mosaics by AOR
# Process in groups, helps for fields like HFF with dozens/hundreds of AORs!
for ch in channels:
aor = aors[(aors['channel'] == ch) & (aors['N'] > 5)]
if len(aor) == 0:
continue
#aors_ch[ch] = []
if ch in ['ch1','ch2']:
NPER, instrument = 500, 'irac'
if ch in ['ch3','ch4']:
NPER, instrument = 500, 'irac'
elif ch in ['mips1']:
NPER, instrument = 400, 'mips'
min_frametime = min_frame[instrument]
nsort = np.cumsum(aor['N']/NPER)
NGROUP = int(np.ceil(nsort.max()))
count = 0
for g in range(NGROUP):
root_i = root+'-{0:02d}'.format(g)
gsel = (nsort > g) & (nsort <= g+1)
aor_ids = list(aor['aor'][gsel])
print('{0}-{1} N_AOR = {2:>2d} N_EXP = {3:>4d}'.format(root_i, ch, len(aor_ids), aor['N'][gsel].sum()))
count += gsel.sum()
files = glob.glob('{0}-{1}*'.format(root_i, ch))
if (len(files) > 0) & (SKIP):
print('Skip {0}-{1}'.format(root_i, ch))
continue
with open('{0}-{1}.log'.format(root_i, ch),'w') as fp:
fp.write(time.ctime())
# Do internal alignment to GAIA.
# Otherwise, set `radec` to the name of a file that has two columns with
# reference ra/dec.
#radec = None
# Pipeline
if instrument == 'mips':
aors_ch[ch] = irac.process_all(channel=ch.replace('mips','ch'), output_root=root_i, driz_scale=initial_pix, kernel=kernel, pixfrac=pixfrac, wcslist=None, pad=0, out_hdu=out_hdu, aor_ids=aor_ids, flat_background=False, two_pass=True, min_frametime=min_frametime, instrument=instrument, align_threshold=0.15, radec=radec, run_alignment=False, mips_ext=mips_ext, ref_seg=ref_seg, global_mask=root+'_mask.reg')
else:
aors_ch[ch] = irac.process_all(channel=ch, output_root=root_i, driz_scale=initial_pix, kernel=kernel, pixfrac=pixfrac, wcslist=None, pad=0, out_hdu=out_hdu, aor_ids=aor_ids, flat_background=False, two_pass=True, min_frametime=min_frametime, instrument=instrument, radec=radec, run_alignment=run_alignment, assume_close=assume_close, ref_seg=ref_seg, global_mask=root+'_mask.reg', med_max_size=med_max_size)
if len(aors_ch[ch]) == 0:
continue
# PSFs
plt.ioff()
if (instrument != 'mips') & make_psf:
ch_num = int(ch[-1])
segmask=True
# psf_size=20
# for p in [0.1, final_pix]:
# irac.mosaic_psf(output_root=root_i, target_pix=p, channel=ch_num, aors=aors_ch[ch], kernel=kernel, pixfrac=pixfrac, size=psf_size, native_orientation=False, instrument=instrument, subtract_background=False, segmentation_mask=segmask, max_R=10)
# plt.close('all')
psf_size=30
p = 0.1
irac.mosaic_psf(output_root=root_i, target_pix=p, channel=ch_num, aors=aors_ch[ch], kernel=kernel, pixfrac=pixfrac, size=psf_size, native_orientation=True, subtract_background=False, segmentation_mask=segmask, max_R=10)
plt.close('all')
if delete_group:
del(aors_ch[ch])
print('Done {0}-{1}, gzip products'.format(root_i, ch))
if zip_outputs:
os.system('gzip {0}*-{1}_drz*fits'.format(root_i, ch))
# PSFs
if (instrument != 'mips') & make_psf:
# Average PSF
p = 0.1
files = glob.glob('*{0}-{1:.1f}*psfr.fits'.format(ch, p))
if len(files) == 0:
continue
files.sort()
avg = None
for file in files:
im = pyfits.open(file)
if avg is None:
wht = im[0].data != 0
avg = im[0].data*wht
else:
wht_i = im[0].data != 0
avg += im[0].data*wht_i
wht += wht_i
im.close()
avg = avg/wht
avg[wht == 0] = 0
# Window
from photutils import (HanningWindow, TukeyWindow,
CosineBellWindow,
SplitCosineBellWindow, TopHatWindow)
coswindow = CosineBellWindow(alpha=1)
avg *= coswindow(avg.shape)**0.05
avg /= avg.sum()
pyfits.writeto('{0}-{1}-{2:0.1f}.psfr_avg.fits'.format(root, ch, p), data=avg, header=im[0].header, overwrite=True)
####
## Show the initial product
plt.ioff()
for i in range(10):
files = glob.glob(f'{root}-{i:02d}-ch*sci.fits')
if len(files) > 0:
break
files.sort()
if len(files) == 1:
subs = 1,1
fs = [7,7]
elif len(files) == 2:
subs = 1,2
fs = [14,7]
elif len(files) == 3:
subs = 2,2
fs = [14,14]
else:
subs = 2,2
fs = [14,14]
fig = plt.figure(figsize=fs)
for i, file in enumerate(files[:4]):
im = pyfits.open(file)
print('{0} {1} {2:.1f} s'.format(file, im[0].header['FILTER'], im[0].header['EXPTIME']))
ax = fig.add_subplot(subs[0], subs[1], 1+i)
ax.imshow(im[0].data, vmin=-0.1, vmax=1, cmap='gray_r', origin='lower')
ax.text(0.05, 0.95, file, ha='left', va='top', color='k',
transform=ax.transAxes)
im.close()
if len(files) > 1:
fig.axes[1].set_yticklabels([])
if len(files) > 2:
fig.axes[0].set_xticklabels([])
fig.axes[1].set_xticklabels([])
if len(files) > 3:
fig.axes[3].set_yticklabels([])
fig.tight_layout(pad=0.5)
fig.savefig(f'{root}.init.png')
plt.close('all')
if stop_at == 'preprocess':
return True
#######
# Make more compact individual exposures and clean directories
wfiles = []
for ch in channels:
if 'mips' in ch:
chq = ch.replace('mips','ch')
wfiles += glob.glob(f'{aor_query}/{chq}/bcd/SPITZER_M*wcs.fits')
else:
wfiles += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*wcs.fits')
#wfiles = glob.glob('r*/*/bcd/*_I[1-4]_*wcs.fits')
#wfiles += glob.glob('r*/*/bcd/*_M[1-4]_*wcs.fits')
wfiles.sort()
for wcsfile in wfiles:
outfile = wcsfile.replace('_wcs.fits', '_xbcd.fits.gz')
if os.path.exists(outfile):
print(outfile)
else:
irac.combine_products(wcsfile)
print('Run: ', outfile)
if os.path.exists(outfile):
remove_files = glob.glob('{0}*fits'.format(wcsfile.split('_wcs')[0]))
for f in remove_files:
print(' rm ', f)
os.remove(f)
if stop_at == 'make_compact':
return True
#############
# Drizzle final mosaics
# Make final mosaic a bit bigger than the HST image
pad = mosaic_pad
# Pixel scale of final mosaic.
# Don't make too small if not many dithers available as in this example.
# But for well-sampled mosaics like RELICS / HFF, can push this to perhaps 0.3" / pix
pixscale = final_pix #0.5
# Again, if have many dithers maybe can use more aggressive drizzle parameters,
# like a 'point' kernel or smaller pixfrac (a 'point' kernel is pixfrac=0)
#kernel, pixfrac = 'square', 0.2
# Correction for bad columns near bright stars
#pulldown_mag = 15.2
##############
# Dilation for CR rejection
dil = np.ones((3,3))
driz_cr = [7, 4]
blot_interp = 'poly5'
bright_fmax = 0.5
### Drizzle
for ch in channels: #[:2]:
###########
# Files and reference image for extra CR rejection
if ch == 'mips1':
files = glob.glob('{0}/ch1/bcd/SPITZER_M1_*xbcd.fits*'.format(drz_query, ch))
files.sort()
pulldown_mag = -10
pixscale = 1.
kernel = 'point'
else:
files = glob.glob('{0}/{1}/bcd/*_I?_*xbcd.fits*'.format(drz_query, ch))
files.sort()
#ref = pyfits.open('{0}-00-{1}_drz_sci.fits'.format(root, ch))
#ref_data = ref[0].data.astype(np.float32)
ref_files = glob.glob(f'{root}-??-{ch}*sci.fits')
if len(ref_files) == 0:
continue
num = None
for ref_file in ref_files:
ref = pyfits.open(ref_file)
wht = pyfits.open(ref_file.replace('_sci.fits', '_wht.fits'))
if num is None:
num = ref[0].data*wht[0].data
den = wht[0].data
else:
num += ref[0].data*wht[0].data
den += wht[0].data
ref_data = (num/den).astype(np.float32)
ref_data[den <= 0] = 0
ref_wcs = pywcs.WCS(ref[0].header, relax=True)
ref_wcs.pscale = utils.get_wcs_pscale(ref_wcs)
if (not hasattr(ref_wcs, '_naxis1')) & hasattr(ref_wcs, '_naxis'):
ref_wcs._naxis1, ref_wcs._naxis2 = ref_wcs._naxis
##############
# Output WCS based on HST footprint
if drizzle_ref_file == '':
try:
hst_im = pyfits.open(glob.glob('{0}-f[01]*_drz_sci.fits*'.format(root))[-1])
except:
hst_im = pyfits.open(glob.glob('{0}-f[578]*_dr*sci.fits*'.format(root))[-1])
hst_wcs = pywcs.WCS(hst_im[0])
hst_wcs.pscale = utils.get_wcs_pscale(hst_wcs)
try:
size = (np.round(np.array([hst_wcs._naxis1, hst_wcs._naxis2])*hst_wcs.pscale*pad/pixscale)*pixscale)
except:
size = (np.round(np.array([hst_wcs._naxis[0], hst_wcs._naxis[1]])*hst_wcs.pscale*pad/pixscale)*pixscale)
hst_rd = hst_wcs.calc_footprint().mean(axis=0)
_x = utils.make_wcsheader(ra=hst_rd[0], dec=hst_rd[1],
size=size,
pixscale=pixscale,
get_hdu=False, theta=0)
out_header, out_wcs = _x
else:
driz_ref_im = pyfits.open(drizzle_ref_file)
out_wcs = pywcs.WCS(driz_ref_im[0].header, relax=True)
out_wcs.pscale = utils.get_wcs_pscale(out_wcs)
out_header = utils.to_header(out_wcs)
if (not hasattr(out_wcs, '_naxis1')) & hasattr(out_wcs, '_naxis'):
out_wcs._naxis1, out_wcs._naxis2 = out_wcs._naxis
##############
# Bright stars for pulldown correction
cat_file = glob.glob(f'{root}-[0-9][0-9]-{ch}.cat.fits')[0]
ph = utils.read_catalog(cat_file)
bright = (ph['mag_auto'] < pulldown_mag) # & (ph['flux_radius'] < 3)
ph = ph[bright]
##############
# Now do the drizzling
yp, xp = np.indices((256, 256))
orig_files = []
out_header['DRIZ_CR0'] = driz_cr[0]
out_header['DRIZ_CR1'] = driz_cr[1]
out_header['KERNEL'] = kernel
out_header['PIXFRAC'] = pixfrac
out_header['NDRIZIM'] = 0
out_header['EXPTIME'] = 0
out_header['BUNIT'] = 'microJy'
out_header['FILTER'] = ch
med_root = 'xxx'
N = len(files)
for i, file in enumerate(files):#[:100]):
print('{0}/{1} {2}'.format(i, N, file))
if file in orig_files:
continue
im = pyfits.open(file)
ivar = 1/im['CBUNC'].data**2
msk = (~np.isfinite(ivar)) | (~np.isfinite(im['CBCD'].data))
im['CBCD'].data[msk] = 0
ivar[msk] = 0
wcs = pywcs.WCS(im['WCS'].header, relax=True)
wcs.pscale = utils.get_wcs_pscale(wcs)
if (not hasattr(wcs, '_naxis1')) & hasattr(wcs, '_naxis'):
wcs._naxis1, wcs._naxis2 = wcs._naxis
fp = Path(wcs.calc_footprint())
med_root_i = im.filename().split('/')[0]
if med_root != med_root_i:
print('\n Read {0}-{1}_med.fits \n'.format(med_root_i, ch))
med = pyfits.open('{0}-{1}_med.fits'.format(med_root_i, ch))
med_data = med[0].data.astype(np.float32)
med_root = med_root_i
med.close()
try:
gaia_rd = utils.read_catalog('{0}-{1}_gaia.radec'.format(med_root_i, ch))
ii, rr = gaia_rd.match_to_catalog_sky(ph)
gaia_rd = gaia_rd[ii][rr.value < 2]
gaia_pts = np.array([gaia_rd['ra'].data,
gaia_rd['dec'].data]).T
except:
gaia_rd = []
#data = im['CBCD'].data - aor_med[0].data
# Change output units to uJy / pix
if ch == 'mips1':
# un = 1*u.MJy/u.sr
# #to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(out_wcs.pscale**2)
# to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(native_scale**2)
to_ujy_px = 146.902690
else:
# native_scale = 1.223
# un = 1*u.MJy/u.sr
# #to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(out_wcs.pscale**2)
# to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(native_scale**2)
to_ujy_px = 35.17517196810
blot_data = ablot.do_blot(ref_data, ref_wcs, wcs, 1, coeffs=True,
interp=blot_interp,
sinscl=1.0, stepsize=10,
wcsmap=None)/to_ujy_px
# mask for bright stars
eblot = 1-np.clip(blot_data, 0, bright_fmax)/bright_fmax
# Initial CR
clean = im[0].data - med_data - im['WCS'].header['PEDESTAL']
dq = (clean - blot_data)*np.sqrt(ivar)*eblot > driz_cr[0]
# Adjacent CRs
dq_dil = binary_dilation(dq, selem=dil)
dq |= ((clean - blot_data)*np.sqrt(ivar)*eblot > driz_cr[1]) & (dq_dil)
# Very negative pixels
dq |= clean*np.sqrt(ivar) < -4
original_dq = im['WCS'].data - (im['WCS'].data & 1)
dq |= original_dq > 0
# Pulldown correction for bright stars
if len(gaia_rd) > 0:
mat = fp.contains_points(gaia_pts)
if mat.sum() > 0:
xg, yg = wcs.all_world2pix(gaia_rd['ra'][mat], gaia_rd['dec'][mat], 0)
sh = dq.shape
mat = (xg > 0) & (xg < sh[1]) & (yg > 0) & (yg < sh[0])
if mat.sum() > 0:
for xi, yi in zip(xg[mat], yg[mat]):
dq |= (np.abs(xp-xi) < 2) & (np.abs(yp-yi) > 10)
if i == 0:
res = utils.drizzle_array_groups([clean], [ivar*(dq == 0)], [wcs], outputwcs=out_wcs, kernel=kernel, pixfrac=pixfrac, data=None, verbose=False)
# Copy header keywords
wcs_header = utils.to_header(wcs)
for k in im[0].header:
if (k not in ['', 'HISTORY', 'COMMENT']) & (k not in out_header) & (k not in wcs_header):
out_header[k] = im[0].header[k]
else:
_ = utils.drizzle_array_groups([clean], [ivar*(dq == 0)], [wcs], outputwcs=out_wcs, kernel=kernel, pixfrac=pixfrac, data=res[:3], verbose=False)
out_header['NDRIZIM'] += 1
out_header['EXPTIME'] += im[0].header['EXPTIME']
im.close()
# Pixel scale factor for weights
wht_scale = (out_wcs.pscale/wcs.pscale)**-4
# Write final images
pyfits.writeto('{0}-{1}_drz_sci.fits'.format(root, ch), data=res[0]*to_ujy_px, header=out_header,
output_verify='fix', overwrite=True)
pyfits.writeto('{0}-{1}_drz_wht.fits'.format(root, ch), data=res[1]*wht_scale/to_ujy_px**2,
header=out_header, output_verify='fix', overwrite=True)
##########
## Show the final drizzled images
plt.ioff()
files = glob.glob(f'{root}-ch*sci.fits')
files.sort()
if len(files) == 1:
subs = 1,1
fs = [7,7]
elif len(files) == 2:
subs = 1,2
fs = [14,7]
elif len(files) == 3:
subs = 2,2
fs = [14,14]
else:
subs = 2,2
fs = [14,14]
fig = plt.figure(figsize=fs)
for i, file in enumerate(files[:4]):
im = pyfits.open(file)
print('{0} {1} {2:.1f} s'.format(file, im[0].header['FILTER'], im[0].header['EXPTIME']))
ax = fig.add_subplot(subs[0], subs[1], 1+i)
scl = (final_pix/initial_pix)**2
ax.imshow(im[0].data, vmin=-0.1*scl, vmax=1*scl, cmap='gray_r', origin='lower')
ax.text(0.05, 0.95, file, ha='left', va='top', color='k',
transform=ax.transAxes)
im.close()
if len(files) > 1:
fig.axes[1].set_yticklabels([])
if len(files) > 2:
fig.axes[0].set_xticklabels([])
fig.axes[1].set_xticklabels([])
if len(files) > 3:
fig.axes[3].set_yticklabels([])
fig.tight_layout(pad=0.5)
fig.savefig(f'{root}.final.png')
plt.close('all')
if sync_results:
print('gzip mosaics')
os.system(f'gzip -f {root}-ch*_drz*fits {root}-mips*_drz*fits')
######## Sync
## Sync
print(f's3://{bucket}/Pipeline/{root}/IRAC/')
make_html(root, bucket=bucket)
os.system(f'aws s3 sync ./ s3://{bucket}/Pipeline/{root}/IRAC/'
f' --exclude "*" --include "{root}-ch*drz*fits*"'
f' --include "{root}-mips*drz*fits*"'
f' --include "{root}.*png"'
' --include "*-ch*psf*" --include "*log.fits"'
' --include "*wcs.[lp]*"'
' --include "*html" --include "*fail*"'
' --acl public-read')
if sync_xbcd:
aor_files = glob.glob('r*-ch*med.fits')
for aor_file in aor_files:
aor = aor_file.split('-ch')[0]
os.system(f'aws s3 sync ./{aor}/ s3://{bucket}/IRAC/AORS/{aor}/ --exclude "*" --include "ch*/bcd/*xbcd.fits.gz" --acl public-read')
os.system(f'aws s3 cp {aor_file} s3://{bucket}/IRAC/AORS/ --acl public-read')
msg = f'### Done: \n https://s3.amazonaws.com/{bucket}/Pipeline/{root}/IRAC/{root}.irac.html'
utils.log_comment(f'/tmp/{root}.success', msg, verbose=True, show_date=True)
def auto_run(root='j023507-040202', flag_global_crs=False):
import os
import glob
import numpy as np
import astropy.io.fits as pyfits
import astropy.wcs as pywcs
from drizzlepac import updatehdr
from stwcs import updatewcs
from grizli import utils, prep
from grizli.pipeline import auto_script
utils.set_warnings()
visit_file = '{0}_visits.npy'.format(root)
visits, all_groups, info = np.load(visit_file)
# Something wrong with some files with bad shifts, reset wcs
for visit in visits:
for file in visit['files']:
utils.fetch_hst_calibs(
file, calib_types=['IDCTAB', 'NPOLFILE', 'IMPHTTAB'])
updatewcs.updatewcs(file, verbose=True, use_db=False)
# Apply shifts
shift_log = '{0}_shifts.log'.format(visit['product'])
if os.path.exists(shift_log):
sh = utils.read_catalog(shift_log)
flt0 = pyfits.open(sh['flt'][0])
wcs_ref = pywcs.WCS(flt0['SCI', 1].header, fobj=flt0, relax=True)
shift_dict = {}
for i in range(len(sh)):
shift_dict[sh['flt'][i]] = [sh['xshift'][i], sh['yshift'][i]]
prep.apply_tweak_shifts(wcs_ref,
shift_dict,
grism_matches={},
verbose=False)
# Redrizzle mosaics
prep.drizzle_overlaps(visits,
check_overlaps=False,
skysub=False,
static=False,
pixfrac=0.5,
scale=None,
final_wcs=False,
fetch_flats=False,
final_rot=None,
include_saturated=True)
####### Alignment
os.system('rm *wcs.*')
# Radec
master_radec = '{0}/../../{1}_master.radec'.format(os.getcwd(), root)
if not os.path.exists(master_radec):
master_radec = None
ref_catalog = 'USER'
if root.startswith('cos-'):
hsc = '{0}/../../{1}'.format(os.getcwd(),
'hsc-udeep-i25_corr_cosmos.radec')
if os.path.exists(hsc):
master_radec = hsc
ref_catalog = 'HSC'
elif root.startswith('uds-'):
hsc = '{0}/../../{1}'.format(os.getcwd(),
'hsc-udeep-sxds_corr_uds.radec')
if os.path.exists(hsc):
master_radec = hsc
ref_catalog = 'HSC'
parent_radec = '{0}/../../{1}_parent.radec'.format(os.getcwd(), root)
if not os.path.exists(parent_radec):
parent_radec = None
if master_radec is not None:
radec = master_radec
elif parent_radec is not None:
radec = parent_radec
else:
radec = None
if radec is None:
needs_gaia = True
else:
needs_gaia = False
REFERENCE = 'GAIA'
REFERENCE = 'PS1'
print('master RADEC file: ', radec)
thresh = 2.5
for visit in visits:
# Clean catalogs
files = glob.glob('{0}.*'.format(visit['product']))
for file in files:
os.remove(file)
# Generate GAIA alignment catalog at the observation epoch
clip = 120
clip = -1
if needs_gaia:
flt = pyfits.open(visit['files'][0])
h = flt['SCI', 1].header
ra_i, dec_i = h['CRVAL1'], h['CRVAL2']
radec, ref_catalog = prep.get_radec_catalog(
ra=ra_i,
dec=dec_i,
product=visit['product'],
date=flt[0].header['EXPSTART'],
date_format='mjd',
reference_catalogs=[REFERENCE],
radius=5.)
flt.close()
if REFERENCE == 'GAIA':
mag_limits = [16, 20]
else:
mag_limits = [18, 22]
#clip = 50
if '_flc' in visit['files'][0]:
triangle_size_limit = [5, 4000 * np.sqrt(2)]
else:
triangle_size_limit = [5, 1300]
else:
mag_limits = [19, 23]
triangle_size_limit = [5, 1300]
# Remake catalogs
cat = prep.make_SEP_catalog(root=visit['product'], threshold=thresh)
# Redo alignment
try:
print('XXX clip', clip, mag_limits, triangle_size_limit)
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=mag_limits,
simple=False,
max_err_percentile=80,
clip=clip,
outlier_threshold=5,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
except:
print('First align failed! Relax parameters')
try:
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=[10, 20],
simple=False,
max_err_percentile=99,
clip=160,
outlier_threshold=20,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
except:
try:
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=[10, 20],
simple=False,
max_err_percentile=99,
clip=160,
outlier_threshold=40,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
except:
radec = '{0}_ps1.radec'.format(visit['product'])
ref_catalog = 'PS1'
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=mag_limits,
simple=False,
max_err_percentile=80,
clip=120,
outlier_threshold=5,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
#continue
orig_wcs, drz_wcs, out_shift, out_rot, out_scale = result
# Propagate shifts
for file in visit['files']:
updatehdr.updatewcs_with_shift(file,
str('{0}_wcs.fits'.format(
visit['product'])),
xsh=out_shift[0],
ysh=out_shift[1],
rot=out_rot,
scale=out_scale,
wcsname=ref_catalog,
force=True,
reusename=True,
verbose=True,
sciext='SCI')
### Bug in astrodrizzle? Dies if the FLT files don't have MJD-OBS
### keywords
im = pyfits.open(file, mode='update')
im[0].header['MJD-OBS'] = im[0].header['EXPSTART']
im.flush()
# Redrizzle mosaics again including new shifts
prep.drizzle_overlaps(visits,
check_overlaps=False,
skysub=False,
static=False,
pixfrac=0.8,
scale=None,
final_wcs=False,
fetch_flats=False,
final_rot=None)
# Remake catalogs
thresh = 2.5
for visit in visits:
# Remake catalogs
cat = prep.make_SEP_catalog(root=visit['product'], threshold=thresh)
prep.table_to_regions(cat, '{0}.cat.reg'.format(visit['product']))
prep.table_to_radec(cat, '{0}.cat.radec'.format(visit['product']))
# Update visits file
v = auto_script.get_visit_exposure_footprints(visit_file=visit_file,
check_paths=['./', '../RAW'],
simplify=1.e-6)
if flag_global_crs:
# Assume everything at same orient
pass
if False:
# Mosaic
auto_script.drizzle_overlaps(root,
filters=['F160W'],
min_nexp=1,
pixfrac=0.8,
scale=0.1,
make_combined=False,
ref_image=None,
static=False)
def apply_catalog_corrections(root, total_flux='flux_auto', auto_corr=True, get_external_photometry=False, aperture_indices='all', suffix='_apcorr', verbose=True, apply_background=True):
"""
Aperture and background corrections to photometric catalog
"""
import os
import eazy
import numpy as np
from grizli import utils
import mastquery.utils
cat = utils.read_catalog('{0}_phot.fits'.format(root))
filters = []
for c in cat.meta:
if c.endswith('_ZP'):
filters.append(c.split('_ZP')[0].lower())
if get_external_photometry:
print('Get external photometry from Vizier')
try:
ext = get_external_catalog(cat, external_limits=external_limits,
timeout=external_timeout,
sys_err=external_sys_err)
for c in ext.colnames:
if c not in cat.colnames:
cat[c] = ext[c]
for k in ext.meta:
cat.meta[k] = ext.meta[k]
except:
print(' - External catalog FAILED')
pass
# Fix: Take flux_auto when flag==0, flux otherwise
if (total_flux == 'flux_auto_fix') & (total_flux not in cat.colnames):
flux = cat['flux_auto']*1.
flagged = (cat['flag'] > 0)
flux[flagged] = cat['flux'][flagged]
cat['flux_auto_fix'] = flux*1.
# Additional auto correction
cat.meta['TOTALCOL'] = total_flux, 'Column for total flux'
#cat.meta['HASTOT'] = (auto_corr & ('tot_corr' in cat.colnames), 'Catalog has full total flux')
apcorr = {}
for NAPER in range(100):
if 'APER_{0}'.format(NAPER) not in cat.meta:
break
if aperture_indices == 'all':
aperture_indices = range(NAPER)
for i in aperture_indices:
if verbose:
print('Compute aperture corrections: i={0}, D={1:.2f}" aperture'.format(i, cat.meta['ASEC_{0}'.format(i)]))
if 'flux_aper_{0}'.format(i) in cat.colnames:
cat['apcorr_{0}'.format(i)] = cat[total_flux]/cat['flux_aper_{0}'.format(i)]
for f in filters:
bkgc = '{0}_bkg_aper_{1}'.format(f, i)
if (bkgc in cat.colnames) & apply_background:
bkg = cat[bkgc]
else:
bkg = 0.
cat['{0}_corr_{1}'.format(f, i)] = (cat['{0}_flux_aper_{1}'.format(f, i)]-bkg)*cat['apcorr_{0}'.format(i)]
cat['{0}_ecorr_{1}'.format(f, i)] = cat['{0}_fluxerr_aper_{1}'.format(f, i)]*cat['apcorr_{0}'.format(i)]
# mask_thresh = np.percentile(cat['{0}_mask_aper_{1}'.format(f, i)], 95)
aper_area = np.pi*(cat.meta['APER_{0}'.format(i)]/2)**2
mask_thresh = aper_area
bad = cat['{0}_mask_aper_{1}'.format(f, i)] > 0.2*mask_thresh
cat['{0}_corr_{1}'.format(f, i)][bad] = -99
cat['{0}_ecorr_{1}'.format(f, i)][bad] = -99
tot_col = '{0}_tot_corr'.format(f.lower())
if auto_corr and (tot_col in cat.colnames):
cat['{0}_tot_{1}'.format(f, i)] = cat['{0}_corr_{1}'.format(f, i)]*cat[tot_col]
cat['{0}_etot_{1}'.format(f, i)] = cat['{0}_ecorr_{1}'.format(f, i)]*cat[tot_col]
cat['{0}_tot_{1}'.format(f, i)][bad] = -99
cat['{0}_etot_{1}'.format(f, i)][bad] = -99
cat.rename_column('number','id')
cat['z_spec'] = cat['id']*0.-1
# Spurious sources, sklearn SVM model trained for a single field
morph_model = os.path.join(os.path.dirname(utils.__file__),
'data/sep_catalog_junk.pkl')
if os.path.exists(morph_model):
if verbose:
print('Apply morphological validity class')
from sklearn.externals import joblib
clf = joblib.load(morph_model)
X = np.hstack([[cat['peak']/cat['flux'],
cat['cpeak']/cat['peak']]]).T
# Predict labels, which where generated for
# bad_bright, bad_faint, stars, big_galaxies, small_galaxies
pred = clf.predict_proba(X)
# Should be >~ 0.9 for valid sources, stars & galaxies in "ir" image
cat['class_valid'] = pred[:,-3:].sum(axis=1)
cat['class_valid'].format = '.2f'
cat['dummy_err'] = 10**(-0.4*(8-23.9))
cat['dummy_flux'] = cat[total_flux] # detection band
if suffix:
if verbose:
print('Write {0}_phot{1}.fits'.format(root, suffix))
cat.write('{0}_phot{1}.fits'.format(root, suffix), overwrite=True)
return cat
def extract_beams_from_flt(root, bucket, id, clean=True, silent=False):
"""
Download GrismFLT files and extract the beams file
"""
import gc
import boto3
import matplotlib.pyplot as plt
import grizli
from grizli import fitting, utils, multifit
from grizli.version import __version__ as grizli__version
utils.set_warnings()
from grizli.pipeline import auto_script
s3 = boto3.resource('s3')
s3_client = boto3.client('s3')
bkt = s3.Bucket(bucket)
# WCS files for ACS
files = [obj.key for obj in bkt.objects.filter(Prefix='Pipeline/{0}/Extractions/j'.format(root))]
files += [obj.key for obj in bkt.objects.filter(Prefix='Pipeline/{0}/Extractions/i'.format(root))]
files += [obj.key for obj in bkt.objects.filter(Prefix='Pipeline/{0}/Extractions/{0}-ir.cat.fits'.format(root))]
files += [obj.key for obj in bkt.objects.filter(Prefix='Pipeline/{0}/Extractions/fit_args.npy'.format(root))]
download_files = []
for file in np.unique(files):
if ('cat.fits' in file) | ('fit_args' in file):
if os.path.exists(os.path.basename(file)):
continue
download_files.append(file)
for file in download_files:
print(file)
bkt.download_file(file, os.path.basename(file),
ExtraArgs={"RequestPayer": "requester"})
# Read the catalog
ircat = utils.read_catalog('{0}-ir.cat.fits'.format(root))
ix = ircat['NUMBER'] == id
object_rd = (ircat['X_WORLD'][ix], ircat['Y_WORLD'][ix])
del(ircat)
# One beam at a time
beams = None
flt_files = []
for file in files:
if 'GrismFLT.fits' in file:
flt_files.append(file)
if not silent:
print('Read {0} GrismFLT files'.format(len(flt_files)))
if os.path.exists('{0}_fit_args.npy'.format(root)):
args_file = '{0}_fit_args.npy'.format(root)
else:
args_file = 'fit_args.npy'
for i, file in enumerate(flt_files):
if not silent:
print('# Read {0}/{1}'.format(i+1, len(flt_files)))
flt, ext, _, _ = os.path.basename(file).split('.')
if flt.startswith('i'):
fl = 'flt'
else:
fl = 'flc'
out_files = ['{0}_{2}.{1}.wcs.fits'.format(flt, ext, fl),
'{0}.{1}.GrismFLT.fits'.format(flt, ext),
'{0}.{1}.GrismFLT.pkl'.format(flt, ext)]
exp_has_id = False
for j, f_j in enumerate(out_files):
aws_file = os.path.join(os.path.dirname(file), f_j)
if not silent:
print(' ', aws_file)
if not os.path.exists(f_j):
bkt.download_file(aws_file, f_j,
ExtraArgs={"RequestPayer": "requester"})
# WCS file, check if object in footprint
if f_j.endswith('.wcs.fits'):
#exp_has_id = check_object_in_footprint(id, f_j, ircat)
exp_has_id = check_object_in_footprint(None, f_j, None, rd=object_rd)
if not exp_has_id:
if clean:
os.remove(f_j)
break
if not exp_has_id:
continue
beams_i = auto_script.extract(field_root=root, maglim=[13,24], prior=None, MW_EBV=0.00, ids=id, pline={}, fit_only_beams=True, run_fit=False, poly_order=7, master_files=[os.path.basename(file)], grp=None, bad_pa_threshold=None, fit_trace_shift=False, size=32, diff=True, min_sens=0.02, skip_complete=True, fit_args={}, args_file=args_file, get_only_beams=True)
# Remove the GrismFLT file
for f_j in out_files:
if ('GrismFLT' in f_j) & clean:
os.remove(f_j)
if beams is None:
beams = beams_i
else:
beams.extend(beams_i)
# Garbage collector
gc.collect()
if not beams:
print('No beams found for {0} id={1}'.format(root, id))
return False
# Grism Object
args = np.load(args_file, allow_pickle=True)[0]
mb = multifit.MultiBeam(beams, **args)
mb.write_master_fits()
# 1D spectrum with R=30 fit
if True:
bin_steps, step_templ = utils.step_templates(wlim=[5000, 18000.0],
R=30, round=10)
tfit = mb.template_at_z(z=0, templates=step_templ,
fit_background=True, fitter='lstsq',
get_uncertainties=2)
fig1 = mb.oned_figure(figsize=[5,3], tfit=tfit, show_beams=True,
scale_on_stacked=True, ylim_percentile=5)
outroot='{0}_{1:05d}.R{2:.0f}'.format(root, id, 30)
hdu = mb.oned_spectrum_to_hdu(outputfile=outroot+'.fits',
tfit=tfit, wave=bin_steps)
fig1.savefig(outroot+'.png')
del(hdu)
# Drizzled spectrum
hdu, fig = mb.drizzle_grisms_and_PAs(fcontam=args['fcontam'],
flambda=False,
kernel='point', size=32,
zfit=tfit, diff=False)
hdu[0].header['GRIZLIV'] = (grizli__version, 'Grizli version')
fig.savefig('{0}_{1:05d}.stack.png'.format(root, id))
hdu.writeto('{0}_{1:05d}.stack.fits'.format(root, id),
overwrite=True)
plt.close('all')
del(hdu)
outfiles = ['{0}_{1:05d}.beams.fits'.format(root, id)]
outfiles += glob.glob(outroot+'*')
outfiles += glob.glob('{0}_{1:05d}.stack*'.format(root, id))
return(outfiles)
def run_root(root='j002532m1223', min_zoom=2, get_grism=True):
"""
Prepare images for fitsmap.convert
"""
from grizli.pipeline import auto_script
from grizli import utils
import eazy.utils
from fitsmap import convert
print('sync')
os.system(
f'aws s3 sync s3://grizli-v1/Pipeline/{root}/Prep/ {root}/ --exclude "*" --include "*sci.fits.gz" --include "*phot.fits" --include "*seg.fits.gz"'
)
os.system(
f'aws s3 sync s3://grizli-v1/Pipeline/{root}/IRAC/ {root}/ --exclude "*" --include "*sci.fits*" --include "*model.fits"'
)
os.system(
f'aws s3 sync s3://grizli-v1/Pipeline/{root}/Map/ {root}/ --exclude "*" --include "{root}.*png"'
)
os.chdir(root)
if not os.path.exists(f'{root}.rgb.png'):
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=True,
HOME_PATH=None,
gzext='*',
suffix='.rgb',
output_format='png')
# IR
files = glob.glob(f'{root}-[if][r01]*sci.fits*')
files.sort()
filts = [file.split(f'{root}-')[1].split('_')[0] for file in files]
for filt in filts:
if os.path.exists(f'{root}.{filt}.png'):
continue
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=True,
HOME_PATH=None,
gzext='*',
filters=[filt],
suffix=f'.{filt}',
output_format='png',
invert=True,
scl=2)
# Optical, 2X pix
files = glob.glob(f'{root}-[f][2-8]*sci.fits*')
files.sort()
filts = [file.split(f'{root}-')[1].split('_')[0] for file in files]
for filt in filts:
if os.path.exists(f'{root}.{filt}.png'):
continue
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=2,
HOME_PATH=None,
gzext='*',
filters=[filt],
suffix=f'.{filt}',
output_format='png',
invert=True,
scl=2)
# Spitzer
if glob.glob(f'{root}-ch*fits*'):
import reproject
out_img = pyfits.open(f'{root}-ir_drz_sci.fits.gz')
repr_hdu = out_img[0]
# repr_hdu = utils.make_maximal_wcs([out_wcs], pixel_scale=0.2,
# verbose=False, pad=0, poly_buffer=0)
repr_wcs = pywcs.WCS(repr_hdu.header)
mosaics = glob.glob(f'{root}-ch[12]*sci.fits*')
mosaics.sort()
for mos in mosaics:
ch = mos.split(f'{root}-')[1].split('_')[0]
if os.path.exists(f'{root}.{ch}.png'):
continue
print(f'Reproject {ch}')
in_img = pyfits.open(mos)
in_wcs = pywcs.WCS(in_img[0].header)
reproj = utils.blot_nearest_exact(in_img[0].data,
in_wcs,
repr_wcs,
scale_by_pixel_area=False)
pyfits.writeto(f'{root}-{ch}s_drz_sci.fits',
data=reproj,
header=repr_hdu.header,
overwrite=True)
ext = [ch + 's']
if os.path.exists(f'{root}-{ch}_model.fits'):
# resid
print(f' {ch} model')
in_img = pyfits.open(f'{root}-{ch}_model.fits')
reproj = utils.blot_nearest_exact(in_img[1].data,
in_wcs,
repr_wcs,
scale_by_pixel_area=False)
pyfits.writeto(f'{root}-{ch}m_drz_sci.fits',
data=reproj,
header=repr_hdu.header,
overwrite=True)
ext.append(ch + 'm')
for filt in ext:
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=True,
HOME_PATH=None,
gzext='',
filters=[filt],
suffix=f'.{filt}',
output_format='png',
invert=True,
scl=2)
if not os.path.exists(f'{root}.seg.png'):
sfig = make_seg(f'{root}-ir_seg.fits.gz', outfile=f'{root}.seg.png')
filelist = []
for q in ['*f[2-8]', '*f[01]*', '*ir*', '*ch[12]', '*seg', '*rgb']:
l_i = glob.glob(q + '*png')
l_i.sort()
filelist.extend(l_i)
ph = utils.read_catalog(f'{root}_phot.fits')
ph['id'] = ph['number']
ph['ra'].format = '.6f'
ph['dec'].format = '.6f'
ph['mag'] = ph['mag_auto']
ph['mag'].format = '.2f'
ph['query'] = [
eazy.utils.query_html(r, d).split(') ')[1]
for r, d in zip(ph['ra'], ph['dec'])
]
ph['id', 'ra', 'dec', 'query', 'mag'].write('phot.cat',
format='ascii.csv',
overwrite=True)
filelist += ['phot.cat']
if get_grism:
from grizli.aws import db
engine = db.get_db_engine()
gr = db.from_sql(
f"select root, id, ra, dec, z_map from redshift_fit where root='{root}'",
engine)
print(f'grism.cat: {len(gr)} sources')
if len(gr) > 0:
gr['query'] = [
eazy.utils.query_html(r, d).split(') ')[1]
for r, d in zip(gr['ra'], gr['dec'])
]
gr['stack'] = [
f'<img src="https://s3.amazonaws.com/grizli-v1/Pipeline/{root}/Extractions/{root}_{id:05d}.stack.png" height="100px"/>'
for id in gr['id']
]
gr['full'] = [
f'<img src="https://s3.amazonaws.com/grizli-v1/Pipeline/{root}/Extractions/{root}_{id:05d}.full.png" height="100px"/>'
for id in gr['id']
]
gr['line'] = [
f'<img src="https://s3.amazonaws.com/grizli-v1/Pipeline/{root}/Extractions/{root}_{id:05d}.line.png" height="80px"/>'
for id in gr['id']
]
gr['ra'].format = '.6f'
gr['dec'].format = '.6f'
gr['z_map'].format = '.4f'
gr['id', 'ra', 'dec', 'query', 'z_map', 'stack', 'full',
'line'].write('grism.cat', format='ascii.csv', overwrite=True)
filelist += ['grism.cat']
convert.MPL_CMAP = 'gray_r'
convert.cartographer.MARKER_HTML_WIDTH = '650px'
convert.cartographer.MARKER_HTML_HEIGHT = '440px'
convert.POPUP_CSS = [
"span { text-decoration:underline; font-weight:bold; line-height:12pt; }",
"tr { line-height: 7pt; }",
"table { width: 100%; }",
"img { height: 100px; width: auto; }",
]
convert.dir_to_map("./",
filelist=filelist,
out_dir="output",
cat_wcs_fits_file=f"{root}-ir_drz_sci.fits.gz",
catalog_delim=',',
min_zoom=min_zoom,
task_procs=False,
image_engine='MPL')
plt.close('all')
if os.path.exists('output/index.html'):
os.system(
f'aws s3 sync output/ s3://grizli-v1/Pipeline/{root}/Map/ --acl public-read --quiet'
)
os.system(
f'aws s3 sync ./ s3://grizli-v1/Pipeline/{root}/Map/ --exclude "*" --include "{root}.*png" --acl public-read'
)
def fit_quasars_lambda(root='j100025+021706', newfunc=True):
import time
import os
import numpy as np
import boto3
import json
from astropy import table
from grizli import utils
s3 = boto3.resource('s3')
bkt = s3.Bucket('aws-grivam')
#bkt.download_file('/Pipeline/wisps-aug10.fits', 'wisps-aug10', ExtraArgs={"RequestPayer": "requester"})
fit = utils.read_catalog('wisps-aug10.fits')
fit = table.vstack([
utils.read_catalog('grizli-3dhst-18.08.05.fits'),
utils.read_catalog('grizli-18.05.17-full.fits')
])
# All
fit = table.vstack([
utils.read_catalog('wisps-aug10.fits'),
utils.read_catalog('grizli-3dhst-18.08.05.fits'),
utils.read_catalog('grizli-18.05.17-full.fits')
])
fit = utils.read_catalog('grizli-3dhst-18.08.05.fits')
fit = utils.read_catalog('grizli-18.05.17-full.fits')
roots = np.unique(fit['root'])
pa = np.polyfit([16, 21, 24, 25.5], [4.5, 2.6, 2.3, 2.1], 2)
py = np.polyval(pa, fit['mag_auto'])
py = np.interp(fit['mag_auto'], [16, 18, 19, 21, 24, 25.5],
[7, 7, 5, 2.6, 2.3, 2.1])
point_source = (fit['flux_radius'] < py) #& (fit['mag_auto'] < 23)
fit['is_point'] = point_source * 2 - 1
bad = (fit['mag_auto'] > 22) & (fit['flux_radius'] < 1.2)
fit['too_small'] = bad * 2 - 1
stars = (fit['is_point'] > 0) & (fit['too_small'] < 0) & (fit['mag_auto'] <
24.5)
stars &= fit['ninput'] < 40
beams = [
'Pipeline/{0}/Extractions/{0}_{1:05d}.beams.fits'.format(root, id)
for root, id in zip(fit['root'][stars], fit['id'][stars])
]
ninput = fit['ninput'][stars]
# Auth to create a Lambda function (credentials are picked up from above .aws/credentials)
session = boto3.Session()
# Make sure Lambda is running in the same region as the HST public dataset
client = session.client('lambda', region_name='us-east-1')
func = 'GrizliFitQuasar'
print('Lambda function: {0}'.format(func))
# Auth to create a Lambda function
session = boto3.Session()
client = session.client('lambda', region_name='us-east-1')
done_files = [
os.path.basename(obj.key)
for obj in bkt.objects.filter(Prefix='Pipeline/QuasarFit/')
]
count = 0
for beam, ninp in zip(beams, ninput):
# if count > 400:
# break
if os.path.basename(beam).replace('.beams', '.full') in done_files:
#print('{0} {1} {2}'.format(count, 'Skip', beam))
continue
else:
count += 1
print('{0} {1} {2} ({3})'.format(count, ' ', beam, ninp))
event = {
's3_object_path': beam,
'verbose': "True",
"use_psf": str(ninp < 15),
}
# Invoke Lambda function
response = client.invoke(FunctionName=func,
InvocationType='Event',
LogType='Tail',
Payload=json.dumps(event))
files = glob.glob('*full.fits')
roots = np.unique([file.split('_')[0] for file in files])
for root in roots:
if os.path.exists('{0}.info.fits'.format(root)):
continue
try:
auto_script.summary_catalog(field_root=root,
dzbin=0.01,
use_localhost=False,
filter_bandpasses=None)
except:
pass
#
#fit = utils.read_catalog('wisps-aug10.fits')
from astropy import table
files = glob.glob('*info.fits')
roots = np.unique([file.split('.info')[0] for file in files])
tabs = []
for root in roots:
print(root)
file = '{0}.info.fits'.format(root)
if os.path.exists(file):
tabs.append(utils.read_catalog(file))
tab = table.vstack(tabs)
tab['bic_diff_spl'] = tab['bic_spl'] - tab['bic_temp']
tab['bic_diff_spl'].format = '.1f'
tab['chinusp'] = tab['chi2spl'] / tab['dof']
tab['chinusp'].format = '.1f'
tab['star14'] = tab['splf03'] / np.maximum(tab['splf04'], tab['sple04'])
tab['star14'].format = '.1f'
tab['star14e'] = tab['splf03'] / np.sqrt(tab['sple03']**2 +
tab['sple04']**2)
tab['star14e'].format = '.2f'
tab['spl14'] = tab['splf03'] / tab['sple03']
tab['spl14'].format = '.0f'
cols = [
'root', 'idx', 'ra', 'dec', 't_g102', 't_g141', 'mag_auto', 'is_point',
'z_map', 'chinu', 'bic_diff', 'chinusp', 'bic_diff_spl', 'star14',
'star14e', 'spl14', 'zwidth1', 'a_image', 'sn_SIII', 'sn_Ha',
'sn_OIII', 'sn_Hb', 'sn_OII', 'log_mass', 'aws_png_stack', 'png_full',
'png_line'
]
newcol = {}
for col in ['png_full', 'png_line']:
newcol[col] = [item.replace('+', '%2B') for item in tab[col]]
tab.remove_column(col)
tab[col] = newcol[col]
idx, dr = fit.match_to_catalog_sky(tab)
for c in cols:
if c not in tab.colnames:
tab[c] = fit[c][idx]
clip = tab['bic_diff'] > 0
outroot = 'point_sources_aug15'
tab.write(outroot + '.fits', overwrite=True)
tab[cols].filled(fill_value=-1).write_sortable_html(
outroot + '.html',
replace_braces=True,
localhost=False,
max_lines=50000,
table_id=None,
table_class='display compact',
css=None,
filter_columns=[
'mag_auto', 'z_map', 'z02', 'z97', 'bic_diff', 'chinusp', 'star14',
'star14e', 'spl14', 'bic_diff_spl', 'chinu', 'a_image',
'flux_radius', 'zwidth1', 'is_point', 'sn_SIII', 'sn_Ha',
'sn_OIII', 'sn_Hb', 'sn_OII'
],
use_json=True)
sleep_time = 303 * np.ceil(len(beams) / 950)
print('{0}: sleep {1}'.format(time.ctime(), sleep_time))
time.sleep(sleep_time)
# Status again to check products
beams, files = get_needed_paths(root)
def eazy_photoz(root, force=False, object_only=True, apply_background=True, aper_ix=1, apply_prior=False, beta_prior=True, get_external_photometry=False, external_limits=3, external_sys_err=0.3, external_timeout=300, sys_err=0.05, z_step=0.01, z_min=0.01, z_max=12, total_flux='flux_auto', auto_corr=True, compute_residuals=False, dummy_prior=False, extra_rf_filters=[]):
import os
import eazy
import numpy as np
from grizli import utils
import mastquery.utils
if (os.path.exists('{0}.eazypy.self.npy'.format(root))) & (not force):
self = np.load('{0}.eazypy.self.npy'.format(root))[0]
zout = utils.read_catalog('{0}.eazypy.zout.fits'.format(root))
cat = utils.read_catalog('{0}_phot_apcorr.fits'.format(root))
return self, cat, zout
trans = {'f098m':201, 'f105w':202, 'f110w':241, 'f125w':203, 'f140w':204, 'f160w':205, 'f435w':233, 'f438w':211, 'f606w':236, 'f625w':237, 'f814w':239, 'f702w':15, 'f555w':235, 'f350lp':339, 'f475w':212, 'f775w':238, 'f850lp':240}
#trans.pop('f814w')
print('Apply catalog corrections')
apply_catalog_corrections(root, suffix='_apcorr')
cat = utils.read_catalog('{0}_phot_apcorr.fits'.format(root))
filters = []
for c in cat.meta:
if c.endswith('_ZP'):
filters.append(c.split('_ZP')[0].lower())
# Translate
fp = open('zphot.translate','w')
for f in filters:
if f in trans:
fp.write('{0}_tot_{1} F{2}\n'.format(f, aper_ix, trans[f]))
fp.write('{0}_etot_{1} E{2}\n'.format(f, aper_ix, trans[f]))
fp.write('irac_ch1_flux F18\n')
fp.write('irac_ch1_err E18\n')
# For zeropoint
if dummy_prior:
fp.write('dummy_flux F205x\n')
fp.write('dummy_err E205x\n')
fp.close()
params = {}
params['CATALOG_FILE'] = '{0}_phot_apcorr.fits'.format(root)
params['Z_STEP'] = z_step
params['MAIN_OUTPUT_FILE'] = '{0}.eazypy'.format(root)
params['Z_MAX'] = z_max
params['MW_EBV'] = mastquery.utils.get_irsa_dust(cat['ra'].mean(), cat['dec'].mean())
params['PRIOR_ABZP'] = 23.9
params['SYS_ERR'] = sys_err
params['CAT_HAS_EXTCORR'] = False
# Pick prior filter, starting from reddest
for f in ['f435w', 'f606w', 'f814w', 'f105w', 'f110w', 'f125w', 'f140w', 'f160w'][::-1]:
if f in filters:
if dummy_prior:
params['PRIOR_FILTER'] = 'dummy_flux'
else:
params['PRIOR_FILTER'] = trans[f]
mag = 23.9-2.5*np.log10(cat['{0}_corr_{1}'.format(f, aper_ix)])
break
#
if os.path.exists('templates/fsps_full/tweak_fsps_QSF_11_v3_noRed.param.fits'):
params['TEMPLATES_FILE'] = 'templates/fsps_full/tweak_fsps_QSF_11_v3_noRed.param'
else:
params['TEMPLATES_FILE'] = 'templates/fsps_full/tweak_fsps_QSF_12_v3.param'
zpfile = None
load_products = False
if (not os.path.exists('FILTER.RES.latest') or
not os.path.exists('templates')):
try:
# should work with eazy-py >= 0.2.0
eazy.symlink_eazy_inputs(path=None)
except:
print("""
The filter file `FILTER.RES.latest` and `templates` directory were not
found in the working directory and the automatic command to retrieve them
failed:
>>> import eazy; eazy.symlink_eazy_inputs(path=None)
Run it with `path` pointing to the location of the `eazy-photoz` repository.""")
return False
self = eazy.photoz.PhotoZ(param_file=None, translate_file='zphot.translate', zeropoint_file=zpfile, params=params, load_prior=True, load_products=load_products)
if object_only:
return self
idx = np.arange(self.NOBJ)
#sample = (mag < 27) #& (self.cat['star_flag'] != 1)
#sample |= (self.cat['z_spec'] > 0)
sample = np.isfinite(self.cat['id']) #mag)
for iter in range(1+(get_external_photometry & compute_residuals)*1):
self.fit_parallel(idx[sample], n_proc=10)
if compute_residuals:
self.error_residuals()
self.standard_output(prior=apply_prior, beta_prior=beta_prior, extra_rf_filters=extra_rf_filters)
zout = utils.read_catalog('{0}.eazypy.zout.fits'.format(root))
np.save('{0}.eazypy.self.npy'.format(root), [self])
return self, cat, zout
def summary_catalog(field_root='', dzbin=0.01, use_localhost=True, filter_bandpasses=None, files=None, cdf_sigmas=None, strip_empty_columns=False, **kwargs):
"""
Make redshift histogram and summary catalog / HTML table
"""
import os
import time
import numpy as np
from matplotlib.ticker import FixedLocator
import matplotlib.pyplot as plt
import astropy.table
try:
from .. import fitting, prep, utils
from . import auto_script
except:
from grizli import prep, utils, fitting
from grizli.pipeline import auto_script
if filter_bandpasses is None:
import pysynphot as S
filter_bandpasses = [S.ObsBandpass(bpstr) for bpstr in ['acs,wfc1,f814w', 'wfc3,ir,f105w', 'wfc3,ir,f110w', 'wfc3,ir,f125w', 'wfc3,ir,f140w', 'wfc3,ir,f160w']]
if os.path.exists('{0}.info.fits'.format(field_root)):
orig = utils.read_catalog('{0}.info.fits'.format(field_root))
all_files = glob.glob('{0}*full.fits'.format(field_root))
all_files.sort()
print('{0}.info.fits: {1} objects. Found {2} full.fits files, checking modify dates.'.format(field_root, len(orig), len(all_files)))
info_mtime = os.stat('{0}.info.fits'.format(field_root)).st_mtime
keep = np.ones(len(orig), dtype=bool)
files = []
for file in all_files:
id = int(file.split('_')[1].split('.full')[0])
if id not in orig['id']:
files.append(file)
else:
full_mtime = os.stat(file).st_mtime
if full_mtime > info_mtime:
files.append(file)
keep[orig['id'] == id] = False
orig = orig[keep]
if len(files) == 0:
print('Found {0}.info.fits, and {1} new objects.\n'.format(field_root, len(files)))
return False
else:
print('Found {0}.info.fits. Adding {1} new objects.\n'.format(field_root, len(files)))
else:
orig = None
# SUmmary catalog
fit = fitting.make_summary_catalog(target=field_root, sextractor=None,
filter_bandpasses=filter_bandpasses,
files=files, cdf_sigmas=cdf_sigmas,
write_table=(dzbin is not None))
fit.meta['root'] = field_root
if orig is not None:
if len(fit) > 0:
fit = astropy.table.vstack([orig, fit])
if dzbin is not None:
fit.write('{0}.info.fits'.format(field_root), overwrite=True)
mtime = []
for i in range(len(fit)):
full_file = '{0}_{1:05d}.full.fits'.format(fit['root'][i], fit['id'][i])
if os.path.exists(full_file):
mtime.append(time.ctime(os.stat(full_file).st_mtime))
else:
mtime.append('-')
fit['mtime'] = mtime
# Add photometric catalog
try:
catalog = glob.glob('{0}-*.cat.fits'.format(field_root))[0]
sex = utils.GTable.gread(catalog)
# try:
# except:
# sex = utils.GTable.gread('../Prep/{0}-ir.cat.fits'.format(field_root), sextractor=True)
idx = np.arange(len(sex))
sex_idx = np.array([idx[sex['NUMBER'] == id][0] for id in fit['id']])
fit['ellipticity'] = (sex['B_IMAGE']/sex['A_IMAGE'])[sex_idx]
for col in ['MAG_AUTO', 'FLUX_RADIUS', 'A_IMAGE']:
fit[col.lower()] = sex[col][sex_idx]
except:
pass
fit = set_column_formats(fit)
if strip_empty_columns:
# Remove float columns with only NaN values
#print('Strip empty columns')
empty_cols = []
for c in fit.colnames:
try:
isfin = np.isfinite(fit[c])
if isfin.sum() == 0:
empty_cols.append(c)
except:
pass
for c in empty_cols:
fit.remove_column(c)
# Just return the table if dzbin parameter not specified
if dzbin is None:
return fit
# Overwrite with additional sextractor keywords
fit.write('{0}.info.fits'.format(field_root), overwrite=True)
clip = (fit['chinu'] < 2.0) & (fit['log_risk'] < -1)
clip = (fit['chinu'] < 2.0) & (fit['zq'] < -3) & (fit['zwidth1']/(1+fit['z_map']) < 0.005)
clip &= fit['bic_diff'] > 30 # -40
bins = utils.log_zgrid(zr=[0.1, 3.5], dz=dzbin)
fig = plt.figure(figsize=[6, 4])
ax = fig.add_subplot(111)
ax.hist(np.log10(1+fit['z_map']), bins=np.log10(1+bins), alpha=0.2, color='k')
ax.hist(np.log10(1+fit['z_map'][clip]), bins=np.log10(1+bins), alpha=0.8)
xt = np.array(np.arange(0.25, 3.55, 0.25))
ax.xaxis.set_minor_locator(FixedLocator(np.log10(1+xt)))
xt = np.array([1, 2, 3])
ax.set_xticks(np.log10(1+xt))
ax.set_xticklabels(xt)
ax.set_xlabel('z')
ax.set_ylabel(r'$N$')
ax.grid()
ax.text(0.05, 0.95, field_root, ha='left', va='top', transform=ax.transAxes)
fig.tight_layout(pad=0.2)
fig.savefig('{0}_zhist.png'.format(field_root))
cols = ['root', 'mtime', 'idx', 'ra', 'dec', 'mag_auto', 't_g800l', 't_g102', 't_g141', 'z_map', 'chinu', 'bic_diff', 'zwidth1', 'd4000', 'd4000_e', 'png_stack', 'png_full', 'png_rgb', 'png_line']
for i in range(len(cols))[::-1]:
if cols[i] not in fit.colnames:
cols.pop(i)
filter_columns = ['ra', 'dec', 'mag_auto', 't_g800l', 't_g102', 't_g141', 'z_map', 'chinu', 'bic_diff', 'zwidth1', 'd4000', 'd4000_e']
fit[cols].write_sortable_html(field_root+'-fit.html', replace_braces=True, localhost=use_localhost, max_lines=50000, table_id=None, table_class='display compact', css=None, filter_columns=filter_columns, use_json=(not use_localhost))
fit[cols][clip].write_sortable_html(field_root+'-fit.zq.html', replace_braces=True, localhost=use_localhost, max_lines=50000, table_id=None, table_class='display compact', css=None, filter_columns=filter_columns, use_json=(not use_localhost))
zstr = ['{0:.3f}'.format(z) for z in fit['z_map'][clip]]
prep.table_to_regions(fit[clip], output=field_root+'-fit.zq.reg', comment=zstr)
if False:
fit = utils.GTable.gread('{0}.info.fits'.format(root))
fit = auto_script.set_column_formats(fit)
cols = ['id', 'ra', 'dec', 'mag_auto', 't_g102', 't_g141', 'sn_Ha', 'sn_OIII', 'sn_Hb', 'z_map', 'log_risk', 'log_pdf_max', 'zq', 'chinu', 'bic_diff', 'zwidth1', 'png_stack', 'png_full', 'png_line']
#clip = ((fit['sn_Ha'] > 5) | (fit['sn_OIII'] > 5)) & (fit['bic_diff'] > 50) & (fit['chinu'] < 2)
#clip = (fit['sn_OIII'] > 5) & (fit['bic_diff'] > 100) & (fit['chinu'] < 3)
test_line = {}
for li in ['Ha', 'OIII', 'OII']:
test_line[l] = (fit['sn_'+li] > 5) & (fit['err_'+li] < 1.e-16)
clip = (test_line['Ha'] | test_line['OIII'] | test_line['OII']) & (fit['bic_diff'] > 50) & (fit['chinu'] < 2)
star = fit['flux_radius'] < 2.3
clip &= ~star
jh = fit['mag_wfc3,ir,f125w'] - fit['mag_wfc3,ir,f160w']
clip = (fit['chinu'] < 2) & (jh > 0.9) & (fit['mag_wfc3,ir,f160w'] < 23)
fit['jh'] = jh
fit['jh'].format = '.1f'
fit['dmag'] = fit['mag_wfc3,ir,f140w'] - fit['mag_auto']
fit['dmag'].format = '.1f'
cols = ['idx', 'ra', 'dec', 'mag_auto', 'jh', 'dmag', 't_g141', 'sn_Ha', 'sn_OIII', 'sn_Hb', 'z_map', 'log_risk', 'log_pdf_max', 'zq', 'chinu', 'bic_diff', 'zwidth1', 'png_stack', 'png_full', 'png_line']
fit[cols][clip].write_sortable_html(root+'-fit.lines.html', replace_braces=True, localhost=False, max_lines=50000, table_id=None, table_class='display compact', css=None)
def full_hawki_query(rd=None, query_result=None, eso=None):
"""
Query all HAWKI observations....
"""
import os
import numpy as np
import matplotlib.pyplot as plt
from shapely.geometry import Polygon, Point
from descartes import PolygonPatch
from shapely import affinity
from grizli import utils
from mastquery import query, overlaps
if eso is None:
eso = get_eso()
if query_result is None:
_, kwargs, res = full_query(eso=eso)
else:
kwargs, res = query_result
# surveys = 092.A-0472
# CHArGE fields
from grizli.aws import db
import astropy.units as u
from astropy.coordinates import SkyCoord
engine = db.get_db_engine()
if rd is None:
ch = db.from_sql(
"SELECT field_root, field_ra as ra, field_dec as dec, log FROM charge_fields where log LIKE '%%Finish%%'",
engine)
else:
ra, dec = rd
ch = utils.GTable()
ch['ra'] = [ra]
ch['dec'] = [dec]
ch['field_root'] = [
utils.radec_to_targname(
ra=ra,
dec=dec,
round_arcsec=(4, 60),
precision=2,
targstr='j{rah}{ram}{ras}{sign}{ded}{dem}',
header=None,
)
]
idx, dr = ch.match_to_catalog_sky(res)
has_hawki = dr < 10 * u.arcmin
import scipy.spatial
ch_rd = SkyCoord(ch['ra'], ch['dec'], unit='deg')
ch_xyz = ch_rd.cartesian.get_xyz().value
ctree = scipy.spatial.cKDTree(ch_xyz.T)
hawki_rd = SkyCoord(res['RA'], res['DEC'], unit='deg')
hawki_xyz = hawki_rd.cartesian.get_xyz().value
htree = scipy.spatial.cKDTree(hawki_xyz.T)
r = 30. / 60 / 360. * 2
tr = ctree.query_ball_tree(htree, r)
n_hawki = np.array([len(t) for t in tr])
# Figures
idx = np.where(n_hawki > 0)[0]
xsize = 5
px, py = 0.45, 0.2
for i in idx:
field = ch['field_root'][i]
print(i, field)
if os.path.exists(f'{field}_hawki.png'):
continue
field = ch['field_root'][i]
#tab = utils.read_catalog(f'../FieldsSummary/{field}_footprint.fits')
if os.path.exists(f'{field}_footprint.fits'):
tab = utils.read_catalog(f'{field}_footprint.fits')
meta = tab.meta
xr = (meta['XMIN'], meta['XMAX'])
yr = (meta['YMIN'], meta['YMAX'])
ra, dec = meta['BOXRA'], meta['BOXDEC']
cosd = np.cos(dec / 180 * np.pi)
dx = (xr[1] - xr[0]) * cosd * 60
dy = (yr[1] - yr[0]) * 60
box_width = np.maximum(dx, dy)
#query_size = np.maximum(min_size, box_width/2)/60.
p_hst = None
p_ir = None
for j, fph in enumerate(tab['footprint']):
ps, is_bad, poly = query.instrument_polygon(tab[j])
if not hasattr(ps, '__len__'):
ps = [ps]
for p in ps:
p_j = Polygon(p).buffer(0.001)
if p_hst is None:
p_hst = p_j
else:
p_hst = p_hst.union(p_j)
if tab['instrument_name'][j] == 'WFC3/IR':
if p_ir is None:
p_ir = p_j
else:
p_ir = p_ir.union(p_j)
else:
cosd = np.cos(dec / 180 * np.pi)
p_hst = None
p_ir = None
##############################
fig = plt.figure(figsize=[6, 6])
ax = fig.add_subplot(111)
ax.scatter(ra, dec, zorder=1000, marker='+', color='k')
# HAWKI
h_p = None
for j in tr[i]:
p = Point(res['RA'][j], res['DEC'][j]).buffer(4.1 / 60)
p = affinity.scale(p, xfact=1. / cosd)
# ax.add_patch(PolygonPatch(p, color='r', alpha=0.1))
x, y = p.boundary.xy
ax.plot(x, y, color=utils.MPL_COLORS['r'], alpha=0.05)
if h_p is None:
h_p = p
else:
h_p = h_p.union(p)
# If overlap between hawki and HST, query all exposures
if p_hst is not None:
hawki_overlap = h_p.intersection(p_hst)
hawki_un = h_p.union(p_hst)
if not hasattr(p_hst, '__len__'):
p_hst = [p_hst]
if not hasattr(h_p, '__len__'):
h_p = [h_p]
for p in p_hst:
#ax.add_patch(PolygonPatch(p, color='k', alpha=0.2))
if not hasattr(p.boundary, '__len__'):
bs = [p.boundary]
else:
bs = p.boundary
for b in bs:
x, y = b.xy
ax.plot(x, y, color=utils.MPL_COLORS['gray'], alpha=0.3)
else:
hawki_overlap = h_p
if not hasattr(h_p, '__len__'):
h_p = [h_p]
if p_ir is not None:
if not hasattr(p_ir, '__len__'):
p_ir = [p_ir]
for p in p_ir:
ax.add_patch(
PolygonPatch(p, color=utils.MPL_COLORS['gray'], alpha=0.2))
x, y = p.boundary.xy
ax.plot(x, y, color=utils.MPL_COLORS['gray'], alpha=0.3)
for p in h_p:
ax.add_patch(
PolygonPatch(p, color=utils.MPL_COLORS['r'], alpha=0.2))
targets = [
'{0} {1}'.format(res['ProgId'][j], res['Object'][j])
for j in tr[i]
]
for j, targ in enumerate(np.unique(targets)):
ixj = np.where(np.array(targets) == targ)[0]
expt = res['DET NDIT'] * res['DET DIT'] * res['TPL NEXP']
ax.text(0.02,
0.98 - j * 0.03,
'{0} {1:.1f}'.format(targ, expt[tr[i]][ixj].sum() / 3600.),
ha='left',
va='top',
transform=ax.transAxes,
fontsize=7)
ax.set_aspect(1. / cosd)
ax.set_title(field)
ax.grid()
#xsize = 4
dx = np.diff(ax.get_xlim())[0] * cosd * 60
dy = np.diff(ax.get_ylim())[0] * 60
fig.set_size_inches(xsize * np.clip(dx / dy, 0.2, 5) + px, xsize + py)
ax.set_xlim(ax.get_xlim()[::-1])
overlaps.draw_axis_labels(ax=ax, nlabel=3)
fig.tight_layout(pad=0.5)
fig.savefig(f'{field}_hawki.png', dpi=120)
plt.close('all')
if (hawki_overlap.area >
0.0) & (not os.path.exists(f'{field}_hawki.fits')):
kws = {}
for k in kwargs:
kws[k] = kwargs[k].copy()
kws['column_filters'].pop('tpl_nexp')
kws['column_filters'].pop('tpl_expno')
_res = eso.query_instrument('hawki',
pi_coi_name='PI_only',
coord1=ra,
coord2=dec,
box='00 30 00',
**kws)
if len(_res) > 0:
print('{0} datasets'.format(len(_res)))
_res['PI'] = [p.split('/')[0].strip() for p in _res['PI/CoI']]
_res.write(f'{field}_hawki.fits', overwrite=True)
def eazy_photoz(root, force=False):
import os
import eazy
import numpy as np
from grizli import utils
if (os.path.exists('{0}.eazypy.self.npy'.format(root))) & (not force):
self = np.load('{0}.eazypy.self.npy'.format(root))[0]
zout = utils.read_catalog('{0}.eazypy.zout.fits'.format(root))
cat = utils.read_catalog('{0}_phot_apcorr.fits'.format(root))
return self, cat, zout
trans = {
'f105w': 202,
'f110w': 241,
'f125w': 203,
'f140w': 204,
'f160w': 205,
'f435w': 233,
'f606w': 236,
'f814w': 239
}
cat = utils.read_catalog('{0}_phot.fits'.format(root))
filters = []
for c in cat.meta:
if c.endswith('_ZP'):
filters.append(c.split('_ZP')[0].lower())
# Total flux
apcorr = {}
for i in range(5):
if 'flux_aper_{0}'.format(i) in cat.colnames:
cat['apcorr_{0}'.format(
i)] = cat['flux_auto'] / cat['flux_aper_{0}'.format(i)]
for f in filters:
cat['{0}_corr_{1}'.format(
f, i)] = cat['{0}_flux_aper_{1}'.format(
f, i)] * cat['apcorr_{0}'.format(i)]
cat['{0}_ecorr_{1}'.format(
f, i)] = cat['{0}_fluxerr_aper_{1}'.format(
f, i)] * cat['apcorr_{0}'.format(i)]
bad = cat['{0}_mask_aper_{1}'.format(
f, i)] > 0.2 * np.percentile(
cat['{0}_mask_aper_{1}'.format(f, i)], 95)
cat['{0}_corr_{1}'.format(f, i)][bad] = -99
cat['{0}_ecorr_{1}'.format(f, i)][bad] = -99
cat.rename_column('number', 'id')
cat['z_spec'] = cat['id'] * 0. - 1
cat.write('{0}_phot_apcorr.fits'.format(root), overwrite=True)
# Translate
ix = 0
fp = open('zphot.translate', 'w')
for f in filters:
if f in trans:
fp.write('{0}_corr_{1} F{2}\n'.format(f, ix, trans[f]))
fp.write('{0}_ecorr_{1} E{2}\n'.format(f, ix, trans[f]))
fp.close()
params = {}
params['CATALOG_FILE'] = '{0}_phot_apcorr.fits'.format(root)
params['Z_STEP'] = 0.01
params['MAIN_OUTPUT_FILE'] = '{0}.eazypy'.format(root)
params['Z_MAX'] = 12
params['MW_EBV'] = 0.
params['PRIOR_ABZP'] = 23.9
# Pick prior filter, starting from reddest
for f in [
'f435w', 'f606w', 'f814w', 'f105w', 'f110w', 'f125w', 'f140w',
'f160w'
][::-1]:
if f in filters:
params['PRIOR_FILTER'] = trans[f]
mag = 23.9 - 2.5 * np.log10(cat['{0}_corr_{1}'.format(f, ix)])
break
#
params['TEMPLATES_FILE'] = 'templates/fsps_full/tweak_fsps_QSF_12_v3.param'
zpfile = None
load_products = False
eazy.symlink_eazy_inputs(
path='/usr/local/share/python/eazy-py/eazy-photoz', path_is_env=False)
self = eazy.photoz.PhotoZ(param_file=None,
translate_file='zphot.translate',
zeropoint_file=zpfile,
params=params,
load_prior=True,
load_products=load_products)
idx = np.arange(self.NOBJ)
#sample = (mag < 27) #& (self.cat['star_flag'] != 1)
#sample |= (self.cat['z_spec'] > 0)
sample = np.isfinite(mag)
for iter in range(2):
self.fit_parallel(idx[sample], n_proc=10)
self.error_residuals()
self.standard_output()
zout = utils.read_catalog('{0}.eazypy.zout.fits'.format(root))
np.save('{0}.eazypy.self.npy'.format(root), [self])
return self, cat, zout
def go():
os.chdir('/home/ec2-user/Mosaics')
root = 'j123656p6215'
root = 'j021732m0512'
root = 'j141956p5255'
root = 'j033236m2748'
root = 'j100012p0210'
# Sync needed files for GOODSN
if root == 'j123656p6215':
# os.system('aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/{0}_*dr*fits.gz" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" s3://grizli/Pipeline/ .'.format(root))
# os.system('aws s3 sync --exclude "*" --include "{0}*/Prep/*wcs.log" --include "{0}*/Prep/*shifts.*" s3://grizli/Pipeline/ .'.format(root))
os.system(
'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" s3://grizli/Pipeline/ .'
.format(root))
else:
#os.system('aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*_fl?.*" --include "{0}*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .'.format(root))
os.system(
'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .'
.format(root))
#os.system('files=`find . |grep "dr[cz]_" |grep fits.gz |grep -v "\-ir_dr"`; for file in $files; do echo $file; gunzip -f $file; done')
# Remake catalogs
mos_files = glob.glob('{0}*/Prep/*_dr*sci.fits'.format(root))
mos_files.sort()
thresh = 10
force = False
for i, file in enumerate(mos_files):
root = file.split('_drc_sci')[0].split('_drz_sci')[0]
print(i, root)
cat_file = root + '.cat.fits'
if (not os.path.exists(cat_file)) | (force):
prep.make_SEP_catalog(root=root,
threshold=thresh,
get_background=True,
phot_apertures=[1 * u.arcsec])
os.chdir('Fine/')
os.system('rsync -avz ../{0}*/Prep/*cat.fits .'.format(root))
os.system('ln -s ../{0}*/Prep/*_dr?_sci.fits .'.format(root))
os.system('rsync -avz ../{0}*/Prep/*visits.npy .'.format(root))
os.system('rsync -avz ../{0}*/Prep/*fail* .'.format(root))
visit_files = glob.glob('*visits.npy')
#visit_files = glob.glob('*-tile-*visits.npy')
visit_files.sort()
filt = 'f850lp'
all_visits = []
for f in visit_files:
root_i = f.split('_visits.npy')[0]
cat_file = root_i + '-' + filt + '.cat.fits'
if not os.path.exists(cat_file):
continue
visits, _, _ = np.load(f, allow_pickle=True)
has_failed = False
for v in visits:
has_failed |= os.path.exists(v['product'] + '.failed')
if has_failed:
continue
all_visits.append({'product': root_i + '-' + filt})
# Fine alignment
radec, ref_catalog = prep.get_radec_catalog(ra=189.2316435,
dec=62.249739865958,
product='xxx',
reference_catalogs=['PS1'],
radius=30)
auto_script.fine_alignment(field_root='xxx',
all_visits=all_visits,
gaia_by_date=False,
catalogs=['PS1'],
redrizzle=False,
shift_only=False,
radec=radec,
tol=1.e-3)
##### Done
import numpy as np
import os
import glob
from grizli import prep, utils
from drizzlepac import updatehdr
import astropy.io.fits as pyfits
import astropy.wcs as pywcs
fine_visits, fine_fit = np.load('xxx_fine.npy', allow_pickle=True)
N = len(fine_visits)
trans = np.reshape(fine_fit.x, (N, -1)) #/10.
sh = trans.shape
if sh[1] == 2:
pscl = np.array([10., 10.])
trans = np.hstack([trans / pscl, np.zeros((N, 1)), np.ones((N, 1))])
elif sh[1] == 3:
pscl = np.array([10., 10., 100])
trans = np.hstack([trans / pscl, np.ones((N, 1))])
elif sh[1] == 4:
pscl = np.array([10., 10., 100, 100])
trans = trans / pscl
# Update WCS
# Update direct WCS
for ix, direct in enumerate(fine_visits):
root_i = direct['product'][:-7]
print('\n\n\n\n##### ', ix, root_i, '#######\n\n\n')
# Sync products
os.system(
'aws s3 sync s3://grizli/Pipeline/{0}/Prep/ ./{0}/Prep/ --exclude "*" --include "{0}*drc_sci.fits.gz" --include "*_flc.*fits"'
.format(root_i))
#os.system('aws s3 ls s3://grizli/Pipeline/{0}/Prep/'.format(root_i))
# Just visits and fail
os.system(
'aws s3 sync s3://grizli/Pipeline/{0}/Prep/ ./{0}/Prep/ --exclude "*" --include "*visits.npy" --include "*fail*"'
.format(root_i))
#direct = visits[ix]
out_shift, out_rot = trans[ix, :2], trans[ix, 2]
out_scale = trans[ix, 3]
xyscale = trans[ix, :4]
wcs_ref_file = str(
'{0}/Prep/{0}-f850lp_drc_sci.fits.gz'.format(root_i))
wcs_ref = pywcs.WCS(pyfits.open(wcs_ref_file)[0].header, relax=True)
for file in glob.glob('{0}/Prep/*flc.fits'.format(root_i)):
prep.update_wcs_fits_log(file,
wcs_ref,
xyscale=xyscale,
initialize=False,
replace=('.fits', '.wcslog.fits'),
wcsname='FINE')
updatehdr.updatewcs_with_shift(file,
wcs_ref_file,
xsh=out_shift[0],
ysh=out_shift[1],
rot=out_rot,
scale=out_scale,
wcsname='FINE',
force=True,
reusename=True,
verbose=True,
sciext='SCI')
### Bug in astrodrizzle? Dies if the FLT files don't have MJD-OBS
### keywords
im = pyfits.open(file, mode='update')
im[0].header['MJD-OBS'] = im[0].header['EXPSTART']
im.flush()
## Fields with catalogs
visit_files = glob.glob('j*/Prep/*f814w*visits.npy')
visit_files.sort()
## Mosaic now that they're aligned
# Shift parameters
if False:
os.system(
'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" --include "{0}*/Prep/*expflag*" s3://grizli-v1/Pipeline/ .'
.format(root))
# No catalogs
os.system(
'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*wcs.log" --include "{0}*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .'
.format(root))
# ACS
os.system(
'aws s3 sync --exclude "*" --include "{0}*acswfc*/Prep/*expflag*" --include "{0}*acswfc*/Prep/*fail*" --include "{0}*acswfc*/Prep/*visits.npy" --include "{0}*acswfc*/Prep/*[._]wcs*" --include "{0}*acswfc*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .'
.format(root))
# Shifts
os.system(
'echo "# root visit exp xsh ysh rot scl n xrms yrms" > shifts_results.log'
)
os.system(
'grep "_fl" j*/Prep/*shifts.log | grep -v "\#" | sed "s/:/ /" | sed "s/.Prep./ /g" | sed "s/_shifts.log//" >> shifts_results.log'
)
sh = utils.read_catalog('shifts_results.log')
badsh = (sh['xrms'] > 1) | (sh['yrms'] > 1) #| (sh['n'] < 10)
shift_skip_visits = list(np.unique(sh['visit'][badsh]))
# Flag
os.system('echo "group,x,expm,flag" > expflag_results.csv')
os.system(
'grep "fits" j*/Prep/*expflag*txt | sed "s/:/,/g" | sed "s/.txt/ /" | sed "s/.Prep//" | sed "s/\//,/g" | sed "s/_raw/_flt/" >> expflag_results.csv'
)
expf = utils.read_catalog('expflag_results.csv')
bad_expf = expf['flag'] != 'NORMAL'
exp_visits = np.unique(expf['group'][bad_expf])
exp_visits = []
# wcs
os.system('echo "# dir visit x xs ys rot scl rms n" > shift_wcs.log')
os.system(
'grep " 0 " j*/Prep/*wcs.log | sed "s/.Prep./ /" | sed "s/_wcs.log://" >> shift_wcs.log'
)
shifts = utils.read_catalog('shift_wcs.log')
skip = (shifts['n'] < 7)
skip |= (shifts['xs'] == 0.) | (np.abs(shifts['rot']) > 0.2)
skip_visits = [
'{0}_{1}'.format('_'.join(d.split('_')[:2]), v)
for d, v in zip(shifts['dir'][skip], shifts['visit'][skip])
]
skip_visits += list(exp_visits)
skip_visits += shift_skip_visits
skip_visits = list(np.unique(skip_visits))
skip_keys = ['uds-12-bhm']
skip_keys += ['94s-245.0'] # bad shifts.txt
skip_keys += ['sn2002zx-'] # bad scale
visit_files = glob.glob('j*/Prep/*visits.npy')
#visit_files = glob.glob('*shizels*/Prep/*visits.npy')
all_visits = []
all_groups = []
all_info = []
bucket = 'grizli-v1'
bucket = 'grizli-cosmos-v2'
for file in visit_files:
root_i = os.path.basename(file).split('_visits')[0]
assoc = '_'.join(os.path.basename(file).split('_')[:2])
visits, groups, info = np.load(file)
info['keep'] = False
ic = 0
for v in visits:
failed = glob.glob('{0}/Prep/{1}*fail*'.format(
root_i, v['product']))
if len(failed) > 0:
print(failed)
continue
for k in skip_keys:
if k in v['product']:
print('SKIP', v['product'])
continue
v['product'] = assoc + '_' + v['product']
if v['product'] in skip_visits:
print('SKIP ', v['product'])
continue
v['awspath'] = ['{0}/Pipeline/{1}/Prep'.format(bucket, root_i)
] * len(v['files'])
all_visits.append(v)
for f in v['files']:
info['keep'][info['FILE'] == f] = True
all_info.append(info[info['keep'] == True])
for g in groups:
failed = glob.glob('{0}/Prep/{1}*fail*'.format(
root_i, g['direct']['product']))
if len(failed) > 0:
print(failed)
continue
# for ext in ['direct', 'grism']:
# g[ext]['product'] = assoc+'_'+g[ext]['product']
all_groups.append(g)
import astropy.table
all_info = astropy.table.vstack(all_info)
if root == 'j123656p6215':
out_root = 'gdn-' + root
elif root == 'j021732m0512':
out_root = 'uds-' + root
elif root == 'j141956p5255':
out_root = 'egs-' + root
elif root == 'j033236m2748':
out_root = 'gds-' + root
elif root == 'j100012p0210':
out_root = 'cos-' + root
else:
out_root = 'xxx-' + root
np.save('{0}_visits.npy'.format(out_root),
[all_visits, all_groups, all_info])
if True:
os.system(
'aws s3 cp {0}_visits.npy s3://{1}/Mosaics/ --acl public-read'.
format(out_root, bucket))
all_visits, all_groups, all_info = np.load(
'{0}_visits.npy'.format(out_root))
########
# Remake catalogs and drizzled images
from drizzlepac.astrodrizzle import AstroDrizzle
import numpy as np
import glob
import os
from grizli import prep
all_visits, all_groups, all_info = np.load(
'{0}_visits.npy'.format(out_root))
filters = ['f814w', 'f140w', 'f160w', 'f105w', 'f098m', 'f850lp']
#filters = ['f850lp']
count, products = 0, []
for ii, direct in enumerate(all_visits[::-1]):
filt = direct['product'].split('-')[-1]
prod_files = glob.glob(direct['product'] + '*sci.fits')
if (filt not in filters):
continue
else:
products.append(direct['product'])
if (len(prod_files) > 0):
print('Skip ', direct['product'])
continue
else:
count += 1
print('\n\n\n\n===========\n\n', ii, count, direct['product'],
'\n\n\n========\n')
#break
isACS = '_flc' in direct['files'][0]
if isACS:
bits = 64 + 32 + 256
driz_cr_snr = '3.5 3.0'
driz_cr_scale = '1.2 0.7'
else:
bits = 576 + 256
driz_cr_snr = '8.0 5.0'
driz_cr_scale = '2.5 0.7'
for aws, file in zip(direct['awspath'], direct['files']):
if not os.path.exists(file):
os.system('aws s3 cp s3://{0}/{1} ./'.format(aws, file))
cr_corr = False
AstroDrizzle(direct['files'],
output=direct['product'],
clean=True,
context=False,
preserve=False,
skysub=True,
driz_separate=cr_corr,
driz_sep_wcs=cr_corr,
median=cr_corr,
blot=cr_corr,
driz_cr=cr_corr,
driz_cr_corr=cr_corr,
driz_cr_snr=driz_cr_snr,
driz_cr_scale=driz_cr_scale,
driz_combine=True,
final_bits=bits,
coeffs=True,
resetbits=4096 * cr_corr,
build=False,
final_kernel='point',
final_wht_type='IVM')
# Remake catalog
cat = prep.make_SEP_catalog(root=direct['product'], threshold=5)
os.system('rm {0}*_[wbs]?[gt].fits'.format(direct['product']))
# Remove mosaic
os.system('rm {0}*_sci.fits'.format(direct['product']))
# Remove FLC
if isACS:
for f in direct['files']:
os.remove(f)
# Make mosaics
kwargs = auto_script.get_yml_parameters()
mos_args = {
'mosaic_args':
kwargs['mosaic_args'],
'fix_stars':
kwargs['visit_prep_args']['fix_stars'],
'mask_spikes':
kwargs['mask_spikes'],
'skip_single_optical_visits':
kwargs['preprocess_args']['skip_single_optical_visits']
}
#mos_args['mosaic_args']['ir_filters'] = ['F140W']
#mos_args['mosaic_args']['optical_filters'] = ['F814W']
mos_args['mosaic_args']['fill_mosaics'] = False
mos_args['mosaic_args']['half_optical_pixscale'] = True
mos_args['mosaic_args']['kernel'] = 'square'
mos_args['mosaic_args']['pixfrac'] = 0.33
mos_args['mosaic_args']['wcs_params']['pixel_scale'] = 0.1
if root == 'j123656p6215':
mos_args['mosaic_args']['kernel'] = 'point'
mos_args['mosaic_args']['pixfrac'] = 0.33
mos_args['mosaic_args']['wcs_params']['pixel_scale'] = 0.1
#mos_args['mosaic_args']['wcs_params']['filters'] = ['F140W']
os.system('ln -s j*/Prep/*_fl?.fits .')
auto_script.make_combined_mosaics(out_root, **mos_args)
