def get_phot_dict(self, ra, dec):
"""
"""
from collections import OrderedDict
try:
from .. import utils
except:
from grizli import utils
icat = utils.GTable()
icat['ra'] = [ra]
icat['dec'] = [dec]
rdcat = utils.GTable()
rdcat['ra'] = self.ra_cat
rdcat['dec'] = self.dec_cat
ix, dr = rdcat.match_to_catalog_sky(icat)
phot = OrderedDict()
phot['flam'] = self.flam[ix[0], :] * 1.e-19
phot['eflam'] = self.eflam[ix[0], :] * 1.e-19
phot['filters'] = self.filters
phot['tempfilt'] = self.tempfilt
return phot, ix[0], dr[0]
def get_phot_dict(self, ra, dec):
"""
Return catalog info for nearest object to supplied coordinates
Returns:
phot - photometry dictionary
ix - index in catalog of nearest match
dr - distance to nearest match
"""
from collections import OrderedDict
try:
from .. import utils
except:
from grizli import utils
icat = utils.GTable()
icat['ra'] = [ra]
icat['dec'] = [dec]
rdcat = utils.GTable()
rdcat['ra'] = self.ra_cat
rdcat['dec'] = self.dec_cat
ix, dr = rdcat.match_to_catalog_sky(icat)
phot = OrderedDict()
apcorr_i = self.apcorr[ix[0]]
try:
phot['source'] = self.source_text
except:
phot['source'] = 'unknown'
phot['flam'] = self.flam[ix[0], :] * 1.e-19 * apcorr_i
phot['eflam'] = self.eflam[ix[0], :] * 1.e-19 * apcorr_i
phot['filters'] = self.filters
phot['tempfilt'] = self.tempfilt
try:
phot['ext_corr'] = self.ext_corr
except:
phot['ext_corr'] = 1
if self.include_pz & (self.pz is not None):
pz = (self.zgrid, self.pz[ix[0], :].flatten())
else:
pz = None
phot['pz'] = pz
try:
phot['z_spec'] = self.z_spec[ix[0]]
except:
phot['z_spec'] = -1
if not self.include_photometry:
phot['flam'] = None
return phot, ix[0], dr[0]
def single_pixel_table(mask, start_id=0):
"""
Get individual pixels
"""
from grizli import utils
import numpy as np
sh = mask.shape
yp,xp = np.indices(sh)
n_obj = mask.sum()
# id, bin, xmin, xmax, ymin, ymax, npix
tab = utils.GTable()
tab['id'] = start_id+np.arange(n_obj,dtype=int)
tab['bin'] = 1
tab['xmin'] = xp[mask]
tab['xmax'] = tab['xmin']+1
tab['ymin'] = yp[mask]
tab['ymax'] = tab['ymin']+1
tab['npix'] = 1
return tab
def make_visit_fits():
import glob
import numpy as np
from grizli import utils
visit_files = glob.glob('[egu]*visits.npy')
visit_files.sort()
all_visits = []
products = []
for file in visit_files:
visits, groups, info = np.load(file)
for v in visits:
has_fp = ('footprints' in v)
if not has_fp:
print('No footprint: {0}'.format(v['product']))
if has_fp & (v['product'] not in products):
all_visits.append(v)
products.append(v['product'])
# WFC3/IR copied to "Exposures" paths
for visit in all_visits:
visit['filter'] = visit['product'].split('-')[-1]
for v in all_visits:
if v['filter'].startswith('f0') | v['filter'].startswith('f1'):
#print(v['product'])
v['awspath'] = [
'grizli-v1/Exposures/{0}/{1}'.format(f[:4],
f.split('_')[0])
for f in v['files']
]
tab = utils.GTable()
for k in ['product', 'filter']:
tab[k] = [visit[k] for visit in all_visits]
coo = np.array([
np.array(visit['footprint'].centroid.xy).flatten()
for visit in all_visits
])
tab['ra'] = coo[:, 0]
tab['dec'] = coo[:, 1]
tab['nexp'] = [len(visit['files']) for visit in all_visits]
root = 'candels-july2019'
root = 'candels-sep2019'
tab.write(root + '_visits.fits', overwrite=True)
np.save(root + '_visits.npy', [all_visits])
os.system(
'echo "# In https://s3.amazonaws.com/grizli-v1/Mosaics/" > candels-july2019.files.txt; ls candels-july2019* |grep -v files.txt >> candels-july2019.files.txt'
)
os.system(
'aws s3 sync --exclude "*" --include "candels-july2019*" ./ s3://grizli-v1/Mosaics/ --acl public-read'
)
def _get_wcs_from_hdrtab(drz_file):
"""
Read tabulated exposure WCS info from the HDRTAB
extension of an AstroDrizzle output file
"""
from shapely.geometry import Polygon, Point
drz = pyfits.open(drz_file)
if 'HDRTAB' not in drz:
print('No HDRTAB extension found in {0}'.format(file))
return None
hdr = utils.GTable(drz['HDRTAB'].data)
wcs = OrderedDict()
footprint = OrderedDict()
flt_keys = []
N = len(hdr)
if 'CCDCHIP' in hdr.colnames:
ext_key = 'CCDCHIP'
else:
ext_key = 'EXTNAME'
for i in range(N):
h = pyfits.Header()
for c in hdr.colnames:
try:
h[c] = hdr[c][i]
except:
h[c] = 1
key = (h['ROOTNAME'], h[ext_key])
flt_keys.append(key)
wcs[key] = pywcs.WCS(h, relax=True)
wcs[key].pscale = utils.get_wcs_pscale(wcs[key])
if 'EXPTIME' in h:
wcs[key].expweight = h['EXPTIME']
else:
wcs[key].expweight = 1
footprint[key] = Polygon(wcs[key].calc_footprint())
return flt_keys, wcs, footprint
def make_summary_html(output_table='grizli_aws.html', verbose=True):
import os
from grizli import utils
import boto3
s3 = boto3.resource('s3')
bucket = s3.Bucket('aws-grivam')
roots, dates = get_roots(verbose=verbose)
# Table
tab = utils.GTable()
tab['root'] = roots
tab['modified'] = dates
tab['Full'] = [
'<a href=https://s3.amazonaws.com/aws-grivam/Pipeline/{0}/Extractions/{0}-full.html?chinu_max=2&bic_diff_min=30&zwidth1_max=0.01>Full</a>'
.format(root.replace('+', '%2B')) for root in roots
]
#tab['Best'] = ['<a href=https://s3.amazonaws.com/aws-grivam/Pipeline/{0}/Extractions/{0}-fit.zq.html>Best</a>'.format(root.replace('+','%2B')) for root in roots]
#tab['footprint'] = ['<a href=https://s3.amazonaws.com/aws-grivam/Pipeline/{0}_footprint.pdf> <img width=400 src=https://s3.amazonaws.com/aws-grivam/Pipeline/{0}_footprint.pdf></a>'.format(root.replace('+','%2B')) for root in roots]
tab['footprint'] = [
'<a href=https://s3.amazonaws.com/aws-grivam/Pipeline/{0}_footprint.png> <img height=300 src=https://s3.amazonaws.com/aws-grivam/Pipeline/{0}_footprint.png></a>'
.format(root.replace('+', '%2B')) for root in roots
]
tab['zhist'] = [
'<a href=https://s3.amazonaws.com/aws-grivam/Pipeline/{0}/Extractions/{0}_zhist.png> <img height=300 src=https://s3.amazonaws.com/aws-grivam/Pipeline/{0}/Extractions/{0}_zhist.png></a>'
.format(root.replace('+', '%2B')) for root in roots
]
tab.write_sortable_html(output_table, localhost=False, max_lines=10000)
print("""
Done: {0}
Sync to s3: aws s3 cp {0} s3://aws-grivam/Pipeline/ --acl public-read
""".format(output_table))
return tab
def get_external_catalog(
phot,
filter_file='/usr/local/share/eazy-photoz/filters/FILTER.RES.latest',
ZP=23.9,
sys_err=0.3,
verbose=True,
external_limits=3,
timeout=300):
"""
Fetch photometry from vizier
"""
import numpy as np
import astropy.units as u
try:
from .. import utils
except:
from grizli import utils
from eazy.filters import FilterFile
res = FilterFile(filter_file)
vizier_catalog = list(utils.VIZIER_VEGA.keys())
ra = np.median(phot['ra'])
dec = np.median(phot['dec'])
dr = np.sqrt((phot['ra'] - ra)**2 * np.cos(phot['dec'] / 180 * np.pi)**2 +
(phot['dec'] - dec)**2) * 60
radius = 1.5 * dr.max() # arcmin
tabs = utils.get_Vizier_photometry(ra,
dec,
templates=None,
radius=radius * 60,
vizier_catalog=vizier_catalog,
filter_file=filter_file,
MW_EBV=0,
convert_vega=False,
raw_query=True,
verbose=True,
timeout=timeout)
extern_phot = utils.GTable()
N = len(phot)
for t_i in tabs:
# Match
if 'RAJ2000' in t_i.colnames:
other_radec = ('RAJ2000', 'DEJ2000')
elif 'RA_ICRS' in t_i.colnames:
other_radec = ('RA_ICRS', 'DE_ICRS')
else:
other_radec = ('ra', 'dec')
idx, dr = phot.match_to_catalog_sky(t_i, other_radec=other_radec)
if (dr < 2 * u.arcsec).sum() == 0:
continue
tab = t_i[dr < 2 * u.arcsec]
idx = idx[dr < 2 * u.arcsec]
# Downweight PS1 if have SDSS
# if (tab.meta['name'] == PS1_VIZIER) & (SDSS_DR12_VIZIER in viz_tables):
# continue
# err_scale = 1
# else:
# err_scale = 1
err_scale = 1
if (tab.meta['name'] == utils.UKIDSS_LAS_VIZIER):
flux_scale = 1.33
else:
flux_scale = 1.
convert_vega = utils.VIZIER_VEGA[tab.meta['name']]
bands = utils.VIZIER_BANDS[tab.meta['name']]
#if verbose:
# print(tab.colnames)
#filters += [res.filters[res.search(b, verbose=False)[0]] for b in bands]
to_flam = 10**(-0.4 * (48.6)) * 3.e18 # / pivot(Ang)**2
for ib, b in enumerate(bands):
f_number = res.search(b, verbose=False)[0] + 1
filt = res.filters[f_number - 1]
#filters.append(filt)
if convert_vega:
to_ab = filt.ABVega()
else:
to_ab = 0.
fcol, ecol = bands[b]
#pivot.append(filt.pivot())
fnu = 10**(-0.4 * (tab[fcol] + to_ab - ZP))
efnu = tab[ecol] * np.log(10) / 2.5 * fnu * err_scale
efnu = np.sqrt(efnu**2 + (sys_err * fnu)**2)
fnu.fill_value = -99
efnu.fill_value = -99
comment = 'Filter {0} from {1} (N={2})'.format(
bands[b][0], t_i.meta['name'], len(idx))
if verbose:
print(comment)
if ((~efnu.mask).sum() > 4) & (external_limits > 0):
fill = np.percentile(efnu.data[~efnu.mask], [90])
efill = external_limits * fill
else:
fill = -99
efill = -99
extern_phot.meta['F{0}'.format(f_number)] = b, comment
extern_phot['F{0}'.format(f_number)] = fill * np.ones(N)
extern_phot['F{0}'.format(f_number)][idx] = fnu.filled()
extern_phot['E{0}'.format(f_number)] = efill * np.ones(N)
extern_phot['E{0}'.format(f_number)][idx] = efnu.filled()
return extern_phot
def summary_table(tabs=None, output='overlap_summary'):
import glob
from collections import OrderedDict
from astropy.table import Table
import astropy.table
try:
from grizli import utils
HAS_GRIZLI = True
except:
HAS_GRIZLI = False
if tabs is None:
tabs = [Table.read(file) for file in glob.glob('*footprint.fits')]
# for tab in tabs:
# tab.remove_column('moving_target')
# #
# for tab in tabs:
# prop = np.cast[int](tab['proposal_id'])
# tab.remove_column('proposal_id')
# tab['proposal_id'] = prop
names, props = parse_overlap_table(tabs[0])
props = []
pdict = OrderedDict()
for name in names:
pdict[name] = []
for i in range(len(tabs)):
print('Parse table ', i)
n_i, p_i = parse_overlap_table(tabs[i])
for name in names:
if name in n_i:
pdict[name].append(p_i[n_i.index(name)])
else:
pdict[name].append('---')
mtab = Table(pdict) #rows=props, names=names)
mtab['RA'].format = '.5f'
mtab['DEC'].format = '.5f'
mtab.rename_column('MW_EBV', 'E(B-V)')
mtab['EclLat'].format = '.1f'
mtab['EclLon'].format = '.1f'
mtab['GalLat'].format = '.1f'
mtab['GalLon'].format = '.1f'
mtab['AreaG102'].format = '.1f'
mtab['AreaG141'].format = '.1f'
mtab['TexpG102'].format = '.1f'
mtab['TexpG141'].format = '.1f'
mtab['TperG102'].format = '.1f'
mtab['TperG141'].format = '.1f'
# Links
mast_link = [
'<a href=https://archive.stsci.edu/hst/search.php?RA={0}&DEC={1}&radius=3.&max_records=5000&sci_aec=S&action=Search>MAST</a>'
.format(t['RA'], t['DEC']) for t in mtab
]
mtab['MAST'] = mast_link
fp_link = [
'<a href={0}_footprint.pdf>Footprint</a>'.format(t['NAME'])
for t in mtab
]
mtab['Footprint'] = fp_link
mtab.write('{0}.fits'.format(output), overwrite=True)
if HAS_GRIZLI:
gtab = utils.GTable(mtab)
gtab.write_sortable_html('{0}.html'.format(output),
replace_braces=True,
localhost=False,
max_lines=len(mtab) + 10,
table_id=None,
table_class='display compact',
css=None)
return gtab
def make_visit_fits():
import glob
import numpy as np
from grizli import utils
visit_files = glob.glob('[egu]*visits.npy')
visit_files.sort()
indiv_files = glob.glob('j*visits.npy')
indiv_files.sort()
visit_files += indiv_files
for p in ['grizli-v1-19.12.04_visits.npy', 'grizli-v1-19.12.05_visits.npy', 'grizli-cosmos-v2_visits.npy']:
if p in visit_files:
visit_files.pop(visit_files.index(p))
all_visits = []
products = []
for extra in ['candels-july2019_visits.npy', 'grizli-cosmos-v2_visits.npy']:
extra_visits = np.load(extra)[0]
extra_products = [v['product'] for v in extra_visits]
for i, p in enumerate(extra_products):
if p not in products:
parent = p.split('_')[0]
print(parent, p)
v = extra_visits[i]
v['parent'] = parent
v['parent_file'] = extra # 'candels-july2019_visits.npy'
all_visits.append(v)
products.append(p)
# COSMOS footprint
cosmos_fp = None
for i, v in enumerate(extra_visits):
if v['product'].endswith('f814w'):
print(v['product'])
if cosmos_fp is None:
cosmos_fp = v['footprint'].buffer(1.e-6)
else:
cosmos_fp = cosmos_fp.union(v['footprint'])
for i, file in enumerate(visit_files):
visits, groups, info = np.load(file)
print(file, len(visits))
for v in visits:
has_fp = ('footprints' in v)
if not has_fp:
print('No footprint: {0}'.format(v['product']))
if has_fp & (v['product'] not in products):
all_visits.append(v)
v['parent'] = file.split("_visits")[0].split('-')[-1]
v['first'] = v['files'][0]
v['parent_file'] = file
products.append(v['product'])
for v in all_visits:
v['filter'] = v['product'].split('-')[-1]
v['first'] = v['files'][0]
# File dictionary
all_files = []
file_products = []
for v in all_visits:
all_files.extend(v['files'])
file_products.extend([v['product']]*len(v['files']))
# duplicates?? seem to be in GOODS-S.
# Exclude them in all but the first product that contains them for now
if True:
_un = np.unique(all_files, return_counts=True, return_index=True, return_inverse=True)
un_file, un_index, un_inv, un_count = _un
dup = un_count > 1
dup_files = un_file[dup]
for file in dup_files:
prods = list(np.array(file_products)[np.array(all_files) == file])
for prod in prods[1:]:
i = products.index(prod)
v = all_visits[i]
j = v['files'].index(file)
print(file, v['parent'], prod, i, j)
pj = all_visits[i]['files'].pop(j)
pj = all_visits[i]['footprints'].pop(j)
if 'awspath' in all_visits[i]:
pj = all_visits[i]['awspath'].pop(j)
#print(file, prods[-1])
# WFC3/IR copied to "Exposures" paths in CANDELS fields
for v in all_visits:
if v['parent_file'] == 'grizli-cosmos-v2_visits.npy':
continue
if v['parent_file'].startswith('j'):
v['awspath'] = ['grizli-v1/Pipeline/{0}/Prep'.format(v['parent']) for f in v['files']]
if v['filter'].startswith('f0') | v['filter'].startswith('f1'):
# print(v['product'])
v['awspath'] = ['grizli-v1/Exposures/{0}/{1}'.format(f[:4], f.split('_')[0]) for f in v['files']]
# Empty visits, seems to be from duplicates above and mostly in CANDELS
nexp = np.array([len(visit['files']) for visit in all_visits])
for i in np.where(nexp == 0)[0][::-1]:
v_i = all_visits.pop(i)
print(i, v_i['product'])
products.pop(i)
tab = utils.GTable()
for k in ['parent', 'product', 'filter', 'first']:
tab[k] = [visit[k] for visit in all_visits]
coo = np.array([np.array(visit['footprint'].centroid.xy).flatten() for visit in all_visits])
tab['ra'] = coo[:, 0]
tab['dec'] = coo[:, 1]
tab['nexp'] = [len(visit['files']) for visit in all_visits]
tab['bounds'] = [np.array(v['footprint'].bounds) for v in all_visits]
root = 'candels-july2019'
root = 'candels-sep2019'
root = 'grizli-v1-19.12.04'
root = 'grizli-v1-19.12.05'
tab.write(root+'_visits.fits', overwrite=True)
np.save(root+'_visits.npy', [all_visits])
# os.system('echo "# In https://s3.amazonaws.com/grizli-v1/Mosaics/" > candels-july2019.files.txt; ls candels-july2019* |grep -v files.txt >> candels-july2019.files.txt')
#os.system('aws s3 sync --exclude "*" --include "candels-july2019*" --include "grizli-v1-19.12.04*" ./ s3://grizli-v1/Mosaics/ --acl public-read')
os.system('echo "# In https://s3.amazonaws.com/grizli-v1/Mosaics/" > {0}.files.txt; ls {0}* |grep -v files.txt >> {0}.files.txt'.format(root))
os.system('aws s3 sync --exclude "*" --include "{0}*" ./ s3://grizli-v1/Mosaics/ --acl public-read'.format(root))
if False:
from shapely.geometry import Point
candels = utils.column_values_in_list(tab['parent'], ['j141956p5255', 'j123656p6215', 'j033236m2748', 'j021732m0512', 'j100012p0210'])
cosmos = np.array([v['footprint'].intersection(cosmos_fp).area > 0 for v in all_visits])
extra = candels
extra = ~(candels | cosmos)
# Area
filter_polys = {}
filt = 'f160w'
for filt in np.unique(tab['filter']):
print(filt)
if filt in filter_polys:
print(filt)
continue
poly = None
count = 0
# Dec strips
di = np.arange(-90, 91, 5)
strips = []
for i in range(len(di)-1):
strip = (tab['dec'] > di[i]) & (tab['dec'] <= di[i+1]) & (tab['filter'] == filt)
strip &= extra
if strip.sum() == 0:
continue
indices = np.arange(len(tab))[strip]
poly = None
for j in indices:
v = all_visits[j]
if v['filter'] != filt:
continue
# for fp in v['footprints']:
for fp in [v['footprint']]:
count += 1
#print(i, v['product'], count)
if poly is None:
poly = fp.buffer(1.e-6)
else:
poly = poly.union(fp.buffer(1.e-6))
poly.dec = di[i]+2.5
strips.append(poly)
if len(strips) == 0:
filter_polys[filt] = Point(0, 0).buffer(1.e-6)
continue
full = strips[0].buffer(1.e-6)
for strip in strips[1:]:
full = full.union(strip.buffer(1.e-6))
filter_polys[filt] = full
optical = filter_polys['f606w'].union(filter_polys['f814w'])
optical = optical.union(filter_polys['f850lp'])
optical = optical.union(filter_polys['f775w'])
yband = filter_polys['f098m'].union(filter_polys['f105w'])
visy = optical.union(yband)
jband = filter_polys['f125w']
jband = jband.union(filter_polys['f110w'])
hband = filter_polys['f140w'].union(filter_polys['f160w'])
filter_polys[r'$\mathrm{opt} = i_{775} | i_{814} | z_{850}$'] = optical
filter_polys[r'$\mathrm{opty} = \mathrm{opt} | Y$'] = visy
filter_polys[r'$Y = y_{098 } | y_{105}$'] = yband
filter_polys[r'$J = j_{110} | j_{125}$'] = jband
filter_polys[r'$H = h_{140} | h_{160}$'] = hband
ydrop = visy.intersection(jband)
ydrop = ydrop.intersection(hband)
filter_polys[r'$Y-\mathrm{drop} = (\mathrm{opt} | Y) + J + H$'] = ydrop
yj = yband.union(jband)
jdrop = yj.intersection(hband)
filter_polys[r'$J-\mathrm{drop} = (Y | J) + H$'] = jdrop
for filt in filter_polys:
full = filter_polys[filt]
try:
areas = [f.area*np.cos(np.array(f.centroid.xy).flatten()[1]/180*np.pi) for f in full]
except:
try:
areas = [f.area*np.cos(np.array(f.centroid.xy).flatten()[1]/180*np.pi) for f in [full]]
except:
areas = [0]
full.total_area = np.sum(areas)
print(filt, filter_polys[filt].total_area)
ta = utils.GTable()
ta['filter'] = [f.upper() for f in filter_polys]
ta['area'] = [filter_polys[f].total_area*3600 for f in filter_polys]
ta['area'].format = '.0f'
# Compare areas
h = fields['a_wfc3_ir_f160w'] > 0
for root, aa in zip(fields['field_root'][h], fields['a_wfc3_ir_f160w'][h]):
sel = (tab['filter'] == 'f160w') & (tab['parent'] == root)
if sel.sum() > 0:
indices = np.where(sel)[0]
a = all_visits[indices[0]]['footprint'].buffer(1.e-6)
for i in indices:
a = a.union(all_visits[i]['footprint'])
a_i = a.area*3600*np.cos(tab['dec'][indices[0]]/180*np.pi)
print(root, aa, a_i, a_i/aa)
def full_hawki_query(rd=None, query_result=None, eso=None):
"""
Query all HAWKI observations....
"""
import os
import numpy as np
import matplotlib.pyplot as plt
from shapely.geometry import Polygon, Point
from descartes import PolygonPatch
from shapely import affinity
from grizli import utils
from mastquery import query, overlaps
if eso is None:
eso = get_eso()
if query_result is None:
_, kwargs, res = full_query(eso=eso)
else:
kwargs, res = query_result
# surveys = 092.A-0472
# CHArGE fields
from grizli.aws import db
import astropy.units as u
from astropy.coordinates import SkyCoord
engine = db.get_db_engine()
if rd is None:
ch = db.from_sql(
"SELECT field_root, field_ra as ra, field_dec as dec, log FROM charge_fields where log LIKE '%%Finish%%'",
engine)
else:
ra, dec = rd
ch = utils.GTable()
ch['ra'] = [ra]
ch['dec'] = [dec]
ch['field_root'] = [
utils.radec_to_targname(
ra=ra,
dec=dec,
round_arcsec=(4, 60),
precision=2,
targstr='j{rah}{ram}{ras}{sign}{ded}{dem}',
header=None,
)
]
idx, dr = ch.match_to_catalog_sky(res)
has_hawki = dr < 10 * u.arcmin
import scipy.spatial
ch_rd = SkyCoord(ch['ra'], ch['dec'], unit='deg')
ch_xyz = ch_rd.cartesian.get_xyz().value
ctree = scipy.spatial.cKDTree(ch_xyz.T)
hawki_rd = SkyCoord(res['RA'], res['DEC'], unit='deg')
hawki_xyz = hawki_rd.cartesian.get_xyz().value
htree = scipy.spatial.cKDTree(hawki_xyz.T)
r = 30. / 60 / 360. * 2
tr = ctree.query_ball_tree(htree, r)
n_hawki = np.array([len(t) for t in tr])
# Figures
idx = np.where(n_hawki > 0)[0]
xsize = 5
px, py = 0.45, 0.2
for i in idx:
field = ch['field_root'][i]
print(i, field)
if os.path.exists(f'{field}_hawki.png'):
continue
field = ch['field_root'][i]
#tab = utils.read_catalog(f'../FieldsSummary/{field}_footprint.fits')
if os.path.exists(f'{field}_footprint.fits'):
tab = utils.read_catalog(f'{field}_footprint.fits')
meta = tab.meta
xr = (meta['XMIN'], meta['XMAX'])
yr = (meta['YMIN'], meta['YMAX'])
ra, dec = meta['BOXRA'], meta['BOXDEC']
cosd = np.cos(dec / 180 * np.pi)
dx = (xr[1] - xr[0]) * cosd * 60
dy = (yr[1] - yr[0]) * 60
box_width = np.maximum(dx, dy)
#query_size = np.maximum(min_size, box_width/2)/60.
p_hst = None
p_ir = None
for j, fph in enumerate(tab['footprint']):
ps, is_bad, poly = query.instrument_polygon(tab[j])
if not hasattr(ps, '__len__'):
ps = [ps]
for p in ps:
p_j = Polygon(p).buffer(0.001)
if p_hst is None:
p_hst = p_j
else:
p_hst = p_hst.union(p_j)
if tab['instrument_name'][j] == 'WFC3/IR':
if p_ir is None:
p_ir = p_j
else:
p_ir = p_ir.union(p_j)
else:
cosd = np.cos(dec / 180 * np.pi)
p_hst = None
p_ir = None
##############################
fig = plt.figure(figsize=[6, 6])
ax = fig.add_subplot(111)
ax.scatter(ra, dec, zorder=1000, marker='+', color='k')
# HAWKI
h_p = None
for j in tr[i]:
p = Point(res['RA'][j], res['DEC'][j]).buffer(4.1 / 60)
p = affinity.scale(p, xfact=1. / cosd)
# ax.add_patch(PolygonPatch(p, color='r', alpha=0.1))
x, y = p.boundary.xy
ax.plot(x, y, color=utils.MPL_COLORS['r'], alpha=0.05)
if h_p is None:
h_p = p
else:
h_p = h_p.union(p)
# If overlap between hawki and HST, query all exposures
if p_hst is not None:
hawki_overlap = h_p.intersection(p_hst)
hawki_un = h_p.union(p_hst)
if not hasattr(p_hst, '__len__'):
p_hst = [p_hst]
if not hasattr(h_p, '__len__'):
h_p = [h_p]
for p in p_hst:
#ax.add_patch(PolygonPatch(p, color='k', alpha=0.2))
if not hasattr(p.boundary, '__len__'):
bs = [p.boundary]
else:
bs = p.boundary
for b in bs:
x, y = b.xy
ax.plot(x, y, color=utils.MPL_COLORS['gray'], alpha=0.3)
else:
hawki_overlap = h_p
if not hasattr(h_p, '__len__'):
h_p = [h_p]
if p_ir is not None:
if not hasattr(p_ir, '__len__'):
p_ir = [p_ir]
for p in p_ir:
ax.add_patch(
PolygonPatch(p, color=utils.MPL_COLORS['gray'], alpha=0.2))
x, y = p.boundary.xy
ax.plot(x, y, color=utils.MPL_COLORS['gray'], alpha=0.3)
for p in h_p:
ax.add_patch(
PolygonPatch(p, color=utils.MPL_COLORS['r'], alpha=0.2))
targets = [
'{0} {1}'.format(res['ProgId'][j], res['Object'][j])
for j in tr[i]
]
for j, targ in enumerate(np.unique(targets)):
ixj = np.where(np.array(targets) == targ)[0]
expt = res['DET NDIT'] * res['DET DIT'] * res['TPL NEXP']
ax.text(0.02,
0.98 - j * 0.03,
'{0} {1:.1f}'.format(targ, expt[tr[i]][ixj].sum() / 3600.),
ha='left',
va='top',
transform=ax.transAxes,
fontsize=7)
ax.set_aspect(1. / cosd)
ax.set_title(field)
ax.grid()
#xsize = 4
dx = np.diff(ax.get_xlim())[0] * cosd * 60
dy = np.diff(ax.get_ylim())[0] * 60
fig.set_size_inches(xsize * np.clip(dx / dy, 0.2, 5) + px, xsize + py)
ax.set_xlim(ax.get_xlim()[::-1])
overlaps.draw_axis_labels(ax=ax, nlabel=3)
fig.tight_layout(pad=0.5)
fig.savefig(f'{field}_hawki.png', dpi=120)
plt.close('all')
if (hawki_overlap.area >
0.0) & (not os.path.exists(f'{field}_hawki.fits')):
kws = {}
for k in kwargs:
kws[k] = kwargs[k].copy()
kws['column_filters'].pop('tpl_nexp')
kws['column_filters'].pop('tpl_expno')
_res = eso.query_instrument('hawki',
pi_coi_name='PI_only',
coord1=ra,
coord2=dec,
box='00 30 00',
**kws)
if len(_res) > 0:
print('{0} datasets'.format(len(_res)))
_res['PI'] = [p.split('/')[0].strip() for p in _res['PI/CoI']]
_res.write(f'{field}_hawki.fits', overwrite=True)
def voronoi_binning(image, obj_name, targetSN = 50, largest_bin = 5, smallest_bin = 0, minimumSN = 7, quiet=True, plot=True):
"""
Function to bin an image using the Voronoi binning method by Cappellari & Copin (2003)
Input as 'image' the target with the filter where S/N is highest
"""
import numpy as np
import astropy.io.fits as pyfits
from grizli import utils
from grizli import prep
import matplotlib.pyplot as plt
from astropy.table import vstack
im = pyfits.open(image)
sci = np.cast[np.float32](im['SCI'].data)
sh = sci.shape
ivar = np.cast[np.float32](im['WHT'].data)
var = 1/ivar
orig_mask = (ivar > 0)
sci[~orig_mask] = 0
var[~orig_mask] = 0
orig_var = var*1
# Simple background
bkg = np.median(sci[orig_mask])
sci -= bkg*orig_mask
cps = sci*im[0].header['EXPTIME']
shot_err = np.sqrt(np.maximum(cps, 4))/im[0].header['EXPTIME']
var2 = var + shot_err**2
var = var2
full_bin_seg = np.zeros(sh, dtype=np.int)-1
yp, xp = np.indices(sci.shape)
xpf = xp.flatten()
ypf = yp.flatten()
# Initialize mask
mask = orig_mask & True
bin_min = 1
full_image = sci*0.
full_err = sci*0.
idx = np.arange(sci.size, dtype=np.int)
full_image = full_image.flatten()
full_err = full_err.flatten()
# id, bin, xmin, xmax, ymin, ymax, npix
full_bin_data = []
SKIP_LAST = False
bin_iter = 0
bin_factor = largest_bin
NO_NEWLINE = '\x1b[1A\x1b[1M'
for bin_iter, bin_factor in enumerate(range(largest_bin+1)[::-1]):
bin = 2**bin_factor
if bin_factor < smallest_bin:
break
if (bin_factor == 0) & SKIP_LAST:
continue
ypb = yp[mask] // bin
xpb = xp[mask] // bin
if bin_factor > 0:
binned_sci = np.zeros((sh[0]//bin+1, sh[1]//bin+1))
binned_npix = binned_sci*0
binned_var = binned_sci*0
ypi, xpi = np.indices(binned_sci.shape)
# Only consider unmasked bins
ij = np.unique(xpb + sh[0]//bin*ypb)
yarr = ij // (sh[0]//bin)
xarr = ij - (sh[0]//bin)*yarr
for xi, yi in zip(xarr, yarr):
if not quiet:
print(NO_NEWLINE+'{0} {1}/{2} {3}/{4}'.format(bin_factor,
xi, xarr.max(),
yi, yarr.max()))
slx = slice(xi*bin, xi*bin+bin)
sly = slice(yi*bin, yi*bin+bin)
mslice = mask[sly, slx]
# Straight average
binned_sci[yi, xi] = sci[sly, slx][mslice].sum()
binned_npix[yi, xi] = mslice.sum()
binned_var[yi, xi] = var[sly, slx][mslice].sum()
binned_err = np.sqrt(binned_var) / binned_npix
binned_avg = binned_sci / binned_npix
mask_i = (binned_npix > 0) & (binned_avg/binned_err > minimumSN)
xpi = xpi[mask_i]
ypi = ypi[mask_i]
binned_avg = binned_avg[mask_i]
binned_err = binned_err[mask_i]
binned_npix = binned_npix[mask_i]
else:
mask_i = mask_j
xpi = xp[mask]
ypi = yp[mask]
binned_avg = sci[mask]
binned_err = np.sqrt(var)[mask]
binned_npix = mask[mask]*1
if True:
# Mask pixels in that don't satisfy S/N cutoff as they are
# unreliable for vorbin
clip_mask = mask < 0
for xi, yi in zip(xpi, ypi):
slx = slice(xi*bin, xi*bin+bin)
sly = slice(yi*bin, yi*bin+bin)
clip_mask[sly, slx] = True
mask &= clip_mask
# Identify blobs (usually the main central galaxies) and
# only consider blobs larger than 20% of the largest blob
if bin_factor == largest_bin:
label_image = label(mask)
label_ids = np.unique(label_image)[1:]
label_sizes = np.array([(label_image == id_i).sum() for id_i in label_ids])
keep_ids = label_ids[label_sizes > 0.2*label_sizes.max()]
keep_mask = mask < -1
for i in keep_ids:
keep_mask |= label_image == i
mask &= keep_mask
# In binned_coords
in_blob = keep_mask[ypi*bin, xpi*bin]
msg = 'Drop {0} bins not in main blob'
print(msg.format((~in_blob).sum()))
xpi = xpi[in_blob]
ypi = ypi[in_blob]
binned_avg = binned_avg[in_blob]
binned_err = binned_err[in_blob]
binned_npix = binned_npix[in_blob]
ypb = yp[mask] // bin
xpb = xp[mask] // bin
print('Run voronoi_2d_binning, bin_factor={0}'.format(bin_factor))
res = voronoi_2d_binning(xpi, ypi, binned_avg, binned_err, targetSN,
quiet=True, plot=False, pixelsize=0.1*bin,
cvt=True, wvt=True)
binNum, xBin, yBin, xBar, yBar, sn, nPixels, scale = res
# Put Voronoi bins with nPixels > 1 back in the original image
NBINS = len(nPixels)
if (bin_factor == smallest_bin) & (smallest_bin > 0):
valid_bins = nPixels > 0
else:
valid_bins = nPixels > 1
large_bin_ids = np.arange(NBINS)[valid_bins]
# Put bin in original 2D array and store info
for b0, bin_id in enumerate(large_bin_ids):
m_i = binNum == bin_id
xi_bin = xpi[m_i]
yi_bin = ypi[m_i]
for xi, yi in zip(xi_bin, yi_bin):
slx = slice(xi*bin, xi*bin+bin)
sly = slice(yi*bin, yi*bin+bin)
mslice = mask[sly, slx]
full_bin_seg[sly, slx][mslice] = b0+bin_min
# Bin properties
id_i = b0+bin_min
xmin = xi_bin.min()*bin-1
xmax = xi_bin.max()*bin+1
ymin = yi_bin.min()*bin-1
ymax = yi_bin.max()*bin+1
npix = m_i.sum()*bin**2
bin_data_i = [id_i, bin, xmin, xmax, ymin, ymax, npix]
full_bin_data.append(bin_data_i)
# Update the mask
not_in_a_bin = full_bin_seg == -1
mask &= not_in_a_bin
bin_min = full_bin_data[-1][0]+1
if not quiet:
print('\n\n\n\n\n bin_factor: {0}, bin_min: {1}\n\n\n\n'.format(bin_factor, bin_min))
## Bin information
bin_data = np.array(full_bin_data)
# bin_data_i = [id_i, bin, xmin, xmax, ymin, ymax, npix]
tab = utils.GTable()
for i, c in enumerate(['id', 'bin', 'xmin', 'xmax', 'ymin', 'ymax', 'npix']):
tab[c] = bin_data[:,i]
if 'min' in c:
tab[c] -= tab['bin']
elif 'max' in c:
tab[c] += tab['bin']
# Make a table for the individual pixels
if mask.sum() > 0:
single_table = single_pixel_table(mask,start_id=1+tab['id'].max())
full_bin_seg[mask] = single_table['id']
tab = vstack([tab,single_table])
tab['flux'], tab['err'], tab['area'] = prep.get_seg_iso_flux(sci, full_bin_seg, tab, err=np.sqrt(var))
binned_flux = prep.get_seg_iso_flux(sci, full_bin_seg, tab, fill=tab['flux']/tab['area'])
binned_err = prep.get_seg_iso_flux(sci, full_bin_seg, tab, fill=tab['err']/tab['area'])
binned_area = prep.get_seg_iso_flux(sci, full_bin_seg, tab, fill=tab['area'])
binned_bin = prep.get_seg_iso_flux(sci, full_bin_seg, tab, fill=tab['bin'])
binned_flux[mask] = sci[mask]
binned_err[mask] = np.sqrt(var)[mask]
binned_area[mask] = 1
binned_bin[mask] = 1
if plot:
plt.figure(figsize=(12,12))
plt.imshow(binned_flux,vmin=-0.5,vmax=2)
# Save output into image fits file
primary_extn = pyfits.PrimaryHDU()
sci_extn = pyfits.ImageHDU(data=binned_flux.astype(np.float32),name='SCI')
err_extn = pyfits.ImageHDU(data=binned_err.astype(np.float32),name='ERR')
hdul = pyfits.HDUList([primary_extn, sci_extn, err_extn])
for ext in [0,1]:
for k in im[ext].header:
if k not in hdul[ext].header:
if k in ['COMMENT','HISTORY','']:
continue
hdul[ext].header[k] = im[ext].header[k]
hdul.writeto('binned_{0}_image.fits'.format(obj_name), output_verify='fix', overwrite=True)
tab.write('binned_{0}_table.fits'.format(obj_name), overwrite=True)
pyfits.writeto('binned_{0}_seg.fits'.format(obj_name),data = full_bin_seg, overwrite=True)
pyfits.writeto('binned_{0}_mask.fits'.format(obj_name),data = mask*1, overwrite=True)
return tab, full_bin_seg, mask
def bin_image(im, tab_in, seg_in, mask_in, bkg_in=None, bg_mask_in=None):
"""
Apply 2D bins specified in "seg_in" to a new image
"""
from grizli import utils
from grizli import prep
import numpy as np
sci_data = im['SCI'].data*1
var_data = 1/im['WHT'].data
wht_mask = im['WHT'].data > 0
IS_ACS = sci_data.shape[0] == 2*seg_in.shape[0]
if IS_ACS:
# ACS
to_flam = im[1].header['PHOTFLAM']
to_fnu = 1. #im[1].header['PHOTFNU']
tab = utils.GTable()
for c in tab_in.colnames:
if c[:2] in ['xm', 'ym']:
tab[c] = 2*tab_in[c]
else:
tab[c] = tab_in[c]
sh = sci_data.shape
mask = np.zeros(sh, dtype=bool)
seg = np.zeros(sh, dtype=np.int)
for i in [0,1]:
for j in [0,1]:
mask[j::2, i::2] = mask_in
seg[j::2, i::2] = seg_in
else:
to_flam = im[0].header['PHOTFLAM']
to_fnu = im[0].header['PHOTFNU']
tab = tab_in
mask = mask_in
seg = seg_in
if bg_mask_in is None:
bg_mask = (~mask) & wht_mask & (seg <= 0)
else:
bg_mask = bg_mask_in
if bkg_in is None:
bkg = np.median(sci_data[bg_mask])
else:
bkg = bkg_in
sci_data -= bkg
var_data[~wht_mask] = 0
bin_flux, bin_err, bin_area = prep.get_seg_iso_flux(sci_data, seg, tab, err=np.sqrt(var_data))
image_flux = prep.get_seg_iso_flux(sci_data, seg, tab, fill=bin_flux/bin_area)
image_err = prep.get_seg_iso_flux(sci_data, seg, tab, fill=bin_err/bin_area)
image_flux[mask] = sci_data[mask]*1
image_err[mask] = np.sqrt(var_data)[mask]
res = {}
res['bin_flux'] = bin_flux
res['bin_err'] = bin_err
res['bin_area'] = bin_area
if IS_ACS:
res['to_flam'] = to_flam
res['to_fnu'] = to_fnu
res['image_flux'] = image_flux[0::2, 0::2]*4
res['image_err'] = image_err[0::2, 0::2]*4
else:
res['to_flam'] = to_flam
res['to_fnu'] = to_fnu
res['image_flux'] = image_flux
res['image_err'] = image_err
res['bkg'] = bkg
res['bg_mask'] = bg_mask
data_tab = {}
data_tab['sci'] = sci_data
data_tab['err'] = np.sqrt(var_data)
data_tab['mask'] = mask
return res, data_tab
def spline_sfh():
"""
Generate a template set with spline-basis SFHs
"""
import os
import numpy as np
import matplotlib.pyplot as plt
from grizli.utils import bspline_templates
import eazy.sps
# Grid encompassing FSPS.log_age
step = 0.022
ages = 10**np.arange(np.log10(3.e-4), np.log10(14.12) + step, step)
ages = np.arange(3.e-4, 14.127, 0.001)
# Spline range between tmin, tmax
tmin, tmax, Ns = 0.02, 8, 7
# For Grid
tmin, tmax, Ns = 0.03, 9, 21
spl = bspline_templates(ages,
get_matrix=True,
log=True,
df=Ns,
clip=0.,
minmax=(np.log(tmin), np.log(tmax)))
Na = len(ages)
# Log-normals
centers = [0.03, 0.1, 0.3, 0.8, 1.4, 3., 8.]
widths = [0.21, 0.21, 0.21, 0.08, 0.08, 0.21, 0.21]
centers = [0.03, 0.10, 0.25, 0.50, 0.8, 1.6, 4.]
widths = [0.17, 0.17, 0.17, 0.09, 0.09, 0.17, 0.17]
# For grid
if 0:
centers = [
0.03, 0.065, 0.1, 0.2, 0.3, 0.55, 0.8, 1.1, 1.4, 2.2, 3., 5., 8.
]
widths = np.diff(np.log10(centers)) / 2.35
widths = np.append([widths[0]], widths)
Ng = len(centers)
gau = np.zeros((Na, Ng))
for i in range(Ng):
log_age = np.log10(ages / centers[i])
w = widths[i]
gau[:, i] = 1. / np.sqrt(2 * np.pi * w**2) * np.exp(
-log_age**2 / 2 / w**2)
spl = gau
age100 = ages <= 0.1
sp = eazy.sps.ExtendedFsps(logzsol=0,
zcontinuous=True,
add_neb_emission=True,
sfh=4)
sp.set_fir_template()
#sp.set_dust(dust_obj_type='KC13')
#sp.params['dust_index'] = 0. #-0.3
sp.set_dust(dust_obj_type='WG00x')
sp.params['imf_type'] = 1.
sp.get_spectrum(tage=0.1)
N = spl.shape[1]
vband = ["v"]
ssfr = np.zeros(N)
logz = np.array([-1., -0.5, 0., 0., 0., 0., 0.])
logu = np.array([-1.5, -2.0, -2.5, -2.5, -2.5, -2.5, -2.5])
dusts = [[0.02, 0.5], [0.02, 0.5, 1.0, 2.0, 3.0, 5.], [0.02, 1., 2.0],
[0.02, 1., 2.0], [0.02], [0.02], [0.02]]
# for i in [0,1,2,3,4]:
# dusts[i] = list(np.arange(0.2, 3.05-0.25*i, 0.1))
# Full grid
if N > 7:
dusts = [[
0.02, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 4., 5., 6., 7., 8., 10., 12
]] * N
logz = np.zeros(N)
logu = np.zeros(N) - 2.5
logz[:2] = -1.
logu[:2] = -1.5
bands = ['u', 'v', '2mass_j', 'i1500', 'i2800']
uvj_mags = np.zeros((N, len(bands)))
band_ages = np.zeros((N, len(bands)))
_ = sp.get_spectrum()
fsps_ages = 10**(sp.log_age - 9)
from grizli.utils_c.interp import interp_conserve_c
norm_sfh = spl.T * 0.
ysfh = np.ones((N, len(fsps_ages)))
# 30 Myr
t0 = 0.03
end_age = np.zeros(N)
end_clip = 1.e-3
sfh_interp = norm_sfh * 0.
for i in range(N):
sfh = spl.T[i, :] * 1.
sfh_t0 = np.interp(t0, ages, sfh)
#sfh[ages < t0] = sfh_t0
sfh_i = sfh / np.trapz(sfh, ages * 1.e9)
# Evaluate tage where sfr is down by some factor from the max
if sfh[-1] / sfh.max() > end_clip:
tstart = ages[-1]
else:
tstart = ages[sfh / sfh.max() > end_clip].max()
end_age[i] = tstart
#end_age[i] = 14.12 #ages.max()
# Constant SFR below t0 (30 Myr)
sfr0 = np.interp(t0, ages, sfh_i)
sfh_i[ages < t0] = sfr0
sfr_min = sfh_i.max() / 1000.
imax = np.argmax(sfh_i)
sfh_i[(sfh_i < sfr_min) & (ages < ages[imax])] = sfr_min
if 1:
# Now evaluate SFH in "forward" direction,rather than lookback
tforward = end_age[i] - ages[::-1]
clip = tforward > 0
sfh_i[~clip[::-1]] = sfr_min
norm_sfh[i, :] = sfh_i * 1.
sfh = np.interp(ages,
tforward,
norm_sfh[i, ::-1],
left=0.,
right=sfr0) #, integrate=0)
sfh[sfh == 0] = sfr_min
sfh_interp[i, :] = sfh
plt.plot(ages, sfh)
else:
sfh_interp[i, :] = sfh_i[::-1]
norm_sfh[i, :] = sfh_i * 1.
plt.plot(ages, sfh_i)
sp.params['sfh'] = 3
sp.set_tabular_sfh(ages, sfh_interp[i, :])
sp.params['logzsol'] = logz[i]
sp.params['gas_logz'] = logz[i]
sp.params['gas_logu'] = logu[i]
sp.params['compute_light_ages'] = True
band_ages[i, :] = sp.get_mags(tage=end_age[i], bands=bands)
sp.params['compute_light_ages'] = False
uvj_mags[i, :] = sp.get_mags(tage=end_age[i], bands=bands)
ssfr[i] = sp.sfr100 / sp.stellar_mass
if False:
#plt.figure()
plt.plot(uvj_mags[:, 1] - uvj_mags[:, 2],
uvj_mags[:, 0] - uvj_mags[:, 1],
color='w',
alpha=1.,
zorder=100)
plt.plot(uvj_mags[:, 1] - uvj_mags[:, 2],
uvj_mags[:, 0] - uvj_mags[:, 1],
color='0.5',
alpha=0.5,
zorder=101)
plt.scatter(uvj_mags[:, 1] - uvj_mags[:, 2],
uvj_mags[:, 0] - uvj_mags[:, 1],
c=np.log10(np.maximum(ssfr, 1.e-12)),
vmin=-12,
vmax=-8,
zorder=1000,
marker='s',
s=100,
edgecolor='w')
### Make all templates
res = eazy.filters.FilterFile('FILTER.RES.latest')
uvj_res = [res[153], res[155], res[161]]
breakme = False
templates = []
sp.params['dust2'] = 0.
sp.params['sfh'] = 3.
sp.params['compute_light_ages'] = False
kwargs = {'scale_lyman_series': 0.1}
plt.figure()
for i in range(N):
#sp.set_tabular_sfh(ages, sfh_interp[i,:])
sp.set_tabular_sfh(ages, sfh_interp[i, :])
sp.params['logzsol'] = logz[i]
sp.params['gas_logz'] = logz[i]
sp.params['gas_logu'] = logu[i]
ssfr_i = sp.sfr100 / sp.stellar_mass
for Av in dusts[i]:
templ = sp.get_full_spectrum(tage=end_age[i],
Av=Av,
get_template=True,
set_all_templates=False,
**kwargs)
templ.ageV = band_ages[i, 1]
jflux = templ.integrate_filter(uvj_res[2], flam=True)
templates.append(templ)
plt.plot(templ.wave,
templ.flux / jflux,
label=templ.name,
alpha=0.8)
if (i == 0) & breakme:
break
plt.loglog()
fig = plt.figure()
plt.scatter(vj[sel],
uv[sel],
c=np.log10(zout['SFR'] / zout['mass'])[sel],
vmin=-12,
vmax=-8)
uvj = np.array(
[templ.integrate_filter_list(uvj_res) for templ in templates])
ssfr_temp = np.array([
templ.meta['sfr100'] / templ.meta['stellar_mass']
for templ in templates
])
plt.scatter(-2.5 * np.log10(uvj[:, 1] / uvj[:, 2]),
-2.5 * np.log10(uvj[:, 0] / uvj[:, 1]),
c=np.log10(ssfr_temp),
vmin=-12,
vmax=-8,
marker='s',
edgecolor='w',
s=100)
# Try NMF decomposition
if False:
from sklearn.decomposition import NMF
model = NMF(n_components=5,
init='nndsvda',
random_state=0,
verbose=True,
solver='cd',
tol=1.e-6,
alpha=0.01,
beta_loss='frobenius',
max_iter=100000)
clip = (templ.wave > 2500) & (templ.wave < 1.6e4)
X = [
templ.flux / (uvj[i, 1] * 3.e18 / 5500**2)
for i, templ in enumerate(templates)
]
dust_array = np.hstack(dusts)
ix = np.array([1, 3, 4, 5, 6, 9, 10, 12, 13])
ix = np.where((dust_array > 0.1) & (dust_array < 4))[0]
W = model.fit_transform(np.array(X)[ix, clip].T)
H = model.components_
uvjr = H.dot((uvj.T / uvj[:, 1]).T[ix, :])
plt.scatter(-2.5 * np.log10(uvj[ix, 1] / uvj[ix, 2]),
-2.5 * np.log10(uvj[ix, 0] / uvj[ix, 1]),
c=np.log10(ssfr_temp)[ix],
vmin=-12,
vmax=-8,
marker='s',
edgecolor='r',
s=100)
plt.scatter(-2.5 * np.log10(uvjr[:, 1] / uvjr[:, 2]),
-2.5 * np.log10(uvjr[:, 0] / uvjr[:, 1]),
vmin=-12,
vmax=-8,
marker='v',
edgecolor='w',
s=100,
alpha=0.8)
# Write templates
if N > 8:
param_file = 'templates/spline_templates/spline.grid.param'
else:
param_file = 'templates/spline_templates/spline.param'
fp = open(param_file, 'w')
for i, templ in enumerate(templates):
tab = templ.to_table()
tab.meta['ageV'] = templ.ageV
name = 'spline_age{0:4.2f}_av{1:3.1f}'.format(tab.meta['ageV'],
tab.meta['Av'])
line = f'{i+1} templates/spline_templates/{name}.fits 1.0'
print(line)
fp.write(line + '\n')
tab.write(f'templates/spline_templates/{name}.fits', overwrite=True)
fp.close()
# Metadata
cols = ('file', 'Av', 'mass', 'Lv', 'sfr', 'LIR', 'energy_abs', 'ageV')
rows = []
for i, templ in enumerate(templates):
vflux = templ.integrate_filter(uvj_res[1], flam=False)
Lv = vflux * 3.e18 / uvj_res[1].pivot
tab = templ.to_table()
name = 'spline_age{0:4.2f}_av{1:3.1f}'.format(templ.ageV,
templ.meta['Av'])
row = [
name, templ.meta['Av'], templ.meta['stellar_mass'], Lv,
templ.meta['sfr100'], templ.meta['energy_absorbed'],
templ.meta['energy_absorbed'], templ.ageV
]
rows.append(row)
par = utils.GTable(names=cols, rows=rows)
for line in ['Ha', 'O3', 'Hb', 'O2', 'Lya']:
par['line_flux_' + line] = 0.
par['line_C_' + line] = 0.
par['line_EW_' + line] = 0.
par.write(param_file + '.fits', overwrite=True)
# color space spanned by random draws
NT = len(templates)
rnd = 10**(np.random.rand(1000 * NT, NT) - 3)
for i in range(NT):
rnd[i * 1000:(i + 1) * 1000, i] = 1.
uvjr = rnd.dot((uvj.T / uvj[:, 0]).T)
plt.scatter(-2.5 * np.log10(uvjr[:, 1] / uvjr[:, 2]),
-2.5 * np.log10(uvjr[:, 0] / uvjr[:, 1]),
c='k',
marker='.',
alpha=0.1)
# Interpolation not working?
for i in range(N):
sp.set_tabular_sfh(ages, sfh_interp[i, :])
#sp.set_tabular_sfh(fsps_ages, ysfh[i,:])
_ = sp.get_spectrum()
if 0:
# Show in lookback time
plt.plot(ages, norm_sfh[i, :], color='r', alpha=0.5)
plt.plot(end_age[i] - 10**(sp.log_age - 9),
sp.sfr,
color='k',
alpha=0.5)
plt.plot(end_age[i] - 10**(sp.log_age - 9),
ysfh[i, :],
color='g',
alpha=0.5)
plt.vlines(end_age[i], 1.e-30, 10, color='b', alpha=0.5)
sp.get_spectrum(tage=end_age[i], zmet=None)
plt.scatter(0.01, sp.sfr, marker='o', color='b')
else:
_ = sp.get_spectrum()
plt.plot(10**(sp.log_age - 9), sp.sfr, color='k', alpha=0.5)
plt.plot(end_age[i] - ages, norm_sfh[i, :], color='r', alpha=0.5)
plt.vlines(end_age[i], 1.e-30, 10, color='b', alpha=0.5)
sp.get_spectrum(tage=end_age[i], zmet=None)
plt.scatter(end_age[i], sp.sfr, marker='o', color='b')
#_ = sp.get_spectrum()
#plt.plot(10**(sp.log_age-9), sp.sfr/np.interp(end_age[i], fsps_ages, sp.sfr)*np.interp(end_age[i], end_age[i]-ages[::-1], norm_sfh[i,::-1]), alpha=0.5)
#####################
# Do light-weighted ages add linearly? - YES!
ij = [1, 3]
coeffs = np.array([0.2, 0.5])
sfh = sfh_i * 0.
for k in range(len(ij)):
i = ij[k]
sfh += coeffs[k] * spl.T[i, :] / np.trapz(spl.T[i, :], ages * 1.e9)
sp.set_tabular_sfh(13 - ages[::-1], sfh[::-1])
sp.params['compute_light_ages'] = True
combined_age = sp.get_mags(tage=13, bands=vband)
vflux = 10**(-0.4 * uvj_mags[ij, 1])
c_norm = (coeffs * vflux)
c_norm /= c_norm.sum()
coeffs_age = vband_ages[ij].dot(c_norm)
print(combined_age, coeffs_age)
def irac_mosaics(root='j000308m3303', home='/GrizliImaging/', pixfrac=0.2, kernel='square', initial_pix=1.0, final_pix=0.5, pulldown_mag=15.2, sync_xbcd=True, skip_fetch=False, radec=None, mosaic_pad=2.5, drizzle_ref_file='', run_alignment=True, assume_close=True, bucket='grizli-v1', aor_query='r*', mips_ext='[_e]bcd.fits', channels=['ch1','ch2','ch3','ch4','mips1'], drz_query='r*', sync_results=True, ref_seg=None, min_frame={'irac':5, 'mips':1.0}, med_max_size=500e6, stop_at='', make_psf=True, **kwargs):
"""
stop_at: preprocess, make_compact
"""
from grizli import utils
from . import irac
from .utils import get_wcslist, fetch_irac
PATH = os.path.join(home, root)
try:
os.mkdir(PATH)
except:
pass
os.chdir(PATH)
if not skip_fetch:
# Fetch IRAC bcds
if not os.path.exists(f'{root}_ipac.fits'):
os.system(f'wget https://s3.amazonaws.com/{bucket}/IRAC/{root}_ipac.fits')
res = fetch_irac(root=root, path='./', channels=channels)
if res in [False, None]:
# Nothing to do
make_html(root, bucket=bucket)
print(f'### Done: \n https://s3.amazonaws.com/{bucket}/Pipeline/{root}/IRAC/{root}.irac.html')
utils.log_comment(f'/tmp/{root}.success', 'Done!',
verbose=True, show_date=True)
return True
# Sync CHArGE HST images
os.system(f'aws s3 sync s3://{bucket}/Pipeline/{root}/Prep/ ./ '
f' --exclude "*" --include "{root}*seg.fits*"'
f' --include "{root}-ir_drz*fits*"'
f' --include "{root}*psf.fits*"'
f' --include "{root}-f[01]*_drz*fits.gz"'
f' --include "{root}*phot.fits"')
# Drizzle properties of the preliminary mosaic
#pixfrac, pix, kernel = 0.2, 1.0, 'square'
# Define an output WCS aligned in pixel phase to the HST mosaic ()
if not os.path.exists('ref_hdu.fits'):
wcslist = get_wcslist(skip=-500)
out_hdu = utils.make_maximal_wcs(wcslist, pixel_scale=initial_pix, theta=0, pad=5, get_hdu=True, verbose=True)
# Make sure pixels align
ref_file = glob.glob('{0}-f[01]*_drz_sci.fits*'.format(root))
if len(ref_file) == 0:
os.system(f'aws s3 sync s3://{bucket}/Pipeline/{root}/Prep/ ./ '
f' --exclude "*"'
f' --include "{root}-f[678]*_dr*fits.gz"')
ref_file = glob.glob('{0}-f[678]*_dr*_sci.fits*'.format(root))
ref_file = ref_file[-1]
print(f'\nHST reference image: {ref_file}\n')
ref_hdu = pyfits.open(ref_file)[0].header
ref_filter = utils.get_hst_filter(ref_hdu).lower()
ref_wcs = pywcs.WCS(ref_hdu)
ref_rd = ref_wcs.all_pix2world(np.array([[-0.5, -0.5]]), 0).flatten()
target_phase = np.array([0.5, 0.5])#/(pix/0.1)
for k in ['RADESYS', 'LATPOLE', 'LONPOLE']:
out_hdu.header[k] = ref_hdu[k]
# Shift CRVAL to same tangent point
out_wcs = pywcs.WCS(out_hdu.header)
out_xy = out_wcs.all_world2pix(np.array([ref_wcs.wcs.crval]), 1).flatten()
out_hdu.header['CRVAL1'], out_hdu.header['CRVAL2'] = tuple(ref_wcs.wcs.crval)
out_hdu.header['CRPIX1'], out_hdu.header['CRPIX2'] = tuple(out_xy)
# Align integer pixel phase
out_wcs = pywcs.WCS(out_hdu.header)
out_xy = out_wcs.all_world2pix(np.array([ref_rd]), 0).flatten()
xy_phase = out_xy - np.floor(out_xy)
new_crpix = out_wcs.wcs.crpix - (xy_phase - target_phase)
out_hdu.header['CRPIX1'], out_hdu.header['CRPIX2'] = tuple(new_crpix)
out_wcs = pywcs.WCS(out_hdu.header)
out_hdu.writeto('ref_hdu.fits', output_verify='Fix')
else:
out_hdu = pyfits.open('ref_hdu.fits')[1]
########
files = []
for ch in channels:
if 'mips' in ch:
mc = ch.replace('mips','ch')
files += glob.glob(f'{aor_query}/{mc}/bcd/SPITZER_M*{mips_ext}')
files += glob.glob(f'{aor_query}/{mc}/bcd/SPITZER_M*xbcd.fits.gz')
else:
files += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*cbcd.fits')
files += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*xbcd.fits.gz')
files.sort()
roots = np.array([file.split('/')[0] for file in files])
with_channels = np.array([file.split('_')[1] for file in files])
all_roots = np.array(['{0}-{1}'.format(r, c.replace('I','ch').replace('M', 'mips')) for r, c in zip(roots, with_channels)])
tab = {'aor':[], 'N':[], 'channel':[]}
for r in np.unique(all_roots):
tab['aor'].append(r.split('-')[0])
tab['N'].append((all_roots == r).sum())
tab['channel'].append(r.split('-')[1])
aors = utils.GTable(tab)
print(aors)
########
SKIP = True # Don't regenerate finished files
delete_group = False # Delete intermediate products from memory
zip_outputs = False # GZip intermediate products
aors_ch = {}
########
# Process mosaics by AOR
# Process in groups, helps for fields like HFF with dozens/hundreds of AORs!
for ch in channels:
aor = aors[(aors['channel'] == ch) & (aors['N'] > 5)]
if len(aor) == 0:
continue
#aors_ch[ch] = []
if ch in ['ch1','ch2']:
NPER, instrument = 500, 'irac'
if ch in ['ch3','ch4']:
NPER, instrument = 500, 'irac'
elif ch in ['mips1']:
NPER, instrument = 400, 'mips'
min_frametime = min_frame[instrument]
nsort = np.cumsum(aor['N']/NPER)
NGROUP = int(np.ceil(nsort.max()))
count = 0
for g in range(NGROUP):
root_i = root+'-{0:02d}'.format(g)
gsel = (nsort > g) & (nsort <= g+1)
aor_ids = list(aor['aor'][gsel])
print('{0}-{1} N_AOR = {2:>2d} N_EXP = {3:>4d}'.format(root_i, ch, len(aor_ids), aor['N'][gsel].sum()))
count += gsel.sum()
files = glob.glob('{0}-{1}*'.format(root_i, ch))
if (len(files) > 0) & (SKIP):
print('Skip {0}-{1}'.format(root_i, ch))
continue
with open('{0}-{1}.log'.format(root_i, ch),'w') as fp:
fp.write(time.ctime())
# Do internal alignment to GAIA.
# Otherwise, set `radec` to the name of a file that has two columns with
# reference ra/dec.
#radec = None
# Pipeline
if instrument == 'mips':
aors_ch[ch] = irac.process_all(channel=ch.replace('mips','ch'), output_root=root_i, driz_scale=initial_pix, kernel=kernel, pixfrac=pixfrac, wcslist=None, pad=0, out_hdu=out_hdu, aor_ids=aor_ids, flat_background=False, two_pass=True, min_frametime=min_frametime, instrument=instrument, align_threshold=0.15, radec=radec, run_alignment=False, mips_ext=mips_ext, ref_seg=ref_seg, global_mask=root+'_mask.reg')
else:
aors_ch[ch] = irac.process_all(channel=ch, output_root=root_i, driz_scale=initial_pix, kernel=kernel, pixfrac=pixfrac, wcslist=None, pad=0, out_hdu=out_hdu, aor_ids=aor_ids, flat_background=False, two_pass=True, min_frametime=min_frametime, instrument=instrument, radec=radec, run_alignment=run_alignment, assume_close=assume_close, ref_seg=ref_seg, global_mask=root+'_mask.reg', med_max_size=med_max_size)
if len(aors_ch[ch]) == 0:
continue
# PSFs
plt.ioff()
if (instrument != 'mips') & make_psf:
ch_num = int(ch[-1])
segmask=True
# psf_size=20
# for p in [0.1, final_pix]:
# irac.mosaic_psf(output_root=root_i, target_pix=p, channel=ch_num, aors=aors_ch[ch], kernel=kernel, pixfrac=pixfrac, size=psf_size, native_orientation=False, instrument=instrument, subtract_background=False, segmentation_mask=segmask, max_R=10)
# plt.close('all')
psf_size=30
p = 0.1
irac.mosaic_psf(output_root=root_i, target_pix=p, channel=ch_num, aors=aors_ch[ch], kernel=kernel, pixfrac=pixfrac, size=psf_size, native_orientation=True, subtract_background=False, segmentation_mask=segmask, max_R=10)
plt.close('all')
if delete_group:
del(aors_ch[ch])
print('Done {0}-{1}, gzip products'.format(root_i, ch))
if zip_outputs:
os.system('gzip {0}*-{1}_drz*fits'.format(root_i, ch))
# PSFs
if (instrument != 'mips') & make_psf:
# Average PSF
p = 0.1
files = glob.glob('*{0}-{1:.1f}*psfr.fits'.format(ch, p))
if len(files) == 0:
continue
files.sort()
avg = None
for file in files:
im = pyfits.open(file)
if avg is None:
wht = im[0].data != 0
avg = im[0].data*wht
else:
wht_i = im[0].data != 0
avg += im[0].data*wht_i
wht += wht_i
im.close()
avg = avg/wht
avg[wht == 0] = 0
# Window
from photutils import (HanningWindow, TukeyWindow,
CosineBellWindow,
SplitCosineBellWindow, TopHatWindow)
coswindow = CosineBellWindow(alpha=1)
avg *= coswindow(avg.shape)**0.05
avg /= avg.sum()
pyfits.writeto('{0}-{1}-{2:0.1f}.psfr_avg.fits'.format(root, ch, p), data=avg, header=im[0].header, overwrite=True)
####
## Show the initial product
plt.ioff()
for i in range(10):
files = glob.glob(f'{root}-{i:02d}-ch*sci.fits')
if len(files) > 0:
break
files.sort()
if len(files) == 1:
subs = 1,1
fs = [7,7]
elif len(files) == 2:
subs = 1,2
fs = [14,7]
elif len(files) == 3:
subs = 2,2
fs = [14,14]
else:
subs = 2,2
fs = [14,14]
fig = plt.figure(figsize=fs)
for i, file in enumerate(files[:4]):
im = pyfits.open(file)
print('{0} {1} {2:.1f} s'.format(file, im[0].header['FILTER'], im[0].header['EXPTIME']))
ax = fig.add_subplot(subs[0], subs[1], 1+i)
ax.imshow(im[0].data, vmin=-0.1, vmax=1, cmap='gray_r', origin='lower')
ax.text(0.05, 0.95, file, ha='left', va='top', color='k',
transform=ax.transAxes)
im.close()
if len(files) > 1:
fig.axes[1].set_yticklabels([])
if len(files) > 2:
fig.axes[0].set_xticklabels([])
fig.axes[1].set_xticklabels([])
if len(files) > 3:
fig.axes[3].set_yticklabels([])
fig.tight_layout(pad=0.5)
fig.savefig(f'{root}.init.png')
plt.close('all')
if stop_at == 'preprocess':
return True
#######
# Make more compact individual exposures and clean directories
wfiles = []
for ch in channels:
if 'mips' in ch:
chq = ch.replace('mips','ch')
wfiles += glob.glob(f'{aor_query}/{chq}/bcd/SPITZER_M*wcs.fits')
else:
wfiles += glob.glob(f'{aor_query}/{ch}/bcd/SPITZER_I*wcs.fits')
#wfiles = glob.glob('r*/*/bcd/*_I[1-4]_*wcs.fits')
#wfiles += glob.glob('r*/*/bcd/*_M[1-4]_*wcs.fits')
wfiles.sort()
for wcsfile in wfiles:
outfile = wcsfile.replace('_wcs.fits', '_xbcd.fits.gz')
if os.path.exists(outfile):
print(outfile)
else:
irac.combine_products(wcsfile)
print('Run: ', outfile)
if os.path.exists(outfile):
remove_files = glob.glob('{0}*fits'.format(wcsfile.split('_wcs')[0]))
for f in remove_files:
print(' rm ', f)
os.remove(f)
if stop_at == 'make_compact':
return True
#############
# Drizzle final mosaics
# Make final mosaic a bit bigger than the HST image
pad = mosaic_pad
# Pixel scale of final mosaic.
# Don't make too small if not many dithers available as in this example.
# But for well-sampled mosaics like RELICS / HFF, can push this to perhaps 0.3" / pix
pixscale = final_pix #0.5
# Again, if have many dithers maybe can use more aggressive drizzle parameters,
# like a 'point' kernel or smaller pixfrac (a 'point' kernel is pixfrac=0)
#kernel, pixfrac = 'square', 0.2
# Correction for bad columns near bright stars
#pulldown_mag = 15.2
##############
# Dilation for CR rejection
dil = np.ones((3,3))
driz_cr = [7, 4]
blot_interp = 'poly5'
bright_fmax = 0.5
### Drizzle
for ch in channels: #[:2]:
###########
# Files and reference image for extra CR rejection
if ch == 'mips1':
files = glob.glob('{0}/ch1/bcd/SPITZER_M1_*xbcd.fits*'.format(drz_query, ch))
files.sort()
pulldown_mag = -10
pixscale = 1.
kernel = 'point'
else:
files = glob.glob('{0}/{1}/bcd/*_I?_*xbcd.fits*'.format(drz_query, ch))
files.sort()
#ref = pyfits.open('{0}-00-{1}_drz_sci.fits'.format(root, ch))
#ref_data = ref[0].data.astype(np.float32)
ref_files = glob.glob(f'{root}-??-{ch}*sci.fits')
if len(ref_files) == 0:
continue
num = None
for ref_file in ref_files:
ref = pyfits.open(ref_file)
wht = pyfits.open(ref_file.replace('_sci.fits', '_wht.fits'))
if num is None:
num = ref[0].data*wht[0].data
den = wht[0].data
else:
num += ref[0].data*wht[0].data
den += wht[0].data
ref_data = (num/den).astype(np.float32)
ref_data[den <= 0] = 0
ref_wcs = pywcs.WCS(ref[0].header, relax=True)
ref_wcs.pscale = utils.get_wcs_pscale(ref_wcs)
if (not hasattr(ref_wcs, '_naxis1')) & hasattr(ref_wcs, '_naxis'):
ref_wcs._naxis1, ref_wcs._naxis2 = ref_wcs._naxis
##############
# Output WCS based on HST footprint
if drizzle_ref_file == '':
try:
hst_im = pyfits.open(glob.glob('{0}-f[01]*_drz_sci.fits*'.format(root))[-1])
except:
hst_im = pyfits.open(glob.glob('{0}-f[578]*_dr*sci.fits*'.format(root))[-1])
hst_wcs = pywcs.WCS(hst_im[0])
hst_wcs.pscale = utils.get_wcs_pscale(hst_wcs)
try:
size = (np.round(np.array([hst_wcs._naxis1, hst_wcs._naxis2])*hst_wcs.pscale*pad/pixscale)*pixscale)
except:
size = (np.round(np.array([hst_wcs._naxis[0], hst_wcs._naxis[1]])*hst_wcs.pscale*pad/pixscale)*pixscale)
hst_rd = hst_wcs.calc_footprint().mean(axis=0)
_x = utils.make_wcsheader(ra=hst_rd[0], dec=hst_rd[1],
size=size,
pixscale=pixscale,
get_hdu=False, theta=0)
out_header, out_wcs = _x
else:
driz_ref_im = pyfits.open(drizzle_ref_file)
out_wcs = pywcs.WCS(driz_ref_im[0].header, relax=True)
out_wcs.pscale = utils.get_wcs_pscale(out_wcs)
out_header = utils.to_header(out_wcs)
if (not hasattr(out_wcs, '_naxis1')) & hasattr(out_wcs, '_naxis'):
out_wcs._naxis1, out_wcs._naxis2 = out_wcs._naxis
##############
# Bright stars for pulldown correction
cat_file = glob.glob(f'{root}-[0-9][0-9]-{ch}.cat.fits')[0]
ph = utils.read_catalog(cat_file)
bright = (ph['mag_auto'] < pulldown_mag) # & (ph['flux_radius'] < 3)
ph = ph[bright]
##############
# Now do the drizzling
yp, xp = np.indices((256, 256))
orig_files = []
out_header['DRIZ_CR0'] = driz_cr[0]
out_header['DRIZ_CR1'] = driz_cr[1]
out_header['KERNEL'] = kernel
out_header['PIXFRAC'] = pixfrac
out_header['NDRIZIM'] = 0
out_header['EXPTIME'] = 0
out_header['BUNIT'] = 'microJy'
out_header['FILTER'] = ch
med_root = 'xxx'
N = len(files)
for i, file in enumerate(files):#[:100]):
print('{0}/{1} {2}'.format(i, N, file))
if file in orig_files:
continue
im = pyfits.open(file)
ivar = 1/im['CBUNC'].data**2
msk = (~np.isfinite(ivar)) | (~np.isfinite(im['CBCD'].data))
im['CBCD'].data[msk] = 0
ivar[msk] = 0
wcs = pywcs.WCS(im['WCS'].header, relax=True)
wcs.pscale = utils.get_wcs_pscale(wcs)
if (not hasattr(wcs, '_naxis1')) & hasattr(wcs, '_naxis'):
wcs._naxis1, wcs._naxis2 = wcs._naxis
fp = Path(wcs.calc_footprint())
med_root_i = im.filename().split('/')[0]
if med_root != med_root_i:
print('\n Read {0}-{1}_med.fits \n'.format(med_root_i, ch))
med = pyfits.open('{0}-{1}_med.fits'.format(med_root_i, ch))
med_data = med[0].data.astype(np.float32)
med_root = med_root_i
med.close()
try:
gaia_rd = utils.read_catalog('{0}-{1}_gaia.radec'.format(med_root_i, ch))
ii, rr = gaia_rd.match_to_catalog_sky(ph)
gaia_rd = gaia_rd[ii][rr.value < 2]
gaia_pts = np.array([gaia_rd['ra'].data,
gaia_rd['dec'].data]).T
except:
gaia_rd = []
#data = im['CBCD'].data - aor_med[0].data
# Change output units to uJy / pix
if ch == 'mips1':
# un = 1*u.MJy/u.sr
# #to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(out_wcs.pscale**2)
# to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(native_scale**2)
to_ujy_px = 146.902690
else:
# native_scale = 1.223
# un = 1*u.MJy/u.sr
# #to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(out_wcs.pscale**2)
# to_ujy_px = un.to(u.uJy/u.arcsec**2).value*(native_scale**2)
to_ujy_px = 35.17517196810
blot_data = ablot.do_blot(ref_data, ref_wcs, wcs, 1, coeffs=True,
interp=blot_interp,
sinscl=1.0, stepsize=10,
wcsmap=None)/to_ujy_px
# mask for bright stars
eblot = 1-np.clip(blot_data, 0, bright_fmax)/bright_fmax
# Initial CR
clean = im[0].data - med_data - im['WCS'].header['PEDESTAL']
dq = (clean - blot_data)*np.sqrt(ivar)*eblot > driz_cr[0]
# Adjacent CRs
dq_dil = binary_dilation(dq, selem=dil)
dq |= ((clean - blot_data)*np.sqrt(ivar)*eblot > driz_cr[1]) & (dq_dil)
# Very negative pixels
dq |= clean*np.sqrt(ivar) < -4
original_dq = im['WCS'].data - (im['WCS'].data & 1)
dq |= original_dq > 0
# Pulldown correction for bright stars
if len(gaia_rd) > 0:
mat = fp.contains_points(gaia_pts)
if mat.sum() > 0:
xg, yg = wcs.all_world2pix(gaia_rd['ra'][mat], gaia_rd['dec'][mat], 0)
sh = dq.shape
mat = (xg > 0) & (xg < sh[1]) & (yg > 0) & (yg < sh[0])
if mat.sum() > 0:
for xi, yi in zip(xg[mat], yg[mat]):
dq |= (np.abs(xp-xi) < 2) & (np.abs(yp-yi) > 10)
if i == 0:
res = utils.drizzle_array_groups([clean], [ivar*(dq == 0)], [wcs], outputwcs=out_wcs, kernel=kernel, pixfrac=pixfrac, data=None, verbose=False)
# Copy header keywords
wcs_header = utils.to_header(wcs)
for k in im[0].header:
if (k not in ['', 'HISTORY', 'COMMENT']) & (k not in out_header) & (k not in wcs_header):
out_header[k] = im[0].header[k]
else:
_ = utils.drizzle_array_groups([clean], [ivar*(dq == 0)], [wcs], outputwcs=out_wcs, kernel=kernel, pixfrac=pixfrac, data=res[:3], verbose=False)
out_header['NDRIZIM'] += 1
out_header['EXPTIME'] += im[0].header['EXPTIME']
im.close()
# Pixel scale factor for weights
wht_scale = (out_wcs.pscale/wcs.pscale)**-4
# Write final images
pyfits.writeto('{0}-{1}_drz_sci.fits'.format(root, ch), data=res[0]*to_ujy_px, header=out_header,
output_verify='fix', overwrite=True)
pyfits.writeto('{0}-{1}_drz_wht.fits'.format(root, ch), data=res[1]*wht_scale/to_ujy_px**2,
header=out_header, output_verify='fix', overwrite=True)
##########
## Show the final drizzled images
plt.ioff()
files = glob.glob(f'{root}-ch*sci.fits')
files.sort()
if len(files) == 1:
subs = 1,1
fs = [7,7]
elif len(files) == 2:
subs = 1,2
fs = [14,7]
elif len(files) == 3:
subs = 2,2
fs = [14,14]
else:
subs = 2,2
fs = [14,14]
fig = plt.figure(figsize=fs)
for i, file in enumerate(files[:4]):
im = pyfits.open(file)
print('{0} {1} {2:.1f} s'.format(file, im[0].header['FILTER'], im[0].header['EXPTIME']))
ax = fig.add_subplot(subs[0], subs[1], 1+i)
scl = (final_pix/initial_pix)**2
ax.imshow(im[0].data, vmin=-0.1*scl, vmax=1*scl, cmap='gray_r', origin='lower')
ax.text(0.05, 0.95, file, ha='left', va='top', color='k',
transform=ax.transAxes)
im.close()
if len(files) > 1:
fig.axes[1].set_yticklabels([])
if len(files) > 2:
fig.axes[0].set_xticklabels([])
fig.axes[1].set_xticklabels([])
if len(files) > 3:
fig.axes[3].set_yticklabels([])
fig.tight_layout(pad=0.5)
fig.savefig(f'{root}.final.png')
plt.close('all')
if sync_results:
print('gzip mosaics')
os.system(f'gzip -f {root}-ch*_drz*fits {root}-mips*_drz*fits')
######## Sync
## Sync
print(f's3://{bucket}/Pipeline/{root}/IRAC/')
make_html(root, bucket=bucket)
os.system(f'aws s3 sync ./ s3://{bucket}/Pipeline/{root}/IRAC/'
f' --exclude "*" --include "{root}-ch*drz*fits*"'
f' --include "{root}-mips*drz*fits*"'
f' --include "{root}.*png"'
' --include "*-ch*psf*" --include "*log.fits"'
' --include "*wcs.[lp]*"'
' --include "*html" --include "*fail*"'
' --acl public-read')
if sync_xbcd:
aor_files = glob.glob('r*-ch*med.fits')
for aor_file in aor_files:
aor = aor_file.split('-ch')[0]
os.system(f'aws s3 sync ./{aor}/ s3://{bucket}/IRAC/AORS/{aor}/ --exclude "*" --include "ch*/bcd/*xbcd.fits.gz" --acl public-read')
os.system(f'aws s3 cp {aor_file} s3://{bucket}/IRAC/AORS/ --acl public-read')
msg = f'### Done: \n https://s3.amazonaws.com/{bucket}/Pipeline/{root}/IRAC/{root}.irac.html'
utils.log_comment(f'/tmp/{root}.success', msg, verbose=True, show_date=True)
