def make_visit_fits():
import glob
import numpy as np
from grizli import utils
visit_files = glob.glob('[egu]*visits.npy')
visit_files.sort()
indiv_files = glob.glob('j*visits.npy')
indiv_files.sort()
visit_files += indiv_files
for p in ['grizli-v1-19.12.04_visits.npy', 'grizli-v1-19.12.05_visits.npy', 'grizli-cosmos-v2_visits.npy']:
if p in visit_files:
visit_files.pop(visit_files.index(p))
all_visits = []
products = []
for extra in ['candels-july2019_visits.npy', 'grizli-cosmos-v2_visits.npy']:
extra_visits = np.load(extra)[0]
extra_products = [v['product'] for v in extra_visits]
for i, p in enumerate(extra_products):
if p not in products:
parent = p.split('_')[0]
print(parent, p)
v = extra_visits[i]
v['parent'] = parent
v['parent_file'] = extra # 'candels-july2019_visits.npy'
all_visits.append(v)
products.append(p)
# COSMOS footprint
cosmos_fp = None
for i, v in enumerate(extra_visits):
if v['product'].endswith('f814w'):
print(v['product'])
if cosmos_fp is None:
cosmos_fp = v['footprint'].buffer(1.e-6)
else:
cosmos_fp = cosmos_fp.union(v['footprint'])
for i, file in enumerate(visit_files):
visits, groups, info = np.load(file)
print(file, len(visits))
for v in visits:
has_fp = ('footprints' in v)
if not has_fp:
print('No footprint: {0}'.format(v['product']))
if has_fp & (v['product'] not in products):
all_visits.append(v)
v['parent'] = file.split("_visits")[0].split('-')[-1]
v['first'] = v['files'][0]
v['parent_file'] = file
products.append(v['product'])
for v in all_visits:
v['filter'] = v['product'].split('-')[-1]
v['first'] = v['files'][0]
# File dictionary
all_files = []
file_products = []
for v in all_visits:
all_files.extend(v['files'])
file_products.extend([v['product']]*len(v['files']))
# duplicates?? seem to be in GOODS-S.
# Exclude them in all but the first product that contains them for now
if True:
_un = np.unique(all_files, return_counts=True, return_index=True, return_inverse=True)
un_file, un_index, un_inv, un_count = _un
dup = un_count > 1
dup_files = un_file[dup]
for file in dup_files:
prods = list(np.array(file_products)[np.array(all_files) == file])
for prod in prods[1:]:
i = products.index(prod)
v = all_visits[i]
j = v['files'].index(file)
print(file, v['parent'], prod, i, j)
pj = all_visits[i]['files'].pop(j)
pj = all_visits[i]['footprints'].pop(j)
if 'awspath' in all_visits[i]:
pj = all_visits[i]['awspath'].pop(j)
#print(file, prods[-1])
# WFC3/IR copied to "Exposures" paths in CANDELS fields
for v in all_visits:
if v['parent_file'] == 'grizli-cosmos-v2_visits.npy':
continue
if v['parent_file'].startswith('j'):
v['awspath'] = ['grizli-v1/Pipeline/{0}/Prep'.format(v['parent']) for f in v['files']]
if v['filter'].startswith('f0') | v['filter'].startswith('f1'):
# print(v['product'])
v['awspath'] = ['grizli-v1/Exposures/{0}/{1}'.format(f[:4], f.split('_')[0]) for f in v['files']]
# Empty visits, seems to be from duplicates above and mostly in CANDELS
nexp = np.array([len(visit['files']) for visit in all_visits])
for i in np.where(nexp == 0)[0][::-1]:
v_i = all_visits.pop(i)
print(i, v_i['product'])
products.pop(i)
tab = utils.GTable()
for k in ['parent', 'product', 'filter', 'first']:
tab[k] = [visit[k] for visit in all_visits]
coo = np.array([np.array(visit['footprint'].centroid.xy).flatten() for visit in all_visits])
tab['ra'] = coo[:, 0]
tab['dec'] = coo[:, 1]
tab['nexp'] = [len(visit['files']) for visit in all_visits]
tab['bounds'] = [np.array(v['footprint'].bounds) for v in all_visits]
root = 'candels-july2019'
root = 'candels-sep2019'
root = 'grizli-v1-19.12.04'
root = 'grizli-v1-19.12.05'
tab.write(root+'_visits.fits', overwrite=True)
np.save(root+'_visits.npy', [all_visits])
# os.system('echo "# In https://s3.amazonaws.com/grizli-v1/Mosaics/" > candels-july2019.files.txt; ls candels-july2019* |grep -v files.txt >> candels-july2019.files.txt')
#os.system('aws s3 sync --exclude "*" --include "candels-july2019*" --include "grizli-v1-19.12.04*" ./ s3://grizli-v1/Mosaics/ --acl public-read')
os.system('echo "# In https://s3.amazonaws.com/grizli-v1/Mosaics/" > {0}.files.txt; ls {0}* |grep -v files.txt >> {0}.files.txt'.format(root))
os.system('aws s3 sync --exclude "*" --include "{0}*" ./ s3://grizli-v1/Mosaics/ --acl public-read'.format(root))
if False:
from shapely.geometry import Point
candels = utils.column_values_in_list(tab['parent'], ['j141956p5255', 'j123656p6215', 'j033236m2748', 'j021732m0512', 'j100012p0210'])
cosmos = np.array([v['footprint'].intersection(cosmos_fp).area > 0 for v in all_visits])
extra = candels
extra = ~(candels | cosmos)
# Area
filter_polys = {}
filt = 'f160w'
for filt in np.unique(tab['filter']):
print(filt)
if filt in filter_polys:
print(filt)
continue
poly = None
count = 0
# Dec strips
di = np.arange(-90, 91, 5)
strips = []
for i in range(len(di)-1):
strip = (tab['dec'] > di[i]) & (tab['dec'] <= di[i+1]) & (tab['filter'] == filt)
strip &= extra
if strip.sum() == 0:
continue
indices = np.arange(len(tab))[strip]
poly = None
for j in indices:
v = all_visits[j]
if v['filter'] != filt:
continue
# for fp in v['footprints']:
for fp in [v['footprint']]:
count += 1
#print(i, v['product'], count)
if poly is None:
poly = fp.buffer(1.e-6)
else:
poly = poly.union(fp.buffer(1.e-6))
poly.dec = di[i]+2.5
strips.append(poly)
if len(strips) == 0:
filter_polys[filt] = Point(0, 0).buffer(1.e-6)
continue
full = strips[0].buffer(1.e-6)
for strip in strips[1:]:
full = full.union(strip.buffer(1.e-6))
filter_polys[filt] = full
optical = filter_polys['f606w'].union(filter_polys['f814w'])
optical = optical.union(filter_polys['f850lp'])
optical = optical.union(filter_polys['f775w'])
yband = filter_polys['f098m'].union(filter_polys['f105w'])
visy = optical.union(yband)
jband = filter_polys['f125w']
jband = jband.union(filter_polys['f110w'])
hband = filter_polys['f140w'].union(filter_polys['f160w'])
filter_polys[r'$\mathrm{opt} = i_{775} | i_{814} | z_{850}$'] = optical
filter_polys[r'$\mathrm{opty} = \mathrm{opt} | Y$'] = visy
filter_polys[r'$Y = y_{098 } | y_{105}$'] = yband
filter_polys[r'$J = j_{110} | j_{125}$'] = jband
filter_polys[r'$H = h_{140} | h_{160}$'] = hband
ydrop = visy.intersection(jband)
ydrop = ydrop.intersection(hband)
filter_polys[r'$Y-\mathrm{drop} = (\mathrm{opt} | Y) + J + H$'] = ydrop
yj = yband.union(jband)
jdrop = yj.intersection(hband)
filter_polys[r'$J-\mathrm{drop} = (Y | J) + H$'] = jdrop
for filt in filter_polys:
full = filter_polys[filt]
try:
areas = [f.area*np.cos(np.array(f.centroid.xy).flatten()[1]/180*np.pi) for f in full]
except:
try:
areas = [f.area*np.cos(np.array(f.centroid.xy).flatten()[1]/180*np.pi) for f in [full]]
except:
areas = [0]
full.total_area = np.sum(areas)
print(filt, filter_polys[filt].total_area)
ta = utils.GTable()
ta['filter'] = [f.upper() for f in filter_polys]
ta['area'] = [filter_polys[f].total_area*3600 for f in filter_polys]
ta['area'].format = '.0f'
# Compare areas
h = fields['a_wfc3_ir_f160w'] > 0
for root, aa in zip(fields['field_root'][h], fields['a_wfc3_ir_f160w'][h]):
sel = (tab['filter'] == 'f160w') & (tab['parent'] == root)
if sel.sum() > 0:
indices = np.where(sel)[0]
a = all_visits[indices[0]]['footprint'].buffer(1.e-6)
for i in indices:
a = a.union(all_visits[i]['footprint'])
a_i = a.area*3600*np.cos(tab['dec'][indices[0]]/180*np.pi)
print(root, aa, a_i, a_i/aa)
def segmentation_figure(label, cat, segfile):
"""
Make a figure showing a cutout of the segmentation file
"""
import matplotlib.pyplot as plt
import numpy as np
import astropy.io.fits as pyfits
import astropy.wcs as pywcs
from grizli import utils
plt.ioff()
seg = pyfits.open(segfile)
seg_data = seg[0].data
seg_wcs = pywcs.WCS(seg[0].header)
# Randomize seg to get dispersion between neighboring objects
np.random.seed(hash(label.split('_')[0]) % (10 ** 8))
rnd_ids = np.append([0], np.argsort(np.random.rand(len(cat)))+1)
# Make cutout
th = pyfits.open('{0}.thumb.fits'.format(label), mode='update')
th_wcs = pywcs.WCS(th[0].header)
blot_seg = utils.blot_nearest_exact(seg_data, seg_wcs, th_wcs,
stepsize=-1, scale_by_pixel_area=False)
rnd_seg = rnd_ids[np.cast[int](blot_seg)]*1.
th_ids = np.unique(blot_seg)
sh = th[0].data.shape
yp, xp = np.indices(sh)
thumb_height = 2.
fig = plt.figure(figsize=[thumb_height*sh[1]/sh[0], thumb_height])
ax = fig.add_subplot(111)
rnd_seg[rnd_seg == 0] = np.nan
ax.imshow(rnd_seg, aspect='equal', cmap='terrain_r',
vmin=-0.05*len(cat), vmax=1.05*len(cat))
ax.set_xticklabels([])
ax.set_yticklabels([])
ix = utils.column_values_in_list(cat['number'], th_ids)
xc, yc = th_wcs.all_world2pix(cat['ra'][ix], cat['dec'][ix], 0)
xc = np.clip(xc, 0.09*sh[1], 0.91*sh[1])
yc = np.clip(yc, 0.08*sh[0], 0.92*sh[0])
for th_id, x_i, y_i in zip(cat['number'][ix], xc, yc):
if th_id == 0:
continue
ax.text(x_i, y_i, '{0:.0f}'.format(th_id), ha='center', va='center', fontsize=8, color='w')
ax.text(x_i, y_i, '{0:.0f}'.format(th_id), ha='center', va='center', fontsize=8, color='k', alpha=0.95)
ax.set_xlim(0, sh[1]-1)
ax.set_ylim(0, sh[0]-1)
ax.set_axis_off()
fig.tight_layout(pad=0.01)
fig.savefig('{0}.seg.png'.format(label))
plt.close(fig)
# Append to thumbs file
seg_hdu = pyfits.ImageHDU(data=np.cast[int](blot_seg), name='SEG')
if 'SEG' in th:
th.pop('SEG')
th.append(seg_hdu)
th.writeto('{0}.thumb.fits'.format(label), overwrite=True,
output_verify='fix')
th.close()