def show_all_thumbnails(label='j022708p4901_00273', filters=['vis','f098m','f105w','f110w','f125w','f140w','f160w'], scale_ab=21, close=True, thumb_height=2., rgb_params=RGB_PARAMS):
"""
Show individual filter and RGB thumbnails
"""
import glob
#from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import astropy.io.fits as pyfits
from astropy.visualization import make_lupton_rgb
from grizli.pipeline import auto_script
from grizli import utils
all_files = glob.glob('{0}-f*sci.fits'.format(label))
all_filters = [f.split('_dr')[0].split('-')[-1] for f in all_files]
ims = {}
for filter in filters:
drz_files = glob.glob('{0}-{1}*_dr*sci.fits'.format(label, filter))
if len(drz_files) > 0:
im = pyfits.open(drz_files[0])
ims[filter] = im
rgb_params['scale_ab'] = scale_ab
slx, sly, rgb_filts, fig = auto_script.field_rgb(root=label, HOME_PATH=None, **rgb_params) #xsize=4, output_dpi=None, HOME_PATH=None, show_ir=False, pl=1, pf=1, scl=1, rgb_scl=[1, 1, 1], ds9=None, force_ir=False, filters=all_filters, add_labels=False, output_format='png', rgb_min=-0.01, xyslice=None, pure_sort=False, verbose=True, force_rgb=None, suffix='.rgb', scale_ab=scale_ab)
if close:
plt.close()
filter_queries = {}
filter_queries['vis'] = '{0}-f[3-8]*sci.fits'.format(label)
filter_queries['y'] = '{0}-f[01][90][85]*sci.fits'.format(label)
filter_queries['j'] = '{0}-f1[12][05]*sci.fits'.format(label)
filter_queries['h'] = '{0}-f1[64]0*sci.fits'.format(label)
if 'vis' in filters:
drz_files = glob.glob('{0}-f[3-8]*sci.fits'.format(label))
drz_files.sort()
vis_filters = [f.split('_dr')[0].split('-')[-1] for f in drz_files]
if len(drz_files) > 0:
drz_files.sort()
for i, file in enumerate(drz_files[::-1]):
drz = pyfits.open(file)
wht = pyfits.open(file.replace('_sci','_wht'))
if i == 0:
photflam = drz[0].header['PHOTFLAM']
num = drz[0].data*wht[0].data
den = wht[0].data
drz_ref = drz
drz_ref[0].header['FILTER{0}'.format(i+1)] = utils.get_hst_filter(drz[0].header)
else:
scl = drz[0].header['PHOTFLAM']/photflam
num += drz[0].data*wht[0].data/scl
den += wht[0].data/scl**2
drz_ref[0].header['FILTER{0}'.format(i+1)] = utils.get_hst_filter(drz[0].header)
sci = num/den
sci[den == 0] = 0
drz_ref[0].data = sci
ims['vis'] = drz_ref
pyfits.writeto('{0}-{1}_drz_sci.fits'.format(label, 'vis'),
data=sci, header=drz_ref[0].header, overwrite=True,
output_verify='fix')
pyfits.writeto('{0}-{1}_drz_wht.fits'.format(label, 'vis'),
data=den, header=drz_ref[0].header, overwrite=True,
output_verify='fix')
#rgb = np.array(Image.open('{0}.rgb.png'.format(label)))
rgb = plt.imread('{0}.rgb.png'.format(label))
NX = (len(filters)+1)
fig = plt.figure(figsize=[thumb_height*NX, thumb_height])
ax = fig.add_subplot(1,NX,NX)
ax.imshow(rgb, origin='upper', interpolation='nearest')
ax.text(0.05, 0.95, label, ha='left', va='top', transform=ax.transAxes, fontsize=7, color='w', bbox=dict(facecolor='k', edgecolor='None', alpha=0.8))
ax.text(0.05, 0.05, ' '.join(rgb_filts), ha='left', va='bottom', transform=ax.transAxes, fontsize=6, color='w', bbox=dict(facecolor='k', edgecolor='None', alpha=0.8))
for i, filter in enumerate(filters):
if filter in ims:
zp_i = utils.calc_header_zeropoint(ims[filter], ext=0)
scl = 10**(-0.4*(zp_i-5-scale_ab))
img = ims[filter][0].data*scl
image = make_lupton_rgb(img, img, img, stretch=0.1, minimum=-0.01)
ax = fig.add_subplot(1,NX,i+1)
ax.imshow(255-image, origin='lower', interpolation='nearest')
if filter == 'vis':
ax.text(0.05, 0.95, '+'.join(vis_filters), ha='left', va='top', transform=ax.transAxes, fontsize=7, bbox=dict(facecolor='w', edgecolor='None', alpha=0.9))
else:
ax.text(0.05, 0.95, filter, ha='left', va='top', transform=ax.transAxes, fontsize=7, bbox=dict(facecolor='w', edgecolor='None', alpha=0.9))
for ax in fig.axes:
ax.set_xticklabels([])
ax.set_yticklabels([])
ax.set_xticks([])
ax.set_yticks([])
fig.tight_layout(pad=0.1)
fig.savefig('{0}.thumb.png'.format(label))
if close:
plt.close()
def auto_run(root='j023507-040202'):
import os
import matplotlib.pyplot as plt
from grizli import utils
from grizli.pipeline import auto_script, photoz
utils.set_warnings()
tab = utils.GTable.gread('{0}_footprint.fits'.format(root))
HOME_PATH = os.getcwd()
auto_script.VALID_FILTERS = [
'F098M', 'F105W', 'F110W', 'F125W', 'F127M', 'F139M', 'F140W', 'F153M',
'F160W', 'F410M', 'F435W', 'F438W', 'F439W', 'F450W', 'F467M', 'F475W',
'F475X', 'F547M', 'F550M', 'F555W', 'F569W', 'F600LP', 'F606W',
'F621M', 'F622W', 'F625W', 'F675W', 'F689M', 'F702W', 'F763M', 'F775W',
'F791W', 'F814W', 'F845M', 'F850LP', 'F350LP'
]
IS_PARALLEL = utils.column_string_operation(
tab['proposal_pi'], 'alkan', method='count', logical='or').sum() > 0
auto_script.go(root=root,
maglim=[19, 23],
HOME_PATH=HOME_PATH,
inspect_ramps=False,
manual_alignment=False,
is_parallel_field=IS_PARALLEL,
reprocess_parallel=False,
only_preprocess=True,
run_extractions=False,
run_fit=False,
s3_sync='cp',
fine_radec=None,
combine_all_filters=False,
gaia_by_date=True,
align_simple=False,
align_clip=100,
master_radec=None,
is_dash=False,
run_parse_visits=True,
reference_wcs_filters=[
'F160W', 'F140W', 'F125W', 'F105W', 'F110W', 'F098M',
'F814W', 'F850LP', 'F606W', 'F435W'
])
plt.ioff()
fig = auto_script.field_rgb(root=root, HOME_PATH=HOME_PATH, xsize=18)
plt.close(fig)
# Photo-z
try:
out = photoz.eazy_photoz(root,
object_only=False,
force=True,
aper_ix=1,
sys_err=0.05,
apply_prior=False,
beta_prior=True,
external_limits=3,
external_sys_err=0.3)
except:
pass
def show_all_thumbnails(label='j022708p4901_00273', filters=['visb', 'visr', 'y', 'j', 'h'], scale_ab=21, close=True, thumb_height=2., rgb_params=RGB_PARAMS, ext='png', xl=0.04, yl=0.98, fs=7):
"""
Show individual filter and RGB thumbnails
"""
import glob
#from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import astropy.io.fits as pyfits
from astropy.visualization import make_lupton_rgb
from grizli.pipeline import auto_script
from grizli import utils
all_files = glob.glob('{0}-f*sci.fits'.format(label))
all_filters = [f.split('_dr')[0].split('-')[-1] for f in all_files]
ims = {}
for filter in filters:
drz_files = glob.glob('{0}-{1}*_dr*sci.fits'.format(label, filter))
if len(drz_files) > 0:
im = pyfits.open(drz_files[0])
ims[filter] = im
rgb_params['scale_ab'] = scale_ab
slx, sly, rgb_filts, fig = auto_script.field_rgb(root=label, HOME_PATH=None, **rgb_params) # xsize=4, output_dpi=None, HOME_PATH=None, show_ir=False, pl=1, pf=1, scl=1, rgb_scl=[1, 1, 1], ds9=None, force_ir=False, filters=all_filters, add_labels=False, output_format='png', rgb_min=-0.01, xyslice=None, pure_sort=False, verbose=True, force_rgb=None, suffix='.rgb', scale_ab=scale_ab)
if close:
plt.close()
#rgb = np.array(Image.open('{0}.rgb.png'.format(label)))
rgb = plt.imread('{0}.rgb.png'.format(label))
NX = (len(filters)+1)
fig = plt.figure(figsize=[thumb_height*NX, thumb_height])
ax = fig.add_subplot(1, NX, NX)
ax.imshow(rgb, origin='upper', interpolation='nearest')
# ax.text(0.05, 0.95, label, ha='left', va='top', transform=ax.transAxes, fontsize=7, color='w', bbox=dict(facecolor='k', edgecolor='None', alpha=0.8))
# ax.text(0.05, 0.05, ' '.join(rgb_filts), ha='left', va='bottom', transform=ax.transAxes, fontsize=6, color='w', bbox=dict(facecolor='k', edgecolor='None', alpha=0.8))
for i, filter in enumerate(filters):
if filter in ims:
zp_i = utils.calc_header_zeropoint(ims[filter], ext=0)
scl = 10**(-0.4*(zp_i-5-scale_ab))
pixscl = utils.get_wcs_pscale(ims[filter][0].header.copy())
scl *= (0.06/pixscl)**2
img = ims[filter][0].data*scl
image = make_lupton_rgb(img, img, img, stretch=0.1, minimum=-0.01)
ax = fig.add_subplot(1, NX, i+1)
ax.imshow(255-image, origin='lower', interpolation='nearest')
for ax in fig.axes:
ax.set_xticklabels([])
ax.set_yticklabels([])
ax.set_xticks([])
ax.set_yticks([])
fig.tight_layout(pad=0.1)
# Add labels
#xl, yl = 0.04, 0.98
for i, filter in enumerate(filters):
if filter in ims:
if filter in ['uv', 'visb', 'visr', 'y', 'j', 'h']:
grouped_filters = []
h_i = ims[filter][0].header
for j in range(h_i['NCOMBINE']):
grouped_filters.append(h_i['CFILT{0}'.format(j+1)])
text_label = '+'.join(grouped_filters)
else:
text_label = filter
fig.text((i+xl)/NX, yl, text_label, fontsize=fs,
ha='left', va='top', transform=fig.transFigure,
bbox=dict(facecolor='w', edgecolor='None', alpha=0.9))
fig.text((i+1+xl)/NX, yl, label, ha='left', va='top', transform=fig.transFigure, fontsize=fs, color='w', bbox=dict(facecolor='k', edgecolor='None', alpha=0.8))
fig.text((i+1+0.04)/NX, 1-yl, ' '.join(rgb_filts), ha='left', va='bottom', transform=fig.transFigure, fontsize=fs, color='w', bbox=dict(facecolor='k', edgecolor='None', alpha=0.8))
fig.savefig('{0}.thumb.{1}'.format(label, ext))
if close:
plt.close()
return fig
def run_root(root='j002532m1223', min_zoom=2, get_grism=True):
"""
Prepare images for fitsmap.convert
"""
from grizli.pipeline import auto_script
from grizli import utils
import eazy.utils
from fitsmap import convert
print('sync')
os.system(
f'aws s3 sync s3://grizli-v1/Pipeline/{root}/Prep/ {root}/ --exclude "*" --include "*sci.fits.gz" --include "*phot.fits" --include "*seg.fits.gz"'
)
os.system(
f'aws s3 sync s3://grizli-v1/Pipeline/{root}/IRAC/ {root}/ --exclude "*" --include "*sci.fits*" --include "*model.fits"'
)
os.system(
f'aws s3 sync s3://grizli-v1/Pipeline/{root}/Map/ {root}/ --exclude "*" --include "{root}.*png"'
)
os.chdir(root)
if not os.path.exists(f'{root}.rgb.png'):
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=True,
HOME_PATH=None,
gzext='*',
suffix='.rgb',
output_format='png')
# IR
files = glob.glob(f'{root}-[if][r01]*sci.fits*')
files.sort()
filts = [file.split(f'{root}-')[1].split('_')[0] for file in files]
for filt in filts:
if os.path.exists(f'{root}.{filt}.png'):
continue
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=True,
HOME_PATH=None,
gzext='*',
filters=[filt],
suffix=f'.{filt}',
output_format='png',
invert=True,
scl=2)
# Optical, 2X pix
files = glob.glob(f'{root}-[f][2-8]*sci.fits*')
files.sort()
filts = [file.split(f'{root}-')[1].split('_')[0] for file in files]
for filt in filts:
if os.path.exists(f'{root}.{filt}.png'):
continue
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=2,
HOME_PATH=None,
gzext='*',
filters=[filt],
suffix=f'.{filt}',
output_format='png',
invert=True,
scl=2)
# Spitzer
if glob.glob(f'{root}-ch*fits*'):
import reproject
out_img = pyfits.open(f'{root}-ir_drz_sci.fits.gz')
repr_hdu = out_img[0]
# repr_hdu = utils.make_maximal_wcs([out_wcs], pixel_scale=0.2,
# verbose=False, pad=0, poly_buffer=0)
repr_wcs = pywcs.WCS(repr_hdu.header)
mosaics = glob.glob(f'{root}-ch[12]*sci.fits*')
mosaics.sort()
for mos in mosaics:
ch = mos.split(f'{root}-')[1].split('_')[0]
if os.path.exists(f'{root}.{ch}.png'):
continue
print(f'Reproject {ch}')
in_img = pyfits.open(mos)
in_wcs = pywcs.WCS(in_img[0].header)
reproj = utils.blot_nearest_exact(in_img[0].data,
in_wcs,
repr_wcs,
scale_by_pixel_area=False)
pyfits.writeto(f'{root}-{ch}s_drz_sci.fits',
data=reproj,
header=repr_hdu.header,
overwrite=True)
ext = [ch + 's']
if os.path.exists(f'{root}-{ch}_model.fits'):
# resid
print(f' {ch} model')
in_img = pyfits.open(f'{root}-{ch}_model.fits')
reproj = utils.blot_nearest_exact(in_img[1].data,
in_wcs,
repr_wcs,
scale_by_pixel_area=False)
pyfits.writeto(f'{root}-{ch}m_drz_sci.fits',
data=reproj,
header=repr_hdu.header,
overwrite=True)
ext.append(ch + 'm')
for filt in ext:
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=True,
HOME_PATH=None,
gzext='',
filters=[filt],
suffix=f'.{filt}',
output_format='png',
invert=True,
scl=2)
if not os.path.exists(f'{root}.seg.png'):
sfig = make_seg(f'{root}-ir_seg.fits.gz', outfile=f'{root}.seg.png')
filelist = []
for q in ['*f[2-8]', '*f[01]*', '*ir*', '*ch[12]', '*seg', '*rgb']:
l_i = glob.glob(q + '*png')
l_i.sort()
filelist.extend(l_i)
ph = utils.read_catalog(f'{root}_phot.fits')
ph['id'] = ph['number']
ph['ra'].format = '.6f'
ph['dec'].format = '.6f'
ph['mag'] = ph['mag_auto']
ph['mag'].format = '.2f'
ph['query'] = [
eazy.utils.query_html(r, d).split(') ')[1]
for r, d in zip(ph['ra'], ph['dec'])
]
ph['id', 'ra', 'dec', 'query', 'mag'].write('phot.cat',
format='ascii.csv',
overwrite=True)
filelist += ['phot.cat']
if get_grism:
from grizli.aws import db
engine = db.get_db_engine()
gr = db.from_sql(
f"select root, id, ra, dec, z_map from redshift_fit where root='{root}'",
engine)
print(f'grism.cat: {len(gr)} sources')
if len(gr) > 0:
gr['query'] = [
eazy.utils.query_html(r, d).split(') ')[1]
for r, d in zip(gr['ra'], gr['dec'])
]
gr['stack'] = [
f'<img src="https://s3.amazonaws.com/grizli-v1/Pipeline/{root}/Extractions/{root}_{id:05d}.stack.png" height="100px"/>'
for id in gr['id']
]
gr['full'] = [
f'<img src="https://s3.amazonaws.com/grizli-v1/Pipeline/{root}/Extractions/{root}_{id:05d}.full.png" height="100px"/>'
for id in gr['id']
]
gr['line'] = [
f'<img src="https://s3.amazonaws.com/grizli-v1/Pipeline/{root}/Extractions/{root}_{id:05d}.line.png" height="80px"/>'
for id in gr['id']
]
gr['ra'].format = '.6f'
gr['dec'].format = '.6f'
gr['z_map'].format = '.4f'
gr['id', 'ra', 'dec', 'query', 'z_map', 'stack', 'full',
'line'].write('grism.cat', format='ascii.csv', overwrite=True)
filelist += ['grism.cat']
convert.MPL_CMAP = 'gray_r'
convert.cartographer.MARKER_HTML_WIDTH = '650px'
convert.cartographer.MARKER_HTML_HEIGHT = '440px'
convert.POPUP_CSS = [
"span { text-decoration:underline; font-weight:bold; line-height:12pt; }",
"tr { line-height: 7pt; }",
"table { width: 100%; }",
"img { height: 100px; width: auto; }",
]
convert.dir_to_map("./",
filelist=filelist,
out_dir="output",
cat_wcs_fits_file=f"{root}-ir_drz_sci.fits.gz",
catalog_delim=',',
min_zoom=min_zoom,
task_procs=False,
image_engine='MPL')
plt.close('all')
if os.path.exists('output/index.html'):
os.system(
f'aws s3 sync output/ s3://grizli-v1/Pipeline/{root}/Map/ --acl public-read --quiet'
)
os.system(
f'aws s3 sync ./ s3://grizli-v1/Pipeline/{root}/Map/ --exclude "*" --include "{root}.*png" --acl public-read'
)