def database_from_query(query):
engine = db.get_db_engine()
utils.set_warnings()
result = db.from_sql(query, engine)
return result
def all_fields():
from grizli.aws import db
engine = db.get_db_engine()
ch = db.from_sql(
"select * from charge_fields where log like 'Finish%%' and field_ra > 0 and filters like '%%F1%%' and nfilt > 1 and field_root like 'j%%'",
engine)
so = np.argsort(ch['field_root'])
with open('fields.txt', 'w') as fp:
for r in ch['field_root'][so]:
fp.write(f'{r}\n')
bash = """
roots=`cat fields.txt | shuf`
for root in ${roots}; do
if [ ! -e "./log/${root}.log" ]; then
date > ./log/${root}.log
rm -rf ${root}/output/catalog_assets ${root}/output/js
python make_fitsmap.py ${root}
rm ${root}/*fits*
fi
done
"""
import json
def reproc():
js = json.load(open('CHArGE-Mar2020.json.bkup'))
count = 0
for row in js['data']:
root = row[0]
map_href = f"<a href=https://s3.amazonaws.com/grizli-v1/Pipeline/{root}/Map/index.html>{root}</a>"
if os.path.exists(f'log/{root}.log'):
row[0] = map_href
count += 1
print(f'Count: {count} / {len(js["data"])}')
with open('CHArGE-Mar2020.json', 'w') as fp:
json.dump(js, fp)
os.system(
'aws s3 cp CHArGE-Mar2020.json s3://grizli-v1/Master/ --acl public-read'
)
os.system(
'aws s3 cp CHArGE-Mar2020.json.bkup s3://grizli-v1/Master/ --acl public-read'
)
def load_grizli_database(db_name = db_name, db_word = db_word, db_base = db_base):
"""
load AWS interface for getting stuff
"""
# readonly DB access
config = {'hostname': 'grizdbinstance.c3prl6czsxrm.us-east-1.rds.amazonaws.com',
'username': db_name,
'password': db_word,
'database': db_base,
'port': 5432}
# sqlalchemy engine for postgresql
engine = grizli_db.get_db_engine(config=config)
#print('Tables: ', engine.table_names())
return engine
def get_visit_files():
import boto3
from grizli.aws import db
engine = db.get_db_engine()
fields = db.from_sql("select field_root, a_wfc3 from charge_fields where log LIKE 'Finished%%'", engine=engine)
s3 = boto3.resource('s3')
bkt = s3.Bucket('grizli-v1')
for i, field in enumerate(fields['field_root']):
s3_file = '{0}_visits.npy'.format(field)
if not os.path.exists(s3_file):
s3_path = 'Pipeline/{0}/Prep'.format(field)
try:
bkt.download_file(s3_path+'/'+s3_file, s3_file,
ExtraArgs={"RequestPayer": "requester"})
print(i, s3_file)
except:
print(i, 'Download failed: {0}'.format(field))
def full_hawki_query(rd=None, query_result=None, eso=None):
"""
Query all HAWKI observations....
"""
import os
import numpy as np
import matplotlib.pyplot as plt
from shapely.geometry import Polygon, Point
from descartes import PolygonPatch
from shapely import affinity
from grizli import utils
from mastquery import query, overlaps
if eso is None:
eso = get_eso()
if query_result is None:
_, kwargs, res = full_query(eso=eso)
else:
kwargs, res = query_result
# surveys = 092.A-0472
# CHArGE fields
from grizli.aws import db
import astropy.units as u
from astropy.coordinates import SkyCoord
engine = db.get_db_engine()
if rd is None:
ch = db.from_sql(
"SELECT field_root, field_ra as ra, field_dec as dec, log FROM charge_fields where log LIKE '%%Finish%%'",
engine)
else:
ra, dec = rd
ch = utils.GTable()
ch['ra'] = [ra]
ch['dec'] = [dec]
ch['field_root'] = [
utils.radec_to_targname(
ra=ra,
dec=dec,
round_arcsec=(4, 60),
precision=2,
targstr='j{rah}{ram}{ras}{sign}{ded}{dem}',
header=None,
)
]
idx, dr = ch.match_to_catalog_sky(res)
has_hawki = dr < 10 * u.arcmin
import scipy.spatial
ch_rd = SkyCoord(ch['ra'], ch['dec'], unit='deg')
ch_xyz = ch_rd.cartesian.get_xyz().value
ctree = scipy.spatial.cKDTree(ch_xyz.T)
hawki_rd = SkyCoord(res['RA'], res['DEC'], unit='deg')
hawki_xyz = hawki_rd.cartesian.get_xyz().value
htree = scipy.spatial.cKDTree(hawki_xyz.T)
r = 30. / 60 / 360. * 2
tr = ctree.query_ball_tree(htree, r)
n_hawki = np.array([len(t) for t in tr])
# Figures
idx = np.where(n_hawki > 0)[0]
xsize = 5
px, py = 0.45, 0.2
for i in idx:
field = ch['field_root'][i]
print(i, field)
if os.path.exists(f'{field}_hawki.png'):
continue
field = ch['field_root'][i]
#tab = utils.read_catalog(f'../FieldsSummary/{field}_footprint.fits')
if os.path.exists(f'{field}_footprint.fits'):
tab = utils.read_catalog(f'{field}_footprint.fits')
meta = tab.meta
xr = (meta['XMIN'], meta['XMAX'])
yr = (meta['YMIN'], meta['YMAX'])
ra, dec = meta['BOXRA'], meta['BOXDEC']
cosd = np.cos(dec / 180 * np.pi)
dx = (xr[1] - xr[0]) * cosd * 60
dy = (yr[1] - yr[0]) * 60
box_width = np.maximum(dx, dy)
#query_size = np.maximum(min_size, box_width/2)/60.
p_hst = None
p_ir = None
for j, fph in enumerate(tab['footprint']):
ps, is_bad, poly = query.instrument_polygon(tab[j])
if not hasattr(ps, '__len__'):
ps = [ps]
for p in ps:
p_j = Polygon(p).buffer(0.001)
if p_hst is None:
p_hst = p_j
else:
p_hst = p_hst.union(p_j)
if tab['instrument_name'][j] == 'WFC3/IR':
if p_ir is None:
p_ir = p_j
else:
p_ir = p_ir.union(p_j)
else:
cosd = np.cos(dec / 180 * np.pi)
p_hst = None
p_ir = None
##############################
fig = plt.figure(figsize=[6, 6])
ax = fig.add_subplot(111)
ax.scatter(ra, dec, zorder=1000, marker='+', color='k')
# HAWKI
h_p = None
for j in tr[i]:
p = Point(res['RA'][j], res['DEC'][j]).buffer(4.1 / 60)
p = affinity.scale(p, xfact=1. / cosd)
# ax.add_patch(PolygonPatch(p, color='r', alpha=0.1))
x, y = p.boundary.xy
ax.plot(x, y, color=utils.MPL_COLORS['r'], alpha=0.05)
if h_p is None:
h_p = p
else:
h_p = h_p.union(p)
# If overlap between hawki and HST, query all exposures
if p_hst is not None:
hawki_overlap = h_p.intersection(p_hst)
hawki_un = h_p.union(p_hst)
if not hasattr(p_hst, '__len__'):
p_hst = [p_hst]
if not hasattr(h_p, '__len__'):
h_p = [h_p]
for p in p_hst:
#ax.add_patch(PolygonPatch(p, color='k', alpha=0.2))
if not hasattr(p.boundary, '__len__'):
bs = [p.boundary]
else:
bs = p.boundary
for b in bs:
x, y = b.xy
ax.plot(x, y, color=utils.MPL_COLORS['gray'], alpha=0.3)
else:
hawki_overlap = h_p
if not hasattr(h_p, '__len__'):
h_p = [h_p]
if p_ir is not None:
if not hasattr(p_ir, '__len__'):
p_ir = [p_ir]
for p in p_ir:
ax.add_patch(
PolygonPatch(p, color=utils.MPL_COLORS['gray'], alpha=0.2))
x, y = p.boundary.xy
ax.plot(x, y, color=utils.MPL_COLORS['gray'], alpha=0.3)
for p in h_p:
ax.add_patch(
PolygonPatch(p, color=utils.MPL_COLORS['r'], alpha=0.2))
targets = [
'{0} {1}'.format(res['ProgId'][j], res['Object'][j])
for j in tr[i]
]
for j, targ in enumerate(np.unique(targets)):
ixj = np.where(np.array(targets) == targ)[0]
expt = res['DET NDIT'] * res['DET DIT'] * res['TPL NEXP']
ax.text(0.02,
0.98 - j * 0.03,
'{0} {1:.1f}'.format(targ, expt[tr[i]][ixj].sum() / 3600.),
ha='left',
va='top',
transform=ax.transAxes,
fontsize=7)
ax.set_aspect(1. / cosd)
ax.set_title(field)
ax.grid()
#xsize = 4
dx = np.diff(ax.get_xlim())[0] * cosd * 60
dy = np.diff(ax.get_ylim())[0] * 60
fig.set_size_inches(xsize * np.clip(dx / dy, 0.2, 5) + px, xsize + py)
ax.set_xlim(ax.get_xlim()[::-1])
overlaps.draw_axis_labels(ax=ax, nlabel=3)
fig.tight_layout(pad=0.5)
fig.savefig(f'{field}_hawki.png', dpi=120)
plt.close('all')
if (hawki_overlap.area >
0.0) & (not os.path.exists(f'{field}_hawki.fits')):
kws = {}
for k in kwargs:
kws[k] = kwargs[k].copy()
kws['column_filters'].pop('tpl_nexp')
kws['column_filters'].pop('tpl_expno')
_res = eso.query_instrument('hawki',
pi_coi_name='PI_only',
coord1=ra,
coord2=dec,
box='00 30 00',
**kws)
if len(_res) > 0:
print('{0} datasets'.format(len(_res)))
_res['PI'] = [p.split('/')[0].strip() for p in _res['PI/CoI']]
_res.write(f'{field}_hawki.fits', overwrite=True)
def run_root(root='j002532m1223', min_zoom=2, get_grism=True):
"""
Prepare images for fitsmap.convert
"""
from grizli.pipeline import auto_script
from grizli import utils
import eazy.utils
from fitsmap import convert
print('sync')
os.system(
f'aws s3 sync s3://grizli-v1/Pipeline/{root}/Prep/ {root}/ --exclude "*" --include "*sci.fits.gz" --include "*phot.fits" --include "*seg.fits.gz"'
)
os.system(
f'aws s3 sync s3://grizli-v1/Pipeline/{root}/IRAC/ {root}/ --exclude "*" --include "*sci.fits*" --include "*model.fits"'
)
os.system(
f'aws s3 sync s3://grizli-v1/Pipeline/{root}/Map/ {root}/ --exclude "*" --include "{root}.*png"'
)
os.chdir(root)
if not os.path.exists(f'{root}.rgb.png'):
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=True,
HOME_PATH=None,
gzext='*',
suffix='.rgb',
output_format='png')
# IR
files = glob.glob(f'{root}-[if][r01]*sci.fits*')
files.sort()
filts = [file.split(f'{root}-')[1].split('_')[0] for file in files]
for filt in filts:
if os.path.exists(f'{root}.{filt}.png'):
continue
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=True,
HOME_PATH=None,
gzext='*',
filters=[filt],
suffix=f'.{filt}',
output_format='png',
invert=True,
scl=2)
# Optical, 2X pix
files = glob.glob(f'{root}-[f][2-8]*sci.fits*')
files.sort()
filts = [file.split(f'{root}-')[1].split('_')[0] for file in files]
for filt in filts:
if os.path.exists(f'{root}.{filt}.png'):
continue
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=2,
HOME_PATH=None,
gzext='*',
filters=[filt],
suffix=f'.{filt}',
output_format='png',
invert=True,
scl=2)
# Spitzer
if glob.glob(f'{root}-ch*fits*'):
import reproject
out_img = pyfits.open(f'{root}-ir_drz_sci.fits.gz')
repr_hdu = out_img[0]
# repr_hdu = utils.make_maximal_wcs([out_wcs], pixel_scale=0.2,
# verbose=False, pad=0, poly_buffer=0)
repr_wcs = pywcs.WCS(repr_hdu.header)
mosaics = glob.glob(f'{root}-ch[12]*sci.fits*')
mosaics.sort()
for mos in mosaics:
ch = mos.split(f'{root}-')[1].split('_')[0]
if os.path.exists(f'{root}.{ch}.png'):
continue
print(f'Reproject {ch}')
in_img = pyfits.open(mos)
in_wcs = pywcs.WCS(in_img[0].header)
reproj = utils.blot_nearest_exact(in_img[0].data,
in_wcs,
repr_wcs,
scale_by_pixel_area=False)
pyfits.writeto(f'{root}-{ch}s_drz_sci.fits',
data=reproj,
header=repr_hdu.header,
overwrite=True)
ext = [ch + 's']
if os.path.exists(f'{root}-{ch}_model.fits'):
# resid
print(f' {ch} model')
in_img = pyfits.open(f'{root}-{ch}_model.fits')
reproj = utils.blot_nearest_exact(in_img[1].data,
in_wcs,
repr_wcs,
scale_by_pixel_area=False)
pyfits.writeto(f'{root}-{ch}m_drz_sci.fits',
data=reproj,
header=repr_hdu.header,
overwrite=True)
ext.append(ch + 'm')
for filt in ext:
_ = auto_script.field_rgb(root=root,
xsize=6,
full_dimensions=True,
HOME_PATH=None,
gzext='',
filters=[filt],
suffix=f'.{filt}',
output_format='png',
invert=True,
scl=2)
if not os.path.exists(f'{root}.seg.png'):
sfig = make_seg(f'{root}-ir_seg.fits.gz', outfile=f'{root}.seg.png')
filelist = []
for q in ['*f[2-8]', '*f[01]*', '*ir*', '*ch[12]', '*seg', '*rgb']:
l_i = glob.glob(q + '*png')
l_i.sort()
filelist.extend(l_i)
ph = utils.read_catalog(f'{root}_phot.fits')
ph['id'] = ph['number']
ph['ra'].format = '.6f'
ph['dec'].format = '.6f'
ph['mag'] = ph['mag_auto']
ph['mag'].format = '.2f'
ph['query'] = [
eazy.utils.query_html(r, d).split(') ')[1]
for r, d in zip(ph['ra'], ph['dec'])
]
ph['id', 'ra', 'dec', 'query', 'mag'].write('phot.cat',
format='ascii.csv',
overwrite=True)
filelist += ['phot.cat']
if get_grism:
from grizli.aws import db
engine = db.get_db_engine()
gr = db.from_sql(
f"select root, id, ra, dec, z_map from redshift_fit where root='{root}'",
engine)
print(f'grism.cat: {len(gr)} sources')
if len(gr) > 0:
gr['query'] = [
eazy.utils.query_html(r, d).split(') ')[1]
for r, d in zip(gr['ra'], gr['dec'])
]
gr['stack'] = [
f'<img src="https://s3.amazonaws.com/grizli-v1/Pipeline/{root}/Extractions/{root}_{id:05d}.stack.png" height="100px"/>'
for id in gr['id']
]
gr['full'] = [
f'<img src="https://s3.amazonaws.com/grizli-v1/Pipeline/{root}/Extractions/{root}_{id:05d}.full.png" height="100px"/>'
for id in gr['id']
]
gr['line'] = [
f'<img src="https://s3.amazonaws.com/grizli-v1/Pipeline/{root}/Extractions/{root}_{id:05d}.line.png" height="80px"/>'
for id in gr['id']
]
gr['ra'].format = '.6f'
gr['dec'].format = '.6f'
gr['z_map'].format = '.4f'
gr['id', 'ra', 'dec', 'query', 'z_map', 'stack', 'full',
'line'].write('grism.cat', format='ascii.csv', overwrite=True)
filelist += ['grism.cat']
convert.MPL_CMAP = 'gray_r'
convert.cartographer.MARKER_HTML_WIDTH = '650px'
convert.cartographer.MARKER_HTML_HEIGHT = '440px'
convert.POPUP_CSS = [
"span { text-decoration:underline; font-weight:bold; line-height:12pt; }",
"tr { line-height: 7pt; }",
"table { width: 100%; }",
"img { height: 100px; width: auto; }",
]
convert.dir_to_map("./",
filelist=filelist,
out_dir="output",
cat_wcs_fits_file=f"{root}-ir_drz_sci.fits.gz",
catalog_delim=',',
min_zoom=min_zoom,
task_procs=False,
image_engine='MPL')
plt.close('all')
if os.path.exists('output/index.html'):
os.system(
f'aws s3 sync output/ s3://grizli-v1/Pipeline/{root}/Map/ --acl public-read --quiet'
)
os.system(
f'aws s3 sync ./ s3://grizli-v1/Pipeline/{root}/Map/ --exclude "*" --include "{root}.*png" --acl public-read'
)
import astropy.units as u
import numpy as np
import pandas as pd
from astropy.cosmology import WMAP9
from astropy.table import Table
from grizli import utils
from grizli.aws import db
from grizli.fitting import compute_cdf_percentiles
from pylab import rcParams
from scipy import stats
from scipy.spatial import cKDTree
from tqdm import tqdm
from .plotting import Plotter
engine = db.get_db_engine()
utils.set_warnings()
rcParams['figure.figsize'] = 12, 8
class PDTable(Table):
"""
Table populated by the values returned from a query of the Grizli database.
Contains methods to calculate the projected density of the given field.
"""
@classmethod
def from_query(cls, query):
"""
Generate a `PDTable` from a database query.