def group_by_filter():
"""
aws s3 sync --exclude "*" --include "cosmos_visits*" s3://grizli-preprocess/CosmosMosaic/ ./
"""
from grizli import prep, utils
import numpy as np
master = 'cosmos'
master = 'grizli-cosmos-v2'
master = 'grizli-jan2019'
master = 'grizli-v1-19.12.04'
master = 'grizli-v1-19.12.05'
tab = utils.read_catalog('{0}_visits.fits'.format(master))
all_visits = np.load('{0}_visits.npy'.format(master), allow_pickle=True)[0]
# By filter
# Exclude DASH
dash = utils.column_string_operation(tab['product'], 'icxe', 'startswith')
dash |= utils.column_string_operation(tab['product'], '_icxe',
'count', 'or')
# Don't exclude DASH
dash = utils.column_string_operation(tab['product'], 'xxxx', 'startswith')
groups = {}
fpstr = {}
for filt in np.unique(tab['filter']):
mat = (tab['filter'] == filt) & (~dash)
groups[filt] = {'filter': filt, 'files': [], 'awspath': [], 'footprints': []}
fpstr[filt] = 'fk5\n'
for ix in np.where(mat)[0]:
fp = all_visits[ix]['footprint']
if hasattr(fp, '__len__'):
fps = fp
else:
fps = [fp]
for fp in fps:
xy = fp.boundary.xy
pstr = 'polygon('+','.join(['{0:.6f}'.format(i) for i in np.array([xy[0].tolist(), xy[1].tolist()]).T.flatten()])+') # text={{{0}}}\n'.format(all_visits[ix]['product'])
fpstr[filt] += pstr
for k in ['files', 'awspath', 'footprints']:
groups[filt][k].extend(all_visits[ix][k])
fp = open('{0}-pointings-{1}.reg'.format(master, filt), 'w')
fp.write(fpstr[filt])
fp.close()
print('{0:6} {1:>3d} {2:>4d} ({3:>4d})'.format(filt, mat.sum(), len(groups[filt]['files']), len(np.unique(groups[filt]['files']))))
np.save('{0}_filter_groups.npy'.format(master), [groups])
def auto_run(root='j023507-040202'):
import os
import glob
import matplotlib.pyplot as plt
from grizli import utils, prep
from grizli.pipeline import auto_script, photoz
utils.set_warnings()
tab = utils.GTable.gread('{0}_footprint.fits'.format(root))
HOME_PATH = os.getcwd()
auto_script.VALID_FILTERS = ['F098M', 'F105W', 'F110W', 'F125W', 'F127M', 'F139M', 'F140W', 'F153M', 'F160W', 'F410M', 'F435W', 'F438W', 'F439W', 'F450W', 'F467M', 'F475W', 'F475X', 'F547M', 'F550M', 'F555W', 'F569W', 'F600LP', 'F606W', 'F621M', 'F622W', 'F625W', 'F675W', 'F689M', 'F702W', 'F763M', 'F775W', 'F791W', 'F814W', 'F845M', 'F850LP', 'F350LP']
# Include grisms
auto_script.VALID_FILTERS += ['G102','G141','G800L']
IS_PARALLEL = utils.column_string_operation(tab['proposal_pi'], 'alkan', method='count', logical='or').sum() > 0
IS_DASH = list(np.unique(np.cast[int](tab['proposal_id']))) == [14114]
master_radec = '{0}/{1}_master.radec'.format(os.getcwd(), root)
if not os.path.exists(master_radec):
if root.startswith('cos-') & os.path.exists('hsc-udeep-i25_corr_cosmos.radec'):
master_radec = '{0}/{1}'.format(os.getcwd(), 'hsc-udeep-i25_corr_cosmos.radec')
else:
master_radec = None
parent_radec = '{0}/{1}_parent.radec'.format(os.getcwd(), root)
if not os.path.exists(parent_radec):
parent_radec = None
BKG_PARAMS = {'bw': 1024, 'bh': 1024, 'fw': 3, 'fh': 3}
catalogs = ['PS1','DES','NSC', 'SDSS','GAIA','WISE']
# MACS0717
if 'macs0717' in root:
catalogs = ['GAIA', 'PS1']
run_fine_alignment=False
auto_script.go(root=root, maglim=[16, 23], HOME_PATH=HOME_PATH,
inspect_ramps=False, manual_alignment=False,
is_parallel_field=IS_PARALLEL, reprocess_parallel=True,
only_preprocess=True, run_extractions=True, run_fit=False,
s3_sync='cp', catalogs=catalogs,
fine_radec=None, run_fine_alignment=run_fine_alignment,
combine_all_filters=False,
align_mag_limits=[16,24],
gaia_by_date=True, align_simple=False, align_clip=120,
master_radec=master_radec, parent_radec=parent_radec,
is_dash=IS_DASH, run_parse_visits=True,
reference_wcs_filters=['F160W','F140W','F125W','F105W','F110W','F098M','F814W','F850LP', 'F606W','F435W'],
make_phot=False, make_mosaics=False, align_rms_limit=4,
align_min_overlap=2, imaging_bkg_params=BKG_PARAMS)
def auto_run(root='j023507-040202'):
import os
import matplotlib.pyplot as plt
from grizli import utils
from grizli.pipeline import auto_script, photoz
utils.set_warnings()
tab = utils.GTable.gread('{0}_footprint.fits'.format(root))
HOME_PATH = os.getcwd()
auto_script.VALID_FILTERS = [
'F098M', 'F105W', 'F110W', 'F125W', 'F127M', 'F139M', 'F140W', 'F153M',
'F160W', 'F410M', 'F435W', 'F438W', 'F439W', 'F450W', 'F467M', 'F475W',
'F475X', 'F547M', 'F550M', 'F555W', 'F569W', 'F600LP', 'F606W',
'F621M', 'F622W', 'F625W', 'F675W', 'F689M', 'F702W', 'F763M', 'F775W',
'F791W', 'F814W', 'F845M', 'F850LP', 'F350LP'
]
IS_PARALLEL = utils.column_string_operation(
tab['proposal_pi'], 'alkan', method='count', logical='or').sum() > 0
auto_script.go(root=root,
maglim=[19, 23],
HOME_PATH=HOME_PATH,
inspect_ramps=False,
manual_alignment=False,
is_parallel_field=IS_PARALLEL,
reprocess_parallel=False,
only_preprocess=True,
run_extractions=False,
run_fit=False,
s3_sync='cp',
fine_radec=None,
combine_all_filters=False,
gaia_by_date=True,
align_simple=False,
align_clip=100,
master_radec=None,
is_dash=False,
run_parse_visits=True,
reference_wcs_filters=[
'F160W', 'F140W', 'F125W', 'F105W', 'F110W', 'F098M',
'F814W', 'F850LP', 'F606W', 'F435W'
])
plt.ioff()
fig = auto_script.field_rgb(root=root, HOME_PATH=HOME_PATH, xsize=18)
plt.close(fig)
# Photo-z
try:
out = photoz.eazy_photoz(root,
object_only=False,
force=True,
aper_ix=1,
sys_err=0.05,
apply_prior=False,
beta_prior=True,
external_limits=3,
external_sys_err=0.3)
except:
pass