# take z4 and divide by 172 to put it in waves as it should be
data['z4'] /= 172.
return data
if __name__ == '__main__':
import matplotlib.pyplot as plt
from WavefrontPSF.wavefront import Wavefront
from WavefrontPSF.psf_evaluator import Moment_Evaluator
PSF_Evaluator = Moment_Evaluator()
directory = '/Users/cpd/Projects/WavefrontPSF/meshes/Science-20140212s2-v1i2'
mesh_name = 'Science-20140212s2-v1i2_All'
PSF_Interpolator = Mesh_Interpolator(mesh_name=mesh_name, directory=directory)
WF = Wavefront(PSF_Interpolator=PSF_Interpolator,
PSF_Evaluator=PSF_Evaluator,
model=PSF_Interpolator.data,
num_bins=3)
""" compare timings of my digestion vs donutana
%timeit WF.PSF_Interpolator.digest_mesh(mesh_name, directory)
from donutlib.donutana import donutana
sensorSet = "ScienceOnly"
method = "idw"
nInterpGrid = 32
methodVal = (4, 1.0)
in_dict = {"sensorSet": sensorSet,
"doTrefoil": True,
"doSpherical": True,
"doQuadrefoil": False,
# <codecell> from WavefrontPSF.colors import blue_red, blues_r, reds from WavefrontPSF.focal_plane_fit import FocalPlaneFit from WavefrontPSF.focal_plane import FocalPlane from WavefrontPSF.decamutil_cpd import decaminfo from os import path from scipy import linalg from Reference.shiftedColorMap import shiftedColorMap, shiftedColorMap_minmax from Reference.Image_GMM_EM import generate_random_params, em_algorithm, model, do_lpr, logsumexp, construct_X_and_I_from_image import ngmix from WavefrontPSF.wavefront import Wavefront WF = Wavefront(background=0, number_electrons=1) # <markdowncell> # # now try convolution model # <markdowncell> # ## first make two single gaussians and convolve and solve # <markdowncell> # # pars correspond to # # [cen1, cen2, g1, g2, T, flux] # # # # note T=ixx+iyy, and the center corresponds to an offset relative to the
from WavefrontPSF.psf_evaluator import Moment_Evaluator
from WavefrontPSF.analytic_interpolator import DECAM_Analytic_Wavefront, r0_guess
from WavefrontPSF.donutengine import DECAM_Model_Wavefront, generate_random_coordinates
from WavefrontPSF.donutengine import correct_dz, correct_dz_theta
digestor = Digestor()
base_directory = '/nfs/slac/g/ki/ki18/des/cpd/psfex_catalogs/SVA1_FINALCUT/psfcat/'
PSF_Evaluator = Moment_Evaluator()
mesh_directory = '/nfs/slac/g/ki/ki18/cpd/Projects/WavefrontPSF/meshes/Science-20140212s2-v1i2'
mesh_name = 'Science-20140212s2-v1i2_All'
PSF_Interpolator = Mesh_Interpolator(mesh_name=mesh_name, directory=mesh_directory)
# This will be our main wavefront
WF = DECAM_Model_Wavefront(PSF_Interpolator=PSF_Interpolator)
# let's create a Wavefront object for the data
WF_data = Wavefront(PSF_Interpolator=None, PSF_Evaluator=PSF_Evaluator)
# premake coordinate list
coords = []
for num_bins in xrange(6):
# create coordinates
x = []
y = []
if num_bins >= 2:
num_bins_make = num_bins + (num_bins-1)
else:
num_bins_make = num_bins
for key in WF.decaminfo.infoDict.keys():
if 'F' in key:
continue
xi, yi = WF.decaminfo.getBounds(key, num_bins_make)
def do_run(expid, aos=False):
if aos:
plot_dir = out_dir + '/plots_aos/{0:08d}'.format(expid)
else:
plot_dir = out_dir + '/plots/{0:08d}'.format(expid)
if not path.exists(plot_dir):
makedirs(plot_dir)
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from WavefrontPSF.psf_interpolator import Mesh_Interpolator
from WavefrontPSF.wavefront import Wavefront
from WavefrontPSF.digestor import Digestor
from WavefrontPSF.psf_evaluator import Moment_Evaluator
from WavefrontPSF.donutengine import DECAM_Model_Wavefront
medsubkeys = ['e0', 'e1', 'e2', 'E1norm', 'E2norm', 'delta1', 'delta2', 'zeta1', 'zeta2']
rows = ['e0', 'e0_medsub',
'e1', 'e1_medsub',
'e2', 'e2_medsub',
'E1norm', 'E1norm_medsub',
'E2norm', 'E2norm_medsub',
'delta1', 'delta1_medsub',
'delta2', 'delta2_medsub',
'zeta1', 'zeta1_medsub',
'zeta2', 'zeta2_medsub']
# set up objects. make sure I get the right mesh
digestor = Digestor()
PSF_Evaluator = Moment_Evaluator()
mesh_name = 'Science-20121120s1-v20i2_All'
PSF_Interpolator = Mesh_Interpolator(mesh_name=mesh_name, directory=mesh_directory)
# This will be our main wavefront
WF = DECAM_Model_Wavefront(PSF_Interpolator=PSF_Interpolator)
# let's create a Wavefront object for the data
WF_data = Wavefront(PSF_Interpolator=None, PSF_Evaluator=PSF_Evaluator)
# premake coordinate list
coords = []
for num_bins in xrange(6):
# create coordinates
x = []
y = []
if num_bins >= 2:
num_bins_make = num_bins + (num_bins-1)
else:
num_bins_make = num_bins
for key in WF.decaminfo.infoDict.keys():
if 'F' in key:
continue
xi, yi = WF.decaminfo.getBounds(key, num_bins_make)
xi = np.array(xi)
xi = 0.5 * (xi[1:] + xi[:-1])
yi = np.array(yi)
yi = 0.5 * (yi[1:] + yi[:-1])
xi, yi = np.meshgrid(xi, yi)
xi = xi.flatten()
yi = yi.flatten()
x += list(xi)
y += list(yi)
x = np.array(x)
y = np.array(y)
coords_i = pd.DataFrame({'x': x, 'y': y})
coords.append(coords_i)
# load up data
expid_path = '{0:08d}/{1:08d}'.format(expid - expid % 1000, expid)
data_directory = base_directory + expid_path
# load up all the data from an exposure. Unfortunately, pandas is stupid and
# can't handle the vignet format, so we don't load those up
# note that you CAN load them up by passing "do_exclude=True", which then
# returns a second variable containing the vignets and aperture fluxes and
# errors
model = digestor.digest_directory(
data_directory,
file_type='_selpsfcat.fits')
# cut the old data appropriately
model = model[(model['SNR_WIN'] > 90) &
(model['SNR_WIN'] < 400)]
# create normalized moments
model['E1norm'] = model['e1'] / model['e0']
model['E2norm'] = model['e2'] / model['e0']
# do med sub and add to WF_data
for key in medsubkeys:
model['{0}_medsub'.format(key)] = model[key] - np.median(model[key])
WF_data.data = model
# set the number of bins from total number of stars
if len(model) < 200:
num_bins = 0
num_bins_mis = 0
num_bins_whisker = 0
elif len(model) < 1000:
num_bins = 1
# num_bins_mis = 0
num_bins_mis = 1
num_bins_whisker = 1
elif len(model) < 10000:
num_bins = 2
# num_bins_mis = 1
num_bins_mis = 2
num_bins_whisker = 2
else:
num_bins = 3
num_bins_mis = 2
num_bins_whisker = 2
# generate optics model from fit data
fit_i = jamierod_results.loc[expid]
# TODO: Add ALL fit_i params that were used?
# TODO: get rzero?
if aos:
misalignment = {'z04d': fit_i['aos_z04d'],
'z05d': fit_i['aos_z05d'], 'z05x': fit_i['aos_z05x'], 'z05y': fit_i['aos_z05y'],
'z06d': fit_i['aos_z06d'], 'z06x': fit_i['aos_z06x'], 'z06y': fit_i['aos_z06y'],
'z07d': fit_i['aos_z07d'], 'z07x': fit_i['aos_z07x'], 'z07y': fit_i['aos_z07y'],
'z08d': fit_i['aos_z08d'], 'z08x': fit_i['aos_z08x'], 'z08y': fit_i['aos_z08y'],
'z09d': fit_i['aos_z09d'],
'z10d': fit_i['aos_z10d'],
'rzero': fit_i['aos_rzero']}
else:
misalignment = {'z04d': fit_i['z04d'], 'z04x': fit_i['z04x'], 'z04y': fit_i['z04y'],
'z05d': fit_i['z05d'], 'z05x': fit_i['z05x'], 'z05y': fit_i['z05y'],
'z06d': fit_i['z06d'], 'z06x': fit_i['z06x'], 'z06y': fit_i['z06y'],
'z07d': fit_i['z07d'], 'z07x': fit_i['z07x'], 'z07y': fit_i['z07y'],
'z08d': fit_i['z08d'], 'z08x': fit_i['z08x'], 'z08y': fit_i['z08y'],
'z09d': fit_i['z09d'], 'z09x': fit_i['z09x'], 'z09y': fit_i['z09y'],
'z10d': fit_i['z10d'], 'z10x': fit_i['z10x'], 'z10y': fit_i['z10y'],
'rzero': fit_i['rzero']}
# create model fit from donuts
WF.data = coords[num_bins].copy()
WF.data['rzero'] = misalignment['rzero']
WF.data = WF(WF.data, misalignment=misalignment)
# add dc factors
WF.data['e0'] += fit_i['e0']
WF.data['e1'] += fit_i['e1']
WF.data['e2'] += fit_i['e2']
WF.data['delta1'] += fit_i['delta1']
WF.data['delta2'] += fit_i['delta2']
WF.data['zeta1'] += fit_i['zeta1']
WF.data['zeta2'] += fit_i['zeta2']
# create normalized moments
WF.data['E1norm'] = WF.data['e1'] / WF.data['e0']
WF.data['E2norm'] = WF.data['e2'] / WF.data['e0']
# update WF medsubs appropriately
for key in medsubkeys:
WF.data['{0}_medsub'.format(key)] = WF.data[key] - np.median(WF.data[key])
# add a couple diagnostic things
WF.data['num_bins'] = num_bins
WF.data['expid'] = expid
# put in data into field
WF.reduce(num_bins=num_bins)
# create another reduced field for setting the color levels
field_model, _, _ = WF.reduce_data_to_field(
WF.data, xkey='x', ykey='y', reducer=np.median,
num_bins=num_bins_mis)
# update WF_data fields
WF_data.reduce(num_bins=num_bins)
# create another reduced field for setting the color levels
field_data, _, _ = WF_data.reduce_data_to_field(
WF_data.data, xkey='x', ykey='y', reducer=np.median,
num_bins=num_bins_mis)
# put in residual of data minus model for field
for row_i, row in enumerate(rows):
WF.field[row + '_data'] = WF_data.field[row]
WF.field[row + '_residual'] = WF_data.field[row] - WF.field[row]
field_model[row + '_residual'] = field_data[row] - field_model[row]
# create plots
for row in rows:
ncols = 3
nrows = 1
fig, axs = plt.subplots(nrows=nrows, ncols=ncols, figsize=(6*ncols, 5*nrows))
fig.suptitle('Expid: {0}, {1}'.format(expid, row))
vmin = np.nanmin((field_model[row].min(),
field_data[row].min()))
vmax = np.nanmax((field_data[row].max(),
field_model[row].max()))
vmindiff = field_model[row + '_residual'].min()
vmaxdiff = field_model[row + '_residual'].max()
ax = axs[0]
ax.set_title('Data')
WF_data.plot_field(row, fig=fig, ax=ax, a=vmin, b=vmax)
ax = axs[1]
ax.set_title('Model')
WF.plot_field(row, fig=fig, ax=ax, a=vmin, b=vmax)
ax = axs[2]
ax.set_title('Residual')
WF.plot_field(row + '_residual', fig=fig, ax=ax, a=vmindiff, b=vmaxdiff)
fig.savefig(plot_dir + '/{0}_{1}.png'.format(row, expid))
fig.savefig(plot_dir + '/{0}_{1}.pdf'.format(row, expid))
# TODO: Save each axis individually as well:
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(12, 10))
ax.set_title('Expid: {0}, {1}'.format(expid, row))
vmin = np.nanmin((field_model[row].min(),
field_data[row].min()))
vmax = np.nanmax((field_data[row].max(),
field_model[row].max()))
vmindiff = field_model[row + '_residual'].min()
vmaxdiff = field_model[row + '_residual'].max()
WF_data.plot_field(row, fig=fig, ax=ax, a=vmin, b=vmax)
fig.savefig(plot_dir + '/{0}_{1}_data.png'.format(row, expid))
fig.savefig(plot_dir + '/{0}_{1}_data.pdf'.format(row, expid))
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(12, 10))
ax.set_title('Expid: {0}, {1}'.format(expid, row))
WF.plot_field(row, fig=fig, ax=ax, a=vmin, b=vmax)
fig.savefig(plot_dir + '/{0}_{1}_model.png'.format(row, expid))
fig.savefig(plot_dir + '/{0}_{1}_model.pdf'.format(row, expid))
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(12, 10))
ax.set_title('Expid: {0}, {1}'.format(expid, row))
WF.plot_field(row + '_residual', fig=fig, ax=ax, a=vmindiff, b=vmaxdiff)
fig.savefig(plot_dir + '/{0}_{1}_residual.png'.format(row, expid))
fig.savefig(plot_dir + '/{0}_{1}_residual.pdf'.format(row, expid))
plt.close('all')
# save whisker plots too
# do w, e, and normalized e.
# for each: data and model separate, data plus model, residual
###########################################################################
# w
###########################################################################
num_spokes = 2
scalefactor = 0.2
scalefactor_residual = 0.5
quiverdict = {'width': 2}
# w
# data
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=False,
fig=None, ax=None,
scalefactor=scalefactor,
num_spokes=num_spokes,
color='black',
e1key='e1_data', e2key='e2_data',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_w_{0}_data.png'.format(expid))
fig.savefig(plot_dir + '/whisker_w_{0}_data.pdf'.format(expid))
# w
# model
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=False,
fig=None, ax=None,
scalefactor=scalefactor,
num_spokes=num_spokes,
color='black',
e1key='e1', e2key='e2',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_w_{0}_model.png'.format(expid))
fig.savefig(plot_dir + '/whisker_w_{0}_model.pdf'.format(expid))
# w
# data + model
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=False,
fig=None, ax=None,
scalefactor=scalefactor,
num_spokes=num_spokes,
color='black',
e1key='e1_data', e2key='e2_data',
do_var=False, legend=False, quiverdict=quiverdict)
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=False,
fig=fig, ax=ax,
scalefactor=scalefactor, num_spokes=num_spokes,
color='red',
e1key='e1', e2key='e2',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_w_{0}_blackdata_redmodel.png'.format(expid))
fig.savefig(plot_dir + '/whisker_w_{0}_blackdata_redmodel.pdf'.format(expid))
# w
# residual
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=False,
fig=None, ax=None,
scalefactor=scalefactor_residual,
num_spokes=num_spokes,
color='black',
e1key='e1_residual', e2key='e2_residual',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_w_{0}_residual.png'.format(expid))
fig.savefig(plot_dir + '/whisker_w_{0}_residual.pdf'.format(expid))
###########################################################################
###########################################################################
# e
###########################################################################
num_spokes = 1
scalefactor = 1
scalefactor_residual = 5
quiverdict = {'width': 2}
# e
# data
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=False,
fig=None, ax=None,
scalefactor=scalefactor,
num_spokes=num_spokes,
color='black',
e1key='e1_data', e2key='e2_data',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_e_{0}_data.png'.format(expid))
fig.savefig(plot_dir + '/whisker_e_{0}_data.pdf'.format(expid))
# e
# model
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=False,
fig=None, ax=None,
scalefactor=scalefactor,
num_spokes=num_spokes,
color='black',
e1key='e1', e2key='e2',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_e_{0}_model.png'.format(expid))
fig.savefig(plot_dir + '/whisker_e_{0}_model.pdf'.format(expid))
# e
# data + model
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=False,
fig=None, ax=None,
scalefactor=scalefactor,
num_spokes=num_spokes,
color='black',
e1key='e1_data', e2key='e2_data',
do_var=False, legend=False, quiverdict=quiverdict)
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=False,
fig=fig, ax=ax,
scalefactor=scalefactor, num_spokes=num_spokes,
color='red',
e1key='e1', e2key='e2',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_e_{0}_blackdata_redmodel.png'.format(expid))
fig.savefig(plot_dir + '/whisker_e_{0}_blackdata_redmodel.pdf'.format(expid))
# e
# residual
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=False,
fig=None, ax=None,
scalefactor=scalefactor_residual,
num_spokes=num_spokes,
color='black',
e1key='e1_residual', e2key='e2_residual',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_e_{0}_residual.png'.format(expid))
fig.savefig(plot_dir + '/whisker_e_{0}_residual.pdf'.format(expid))
###########################################################################
###########################################################################
# E
###########################################################################
num_spokes = 1
scalefactor = 1
scalefactor_residual = 5
quiverdict = {'width': 2}
# E
# data
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=True,
fig=None, ax=None,
scalefactor=scalefactor,
num_spokes=num_spokes,
color='black',
e1key='E1norm_data', e2key='E2norm_data',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_Enorm_{0}_data.png'.format(expid))
fig.savefig(plot_dir + '/whisker_Enorm_{0}_data.pdf'.format(expid))
# E
# model
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=True,
fig=None, ax=None,
scalefactor=scalefactor,
num_spokes=num_spokes,
color='black',
e1key='E1norm', e2key='E2norm',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_Enorm_{0}_model.png'.format(expid))
fig.savefig(plot_dir + '/whisker_Enorm_{0}_model.pdf'.format(expid))
# E
# data + model
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=True,
fig=None, ax=None,
scalefactor=scalefactor,
num_spokes=num_spokes,
color='black',
e1key='E1norm_data', e2key='E2norm_data',
do_var=False, legend=False, quiverdict=quiverdict)
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=True,
fig=fig, ax=ax,
scalefactor=scalefactor, num_spokes=num_spokes,
color='red',
e1key='E1norm', e2key='E2norm',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_Enorm_{0}_blackdata_redmodel.png'.format(expid))
fig.savefig(plot_dir + '/whisker_Enorm_{0}_blackdata_redmodel.pdf'.format(expid))
# E
# residual
fig, ax = WF.plot_whisker(WF.field, num_bins=num_bins_whisker,
normalized_ellipticity=True,
fig=None, ax=None,
scalefactor=scalefactor_residual,
num_spokes=num_spokes,
color='black',
e1key='E1norm_residual', e2key='E2norm_residual',
do_var=False, legend=True, quiverdict=quiverdict)
fig.savefig(plot_dir + '/whisker_Enorm_{0}_residual.png'.format(expid))
fig.savefig(plot_dir + '/whisker_Enorm_{0}_residual.pdf'.format(expid))
###########################################################################
# make plot of N
fig, ax = plt.subplots(figsize=(6,5))
WF_data.plot_field('N', fig=fig, ax=ax)
fig.savefig(plot_dir + '/N_{0}.png'.format(expid))
plt.close('all')
# save summary statistics of stars to npy file for later collection
if aos:
field_jamie = pd.read_pickle(pkl_dir + '/{0:08d}.pkl'.format(expid))
for row in rows:
WF.field[row + '_jamie'] = field_jamie[row]
WF.field[row + '_jamie_residual'] = WF.field[row + '_data'] - \
WF.field[row + '_jamie']
WF.field.to_pickle(pkl_dir + '/aos_{0:08d}.pkl'.format(expid))
else:
WF.field.to_pickle(pkl_dir + '/{0:08d}.pkl'.format(expid))
out_dir_base = '/nfs/slac/g/ki/ki18/des/cpd/psfex_catalogs/SVA1_FINALCUT/treerings_15_09_30/'
out_dir_base = '/nfs/slac/g/ki/ki18/des/cpd/psfex_catalogs/SVA1_FINALCUT/treerings_15_10_20/'
if not path.exists(out_dir_base):
makedirs(out_dir_base)
if not path.exists(out_dir_base + 'logs'):
makedirs(out_dir_base + 'logs')
if not path.exists(out_dir_base + 'individual'):
makedirs(out_dir_base + 'individual')
if not path.exists(out_dir_base + 'individual_stamps'):
makedirs(out_dir_base + 'individual_stamps')
PSF_Evaluator = Moment_Evaluator()
digestor = Digestor()
WF = Wavefront(PSF_Interpolator=None, PSF_Evaluator=PSF_Evaluator)
# what columns will we extract
model_keys = ['XWIN_IMAGE', 'YWIN_IMAGE', 'ext', 'x', 'y', 'expnum', 'FLAGS', 'SNR_WIN']
model_keys += ['MAG_APER_6']
moment_keys = ['Mx', 'My', 'flux', 'e0', 'e1', 'e2', 'e0prime', 'delta1', 'delta2', 'zeta1', 'zeta2', 'fwhm', 'a4']
def clean_stamps(stamps, sexhdu):
# find background
skycol = 'NONE'
for coli, col in enumerate(sexhdu):
if 'SKYBRITE' in col:
skycol = col
if skycol == 'NONE':
background = 0
