def read_data(exps, work, keys, limit_bands=None, prefix='piff', use_reserved=False, frac=1.):
RESERVED = 64
NOT_STAR = 128
BAD_CCDS = [2, 31, 61]
MAX_TILING = 10
all_keys = keys
if 'x' in keys:
# Note: use this syntax rather than += to avoid changing input keys
all_keys = all_keys + ['fov_x', 'fov_y']
all_keys = all_keys + ['exp', 'ccd', 'band', 'tiling']
all_data = { key : [] for key in all_keys }
bands = set() # This is the set of all bands being used
tilings = set() # This is the set of all tilings being used
n_reject_mean_dt = 0
n_reject_mean_de1 = 0
n_reject_mean_de2 = 0
n_reject_mean_e1 = 0
n_reject_mean_e2 = 0
n_reject_std_dt = 0
n_reject_std_de1 = 0
n_reject_std_de2 = 0
n_reject_rho2 = 0
nrows = 0
n_good_obj = 0
n_bad_obj = 0
for exp in sorted(exps):
expnum = int(exp)
old = False
expname = os.path.join(work, exp, 'exp_psf_cat_%d.fits'%expnum)
if not os.path.exists(expname):
# Old file name (v21 and earlier)
old = True
expname = os.path.join(work, exp, 'exp_info_%d.fits'%expnum)
if not os.path.exists(expname):
print('Neither kind of exposure catalog exists.')
print(os.path.join(work, exp, 'exp_psf_cat_%d.fits'%expnum))
print(expname)
print('Skip this exposure')
continue
try:
if old:
expcat = fitsio.read(expname)
else:
print(expname)
expcat = fitsio.read(expname, ext='info')
except Exception as e:
print('Error reading exposure catalog')
print('Caught ',e)
print('Skip this exposure')
continue
if len(expcat) == 0:
print('expcat for exp=%d has no entries!',expnum)
continue
band = expcat['band'][0]
if (limit_bands is not None) and (band not in limit_bands):
#print('Not doing band = %s.'%band)
continue
if expcat['expnum'][0] != expnum:
print('%s != %s'%(expcat['expnum'][0], expnum))
print('expnum in catalog does not match expected value')
sys.exit()
print('Start work on exp = ',exp)
print('band = ',band)
if 'tiling' in expcat.dtype.names:
tiling = int(expcat['tiling'][0])
if tiling == 0:
# This shouldn't happen, but it did for a few exposures. Just skip them, since this
# might indicate some kind of problem.
print('tiling == 0. Skip this exposure.')
continue
if tiling > MAX_TILING:
print('tiling is > %d. Skip this exposure.'%MAX_TILING)
continue
print('tiling = ',tiling)
else:
tiling = 0
if old:
mask = ~np.in1d(expcat['ccdnum'], BAD_CCDS)
mask &= expcat['flag'] == 0
data, ccdnums = old_read_ccd_data(expcat[mask], work, expnum)
else:
data = fitsio.read(expname, ext='stars')
ccdnums = data['ccdnum'].astype(int)
flag = data[prefix+'_flag'].astype(int)
ntot = len(data)
nused = np.sum((flag & 1) != 0)
nreserved = np.sum((flag & RESERVED) != 0)
ngood = np.sum(flag == 0)
print('ntot = ',ntot)
print('nused = ',nused)
print('nreserved = ',nreserved)
print('ngood = ',ngood)
mask = (flag & NOT_STAR) == 0
mask &= ~np.in1d(ccdnums, BAD_CCDS)
if use_reserved:
mask &= (flag & RESERVED) != 0
used = (flag & ~RESERVED) == 0
#print('flag = ',flag)
#print('mask = ',mask)
#print('used = ',used)
#print('flag where flag == RESERVED: ',flag[flag==RESERVED+1])
#print('mask where flag == RESERVED: ',mask[flag==RESERVED+1])
#print('used where flag == RESERVED: ',mask[flag==RESERVED+1])
print('nmask = ',np.sum(mask))
print('nused = ',np.sum(used))
T = data['obs_T']
e1 = data['obs_e1']
e2 = data['obs_e2']
dT = data['obs_T'] - data[prefix + '_T']
de1 = data['obs_e1'] - data[prefix + '_e1']
de2 = data['obs_e2'] - data[prefix + '_e2']
print(expnum, len(dT), band)
#print('T = ',np.mean(T[used]),np.std(T[used]))
#print('e1 = ',np.mean(e1[used]),np.std(e1[used]))
#print('e2 = ',np.mean(e2[used]),np.std(e2[used]))
#print('dT/T = ',np.mean(dT[used]/T[used]),np.std(dT[used]/T[used]))
#print('de1 = ',np.mean(de1[used]),np.std(de1[used]))
#print('de2 = ',np.mean(de2[used]),np.std(de2[used]))
rho1 = (de1 - 1j*de2) * (de1 + 1j*de2)
#print('mean rho1 = ',np.mean(rho1[used]))
rho2 = (e1 - 1j*e2) * (de1 + 1j*de2)
#print('mean rho2 = ',np.mean(rho2[used]))
if abs(np.mean(dT[used]/T[used])) > 0.01:
print('mean dT/T = %f.'%(np.mean(dT[used]/T[used])))
n_reject_mean_dt += 1
#continue
if abs(np.mean(de1[used])) > 0.01:
print('mean de1 = %f.'%(np.mean(de1[used])))
n_reject_mean_de1 += 1
#continue
if abs(np.mean(de2[used])) > 0.01:
print('mean de2 = %f.'%(np.mean(de2[used])))
n_reject_mean_de2 += 1
#continue
if abs(np.std(dT[used]/T[used])) > 0.1:
print('std dT/T = %f.'%(np.std(dT[used]/T[used])))
n_reject_std_dt += 1
#continue
if abs(np.std(de1[used])) > 0.1:
print('std de1 = %f.'%(np.std(de1[used])))
n_reject_std_de1 += 1
#continue
if abs(np.std(de2[used])) > 0.1:
print('std de2 = %f.'%(np.std(de2[used])))
n_reject_std_de2 += 1
#continue
if abs(np.mean(rho1[used])) > 5.e-4:
print('mean rho1 = %s.'%(np.mean(rho1[used])))
n_reject_rho2 += 1
#continue
if abs(np.mean(rho2[used])) > 5.e-4:
print('mean rho2 = %s.'%(np.mean(rho2[used])))
n_reject_rho2 += 1
#continue
if abs(np.mean(e1[used])) > 0.03:
print('mean e1 = %f.'%(np.mean(e1[used])))
n_reject_mean_e1 += 1
#continue
if abs(np.mean(e2[used])) > 0.03:
print('mean e2 = %f.'%(np.mean(e2[used])))
n_reject_mean_e2 += 1
#continue
# Filter out egregiously bad values. Just in case.
good = (abs(dT/T) < 0.1) & (abs(de1) < 0.1) & (abs(de2) < 0.1)
n1 = np.sum(mask)
mask = mask & good
n2 = np.sum(mask)
print('"good" filter removed %d/%d objects'%(n1-n2,n1))
n_good_obj += n2
n_bad_obj += n1-n2
#print('mask = ',len(mask),np.sum(mask),mask)
mask = np.where(mask)[0]
#print('mask = ',len(mask),mask)
if frac != 1.:
mask = np.random.choice(mask, int(frac * len(mask)), replace=False)
#print('mask = ',len(mask),mask)
ngood = len(mask)
print('ngood = ',ngood,'/',len(data))
assert ngood == len(data[mask])
if ngood == 0:
print('All objects in exp %d are flagged.'%expnum)
continue
# Start with just the input keys, which should be columns in data.
for key in keys:
all_data[key].append(data[key][mask])
# Now add the extra ones we added to all_keys
if 'x' in keys:
# Convert to focal position.
x,y = toFocal(ccdnums[mask], data['x'][mask], data['y'][mask])
# This comes back in units of mm. Convert to arcsec.
# 1 pixel = 15e-3 mm = 0.263 arcsec
x *= 0.263/15e-3
y *= 0.263/15e-3
all_data['fov_x'].append(x)
all_data['fov_y'].append(y)
all_data['ccd'].append(ccdnums[mask])
all_data['exp'].append([expnum] * ngood)
all_data['band'].append([band] * ngood)
all_data['tiling'].append([tiling] * ngood)
bands.add(band)
tilings.add(tiling)
nrows += ngood
print('\nFinished processing %d exposures'%len(exps))
print('bands = ',bands)
print('tilings = ',tilings)
print('total "good" stars = ',n_good_obj)
print('total "bad" stars = ',n_bad_obj)
print('total good stars selected = ',nrows)
print('Potential rejections: (not enabled):')
print('n_reject_mean_dt = ',n_reject_mean_dt)
print('n_reject_mean_de1 = ',n_reject_mean_de1)
print('n_reject_mean_de2 = ',n_reject_mean_de2)
print('n_reject_mean_e1 = ',n_reject_mean_de1)
print('n_reject_mean_e2 = ',n_reject_mean_de2)
print('n_reject_std_dt = ',n_reject_std_dt)
print('n_reject_std_de1 = ',n_reject_std_de1)
print('n_reject_std_de2 = ',n_reject_std_de2)
print('n_reject_rho2 = ',n_reject_rho2)
# Turn the data into a recarray
# Pick appropriate formats for each kind of data
formats = []
for key in all_keys:
if key == 'ccd' or key == 'tiling':
formats.append('i2')
elif key == 'exp' or 'flag' in key:
formats.append('i4')
elif key == 'band':
formats.append('a1')
else:
formats.append('f8')
data = np.recarray(shape=(nrows,), formats=formats, names=all_keys)
#print('data.dtype = ',data.dtype)
for key in all_keys:
data[key] = np.concatenate(all_data[key])
print('made recarray')
return data, bands, tilings
def bin_by_fov(ccd, x, y, e1, e2, s, w=None, nwhisk=5):
from toFocal import toFocal
all_x = np.array([])
all_y = np.array([])
all_e1 = np.array([])
all_e2 = np.array([])
all_s = np.array([])
if w is None:
w = np.ones(len(x))
x_bins = np.linspace(0,2048,nwhisk+1)
y_bins = np.linspace(0,4096,2*nwhisk+1)
print('x_bins = ',x_bins)
print('y_bins = ',y_bins)
ccdnums = np.unique(ccd)
print('ccdnums = ',ccdnums)
for ccdnum in ccdnums:
mask = np.where(ccd == ccdnum)[0]
print('ccdnum = ',ccdnum,', nstar = ',len(mask))
if mask.sum() < 100: continue
if ccdnum in [31, 61]: continue
x_index = np.digitize(x[mask], x_bins)
y_index = np.digitize(y[mask], y_bins)
nbins = (len(x_bins)-1) * (len(y_bins)-1)
bin_e1 = np.empty(nbins)
bin_e2 = np.empty(nbins)
bin_s = np.empty(nbins)
bin_x = np.empty(nbins)
bin_y = np.empty(nbins)
bin_nz = np.empty(nbins, dtype=bool)
sumesq = 0.
for i in range(1, len(x_bins)):
for j in range(1, len(y_bins)):
k = (i-1)*(len(y_bins)-1) + (j-1)
mask2 = np.where( (x_index == i) & (y_index == j) )[0]
ww = w[mask][mask2]
ws = ww.sum()
bin_e1[k] = (ww * e1[mask][mask2]).sum() / ws
bin_e2[k] = (ww * e2[mask][mask2]).sum() / ws
bin_s[k] = (ww * s[mask][mask2]).sum() / ws
bin_x[k] = (ww * x[mask][mask2]).sum() / ws
bin_y[k] = (ww * y[mask][mask2]).sum() / ws
bin_nz[k] = len(mask2) > 0
print(i,j,k,len(mask2), bin_e1[k], bin_e2[k])
sumesq += bin_e1[k]**2 + bin_e2[k]**2
print('rms e = ',np.sqrt(sumesq / nbins))
print('ccdnum = ',ccdnum)
print('bin_x,y = ',bin_x,bin_y)
focal_x, focal_y = toFocal(int(ccdnum), bin_x, bin_y)
print('x,y = ',focal_x, focal_y)
print('num with count > 0 = ',bin_nz.sum())
all_x = np.append(all_x, focal_x[bin_nz])
all_y = np.append(all_y, focal_y[bin_nz])
all_e1 = np.append(all_e1, bin_e1[bin_nz])
all_e2 = np.append(all_e2, bin_e2[bin_nz])
all_s = np.append(all_s, bin_s[bin_nz])
return all_x, all_y, all_e1, all_e2, all_s
def read_data(args, work, limit_filters=None, subtract_mean=True, reserved=False):
import astropy.io.fits as pyfits
datadir = '/astro/u/astrodat/data/DES'
RESERVED = 64
if args.file != '':
print 'Read file ',args.file
with open(args.file) as fin:
data = [ line.split() for line in fin ]
runs, exps = zip(*data)
else:
runs = args.runs
exps = args.exps
expinfo_file = '/astro/u/mjarvis/work/exposure_info_' + args.tag + '.fits'
#expinfo_file = '/astro/u/mjarvis/work/exposure_info_y1a1-v01.fits'
#expinfo_file = 'exposure_info_y1spte-v02.fits'
print 'reading exposure_info file: ',expinfo_file
with pyfits.open(expinfo_file) as pyf:
expinfo = pyf[1].data
keys = ['ra', 'dec', 'x', 'y', 'e1', 'e2', 'size', 'psf_e1', 'psf_e2', 'psf_size']
all_data = { key : [] for key in keys }
all_keys = keys
all_data['exp'] = []
all_data['ccd'] = []
all_keys = all_keys + ['exp', 'ccd' ]
if subtract_mean:
all_data['alt_e1'] = []
all_data['alt_e2'] = []
all_data['alt_size'] = []
all_keys = all_keys + ['alt_e1', 'alt_e2', 'alt_size']
if 'x' in keys:
all_data['fov_x'] = []
all_data['fov_y'] = []
all_keys = all_keys + ['fov_x', 'fov_y']
if args.oldkeys:
inkeys = [ k.replace('psf_','psfex_') for k in keys ]
else:
inkeys = keys
all_filters = [] # This keeps track of the filter for each record
all_tilings = [] # This keeps track of the tiling for each record
filters = set() # This is the set of all filters being used
tilings = set() # This is the set of all tilings being used
ntot = 0
nused = 0
nreserved = 0
ngood = 0
cat_dir = os.path.join(work,'psf_cats')
for run,exp in zip(runs,exps):
print 'Start work on run, exp = ',run,exp
expnum = int(exp[6:])
print 'expnum = ',expnum
if expnum not in expinfo['expnum']:
print 'expnum is not in expinfo!'
print 'expinfo[expnum] = ',expinfo['expnum']
print 'Could not find information about this expnum. Skipping ',run,exp
continue
k = numpy.nonzero(expinfo['expnum'] == expnum)[0][0]
print 'k = ',k
filter = expinfo['filter'][k]
print 'filter[k] = ',filter
if (limit_filters is not None) and (filter not in limit_filters):
print 'Not doing this filter.'
continue
tiling = int(expinfo['tiling'][k])
print 'tiling[k] = ',tiling
if tiling == 0:
# This shouldn't happen, but it did for a few exposures. Just skip them, since this
# might indicate some kind of problem.
print 'tiling == 0. Skip this exposure.'
continue
if tiling > args.max_tiling:
print 'tiling is > %d. Skip this exposure.'%args.max_tiling
continue
cat_file = os.path.join(cat_dir, exp + "_exppsf.fits")
if not os.path.exists(cat_file):
cat_file = os.path.join(cat_dir, exp + "_psf.fits")
print 'cat_file = ',cat_file
try:
with pyfits.open(cat_file) as pyf:
data = pyf[1].data
except:
print 'Unable to open cat_file %s. Skipping this file.'%cat_file
continue
#ccdnums = numpy.unique(data['ccdnum'])
#print 'ccdnums = ',ccdnums
print 'n = ',len(data)
print 'nused = ',numpy.sum((data['flag'] & 1) != 0)
print 'nreserved = ',numpy.sum((data['flag'] & 64) != 0)
print 'ngood = ',numpy.sum(data['flag'] == 0)
ntot += len(data)
nused += numpy.sum((data['flag'] & 1) != 0)
nreserved += numpy.sum((data['flag'] & 64) != 0)
ngood += numpy.sum(data['flag'] == 0)
if args.use_reserved:
mask = (data['flag'] == RESERVED)
else:
mask = (data['flag'] == 0)
ngood = numpy.sum(mask)
assert ngood == len(data[mask])
if ngood == 0:
print 'All objects in this exposure are flagged.'
print 'Probably due to astrometry flags. Skip this exposure.'
continue
for key, inkey in zip(keys, inkeys):
all_data[key].append(data[inkey][mask])
all_data['exp'].append([expnum] * ngood)
all_data['ccd'].append(data['ccdnum'][mask])
if subtract_mean:
e1 = data['e1'][mask]
e2 = data['e2'][mask]
s = data['size'][mask]
p_e1 = data['psf_e1'][mask]
p_e2 = data['psf_e2'][mask]
p_s = data['psf_size'][mask]
de1 = numpy.mean(e1-p_e1)
de2 = numpy.mean(e2-p_e2)
# Really want <(s^2 - p_s^2)/s^2> => 0 after subtracting ds
# <1 - p_s^2/(s-ds)^2> = 0
# <1 - p_s^2/s^2 (1 + 2ds/s + 3ds^2/s^2 + ...) > = 0
# 1 - <p_s^2/s^2> - 2ds<p_s^2/s^3> - 3ds^2<p_s^2/s^4> = 0
a1 = numpy.mean(p_s**2/s**2)
a2 = numpy.mean(p_s**2/s**3)
a3 = numpy.mean(p_s**2/s**4)
ds = (1. - a1) / (2.*a2)
# Iterate once to refine
ds = (1. - a1 - 3.*ds**2*a3) / (2.*a2)
print 'de = ',de1,de2, 'mean e = ',numpy.mean(e1),numpy.mean(e2),
print ' -> ',numpy.mean(e1-de1), numpy.mean(e2-de2)
print 'ds = ',ds, 'mean s = ',numpy.mean(s),' -> ',numpy.mean(s-ds)
print 'mean dt = ',numpy.mean( (s**2 - p_s**2) / s**2 ),
print ' -> ',numpy.mean( ((s-ds)**2 - p_s**2) / (s-ds)**2 )
all_data['alt_e1'].append(e1 - de1)
all_data['alt_e2'].append(e2 - de2)
all_data['alt_size'].append(s - ds)
if 'x' in keys:
# Convert to focal position.
x,y = toFocal(data['ccdnum'][mask], data['x'][mask], data['y'][mask])
# This comes back in units of mm. Convert to arcsec.
# 1 pixel = 15e-3 mm = 0.263 arcsec
x *= 0.263/15e-3
y *= 0.263/15e-3
all_data['fov_x'].append(x)
all_data['fov_y'].append(y)
all_filters.extend( ([filter] * ngood) )
all_tilings.extend( ([tiling] * ngood) )
filters.add(filter)
tilings.add(tiling)
print '\nFinished processing all exposures'
print 'filters = ',filters
print 'tilings = ',tilings
# Turn the data into a recarray
print 'all_data.keys = ',all_data.keys()
formats = ['f8'] * len(all_keys) + ['a1', 'i2']
names = all_keys + ['filter', 'tiling']
data = numpy.recarray(shape = (len(all_filters),),
formats = formats, names = names)
print 'data.dtype = ',data.dtype
for key in all_keys:
data[key] = numpy.concatenate(all_data[key])
data['filter'] = all_filters
data['tiling'] = all_tilings
print 'made recarray'
print 'ntot = ',ntot
print 'nused = ',nused
print 'nreserved = ',nreserved
print 'ngood = ',ngood
return data, filters, tilings
def bin_by_fov(ccd, x, y, dT, de1, de2, bands):
import numpy as np
import matplotlib.pyplot as plt
from toFocal import toFocal
all_x = np.array([])
all_y = np.array([])
all_e1 = np.array([])
all_e2 = np.array([])
all_T = np.array([])
nwhisk = 5
x_bins = np.linspace(0,2048,nwhisk+1)
y_bins = np.linspace(0,4096,2*nwhisk+1)
ccdnums = np.unique(ccd)
for ccdnum in ccdnums:
mask = np.where(ccd == ccdnum)[0]
print('ccdnum = ',ccdnum,', nstar = ',mask.sum())
if mask.sum() < 100: continue
x_index = np.digitize(x[mask], x_bins)
y_index = np.digitize(y[mask], y_bins)
bin_de1 = np.array([ de1[mask][(x_index == i) & (y_index == j)].mean()
for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ])
print('bin_de1 = ',bin_de1)
bin_de2 = np.array([ de2[mask][(x_index == i) & (y_index == j)].mean()
for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ])
print('bin_de2 = ',bin_de2)
bin_dT = np.array([ dT[mask][(x_index == i) & (y_index == j)].mean()
for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ])
print('bin_dT = ',bin_dT)
bin_x = np.array([ x[mask][(x_index == i) & (y_index == j)].mean()
for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ])
print('bin_x = ',bin_x)
bin_y = np.array([ y[mask][(x_index == i) & (y_index == j)].mean()
for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ])
print('bin_y = ',bin_y)
bin_count = np.array([ ((x_index == i) & (y_index == j)).sum()
for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ])
print('bin_count = ',bin_count)
focal_x, focal_y = toFocal(ccdnum, bin_x, bin_y)
mask2 = np.where(bin_count > 0)[0]
print('num with count > 0 = ',mask2.sum())
all_x = np.append(all_x, focal_x[mask2])
all_y = np.append(all_y, focal_y[mask2])
all_e1 = np.append(all_e1, bin_de1[mask2])
all_e2 = np.append(all_e2, bin_de2[mask2])
all_T = np.append(all_T, bin_dT[mask2])
plt.clf()
#plt.title('PSF Ellipticity residuals in DES focal plane')
theta = np.arctan2(all_e2,all_e1)/2.
r = np.sqrt(all_e1**2 + all_e2**2)
u = r*np.cos(theta)
v = r*np.sin(theta)
plt.xlim(-250,250)
plt.ylim(-250,250)
print('all_x = ',all_x)
print('len(all_x) = ',len(all_x))
qv = plt.quiver(all_x,all_y,u,v, pivot='middle', scale_units='xy',
headwidth=0., headlength=0., headaxislength=0.,
width=0.001, scale=1.e-3, color='blue')
ref_scale = 0.01
if 'raw' in bands:
ref_label = 'e = ' + str(ref_scale)
else:
ref_label = 'de = ' + str(ref_scale)
plt.quiverkey(qv, 0.10, 0.08, ref_scale, ref_label,
coordinates='axes', color='darkred', labelcolor='darkred',
labelpos='E', fontproperties={'size':'x-small'})
plt.axis('off')
plt.savefig('de_fov_' + bands + '.png')
plt.savefig('de_fov_' + bands + '.pdf')
plt.savefig('de_fov_' + bands + '.eps')
def read_data(exps,
work,
keys,
limit_bands=None,
prefix='piff',
use_reserved=False,
frac=1.):
RESERVED = 64
NOT_STAR = 128
BAD_CCDS = [2, 31, 61]
MAX_TILING = 10
all_keys = keys
if 'x' in keys:
# Note: use this syntax rather than += to avoid changing input keys
all_keys = all_keys + ['fov_x', 'fov_y']
all_keys = all_keys + ['exp', 'ccd', 'band', 'tiling']
all_data = {key: [] for key in all_keys}
bands = set() # This is the set of all bands being used
tilings = set() # This is the set of all tilings being used
n_reject_mean_dt = 0
n_reject_mean_de1 = 0
n_reject_mean_de2 = 0
n_reject_mean_e1 = 0
n_reject_mean_e2 = 0
n_reject_std_dt = 0
n_reject_std_de1 = 0
n_reject_std_de2 = 0
n_reject_rho2 = 0
nrows = 0
n_good_obj = 0
n_bad_obj = 0
for exp in sorted(exps):
expnum = int(exp)
old = False
expname = os.path.join(work, exp, 'exp_psf_cat_%d.fits' % expnum)
if not os.path.exists(expname):
# Old file name (v21 and earlier)
old = True
expname = os.path.join(work, exp, 'exp_info_%d.fits' % expnum)
if not os.path.exists(expname):
print('Neither kind of exposure catalog exists.')
print(os.path.join(work, exp, 'exp_psf_cat_%d.fits' % expnum))
print(expname)
print('Skip this exposure')
continue
try:
if old:
expcat = fitsio.read(expname)
else:
print(expname)
expcat = fitsio.read(expname, ext='info')
except Exception as e:
print('Error reading exposure catalog')
print('Caught ', e)
print('Skip this exposure')
continue
if len(expcat) == 0:
print('expcat for exp=%d has no entries!', expnum)
continue
band = expcat['band'][0]
if (limit_bands is not None) and (band not in limit_bands):
#print('Not doing band = %s.'%band)
continue
if expcat['expnum'][0] != expnum:
print('%s != %s' % (expcat['expnum'][0], expnum))
print('expnum in catalog does not match expected value')
sys.exit()
print('Start work on exp = ', exp)
print('band = ', band)
if 'tiling' in expcat.dtype.names:
tiling = int(expcat['tiling'][0])
if tiling == 0:
# This shouldn't happen, but it did for a few exposures. Just skip them, since this
# might indicate some kind of problem.
print('tiling == 0. Skip this exposure.')
continue
if tiling > MAX_TILING:
print('tiling is > %d. Skip this exposure.' % MAX_TILING)
continue
print('tiling = ', tiling)
else:
tiling = 0
if old:
mask = ~np.in1d(expcat['ccdnum'], BAD_CCDS)
mask &= expcat['flag'] == 0
data, ccdnums = old_read_ccd_data(expcat[mask], work, expnum)
else:
data = fitsio.read(expname, ext='stars')
ccdnums = data['ccdnum'].astype(int)
flag = data[prefix + '_flag'].astype(int)
ntot = len(data)
nused = np.sum((flag & 1) != 0)
nreserved = np.sum((flag & RESERVED) != 0)
ngood = np.sum(flag == 0)
print('ntot = ', ntot)
print('nused = ', nused)
print('nreserved = ', nreserved)
print('ngood = ', ngood)
mask = (flag & NOT_STAR) == 0
mask &= ~np.in1d(ccdnums, BAD_CCDS)
if use_reserved:
mask &= (flag & RESERVED) != 0
used = (flag & ~RESERVED) == 0
#print('flag = ',flag)
#print('mask = ',mask)
#print('used = ',used)
#print('flag where flag == RESERVED: ',flag[flag==RESERVED+1])
#print('mask where flag == RESERVED: ',mask[flag==RESERVED+1])
#print('used where flag == RESERVED: ',mask[flag==RESERVED+1])
print('nmask = ', np.sum(mask))
print('nused = ', np.sum(used))
T = data['obs_T']
e1 = data['obs_e1']
e2 = data['obs_e2']
dT = data['obs_T'] - data[prefix + '_T']
de1 = data['obs_e1'] - data[prefix + '_e1']
de2 = data['obs_e2'] - data[prefix + '_e2']
print(expnum, len(dT), band)
#print('T = ',np.mean(T[used]),np.std(T[used]))
#print('e1 = ',np.mean(e1[used]),np.std(e1[used]))
#print('e2 = ',np.mean(e2[used]),np.std(e2[used]))
#print('dT/T = ',np.mean(dT[used]/T[used]),np.std(dT[used]/T[used]))
#print('de1 = ',np.mean(de1[used]),np.std(de1[used]))
#print('de2 = ',np.mean(de2[used]),np.std(de2[used]))
rho1 = (de1 - 1j * de2) * (de1 + 1j * de2)
#print('mean rho1 = ',np.mean(rho1[used]))
rho2 = (e1 - 1j * e2) * (de1 + 1j * de2)
#print('mean rho2 = ',np.mean(rho2[used]))
if abs(np.mean(dT[used] / T[used])) > 0.01:
print('mean dT/T = %f.' % (np.mean(dT[used] / T[used])))
n_reject_mean_dt += 1
#continue
if abs(np.mean(de1[used])) > 0.01:
print('mean de1 = %f.' % (np.mean(de1[used])))
n_reject_mean_de1 += 1
#continue
if abs(np.mean(de2[used])) > 0.01:
print('mean de2 = %f.' % (np.mean(de2[used])))
n_reject_mean_de2 += 1
#continue
if abs(np.std(dT[used] / T[used])) > 0.1:
print('std dT/T = %f.' % (np.std(dT[used] / T[used])))
n_reject_std_dt += 1
#continue
if abs(np.std(de1[used])) > 0.1:
print('std de1 = %f.' % (np.std(de1[used])))
n_reject_std_de1 += 1
#continue
if abs(np.std(de2[used])) > 0.1:
print('std de2 = %f.' % (np.std(de2[used])))
n_reject_std_de2 += 1
#continue
if abs(np.mean(rho1[used])) > 5.e-4:
print('mean rho1 = %s.' % (np.mean(rho1[used])))
n_reject_rho2 += 1
#continue
if abs(np.mean(rho2[used])) > 5.e-4:
print('mean rho2 = %s.' % (np.mean(rho2[used])))
n_reject_rho2 += 1
#continue
if abs(np.mean(e1[used])) > 0.03:
print('mean e1 = %f.' % (np.mean(e1[used])))
n_reject_mean_e1 += 1
#continue
if abs(np.mean(e2[used])) > 0.03:
print('mean e2 = %f.' % (np.mean(e2[used])))
n_reject_mean_e2 += 1
#continue
# Filter out egregiously bad values. Just in case.
good = (abs(dT / T) < 0.1) & (abs(de1) < 0.1) & (abs(de2) < 0.1)
n1 = np.sum(mask)
mask = mask & good
n2 = np.sum(mask)
print('"good" filter removed %d/%d objects' % (n1 - n2, n1))
n_good_obj += n2
n_bad_obj += n1 - n2
#print('mask = ',len(mask),np.sum(mask),mask)
mask = np.where(mask)[0]
#print('mask = ',len(mask),mask)
if frac != 1.:
mask = np.random.choice(mask, int(frac * len(mask)), replace=False)
#print('mask = ',len(mask),mask)
ngood = len(mask)
print('ngood = ', ngood, '/', len(data))
assert ngood == len(data[mask])
if ngood == 0:
print('All objects in exp %d are flagged.' % expnum)
continue
# Start with just the input keys, which should be columns in data.
for key in keys:
all_data[key].append(data[key][mask])
# Now add the extra ones we added to all_keys
if 'x' in keys:
# Convert to focal position.
x, y = toFocal(ccdnums[mask], data['x'][mask], data['y'][mask])
# This comes back in units of mm. Convert to arcsec.
# 1 pixel = 15e-3 mm = 0.263 arcsec
x *= 0.263 / 15e-3
y *= 0.263 / 15e-3
all_data['fov_x'].append(x)
all_data['fov_y'].append(y)
all_data['ccd'].append(ccdnums[mask])
all_data['exp'].append([expnum] * ngood)
all_data['band'].append([band] * ngood)
all_data['tiling'].append([tiling] * ngood)
bands.add(band)
tilings.add(tiling)
nrows += ngood
print('\nFinished processing %d exposures' % len(exps))
print('bands = ', bands)
print('tilings = ', tilings)
print('total "good" stars = ', n_good_obj)
print('total "bad" stars = ', n_bad_obj)
print('total good stars selected = ', nrows)
print('Potential rejections: (not enabled):')
print('n_reject_mean_dt = ', n_reject_mean_dt)
print('n_reject_mean_de1 = ', n_reject_mean_de1)
print('n_reject_mean_de2 = ', n_reject_mean_de2)
print('n_reject_mean_e1 = ', n_reject_mean_de1)
print('n_reject_mean_e2 = ', n_reject_mean_de2)
print('n_reject_std_dt = ', n_reject_std_dt)
print('n_reject_std_de1 = ', n_reject_std_de1)
print('n_reject_std_de2 = ', n_reject_std_de2)
print('n_reject_rho2 = ', n_reject_rho2)
# Turn the data into a recarray
# Pick appropriate formats for each kind of data
formats = []
for key in all_keys:
if key == 'ccd' or key == 'tiling':
formats.append('i2')
elif key == 'exp' or 'flag' in key:
formats.append('i4')
elif key == 'band':
formats.append('a1')
else:
formats.append('f8')
data = np.recarray(shape=(nrows, ), formats=formats, names=all_keys)
#print('data.dtype = ',data.dtype)
for key in all_keys:
data[key] = np.concatenate(all_data[key])
print('made recarray')
return data, bands, tilings
def bin_by_fov(ccd, x, y, dT, de1, de2, bands):
import numpy as np
import matplotlib.pyplot as plt
from toFocal import toFocal
all_x = np.array([])
all_y = np.array([])
all_e1 = np.array([])
all_e2 = np.array([])
all_T = np.array([])
nwhisk = 5
x_bins = np.linspace(0, 2048, nwhisk + 1)
y_bins = np.linspace(0, 4096, 2 * nwhisk + 1)
ccdnums = np.unique(ccd)
for ccdnum in ccdnums:
mask = np.where(ccd == ccdnum)[0]
print('ccdnum = ', ccdnum, ', nstar = ', mask.sum())
if mask.sum() < 100: continue
x_index = np.digitize(x[mask], x_bins)
y_index = np.digitize(y[mask], y_bins)
bin_de1 = np.array([
de1[mask][(x_index == i) & (y_index == j)].mean()
for i in range(1, len(x_bins)) for j in range(1, len(y_bins))
])
print('bin_de1 = ', bin_de1)
bin_de2 = np.array([
de2[mask][(x_index == i) & (y_index == j)].mean()
for i in range(1, len(x_bins)) for j in range(1, len(y_bins))
])
print('bin_de2 = ', bin_de2)
bin_dT = np.array([
dT[mask][(x_index == i) & (y_index == j)].mean()
for i in range(1, len(x_bins)) for j in range(1, len(y_bins))
])
print('bin_dT = ', bin_dT)
bin_x = np.array([
x[mask][(x_index == i) & (y_index == j)].mean()
for i in range(1, len(x_bins)) for j in range(1, len(y_bins))
])
print('bin_x = ', bin_x)
bin_y = np.array([
y[mask][(x_index == i) & (y_index == j)].mean()
for i in range(1, len(x_bins)) for j in range(1, len(y_bins))
])
print('bin_y = ', bin_y)
bin_count = np.array([((x_index == i) & (y_index == j)).sum()
for i in range(1, len(x_bins))
for j in range(1, len(y_bins))])
print('bin_count = ', bin_count)
focal_x, focal_y = toFocal(ccdnum, bin_x, bin_y)
mask2 = np.where(bin_count > 0)[0]
print('num with count > 0 = ', mask2.sum())
all_x = np.append(all_x, focal_x[mask2])
all_y = np.append(all_y, focal_y[mask2])
all_e1 = np.append(all_e1, bin_de1[mask2])
all_e2 = np.append(all_e2, bin_de2[mask2])
all_T = np.append(all_T, bin_dT[mask2])
plt.clf()
#plt.title('PSF Ellipticity residuals in DES focal plane')
theta = np.arctan2(all_e2, all_e1) / 2.
r = np.sqrt(all_e1**2 + all_e2**2)
u = r * np.cos(theta)
v = r * np.sin(theta)
plt.xlim(-250, 250)
plt.ylim(-250, 250)
print('all_x = ', all_x)
print('len(all_x) = ', len(all_x))
qv = plt.quiver(all_x,
all_y,
u,
v,
pivot='middle',
scale_units='xy',
headwidth=0.,
headlength=0.,
headaxislength=0.,
width=0.001,
scale=1.e-3,
color='blue')
ref_scale = 0.01
if 'raw' in bands:
ref_label = 'e = ' + str(ref_scale)
else:
ref_label = 'de = ' + str(ref_scale)
plt.quiverkey(qv,
0.10,
0.08,
ref_scale,
ref_label,
coordinates='axes',
color='darkred',
labelcolor='darkred',
labelpos='E',
fontproperties={'size': 'x-small'})
plt.axis('off')
plt.savefig('de_fov_' + bands + '.png')
plt.savefig('de_fov_' + bands + '.pdf')
plt.savefig('de_fov_' + bands + '.eps')
def bin_by_fov(ccd, x, y, e1, e2, s, w=None, nwhisk=5):
from toFocal import toFocal
all_x = np.array([])
all_y = np.array([])
all_e1 = np.array([])
all_e2 = np.array([])
all_s = np.array([])
if w is None:
w = np.ones(len(x))
x_bins = np.linspace(0,2048,nwhisk+1)
y_bins = np.linspace(0,4096,2*nwhisk+1)
print('x_bins = ',x_bins)
print('y_bins = ',y_bins)
ccdnums = np.unique(ccd)
print('ccdnums = ',ccdnums)
for ccdnum in ccdnums:
mask = np.where(ccd == ccdnum)[0]
print('ccdnum = ',ccdnum,', nstar = ',len(mask))
if mask.sum() < 100: continue
if ccdnum in [31, 61]: continue
x_index = np.digitize(x[mask], x_bins)
y_index = np.digitize(y[mask], y_bins)
nbins = (len(x_bins)-1) * (len(y_bins)-1)
bin_e1 = np.empty(nbins)
bin_e2 = np.empty(nbins)
bin_s = np.empty(nbins)
bin_x = np.empty(nbins)
bin_y = np.empty(nbins)
bin_nz = np.empty(nbins, dtype=bool)
sumesq = 0.
for i in range(1, len(x_bins)):
for j in range(1, len(y_bins)):
k = (i-1)*(len(y_bins)-1) + (j-1)
mask2 = np.where( (x_index == i) & (y_index == j) )[0]
ww = w[mask][mask2]
ws = ww.sum()
bin_e1[k] = (ww * e1[mask][mask2]).sum() / ws
bin_e2[k] = (ww * e2[mask][mask2]).sum() / ws
bin_s[k] = (ww * s[mask][mask2]).sum() / ws
bin_x[k] = (ww * x[mask][mask2]).sum() / ws
bin_y[k] = (ww * y[mask][mask2]).sum() / ws
bin_nz[k] = len(mask2) > 0
print(i,j,k,len(mask2), bin_e1[k], bin_e2[k])
sumesq += bin_e1[k]**2 + bin_e2[k]**2
print('rms e = ',np.sqrt(sumesq / nbins))
print('ccdnum = ',ccdnum)
print('bin_x,y = ',bin_x,bin_y)
focal_x, focal_y = toFocal(int(ccdnum), bin_x, bin_y)
print('x,y = ',focal_x, focal_y)
print('num with count > 0 = ',bin_nz.sum())
all_x = np.append(all_x, focal_x[bin_nz])
all_y = np.append(all_y, focal_y[bin_nz])
all_e1 = np.append(all_e1, bin_e1[bin_nz])
all_e2 = np.append(all_e2, bin_e2[bin_nz])
all_s = np.append(all_s, bin_s[bin_nz])
return all_x, all_y, all_e1, all_e2, all_s
def read_data(args, work, limit_bands=None, prefix='piff'):
import fitsio
RESERVED = 64
BAD_CCDS = [2, 31, 61]
if args.file != '':
print('Read file ',args.file)
with open(args.file) as fin:
exps = [ line.strip() for line in fin if line[0] != '#' ]
print('File included %d exposures'%len(exps))
else:
exps = args.exps
print('Explicit listing of %d exposures'%len(exps))
exps = sorted(exps)
keys = ['ra', 'dec', 'x', 'y', 'mag', 'obs_e1', 'obs_e2', 'obs_T',
prefix+'_e1', prefix+'_e2', prefix+'_T']
all_data = { key : [] for key in keys }
all_keys = keys
all_data['exp'] = []
all_data['ccd'] = []
all_keys = all_keys + ['exp', 'ccd' ]
if 'x' in keys:
all_data['fov_x'] = []
all_data['fov_y'] = []
all_keys = all_keys + ['fov_x', 'fov_y']
inkeys = keys
all_bands = [] # This keeps track of the band for each record
#all_tilings = [] # This keeps track of the tiling for each record
bands = set() # This is the set of all bands being used
#tilings = set() # This is the set of all tilings being used
for exp in exps:
print('Start work on exp = ',exp)
expnum = int(exp)
print('expnum = ',expnum)
expinfo = fitsio.read(os.path.join(work, exp, 'exp_info_%d.fits'%expnum))
if expnum not in expinfo['expnum']:
print('expnum is not in expinfo!')
print('expinfo[expnum] = ',expinfo['expnum'])
print('Could not find information about this expnum. Skipping ',run,exp)
continue
i = np.nonzero(expinfo['expnum'] == expnum)[0][0]
#print('i = ',i)
band = expinfo['band'][i]
#print('band[k] = ',band)
if (limit_bands is not None) and (band not in limit_bands):
print('Not doing band = %s.'%band)
continue
#tiling = int(expinfo['tiling'][k])
#print('tiling[k] = ',tiling)
#if tiling == 0:
# This shouldn't happen, but it did for a few exposures. Just skip them, since this
# might indicate some kind of problem.
#print('tiling == 0. Skip this exposure.')
#continue
#if tiling > args.max_tiling:
#print('tiling is > %d. Skip this exposure.'%args.max_tiling)
#continue
for k in range(len(expinfo)):
ccdnum = expinfo[k]['ccdnum']
if expinfo[k]['flag'] != 0:
print('Skipping ccd %d because it is blacklisted: '%ccdnum, expinfo[k]['flag'])
continue
if ccdnum in BAD_CCDS:
print('Skipping ccd %d because it is BAD'%ccdnum)
continue
cat_file = os.path.join(work, exp, "psf_cat_%d_%d.fits"%(expnum,ccdnum))
#print('cat_file = ',cat_file)
try:
data = fitsio.read(cat_file)
flag = data[prefix+'_flag']
except (OSError, IOError):
print('Unable to open cat_file %s. Skipping this file.'%cat_file)
continue
ntot = len(data)
nused = np.sum((flag & 1) != 0)
nreserved = np.sum((flag & RESERVED) != 0)
ngood = np.sum(flag == 0)
#print('nused = ',nused)
#print('nreserved = ',nreserved)
#print('ngood = ',ngood)
if args.use_reserved:
mask = (flag == RESERVED) | (flag == RESERVED+1)
else:
mask = (flag == 0)
#print('mask = ',mask)
T = data['obs_T']
dT = (data[prefix + '_T'] - data['obs_T'])
de1 = (data[prefix + '_e1'] - data['obs_e1'])
de2 = (data[prefix + '_e2'] - data['obs_e2'])
used = (flag == 0)
#if np.std(dT[used]/T[used]) > 0.03:
#continue
#if np.std(de1[used]) > 0.02:
#continue
#if np.std(de2[used]) > 0.02:
#continue
good = (abs(dT/data['obs_T']) < 0.1) & (abs(de1) < 0.1) & (abs(de2) < 0.1)
mask = mask & good
ngood = np.sum(mask)
#print('ngood = ',ngood,'/',len(data))
assert ngood == len(data[mask])
if ngood == 0:
print('All objects in ccd %d are flagged.'%ccdnum)
print('Probably due to astrometry flags. Skip this exposure.')
continue
for key, inkey in zip(keys, inkeys):
all_data[key].append(data[inkey][mask])
all_data['exp'].append([expnum] * ngood)
all_data['ccd'].append([ccdnum] * ngood)
if 'x' in keys:
# Convert to focal position.
x,y = toFocal(ccdnum, data['x'][mask], data['y'][mask])
# This comes back in units of mm. Convert to arcsec.
# 1 pixel = 15e-3 mm = 0.263 arcsec
x *= 0.263/15e-3
y *= 0.263/15e-3
all_data['fov_x'].append(x)
all_data['fov_y'].append(y)
all_bands.extend( ([band] * ngood) )
#all_tilings.extend( ([tiling] * ngood) )
bands.add(band)
#tilings.add(tiling)
print('\nFinished processing all exposures')
print('bands = ',bands)
#print('tilings = ',tilings)
# Turn the data into a recarray
print('all_data.keys = ',all_data.keys())
formats = ['f8'] * len(all_keys) + ['a1', 'i2']
#names = all_keys + ['band', 'tiling']
names = all_keys + ['band']
data = np.recarray(shape = (len(all_bands),),
formats = formats, names = names)
print('data.dtype = ',data.dtype)
for key in all_keys:
data[key] = np.concatenate(all_data[key])
data['band'] = all_bands
#data['tiling'] = all_tilings
print('made recarray')
tilings = None
return data, bands, tilings
