def do_cross_tiling_stats(data,
bands,
tilings,
work,
prefix='piff',
name='cross'):
print('Start CROSS_TILING: ', prefix, name)
# Measure the rho stats using only cross-correlations between tiles.
use_bands = band_combinations(bands)
for band in use_bands:
tile_data = []
for til in tilings:
print('til = ', til)
mask = np.in1d(data['band'], band) & (data['tiling'] == til)
print('sum(mask) = ', np.sum(mask))
print('len(data[mask]) = ', len(data[mask]))
tile_data.append(data[mask])
tag = ''.join(band)
tags = [tag + ":" + str(til) for til in tilings]
stats = measure_cross_rho(tile_data,
max_sep=300,
tags=tags,
prefix=prefix)
stat_file = os.path.join(work, "rho_%s_%s.json" % (name, tag))
write_stats(stat_file, *stats)
def do_canonical_stats(data,
bands,
tilings,
work,
max_mag,
prefix='piff',
name='all',
alt_tt=False,
opt=None,
subtract_mean=False,
do_rho0=False):
print('Start CANONICAL: ', prefix, name)
# Measure the canonical rho stats using all pairs:
use_bands = band_combinations(bands)
for band in use_bands:
print('band ', band)
if len(band) > 1: band = list(band)
mask = np.in1d(data['band'], band)
print('sum(mask) = ', np.sum(mask))
print('len(data[mask]) = ', len(data[mask]))
tag = ''.join(band)
stats = measure_rho(data[mask],
max_sep=300,
max_mag=max_mag,
tag=tag,
prefix=prefix,
alt_tt=alt_tt,
opt=opt,
subtract_mean=subtract_mean,
do_rho0=do_rho0)
stat_file = os.path.join(work, "rho_%s_%s.json" % (name, tag))
write_stats(stat_file, *stats)
def do_fov_stats(data, bands, tilings, work, prefix='piff', name='fov'):
print('Start FOV: ',prefix,name)
# Measure the rho stats using the field-of-view positions.
use_bands = band_combinations(bands)
for band in use_bands:
print('band ',band)
mask = np.in1d(data['band'],band)
print('sum(mask) = ',np.sum(mask))
print('len(data[mask]) = ',len(data[mask]))
tag = ''.join(band)
stats = measure_rho(data[mask], max_sep=300, tag=tag, prefix=prefix, use_xy=True)
stat_file = os.path.join(work, "rho_%s_%s.json"%(name,tag))
write_stats(stat_file,*stats)
def do_fov_stats(data, bands, tilings, work, max_mag, prefix='piff', name='fov'):
print('Start FOV: ',prefix,name)
# Measure the rho stats using the field-of-view positions.
use_bands = band_combinations(bands)
for band in use_bands:
print('band ',band)
mask = np.in1d(data['band'],band)
print('sum(mask) = ',np.sum(mask))
print('len(data[mask]) = ',len(data[mask]))
tag = ''.join(band)
stats = measure_rho(data[mask], max_sep=300, max_mag=max_mag, tag=tag, prefix=prefix,
use_xy=True)
stat_file = os.path.join(work, "rho_%s_%s.json"%(name,tag))
write_stats(stat_file,*stats)
def do_odd_even_stats(data, bands, tilings, work, prefix='piff', name='oddeven'):
print('Start ODD_EVEN: ',prefix,name)
# Measure the rho stats using only cross-correlations between odd vs even tilings.
use_bands = band_combinations(bands)
for band in use_bands:
print('odd/even ',band)
odd = np.in1d(data['band'], band) & (data['tiling'] % 2 == 1)
even = np.in1d(data['band'], band) & (data['tiling'] % 2 == 0)
cats = [ data[odd], data[even] ]
tag = ''.join(band)
tags = [ tag + ":odd", tag + ":even" ]
stats = measure_cross_rho(cats, max_sep=300, tags=tags, prefix=prefix)
stat_file = os.path.join(work, "rho_%s_%s.json"%(name,tag))
write_stats(stat_file,*stats)
def do_cross_band_stats(data, bands, tilings, work, prefix='piff', name='crossband'):
print('Start CROSS_BAND: ',prefix,name)
# Measure the rho stats cross-correlating the different bands.
use_bands = band_combinations(bands, single=False)
for band in use_bands:
print('cross bands ',band)
band_data = []
for b in band:
mask = data['band'] == b
band_data.append(data[mask])
stats = measure_cross_rho(band_data, max_sep=300, tags=band, prefix=prefix)
tag = ''.join(band)
stat_file = os.path.join(work, "rho_%s_%s.json"%(name,tag))
write_stats(stat_file,*stats)
def do_odd_even_stats(data, bands, tilings, work, prefix='piff', name='oddeven'):
print('Start ODD_EVEN: ',prefix,name)
# Measure the rho stats using only cross-correlations between odd vs even tilings.
use_bands = band_combinations(bands)
for band in use_bands:
print('odd/even ',band)
odd = np.in1d(data['band'], band) & (data['tiling'] % 2 == 1)
even = np.in1d(data['band'], band) & (data['tiling'] % 2 == 0)
cats = [ data[odd], data[even] ]
tag = ''.join(band)
tags = [ tag + ":odd", tag + ":even" ]
stats = measure_cross_rho(cats, max_sep=300, tags=tags, prefix=prefix)
stat_file = os.path.join(work, "rho_%s_%s.json"%(name,tag))
write_stats(stat_file,*stats)
def do_cross_band_stats(data, bands, tilings, work, prefix='piff', name='crossband'):
print('Start CROSS_BAND: ',prefix,name)
# Measure the rho stats cross-correlating the different bands.
use_bands = band_combinations(bands, single=False)
for band in use_bands:
print('cross bands ',band)
band_data = []
for b in band:
mask = data['band'] == b
band_data.append(data[mask])
stats = measure_cross_rho(band_data, max_sep=300, tags=band, prefix=prefix)
tag = ''.join(band)
stat_file = os.path.join(work, "rho_%s_%s.json"%(name,tag))
write_stats(stat_file,*stats)
def do_cross_tiling_stats(data, bands, tilings, work, prefix='piff', name='cross'):
print('Start CROSS_TILING: ',prefix,name)
# Measure the rho stats using only cross-correlations between tiles.
use_bands = band_combinations(bands)
for band in use_bands:
tile_data = []
for til in tilings:
print('til = ',til)
mask = np.in1d(data['band'],band) & (data['tiling'] == til)
print('sum(mask) = ',np.sum(mask))
print('len(data[mask]) = ',len(data[mask]))
tile_data.append(data[mask])
tag = ''.join(band)
tags = [ tag + ":" + str(til) for til in tilings ]
stats = measure_cross_rho(tile_data, max_sep=300, tags=tags, prefix=prefix)
stat_file = os.path.join(work, "rho_%s_%s.json"%(name,tag))
write_stats(stat_file,*stats)
def do_canonical_stats(data, bands, tilings, work, max_mag, prefix='piff', name='all',
alt_tt=False, lucas=False, subtract_mean=False, do_rho0=False):
print('Start CANONICAL: ',prefix,name)
# Measure the canonical rho stats using all pairs:
use_bands = band_combinations(bands)
for band in use_bands:
print('band ',band)
if len(band) > 1: band = list(band)
mask = np.in1d(data['band'],band)
print('sum(mask) = ',np.sum(mask))
print('len(data[mask]) = ',len(data[mask]))
tag = ''.join(band)
stats = measure_rho(data[mask], max_sep=300, max_mag=max_mag, tag=tag, prefix=prefix,
alt_tt=alt_tt, lucas=lucas, subtract_mean=subtract_mean,
do_rho0=do_rho0)
stat_file = os.path.join(work, "rho_%s_%s.json"%(name,tag))
write_stats(stat_file,*stats)
def main():
matplotlib.use('Agg') # needs to be done before import pyplot
args = parse_args()
work = os.path.expanduser(args.work)
print('work dir = ',work)
if args.file != '':
print('Read file ',args.file)
with open(args.file) as fin:
exps = [ line.strip() for line in fin if line[0] != '#' ]
else:
exps = args.exps
exps = sorted(exps)
if args.use_psfex:
prefix = 'psfex'
else:
prefix = 'piff'
keys = ['ra', 'dec', 'x', 'y', 'mag', 'obs_e1', 'obs_e2', 'obs_T', 'obs_flag',
prefix+'_e1', prefix+'_e2', prefix+'_T', prefix+'_flag']
data, bands, tilings = read_data(exps, work, keys, limit_bands=args.bands, prefix=prefix,
use_reserved=args.use_reserved, frac=args.frac)
use_bands = band_combinations(args.bands)
for bands in use_bands:
this_data = data[np.in1d(data['band'], list(bands))]
if len(this_data) == 0:
print('No files with bands ',bands)
continue
used = this_data[prefix+'_flag'] & ~RESERVED == 0
#airmass = this_data['airmass']
#sky = this_data['sky']
#sigsky = this_data['sigsky']
#fwhm = this_data['fwhm']
#med_airmass = np.median(airmass)
#med_sky = np.median(sky)
#med_sigsky = np.median(sigsky)
#med_fwhm = np.median(fwhm)
#print('airmass: ',min(airmass),med_airmass,max(airmass))
#print('sky: ',min(sky),med_sky,max(sky))
#print('sigsky: ',min(sigsky),med_sigsky,max(sigsky))
#print('fwhm: ',min(fwhm),med_fwhm,max(fwhm))
ra = this_data['ra']
dec = this_data['dec']
x = this_data['x']
y = this_data['y']
m = this_data['mag']
print('full mag range = ',np.min(m),np.max(m))
print('used mag range = ',np.min(m[used]),np.max(m[used]))
e1 = this_data['obs_e1']
print('mean e1 = ',np.mean(e1))
e2 = this_data['obs_e2']
print('mean e2 = ',np.mean(e2))
T = this_data['obs_T']
print('mean s = ',np.mean(T))
pe1 = this_data[prefix+'_e1']
print('mean pe1 = ',np.mean(pe1))
pe2 = this_data[prefix+'_e2']
print('mean pe2 = ',np.mean(pe2))
pT = this_data[prefix+'_T']
print('mean pT = ',np.mean(pT))
print('min mag = ',np.min(m))
print('max mag = ',np.max(m))
print('mean T (used) = ',np.mean(T[used]))
print('mean e1 (used) = ',np.mean(e1[used]))
print('mean e2 (used) = ',np.mean(e2[used]))
print('mean pT (used) = ',np.mean(pT[used]))
print('mean pe1 (used) = ',np.mean(pe1[used]))
print('mean pe2 (used) = ',np.mean(pe2[used]))
de1 = e1 - pe1
de2 = e2 - pe2
dT = T - pT
print('mean dT (used) = ',np.mean(dT[used]))
print('mean de1 (used) = ',np.mean(de1[used]))
print('mean de2 (used) = ',np.mean(de2[used]))
if args.use_psfex:
min_mused = 0
else:
min_mused = np.min(m[used])
print('min_mused = ',min_mused)
bin_by_mag(m, dT, de1, de2, min_mused, bands)
bin_by_mag(m[used], dT[used], de1[used], de2[used], min_mused, bands+'_used')
make_hist(dT, T, de1, de2, bands)
make_hist(dT[used], T[used], de1[used], de2[used], bands+'_used')
def main():
matplotlib.use('Agg') # needs to be done before import pyplot
args = parse_args()
work = os.path.expanduser(args.work)
print('work dir = ', work)
if args.file != '':
print('Read file ', args.file)
with open(args.file) as fin:
exps = [line.strip() for line in fin if line[0] != '#']
else:
exps = args.exps
exps = sorted(exps)
if args.use_psfex:
prefix = 'psfex'
else:
prefix = 'piff'
keys = [
'ra', 'dec', 'x', 'y', 'mag', 'obs_e1', 'obs_e2', 'obs_T', 'obs_flag',
prefix + '_e1', prefix + '_e2', prefix + '_T', prefix + '_flag'
]
if not args.use_dat:
data, bands, tilings = read_data(exps,
work,
keys,
limit_bands=args.bands,
prefix=prefix,
use_reserved=args.use_reserved,
frac=args.frac)
use_bands = band_combinations(args.bands)
for bands in use_bands:
if not args.use_dat:
this_data = data[np.in1d(data['band'], list(bands))]
if len(this_data) == 0:
print('No files with bands ', bands)
continue
print('bands = ', bands)
print('unique flags = ', np.unique(this_data[prefix + '_flag']))
if args.use_reserved:
RESERVED = 65
used = this_data[prefix + '_flag'] & ~RESERVED == 0
else:
used = this_data[prefix + '_flag'] == 0
print('used = ', used)
print('unique used = ', np.unique(used))
ra = this_data['ra']
dec = this_data['dec']
x = this_data['x']
y = this_data['y']
m = this_data['mag']
print('full mag range = ', np.min(m), np.max(m))
print('used mag range = ', np.min(m[used]), np.max(m[used]))
e1 = this_data['obs_e1']
print('mean e1 = ', np.mean(e1))
e2 = this_data['obs_e2']
print('mean e2 = ', np.mean(e2))
T = this_data['obs_T']
print('mean s = ', np.mean(T))
pe1 = this_data[prefix + '_e1']
print('mean pe1 = ', np.mean(pe1))
pe2 = this_data[prefix + '_e2']
print('mean pe2 = ', np.mean(pe2))
pT = this_data[prefix + '_T']
print('mean pT = ', np.mean(pT))
print('min mag = ', np.min(m))
print('max mag = ', np.max(m))
print('mean T (used) = ', np.mean(T[used]))
print('mean e1 (used) = ', np.mean(e1[used]))
print('mean e2 (used) = ', np.mean(e2[used]))
print('mean pT (used) = ', np.mean(pT[used]))
print('mean pe1 (used) = ', np.mean(pe1[used]))
print('mean pe2 (used) = ', np.mean(pe2[used]))
de1 = e1 - pe1
de2 = e2 - pe2
dT = T - pT
print('mean dT (used) = ', np.mean(dT[used]))
print('mean de1 (used) = ', np.mean(de1[used]))
print('mean de2 (used) = ', np.mean(de2[used]))
if args.use_psfex:
min_mused = 0
elif args.use_dat:
# Save these by hand. TODO: put this in output file?
d = {'r': 15.0976, 'i': 15.2468, 'z': 14.8781, 'riz': 14.8781}
if not bands in d: continue
min_mused = d[bands]
else:
min_mused = np.min(m[used])
print('min_mused = ', min_mused)
if args.use_dat:
infile1 = 'dpsf_mag_' + bands + '.dat'
bin_data1 = read_bins(infile1)
infile2 = 'dpsf_mag_' + bands + '_used.dat'
bin_data2 = read_bins(infile2)
else:
bin_data1 = bin_by_mag(m, dT, de1, de2, min_mused)
outfile1 = 'dpsf_mag_' + bands + '.dat'
write_bins(bin_data1, outfile1)
bin_data2 = bin_by_mag(m[used], dT[used], de1[used], de2[used],
min_mused)
outfile2 = 'dpsf_mag_' + bands + '_used.dat'
write_bins(bin_data2, outfile2)
pdffile1 = 'dpsf_mag_' + bands + '.pdf'
plot_bins(bin_data1, pdffile1)
pdffile2 = 'dpsf_mag_' + bands + '_used.pdf'
plot_bins(bin_data2, pdffile2, min_mused)
def main():
matplotlib.use('Agg') # needs to be done before import pyplot
args = parse_args()
work = os.path.expanduser(args.work)
print('work dir = ', work)
if args.file != '':
print('Read file ', args.file)
with open(args.file) as fin:
exps = [line.strip() for line in fin if line[0] != '#']
else:
exps = args.exps
exps = sorted(exps)
if args.use_psfex:
prefix = 'psfex'
else:
prefix = 'piff'
keys = [
'ra', 'dec', 'x', 'y', 'mag', 'obs_e1', 'obs_e2', 'obs_T', 'obs_flag',
prefix + '_e1', prefix + '_e2', prefix + '_T', prefix + '_flag'
]
data, bands, tilings = read_data(exps,
work,
keys,
limit_bands=args.bands,
prefix=prefix,
use_reserved=args.use_reserved,
frac=args.frac)
use_bands = band_combinations(args.bands)
for bands in use_bands:
this_data = data[np.in1d(data['band'], list(bands))]
if len(this_data) == 0:
print('No files with bands ', bands)
continue
used = this_data[prefix + '_flag'] & ~RESERVED == 0
#airmass = this_data['airmass']
#sky = this_data['sky']
#sigsky = this_data['sigsky']
#fwhm = this_data['fwhm']
#med_airmass = np.median(airmass)
#med_sky = np.median(sky)
#med_sigsky = np.median(sigsky)
#med_fwhm = np.median(fwhm)
#print('airmass: ',min(airmass),med_airmass,max(airmass))
#print('sky: ',min(sky),med_sky,max(sky))
#print('sigsky: ',min(sigsky),med_sigsky,max(sigsky))
#print('fwhm: ',min(fwhm),med_fwhm,max(fwhm))
ra = this_data['ra']
dec = this_data['dec']
x = this_data['x']
y = this_data['y']
m = this_data['mag']
print('full mag range = ', np.min(m), np.max(m))
print('used mag range = ', np.min(m[used]), np.max(m[used]))
e1 = this_data['obs_e1']
print('mean e1 = ', np.mean(e1))
e2 = this_data['obs_e2']
print('mean e2 = ', np.mean(e2))
T = this_data['obs_T']
print('mean s = ', np.mean(T))
pe1 = this_data[prefix + '_e1']
print('mean pe1 = ', np.mean(pe1))
pe2 = this_data[prefix + '_e2']
print('mean pe2 = ', np.mean(pe2))
pT = this_data[prefix + '_T']
print('mean pT = ', np.mean(pT))
print('min mag = ', np.min(m))
print('max mag = ', np.max(m))
print('mean T (used) = ', np.mean(T[used]))
print('mean e1 (used) = ', np.mean(e1[used]))
print('mean e2 (used) = ', np.mean(e2[used]))
print('mean pT (used) = ', np.mean(pT[used]))
print('mean pe1 (used) = ', np.mean(pe1[used]))
print('mean pe2 (used) = ', np.mean(pe2[used]))
de1 = e1 - pe1
de2 = e2 - pe2
dT = T - pT
print('mean dT (used) = ', np.mean(dT[used]))
print('mean de1 (used) = ', np.mean(de1[used]))
print('mean de2 (used) = ', np.mean(de2[used]))
if args.use_psfex:
min_mused = 0
else:
min_mused = np.min(m[used])
print('min_mused = ', min_mused)
bin_by_mag(m, dT, de1, de2, min_mused, bands)
bin_by_mag(m[used], dT[used], de1[used], de2[used], min_mused,
bands + '_used')
make_hist(dT, T, de1, de2, bands)
make_hist(dT[used], T[used], de1[used], de2[used], bands + '_used')
