def write(self):
# Load 2point fits file with n(z) info.
fits = twopoint.TwoPointFile.from_fits(self.input_path("2pt"),
covmat_name=None)
# Write file without covariance (all data vectors)
fits.spectra = self.exts
fits.to_fits(self.output_path("2pt_extended"), clobber=True)
if self.params['lensfile'] != 'None':
self.strip_wtheta(fits)
import pdb
pdb.set_trace()
self.strip_missing_gglensing(fits)
length = self.get_cov_lengths(fits)
self.sort_2pt(fits, length)
# Writes the covariance info into a covariance object and saves to 2point fits file.
fits.covmat_info = twopoint.CovarianceMatrixInfo(
'COVMAT', TWO_POINT_NAMES, length, self.covmat[0])
fits.to_fits(self.output_path("2pt_g"), clobber=True)
fits.covmat_info = twopoint.CovarianceMatrixInfo(
'COVMAT', TWO_POINT_NAMES, length, self.covmat[1])
fits.to_fits(self.output_path("2pt_ng"), clobber=True)
print "Have disabled covmat cleanup"
def covmat_from_block(block, spectra, sky_area, number_density_shear_bin,
number_density_lss_bin, sigma_e_bin):
getter = ObservedClGetter(number_density_shear_bin, number_density_lss_bin,
sigma_e_bin)
C = []
names = []
starts = []
lengths = []
# s and t index the spectra that we have. e.g. s or t=1 might be the full set of
# shear-shear measuremnts
x = 0
for s, AB in enumerate(spectra[:]):
M = []
starts.append(x)
L = len(AB)
lengths.append(L)
x += L
names.append(AB.name)
for t, CD in enumerate(spectra[:]):
print(
"Looking at covariance between {} and {} (s={}, t={})".format(
AB.name, CD.name, s, t))
# We only calculate the upper triangular.
# Get the lower triangular here. We have to
# transpose it compared to the upper one.
if s > t:
MI = C[t][s].T
else:
MI = gaussian_covariance.compute_gaussian_covariance(
sky_area, getter.lookup, block, AB, CD)
M.append(MI)
C.append(M)
C = np.vstack([np.hstack(CI) for CI in C])
info = twopoint.CovarianceMatrixInfo("COVMAT", names, lengths, C)
return info
twopoint_filename = sys.argv[1]
covmat_filename = sys.argv[2]
new_filename = sys.argv[3]
print("Loading 2pt data in {}".format(twopoint_filename))
data = twopoint.TwoPointFile.from_fits(twopoint_filename, covmat_name=None)
print("Loading text covariance file from {}".format(covmat_filename))
covmat = np.loadtxt(covmat_filename)
print("Now you need to make sure that the ordering of the covariance matrix")
print("matches the ordering in the text file!")
print("Expecting this order:")
print("#Name Bin1 Bin2 Angle")
for s in data.spectra:
for b1, b2, ang in zip(s.bin1, s.bin2, s.angle):
print(s.name, b1, b2, ang)
replace_this_with_some_reordering_if_needed()
# The ordering of the
names = [s.name for s in data.spectra]
lengths = [len(s) for s in data.spectra]
n = sum(lengths)
assert covmat.shape == (n, n)
data.covmat_info = twopoint.CovarianceMatrixInfo("COVMAT", names, lengths,
covmat)
data.to_fits(new_filename)
def execute(block, config):
real_space = config['real_space']
fourier_space = not real_space
filename = config['filename']
#shear_nz = config['shear_nz']
#position_nz = config['position_nz']
overwrite = config['overwrite']
make_covariance = config['make_covariance']
print("Saving two-point data to {}".format(filename))
theory_spec_list = []
cl_theory_spec_list = []
cl_to_xi_type_list = []
spec_meas_list = []
kernels = []
no_kernel_found = []
#Loop through spectrum_sections, generating a SpectrumMeasurement
#for each.
#If we're generating the covariance for real space spectra, also
#generate a TheorySpectrum for the corresponding Cl.
print("Generating twopoint file with the following spectra:")
print(config['spectrum_sections'])
for i_spec in range( len(config["spectrum_sections"]) ):
spectrum_section = config["spectrum_sections"][i_spec]
output_extension = config["output_extensions"][i_spec]
#Read in sample information from block
sample_a, sample_b = ( block[spectrum_section, "sample_a"],
block[spectrum_section, "sample_b"] )
kernel_name_a, kernel_name_b = "nz_"+sample_a, "nz_"+sample_b
#Get kernels
if (kernel_name_a not in [ k.name for k in kernels ]) and (kernel_name_a not in no_kernel_found):
if block.has_section(kernel_name_a):
kernels.append( twopoint.NumberDensity.from_block( block, kernel_name_a ) )
else:
no_kernel_found.append(kernel_name_a)
if kernel_name_b not in [ k.name for k in kernels ]:
if block.has_section(kernel_name_b):
kernels.append( twopoint.NumberDensity.from_block( block, kernel_name_b ) )
else:
no_kernel_found.append(kernel_name_b)
if len(no_kernel_found)>0:
print("No kernel found for kernel names:", no_kernel_found)
print("This might not be a problem e.g. for CMB lensing.")
theory_spec = TheorySpectrum.from_block( block, spectrum_section )
theory_spec_list.append(theory_spec)
#get angle_units
if config["angle_units"] is not None:
angle_units = config['angle_units'].name
else:
angle_units = None
spec_meas_list.append( theory_spec.get_spectrum_measurement( config['angle_mids_userunits'],
(kernel_name_a, kernel_name_b), output_extension, angle_lims = config['angle_lims_userunits'],
angle_units=angle_units ) )
if make_covariance:
if real_space:
#In this case we also need the corresponding Cl spectra to generate the covariance
cl_section = config["cl_sections"][i_spec]
cl_spec = TheorySpectrum.from_block( block, cl_section )
cl_theory_spec_list.append( cl_spec )
#Check cls have the same bin pairings as their corresponding real-space spectra
try:
assert cl_spec.bin_pairs == theory_spec_list[i_spec].bin_pairs
except AssertionError as e:
print( "cl and xi specs have different bin_pairs:" )
print( "sections were %s and %s"%(cl_section, spectrum_section))
print( "cl bin pairs:", cl_spec.bin_pairs )
print( "xi bin pairs:", theory_spec_list[i_spec].bin_pairs)
raise(e)
if not real_space:
cl_theory_spec_list = theory_spec_list
if make_covariance:
#First we need to get the ClCov
#For the covariance matrix, we may need to read in more Cls - e.g. if we want
#a covariance for Cl_ab and Cl_cd, we require Cl_ad, Cl_ac, Cl_bc and Cl_bd for
#the covariance calculation.
#So the following checks the types of Cl_ab and Cl_cd, and from that infers
#the required spectra.
types = []
for spec in cl_theory_spec_list:
type_i, type_j = spec.types
if type_i not in types:
types.append(type_i)
if type_j not in types:
types.append(type_j)
cl_specs = []
for (i,type_1) in enumerate(types):
for (j,type_2) in enumerate(types[i:]):
print("Getting cls for cov:")
print("type_1:", type_1)
print("type_2:", type_2)
#Get the cl section name for these types from the type_table (man
#we need to sort out easy access to this type_table info...it sucks right now)
try:
cl_section = type_table[(type_1.name, type_2.name)][0]
except KeyError:
cl_section = type_table[(type_2.name, type_1.name)][0]
assert cl_section not in [s.name for s in cl_specs]
cl_spec = TheorySpectrum.from_block( block, cl_section )
#Add noise if necessary
if (cl_spec.types[0] == cl_spec.types[1]):
if cl_spec.types[0].name == "galaxy_shear_emode_fourier":
noise = ([ (s**2 / 2 / n) for (s,n) in
zip(config['sigma_e'],config['number_density_shear_rad2']) ])
elif cl_spec.types[0].name == "galaxy_position_fourier":
noise = [ 1./n for n in config['number_density_lss_rad2'] ]
else:
print("Tried to, but can't generate noise for spectrum %s"%cl_section)
raise ValueError
cl_spec.set_noise(noise)
cl_specs.append( cl_spec )
cl_cov = ClCov(cl_specs, fsky=config['fsky'])
if real_space:
#If requested, apply bin cuts now - this will speed up the covariance calculation
#Need to apply cuts to real space spectra and cls
for (name, b1, b2) in config['bin_cuts']:
print("cutting %d,%d from %s"%(b1,b2,name))
spec_index = config['output_extensions'].index(name)
spec_meas_list[spec_index].cut_bin_pair( (b1,b2), complain=True )
cl_theory_spec_list[spec_index].cut_bin_pair( (b1,b2) )
cov_blocks, covmat, xi_starts, xi_lengths = real_space_cov( cl_cov,
cl_theory_spec_list, config['cl_to_xi_types'],
config['ell_max'], config['angle_lims'],
upsample=config['upsample_cov'],
high_l_filter = config['high_l_filter'] )
covmat_info = twopoint.CovarianceMatrixInfo( 'COVMAT', [s.name for s in spec_meas_list],
xi_lengths, covmat )
else:
covmat, cl_lengths = cl_cov.get_binned_cl_cov(config['angle_lims'])
assert covmat.shape[0] == sum([len(s.value) for s in spec_meas_list])
covmat_info = twopoint.CovarianceMatrixInfo( 'COVMAT', [s.name for s in spec_meas_list],
[len(s.value) for s in spec_meas_list], covmat )
else:
covmat_info = None
if not spec_meas_list:
raise ValueError("Sorry - I couldn't find any spectra to save.")
windows = []
data = twopoint.TwoPointFile(spec_meas_list, kernels, windows, covmat_info)
# Apply cuts
scale_cuts = config['scale_cuts']
bin_cuts = config['bin_cuts']
if scale_cuts or bin_cuts:
data.mask_scales(scale_cuts, bin_cuts)
data.to_fits(filename, overwrite=overwrite)
return 0
def main():
args = parse_args()
if args.q_cat is not None:
assert args.q_coeff_cat is not None
psf_data = read_psf_data(args)
#if requested, apply gold footprint mask and/or gold badregions mask
use = np.ones(len(psf_data), dtype=bool)
if args.gold_fp_mask:
fp = hu.Map("ring", hp.read_map(args.gold_fp_mask))
fp_vals = fp.get_mapval(psf_data['ra'], psf_data['dec'])
use[fp_vals < 1] = False
print float(use.sum()) / len(use)
if args.gold_br_mask:
br = hu.Map("ring", hp.read_map(args.gold_br_mask))
br_vals = br.get_mapval(psf_data['ra'], psf_data['dec'])
use[br_vals > 0] = False
print float(use.sum()) / len(use)
if use.sum() != len(use):
print 'gold masks leave fraction %f of stars' % (float(use.sum()) /
len(use))
psf_data = psf_data[use]
theta_min, theta_max, nbins = 0.25, 250., args.nbins
bin_slop = args.bin_slop
sep_units = 'arcmin'
gg = treecorr.GGCorrelation(nbins=nbins,
min_sep=theta_min,
max_sep=theta_max,
sep_units=sep_units,
verbose=1,
bin_slop=bin_slop)
if args.z_bin_lims:
num_z_bins = len(args.z_bin_lims) - 1
else:
num_z_bins = 1
shape_colnames = homog_colnames
#read data
if args.pipeline == 'metacal':
shape_data, shape_mask_0, shape_masks_sel, common_mask, read_rows_union, area_deg = get_mcal_cat(
args.shape_cat,
args.cut_dict,
mcal_cols=[
'e1', 'e2', 'psf_e1', 'psf_e2', 'R11', 'R22', 'ra', 'dec'
],
test_nrows=args.test_nrows,
add_Rmean_col=True)
if args.z_bin_lims:
#supplying z_bin_lims implies you want to some 'tomographic' redshift bin splitting, so read in some redshift info too
z_arrays = get_mcal_photoz(read_rows=read_rows_union)
#pylab.hist(z_arrays[0],bins=20)
#pylab.show()
#first one is the one used for the unsheared binning (and therefore the measurement), so add this to the shape_data array
shape_data = nmbot.add_field(shape_data, [("mean_z", float)],
[z_arrays[0]])
#now make a McalCat and apply correction
shape_cat = corrtools.McalCat(shape_data,
shape_mask_0,
shape_masks_sel,
quantities=[('e1', 'e2'),
('psf_e1', 'psf_e2')])
print len(shape_cat.arr_data)
if args.z_bin_lims:
print shape_cat.arr_data['mean_z'].min(
), shape_cat.arr_data['mean_z'].max()
shape_cat.redshift_split(zcol='mean_z', z_bin_lims=args.z_bin_lims)
shape_cat.sheared_redshift_split(z_arrs_sheared=z_arrays[1:])
shape_cat.apply_mcal_correction()
print shape_cat.bin_stats
else:
assert args.pipeline == "im3shape"
shape_data, read_rows, area_deg = get_im3_cat(
args.shape_cat,
args.cut_dict,
im3_cols=IM3_COLS_DEFAULT + ['psf_e1', 'psf_e2'],
test_nrows=None,
apply_c=True)
print shape_data.dtype.names
if args.z_bin_lims:
z_array = get_photoz(read_rows=read_rows)
shape_data = nmbot.add_field(shape_data, [("mean_z", float)],
[z_array])
shape_cat = corrtools.GalCat(shape_data,
quantities=[('e1', 'e2'),
('psf_e1', 'psf_e2')])
if args.z_bin_lims:
shape_cat.redshift_split(zcol='mean_z', z_bin_lims=args.z_bin_lims)
shape_cat.apply_m_correction()
print shape_cat.bin_stats
weight_col = 'weight'
if args.q_cat:
q_data = fitsio.read(args.q_cat, rows=read_rows_union)[shape_mask_0]
if args.z_bin_lims:
q_data = q_data[
(shape_data[shape_mask_0]["mean_z"] > args.z_bin_lims[0]) *
(shape_data["mean_z"][shape_mask_0] < args.z_bin_lims[-1])]
print "len(q_data), len(shape_cat.arr_data)", len(q_data), len(
shape_cat.arr_data)
#shape_data,q_data=shape_data[shape_mask_0],q_data[shape_mask_0]
#shape_data = nmbot.add_field(shape_data, [('de1',float),('de2',float)], [q_data['de1'],q_data['de2']])
use = np.ones(len(q_data), dtype='bool')
use[np.isnan(q_data['e1'])] = False
use[q_data['de1'] < -1.] = False
use[q_data['de2'] < -1.] = False
print 'fraction %f has bad q data, will use mean qs for these objets' % (
1 - float(use.sum()) / len(use))
mean_q1 = q_data['de1'][use].mean()
mean_q2 = q_data['de2'][use].mean()
q_data['de1'][~use] = mean_q1
q_data['de2'][~use] = mean_q2
#now compute corrections
with open(args.q_coeff_cat, 'r') as f:
coeff_lines = f.readlines()
for l in coeff_lines:
if l[0] == '#':
continue
l_entries = (l.strip()).split()
zbin, x, y, alpha = int(
l_entries[0]), l_entries[1], l_entries[2], float(l_entries[3])
if (x == 'de1' and y == 'e1'):
shape_cat.arr_data['e1'][shape_cat.zbin_masks[
zbin]] -= alpha * q_data['de1'][shape_cat.zbin_masks[zbin]]
elif (x == 'de2' and y == 'e2'):
shape_cat.arr_data['e2'][shape_cat.zbin_masks[
zbin]] -= alpha * q_data['de2'][shape_cat.zbin_masks[zbin]]
else:
continue
print 'zbin, x, y, alpha', zbin, x, y, alpha
#sh_cat,im3_psf_cat = make_shape_treecorr_cats(shape_data, shape_colnames)
if not os.path.isdir(args.outdir):
os.makedirs(args.outdir)
#recalculate post-correction bin stats and save
shape_cat.get_bin_stats()
with open(pj(args.outdir, "bin_stats_shape.pkl"), "wb") as f:
pickle.dump(shape_cat.bin_stats, f)
print psf_data.dtype.names
psf_corr_names = ['P', 'p', 'q']
#psf_quantities = [(args.psf_model_prefix+'_e1', args.psf_model_prefix+'_e2'),('de1','de2')]
psf_quantities = [('e1', 'e2'),
(args.psf_model_prefix + '_e1',
args.psf_model_prefix + '_e2'), ('de1', 'de2')]
if args.dTpT:
psf_data = nmbot.add_field(
psf_data, [('dse_1', float), ('dse_2', float)], [
psf_data['dsize'] * psf_data['e1'] / psf_data['size'],
psf_data['dsize'] * psf_data['e2'] / psf_data['size']
])
psf_quantities.append(('dse_1', 'dse_2'))
psf_corr_names.append('dse')
elif args.dTp:
psf_data = nmbot.add_field(psf_data,
[('dse_1', float), ('dse_2', float)], [
psf_data['dsize'] * psf_data['e1'],
psf_data['dsize'] * psf_data['e2']
])
psf_quantities.append(('dse_1', 'dse_2'))
psf_corr_names.append('dse')
psf_cat = corrtools.GalCat(psf_data,
x_col='ra',
y_col='dec',
quantities=psf_quantities,
w_col=args.star_weight_col)
psf_cat.redshift_split()
with open(pj(args.outdir, "bin_stats_psf.pkl"), "wb") as f:
pickle.dump(psf_cat.bin_stats, f)
sh = (shape_colnames['e1'], shape_colnames['e2'])
sh_psf = (shape_colnames['psf_e1'], shape_colnames['psf_e2'])
sh_quantities = [sh, sh_psf]
cross_corrs = []
cross_specs = []
psf_auto_specs = []
varxi_arr = []
#first do shear cross psf(gal)
epg = corrtools.CorrMeasurement(shape_cat,
sh,
q2=sh_psf,
XX=gg,
sub_mean=True)
spec_epg, varxis = epg.get_spec(['epg'] * 2,
'NZ_DUMMY',
kernel_name2='NZ_DUMMY',
ret_varxi_arrs=True)
cross_corrs.append(epg)
cross_specs.append(spec_epg[0])
varxi_arr += list(varxis[0].copy())
#and psf(gal) auto
#do gp auto
pg = corrtools.CorrMeasurement(shape_cat, sh_psf, XX=gg, sub_mean=True)
spec_pg, varxis = pg.get_spec(['pgpg'] * 2,
'NZ_DUMMY',
ret_varxi_arrs=True)
varxi_arr += list(varxis[0].copy())
psf_auto_specs.append(spec_pg[0])
for i in range(len(psf_quantities)):
q2_i, name_i = psf_quantities[i], psf_corr_names[i]
# e x p
ep = corrtools.CorrMeasurement(shape_cat,
sh,
gal_cat2=psf_cat,
q2=q2_i,
XX=gg,
sub_mean=True)
sp, varxis = ep.get_spec(['e' + name_i] * 2,
'NZ_DUMMY',
kernel_name2='NZ_DUMMY',
ret_varxi_arrs=True)
cross_corrs.append(ep)
cross_specs.append(sp[0]) #just use xip
varxi_arr += list(varxis[0].copy())
#do p(gal) x p
pgp = corrtools.CorrMeasurement(shape_cat,
sh_psf,
gal_cat2=psf_cat,
q2=q2_i,
XX=gg,
sub_mean=True)
spec_pgp, varxis = pgp.get_spec(['pg' + name_i] * 2,
'NZ_DUMMY',
kernel_name2='NZ_DUMMY',
ret_varxi_arrs=True)
psf_auto_specs.append(spec_pgp[0])
varxi_arr += list(varxis[0].copy())
#sp = corrtools.SpectrumMeasurement.from_galcats(['e'+name_i]*2, shape_cat, sh, 'NZ_DUMMY', gal_cat2=psf_cat, q2=q2_i,
# XX=gg, kernel_name2='NZ_DUMMY')
for j in range(i, len(psf_quantities)):
print name_i, psf_corr_names[j]
if i == j:
pp = corrtools.CorrMeasurement(psf_cat,
q2_i,
XX=gg,
sub_mean=True)
sp, varxis = pp.get_spec([name_i + name_i] * 2,
'NZ_DUMMY',
ret_varxi_arrs=True)
#pp = corrtools.SpectrumMeasurement.from_galcats([name_i+name_i]*2, psf_cat, q2_i, 'NZ_DUMMY', XX=gg)
else:
q2_j, name_j = psf_quantities[j], psf_corr_names[j]
pp = corrtools.CorrMeasurement(psf_cat,
q2_i,
XX=gg,
q2=q2_j,
sub_mean=True)
sp, varxis = pp.get_spec([name_i + name_j] * 2,
'NZ_DUMMY',
ret_varxi_arrs=True)
#pp = corrtools.SpectrumMeasurement.from_galcats([name_i+name_j]*2, psf_cat, q2_i, 'NZ_DUMMY', XX=gg, q2=q2_j, kernel_name2='NZ_DUMMY')
psf_auto_specs.append(sp[0])
varxi_arr += list(varxis[0].copy())
print 'varxi_arr', varxi_arr
print 'cross_specs', cross_specs
print 'psf_auto_specs', psf_auto_specs
#make shape noise covariance
shape_noise_cov = np.diag(np.array(varxi_arr))
shape_noise_cov_info = twopoint.CovarianceMatrixInfo(
"COV_SHAPENOISE",
[s.name for s in cross_specs] + [s.name for s in psf_auto_specs],
[len(s.value)
for s in cross_specs] + [len(s.value)
for s in psf_auto_specs], shape_noise_cov)
#No covariance compuation, just save measurement
t = twopoint.TwoPointFile(cross_specs + psf_auto_specs,
[twopoint.dummy_kernel("NZ_DUMMY")], None,
shape_noise_cov_info)
t.to_fits(pj(args.outdir, 'corrs_covsn.fits'), clobber=True)