def batch_gbtxwigglez_data_run(gbt_map_key,
wigglez_map_key,
wigglez_mock_key,
wigglez_selection_key,
inifile=None,
datapath_db=None,
outdir="./plots",
output_tag=None,
beam_transfer=None,
mode_transfer_1d=None,
mode_transfer_2d=None,
theory_curve=None):
r"""assemble the pairs of GBT and WiggleZ and calculate the cross-power"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(gbt_map_key, "type;treatment")
mock_cases = datapath_db.fileset_cases(wigglez_mock_key, "realization")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
if output_tag:
file_tools.mkparents(outdir)
for treatment in map_cases['treatment']:
# TODO: make this more elegant
# TODO: convert treatment into mode num
transfer_2d = None
if (mode_transfer_2d is not None) and (beam_transfer is None):
transfer_2d = mode_transfer_2d[treatment][2]
if (mode_transfer_2d is None) and (beam_transfer is not None):
transfer_2d = beam_transfer
if (mode_transfer_2d is not None) and (beam_transfer is not None):
transfer_2d = mode_transfer_2d[treatment][2] * beam_transfer
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for index in mock_cases['realization']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = "%s:%s" % (wigglez_mock_key, index)
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \
(gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(
pe.convert_2d_to_1d(pwrspec_out[0], transfer=transfer_2d))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
if mode_transfer_1d is not None:
transfunc = mode_transfer_1d[treatment][1]
else:
transfunc = None
mtag = output_tag + "_%s_mock" % treatment
agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d,
pwr_2d,
mtag,
outdir=outdir,
apply_1d_transfer=transfunc)
mean1dmock = agg_pwrspec["mean_1d"]
std1dmock = agg_pwrspec["std_1d"]
covmock = agg_pwrspec["covmat_1d"]
# now recover the xspec with the real data
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = wigglez_map_key
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0],
transfer=transfer_2d)
pwr_1d = pwrspec_out_signal[1]['binavg']
pwr_1d_from_2d = pwr_1d_from_2d['binavg']
if mode_transfer_1d is not None:
pwr_1d /= mode_transfer_1d[treatment][1]
pwr_1d_from_2d /= mode_transfer_1d[treatment][1]
# assume that they all have the same binning
bin_left = pwrspec_out_signal[1]['bin_left']
bin_center = pwrspec_out_signal[1]['bin_center']
bin_right = pwrspec_out_signal[1]['bin_right']
counts_histo = pwrspec_out_signal[1]['counts_histo']
filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment)
outfile = open(filename, "w")
for specdata in zip(bin_left, bin_center, bin_right, counts_histo,
pwr_1d, pwr_1d_from_2d, mean1dmock, std1dmock):
outfile.write(("%10.15g " * 8 + "\n") % specdata)
outfile.close()
# TODO: kludge to make a fast fit; remove
theory_curve = np.genfromtxt(
"plots/sim_15hr_oldmap_str_temperature_xWigglez/sim_15hr_oldmap_str_temperature_xWigglez_avg_from2d.dat"
)
theory_curve = theory_curve[:, 4]
if theory_curve is not None:
restrict = np.where(
np.logical_and(bin_center > 0.09, bin_center < 1.1))
res_slice = slice(min(restrict[0]), max(restrict[0]))
#restrict_alt = np.where(restrict)[0][np.newaxis, :]
#restricted_cov = covmock[restrict_alt][0]
from core import utils
amplitude = utils.ampfit(pwr_1d_from_2d[res_slice],
covmock[res_slice, res_slice],
theory_curve[res_slice])
print "AMP:", mtag, treatment, amplitude
if not output_tag:
caller.multiprocess_stack()
return None
def batch_gbtxwigglez_data_run(gbt_map_key, wigglez_map_key,
wigglez_mock_key, wigglez_selection_key,
inifile=None, datapath_db=None,
outdir="./plots",
output_tag=None,
beam_transfer=None,
mode_transfer_1d=None,
mode_transfer_2d=None,
theory_curve=None):
r"""assemble the pairs of GBT and WiggleZ and calculate the cross-power"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(gbt_map_key, "type;treatment")
mock_cases = datapath_db.fileset_cases(wigglez_mock_key, "realization")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
if output_tag:
file_tools.mkparents(outdir)
for treatment in map_cases['treatment']:
# TODO: make this more elegant
# TODO: convert treatment into mode num
transfer_2d = None
if (mode_transfer_2d is not None) and (beam_transfer is None):
transfer_2d = mode_transfer_2d[treatment][2]
if (mode_transfer_2d is None) and (beam_transfer is not None):
transfer_2d = beam_transfer
if (mode_transfer_2d is not None) and (beam_transfer is not None):
transfer_2d = mode_transfer_2d[treatment][2] * beam_transfer
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for index in mock_cases['realization']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = "%s:%s" % (wigglez_mock_key, index)
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \
(gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'], files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
transfer=transfer_2d))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
if mode_transfer_1d is not None:
transfunc = mode_transfer_1d[treatment][1]
else:
transfunc = None
mtag = output_tag + "_%s_mock" % treatment
agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d, pwr_2d, mtag,
outdir=outdir,
apply_1d_transfer=transfunc)
mean1dmock = agg_pwrspec["mean_1d"]
std1dmock = agg_pwrspec["std_1d"]
covmock = agg_pwrspec["covmat_1d"]
# now recover the xspec with the real data
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = wigglez_map_key
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0],
transfer=transfer_2d)
pwr_1d = pwrspec_out_signal[1]['binavg']
pwr_1d_from_2d = pwr_1d_from_2d['binavg']
if mode_transfer_1d is not None:
pwr_1d /= mode_transfer_1d[treatment][1]
pwr_1d_from_2d /= mode_transfer_1d[treatment][1]
# assume that they all have the same binning
bin_left = pwrspec_out_signal[1]['bin_left']
bin_center = pwrspec_out_signal[1]['bin_center']
bin_right = pwrspec_out_signal[1]['bin_right']
counts_histo = pwrspec_out_signal[1]['counts_histo']
filename = "%s/%s_%s.dat" % (outdir,
output_tag,
treatment)
outfile = open(filename, "w")
for specdata in zip(bin_left, bin_center,
bin_right, counts_histo,
pwr_1d, pwr_1d_from_2d,
mean1dmock, std1dmock):
outfile.write(("%10.15g " * 8 + "\n") % specdata)
outfile.close()
# TODO: kludge to make a fast fit; remove
theory_curve = np.genfromtxt("plots/sim_15hr_oldmap_str_temperature_xWigglez/sim_15hr_oldmap_str_temperature_xWigglez_avg_from2d.dat")
theory_curve = theory_curve[:, 4]
if theory_curve is not None:
restrict = np.where(np.logical_and(bin_center > 0.09,
bin_center < 1.1))
res_slice = slice(min(restrict[0]), max(restrict[0]))
#restrict_alt = np.where(restrict)[0][np.newaxis, :]
#restricted_cov = covmock[restrict_alt][0]
from core import utils
amplitude = utils.ampfit(pwr_1d_from_2d[res_slice],
covmock[res_slice, res_slice],
theory_curve[res_slice])
print "AMP:", mtag, treatment, amplitude
if not output_tag:
caller.multiprocess_stack()
return None
def make_xcorr_plotdata():
# find the mean brightness used in simulations
corrobj = corr21cm.Corr21cm()
T_b_sim = corrobj.T_b(1420./800.-1)
print T_b_sim
#splt.process_batch_correlations(xcorr_real_param,
# multiplier=1., cross_power=True)
##splt.process_batch_correlations(xcorr_sim_param, cross_power=True,
## multiplier=1./(T_b_sim/1.e3))
#splt.process_batch_correlations(xcorr_sim_param, cross_power=True,
# multiplier=1./(T_b_sim))
#splt.process_batch_correlations(xcorr_loss_sim_param,
# multiplier=1./T_b_sim*1.e-3, cross_power=True)
#splt.process_batch_correlations(xcorr_variants_param,
# multiplier=1., cross_power=True)
#splt.plot_batch_correlations(xcorr_real_param,
# dir_prefix="plots/xcorr_real/",
# color_range=[-0.04, 0.04], cross_power=True)
#splt.plot_batch_correlations(xcorr_sim_param,
# dir_prefix="plots/xcorr_sim/",
# color_range=[-0.04, 0.04], cross_power=True)
#splt.plot_batch_correlations(xcorr_loss_param,
# dir_prefix="plots/xcorr_loss/",
# color_range=[-0.04, 0.04], cross_power=True)
#splt.plot_batch_correlations(xcorr_loss_sim_param,
# dir_prefix="plots/xcorr_loss_sim/",
# color_range=[-0.04, 0.04], cross_power=True)
#splt.plot_batch_correlations(xcorr_variants_param,
# dir_prefix="plots/xcorr_variants/",
# color_range=[-0.04, 0.04], cross_power=True)
# find the real xcorr signal with errors
xcorr_data = splt.batch_correlations_statistics(xcorr_real_param,
randtoken="rand",
include_signal=True)
# find the simulated xcorr signal with errors
xcorr_sim_data = splt.batch_correlations_statistics(xcorr_sim_param,
randtoken="rand",
include_signal=False)
# find the compensation function from simulations
compmode = splt.batch_compensation_function(xcorr_loss_sim_param)
lagl = xcorr_data[0]
lagc = xcorr_data[1]
lagr = xcorr_data[2]
xcorr_null = xcorr_data[3]
xcorr_signal = xcorr_data[6]
xcorr_cov = xcorr_data[5]
xcorr_err = xcorr_data[4]
xcorr_sim = xcorr_sim_data[3]
xcorr_sim_err = xcorr_sim_data[4]
xcorr_sim_cov = xcorr_sim_data[5]
# 0=0, 5=1, 10=2, 15=3, etc.
compensation = compmode[3 ,:]
(amp, amp_err) = utils.ampfit(xcorr_signal, xcorr_cov + xcorr_sim_cov, xcorr_sim)
print amp, amp_err
# now convert from the 1e-3 Omega_HI in simulations to 0.5e-3
#xcorr_sim /= 2.
#xcorr_sim_err /= 2.
xcorr_sim *= amp
xcorr_sim_err *= amp
for correntry in zip(lagl, lagc, lagr, xcorr_signal, xcorr_null, xcorr_err,
xcorr_sim, xcorr_sim_err, compensation):
print "%5.3g %5.3g %5.3g %5.3g %5.3g %5.3g %5.3g %5.3g %5.3g" % correntry
