def batch_single_crosspwr(left_mapkey,
right_simkey, right_weightkey,
multiplier=1.,
inifile=None, datapath_db=None,
outdir="./plots",
output_tag=None):
r"""w_left m_left x w_right m_right"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(left_mapkey, "type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
crosspwr_collection = {}
for treatment in map_cases['treatment']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (left_mapkey, treatment)
dbkeydict['map2_key'] = right_simkey
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (left_mapkey, treatment)
dbkeydict['noiseinv2_key'] = right_weightkey
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwrspec_out_signal[0]['binavg'] *= multiplier
pwrspec_out_signal[1]['binavg'] *= multiplier
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0])
mtag = "%s_%s" % (output_tag, treatment)
pe.summarize_pwrspec(pwrspec_out_signal[1],
pwr_1d_from_2d,
pwrspec_out_signal[0],
mtag, outdir=outdir)
crosspwr_collection[treatment] = (pwrspec_out_signal[1],
pwr_1d_from_2d,
pwrspec_out_signal[0])
if not output_tag:
caller.multiprocess_stack()
return None
else:
return crosspwr_collection
def batch_physical_sim_run(sim_key,
inifile=None,
datapath_db=None,
output_tag=None,
outdir="./plots/"):
"""Test the power spectral estimator using simulations"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
mock_cases = datapath_db.fileset_cases(sim_key, "realization")
funcname = "correlate.batch_quadratic.call_phys_space_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
if output_tag:
output_root = "%s/%s/" % (outdir, output_tag)
file_tools.mkparents(output_root)
pwr_1d = []
pwr_1d_from_2d = []
pwr_2d = []
for index in mock_cases['realization']:
mapfile = datapath_db.fetch("%s:%s" % (sim_key, index))
pwrspec_out = caller.execute(mapfile, mapfile, inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(
pe.convert_2d_to_1d(pwrspec_out[0], transfer=None))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d,
pwr_2d,
output_tag,
outdir=output_root)
retval = (pwr_1d, pwr_1d_from_2d, pwr_2d)
else:
caller.multiprocess_stack()
retval = None
return retval
def batch_physical_sim_run(sim_key, inifile=None, datapath_db=None,
output_tag=None,
outdir="./plots/"):
"""Test the power spectral estimator using simulations"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
mock_cases = datapath_db.fileset_cases(sim_key, "realization")
funcname = "correlate.batch_quadratic.call_phys_space_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
if output_tag:
output_root = "%s/%s/" % (outdir, output_tag)
file_tools.mkparents(output_root)
pwr_1d = []
pwr_1d_from_2d = []
pwr_2d = []
for index in mock_cases['realization']:
mapfile = datapath_db.fetch("%s:%s" % (sim_key, index))
pwrspec_out = caller.execute(mapfile, mapfile, inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
transfer=None))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
pe.summarize_agg_pwrspec(pwr_1d, pwr_1d_from_2d, pwr_2d,
output_tag, outdir=output_root)
retval = (pwr_1d, pwr_1d_from_2d, pwr_2d)
else:
caller.multiprocess_stack()
retval = None
return retval
def gather_batch_datasim_run(tag,
subtract_mean=False,
degrade_resolution=False,
unitless=True,
return_3d=False,
truncate=False,
window=None,
n_modes=None,
refinement=2,
pad=5,
order=2,
outdir="./plot_data",
alt=""):
datapath_db = data_paths.DataPath()
outpath = datapath_db.fetch("quadratic_batch_simulations")
print "reading from: " + outpath
funcname = "correlate.batch_quadratic.call_xspec_run"
caller = batch_handler.MemoizeBatch(funcname, outpath, verbose=True)
transfer_functions = {}
for mode_num in range(0, 55, 5):
mapsim = "sim_15hr"
map1_key = "%s_cleaned_%s%dmode" % (mapsim, alt, mode_num)
map2_key = "%s_cleaned_%s%dmode" % (mapsim, alt, mode_num)
noise1_key = "%s_cleaned_%s%dmode" % (mapsim, alt, mode_num)
noise2_key = "%s_cleaned_%s%dmode" % (mapsim, alt, mode_num)
(pairlist, pairdict) = \
data_paths.cross_maps(map1_key, map2_key,
noise1_key, noise2_key,
map_suffix=";map",
noise_inv_suffix=";noise_inv",
cross_sym="_x_",
pair_former="GBTauto_cross_pairs",
ignore=['param'],
tag1prefix=map1_key + "_",
tag2prefix=map2_key + "_",
verbose=False)
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for item in pairdict.keys():
pairrun = pairdict[item]
print pairrun['tag1'], pairrun['map1'], pairrun['noise_inv1']
pwr2d_run, pwr1d_run = caller.execute(
pairrun['map1'],
pairrun['map2'],
pairrun['noise_inv1'],
pairrun['noise_inv2'],
subtract_mean=subtract_mean,
degrade_resolution=degrade_resolution,
unitless=unitless,
return_3d=return_3d,
truncate=truncate,
window=window,
n_modes=n_modes,
refinement=refinement,
pad=pad,
order=order)
pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwr2d_run))
pwr_2d.append(pwr2d_run)
pwr_1d.append(pwr1d_run)
if (mode_num == 0):
pwr_1d_from2d_zero_mode = copy.deepcopy(pwr_1d_from_2d)
pwr_1d_zero_mode = copy.deepcopy(pwr_1d)
pwr_2d_zero_mode = copy.deepcopy(pwr_2d)
ttag = tag + "_%dmodes_2dtrans" % mode_num
trans2d_mode = tf.calculate_2d_transfer_function(
pwr_2d, pwr_2d_zero_mode, ttag)
ttag = tag + "_%dmodes_1dtrans" % mode_num
trans1d_mode = tf.calculate_1d_transfer_function(
pwr_1d, pwr_1d_zero_mode, ttag)
ttag = tag + "_%dmodes_1dtrans_from2d" % mode_num
trans1d_mode = tf.calculate_1d_transfer_function(
pwr_1d_from_2d, pwr_1d_from2d_zero_mode, ttag)
transfer_functions[mode_num] = (trans1d_mode, trans2d_mode)
mtag = tag + "_%dmodes" % mode_num
pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d,
pwr_2d,
mtag,
outdir="./plot_data")
return transfer_functions
def batch_gbtpwrspec_data_run(map_key, inifile=None, datapath_db=None,
output_tag=None, beam_transfer=None,
outdir="./plots/",
square_1dmodetrans=False,
mode_transfer_1d=None,
mode_transfer_2d=None):
r"""Form the pairs of maps1*weight1 x map0*weight0 for calculating the
auto-power of the GBT data"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(map_key, "pair;type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
if output_tag:
file_tools.mkparents(outdir)
pwrspec_collection = {}
for treatment in map_cases['treatment']:
unique_pairs = data_paths.GBTauto_cross_pairs(map_cases['pair'],
map_cases['pair'],
cross_sym="_with_")
# TODO: make this more elegant
transfer_2d = None
if (mode_transfer_2d is not None) and (beam_transfer is None):
transfer_2d = mode_transfer_2d[treatment][1]
if (mode_transfer_2d is None) and (beam_transfer is not None):
transfer_2d = beam_transfer
if (mode_transfer_2d is not None) and (beam_transfer is not None):
transfer_2d = mode_transfer_2d[treatment][1] * beam_transfer
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for item in unique_pairs:
dbkeydict = {}
# NOTE: formerly had "weight" instead of noise_inv
mapset0 = (map_key, item[0], treatment)
mapset1 = (map_key, item[1], treatment)
dbkeydict['map1_key'] = "%s:%s;map;%s" % mapset0
dbkeydict['map2_key'] = "%s:%s;map;%s" % mapset1
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
transfer=transfer_2d))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
mtag = output_tag + "_%s" % treatment
if mode_transfer_1d is not None:
transfunc = mode_transfer_1d[treatment][0]
if square_1dmodetrans:
transfunc *= transfunc
else:
transfunc = None
agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d, pwr_2d, mtag,
outdir=outdir,
apply_1d_transfer=transfunc)
# (mean_1d, std_1d, covmat_1d)
pwrspec_collection[treatment] = agg_pwrspec
if output_tag:
return pwrspec_collection
else:
caller.multiprocess_stack()
return None
def batch_sim_run(simleft_key, simright_key,
weightleft_key, weightright_key,
inifile=None, datapath_db=None,
outdir="./plots/",
usecache_output_tag=None, transfer=None):
r"""
typical weight matrix:
db:GBT_15hr_map_cleaned_0mode:A_with_B;noise_inv
"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
mock_cases = datapath_db.fileset_cases(simleft_key, "realization")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if usecache_output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
if usecache_output_tag:
output_root = "%s/%s/" % (outdir, usecache_output_tag)
file_tools.mkparents(output_root)
pwr_1d = []
pwr_1d_from_2d = []
pwr_2d = []
for index in mock_cases['realization']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:%s" % (simleft_key, index)
dbkeydict['map2_key'] = "%s:%s" % (simright_key, index)
dbkeydict['noiseinv1_key'] = weightleft_key
dbkeydict['noiseinv2_key'] = weightright_key
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'], files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if usecache_output_tag:
pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
transfer=transfer))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if usecache_output_tag:
pe.summarize_agg_pwrspec(pwr_1d, pwr_1d_from_2d, pwr_2d,
usecache_output_tag, outdir=output_root)
retval = (pwr_1d, pwr_1d_from_2d, pwr_2d)
else:
caller.multiprocess_stack()
retval = None
return retval
def batch_gbtxwigglez_data_run(gbt_map_key, wigglez_map_key,
wigglez_mock_key, wigglez_selection_key,
inifile=None, datapath_db=None,
outdir="./plots",
output_tag=None,
beam_transfer=None,
mode_transfer_1d=None,
mode_transfer_2d=None,
theory_curve=None):
r"""assemble the pairs of GBT and WiggleZ and calculate the cross-power"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(gbt_map_key, "type;treatment")
mock_cases = datapath_db.fileset_cases(wigglez_mock_key, "realization")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
if output_tag:
file_tools.mkparents(outdir)
for treatment in map_cases['treatment']:
# TODO: make this more elegant
# TODO: convert treatment into mode num
transfer_2d = None
if (mode_transfer_2d is not None) and (beam_transfer is None):
transfer_2d = mode_transfer_2d[treatment][2]
if (mode_transfer_2d is None) and (beam_transfer is not None):
transfer_2d = beam_transfer
if (mode_transfer_2d is not None) and (beam_transfer is not None):
transfer_2d = mode_transfer_2d[treatment][2] * beam_transfer
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for index in mock_cases['realization']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = "%s:%s" % (wigglez_mock_key, index)
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \
(gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'], files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
transfer=transfer_2d))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
if mode_transfer_1d is not None:
transfunc = mode_transfer_1d[treatment][1]
else:
transfunc = None
mtag = output_tag + "_%s_mock" % treatment
agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d, pwr_2d, mtag,
outdir=outdir,
apply_1d_transfer=transfunc)
mean1dmock = agg_pwrspec["mean_1d"]
std1dmock = agg_pwrspec["std_1d"]
covmock = agg_pwrspec["covmat_1d"]
# now recover the xspec with the real data
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = wigglez_map_key
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0],
transfer=transfer_2d)
pwr_1d = pwrspec_out_signal[1]['binavg']
pwr_1d_from_2d = pwr_1d_from_2d['binavg']
if mode_transfer_1d is not None:
pwr_1d /= mode_transfer_1d[treatment][1]
pwr_1d_from_2d /= mode_transfer_1d[treatment][1]
# assume that they all have the same binning
bin_left = pwrspec_out_signal[1]['bin_left']
bin_center = pwrspec_out_signal[1]['bin_center']
bin_right = pwrspec_out_signal[1]['bin_right']
counts_histo = pwrspec_out_signal[1]['counts_histo']
filename = "%s/%s_%s.dat" % (outdir,
output_tag,
treatment)
outfile = open(filename, "w")
for specdata in zip(bin_left, bin_center,
bin_right, counts_histo,
pwr_1d, pwr_1d_from_2d,
mean1dmock, std1dmock):
outfile.write(("%10.15g " * 8 + "\n") % specdata)
outfile.close()
# TODO: kludge to make a fast fit; remove
theory_curve = np.genfromtxt("plots/sim_15hr_oldmap_str_temperature_xWigglez/sim_15hr_oldmap_str_temperature_xWigglez_avg_from2d.dat")
theory_curve = theory_curve[:, 4]
if theory_curve is not None:
restrict = np.where(np.logical_and(bin_center > 0.09,
bin_center < 1.1))
res_slice = slice(min(restrict[0]), max(restrict[0]))
#restrict_alt = np.where(restrict)[0][np.newaxis, :]
#restricted_cov = covmock[restrict_alt][0]
from core import utils
amplitude = utils.ampfit(pwr_1d_from_2d[res_slice],
covmock[res_slice, res_slice],
theory_curve[res_slice])
print "AMP:", mtag, treatment, amplitude
if not output_tag:
caller.multiprocess_stack()
return None
def batch_single_crosspwr(left_mapkey,
right_simkey,
right_weightkey,
multiplier=1.,
inifile=None,
datapath_db=None,
outdir="./plots",
output_tag=None):
r"""w_left m_left x w_right m_right"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(left_mapkey, "type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
crosspwr_collection = {}
for treatment in map_cases['treatment']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (left_mapkey, treatment)
dbkeydict['map2_key'] = right_simkey
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (left_mapkey, treatment)
dbkeydict['noiseinv2_key'] = right_weightkey
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwrspec_out_signal[0]['binavg'] *= multiplier
pwrspec_out_signal[1]['binavg'] *= multiplier
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0])
mtag = "%s_%s" % (output_tag, treatment)
pe.summarize_pwrspec(pwrspec_out_signal[1],
pwr_1d_from_2d,
pwrspec_out_signal[0],
mtag,
outdir=outdir)
crosspwr_collection[treatment] = (pwrspec_out_signal[1],
pwr_1d_from_2d,
pwrspec_out_signal[0])
if not output_tag:
caller.multiprocess_stack()
return None
else:
return crosspwr_collection
def batch_gbtpwrspec_data_run(map_key,
inifile=None,
datapath_db=None,
output_tag=None,
beam_transfer=None,
outdir="./plots/",
square_1dmodetrans=False,
mode_transfer_1d=None,
mode_transfer_2d=None):
r"""Form the pairs of maps1*weight1 x map0*weight0 for calculating the
auto-power of the GBT data"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(map_key, "pair;type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
if output_tag:
file_tools.mkparents(outdir)
pwrspec_collection = {}
for treatment in map_cases['treatment']:
unique_pairs = data_paths.GBTauto_cross_pairs(map_cases['pair'],
map_cases['pair'],
cross_sym="_with_")
# TODO: make this more elegant
transfer_2d = None
if (mode_transfer_2d is not None) and (beam_transfer is None):
transfer_2d = mode_transfer_2d[treatment][1]
if (mode_transfer_2d is None) and (beam_transfer is not None):
transfer_2d = beam_transfer
if (mode_transfer_2d is not None) and (beam_transfer is not None):
transfer_2d = mode_transfer_2d[treatment][1] * beam_transfer
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for item in unique_pairs:
dbkeydict = {}
# NOTE: formerly had "weight" instead of noise_inv
mapset0 = (map_key, item[0], treatment)
mapset1 = (map_key, item[1], treatment)
dbkeydict['map1_key'] = "%s:%s;map;%s" % mapset0
dbkeydict['map2_key'] = "%s:%s;map;%s" % mapset1
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(
pe.convert_2d_to_1d(pwrspec_out[0], transfer=transfer_2d))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
mtag = output_tag + "_%s" % treatment
if mode_transfer_1d is not None:
transfunc = mode_transfer_1d[treatment][0]
if square_1dmodetrans:
transfunc *= transfunc
else:
transfunc = None
agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d,
pwr_2d,
mtag,
outdir=outdir,
apply_1d_transfer=transfunc)
# (mean_1d, std_1d, covmat_1d)
pwrspec_collection[treatment] = agg_pwrspec
if output_tag:
return pwrspec_collection
else:
caller.multiprocess_stack()
return None
def batch_crosspwr_transfer(cleaned_simkey,
truesignal_simkey,
truesignal_weightkey,
reference_simkey,
reference_weightkey,
inifile=None,
datapath_db=None,
outdir="./plots",
output_tag=None):
r"""take relevant cross-powers
cleaned_simkey(map) * cleaned_simkey(weight) x
truesignal_weightkey * truesignal_simkey
divided by:
reference_simkey * reference_weightkey x truesignal_weightkey * truesignal_simkey
"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(cleaned_simkey, "type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
dbkeydict = {}
dbkeydict['map1_key'] = reference_simkey
dbkeydict['map2_key'] = truesignal_simkey
dbkeydict['noiseinv1_key'] = reference_weightkey
dbkeydict['noiseinv2_key'] = truesignal_weightkey
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
reference_pwrspec_out = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
file_tools.mkparents(outdir)
ref_pwr_1d_from_2d = pe.convert_2d_to_1d(reference_pwrspec_out[0])
ref_pwr_1d = reference_pwrspec_out[1]['binavg']
ref_pwr_2d = reference_pwrspec_out[0]['binavg']
ref_pwr_1d_from_2d = ref_pwr_1d_from_2d['binavg']
bin_left = reference_pwrspec_out[1]['bin_left']
bin_center = reference_pwrspec_out[1]['bin_center']
bin_right = reference_pwrspec_out[1]['bin_right']
counts_histo = reference_pwrspec_out[1]['counts_histo']
transfer_functions = {}
for treatment in map_cases['treatment']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (cleaned_simkey, treatment)
dbkeydict['map2_key'] = truesignal_simkey
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (cleaned_simkey,
treatment)
dbkeydict['noiseinv2_key'] = truesignal_weightkey
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0])
pwr_1d = pwrspec_out_signal[1]['binavg']
pwr_2d = pwrspec_out_signal[0]['binavg']
pwr_1d_from_2d = pwr_1d_from_2d['binavg']
trans1d_mode = pwr_1d / ref_pwr_1d
trans1d_from2d_mode = pwr_1d_from_2d / ref_pwr_1d_from_2d
trans2d_mode = pwr_2d / ref_pwr_2d
transfer_functions[treatment] = (trans1d_mode, trans1d_from2d_mode,
trans2d_mode)
filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment)
outfile = open(filename, "w")
for specdata in zip(bin_left, bin_center, bin_right, counts_histo,
pwr_1d, pwr_1d_from_2d, trans1d_mode,
trans1d_from2d_mode):
outfile.write(("%10.15g " * 8 + "\n") % specdata)
outfile.close()
if not output_tag:
caller.multiprocess_stack()
return None
else:
return transfer_functions
def batch_gbtxwigglez_data_run(gbt_map_key,
wigglez_map_key,
wigglez_mock_key,
wigglez_selection_key,
inifile=None,
datapath_db=None,
outdir="./plots",
output_tag=None,
beam_transfer=None,
mode_transfer_1d=None,
mode_transfer_2d=None,
theory_curve=None):
r"""assemble the pairs of GBT and WiggleZ and calculate the cross-power"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(gbt_map_key, "type;treatment")
mock_cases = datapath_db.fileset_cases(wigglez_mock_key, "realization")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
if output_tag:
file_tools.mkparents(outdir)
for treatment in map_cases['treatment']:
# TODO: make this more elegant
# TODO: convert treatment into mode num
transfer_2d = None
if (mode_transfer_2d is not None) and (beam_transfer is None):
transfer_2d = mode_transfer_2d[treatment][2]
if (mode_transfer_2d is None) and (beam_transfer is not None):
transfer_2d = beam_transfer
if (mode_transfer_2d is not None) and (beam_transfer is not None):
transfer_2d = mode_transfer_2d[treatment][2] * beam_transfer
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for index in mock_cases['realization']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = "%s:%s" % (wigglez_mock_key, index)
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \
(gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(
pe.convert_2d_to_1d(pwrspec_out[0], transfer=transfer_2d))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
if mode_transfer_1d is not None:
transfunc = mode_transfer_1d[treatment][1]
else:
transfunc = None
mtag = output_tag + "_%s_mock" % treatment
agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d,
pwr_2d,
mtag,
outdir=outdir,
apply_1d_transfer=transfunc)
mean1dmock = agg_pwrspec["mean_1d"]
std1dmock = agg_pwrspec["std_1d"]
covmock = agg_pwrspec["covmat_1d"]
# now recover the xspec with the real data
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = wigglez_map_key
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0],
transfer=transfer_2d)
pwr_1d = pwrspec_out_signal[1]['binavg']
pwr_1d_from_2d = pwr_1d_from_2d['binavg']
if mode_transfer_1d is not None:
pwr_1d /= mode_transfer_1d[treatment][1]
pwr_1d_from_2d /= mode_transfer_1d[treatment][1]
# assume that they all have the same binning
bin_left = pwrspec_out_signal[1]['bin_left']
bin_center = pwrspec_out_signal[1]['bin_center']
bin_right = pwrspec_out_signal[1]['bin_right']
counts_histo = pwrspec_out_signal[1]['counts_histo']
filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment)
outfile = open(filename, "w")
for specdata in zip(bin_left, bin_center, bin_right, counts_histo,
pwr_1d, pwr_1d_from_2d, mean1dmock, std1dmock):
outfile.write(("%10.15g " * 8 + "\n") % specdata)
outfile.close()
# TODO: kludge to make a fast fit; remove
theory_curve = np.genfromtxt(
"plots/sim_15hr_oldmap_str_temperature_xWigglez/sim_15hr_oldmap_str_temperature_xWigglez_avg_from2d.dat"
)
theory_curve = theory_curve[:, 4]
if theory_curve is not None:
restrict = np.where(
np.logical_and(bin_center > 0.09, bin_center < 1.1))
res_slice = slice(min(restrict[0]), max(restrict[0]))
#restrict_alt = np.where(restrict)[0][np.newaxis, :]
#restricted_cov = covmock[restrict_alt][0]
from core import utils
amplitude = utils.ampfit(pwr_1d_from_2d[res_slice],
covmock[res_slice, res_slice],
theory_curve[res_slice])
print "AMP:", mtag, treatment, amplitude
if not output_tag:
caller.multiprocess_stack()
return None
def gather_batch_datasim_run(tag, subtract_mean=False,
degrade_resolution=False, unitless=True,
return_3d=False, truncate=False, window=None,
n_modes=None, refinement=2, pad=5, order=2,
outdir="./plot_data", alt=""):
datapath_db = data_paths.DataPath()
outpath = datapath_db.fetch("quadratic_batch_simulations")
print "reading from: " + outpath
funcname = "correlate.batch_quadratic.call_xspec_run"
caller = batch_handler.MemoizeBatch(funcname, outpath,
verbose=True)
transfer_functions = {}
for mode_num in range(0,55,5):
mapsim = "sim_15hr"
map1_key = "%s_cleaned_%s%dmode" % (mapsim, alt, mode_num)
map2_key = "%s_cleaned_%s%dmode" % (mapsim, alt, mode_num)
noise1_key = "%s_cleaned_%s%dmode" % (mapsim, alt, mode_num)
noise2_key = "%s_cleaned_%s%dmode" % (mapsim, alt, mode_num)
(pairlist, pairdict) = \
data_paths.cross_maps(map1_key, map2_key,
noise1_key, noise2_key,
map_suffix=";map",
noise_inv_suffix=";noise_inv",
cross_sym="_x_",
pair_former="GBTauto_cross_pairs",
ignore=['param'],
tag1prefix=map1_key + "_",
tag2prefix=map2_key + "_",
verbose=False)
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for item in pairdict.keys():
pairrun = pairdict[item]
print pairrun['tag1'], pairrun['map1'], pairrun['noise_inv1']
pwr2d_run, pwr1d_run = caller.execute(pairrun['map1'],
pairrun['map2'],
pairrun['noise_inv1'],
pairrun['noise_inv2'],
subtract_mean=subtract_mean,
degrade_resolution=degrade_resolution,
unitless=unitless,
return_3d=return_3d,
truncate=truncate,
window=window, n_modes=n_modes,
refinement=refinement,
pad=pad, order=order)
pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwr2d_run))
pwr_2d.append(pwr2d_run)
pwr_1d.append(pwr1d_run)
if (mode_num == 0):
pwr_1d_from2d_zero_mode = copy.deepcopy(pwr_1d_from_2d)
pwr_1d_zero_mode = copy.deepcopy(pwr_1d)
pwr_2d_zero_mode = copy.deepcopy(pwr_2d)
ttag = tag + "_%dmodes_2dtrans" % mode_num
trans2d_mode = tf.calculate_2d_transfer_function(
pwr_2d, pwr_2d_zero_mode,
ttag)
ttag = tag + "_%dmodes_1dtrans" % mode_num
trans1d_mode = tf.calculate_1d_transfer_function(
pwr_1d, pwr_1d_zero_mode,
ttag)
ttag = tag + "_%dmodes_1dtrans_from2d" % mode_num
trans1d_mode = tf.calculate_1d_transfer_function(
pwr_1d_from_2d, pwr_1d_from2d_zero_mode,
ttag)
transfer_functions[mode_num] = (trans1d_mode, trans2d_mode)
mtag = tag + "_%dmodes" % mode_num
pe.summarize_agg_pwrspec(pwr_1d, pwr_1d_from_2d, pwr_2d, mtag,
outdir="./plot_data")
return transfer_functions
def batch_crosspwr_transfer(cleaned_simkey,
truesignal_simkey, truesignal_weightkey,
reference_simkey, reference_weightkey,
inifile=None, datapath_db=None,
outdir="./plots",
output_tag=None):
r"""take relevant cross-powers
cleaned_simkey(map) * cleaned_simkey(weight) x
truesignal_weightkey * truesignal_simkey
divided by:
reference_simkey * reference_weightkey x truesignal_weightkey * truesignal_simkey
"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(cleaned_simkey, "type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
dbkeydict = {}
dbkeydict['map1_key'] = reference_simkey
dbkeydict['map2_key'] = truesignal_simkey
dbkeydict['noiseinv1_key'] = reference_weightkey
dbkeydict['noiseinv2_key'] = truesignal_weightkey
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
reference_pwrspec_out = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
file_tools.mkparents(outdir)
ref_pwr_1d_from_2d = pe.convert_2d_to_1d(reference_pwrspec_out[0])
ref_pwr_1d = reference_pwrspec_out[1]['binavg']
ref_pwr_2d = reference_pwrspec_out[0]['binavg']
ref_pwr_1d_from_2d = ref_pwr_1d_from_2d['binavg']
bin_left = reference_pwrspec_out[1]['bin_left']
bin_center = reference_pwrspec_out[1]['bin_center']
bin_right = reference_pwrspec_out[1]['bin_right']
counts_histo = reference_pwrspec_out[1]['counts_histo']
transfer_functions = {}
for treatment in map_cases['treatment']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (cleaned_simkey, treatment)
dbkeydict['map2_key'] = truesignal_simkey
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (cleaned_simkey, treatment)
dbkeydict['noiseinv2_key'] = truesignal_weightkey
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0])
pwr_1d = pwrspec_out_signal[1]['binavg']
pwr_2d = pwrspec_out_signal[0]['binavg']
pwr_1d_from_2d = pwr_1d_from_2d['binavg']
trans1d_mode = pwr_1d / ref_pwr_1d
trans1d_from2d_mode = pwr_1d_from_2d / ref_pwr_1d_from_2d
trans2d_mode = pwr_2d / ref_pwr_2d
transfer_functions[treatment] = (trans1d_mode,
trans1d_from2d_mode,
trans2d_mode)
filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment)
outfile = open(filename, "w")
for specdata in zip(bin_left, bin_center,
bin_right, counts_histo,
pwr_1d, pwr_1d_from_2d,
trans1d_mode, trans1d_from2d_mode):
outfile.write(("%10.15g " * 8 + "\n") % specdata)
outfile.close()
if not output_tag:
caller.multiprocess_stack()
return None
else:
return transfer_functions
