def batch_wigglez_automock_run(mock_key, sel_key,
inifile=None, datapath_db=None,
output_tag=None):
r"""TODO: make this work; wrote this but never really needed it yet
"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
mock_cases = datapath_db.fileset_cases(mock_key, "realization")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
for index in mock_cases['realization']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:%s" % (mock_key, index)
dbkeydict['map2_key'] = "%s:%s" % (mock_key, index)
dbkeydict['noiseinv1_key'] = sel_key
dbkeydict['noiseinv2_key'] = sel_key
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
caller.execute(files['map1_key'], files['map2_key'],
files['noiseinv1_key'], files['noiseinv2_key'],
inifile=inifile)
caller.multiprocess_stack()
def batch_single_crosspwr(left_mapkey,
right_simkey, right_weightkey,
multiplier=1.,
inifile=None, datapath_db=None,
outdir="./plots",
output_tag=None):
r"""w_left m_left x w_right m_right"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(left_mapkey, "type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
crosspwr_collection = {}
for treatment in map_cases['treatment']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (left_mapkey, treatment)
dbkeydict['map2_key'] = right_simkey
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (left_mapkey, treatment)
dbkeydict['noiseinv2_key'] = right_weightkey
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwrspec_out_signal[0]['binavg'] *= multiplier
pwrspec_out_signal[1]['binavg'] *= multiplier
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0])
mtag = "%s_%s" % (output_tag, treatment)
pe.summarize_pwrspec(pwrspec_out_signal[1],
pwr_1d_from_2d,
pwrspec_out_signal[0],
mtag, outdir=outdir)
crosspwr_collection[treatment] = (pwrspec_out_signal[1],
pwr_1d_from_2d,
pwrspec_out_signal[0])
if not output_tag:
caller.multiprocess_stack()
return None
else:
return crosspwr_collection
def execute(self, processes):
funcname = "quadratic_products.pwrspec_combinations.pwrspec_caller"
caller = aggregate_outputs.AggregateOutputs(funcname)
# TODO: simplify this
try:
dbkeydict = {}
dbkeydict['noiseinv1_key'] = "%s:weight;0modes" % \
self.params['map_key']
files = dp.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=self.datapath_db)
except KeyError:
print "WARNING: NO 21cm weight in WiggleZxGBT_modesim"
files = {'noiseinv1_key': False}
if self.params['wigglez_sel_key']:
sel_key = self.params['wigglez_sel_key']
wigglez_selfile = self.datapath_db.fetch(sel_key)
else:
print "Warning: in WiggleZxGBT_modesim no selection function"
wigglez_selfile = False
execute_key = "sim:0modes"
caller.execute(self.params['sim_file'],
self.params['wigglez_sim_file'],
files['noiseinv1_key'],
wigglez_selfile,
self.params,
execute_key=execute_key)
caller.multiprocess_stack(self.params["outfile"],
debug=False, ncpu=self.params['ncpu'])
def find_avg_fsky(map_key, tack_on=None, refinement=2, pad=5, order=1):
"""Take all the pairs that enter the autopower, open their weight files and
find the fsky for each data treatment
In a pipeline
fsky = find_avg_fsky(self.params["map_key"],
tack_on=self.params["tack_on"],
refinement=self.params["refinement"],
pad=self.params["pad"],
order=self.params["order"])
"""
fsky = {}
datapath_db = dp.DataPath()
map_cases = datapath_db.fileset_cases(map_key, "pair;type;treatment")
# This code is essentially verbatim for the permutation in the real
# autopower
unique_pairs = dp.GBTauto_cross_pairs(map_cases['pair'],
map_cases['pair'],
cross_sym="_with_")
treatment_list = map_cases['treatment']
for treatment in treatment_list:
for item in unique_pairs:
dbkeydict = {}
mapset0 = (map_key, item[0], treatment)
mapset1 = (map_key, item[1], treatment)
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = dp.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db,
tack_on=tack_on)
print files['noiseinv1_key'], files['noiseinv2_key']
weight1 = algebra.make_vect(algebra.load(files['noiseinv1_key']))
weight2 = algebra.make_vect(algebra.load(files['noiseinv2_key']))
physweight1 = bh.repackage_kiyo(
pg.physical_grid(weight1,
refinement=refinement,
pad=pad,
order=order))
physweight2 = bh.repackage_kiyo(
pg.physical_grid(weight2,
refinement=refinement,
pad=pad,
order=order))
#fsky = np.sum(physweight1 * physweight2)**2
#fsky /= np.sum(physweight1**2 * physweight2**2)
#fsky /= float(physweight1.size)
fsky = np.sum(weight1 * weight2)**2
fsky /= np.sum(weight1**2 * weight2**2)
fsky /= float(weight1.size)
print "volume factor in noise weight: ", fsky
return fsky
def generate_runlist(self):
map_key = self.params['map_key']
map_cases = self.datapath_db.fileset_cases(map_key,
"pair;type;treatment")
# first compute the mixing matrix for the crossed (AxB) weightings
unique_pairs = dp.GBTauto_cross_pairs(map_cases['pair'],
map_cases['pair'],
cross_sym="_with_")
# assume the weights are the same for all cleaning treatments
# TODO: this may change in the future
treatment = "0modes"
self.all_pairs = {}
for item in unique_pairs:
dbkeydict = {}
mapset0 = (map_key, item[0], treatment)
mapset1 = (map_key, item[1], treatment)
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = dp.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=self.datapath_db)
self.all_pairs[item[0]] = (files['noiseinv1_key'],
files['noiseinv2_key'])
# For the autopower (in noise assessment), we use the same cleaned maps
# and the weights are the same for various pairs, e.g.
# A_with_B is the same as A_with_C, etc. because the mode cleaning does
# not impact the weighting functions
A_file = (self.datapath_db.fetch('%s:A_with_B;noise_inv;0modes' %
map_key,
silent=True))
self.all_pairs["A_with_A"] = (A_file, A_file)
B_file = (self.datapath_db.fetch('%s:B_with_A;noise_inv;0modes' %
map_key,
silent=True))
self.all_pairs["B_with_B"] = (B_file, B_file)
C_file = (self.datapath_db.fetch('%s:C_with_A;noise_inv;0modes' %
map_key,
silent=True))
self.all_pairs["C_with_C"] = (C_file, C_file)
D_file = (self.datapath_db.fetch('%s:D_with_A;noise_inv;0modes' %
map_key,
silent=True))
self.all_pairs["D_with_D"] = (D_file, D_file)
self.mixing_fileout = {}
for pair in self.all_pairs:
self.mixing_fileout[pair] = "%s_%s.shelve" % \
(self.params['perpair_base'], pair)
def generate_runlist(self):
map_key = self.params['map_key']
map_cases = self.datapath_db.fileset_cases(map_key,
"pair;type;treatment")
# first compute the mixing matrix for the crossed (AxB) weightings
unique_pairs = dp.GBTauto_cross_pairs(map_cases['pair'],
map_cases['pair'],
cross_sym="_with_")
# assume the weights are the same for all cleaning treatments
# TODO: this may change in the future
treatment = "0modes"
self.all_pairs = {}
for item in unique_pairs:
dbkeydict = {}
mapset0 = (map_key, item[0], treatment)
mapset1 = (map_key, item[1], treatment)
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = dp.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=self.datapath_db)
self.all_pairs[item[0]] = (files['noiseinv1_key'],
files['noiseinv2_key'])
# For the autopower (in noise assessment), we use the same cleaned maps
# and the weights are the same for various pairs, e.g.
# A_with_B is the same as A_with_C, etc. because the mode cleaning does
# not impact the weighting functions
A_file = (self.datapath_db.fetch(
'%s:A_with_B;noise_inv;0modes' % map_key, silent=True))
self.all_pairs["A_with_A"] = (A_file, A_file)
B_file = (self.datapath_db.fetch(
'%s:B_with_A;noise_inv;0modes' % map_key, silent=True))
self.all_pairs["B_with_B"] = (B_file, B_file)
C_file = (self.datapath_db.fetch(
'%s:C_with_A;noise_inv;0modes' % map_key, silent=True))
self.all_pairs["C_with_C"] = (C_file, C_file)
D_file = (self.datapath_db.fetch(
'%s:D_with_A;noise_inv;0modes' % map_key, silent=True))
self.all_pairs["D_with_D"] = (D_file, D_file)
self.mixing_fileout = {}
for pair in self.all_pairs:
self.mixing_fileout[pair] = "%s_%s.shelve" % \
(self.params['perpair_base'], pair)
def execute(self, processes):
funcname = "quadratic_products.pwrspec_combinations.pwrspec_caller"
caller = aggregate_outputs.AggregateOutputs(funcname)
map_key = self.params['map_key']
tack_on = self.params['tack_on']
map_cases = self.datapath_db.fileset_cases(map_key,
"pair;type;treatment")
# We use the same cleaned maps and the weights are the same for
# various pairs, e.g. A_with_B is the same as A_with_C, etc. because
# the mode cleaning does not impact the weighting functions
noise_pairs = {"A_with_A": "A_with_B",
"B_with_B": "B_with_A",
"C_with_C": "C_with_A",
"D_with_D": "D_with_A"
}
# unless specified otherwise find power spectra for all treatments
if self.params['treatments'] is None:
treatment_list = map_cases['treatment']
else:
treatment_list = self.params['treatments']
for treatment in treatment_list:
for item in noise_pairs:
dbkeydict = {}
mapset0 = (map_key, noise_pairs[item], treatment)
mapset1 = (map_key, noise_pairs[item], treatment)
dbkeydict['map1_key'] = "%s:%s;map;%s" % mapset0
dbkeydict['map2_key'] = "%s:%s;map;%s" % mapset1
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = dp.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=self.datapath_db,
tack_on=tack_on)
execute_key = "%s:%s" % (item, treatment)
caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
self.params,
execute_key=execute_key)
caller.multiprocess_stack(self.params["outfile"],
debug=False, ncpu=self.params['ncpu'])
def execute(self, processes):
funcname = "quadratic_products.pwrspec_combinations.pwrspec_caller"
caller = aggregate_outputs.AggregateOutputs(funcname)
map_key = self.params['map_key']
tack_on = self.params['tack_on']
map_cases = self.datapath_db.fileset_cases(map_key,
"pair;type;treatment")
unique_pairs = dp.GBTauto_cross_pairs(map_cases['pair'],
map_cases['pair'],
cross_sym="_with_")
# unless specified otherwise find power spectra for all treatments
if self.params['treatments'] is None:
treatment_list = map_cases['treatment']
else:
treatment_list = self.params['treatments']
for treatment in treatment_list:
for item in unique_pairs:
dbkeydict = {}
mapset0 = (map_key, item[0], treatment)
mapset1 = (map_key, item[1], treatment)
dbkeydict['map1_key'] = "%s:%s;map;%s" % mapset0
dbkeydict['map2_key'] = "%s:%s;map;%s" % mapset1
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = dp.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=self.datapath_db,
tack_on=tack_on)
execute_key = "%s:%s" % (item[0], treatment)
caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
self.params,
execute_key=execute_key)
caller.multiprocess_stack(self.params["outfile"],
debug=False, ncpu=self.params['ncpu'])
def batch_wigglez_automock_run(mock_key,
sel_key,
inifile=None,
datapath_db=None,
output_tag=None):
r"""TODO: make this work; wrote this but never really needed it yet
"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
mock_cases = datapath_db.fileset_cases(mock_key, "realization")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
for index in mock_cases['realization']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:%s" % (mock_key, index)
dbkeydict['map2_key'] = "%s:%s" % (mock_key, index)
dbkeydict['noiseinv1_key'] = sel_key
dbkeydict['noiseinv2_key'] = sel_key
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
caller.multiprocess_stack()
def execute(self, processes):
funcname = "quadratic_products.pwrspec_combinations.pwrspec_caller"
caller = aggregate_outputs.AggregateOutputs(funcname)
map_key = self.params['map_key']
sim_file = self.params['sim_file']
map_cases = self.datapath_db.fileset_cases(map_key,
"pair;type;treatment")
unique_pairs = dp.GBTauto_cross_pairs(map_cases['pair'],
map_cases['pair'],
cross_sym="_with_")
treatment = "0modes"
for item in unique_pairs:
dbkeydict = {}
mapset0 = (map_key, item[0], treatment)
mapset1 = (map_key, item[1], treatment)
dbkeydict['map1_key'] = "%s:%s;map;%s" % mapset0
dbkeydict['map2_key'] = "%s:%s;map;%s" % mapset1
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = dp.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=self.datapath_db)
execute_key = "%s:%s" % (item[0], treatment)
caller.execute(sim_file,
sim_file,
files['noiseinv1_key'],
files['noiseinv2_key'],
self.params,
execute_key=execute_key)
caller.multiprocess_stack(self.params["outfile"],
debug=False, ncpu=self.params['ncpu'])
def execute(self, processes):
funcname = "quadratic_products.pwrspec_combinations.pwrspec_caller"
caller_data = aggregate_outputs.AggregateOutputs(funcname)
caller_mock = aggregate_outputs.AggregateOutputs(funcname)
wigglez_key = self.params['wigglez_key']
sel_key = self.params['wigglez_sel_key']
mock_key = self.params['wigglez_mock_key']
mock_files = self.datapath_db.fetch(mock_key)
map_key = self.params['map_key']
map_cases = self.datapath_db.fileset_cases(map_key,
"type;treatment")
for treatment in map_cases['treatment']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (map_key, treatment)
dbkeydict['map2_key'] = wigglez_key
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \
(map_key, treatment)
dbkeydict['noiseinv2_key'] = sel_key
files = dp.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=self.datapath_db)
execute_key = "data:%s" % treatment
#print files, execute_key
caller_data.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
self.params,
execute_key=execute_key)
caller_data.multiprocess_stack(self.params["outfile_data"],
debug=False, ncpu=self.params['ncpu'])
for treatment in map_cases['treatment']:
for item in mock_files[0]:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (map_key, treatment)
dbkeydict['map2_key'] = "%s:%s" % (mock_key, item)
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \
(map_key, treatment)
dbkeydict['noiseinv2_key'] = sel_key
files = dp.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=self.datapath_db)
execute_key = "mock%s:%s" % (item, treatment)
#print files, execute_key
caller_mock.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
self.params,
execute_key=execute_key)
caller_mock.multiprocess_stack(self.params["outfile_mock"],
debug=False, ncpu=self.params['ncpu'])
def batch_crosspwr_transfer(cleaned_simkey,
truesignal_simkey, truesignal_weightkey,
reference_simkey, reference_weightkey,
inifile=None, datapath_db=None,
outdir="./plots",
output_tag=None):
r"""take relevant cross-powers
cleaned_simkey(map) * cleaned_simkey(weight) x
truesignal_weightkey * truesignal_simkey
divided by:
reference_simkey * reference_weightkey x truesignal_weightkey * truesignal_simkey
"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(cleaned_simkey, "type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
dbkeydict = {}
dbkeydict['map1_key'] = reference_simkey
dbkeydict['map2_key'] = truesignal_simkey
dbkeydict['noiseinv1_key'] = reference_weightkey
dbkeydict['noiseinv2_key'] = truesignal_weightkey
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
reference_pwrspec_out = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
file_tools.mkparents(outdir)
ref_pwr_1d_from_2d = pe.convert_2d_to_1d(reference_pwrspec_out[0])
ref_pwr_1d = reference_pwrspec_out[1]['binavg']
ref_pwr_2d = reference_pwrspec_out[0]['binavg']
ref_pwr_1d_from_2d = ref_pwr_1d_from_2d['binavg']
bin_left = reference_pwrspec_out[1]['bin_left']
bin_center = reference_pwrspec_out[1]['bin_center']
bin_right = reference_pwrspec_out[1]['bin_right']
counts_histo = reference_pwrspec_out[1]['counts_histo']
transfer_functions = {}
for treatment in map_cases['treatment']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (cleaned_simkey, treatment)
dbkeydict['map2_key'] = truesignal_simkey
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (cleaned_simkey, treatment)
dbkeydict['noiseinv2_key'] = truesignal_weightkey
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0])
pwr_1d = pwrspec_out_signal[1]['binavg']
pwr_2d = pwrspec_out_signal[0]['binavg']
pwr_1d_from_2d = pwr_1d_from_2d['binavg']
trans1d_mode = pwr_1d / ref_pwr_1d
trans1d_from2d_mode = pwr_1d_from_2d / ref_pwr_1d_from_2d
trans2d_mode = pwr_2d / ref_pwr_2d
transfer_functions[treatment] = (trans1d_mode,
trans1d_from2d_mode,
trans2d_mode)
filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment)
outfile = open(filename, "w")
for specdata in zip(bin_left, bin_center,
bin_right, counts_histo,
pwr_1d, pwr_1d_from_2d,
trans1d_mode, trans1d_from2d_mode):
outfile.write(("%10.15g " * 8 + "\n") % specdata)
outfile.close()
if not output_tag:
caller.multiprocess_stack()
return None
else:
return transfer_functions
def batch_gbtxwigglez_data_run(gbt_map_key, wigglez_map_key,
wigglez_mock_key, wigglez_selection_key,
inifile=None, datapath_db=None,
outdir="./plots",
output_tag=None,
beam_transfer=None,
mode_transfer_1d=None,
mode_transfer_2d=None,
theory_curve=None):
r"""assemble the pairs of GBT and WiggleZ and calculate the cross-power"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(gbt_map_key, "type;treatment")
mock_cases = datapath_db.fileset_cases(wigglez_mock_key, "realization")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
if output_tag:
file_tools.mkparents(outdir)
for treatment in map_cases['treatment']:
# TODO: make this more elegant
# TODO: convert treatment into mode num
transfer_2d = None
if (mode_transfer_2d is not None) and (beam_transfer is None):
transfer_2d = mode_transfer_2d[treatment][2]
if (mode_transfer_2d is None) and (beam_transfer is not None):
transfer_2d = beam_transfer
if (mode_transfer_2d is not None) and (beam_transfer is not None):
transfer_2d = mode_transfer_2d[treatment][2] * beam_transfer
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for index in mock_cases['realization']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = "%s:%s" % (wigglez_mock_key, index)
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \
(gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'], files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
transfer=transfer_2d))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
if mode_transfer_1d is not None:
transfunc = mode_transfer_1d[treatment][1]
else:
transfunc = None
mtag = output_tag + "_%s_mock" % treatment
agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d, pwr_2d, mtag,
outdir=outdir,
apply_1d_transfer=transfunc)
mean1dmock = agg_pwrspec["mean_1d"]
std1dmock = agg_pwrspec["std_1d"]
covmock = agg_pwrspec["covmat_1d"]
# now recover the xspec with the real data
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = wigglez_map_key
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0],
transfer=transfer_2d)
pwr_1d = pwrspec_out_signal[1]['binavg']
pwr_1d_from_2d = pwr_1d_from_2d['binavg']
if mode_transfer_1d is not None:
pwr_1d /= mode_transfer_1d[treatment][1]
pwr_1d_from_2d /= mode_transfer_1d[treatment][1]
# assume that they all have the same binning
bin_left = pwrspec_out_signal[1]['bin_left']
bin_center = pwrspec_out_signal[1]['bin_center']
bin_right = pwrspec_out_signal[1]['bin_right']
counts_histo = pwrspec_out_signal[1]['counts_histo']
filename = "%s/%s_%s.dat" % (outdir,
output_tag,
treatment)
outfile = open(filename, "w")
for specdata in zip(bin_left, bin_center,
bin_right, counts_histo,
pwr_1d, pwr_1d_from_2d,
mean1dmock, std1dmock):
outfile.write(("%10.15g " * 8 + "\n") % specdata)
outfile.close()
# TODO: kludge to make a fast fit; remove
theory_curve = np.genfromtxt("plots/sim_15hr_oldmap_str_temperature_xWigglez/sim_15hr_oldmap_str_temperature_xWigglez_avg_from2d.dat")
theory_curve = theory_curve[:, 4]
if theory_curve is not None:
restrict = np.where(np.logical_and(bin_center > 0.09,
bin_center < 1.1))
res_slice = slice(min(restrict[0]), max(restrict[0]))
#restrict_alt = np.where(restrict)[0][np.newaxis, :]
#restricted_cov = covmock[restrict_alt][0]
from core import utils
amplitude = utils.ampfit(pwr_1d_from_2d[res_slice],
covmock[res_slice, res_slice],
theory_curve[res_slice])
print "AMP:", mtag, treatment, amplitude
if not output_tag:
caller.multiprocess_stack()
return None
def batch_single_crosspwr(left_mapkey,
right_simkey,
right_weightkey,
multiplier=1.,
inifile=None,
datapath_db=None,
outdir="./plots",
output_tag=None):
r"""w_left m_left x w_right m_right"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(left_mapkey, "type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
crosspwr_collection = {}
for treatment in map_cases['treatment']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (left_mapkey, treatment)
dbkeydict['map2_key'] = right_simkey
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (left_mapkey, treatment)
dbkeydict['noiseinv2_key'] = right_weightkey
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwrspec_out_signal[0]['binavg'] *= multiplier
pwrspec_out_signal[1]['binavg'] *= multiplier
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0])
mtag = "%s_%s" % (output_tag, treatment)
pe.summarize_pwrspec(pwrspec_out_signal[1],
pwr_1d_from_2d,
pwrspec_out_signal[0],
mtag,
outdir=outdir)
crosspwr_collection[treatment] = (pwrspec_out_signal[1],
pwr_1d_from_2d,
pwrspec_out_signal[0])
if not output_tag:
caller.multiprocess_stack()
return None
else:
return crosspwr_collection
def batch_gbtpwrspec_data_run(map_key,
inifile=None,
datapath_db=None,
output_tag=None,
beam_transfer=None,
outdir="./plots/",
square_1dmodetrans=False,
mode_transfer_1d=None,
mode_transfer_2d=None):
r"""Form the pairs of maps1*weight1 x map0*weight0 for calculating the
auto-power of the GBT data"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(map_key, "pair;type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
if output_tag:
file_tools.mkparents(outdir)
pwrspec_collection = {}
for treatment in map_cases['treatment']:
unique_pairs = data_paths.GBTauto_cross_pairs(map_cases['pair'],
map_cases['pair'],
cross_sym="_with_")
# TODO: make this more elegant
transfer_2d = None
if (mode_transfer_2d is not None) and (beam_transfer is None):
transfer_2d = mode_transfer_2d[treatment][1]
if (mode_transfer_2d is None) and (beam_transfer is not None):
transfer_2d = beam_transfer
if (mode_transfer_2d is not None) and (beam_transfer is not None):
transfer_2d = mode_transfer_2d[treatment][1] * beam_transfer
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for item in unique_pairs:
dbkeydict = {}
# NOTE: formerly had "weight" instead of noise_inv
mapset0 = (map_key, item[0], treatment)
mapset1 = (map_key, item[1], treatment)
dbkeydict['map1_key'] = "%s:%s;map;%s" % mapset0
dbkeydict['map2_key'] = "%s:%s;map;%s" % mapset1
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(
pe.convert_2d_to_1d(pwrspec_out[0], transfer=transfer_2d))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
mtag = output_tag + "_%s" % treatment
if mode_transfer_1d is not None:
transfunc = mode_transfer_1d[treatment][0]
if square_1dmodetrans:
transfunc *= transfunc
else:
transfunc = None
agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d,
pwr_2d,
mtag,
outdir=outdir,
apply_1d_transfer=transfunc)
# (mean_1d, std_1d, covmat_1d)
pwrspec_collection[treatment] = agg_pwrspec
if output_tag:
return pwrspec_collection
else:
caller.multiprocess_stack()
return None
def batch_crosspwr_transfer(cleaned_simkey,
truesignal_simkey,
truesignal_weightkey,
reference_simkey,
reference_weightkey,
inifile=None,
datapath_db=None,
outdir="./plots",
output_tag=None):
r"""take relevant cross-powers
cleaned_simkey(map) * cleaned_simkey(weight) x
truesignal_weightkey * truesignal_simkey
divided by:
reference_simkey * reference_weightkey x truesignal_weightkey * truesignal_simkey
"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(cleaned_simkey, "type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
dbkeydict = {}
dbkeydict['map1_key'] = reference_simkey
dbkeydict['map2_key'] = truesignal_simkey
dbkeydict['noiseinv1_key'] = reference_weightkey
dbkeydict['noiseinv2_key'] = truesignal_weightkey
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
reference_pwrspec_out = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
file_tools.mkparents(outdir)
ref_pwr_1d_from_2d = pe.convert_2d_to_1d(reference_pwrspec_out[0])
ref_pwr_1d = reference_pwrspec_out[1]['binavg']
ref_pwr_2d = reference_pwrspec_out[0]['binavg']
ref_pwr_1d_from_2d = ref_pwr_1d_from_2d['binavg']
bin_left = reference_pwrspec_out[1]['bin_left']
bin_center = reference_pwrspec_out[1]['bin_center']
bin_right = reference_pwrspec_out[1]['bin_right']
counts_histo = reference_pwrspec_out[1]['counts_histo']
transfer_functions = {}
for treatment in map_cases['treatment']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (cleaned_simkey, treatment)
dbkeydict['map2_key'] = truesignal_simkey
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (cleaned_simkey,
treatment)
dbkeydict['noiseinv2_key'] = truesignal_weightkey
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0])
pwr_1d = pwrspec_out_signal[1]['binavg']
pwr_2d = pwrspec_out_signal[0]['binavg']
pwr_1d_from_2d = pwr_1d_from_2d['binavg']
trans1d_mode = pwr_1d / ref_pwr_1d
trans1d_from2d_mode = pwr_1d_from_2d / ref_pwr_1d_from_2d
trans2d_mode = pwr_2d / ref_pwr_2d
transfer_functions[treatment] = (trans1d_mode, trans1d_from2d_mode,
trans2d_mode)
filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment)
outfile = open(filename, "w")
for specdata in zip(bin_left, bin_center, bin_right, counts_histo,
pwr_1d, pwr_1d_from_2d, trans1d_mode,
trans1d_from2d_mode):
outfile.write(("%10.15g " * 8 + "\n") % specdata)
outfile.close()
if not output_tag:
caller.multiprocess_stack()
return None
else:
return transfer_functions
def batch_gbtpwrspec_data_run(map_key, inifile=None, datapath_db=None,
output_tag=None, beam_transfer=None,
outdir="./plots/",
square_1dmodetrans=False,
mode_transfer_1d=None,
mode_transfer_2d=None):
r"""Form the pairs of maps1*weight1 x map0*weight0 for calculating the
auto-power of the GBT data"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(map_key, "pair;type;treatment")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
if output_tag:
file_tools.mkparents(outdir)
pwrspec_collection = {}
for treatment in map_cases['treatment']:
unique_pairs = data_paths.GBTauto_cross_pairs(map_cases['pair'],
map_cases['pair'],
cross_sym="_with_")
# TODO: make this more elegant
transfer_2d = None
if (mode_transfer_2d is not None) and (beam_transfer is None):
transfer_2d = mode_transfer_2d[treatment][1]
if (mode_transfer_2d is None) and (beam_transfer is not None):
transfer_2d = beam_transfer
if (mode_transfer_2d is not None) and (beam_transfer is not None):
transfer_2d = mode_transfer_2d[treatment][1] * beam_transfer
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for item in unique_pairs:
dbkeydict = {}
# NOTE: formerly had "weight" instead of noise_inv
mapset0 = (map_key, item[0], treatment)
mapset1 = (map_key, item[1], treatment)
dbkeydict['map1_key'] = "%s:%s;map;%s" % mapset0
dbkeydict['map2_key'] = "%s:%s;map;%s" % mapset1
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
transfer=transfer_2d))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
mtag = output_tag + "_%s" % treatment
if mode_transfer_1d is not None:
transfunc = mode_transfer_1d[treatment][0]
if square_1dmodetrans:
transfunc *= transfunc
else:
transfunc = None
agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d, pwr_2d, mtag,
outdir=outdir,
apply_1d_transfer=transfunc)
# (mean_1d, std_1d, covmat_1d)
pwrspec_collection[treatment] = agg_pwrspec
if output_tag:
return pwrspec_collection
else:
caller.multiprocess_stack()
return None
def batch_sim_run(simleft_key, simright_key,
weightleft_key, weightright_key,
inifile=None, datapath_db=None,
outdir="./plots/",
usecache_output_tag=None, transfer=None):
r"""
typical weight matrix:
db:GBT_15hr_map_cleaned_0mode:A_with_B;noise_inv
"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
mock_cases = datapath_db.fileset_cases(simleft_key, "realization")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if usecache_output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname, cache_path,
generate=generate, verbose=True)
if usecache_output_tag:
output_root = "%s/%s/" % (outdir, usecache_output_tag)
file_tools.mkparents(output_root)
pwr_1d = []
pwr_1d_from_2d = []
pwr_2d = []
for index in mock_cases['realization']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:%s" % (simleft_key, index)
dbkeydict['map2_key'] = "%s:%s" % (simright_key, index)
dbkeydict['noiseinv1_key'] = weightleft_key
dbkeydict['noiseinv2_key'] = weightright_key
files = data_paths.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'], files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if usecache_output_tag:
pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0],
transfer=transfer))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if usecache_output_tag:
pe.summarize_agg_pwrspec(pwr_1d, pwr_1d_from_2d, pwr_2d,
usecache_output_tag, outdir=output_root)
retval = (pwr_1d, pwr_1d_from_2d, pwr_2d)
else:
caller.multiprocess_stack()
retval = None
return retval
def batch_gbtxwigglez_data_run(gbt_map_key,
wigglez_map_key,
wigglez_mock_key,
wigglez_selection_key,
inifile=None,
datapath_db=None,
outdir="./plots",
output_tag=None,
beam_transfer=None,
mode_transfer_1d=None,
mode_transfer_2d=None,
theory_curve=None):
r"""assemble the pairs of GBT and WiggleZ and calculate the cross-power"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
map_cases = datapath_db.fileset_cases(gbt_map_key, "type;treatment")
mock_cases = datapath_db.fileset_cases(wigglez_mock_key, "realization")
funcname = "correlate.batch_quadratic.call_xspec_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
if output_tag:
file_tools.mkparents(outdir)
for treatment in map_cases['treatment']:
# TODO: make this more elegant
# TODO: convert treatment into mode num
transfer_2d = None
if (mode_transfer_2d is not None) and (beam_transfer is None):
transfer_2d = mode_transfer_2d[treatment][2]
if (mode_transfer_2d is None) and (beam_transfer is not None):
transfer_2d = beam_transfer
if (mode_transfer_2d is not None) and (beam_transfer is not None):
transfer_2d = mode_transfer_2d[treatment][2] * beam_transfer
pwr_1d = []
pwr_2d = []
pwr_1d_from_2d = []
for index in mock_cases['realization']:
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = "%s:%s" % (wigglez_mock_key, index)
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \
(gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(
pe.convert_2d_to_1d(pwrspec_out[0], transfer=transfer_2d))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
if mode_transfer_1d is not None:
transfunc = mode_transfer_1d[treatment][1]
else:
transfunc = None
mtag = output_tag + "_%s_mock" % treatment
agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d,
pwr_2d,
mtag,
outdir=outdir,
apply_1d_transfer=transfunc)
mean1dmock = agg_pwrspec["mean_1d"]
std1dmock = agg_pwrspec["std_1d"]
covmock = agg_pwrspec["covmat_1d"]
# now recover the xspec with the real data
dbkeydict = {}
dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment)
dbkeydict['map2_key'] = wigglez_map_key
dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (gbt_map_key, treatment)
dbkeydict['noiseinv2_key'] = wigglez_selection_key
files = data_paths.convert_dbkeydict_to_filedict(
dbkeydict, datapath_db=datapath_db)
pwrspec_out_signal = caller.execute(files['map1_key'],
files['map2_key'],
files['noiseinv1_key'],
files['noiseinv2_key'],
inifile=inifile)
if output_tag:
pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0],
transfer=transfer_2d)
pwr_1d = pwrspec_out_signal[1]['binavg']
pwr_1d_from_2d = pwr_1d_from_2d['binavg']
if mode_transfer_1d is not None:
pwr_1d /= mode_transfer_1d[treatment][1]
pwr_1d_from_2d /= mode_transfer_1d[treatment][1]
# assume that they all have the same binning
bin_left = pwrspec_out_signal[1]['bin_left']
bin_center = pwrspec_out_signal[1]['bin_center']
bin_right = pwrspec_out_signal[1]['bin_right']
counts_histo = pwrspec_out_signal[1]['counts_histo']
filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment)
outfile = open(filename, "w")
for specdata in zip(bin_left, bin_center, bin_right, counts_histo,
pwr_1d, pwr_1d_from_2d, mean1dmock, std1dmock):
outfile.write(("%10.15g " * 8 + "\n") % specdata)
outfile.close()
# TODO: kludge to make a fast fit; remove
theory_curve = np.genfromtxt(
"plots/sim_15hr_oldmap_str_temperature_xWigglez/sim_15hr_oldmap_str_temperature_xWigglez_avg_from2d.dat"
)
theory_curve = theory_curve[:, 4]
if theory_curve is not None:
restrict = np.where(
np.logical_and(bin_center > 0.09, bin_center < 1.1))
res_slice = slice(min(restrict[0]), max(restrict[0]))
#restrict_alt = np.where(restrict)[0][np.newaxis, :]
#restricted_cov = covmock[restrict_alt][0]
from core import utils
amplitude = utils.ampfit(pwr_1d_from_2d[res_slice],
covmock[res_slice, res_slice],
theory_curve[res_slice])
print "AMP:", mtag, treatment, amplitude
if not output_tag:
caller.multiprocess_stack()
return None
