def wrap_batch_physical_sim_run(inifile, generate=False, outdir="./plots/"):
r"""Wrapper to the physical sim processing
"""
params_init = {
"simkey": "point to the sims to run",
"spec_ini": "ini file for the spectral estimation",
"output_tag": "tag identifying the output somehow"
}
prefix = "csp_"
params = parse_ini.parse(inifile, params_init, prefix=prefix)
print params
output_tag = "%s_%s" % (params['simkey'], params['output_tag'])
output_root = "%s/%s/" % (outdir, output_tag)
if generate:
output_tag = None
print output_root
print output_tag
file_tools.mkparents(output_root)
parse_ini.write_params(params, output_root + 'params.ini', prefix=prefix)
datapath_db = data_paths.DataPath()
return batch_physical_sim_run(params["simkey"],
inifile=params["spec_ini"],
datapath_db=datapath_db,
outdir=output_root,
output_tag=output_tag)
def generate_windows(window="blackman"):
datapath_db = data_paths.DataPath()
# first generate a window for the full physical volume
filename = datapath_db.fetch('simideal_15hr_physical', intend_read=True,
pick='1')
print filename
pcube = algebra.make_vect(algebra.load(filename))
pwindow = algebra.make_vect(fftutil.window_nd(pcube.shape, name=window),
axis_names=('freq', 'ra', 'dec'))
pwindow.copy_axis_info(pcube)
print pwindow.shape
algebra.save("physical_window.npy", pwindow)
# now generate one for the observed region and project onto the physical
# volume.
filename = datapath_db.fetch('simideal_15hr_beam', intend_read=True,
pick='1')
print filename
ocube = algebra.make_vect(algebra.load(filename))
owindow = algebra.make_vect(fftutil.window_nd(ocube.shape, name=window),
axis_names=('freq', 'ra', 'dec'))
owindow.copy_axis_info(ocube)
print owindow.shape
print owindow.axes
algebra.save("observed_window.npy", owindow)
pwindow = physical_gridding.physical_grid(owindow, refinement=2)
print pwindow.shape
algebra.save("observed_window_physreg.npy", pwindow)
def call_data_autopower(basemaps,
treatments,
inifile=None,
generate=False,
outdir="./plots/",
mode_transfer_1d=None,
mode_transfer_2d=None,
beam_transfer=None,
alttag=None):
r"""Call a chunk of batch data runs for e.g. different map products
"""
datapath_db = data_paths.DataPath()
for base in basemaps:
for treatment in treatments:
output_tag = base + treatment
if alttag:
output_tag += "_" + output_tag
output_root = "%s/%s/" % (outdir, output_tag)
if generate:
output_tag = None
batch_gbtpwrspec_data_run(mapname,
inifile=inifile,
datapath_db=datapath_db,
output_tag=output_tag,
outdir=output_root,
beam_transfer=beam_transfer,
mode_transfer_1d=mode_transfer_1d,
mode_transfer_2d=mode_transfer_2d)
def __init__(self, parameter_file_or_dict=None):
# recordkeeping
self.pairs = {}
self.pairs_nosim = {}
self.pairlist = []
self.noisefiledict = {}
self.datapath_db = dp.DataPath()
self.params = parse_ini.parse(parameter_file_or_dict,
params_init,
prefix=prefix)
self.freq_list = sp.array(self.params['freq_list'], dtype=int)
self.lags = sp.array(self.params['lags'])
self.output_root = self.datapath_db.fetch(self.params['output_root'],
intend_write=True)
if self.params['SVD_root']:
self.SVD_root = self.datapath_db.fetch(self.params['SVD_root'],
intend_write=True)
print "WARNING: using %s to clean (intended?)" % self.SVD_root
else:
self.SVD_root = self.output_root
# Write parameter file.
kiyopy.utils.mkparents(self.output_root)
parse_ini.write_params(self.params,
self.output_root + 'params.ini',
prefix=prefix)
def find_wigglez_region(fieldname):
"""Find the RA/Dec edges of a WiggleZ field by exhaustion
"""
datapath_db = data_paths.DataPath()
db_key = "WiggleZ_%s_mock_catalog" % fieldname
infile_mock = datapath_db.fetch(db_key, intend_read=True,
purpose="WiggleZ real data catalog", silent=True)
n_rand_cats = len(infile_mock[0])
ra_minmax = [None, None]
dec_minmax = [None, None]
freq_minmax = [None, None]
ndtype = [('RA', float), ('Dec', float), ('z', float),
('r-mag', float), ('ijack', int), ('sec', int)]
for index in infile_mock[0]:
filename = infile_mock[1][index]
#print "loading: " + filename
catalog = np.genfromtxt(filename, dtype=ndtype,
skiprows=1)
freq_vec = cc.freq_21cm_MHz * 1.e6 / (1 + catalog['z'])
ra_vec = catalog['RA']
dec_vec = catalog['Dec']
ra_minmax = minmax(ra_vec, ra_minmax)
dec_minmax = minmax(dec_vec, dec_minmax)
freq_minmax = minmax(freq_vec, freq_minmax)
#print ra_vec.min(), ra_vec.max(), ra_minmax
#print dec_vec.min(), dec_vec.max(), dec_minmax
#print freq_vec.min(), freq_vec.max(), freq_minmax
return ra_minmax, dec_minmax, freq_minmax
def combine_maps(source_key, combined_key,
signal='map', weight='noise_inv', divider=";",
fullcov=False, batchsim=None):
r"""
`source_key` is the file db key for the maps to combine
`combined_key` is the file db key for the combined maps
`signal` is the tag in the file db entry for the signal maps
`weight` is the tag in the file db entry for the N^-1 weights
`fullcov` uses a memory map for large N^-1 and pulls the diagonal
`divider` is the token that divides the map section name
from the data type e.g. "A_with_B;noise_inv"
"""
datapath_db = data_paths.DataPath()
input_fdb = datapath_db.fetch(source_key, intend_read=True,
silent=True)
output_fdb = datapath_db.fetch(combined_key, intend_write=True,
silent=True)
(input_fdict, output_fdict) = (input_fdb[1], output_fdb[1])
# determine the number of foreground cleaning options (modes removed)
input_map_cases = datapath_db.fileset_cases(source_key,
"pair;type;treatment", divider=divider)
if batchsim:
output_map_cases = datapath_db.fileset_cases(combined_key,
"type;treatment;simnum", divider=divider)
else:
output_map_cases = datapath_db.fileset_cases(combined_key,
"type;treatment", divider=divider)
if input_map_cases['treatment'] != output_map_cases['treatment']:
print "the source map does not match the requested combined map output"
sys.exit()
print input_map_cases['pair'], input_map_cases['treatment'], \
output_map_cases['type']
for treatment in input_map_cases['treatment']:
inputmap_dict = {}
inputweight_dict = {}
for split in input_map_cases['pair']:
mapkey = "%s;%s;%s" % (split, signal, treatment)
weightkey = "%s;%s;%s" % (split, weight, treatment)
inputmap_dict[split] = input_fdict[mapkey]
inputweight_dict[split] = input_fdict[weightkey]
output_dict = {}
for product in output_map_cases['type']:
if batchsim:
mapkey = "%s;%s;%s" % (product, treatment, batchsim)
else:
mapkey = "%s;%s" % (product, treatment)
output_dict[product] = output_fdict[mapkey]
#print "-"*80
#print inputmap_dict
#print inputweight_dict
#print output_dict
combine_maps_driver(inputmap_dict, inputweight_dict, output_dict,
fullcov=fullcov, datapath_db=datapath_db)
def __init__(self, parameter_file=None, params_dict=None, feedback=0):
# recordkeeping
self.pairs = {}
self.pairs_parallel_track = {}
self.pairlist = []
self.datapath_db = dp.DataPath()
self.params = params_dict
if parameter_file:
self.params = parse_ini.parse(parameter_file,
params_init,
prefix=prefix)
self.freq_list = sp.array(self.params['freq_list'], dtype=int)
self.tack_on_input = self.params['tack_on_input']
self.output_root = self.datapath_db.fetch(
self.params['output_root'], tack_on=self.params['tack_on_output'])
#self.output_root = self.params['output_root']
print "foreground cleaning writing to output root", self.output_root
if not os.path.isdir(self.output_root):
os.mkdir(self.output_root)
if self.params['svd_filename'] is not None:
self.svd_filename = self.params['svd_filename']
print "WARNING: using %s to clean (intended?)" % self.svd_filename
else:
self.svd_filename = self.output_root + "/" + "SVD.hd5"
# Write parameter file.
parse_ini.write_params(self.params,
self.output_root + 'params.ini',
prefix=prefix)
def find_avg_fsky(map_key, tack_on=None, refinement=2, pad=5, order=1):
"""Take all the pairs that enter the autopower, open their weight files and
find the fsky for each data treatment
In a pipeline
fsky = find_avg_fsky(self.params["map_key"],
tack_on=self.params["tack_on"],
refinement=self.params["refinement"],
pad=self.params["pad"],
order=self.params["order"])
"""
fsky = {}
datapath_db = dp.DataPath()
map_cases = datapath_db.fileset_cases(map_key, "pair;type;treatment")
# This code is essentially verbatim for the permutation in the real
# autopower
unique_pairs = dp.GBTauto_cross_pairs(map_cases['pair'],
map_cases['pair'],
cross_sym="_with_")
treatment_list = map_cases['treatment']
for treatment in treatment_list:
for item in unique_pairs:
dbkeydict = {}
mapset0 = (map_key, item[0], treatment)
mapset1 = (map_key, item[1], treatment)
dbkeydict['noiseinv1_key'] = "%s:%s;noise_inv;%s" % mapset0
dbkeydict['noiseinv2_key'] = "%s:%s;noise_inv;%s" % mapset1
files = dp.convert_dbkeydict_to_filedict(dbkeydict,
datapath_db=datapath_db,
tack_on=tack_on)
print files['noiseinv1_key'], files['noiseinv2_key']
weight1 = algebra.make_vect(algebra.load(files['noiseinv1_key']))
weight2 = algebra.make_vect(algebra.load(files['noiseinv2_key']))
physweight1 = bh.repackage_kiyo(
pg.physical_grid(weight1,
refinement=refinement,
pad=pad,
order=order))
physweight2 = bh.repackage_kiyo(
pg.physical_grid(weight2,
refinement=refinement,
pad=pad,
order=order))
#fsky = np.sum(physweight1 * physweight2)**2
#fsky /= np.sum(physweight1**2 * physweight2**2)
#fsky /= float(physweight1.size)
fsky = np.sum(weight1 * weight2)**2
fsky /= np.sum(weight1**2 * weight2**2)
fsky /= float(weight1.size)
print "volume factor in noise weight: ", fsky
return fsky
def generate_aux_simset(params, silent=False, datapath_db=None):
if datapath_db is None:
datapath_db = data_paths.DataPath()
weightfile = datapath_db.fetch(params['weight_key'],
intend_read=True,
purpose="weight map",
silent=silent)
input_rawsimset = datapath_db.fetch(params['sim_key'],
intend_read=True,
silent=silent)
output_deltasimset = datapath_db.fetch(params['sim_delta_key'],
intend_write=True,
silent=silent)
input_beamsimset = datapath_db.fetch(params['sim_beam_key'],
intend_read=True,
silent=silent)
output_meansubsimset = datapath_db.fetch(params['sim_beam_meansub_key'],
intend_write=True,
silent=silent)
output_convsimset = datapath_db.fetch(params['sim_beam_conv_key'],
intend_write=True,
silent=silent)
output_meansubconvsimset = datapath_db.fetch(
params['sim_beam_meansubconv_key'], intend_write=True, silent=silent)
# TODO: actually implement the foreground simulations
output_fgsimset = datapath_db.fetch(params['sim_beam_plus_fg_key'],
intend_write=True,
silent=silent)
for index in input_rawsimset[0]:
sg.generate_delta_sim(input_rawsimset[1][index],
output_deltasimset[1][index])
sg.generate_proc_sim(input_beamsimset[1][index],
weightfile,
output_meansubsimset[1][index],
meansub=True,
degrade=False)
sg.generate_proc_sim(input_beamsimset[1][index],
weightfile,
output_convsimset[1][index],
meansub=False,
degrade=True)
sg.generate_proc_sim(input_beamsimset[1][index],
weightfile,
output_meansubconvsimset[1][index],
meansub=True,
degrade=True)
def __init__(self, parameter_file=None, params_dict=None, feedback=0):
self.params = params_dict
self.datapath_db = dp.DataPath()
if parameter_file:
self.params = parse_ini.parse(parameter_file,
subtractmap_init,
prefix=subtractmap_prefix)
def __init__(self, parameter_file=None, params_dict=None, feedback=0):
self.params = params_dict
if parameter_file:
self.params = parse_ini.parse(parameter_file, params_init,
prefix=prefix)
self.template_key = self.params['template_key']
self.output_key = self.params['output_key']
self.total_integration = self.params['total_integration']
self.scenario = self.params['scenario']
self.refinement = self.params['refinement']
self.multiplier = self.params['multiplier']
self.tack_on = self.params['tack_on']
# set the random seed
if (self.params['seed'] < 0):
print "no seed given; generating one (are you sure?)"
# The usual seed is not fine enough for parallel jobs
randsource = open("/dev/random", "rb")
self.seed = struct.unpack("I", randsource.read(4))[0]
#self.seed = abs(long(outfile_physical.__hash__()))
else:
self.seed = self.params['seed']
random.seed(self.seed)
self.datapath_db = data_paths.DataPath()
self.input_weight_maps = self.return_maplist(self.template_key,
"noise_weight")
self.output_weight_maps = self.return_maplist(self.output_key,
"noise_weight",
tack_on=self.tack_on)
self.output_maps = self.return_maplist(self.output_key,
"clean_map",
tack_on=self.tack_on)
self.output_delta_thermal = []
self.output_thermal = []
for mapfile in self.output_maps:
basename = os.path.splitext(mapfile)[0]
self.output_delta_thermal.append(basename + "_deltat.npy")
self.output_thermal.append(basename + "_thermal.npy")
self.output_signal = "gaussian_signal_simulation.npy"
print "input weight maps: ", self.input_weight_maps
print "output weight maps: ", self.output_weight_maps
self.output_root = os.path.dirname(self.output_weight_maps[0])
self.output_root += "/"
print "output directory: ", self.output_root
if not os.path.isdir(self.output_root):
os.mkdir(self.output_root)
def batch_physical_sim_run(sim_key,
inifile=None,
datapath_db=None,
output_tag=None,
outdir="./plots/"):
"""Test the power spectral estimator using simulations"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
cache_path = datapath_db.fetch("quadratic_batch_data")
mock_cases = datapath_db.fileset_cases(sim_key, "realization")
funcname = "correlate.batch_quadratic.call_phys_space_run"
generate = False if output_tag else True
if generate:
print "REGENERATING the power spectrum result cache: "
caller = batch_handler.MemoizeBatch(funcname,
cache_path,
generate=generate,
verbose=True)
if output_tag:
output_root = "%s/%s/" % (outdir, output_tag)
file_tools.mkparents(output_root)
pwr_1d = []
pwr_1d_from_2d = []
pwr_2d = []
for index in mock_cases['realization']:
mapfile = datapath_db.fetch("%s:%s" % (sim_key, index))
pwrspec_out = caller.execute(mapfile, mapfile, inifile=inifile)
if output_tag:
pwr_1d_from_2d.append(
pe.convert_2d_to_1d(pwrspec_out[0], transfer=None))
pwr_2d.append(pwrspec_out[0])
pwr_1d.append(pwrspec_out[1])
if output_tag:
pe.summarize_agg_pwrspec(pwr_1d,
pwr_1d_from_2d,
pwr_2d,
output_tag,
outdir=output_root)
retval = (pwr_1d, pwr_1d_from_2d, pwr_2d)
else:
caller.multiprocess_stack()
retval = None
return retval
def wigglez_xspec(inifile):
r"""required parameters:
general:
`inifile`: .ini file for the power spectrum estimator (bins, etc.) fixed
`outdir`: place to dump plot data
for the cross power:
`selection_function`
`crosspwr_tag`: unique tag (on top of the input data name) for this run
for the transfer function:
`cleaned_simkey`
`truesignal_simkey`
`truesignal_weightkey`
`reference_simkey`
`reference_weightkey`
`crosspwr_trans_tag`: unique tag (on top of the input data name) for this run
`generate`: True is regenerate the cache for this
applied as:
cleaned_simkey(map) * cleaned_simkey(weight) x
truesignal_weightkey * truesignal_simkey
divided by:
reference_simkey * reference_weightkey x truesignal_weightkey * truesignal_simkey
"""
datapath_db = data_paths.DataPath()
#inifile=None, generate=False, alttag=None,
# mode_transfer_1d=None, mode_transfer_2d=None,
# beam_transfer=None):
#datapath_db = data_paths.DataPath()
# cleaned_mapkey "GBT_15hr_map_fdgcal_cleaned_noconv_combined"
# tack "_xWigglez" onto the output_tag name and an optional alttag after that
#cleaned_mapkey, wigglez_signalkey, wigglez_mockkey, wigglez_selection
mapname = "GBT_15hr_map_fdgcal_cleaned_noconv_combined"
wigglez_map_key = "WiggleZ_15hr_binned_delta"
wigglez_mock_key = "WiggleZ_15hr_delta_mock"
wigglez_selection_key = "WiggleZ_15hr_montecarlo"
cwx.batch_gbtxwigglez_data_run(mapname,
wigglez_map_key,
wigglez_mock_key,
wigglez_selection_key,
inifile=inifile,
datapath_db=datapath_db,
outdir="./plots/",
output_tag=output_tag,
beam_transfer=beam_transfer,
mode_transfer_1d=mode_transfer_1d,
mode_transfer_2d=mode_transfer_2d,
theory_curve=None)
def __init__(self, parameter_file=None, params_dict=None, feedback=0):
self.params = params_dict
if parameter_file:
self.params = parse_ini.parse(parameter_file,
params_init,
prefix=prefix)
if not os.path.isdir(self.params['output_root']):
os.mkdir(self.params['output_root'])
self.refinement = self.params['refinement']
self.scenario = self.params['scenario']
self.template_file = self.params['template_file']
self.output_root = self.params['output_root']
# here we use 300 h km/s from WiggleZ for streaming dispersion
self.streaming_dispersion = 300. * 0.72
#self.template_map = algebra.make_vect(
# algebra.load(self.template_file))
self.datapath_db = data_paths.DataPath()
self.template_map = self.datapath_db.fetch_multi(self.template_file)
# determine the beam model
self.beam_data = np.array([
0.316148488246, 0.306805630985, 0.293729620792, 0.281176247549,
0.270856788455, 0.26745856078, 0.258910010848, 0.249188429031
])
self.freq_data = np.array([695, 725, 755, 785, 815, 845, 875, 905],
dtype=float)
self.freq_data *= 1.0e6
# set the random seed
if (self.params['seed'] < 0):
# The usual seed is not fine enough for parallel jobs
randsource = open("/dev/random", "rb")
self.seed = struct.unpack("I", randsource.read(4))[0]
#self.seed = abs(long(outfile_physical.__hash__()))
else:
self.seed = self.params['seed']
random.seed(self.seed)
# register any maps that need to be produced
self.sim_map_phys = None
self.sim_map = None
self.sim_map_delta = None
self.sim_map_optsim = None
self.sim_map_withbeam = None
self.sim_map_meansub = None
self.sim_map_degrade = None
def process_mode_files(clean_path, outpath, n_modes=20, average=False):
if not os.path.isdir(outpath):
os.mkdir(outpath)
datapath_db = dp.DataPath()
output_root = datapath_db.fetch(clean_path)
filename = output_root + "/" + "SVD.hd5"
print filename
svd_data = h5py.File(filename, "r")
compilation = {}
for fieldname in svd_data:
acc = []
if average:
for pair in svd_data[fieldname]:
acc.append(svd_data[fieldname][pair].value)
else:
pair0 = svd_data[fieldname].keys()[0]
acc.append(svd_data[fieldname][pair0].value)
compilation[fieldname] = np.mean(np.dstack(acc), axis=-1)
print fieldname, compilation[fieldname].shape
# handle the eigenvalues
if "val" in fieldname:
filename = outpath + "/" + fieldname + ".dat"
print "writing ", filename
fileobj = open(filename, "w")
#data = compilation[fieldname][0]
data = np.abs(np.sort(-compilation[fieldname][0]))
data /= data[0]
for index in range(data.shape[0]):
fileobj.write("%10.15g\n" % data[index])
fileobj.close()
# handle the modes
if "modes" in fieldname:
filename = outpath + "/" + fieldname + ".dat"
print "writing ", filename
fileobj = open(filename, "w")
data = compilation[fieldname][0:n_modes]
for index in range(data.shape[1]):
entry = tuple(data[:, index].tolist())
fileobj.write(("%10.5g " * n_modes + "\n") % entry)
fileobj.close()
svd_data.close()
def __init__(self, parameter_file=None, params_dict=None, feedback=0):
self.params = params_dict
self.datapath_db = dp.DataPath()
if parameter_file:
self.params = parse_ini.parse(parameter_file,
calc_mixing_init,
prefix=calc_mixing_prefix)
bin_spec = self.params["bins"]
self.bins = np.logspace(math.log10(bin_spec[0]),
math.log10(bin_spec[1]),
num=bin_spec[2],
endpoint=True)
def wrap_batch_gbtxwigglez_data_run(inifile,
generate=False,
outdir="./plots/"):
r"""Wrapper to the GBT x WiggleZ calculation"""
params_init = {
"gbt_mapkey": "cleaned GBT map",
"wigglez_deltakey": "WiggleZ overdensity map",
"wigglez_mockkey": "WiggleZ overdensities from mocks",
"wigglez_selectionkey": "WiggleZ selection function",
"mode_transfer_1d_ini": "ini file -> 1d trans. function",
"mode_transfer_2d_ini": "ini file -> 2d trans. function",
"beam_transfer_ini": "ini file -> 2d beam trans. function",
"spec_ini": "ini file for the spectral estimation",
"output_tag": "tag identifying the output somehow"
}
prefix = "cwx_"
params = parse_ini.parse(inifile, params_init, prefix=prefix)
print params
output_tag = "%s_%s" % (params['gbt_mapkey'], params['output_tag'])
output_root = "%s/%s/" % (outdir, output_tag)
if generate:
output_tag = None
print output_root
print output_tag
file_tools.mkparents(output_root)
parse_ini.write_params(params, output_root + 'params.ini', prefix=prefix)
datapath_db = data_paths.DataPath()
mode_transfer_1d = None
if params["mode_transfer_1d_ini"]:
mode_transfer_1d = cct.wrap_batch_crosspwr_transfer(
params["mode_transfer_1d_ini"], generate=generate, outdir=outdir)
batch_gbtxwigglez_data_run(params["gbt_mapkey"],
params["wigglez_deltakey"],
params["wigglez_mockkey"],
params["wigglez_selectionkey"],
inifile=params["spec_ini"],
datapath_db=datapath_db,
outdir=output_root,
output_tag=output_tag,
beam_transfer=None,
mode_transfer_1d=mode_transfer_1d,
mode_transfer_2d=None,
theory_curve=None)
def plot_mode_amplitudes(mapkey,
outputdir="/cita/d/www/home/eswitzer/movies/"):
datapath_db = data_paths.DataPath()
map_cases = datapath_db.fileset_cases(mapkey, "pair;product;treatment")
print map_cases
for pair in map_cases['pair']:
source_key = "db:%s:%s;modes;100modes" % (mapkey, pair)
pc.make_cube_movie(source_key,
"Mode amp",
pc.cube_frame_dir,
sigmarange=-1,
outputdir=outputdir,
multiplier=1.,
transverse=False,
convolve=False,
logscale=True)
def plot_first_few_amps(dbitem, modelist):
datapath_db = data_paths.DataPath()
frame_dir = "/scratch/eswitzer/cube_frames/"
dbname = "db:%s:A_with_B;modes;100modes" % dbitem
for modeindex in modelist:
filename = "%s_%d.eps" % (dbitem, modeindex)
plot_cube.make_cube_movie(dbname,
"Amplitude",
frame_dir,
saveslice=modeindex,
saveslice_file=filename,
sigmarange=-1,
multiplier=1.,
title="%s" % filename)
def gather_batch_data_run():
datapath_db = data_paths.DataPath()
outpath = datapath_db.fetch("quadratic_batch_data")
print "reading from to: " + outpath
funcname = "correlate.batch_quadratic.call_xspec_run"
caller = batch_handler.MemoizeBatch(funcname, outpath, verbose=True)
for mode_num in range(0, 55, 5):
map1_key = "GBT_15hr_map_cleaned_%dmode" % mode_num
map2_key = "GBT_15hr_map_cleaned_%dmode" % mode_num
noise1_key = "GBT_15hr_map_cleaned_%dmode" % mode_num
noise2_key = "GBT_15hr_map_cleaned_%dmode" % mode_num
(pairlist, pairdict) = \
data_paths.cross_maps(map1_key, map2_key,
noise1_key, noise2_key,
map_suffix=";map",
noise_inv_suffix=";noise_inv",
cross_sym="_x_",
pair_former="GBTauto_cross_pairs",
ignore=['param'],
tag1prefix=map1_key + "_",
tag2prefix=map2_key + "_",
verbose=False)
pwr_1d = []
pwr_2d = []
for item in pairdict.keys():
pairrun = pairdict[item]
print pairrun['tag1']
print pairrun['map1']
print pairrun['noise_inv1']
print pairdict[item].keys()
pwr2d_run, pwr1d_run = caller.execute(pairrun['map1'],
pairrun['map2'],
pairrun['noise_inv1'],
pairrun['noise_inv2'])
pwr_2d.append(pwr2d_run)
pwr_1d.append(pwr1d_run)
pe.summarize_1d_agg_pwrspec(pwr_1d,
"pwr_data_%dmodes_1d.dat" % mode_num)
pe.summarize_2d_agg_pwrspec(pwr_2d,
"pwr_data_%dmodes_2d.dat" % mode_num)
def plot_gbt_comb_modeset(map_key,
outputdir="/cita/d/www/home/eswitzer/movies/",
convolve=False,
divider_token=";"):
datapath_db = data_paths.DataPath()
input_fdb = datapath_db.fetch(map_key, intend_read=True, silent=True)
map_treatments = []
for filekey in input_fdb[0]:
keyparse = filekey.split(divider_token)
if len(keyparse) == 2:
treatment = keyparse[-1]
if "modes" in treatment:
map_treatments.append(treatment)
map_treatments = unique_list(map_treatments)
for treatment in map_treatments:
source_key = "db:%s:map;%s" % (map_key, treatment)
print source_key, treatment
pc.make_cube_movie(source_key,
"Temperature (mK)",
pc.cube_frame_dir,
sigmarange=3.,
outputdir=outputdir,
multiplier=1000.,
transverse=False,
convolve=convolve)
source_key = "db:%s:product;%s" % (map_key, treatment)
pc.make_cube_movie(source_key,
"Cleaned map times weights",
pc.cube_frame_dir,
sigmarange=-1,
outputdir=outputdir,
multiplier=1000.,
transverse=False,
convolve=convolve)
source_key = "db:%s:weight;%s" % (map_key, treatment)
pc.make_cube_movie(source_key,
"inverse variance weight",
pc.cube_frame_dir,
sigmarange=-1,
outputdir=outputdir,
multiplier=1.,
transverse=False)
def generate_sim(params, parallel=True, silent=True, datapath_db=None):
"""generate simulations
here, assuming the sec A of the set is the template map
"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
template_mapname = datapath_db.fetch(params['template_key'],
intend_read=True,
purpose="template_map",
silent=silent)
physlist = datapath_db.fetch(params['sim_physical_key'],
intend_write=True,
purpose="output sim (physical)",
silent=silent)
rawlist = datapath_db.fetch(params['sim_key'],
intend_write=True,
purpose="output sim",
silent=silent)
beamlist = datapath_db.fetch(params['sim_beam_key'],
intend_write=True,
purpose="output sim+beam",
silent=silent)
bpdlist = datapath_db.fetch(params['sim_beam_plus_data_key'],
intend_write=True,
purpose="output sim+beam+data",
silent=silent)
runlist = [(template_mapname, physlist[1][index], rawlist[1][index],
beamlist[1][index], bpdlist[1][index],
params['pwrspec_scenario'], params['refinement'])
for index in rawlist[0]]
print runlist
#sys.exit()
if parallel:
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count() - 4)
pool.map(wrap_sim, runlist)
else:
for runitem in runlist:
wrap_sim(runitem)
def wrap_batch_gbtpwrspec_data_run(inifile, generate=False, outdir="./plots/"):
r"""Wrapper to the GBT x GBT calculation"""
params_init = {
"gbt_mapkey": "cleaned GBT map",
"mode_transfer_1d_ini": "ini file -> 1d trans. function",
"mode_transfer_2d_ini": "ini file -> 2d trans. function",
"beam_transfer_ini": "ini file -> 2d beam trans. function",
"square_1dmodetrans": False,
"spec_ini": "ini file for the spectral estimation",
"output_tag": "tag identifying the output somehow"
}
prefix = "cp_"
params = parse_ini.parse(inifile, params_init, prefix=prefix)
print params
output_tag = "%s_%s" % (params['gbt_mapkey'], params['output_tag'])
output_root = "%s/%s/" % (outdir, output_tag)
if generate:
output_tag = None
print output_root
print output_tag
file_tools.mkparents(output_root)
parse_ini.write_params(params, output_root + 'params.ini', prefix=prefix)
datapath_db = data_paths.DataPath()
mode_transfer_1d = None
if params["mode_transfer_1d_ini"]:
mode_transfer_1d = cct.wrap_batch_crosspwr_transfer(
params["mode_transfer_1d_ini"], generate=generate, outdir=outdir)
return batch_gbtpwrspec_data_run(
params["gbt_mapkey"],
inifile=params["spec_ini"],
datapath_db=datapath_db,
outdir=output_root,
output_tag=output_tag,
beam_transfer=None,
square_1dmodetrans=params["square_1dmodetrans"],
mode_transfer_1d=mode_transfer_1d,
mode_transfer_2d=None)
def add_sim_to_data(simkey, datakey, replace=False):
datapath_db = data_paths.DataPath()
mapA_file = datapath_db.fetch(datakey + ":A;clean_map", intend_read=True)
mapB_file = datapath_db.fetch(datakey + ":B;clean_map", intend_read=True)
mapC_file = datapath_db.fetch(datakey + ":C;clean_map", intend_read=True)
mapD_file = datapath_db.fetch(datakey + ":D;clean_map", intend_read=True)
simfile = datapath_db.fetch(simkey + ":1", intend_read=True)
simmap = algebra.make_vect(algebra.load(simfile))
mapset = [mapA_file, mapB_file, mapC_file, mapD_file]
for mapfile in mapset:
print mapfile, simfile
origmap = algebra.make_vect(algebra.load(mapfile))
if replace:
algebra.save(mapfile, simmap)
else:
algebra.save(mapfile, origmap + simmap)
def plot_cleaned_maps(source_key, alt_weight=None,
signal='map', weight='noise_inv', divider_token=";",
outputdir="/cita/d/www/home/eswitzer/movies/",
convolve=False,
transverse=False):
r"""
`source_key` is the file db key for the maps to combine
`signal` is the tag in the file db entry for the signal maps
`weight` is the tag in the file db entry for the N^-1 weights
`divider_token` is the token that divides the map section name
from the data type e.g. "A_with_B;noise_inv"
"""
datapath_db = data_paths.DataPath()
source_fdb = datapath_db.fetch(source_key, intend_read=True,
silent=True)
source_fdict = source_fdb[1]
# accumulate all the files to combine
weightkeys = {}
signalkeys = {}
for filekey in source_fdb[0]:
if divider_token in filekey:
data_type = filekey.split(divider_token)[1]
map_section = filekey.split(divider_token)[0]
if data_type == signal:
signalkeys[map_section] = source_fdict[filekey]
if data_type == weight:
weightkeys[map_section] = source_fdict[filekey]
for mapkey in signalkeys:
signalfile = signalkeys[mapkey]
weightfile = weightkeys[mapkey]
print "loading pair: %s %s" % (signalfile, weightfile)
make_cube_movie(signalfile, "Temperature (mK)", cube_frame_dir,
sigmarange=2.5, outputdir=outputdir, multiplier=1000.,
transverse=transverse, convolve=convolve)
make_cube_movie(weightfile, "Inverse variance weight", cube_frame_dir,
sigmarange=2.5, outputdir=outputdir, multiplier=1.,
transverse=transverse)
def batch_crosspwr_transfer_run(cleaned_simkey,
rootsim,
selection_function,
simindex="1",
weightmap="15modes",
include_beam=True,
inifile=None,
generate=False,
alttag=None):
r"""This provides some basic uniformity in how the WiggleZ transfer
functions are derived. TODO: This can probably be phased out.
"""
datapath_db = data_paths.DataPath()
truesignal_simkey = "%s_delta:%s" % (rootsim, simindex)
truesignal_weightkey = selection_function
if include_beam:
reference_simkey = "%s_temperature:%s" % (rootsim, simindex)
else:
reference_simkey = "%s_beam:%s" % (rootsim, simindex)
output_tag = cleaned_simkey
if alttag:
output_tag += "_" + alttag
reference_weightkey = "%s:weight;%s" % (cleaned_simkey, weightmap)
if generate:
output_tag = None
outdir = "./plots/" + output_tag
return batch_crosspwr_transfer(cleaned_simkey,
truesignal_simkey,
truesignal_weightkey,
reference_simkey,
reference_weightkey,
inifile=inifile,
datapath_db=datapath_db,
outdir="./plots/",
output_tag=output_tag)
def plot_gbt_simset(fieldname, outputdir="/cita/d/www/home/eswitzer/movies/"):
datapath_db = data_paths.DataPath()
#keyname = "%s" % fieldname
#filename = datapath_db.fetch(keyname, pick='0')
#pc.make_cube_movie(filename, "Temperature (mK)", pc.cube_frame_dir,
# sigmarange=3., outputdir=outputdir, multiplier=1000.,
# transverse=False, filetag_suffix="_"+fieldname)
#keyname = "%s_beam" % fieldname
#filename = datapath_db.fetch(keyname, pick='0')
#pc.make_cube_movie(filename, "Temperature (mK)", pc.cube_frame_dir,
# sigmarange=3., outputdir=outputdir, multiplier=1000.,
# transverse=False, filetag_suffix="_"+fieldname)
#keyname = "%s_beam_conv" % fieldname
#filename = datapath_db.fetch(keyname, pick='0')
#pc.make_cube_movie(filename, "Temperature (mK)", pc.cube_frame_dir,
# sigmarange=3., outputdir=outputdir, multiplier=1000.,
# transverse=False, filetag_suffix="_"+fieldname)
keyname = "%s_beam_meansub" % fieldname
filename = datapath_db.fetch(keyname, pick='0')
pc.make_cube_movie(filename,
"Temperature (mK)",
pc.cube_frame_dir,
sigmarange=3.,
outputdir=outputdir,
multiplier=1000.,
transverse=False,
filetag_suffix="_" + fieldname)
keyname = "%s_beam_meansubconv" % fieldname
filename = datapath_db.fetch(keyname, pick='0')
pc.make_cube_movie(filename,
"Temperature (mK)",
pc.cube_frame_dir,
sigmarange=3.,
outputdir=outputdir,
multiplier=1000.,
transverse=False,
filetag_suffix="_" + fieldname)
def generate_aux_simset(fieldname, silent=False):
datapath_db = data_paths.DataPath()
weightfile = datapath_db.fetch("GBT_15hr_map_combined_cleaned_0mode_weight",
intend_read=True, silent=silent)
input_rawsimset = datapath_db.fetch("%s" % fieldname, intend_read=True,
silent=silent)
output_deltasimset = datapath_db.fetch("%s_delta" % fieldname,
intend_write=True, silent=silent)
input_beamsimset = datapath_db.fetch("%s_beam" % fieldname,
intend_read=True, silent=silent)
output_meansubsimset = datapath_db.fetch("%s_beam_meansub" % fieldname,
intend_write=True, silent=silent)
output_convsimset = datapath_db.fetch("%s_beam_conv" % fieldname,
intend_write=True, silent=silent)
output_meansubconvsimset = datapath_db.fetch("%s_beam_meansubconv" % fieldname,
intend_write=True, silent=silent)
for index in input_rawsimset[0]:
generate_delta_sim(input_rawsimset[1][index],
output_deltasimset[1][index])
generate_proc_sim(input_beamsimset[1][index], weightfile,
output_meansubsimset[1][index],
meansub=True, degrade=False)
generate_proc_sim(input_beamsimset[1][index], weightfile,
output_convsimset[1][index],
meansub=False, degrade=True)
generate_proc_sim(input_beamsimset[1][index], weightfile,
output_meansubconvsimset[1][index],
meansub=True, degrade=True)
def wrap_batch_crosspwr_transfer(inifile, generate=False, outdir="./plots/"):
r"""Wrapper to the transfer function calculator
"""
params_init = {
"cleaned_simkey": "cleaned sims for transfer func",
"truesignal_simkey": "pure signal",
"truesignal_weightkey": "weight to use for pure signal",
"reference_simkey": "reference signal",
"reference_weightkey": "weight to use for reference signal",
"spec_ini": "ini file for the spectral estimation",
"output_tag": "tag identifying the output somehow"
}
prefix = "cct_"
params = parse_ini.parse(inifile, params_init, prefix=prefix)
print params
output_tag = "%s_%s" % (params['cleaned_simkey'], params['output_tag'])
output_root = "%s/%s/" % (outdir, output_tag)
if generate:
output_tag = None
print output_root
print output_tag
file_tools.mkparents(output_root)
parse_ini.write_params(params, output_root + 'params.ini', prefix=prefix)
datapath_db = data_paths.DataPath()
return batch_crosspwr_transfer(params["cleaned_simkey"],
params["truesignal_simkey"],
params["truesignal_weightkey"],
params["reference_simkey"],
params["reference_weightkey"],
inifile=params["spec_ini"],
datapath_db=datapath_db,
outdir=output_root,
output_tag=output_tag)
def wrap_batch_sim_run(inifile, generate=False, outdir="./plots/"):
r"""Wrapper to the sim processing
TODO: add transfer function to test its application
"""
params_init = {"left_simkey": "sim to put on the left side",
"right_simkey": "sim to put on the right side",
"left_weightkey": "weight to use on the left side",
"right_weightkey": "weight to use on the right side",
"spec_ini": "ini file for the spectral estimation",
"output_tag": "tag identifying the output somehow"}
prefix="csp_"
params = parse_ini.parse(inifile, params_init, prefix=prefix)
print params
output_tag = "%sx%s_%s" % (params['left_simkey'], \
params['right_simkey'], \
params['output_tag'])
output_root = "%s/%s/" % (outdir, output_tag)
if generate:
output_tag = None
print output_root
print output_tag
file_tools.mkparents(output_root)
parse_ini.write_params(params, output_root + 'params.ini',
prefix=prefix)
datapath_db = data_paths.DataPath()
return batch_sim_run(params["left_simkey"],
params["right_simkey"],
params["left_weightkey"],
params["right_weightkey"],
inifile=params["spec_ini"],
datapath_db=datapath_db,
outdir=output_root,
output_tag=output_tag)
