def add_manual_mask(source_key, cut_freq_list=None,
signal_name='map', noise_inv_name='noise_inv',
weight_name='weight', divider_token=";"):
r"""
`source_key` is the file db key for the maps to combine
`signal_name` is the tag in the file db entry for the signal maps
`noise_inv_name` is the tag in the file db entry for the N^-1 weights
`weight_name` is the tag in the file db entry for the weights to write out
`divider_token` is the token that divides the map section name
from the data type e.g. "A_with_B;noise_inv"
"""
datapath_db = data_paths.DataPath()
source_fdb = datapath_db.fetch(source_key, silent=True)
source_fdict = source_fdb[1]
# accumulate all the files to combine
noise_inv_keys = {}
weight_keys = {}
signal_keys = {}
for filekey in source_fdb[0]:
if divider_token in filekey:
data_type = filekey.split(divider_token)[1]
map_section = filekey.split(divider_token)[0]
if data_type == signal_name:
signal_keys[map_section] = source_fdict[filekey]
if data_type == noise_inv_name:
noise_inv_keys[map_section] = source_fdict[filekey]
if data_type == weight_name:
weight_keys[map_section] = source_fdict[filekey]
for mapkey in signal_keys:
signal_file = signal_keys[mapkey]
noise_inv_file = noise_inv_keys[mapkey]
weight_file = weight_keys[mapkey]
print "loading pair: %s %s -> %s" % \
(signal_file, noise_inv_file, weight_file)
signal_map = algebra.make_vect(algebra.load(signal_file))
weightmap = algebra.make_vect(algebra.load(noise_inv_file))
# set the new weights to zero where the N^-1 is small
# or the signal map is inf or nan
weightmap[np.isnan(weightmap)] = 0.
weightmap[np.isinf(weightmap)] = 0.
weightmap[np.isnan(signal_map)] = 0.
weightmap[np.isinf(signal_map)] = 0.
weightmap[weightmap < 1.e-20] = 0.
if cut_freq_list is not None:
for cutindex in cut_freq_list:
weightmap[cutindex, :, :] = 0.
# could also determine the filename here, outside of the database
#outputdir = datapath_db.fetch_parent(source_key, return_path=True)
#weight_out = "%s/%s" % (outputdir, source_key)
algebra.compressed_array_summary(weightmap, "new weight map")
algebra.save(weight_file, weightmap)
def make_combined(self, map_list, targetroot, file):
imap_temp = algebra.make_vect(algebra.load(map_list[0]))
cumulative_product = algebra.zeros_like(imap_temp)
cumulative_weight = algebra.zeros_like(imap_temp)
for dir in map_list:
imap = algebra.make_vect(algebra.load(dir))
wmap = algebra.make_vect(algebra.load(dir.replace('clean_map', 'noise_inv')))
cumulative_product += imap*wmap
cumulative_weight += wmap
algebra.compressed_array_summary(cumulative_weight, "weight map")
algebra.compressed_array_summary(cumulative_product, "product map")
cumulative_weight[cumulative_weight < 1.e-20] = 0.
cumulative_product[cumulative_weight < 1.e-20] = 0.
cumulative_weight[cumulative_weight == 0] = np.inf
cumulative_weight[np.isnan(cumulative_weight)] = np.inf
newmap = cumulative_product / cumulative_weight
cumulative_weight[np.isinf(cumulative_weight)] = 0.
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.
cumulative_product[nan_array] = 0.
cumulative_weight[nan_array] = 0.
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.
cumulative_product[inf_array] = 0.
cumulative_weight[inf_array] = 0.
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(cumulative_product,"final map * weight")
algebra.compressed_array_summary(cumulative_weight, "final weight map")
combined_map_file = targetroot + file
combined_weight_file = targetroot + file.replace('map', 'weight')
print combined_map_file
print combined_weight_file
algebra.save(combined_map_file, newmap)
algebra.save(combined_weight_file, cumulative_weight)
def save_data(self, n_modes):
prodmap_list = []
weight_list = []
n_modes = "%dmodes" % n_modes
for pairitem in self.pairlist:
pair = self.pairs[pairitem]
(tag1, tag2) = (pair.map1_name, pair.map2_name)
clnoise = "cleaned_noise_inv"
map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag1, clnoise, tag2, n_modes)
noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag2, clnoise, tag1, n_modes)
modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
if self.params['subtract_inputmap_from_sim'] or \
self.params['subtract_sim_from_inputmap']:
map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1
map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2
else:
map1 = copy.deepcopy(pair.map1)
map2 = copy.deepcopy(pair.map2)
prodmap_list.append(map1 * pair.noise_inv1)
prodmap_list.append(map2 * pair.noise_inv2)
weight_list.append(pair.noise_inv1)
weight_list.append(pair.noise_inv2)
algebra.save(map1_file, map1)
algebra.save(map2_file, map2)
algebra.save(noise_inv1_file, pair.noise_inv1)
algebra.save(noise_inv2_file, pair.noise_inv2)
algebra.save(modes1_file, pair.left_modes)
algebra.save(modes2_file, pair.right_modes)
cumulative_product = algebra.zeros_like(prodmap_list[0])
cumulative_weight = algebra.zeros_like(prodmap_list[0])
for mapind in range(0, len(prodmap_list)):
cumulative_product += prodmap_list[mapind]
cumulative_weight += weight_list[mapind]
algebra.compressed_array_summary(cumulative_weight, "weight map")
algebra.compressed_array_summary(cumulative_product, "product map")
cumulative_weight[cumulative_weight < 1.e-20] = 0.
cumulative_product[cumulative_weight < 1.e-20] = 0.
newmap = cumulative_product / cumulative_weight
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.
cumulative_product[nan_array] = 0.
cumulative_weight[nan_array] = 0.
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.
cumulative_product[inf_array] = 0.
cumulative_weight[inf_array] = 0.
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(cumulative_product,
"final map * weight")
algebra.compressed_array_summary(cumulative_weight, "final weight map")
combined = "combined_clean"
combined_map_file = "%s/%s_map_%s.npy" % \
(self.output_root, combined, n_modes)
combined_weight_file = "%s/%s_weight_%s.npy" % \
(self.output_root, combined, n_modes)
combined_product_file = "%s/%s_product_%s.npy" % \
(self.output_root, combined, n_modes)
combined_ones_file = "%s/%s_ones_%s.npy" % \
(self.output_root, combined, n_modes)
algebra.save(combined_map_file, newmap)
algebra.save(combined_product_file, cumulative_product)
algebra.save(combined_weight_file, cumulative_weight)
algebra.save(combined_ones_file, algebra.ones_like(newmap))
def add_manual_mask(source_key,
cut_freq_list=None,
signal_name='map',
noise_inv_name='noise_inv',
weight_name='weight',
divider_token=";"):
r"""
`source_key` is the file db key for the maps to combine
`signal_name` is the tag in the file db entry for the signal maps
`noise_inv_name` is the tag in the file db entry for the N^-1 weights
`weight_name` is the tag in the file db entry for the weights to write out
`divider_token` is the token that divides the map section name
from the data type e.g. "A_with_B;noise_inv"
"""
datapath_db = data_paths.DataPath()
source_fdb = datapath_db.fetch(source_key, silent=True)
source_fdict = source_fdb[1]
# accumulate all the files to combine
noise_inv_keys = {}
weight_keys = {}
signal_keys = {}
for filekey in source_fdb[0]:
if divider_token in filekey:
data_type = filekey.split(divider_token)[1]
map_section = filekey.split(divider_token)[0]
if data_type == signal_name:
signal_keys[map_section] = source_fdict[filekey]
if data_type == noise_inv_name:
noise_inv_keys[map_section] = source_fdict[filekey]
if data_type == weight_name:
weight_keys[map_section] = source_fdict[filekey]
for mapkey in signal_keys:
signal_file = signal_keys[mapkey]
noise_inv_file = noise_inv_keys[mapkey]
weight_file = weight_keys[mapkey]
print "loading pair: %s %s -> %s" % \
(signal_file, noise_inv_file, weight_file)
signal_map = algebra.make_vect(algebra.load(signal_file))
weightmap = algebra.make_vect(algebra.load(noise_inv_file))
# set the new weights to zero where the N^-1 is small
# or the signal map is inf or nan
weightmap[np.isnan(weightmap)] = 0.
weightmap[np.isinf(weightmap)] = 0.
weightmap[np.isnan(signal_map)] = 0.
weightmap[np.isinf(signal_map)] = 0.
weightmap[weightmap < 1.e-20] = 0.
if cut_freq_list is not None:
for cutindex in cut_freq_list:
weightmap[cutindex, :, :] = 0.
# could also determine the filename here, outside of the database
#outputdir = datapath_db.fetch_parent(source_key, return_path=True)
#weight_out = "%s/%s" % (outputdir, source_key)
algebra.compressed_array_summary(weightmap, "new weight map")
algebra.save(weight_file, weightmap)
def wigglez_correlation(init_filename):
"""Perform the cross-correlation between the WiggleZ dataset and GBT
"""
np.set_printoptions(threshold=np.nan)
print "starting wigglez correlation run with file: " + init_filename
params = parse_ini.parse(init_filename, params_default,
prefix=prefix, feedback=10)
radio_file = params['radio_root'] + params['radio_data_file']
noiseinv_file = params['radio_root'] + params['radio_noiseinv_file']
optical_file = params['optical_root'] + params['optical_data_file']
optical_selection_file = params['optical_root'] + \
params['optical_selection_file']
map_radio = algebra.make_vect(algebra.load(radio_file))
#noiseinv_radio = np.abs(algebra.make_vect(algebra.load(noiseinv_file)))
noiseinv_radio = algebra.make_vect(algebra.load(noiseinv_file))
map_opt = algebra.make_vect(algebra.load(optical_file))
map_nbar = algebra.make_vect(algebra.load(optical_selection_file))
algebra.compressed_array_summary(map_opt, "opt map as loaded")
algebra.compressed_array_summary(map_nbar, "nbar map as loaded")
algebra.compressed_array_summary(map_radio, "radio map as loaded")
algebra.compressed_array_summary(noiseinv_radio, "radio N^-1 as loaded")
noiseinv_radio[noiseinv_radio < 1.e-20] = 0.
nan_array = np.isnan(noiseinv_radio)
noiseinv_radio[nan_array] = 0.
inf_array = np.isinf(noiseinv_radio)
noiseinv_radio[inf_array] = 0.
if params['convolve']:
beam_data = sp.array([0.316148488246, 0.306805630985, 0.293729620792,
0.281176247549, 0.270856788455, 0.26745856078,
0.258910010848, 0.249188429031])
# also consider the beam model where everything is degraded to the
# lowest resolution; e.g. in a MapPair
beam_data_degrade = sp.zeros(8) + max(beam_data)
freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905],
dtype=float)
freq_data *= 1.e6
psf = beam.GaussianBeam(beam_data_degrade, freq_data)
# Convolve the optical data to the lowest radio resolution
print "convolving the map"
map_opt = psf.apply(map_opt)
algebra.compressed_array_summary(map_opt, "opt after convolution")
# how should the covariance be convolved?
# nbar is N^-1; convolve B N B^T?,
# or just convolve nbar? use?:
#map_nbar[map_nbar<1.e-30] = 1.e-30
#map_nbar = 1./map_nbar
#map_nbar = psf.apply(map_nbar, cval=1.e30)
#map_nbar = 1./map_nbar
#map_nbar[map_nbar<1.e-20] = 0
print "convolving the covariance/selection"
map_nbar = psf.apply(map_nbar)
algebra.compressed_array_summary(map_nbar, "nbar after convolution")
# convert to delta-overdensity
map_opt = map_opt / map_nbar - 1.
#algebra.compressed_array_summary(map_opt, "opt after conversion to delta")
# set the NaNs and infs to zero in data and weights
nan_array = np.isnan(map_opt)
map_opt[nan_array] = 0.
map_nbar[nan_array] = 0.
inf_array = np.isinf(map_opt)
map_opt[inf_array] = 0.
map_nbar[inf_array] = 0.
freqlist = params['freq'] # full: range(map_radio.shape[0])
algebra.compressed_array_summary(map_opt[freqlist, :, :],
"opt map as entering the correlation function")
algebra.compressed_array_summary(map_nbar[freqlist, :, :],
"nbar as entering the correlation function")
algebra.compressed_array_summary(map_radio[freqlist, :, :],
"radio map as entering the correlation function")
algebra.compressed_array_summary(noiseinv_radio[freqlist, :, :],
"radio map N^-1 as entering the correlation function")
cross_pair = fs.MapPair(map_opt, map_radio, map_nbar, noiseinv_radio,
freqlist)
if params['subtract_mean']:
cross_pair.subtract_weighted_mean()
(corr, counts) = cross_pair.correlate(params['lags'],
speedup=params['speedup'])
corr_shelve = shelve.open(params['output_shelve_file'])
corr_shelve["corr"] = corr
corr_shelve["counts"] = counts
corr_shelve["freq_axis"] = map_radio.get_axis('freq')
corr_shelve["params"] = params
corr_shelve.close()
def radio_correlation(init_filename):
"""Perform the cross-correlation between two mock GBT cubes
"""
np.set_printoptions(threshold=np.nan)
print "starting wigglez correlation run with file: " + init_filename
params = parse_ini.parse(init_filename, params_default, prefix=prefix, feedback=10)
radio_file1 = params["radio_root1"] + params["radio_data_file1"]
noiseinv_file1 = params["radio_noiseroot1"] + params["radio_noiseinv_file1"]
radio_file2 = params["radio_root2"] + params["radio_data_file2"]
noiseinv_file2 = params["radio_noiseroot2"] + params["radio_noiseinv_file2"]
map_radio1 = algebra.make_vect(algebra.load(radio_file1))
noiseinv_radio1 = algebra.make_vect(algebra.load(noiseinv_file1))
map_radio2 = algebra.make_vect(algebra.load(radio_file2))
noiseinv_radio2 = algebra.make_vect(algebra.load(noiseinv_file2))
algebra.compressed_array_summary(map_radio1, "radio 1 map as loaded")
algebra.compressed_array_summary(noiseinv_radio1, "radio 1 N^-1 as loaded")
algebra.compressed_array_summary(map_radio2, "radio 2 map as loaded")
algebra.compressed_array_summary(noiseinv_radio2, "radio 2 N^-1 as loaded")
noiseinv_radio1[noiseinv_radio1 < 1.0e-20] = 0.0
nan_array = np.isnan(noiseinv_radio1)
noiseinv_radio1[nan_array] = 0.0
inf_array = np.isinf(noiseinv_radio1)
noiseinv_radio1[inf_array] = 0.0
noiseinv_radio2[noiseinv_radio2 < 1.0e-20] = 0.0
nan_array = np.isnan(noiseinv_radio2)
noiseinv_radio2[nan_array] = 0.0
inf_array = np.isinf(noiseinv_radio2)
noiseinv_radio2[inf_array] = 0.0
if params["convolve"]:
print "ERROR: simulations should be convolved by the beam already!"
freqlist = params["freq"] # full: range(map_radio.shape[0])
algebra.compressed_array_summary(map_radio1[freqlist, :, :], "radio map 1 as entering the correlation function")
algebra.compressed_array_summary(
noiseinv_radio1[freqlist, :, :], "radio map 1 N^-1 as entering the correlation function"
)
algebra.compressed_array_summary(map_radio2[freqlist, :, :], "radio map 2 as entering the correlation function")
algebra.compressed_array_summary(
noiseinv_radio2[freqlist, :, :], "radio map 2 N^-1 as entering the correlation function"
)
cross_pair = fs.MapPair(map_radio1, map_radio2, noiseinv_radio1, noiseinv_radio2, freqlist)
if params["subtract_mean"]:
cross_pair.subtract_weighted_mean()
(corr, counts) = cross_pair.correlate(params["lags"], speedup=params["speedup"])
corr_shelve = shelve.open(params["output_shelve_file"])
corr_shelve["corr"] = corr
corr_shelve["counts"] = counts
corr_shelve["freq_axis"] = map_radio1.get_axis("freq")
corr_shelve["params"] = params
corr_shelve.close()
def combine_maps_driver(inputmap_dict, inputweight_dict, output_dict,
fullcov=False, datapath_db=None):
r"""Combine a list of weights, maps specified by their database keys
"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
signal_list = []
weight_list = []
for mapkey in inputmap_dict:
signalfile = inputmap_dict[mapkey]
weightfile = inputweight_dict[mapkey]
print "loading pair: %s %s" % (signalfile, weightfile)
signal_list.append(algebra.make_vect(algebra.load(signalfile)))
if fullcov:
raw_weight = algebra.make_mat(
algebra.open_memmap(weightfile))
raw_weight = raw_weight.mat_diag()
else:
raw_weight = algebra.make_vect(algebra.load(weightfile))
# zero out any messy stuff
raw_weight[raw_weight < 1.e-20] = 0.
raw_weight[np.isnan(raw_weight)] = 0.
raw_weight[np.isinf(raw_weight)] = 0.
weight_list.append(raw_weight)
prodmap = []
for mapind in range(0, len(signal_list)):
prodmap.append(signal_list[mapind] * weight_list[mapind])
print "CHECK THESE: %d %d %d" % (len(signal_list), len(weight_list),
len(prodmap))
cumulative_product = algebra.zeros_like(prodmap[0])
cumulative_weight = algebra.zeros_like(prodmap[0])
for mapind in range(0, len(signal_list)):
cumulative_product += prodmap[mapind]
cumulative_weight += weight_list[mapind]
algebra.compressed_array_summary(cumulative_weight, "weight map")
algebra.compressed_array_summary(cumulative_product, "product map")
newmap = cumulative_product / cumulative_weight
cumulative_weight[cumulative_weight < 1.e-20] = 0.
cumulative_product[cumulative_weight < 1.e-20] = 0.
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.
cumulative_product[nan_array] = 0.
cumulative_weight[nan_array] = 0.
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.
cumulative_product[inf_array] = 0.
cumulative_weight[inf_array] = 0.
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(cumulative_product, "final map * weight")
algebra.compressed_array_summary(cumulative_weight, "final weight map")
print output_dict
algebra.save(output_dict['map'], newmap)
algebra.save(output_dict['product'], cumulative_product)
algebra.save(output_dict['weight'], cumulative_weight)
algebra.save(output_dict['ones'], algebra.ones_like(newmap))
def save_data(self, n_modes):
prodmap_list = []
weight_list = []
n_modes = "%dmodes" % n_modes
for pairitem in self.pairlist:
pair = self.pairs[pairitem]
(tag1, tag2) = (pair.map1_name, pair.map2_name)
clnoise = "cleaned_noise_inv"
map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag1, clnoise, tag2, n_modes)
noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag2, clnoise, tag1, n_modes)
modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
if self.params['subtract_inputmap_from_sim'] or \
self.params['subtract_sim_from_inputmap']:
map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1
map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2
else:
map1 = copy.deepcopy(pair.map1)
map2 = copy.deepcopy(pair.map2)
prodmap_list.append(map1 * pair.noise_inv1)
prodmap_list.append(map2 * pair.noise_inv2)
weight_list.append(pair.noise_inv1)
weight_list.append(pair.noise_inv2)
algebra.save(map1_file, map1)
algebra.save(map2_file, map2)
algebra.save(noise_inv1_file, pair.noise_inv1)
algebra.save(noise_inv2_file, pair.noise_inv2)
algebra.save(modes1_file, pair.left_modes)
algebra.save(modes2_file, pair.right_modes)
cumulative_product = algebra.zeros_like(prodmap_list[0])
cumulative_weight = algebra.zeros_like(prodmap_list[0])
for mapind in range(0, len(prodmap_list)):
cumulative_product += prodmap_list[mapind]
cumulative_weight += weight_list[mapind]
algebra.compressed_array_summary(cumulative_weight, "weight map")
algebra.compressed_array_summary(cumulative_product, "product map")
cumulative_weight[cumulative_weight < 1.e-20] = 0.
cumulative_product[cumulative_weight < 1.e-20] = 0.
newmap = cumulative_product / cumulative_weight
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.
cumulative_product[nan_array] = 0.
cumulative_weight[nan_array] = 0.
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.
cumulative_product[inf_array] = 0.
cumulative_weight[inf_array] = 0.
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(cumulative_product, "final map * weight")
algebra.compressed_array_summary(cumulative_weight, "final weight map")
combined = "combined_clean"
combined_map_file = "%s/%s_map_%s.npy" % \
(self.output_root, combined, n_modes)
combined_weight_file = "%s/%s_weight_%s.npy" % \
(self.output_root, combined, n_modes)
combined_product_file = "%s/%s_product_%s.npy" % \
(self.output_root, combined, n_modes)
combined_ones_file = "%s/%s_ones_%s.npy" % \
(self.output_root, combined, n_modes)
algebra.save(combined_map_file, newmap)
algebra.save(combined_product_file, cumulative_product)
algebra.save(combined_weight_file, cumulative_weight)
algebra.save(combined_ones_file, algebra.ones_like(newmap))
def radio_correlation(init_filename):
"""Perform the cross-correlation between two mock GBT cubes
"""
np.set_printoptions(threshold=np.nan)
print "starting wigglez correlation run with file: " + init_filename
params = parse_ini.parse(init_filename,
params_default,
prefix=prefix,
feedback=10)
radio_file1 = params['radio_root1'] + params['radio_data_file1']
noiseinv_file1 = params['radio_noiseroot1'] + params['radio_noiseinv_file1']
radio_file2 = params['radio_root2'] + params['radio_data_file2']
noiseinv_file2 = params['radio_noiseroot2'] + params['radio_noiseinv_file2']
map_radio1 = algebra.make_vect(algebra.load(radio_file1))
noiseinv_radio1 = algebra.make_vect(algebra.load(noiseinv_file1))
map_radio2 = algebra.make_vect(algebra.load(radio_file2))
noiseinv_radio2 = algebra.make_vect(algebra.load(noiseinv_file2))
algebra.compressed_array_summary(map_radio1, "radio 1 map as loaded")
algebra.compressed_array_summary(noiseinv_radio1, "radio 1 N^-1 as loaded")
algebra.compressed_array_summary(map_radio2, "radio 2 map as loaded")
algebra.compressed_array_summary(noiseinv_radio2, "radio 2 N^-1 as loaded")
noiseinv_radio1[noiseinv_radio1 < 1.e-20] = 0.
nan_array = np.isnan(noiseinv_radio1)
noiseinv_radio1[nan_array] = 0.
inf_array = np.isinf(noiseinv_radio1)
noiseinv_radio1[inf_array] = 0.
noiseinv_radio2[noiseinv_radio2 < 1.e-20] = 0.
nan_array = np.isnan(noiseinv_radio2)
noiseinv_radio2[nan_array] = 0.
inf_array = np.isinf(noiseinv_radio2)
noiseinv_radio2[inf_array] = 0.
freqlist = params['freq'] # full: range(map_radio.shape[0])
algebra.compressed_array_summary(
map_radio1[freqlist, :, :],
"radio map 1 as entering the correlation function")
algebra.compressed_array_summary(
noiseinv_radio1[freqlist, :, :],
"radio map 1 N^-1 as entering the correlation function")
algebra.compressed_array_summary(
map_radio2[freqlist, :, :],
"radio map 2 as entering the correlation function")
algebra.compressed_array_summary(
noiseinv_radio2[freqlist, :, :],
"radio map 2 N^-1 as entering the correlation function")
cross_pair = fs.MapPair(map_radio1, map_radio2, noiseinv_radio1,
noiseinv_radio2, freqlist)
if params['subtract_mean']:
cross_pair.subtract_weighted_mean()
if params['convolve']:
print "WARNING: you are degrading the beams"
cross_pair.degrade_resolution()
(corr, counts) = cross_pair.correlate(params['lags'],
speedup=params['speedup'])
corr_shelve = shelve.open(params['output_shelve_file'])
corr_shelve["corr"] = corr
corr_shelve["counts"] = counts
corr_shelve["freq_axis"] = map_radio1.get_axis('freq')
corr_shelve["params"] = params
corr_shelve.close()
def combine_maps(param_dict, fullcov=False, verbose=False):
"""combines a list of maps as a weighted mean using a specified list of
inverse covariance weights
fullcov indicates that it is not just the diagonal and should be squashed
"""
print param_dict
covlist = param_dict["covlist"]
try:
mul_cov_list = zip(covlist, param_dict["multiply_cov"])
print "using user-specified covariance multipliers" + repr(param_dict["multiply_cov"])
except KeyError:
mul_cov_list = zip(covlist, [1.0] * len(covlist))
maps = []
for tagname in param_dict["maplist"]:
if verbose:
print tagname
maps.append(algebra.make_vect(algebra.load(param_dict["root_data"] + tagname + ".npy")))
weights = []
for cov_entry in mul_cov_list:
if verbose:
print cov_entry
(tagname, multiplier) = cov_entry
if fullcov:
raw_weight = algebra.make_mat(algebra.open_memmap(param_dict["root_cov"] + tagname + ".npy", mode="r"))
raw_weight = raw_weight.mat_diag()
else:
raw_weight = algebra.make_vect(algebra.load(param_dict["root_cov"] + tagname + ".npy"))
# zero out any messy stuff
raw_weight *= multiplier
raw_weight[raw_weight < 1.0e-20] = 0.0
raw_weight[np.isnan(raw_weight)] = 0.0
raw_weight[np.isinf(raw_weight)] = 0.0
weights.append(raw_weight)
prodmap = []
for mapind in range(0, len(maps)):
prodmap.append(maps[mapind] * weights[mapind])
for mapind in range(1, len(maps)):
prodmap[0] += prodmap[mapind]
weights[0] += weights[mapind]
algebra.compressed_array_summary(weights[0], "weight map")
algebra.compressed_array_summary(prodmap[0], "product map")
newmap = prodmap[0] / weights[0]
newweights = weights[0]
newweights[newweights < 1.0e-20] = 0.0
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.0
newweights[nan_array] = 0.0
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.0
newweights[inf_array] = 0.0
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(newweights, "final weight map")
return (newmap, newweights, prodmap[0])
def save_data(self, n_modes):
prodmap_list = []
weight_list = []
n_modes = "%dmodes" % n_modes
for pairitem in self.pairlist:
pair = self.pairs[pairitem]
(tag1, tag2) = (pair.map1_name, pair.map2_name)
clnoise = "cleaned_noise_inv"
map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag1, clnoise, tag2, n_modes)
modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
#if pair.map1.shape == pair.map2.shape:
map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag2, clnoise, tag1, n_modes)
modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
if self.params['subtract_inputmap_from_sim'] or \
self.params['subtract_sim_from_inputmap']:
map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1
map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2
elif self.params['subtract_realmap_from_sim']:
if not os.path.exists(self.params['realmap_dir']):
print "Error: Real map directory does not exists"
exit()
else:
realmap_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy"%\
(self.params['realmap_dir'], tag1, tag2, n_modes)
realmap = algebra.make_vect(algebra.load(realmap_file))
print "Subtract realmap from result"
map1 = copy.deepcopy(pair.map1) - realmap
map2 = copy.deepcopy(pair.map2)
if map2.shape == map1.shape:
map2 -= realmap
else:
map1 = copy.deepcopy(pair.map1)
map2 = copy.deepcopy(pair.map2)
prodmap_list.append(map1 * pair.noise_inv1)
prodmap_list.append(map2 * pair.noise_inv2)
weight_list.append(pair.noise_inv1)
weight_list.append(pair.noise_inv2)
if self.params['save_section']:
algebra.save(map1_file, map1)
algebra.save(noise_inv1_file, pair.noise_inv1)
algebra.save(modes1_file, pair.left_modes)
if pair.map1.shape == pair.map2.shape:
algebra.save(map2_file, map2)
algebra.save(noise_inv2_file, pair.noise_inv2)
algebra.save(modes2_file, pair.right_modes)
#if map2.shape[0] == 3*map1.shape[0]:
# #source_dict = {}
# #source_dict['map'] = map2_file
# #source_dict['weight'] = noise_inv2_file
# map_dict = {}
# map_dict['map'] = map2
# map_dict['weight'] = pair.noise_inv2
# target_dict = {}
# target_dict['imap'] = map2_file.replace('_'+tag2, '_'+tag2+'_I')
# target_dict['qmap'] = map2_file.replace('_'+tag2, '_'+tag2+'_Q')
# target_dict['umap'] = map2_file.replace('_'+tag2, '_'+tag2+'_U')
# target_dict['imap_weight'] =\
# noise_inv2_file.replace('_'+tag2, '_'+tag2+'_I')
# target_dict['qmap_weight'] =\
# noise_inv2_file.replace('_'+tag2, '_'+tag2+'_Q')
# target_dict['umap_weight'] =\
# noise_inv2_file.replace('_'+tag2, '_'+tag2+'_U')
# divide_iqu_map(map_dict=map_dict, target_dict=target_dict)
if map1.shape != map2.shape:
print "Shape of map1 and map2 are different, can not get combined map."
else:
cumulative_product = algebra.zeros_like(prodmap_list[0])
cumulative_weight = algebra.zeros_like(prodmap_list[0])
for mapind in range(0, len(prodmap_list)):
cumulative_product += prodmap_list[mapind]
cumulative_weight += weight_list[mapind]
algebra.compressed_array_summary(cumulative_weight, "weight map")
algebra.compressed_array_summary(cumulative_product, "product map")
cumulative_weight[cumulative_weight < 1.e-20] = 0.
cumulative_product[cumulative_weight < 1.e-20] = 0.
newmap = cumulative_product / cumulative_weight
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.
cumulative_product[nan_array] = 0.
cumulative_weight[nan_array] = 0.
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.
cumulative_product[inf_array] = 0.
cumulative_weight[inf_array] = 0.
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(cumulative_product,"final map * weight")
algebra.compressed_array_summary(cumulative_weight, "final weight map")
combined = "combined_clean"
combined_map_file = "%s/%s_map_%s.npy" % \
(self.output_root, combined, n_modes)
combined_weight_file = "%s/%s_weight_%s.npy" % \
(self.output_root, combined, n_modes)
combined_product_file = "%s/%s_product_%s.npy" % \
(self.output_root, combined, n_modes)
combined_ones_file = "%s/%s_ones_%s.npy" % \
(self.output_root, combined, n_modes)
algebra.save(combined_map_file, newmap)
algebra.save(combined_product_file, cumulative_product)
algebra.save(combined_weight_file, cumulative_weight)
algebra.save(combined_ones_file, algebra.ones_like(newmap))
def wigglez_correlation(init_filename):
"""Perform the cross-correlation between the WiggleZ dataset and GBT
"""
np.set_printoptions(threshold=np.nan)
print "starting wigglez correlation run with file: " + init_filename
params = parse_ini.parse(init_filename, params_default,
prefix=prefix, feedback=10)
optical_file1 = params['optical_root1'] + params['optical_data_file1']
optical_selection_file1 = params['optical_root1'] + \
params['optical_selection_file1']
optical_file2 = params['optical_root2'] + params['optical_data_file2']
optical_selection_file2 = params['optical_root2'] + \
params['optical_selection_file2']
map_opt1 = algebra.make_vect(algebra.load(optical_file1))
map_nbar1 = algebra.make_vect(algebra.load(optical_selection_file1))
map_opt2 = algebra.make_vect(algebra.load(optical_file2))
map_nbar2 = algebra.make_vect(algebra.load(optical_selection_file2))
algebra.compressed_array_summary(map_opt1, "opt map 1 as loaded")
algebra.compressed_array_summary(map_nbar1, "nbar map 1 as loaded")
algebra.compressed_array_summary(map_opt2, "opt map 2 as loaded")
algebra.compressed_array_summary(map_nbar2, "nbar map 2 as loaded")
if params['convolve']:
beam_data = sp.array([0.316148488246, 0.306805630985, 0.293729620792,
0.281176247549, 0.270856788455, 0.26745856078,
0.258910010848, 0.249188429031])
# also consider the beam model where everything is degraded to the
# lowest resolution; e.g. in a MapPair
beam_data_degrade = sp.zeros(8) + max(beam_data)
freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905],
dtype=float)
freq_data *= 1.e6
psf = beam.GaussianBeam(beam_data_degrade, freq_data)
# Convolve the optical data to the lowest radio resolution
print "convolving the first map"
map_opt1 = psf.apply(map_opt1)
algebra.compressed_array_summary(map_opt1, "opt 1 after convolution")
print "convolving the second map"
map_opt2 = psf.apply(map_opt2)
algebra.compressed_array_summary(map_opt2, "opt 2 after convolution")
# how should the covariance be convolved?
# nbar is N^-1; convolve B N B^T?,
# or just convolve nbar? use?:
#map_nbar[map_nbar<1.e-30] = 1.e-30
#map_nbar = 1./map_nbar
#map_nbar = psf.apply(map_nbar, cval=1.e30)
#map_nbar = 1./map_nbar
#map_nbar[map_nbar<1.e-20] = 0
print "convolving the first covariance/selection"
map_nbar1 = psf.apply(map_nbar1)
algebra.compressed_array_summary(map_nbar1, "nbar 1 after convolution")
print "convolving the second covariance/selection"
map_nbar2 = psf.apply(map_nbar2)
algebra.compressed_array_summary(map_nbar2, "nbar 2 after convolution")
# convert to delta-overdensity
map_opt1 = map_opt1 / map_nbar1 - 1.
map_opt2 = map_opt2 / map_nbar2 - 1.
#algebra.compressed_array_summary(map_opt1, "opt 1 after conversion to delta")
#algebra.compressed_array_summary(map_opt2, "opt 2 after conversion to delta")
# set the NaNs and infs to zero in data and weights
nan_array = np.isnan(map_opt1)
map_opt1[nan_array] = 0.
map_nbar1[nan_array] = 0.
inf_array = np.isinf(map_opt1)
map_opt1[inf_array] = 0.
map_nbar1[inf_array] = 0.
nan_array = np.isnan(map_opt2)
map_opt2[nan_array] = 0.
map_nbar2[nan_array] = 0.
inf_array = np.isinf(map_opt2)
map_opt2[inf_array] = 0.
map_nbar2[inf_array] = 0.
freqlist = params['freq'] # full: range(map_radio.shape[0])
algebra.compressed_array_summary(map_opt1[freqlist, :, :],
"opt map 1 as entering the correlation function")
algebra.compressed_array_summary(map_nbar1[freqlist, :, :],
"nbar 1 as entering the correlation function")
algebra.compressed_array_summary(map_opt2[freqlist, :, :],
"opt map 2 as entering the correlation function")
algebra.compressed_array_summary(map_nbar2[freqlist, :, :],
"nbar 2 N^-1 as entering the correlation function")
cross_pair = fs.MapPair(map_opt1, map_opt2, map_nbar1, map_nbar2,
freqlist)
if params['subtract_mean']:
cross_pair.subtract_weighted_mean()
(corr, counts) = cross_pair.correlate(params['lags'],
speedup=params['speedup'])
corr_shelve = shelve.open(params['output_shelve_file'])
corr_shelve["corr"] = corr
corr_shelve["counts"] = counts
corr_shelve["freq_axis"] = map_opt1.get_axis('freq')
corr_shelve["params"] = params
corr_shelve.close()
def combine_maps(source_key, alt_weight=None,
signal='map', weight='weight', divider_token=";",
fullcov=False):
r"""
`source_key` is the file db key for the maps to combine
`alt_weight` is an optional alternate weighting for the maps
`signal` is the tag in the file db entry for the signal maps
`weight` is the tag in the file db entry for the N^-1 weights
`fullcov` uses a memory map for large N^-1 and pulls the diagonal
`divider_token` is the token that divides the map section name
from the data type e.g. "A_with_B;noise_inv"
"""
datapath_db = data_paths.DataPath()
source_fdb = datapath_db.fetch(source_key, intend_read=True,
silent=True)
source_fdict = source_fdb[1]
# accumulate all the files to combine
weightkeys = {}
signalkeys = {}
for filekey in source_fdb[0]:
if divider_token in filekey:
data_type = filekey.split(divider_token)[1]
map_section = filekey.split(divider_token)[0]
if data_type == signal:
signalkeys[map_section] = source_fdict[filekey]
if data_type == weight:
weightkeys[map_section] = source_fdict[filekey]
signal_list = []
weight_list = []
for mapkey in signalkeys:
signalfile = signalkeys[mapkey]
weightfile = weightkeys[mapkey]
print "loading pair: %s %s" % (signalfile, weightfile)
signal_list.append(algebra.make_vect(algebra.load(signalfile)))
if fullcov:
raw_weight = algebra.make_mat(
algebra.open_memmap(weightfile))
raw_weight = raw_weight.mat_diag()
else:
raw_weight = algebra.make_vect(algebra.load(weightfile))
# zero out any messy stuff (should have been done already)
raw_weight[raw_weight < 1.e-20] = 0.
raw_weight[np.isnan(raw_weight)] = 0.
raw_weight[np.isinf(raw_weight)] = 0.
weight_list.append(raw_weight)
prodmap = []
for mapind in range(0, len(signal_list)):
prodmap.append(signal_list[mapind] * weight_list[mapind])
for mapind in range(1, len(signal_list)):
prodmap[0] += prodmap[mapind]
weight_list[0] += weight_list[mapind]
algebra.compressed_array_summary(weight_list[0], "weight map")
algebra.compressed_array_summary(prodmap[0], "product map")
newmap = prodmap[0] / weight_list[0]
newweights = weight_list[0]
prodmap = prodmap[0]
newweights[newweights < 1.e-20] = 0.
prodmap[newweights < 1.e-20] = 0.
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.
prodmap[nan_array] = 0.
newweights[nan_array] = 0.
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.
prodmap[inf_array] = 0.
newweights[inf_array] = 0.
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(prodmap, "final map * weight")
algebra.compressed_array_summary(newweights, "final weight map")
# write out the three maps (combined map, map * weight, weight)
signal_out = source_fdict['combined_map']
prodmap_out = source_fdict['combined_mapxweight']
weight_out = source_fdict['combined_weight']
# alternately
#signal_out = "%s/%s_signal.npy" % (outputdir, source_key)
#prodmap_out = "%s/%s_product.npy" % (outputdir, source_key)
#weight_out = "%s/%s_weight.npy" % (outputdir, source_key)
algebra.save(signal_out, newmap)
algebra.save(prodmap_out, prodmap)
algebra.save(weight_out, newweights)
def combine_maps(param_dict, fullcov=False, verbose=False):
"""combines a list of maps as a weighted mean using a specified list of
inverse covariance weights
fullcov indicates that it is not just the diagonal and should be squashed
"""
print param_dict
covlist = param_dict["covlist"]
try:
mul_cov_list = zip(covlist, param_dict["multiply_cov"])
print "using user-specified covariance multipliers" + \
repr(param_dict["multiply_cov"])
except KeyError:
mul_cov_list = zip(covlist, [1.] * len(covlist))
maps = []
for tagname in param_dict["maplist"]:
if verbose:
print tagname
maps.append(algebra.make_vect(
algebra.load(param_dict["root_data"] + tagname + ".npy")))
weights = []
for cov_entry in mul_cov_list:
if verbose:
print cov_entry
(tagname, multiplier) = cov_entry
if fullcov:
raw_weight = algebra.make_mat(
algebra.open_memmap(param_dict["root_cov"] + \
tagname + ".npy", mode='r'))
raw_weight = raw_weight.mat_diag()
else:
raw_weight = algebra.make_vect(algebra.load(
param_dict["root_cov"] + tagname + ".npy"))
# zero out any messy stuff
raw_weight *= multiplier
raw_weight[raw_weight < 1.e-20] = 0.
raw_weight[np.isnan(raw_weight)] = 0.
raw_weight[np.isinf(raw_weight)] = 0.
weights.append(raw_weight)
prodmap = []
for mapind in range(0, len(maps)):
prodmap.append(maps[mapind] * weights[mapind])
for mapind in range(1, len(maps)):
prodmap[0] += prodmap[mapind]
weights[0] += weights[mapind]
algebra.compressed_array_summary(weights[0], "weight map")
algebra.compressed_array_summary(prodmap[0], "product map")
newmap = prodmap[0] / weights[0]
newweights = weights[0]
newweights[newweights < 1.e-20] = 0.
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.
newweights[nan_array] = 0.
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.
newweights[inf_array] = 0.
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(newweights, "final weight map")
return (newmap, newweights, prodmap[0])
def radio_correlation(init_filename):
"""Perform the cross-correlation between two mock GBT cubes
"""
np.set_printoptions(threshold=np.nan)
print "starting wigglez correlation run with file: " + init_filename
params = parse_ini.parse(init_filename, params_default,
prefix=prefix, feedback=10)
radio_file1 = params['radio_root1'] + params['radio_data_file1']
noiseinv_file1 = params['radio_noiseroot1'] + params['radio_noiseinv_file1']
radio_file2 = params['radio_root2'] + params['radio_data_file2']
noiseinv_file2 = params['radio_noiseroot2'] + params['radio_noiseinv_file2']
map_radio1 = algebra.make_vect(algebra.load(radio_file1))
noiseinv_radio1 = algebra.make_vect(algebra.load(noiseinv_file1))
map_radio2 = algebra.make_vect(algebra.load(radio_file2))
noiseinv_radio2 = algebra.make_vect(algebra.load(noiseinv_file2))
algebra.compressed_array_summary(map_radio1, "radio 1 map as loaded")
algebra.compressed_array_summary(noiseinv_radio1, "radio 1 N^-1 as loaded")
algebra.compressed_array_summary(map_radio2, "radio 2 map as loaded")
algebra.compressed_array_summary(noiseinv_radio2, "radio 2 N^-1 as loaded")
noiseinv_radio1[noiseinv_radio1 < 1.e-20] = 0.
nan_array = np.isnan(noiseinv_radio1)
noiseinv_radio1[nan_array] = 0.
inf_array = np.isinf(noiseinv_radio1)
noiseinv_radio1[inf_array] = 0.
noiseinv_radio2[noiseinv_radio2 < 1.e-20] = 0.
nan_array = np.isnan(noiseinv_radio2)
noiseinv_radio2[nan_array] = 0.
inf_array = np.isinf(noiseinv_radio2)
noiseinv_radio2[inf_array] = 0.
freqlist = params['freq'] # full: range(map_radio.shape[0])
algebra.compressed_array_summary(map_radio1[freqlist, :, :],
"radio map 1 as entering the correlation function")
algebra.compressed_array_summary(noiseinv_radio1[freqlist, :, :],
"radio map 1 N^-1 as entering the correlation function")
algebra.compressed_array_summary(map_radio2[freqlist, :, :],
"radio map 2 as entering the correlation function")
algebra.compressed_array_summary(noiseinv_radio2[freqlist, :, :],
"radio map 2 N^-1 as entering the correlation function")
cross_pair = fs.MapPair(map_radio1, map_radio2,
noiseinv_radio1, noiseinv_radio2,
freqlist)
if params['subtract_mean']:
cross_pair.subtract_weighted_mean()
if params['convolve']:
print "WARNING: you are degrading the beams"
cross_pair.degrade_resolution()
(corr, counts) = cross_pair.correlate(params['lags'],
speedup=params['speedup'])
corr_shelve = shelve.open(params['output_shelve_file'])
corr_shelve["corr"] = corr
corr_shelve["counts"] = counts
corr_shelve["freq_axis"] = map_radio1.get_axis('freq')
corr_shelve["params"] = params
corr_shelve.close()
