def calculate_xspec_file(cube1_file,
cube2_file,
bins,
weight1_file=None,
weight2_file=None,
truncate=False,
window="blackman",
return_3d=False,
unitless=True):
cube1 = algebra.make_vect(algebra.load(cube1_file))
cube2 = algebra.make_vect(algebra.load(cube2_file))
if weight1_file is None:
weight1 = algebra.ones_like(cube1)
else:
weight1 = algebra.make_vect(algebra.load(weight1_file))
if weight2_file is None:
weight2 = algebra.ones_like(cube2)
else:
weight2 = algebra.make_vect(algebra.load(weight2_file))
print cube1.shape, cube2.shape, weight1.shape, weight2.shape
return calculate_xspec(cube1,
cube2,
weight1,
weight2,
bins=bins,
window=window,
unitless=unitless,
truncate=truncate,
return_3d=return_3d)
def __init__(self,
map1,
map2,
noise_inv1,
noise_inv2,
freq,
input_filenames=False,
conv_factor=1.1):
r"""
arguments: map1, map2, noise_inv1, noise_inv2, freq
conv_factor is the factor by which to multiply the largest beam
in the convolution to a common resolution
"""
if input_filenames:
self.map1 = algebra.make_vect(algebra.load(map1))
self.map2 = algebra.make_vect(algebra.load(map2))
if noise_inv1:
print "loading noise1 file: " + noise_inv1
self.noise_inv1 = algebra.make_vect(algebra.load(noise_inv1))
else:
print "WARNING: map1 has unity weight; no file given"
self.noise_inv1 = algebra.ones_like(self.map1)
if noise_inv2:
print "loading noise2 file: " + noise_inv2
self.noise_inv2 = algebra.make_vect(algebra.load(noise_inv2))
else:
print "WARNING: map2 has unity weight; no file given"
self.noise_inv2 = algebra.ones_like(self.map2)
else:
self.map1 = map1
self.map2 = map2
self.noise_inv1 = noise_inv1
self.noise_inv2 = noise_inv2
self.freq = freq
self.conv_factor = conv_factor
# maps in physical coordinates (derived)
self.phys_map1 = None
self.phys_map2 = None
self.phys_noise_inv1 = None
self.phys_noise_inv2 = None
# give infinite noise to masked bands
self.sanitize()
# Set attributes.
self.left_modes = 0
self.right_modes = 0
# For saving, to keep track of each mapname.
self.map1_name = ''
self.map2_name = ''
# Which section [A, B, C, D...] the maps is from.
self.map1_code = ''
self.map2_code = ''
def __init__(self, map1, map2, noise_inv1, noise_inv2, freq,
input_filenames=False, conv_factor=1.1):
r"""
arguments: map1, map2, noise_inv1, noise_inv2, freq
conv_factor is the factor by which to multiply the largest beam
in the convolution to a common resolution
"""
if input_filenames:
self.map1 = algebra.make_vect(algebra.load(map1))
self.map2 = algebra.make_vect(algebra.load(map2))
if noise_inv1:
print "loading noise1 file: " + noise_inv1
self.noise_inv1 = algebra.make_vect(algebra.load(noise_inv1))
else:
print "WARNING: map1 has unity weight; no file given"
self.noise_inv1 = algebra.ones_like(self.map1)
if noise_inv2:
print "loading noise2 file: " + noise_inv2
self.noise_inv2 = algebra.make_vect(algebra.load(noise_inv2))
else:
print "WARNING: map2 has unity weight; no file given"
self.noise_inv2 = algebra.ones_like(self.map2)
else:
self.map1 = map1
self.map2 = map2
self.noise_inv1 = noise_inv1
self.noise_inv2 = noise_inv2
self.freq = freq
self.conv_factor = conv_factor
# maps in physical coordinates (derived)
self.phys_map1 = None
self.phys_map2 = None
self.phys_noise_inv1 = None
self.phys_noise_inv2 = None
# give infinite noise to masked bands
self.sanitize()
# Set attributes.
self.left_modes = 0
self.right_modes = 0
# For saving, to keep track of each mapname.
self.map1_name = ''
self.map2_name = ''
# Which section [A, B, C, D...] the maps is from.
self.map1_code = ''
self.map2_code = ''
def load_pairs(self, regenerate=True):
r"""load the set of map/noise pairs specified by keys handed to the
database. This sets up operations on the quadratic product
Q = map1^T noise_inv1 B noise_inv2 map2
"""
par = self.params
(self.pairlist, pairdict) = dp.cross_maps(par['map1'],
par['map2'],
par['noise_inv1'],
par['noise_inv2'],
verbose=False)
for pairitem in self.pairlist:
pdict = pairdict[pairitem]
print "-" * 80
dp.print_dictionary(
pdict,
sys.stdout,
key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2'])
map1 = algebra.make_vect(algebra.load(pdict['map1']))
map2 = algebra.make_vect(algebra.load(pdict['map2']))
sim = algebra.make_vect(algebra.load(par['simfile']))
if not par['no_weights']:
noise_inv1 = self.process_noise_inv(pdict['noise_inv1'],
regenerate=regenerate)
noise_inv2 = self.process_noise_inv(pdict['noise_inv2'],
regenerate=regenerate)
else:
noise_inv1 = algebra.ones_like(map1)
noise_inv2 = algebra.ones_like(map2)
pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1,
noise_inv2, self.freq_list)
pair.set_names(pdict['tag1'], pdict['tag2'])
pair.lags = self.lags
pair.params = self.params
self.pairs[pairitem] = pair
pair_nosim = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2,
self.freq_list)
pair_nosim.set_names(pdict['tag1'], pdict['tag2'])
pair_nosim.lags = self.lags
pair_nosim.params = self.params
self.pairs_nosim[pairitem] = pair_nosim
def test_with_random(unitless=True):
"""Test the power spectral estimator using a random noise cube"""
delta = 1.33333
cube1 = algebra.make_vect(np.random.normal(0, 1, size=(257, 124, 68)))
info = {
'axes': ["freq", "ra", "dec"],
'type': 'vect',
'freq_delta': delta / 3.78,
'freq_centre': 0.,
'ra_delta': delta / 1.63,
'ra_centre': 0.,
'dec_delta': delta,
'dec_centre': 0.
}
cube1.info = info
cube2 = copy.deepcopy(cube1)
weight1 = algebra.ones_like(cube1)
weight2 = algebra.ones_like(cube2)
bin_left, bin_center, bin_right, counts_histo, binavg = \
calculate_xspec(cube1, cube2, weight1, weight2,
window="blackman",
truncate=False,
nbins=40,
unitless=unitless,
logbins=True)
if unitless:
pwrspec_input = bin_center**3. / 2. / math.pi / math.pi
else:
pwrspec_input = np.ones_like(bin_center)
volume = 1.
for axis_name in cube1.axes:
axis_vector = cube1.get_axis(axis_name)
volume *= abs(axis_vector[1] - axis_vector[0])
pwrspec_input *= volume
for specdata in zip(bin_left, bin_center, bin_right, counts_histo, binavg,
pwrspec_input):
print("%10.15g " * 6) % specdata
def test_with_random(unitless=True):
"""Test the power spectral estimator using a random noise cube"""
delta = 1.33333
cube1 = algebra.make_vect(np.random.normal(0, 1,
size=(257, 124, 68)))
info = {'axes': ["freq", "ra", "dec"], 'type': 'vect',
'freq_delta': delta / 3.78, 'freq_centre': 0.,
'ra_delta': delta / 1.63, 'ra_centre': 0.,
'dec_delta': delta, 'dec_centre': 0.}
cube1.info = info
cube2 = copy.deepcopy(cube1)
weight1 = algebra.ones_like(cube1)
weight2 = algebra.ones_like(cube2)
bin_left, bin_center, bin_right, counts_histo, binavg = \
calculate_xspec(cube1, cube2, weight1, weight2,
window="blackman",
truncate=False,
nbins=40,
unitless=unitless,
logbins=True)
if unitless:
pwrspec_input = bin_center ** 3. / 2. / math.pi / math.pi
else:
pwrspec_input = np.ones_like(bin_center)
volume = 1.
for axis_name in cube1.axes:
axis_vector = cube1.get_axis(axis_name)
volume *= abs(axis_vector[1] - axis_vector[0])
pwrspec_input *= volume
for specdata in zip(bin_left, bin_center,
bin_right, counts_histo, binavg,
pwrspec_input):
print ("%10.15g " * 6) % specdata
def load_pairs(self, regenerate=True):
r"""load the set of map/noise pairs specified by keys handed to the
database. This sets up operations on the quadratic product
Q = map1^T noise_inv1 B noise_inv2 map2
"""
par = self.params
(self.pairlist, pairdict) = dp.cross_maps(
par["map1"], par["map2"], par["noise_inv1"], par["noise_inv2"], verbose=False
)
for pairitem in self.pairlist:
pdict = pairdict[pairitem]
print "-" * 80
dp.print_dictionary(pdict, sys.stdout, key_list=["map1", "noise_inv1", "map2", "noise_inv2"])
map1 = algebra.make_vect(algebra.load(pdict["map1"]))
map2 = algebra.make_vect(algebra.load(pdict["map2"]))
sim = algebra.make_vect(algebra.load(par["simfile"]))
if not par["no_weights"]:
noise_inv1 = self.process_noise_inv(pdict["noise_inv1"], regenerate=regenerate)
noise_inv2 = self.process_noise_inv(pdict["noise_inv2"], regenerate=regenerate)
else:
noise_inv1 = algebra.ones_like(map1)
noise_inv2 = algebra.ones_like(map2)
pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1, noise_inv2, self.freq_list)
pair.set_names(pdict["tag1"], pdict["tag2"])
pair.lags = self.lags
pair.params = self.params
self.pairs[pairitem] = pair
pair_nosim = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list)
pair_nosim.set_names(pdict["tag1"], pdict["tag2"])
pair_nosim.lags = self.lags
pair_nosim.params = self.params
self.pairs_nosim[pairitem] = pair_nosim
def calculate_xspec_file(cube1_file, cube2_file, bins,
weight1_file=None, weight2_file=None,
truncate=False, window="blackman",
return_3d=False, unitless=True):
cube1 = algebra.make_vect(algebra.load(cube1_file))
cube2 = algebra.make_vect(algebra.load(cube2_file))
if weight1_file is None:
weight1 = algebra.ones_like(cube1)
else:
weight1 = algebra.make_vect(algebra.load(weight1_file))
if weight2_file is None:
weight2 = algebra.ones_like(cube2)
else:
weight2 = algebra.make_vect(algebra.load(weight2_file))
print cube1.shape, cube2.shape, weight1.shape, weight2.shape
return calculate_xspec(cube1, cube2, weight1, weight2, bins=bins,
window=window, unitless=unitless,
truncate=truncate, return_3d=return_3d)
def extend_iqu_map(source_dict=None, target_dict=None, map_dict=None):
if source_dict != None:
imap = algebra.make_vect(algebra.load(source_dict['imap']))
qmap = algebra.make_vect(algebra.load(source_dict['qmap']))
umap = algebra.make_vect(algebra.load(source_dict['umap']))
if source_dict.has_key('imap_weight'):
imap_weight = algebra.make_vect(algebra.load(source_dict['imap_weight']))
qmap_weight = algebra.make_vect(algebra.load(source_dict['qmap_weight']))
umap_weight = algebra.make_vect(algebra.load(source_dict['umap_weight']))
elif source_dict.has_key('imap_inv'):
imap_weight, info = find_weight_re_diagnal(source_dict['imap_inv'])
qmap_weight, info = find_weight_re_diagnal(source_dict['qmap_inv'])
umap_weight, info = find_weight_re_diagnal(source_dict['umap_inv'])
else:
print 'Warning: no weight'
imap_weight = algebra.ones_like(imap)
qmap_weight = algebra.ones_like(imap)
umap_weight = algebra.ones_like(imap)
elif map_dict != None:
imap = map_dict['imap']
qmap = map_dict['qmap']
umap = map_dict['umap']
if 'imap_weight' in map_dict.keys():
imap_weight = map_dict['imap_weight']
qmap_weight = map_dict['qmap_weight']
umap_weight = map_dict['umap_weight']
else:
print 'Warning: no weight'
imap_weight = algebra.ones_like(imap)
qmap_weight = algebra.ones_like(imap)
umap_weight = algebra.ones_like(imap)
else:
print "Error: Can not find I Q U maps"
exit()
iqu = algebra.info_array(imap.tolist() + qmap.tolist() + umap.tolist())
iqu = algebra.make_vect(iqu)
iqu.info = imap.info
iqu.copy_axis_info(imap)
iqu_weight = algebra.info_array(imap_weight.tolist() +
qmap_weight.tolist() +
umap_weight.tolist())
iqu_weight = algebra.make_vect(iqu_weight)
iqu_weight.info = imap_weight.info
iqu_weight.copy_axis_info(imap_weight)
if target_dict != None:
algebra.save(target_dict['map'], iqu)
algebra.save(target_dict['weight'], iqu_weight)
else:
map_dict = {}
map_dict['map'] = iqu
map_dict['weight'] = iqu_weight
return map_dict
def save_data(self, n_modes):
prodmap_list = []
weight_list = []
n_modes = "%dmodes" % n_modes
for pairitem in self.pairlist:
pair = self.pairs[pairitem]
(tag1, tag2) = (pair.map1_name, pair.map2_name)
clnoise = "cleaned_noise_inv"
map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag1, clnoise, tag2, n_modes)
noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag2, clnoise, tag1, n_modes)
modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
if self.params['subtract_inputmap_from_sim'] or \
self.params['subtract_sim_from_inputmap']:
map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1
map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2
else:
map1 = copy.deepcopy(pair.map1)
map2 = copy.deepcopy(pair.map2)
prodmap_list.append(map1 * pair.noise_inv1)
prodmap_list.append(map2 * pair.noise_inv2)
weight_list.append(pair.noise_inv1)
weight_list.append(pair.noise_inv2)
algebra.save(map1_file, map1)
algebra.save(map2_file, map2)
algebra.save(noise_inv1_file, pair.noise_inv1)
algebra.save(noise_inv2_file, pair.noise_inv2)
algebra.save(modes1_file, pair.left_modes)
algebra.save(modes2_file, pair.right_modes)
cumulative_product = algebra.zeros_like(prodmap_list[0])
cumulative_weight = algebra.zeros_like(prodmap_list[0])
for mapind in range(0, len(prodmap_list)):
cumulative_product += prodmap_list[mapind]
cumulative_weight += weight_list[mapind]
algebra.compressed_array_summary(cumulative_weight, "weight map")
algebra.compressed_array_summary(cumulative_product, "product map")
cumulative_weight[cumulative_weight < 1.e-20] = 0.
cumulative_product[cumulative_weight < 1.e-20] = 0.
newmap = cumulative_product / cumulative_weight
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.
cumulative_product[nan_array] = 0.
cumulative_weight[nan_array] = 0.
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.
cumulative_product[inf_array] = 0.
cumulative_weight[inf_array] = 0.
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(cumulative_product, "final map * weight")
algebra.compressed_array_summary(cumulative_weight, "final weight map")
combined = "combined_clean"
combined_map_file = "%s/%s_map_%s.npy" % \
(self.output_root, combined, n_modes)
combined_weight_file = "%s/%s_weight_%s.npy" % \
(self.output_root, combined, n_modes)
combined_product_file = "%s/%s_product_%s.npy" % \
(self.output_root, combined, n_modes)
combined_ones_file = "%s/%s_ones_%s.npy" % \
(self.output_root, combined, n_modes)
algebra.save(combined_map_file, newmap)
algebra.save(combined_product_file, cumulative_product)
algebra.save(combined_weight_file, cumulative_weight)
algebra.save(combined_ones_file, algebra.ones_like(newmap))
def calculate_mixing(weight_file1, weight_file2, bins, xspec_fileout,
mixing_fileout,
unitless=False, refinement=2, pad=5, order=1,
window='blackman', zero_pad=False, identity_test=False):
print "loading the weights and converting to physical coordinates"
weight1_obs = algebra.make_vect(algebra.load(weight_file1))
weight1 = bh.repackage_kiyo(pg.physical_grid(
weight1_obs,
refinement=refinement,
pad=pad, order=order))
weight2_obs = algebra.make_vect(algebra.load(weight_file2))
weight2 = bh.repackage_kiyo(pg.physical_grid(
weight2_obs,
refinement=refinement,
pad=pad, order=order))
if window:
window_function = fftutil.window_nd(weight1.shape, name=window)
weight1 *= window_function
weight2 *= window_function
print "calculating the cross-power of the spatial weighting functions"
arr1 = algebra.ones_like(weight1)
arr2 = algebra.ones_like(weight2)
# no window applied here (applied above)
xspec = pe.cross_power_est(weight1, weight2, arr1, arr2,
window=None, nonorm=True)
# for each point in the cube, find |k|, k_perp, k_parallel
# TODO: speed this up by using one direct numpy call (not limiting)
k_mag_arr = binning.radius_array(xspec)
k_perp_arr = binning.radius_array(xspec, zero_axes=[0])
k_parallel_arr = binning.radius_array(xspec, zero_axes=[1, 2])
if unitless:
xspec = pe.make_unitless(xspec, radius_arr=k_mag_arr)
# NOTE: assuming lowest k bin has only one point in 3D k-space
# could make this floor of dimensions divided by 2 also
center_3d = np.transpose(np.transpose(np.where(k_mag_arr == 0.))[0])
# In the estimator, we devide by 1/sum(w1 * w2) to get most of the effect
# of the weighing. The mixing matrix here can be thought of as a correction
# that that diagonal-only estimate.
leakage_ratio = xspec[center_3d[0], center_3d[1], center_3d[2]] / \
np.sum(weight1 * weight2)
print "power leakage ratio: %10.5g" % leakage_ratio
xspec /= np.sum(weight1 * weight2)
print "partitioning the 3D kspace up into the 2D k bins"
(kflat, ret_indices) = bin_indices_2d(k_perp_arr, k_parallel_arr,
bins, bins)
# perform a test where the window function is a delta function at the
# origin so that the mixing matrix is identity
if identity_test:
xspec = algebra.zeros_like(xspec)
xspec[center_3d[0], center_3d[1], center_3d[2]] = 1.
# now save the window cross-power for downstream pooled users
algebra.save(xspec_fileout, xspec)
runlist = []
for bin_index in range(kflat.shape[0]):
bin_3d = ret_indices[repr(bin_index)]
if bin_3d is not None:
runlist.append((xspec_fileout, bin_index, bins, bin_3d, center_3d))
pool = multiprocessing.Pool(processes=(multiprocessing.cpu_count() - 4))
# the longest runs get pushed to the end; randomize for better job packing
random.shuffle(runlist)
results = pool.map(sum_window, runlist)
#gnuplot_single_slice(runlist[0]) # for troubleshooting
# now save the results for post-processing
params = {"unitless": unitless, "refinement": refinement, "pad": pad,
"order": order, "window": window, "zero_pad": zero_pad,
"identity_test": identity_test, "weight_file1": weight_file1,
"weight_file2": weight_file2, "bins": bins}
outshelve = shelve.open(mixing_fileout, "n")
outshelve["params"] = params # parameters for this run
outshelve["weight1"] = weight1 # weight map 1
outshelve["weight2"] = weight2 # weight map 2
outshelve["xspec"] = xspec # copy of the weight spectra
outshelve["kflat"] = kflat # 2D k bin vector
outshelve["bins_3d"] = ret_indices # indices to k3d for a 2d k bin
outshelve["results"] = results # mixing matrix columns
outshelve.close()
def execute(self):
'''Clean the maps of foregrounds, save the results, and get the
autocorrelation.'''
params = self.params
freq_list = sp.array(params['freq_list'], dtype=int)
lags = sp.array(params['lags'])
# Write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params,
params['output_root'] + 'params.ini',
prefix=prefix)
# Get the map data from file as well as the noise inverse.
if len(params['file_middles']) == 1:
fmid_name = params['file_middles'][0]
params['file_middles'] = (fmid_name, fmid_name)
if len(params['file_middles']) >= 2:
# Deal with multiple files.
num_maps = len(params['file_middles'])
maps = []
noise_invs = []
# Load all maps and noises once.
for map_index in range(0, num_maps):
map_file = (params['input_root'] +
params['file_middles'][map_index] +
params['input_end_map'])
print "Loading map %d of %d." % (map_index + 1, num_maps)
map_in = algebra.make_vect(algebra.load(map_file))
maps.append(map_in)
if not params["no_weights"]:
noise_file = (params['input_root'] +
params['file_middles'][map_index] +
params['input_end_noise'])
print "Loading noise %d of %d." % (map_index + 1, num_maps)
noise_inv = algebra.make_mat(
algebra.open_memmap(noise_file, mode='r'))
noise_inv = noise_inv.mat_diag()
else:
noise_inv = algebra.ones_like(map_in)
noise_invs.append(noise_inv)
pairs = []
# Make pairs with deepcopies to not make mutability mistakes.
for map1_index in range(0, num_maps):
for map2_index in range(0, num_maps):
if (map2_index > map1_index):
map1 = copy.deepcopy(maps[map1_index])
map2 = copy.deepcopy(maps[map2_index])
noise_inv1 = copy.deepcopy(noise_invs[map1_index])
noise_inv2 = copy.deepcopy(noise_invs[map2_index])
pair = map_pair.MapPair(map1, map2, noise_inv1,
noise_inv2, freq_list)
pair.lags = lags
pair.params = params
# Keep track of the names of maps in pairs so
# it knows what to save later.
pair.set_names(params['file_middles'][map1_index],
params['file_middles'][map2_index])
pairs.append(pair)
num_map_pairs = len(pairs)
print "%d map pairs created from %d maps." % (len(pairs), num_maps)
# Hold a reference in self.
self.pairs = pairs
# Get maps/ noise inv ready for running.
if params["convolve"]:
for pair in pairs:
pair.degrade_resolution()
if params['factorizable_noise']:
for pair in pairs:
pair.make_noise_factorizable()
if params['sub_weighted_mean']:
for pair in pairs:
pair.subtract_weighted_mean()
self.pairs = pairs
# Since correlating takes so long, if you already have the svds
# you can skip this first correlation [since that's all it's really
# for and it is the same no matter how many modes you want].
# Note: map_pairs will not have anything saved in 'fore_corr' if you
# skip this correlation.
if not params['skip_fore_corr']:
# Correlate the maps with multiprocessing. Note that the
# correlations are saved to file separately then loaded in
# together because that's (one way) how multiprocessing works.
fore_pairs = []
processes_list = []
for pair_index in range(0, num_map_pairs):
# Calls 1 multiproc (which governs the correlating) for each
# pair on a new CPU so you can have all pairs working at once.
multi = multiprocessing.Process(target=multiproc,
args=([
pairs[pair_index],
params['output_root'],
pair_index, False
]))
processes_list.append(multi)
multi.start()
# Waits for all correlations to finish before continuing.
while True in [multi.is_alive() for multi in processes_list]:
print "processing"
time.sleep(5)
# just to be safe
time.sleep(1)
# more concise call, but multiprocessing does not behave well with
# complex objects...........
#runlist = [(pair_index,
# params['output_root'],
# False) for
# pair_index in range(0, num_map_pairs)]
#pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
#pool.map(self.multiproc, runlist)
# Load the correlations and save them to each pair. The pairs that
# got passed to multiproc are not the same ones as ones in
# self.pairs, so this must be done to have actual values.
print "Loading map pairs back into program."
file_name = params['output_root']
file_name += "map_pair_for_freq_slices_fore_corr_"
for count in range(0, num_map_pairs):
print "Loading correlation for pair %d" % (count)
pickle_handle = open(file_name + str(count) + ".pkl", "r")
correlate_results = cPickle.load(pickle_handle)
pairs[count].fore_corr = correlate_results[0]
pairs[count].fore_counts = correlate_results[1]
fore_pairs.append(pairs[count])
pickle_handle.close()
self.fore_pairs = copy.deepcopy(fore_pairs)
# With this, you do not need fore_pairs anymore.
self.pairs = copy.deepcopy(fore_pairs)
pairs = self.pairs
# Get foregrounds.
# svd_info_list keeps track of all of the modes of all maps in
# all pairs. This means if you want to subract a different number
# of modes for the same maps/noises/frequencies, you have the modes
# already saved and do not need to run the first correlation again.
svd_info_list = []
for pair in pairs:
vals, modes1, modes2 = cf.get_freq_svd_modes(
pair.fore_corr, len(freq_list))
pair.vals = vals
# Save ALL of the modes for reference.
pair.all_modes1 = modes1
pair.all_modes2 = modes2
svd_info = (vals, modes1, modes2)
svd_info_list.append(svd_info)
# Save only the modes you want to subtract.
n_modes = params['modes']
pair.modes1 = modes1[:n_modes]
pair.modes2 = modes2[:n_modes]
self.svd_info_list = svd_info_list
self.pairs = pairs
if params['save_svd_info']:
ft.save_pickle(self.svd_info_list, params['svd_file'])
else:
# The first correlation and svd has been skipped.
# This means you already have the modes so you can just load
# them from file.
self.svd_info_list = ft.load_pickle(params['svd_file'])
# Set the svd info to the pairs.
for i in range(0, len(pairs)):
svd_info = self.svd_info_list[i]
pairs[i].vals = svd_info[0]
pairs[i].all_modes1 = svd_info[1]
pairs[i].all_modes2 = svd_info[2]
n_modes = params['modes']
pairs[i].modes1 = svd_info[1][:n_modes]
pairs[i].modes2 = svd_info[2][:n_modes]
self.pairs = pairs
# Subtract foregrounds.
for pair_index in range(0, len(pairs)):
pairs[pair_index].subtract_frequency_modes(
pairs[pair_index].modes1, pairs[pair_index].modes2)
# Save cleaned clean maps, cleaned noises, and modes.
self.save_data(save_maps=params['save_maps'],
save_noises=params['save_noises'],
save_modes=params['save_modes'])
# Finish if this was just first pass.
if params['first_pass_only']:
self.pairs = pairs
return
# Correlate the cleaned maps.
# Here we could calculate the power spectrum instead eventually.
temp_pair_list = []
processes_list = []
for pair_index in range(0, num_map_pairs):
multi = multiprocessing.Process(target=multiproc,
args=([
pairs[pair_index],
params['output_root'],
pair_index, True
]))
processes_list.append(multi)
multi.start()
while True in [multi.is_alive() for multi in processes_list]:
print "processing"
time.sleep(5)
# just to be safe
time.sleep(1)
# ugh, would really rathter use implementation below except multiprocessing
# does not behave.................
#runlist = [(pairs[pair_index],
# params['output_root'],
# pair_index, True) for
# pair_index in range(0, num_map_pairs)]
#pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
#pool.map(multiproc, runlist)
print "Loading map pairs back into program."
file_name = params['output_root']
file_name += "map_pair_for_freq_slices_corr_"
for count in range(0, num_map_pairs):
print "Loading correlation for pair %d" % (count)
pickle_handle = open(file_name + str(count) + ".pkl", "r")
correlate_results = cPickle.load(pickle_handle)
pairs[count].corr = correlate_results[0]
pairs[count].counts = correlate_results[1]
temp_pair_list.append(pairs[count])
pickle_handle.close()
self.pairs = copy.deepcopy(temp_pair_list)
# Get the average correlation and its standard deviation.
corr_list = []
for pair in self.pairs:
corr_list.append(pair.corr)
self.corr_final, self.corr_std = cf.get_corr_and_std_3d(corr_list)
if params['pickle_slices']:
ft.save_pickle(self, self.params['output_root'] + \
'New_Slices_object.pkl')
return
def load_pairs(self):
r"""load the set of map/noise pairs specified by keys handed to the
database. This sets up operations on the quadratic product
Q = map1^T noise_inv1 B noise_inv2 map2
"""
par = self.params
if not par['pairlist']:
if par['calc_diagnal']:
noise_inv_suffix = ";noise_inv"
else:
noise_inv_suffix = ";noise_weight"
(self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'],
par['noise_inv1'],
par['noise_inv2'],
noise_inv_suffix=noise_inv_suffix,
verbose=False,
db_to_use=self.datapath_db)
else:
self.pairlist = par['pairlist']
pairdict = par['pairdict']
for pairitem in self.pairlist:
pdict = pairdict[pairitem]
print "-" * 80
dp.print_dictionary(pdict, sys.stdout,
key_list=['map1', 'noise_inv1',
'map2', 'noise_inv2'])
# map1 & noise_inv1
map1 = algebra.make_vect(algebra.load(pdict['map1']))
if par['simfile1'] is not None:
print "adding %s with multiplier %s" % (par['simfile1'],
par['sim_multiplier'])
sim1 = algebra.make_vect(algebra.load(par['simfile1']))
sim1 *= par['sim_multiplier']
else:
sim1 = algebra.zeros_like(map1)
if not par['no_weights']:
noise_inv1 = wrap_find_weight(pdict['noise_inv1'],
regenerate=par['regenerate_noise_inv'],
calc_diagnal = par['calc_diagnal'])
else:
noise_inv1 = algebra.ones_like(map1)
# map2 & noise_inv2
#if pairitem == 'I_with_E':
if len(self.freq_list2) == 4*len(self.freq_list1):
'''For IQUV case'''
print 'Construct E map using I Q U V'
iquvdict = {}
iquvdict['imap'] = pdict['map2'].replace('_E', '_I')
iquvdict['qmap'] = pdict['map2'].replace('_E', '_Q')
iquvdict['umap'] = pdict['map2'].replace('_E', '_U')
iquvdict['vmap'] = pdict['map2'].replace('_E', '_V')
iquvdict['imap_weight'] = pdict['noise_inv2'].replace('_E', '_I')
iquvdict['qmap_weight'] = pdict['noise_inv2'].replace('_E', '_Q')
iquvdict['umap_weight'] = pdict['noise_inv2'].replace('_E', '_U')
iquvdict['vmap_weight'] = pdict['noise_inv2'].replace('_E', '_V')
map_dict = extend_iquv_map(source_dict=iquvdict)
map2 = map_dict['map']
noise_inv2 = map_dict['weight']
sim2 = copy.deepcopy(sim1)
map_dict = {}
map_dict['imap'] = sim2
map_dict['qmap'] = algebra.zeros_like(sim1)
map_dict['umap'] = algebra.zeros_like(sim1)
map_dict['vmap'] = algebra.zeros_like(sim1)
map_dict = extend_iquv_map(map_dict=map_dict)
sim2 = map_dict['map']
elif len(self.freq_list2) == 3*len(self.freq_list1):
'''For IQU case'''
print 'Construct E map using I Q U'
iquvdict = {}
iquvdict['imap'] = pdict['map2'].replace('_E', '_I')
iquvdict['qmap'] = pdict['map2'].replace('_E', '_Q')
iquvdict['umap'] = pdict['map2'].replace('_E', '_U')
iquvdict['imap_weight'] = pdict['noise_inv2'].replace('_E', '_I')
iquvdict['qmap_weight'] = pdict['noise_inv2'].replace('_E', '_Q')
iquvdict['umap_weight'] = pdict['noise_inv2'].replace('_E', '_U')
map_dict = extend_iqu_map(source_dict=iquvdict)
map2 = map_dict['map']
noise_inv2 = map_dict['weight']
sim2 = copy.deepcopy(sim1)
map_dict = {}
map_dict['imap'] = sim2
map_dict['qmap'] = algebra.zeros_like(sim1)
map_dict['umap'] = algebra.zeros_like(sim1)
map_dict = extend_iqu_map(map_dict=map_dict)
sim2 = map_dict['map']
else:
'''For common case'''
map2 = algebra.make_vect(algebra.load(pdict['map2']))
if par['simfile2'] is not None:
print "adding %s with multiplier %s" % (par['simfile2'],
par['sim_multiplier'])
sim2 = algebra.make_vect(algebra.load(par['simfile2']))
sim2 *= par['sim_multiplier']
else:
sim2 = algebra.zeros_like(map2)
if not par['no_weights']:
noise_inv2 = wrap_find_weight(pdict['noise_inv2'],
regenerate=par['regenerate_noise_inv'],
calc_diagnal = par['calc_diagnal'])
else:
noise_inv2 = algebra.ones_like(map2)
#if self.params['clip_weight_percent'] is not None:
# print "Note: your are clipping the weight maps"
# mask1 = self.define_weightmask(noise_inv1,
# percentile=self.params['clip_weight_percent'])
# mask2 = self.define_weightmask(noise_inv2,
# percentile=self.params['clip_weight_percent'])
# noise_inv1 = self.saturate_weight(noise_inv1, mask1)
# noise_inv2 = self.saturate_weight(noise_inv2, mask2)
pair = map_pair.MapPair(map1 + sim1, map2 + sim2,
noise_inv1, noise_inv2,
self.freq_list1, self.freq_list2)
pair.set_names(pdict['tag1'], pdict['tag2'])
pair.params = self.params
self.pairs[pairitem] = pair
if par['subtract_inputmap_from_sim'] or \
par['subtract_sim_from_inputmap']:
if par['subtract_inputmap_from_sim']:
pair_parallel_track = map_pair.MapPair(map1, map2,
noise_inv1, noise_inv2,
self.freq_list1, self.freq_list2)
if par['subtract_sim_from_inputmap']:
pair_parallel_track = map_pair.MapPair(sim1, sim2,
noise_inv1, noise_inv2,
self.freq_list1, self.freq_list2)
pair_parallel_track.set_names(pdict['tag1'], pdict['tag2'])
pair_parallel_track.params = self.params
self.pairs_parallel_track[pairitem] = pair_parallel_track
def execute(self):
'''Clean the maps of foregrounds, save the results, and get the
autocorrelation.'''
params = self.params
freq_list = sp.array(params['freq_list'], dtype=int)
lags = sp.array(params['lags'])
# Write parameter file.
kiyopy.utils.mkparents(params['output_root'])
parse_ini.write_params(params, params['output_root'] + 'params.ini',
prefix=prefix)
# Get the map data from file as well as the noise inverse.
if len(params['file_middles']) == 1:
fmid_name = params['file_middles'][0]
params['file_middles'] = (fmid_name, fmid_name)
if len(params['file_middles']) >= 2:
# Deal with multiple files.
num_maps = len(params['file_middles'])
maps = []
noise_invs = []
# Load all maps and noises once.
for map_index in range(0, num_maps):
map_file = (params['input_root'] +
params['file_middles'][map_index] +
params['input_end_map'])
print "Loading map %d of %d." % (map_index + 1, num_maps)
map_in = algebra.make_vect(algebra.load(map_file))
maps.append(map_in)
if not params["no_weights"]:
noise_file = (params['input_root'] +
params['file_middles'][map_index] +
params['input_end_noise'])
print "Loading noise %d of %d." % (map_index + 1, num_maps)
noise_inv = algebra.make_mat(
algebra.open_memmap(noise_file, mode='r'))
noise_inv = noise_inv.mat_diag()
else:
noise_inv = algebra.ones_like(map_in)
noise_invs.append(noise_inv)
pairs = []
# Make pairs with deepcopies to not make mutability mistakes.
for map1_index in range(0, num_maps):
for map2_index in range(0, num_maps):
if (map2_index > map1_index):
map1 = copy.deepcopy(maps[map1_index])
map2 = copy.deepcopy(maps[map2_index])
noise_inv1 = copy.deepcopy(noise_invs[map1_index])
noise_inv2 = copy.deepcopy(noise_invs[map2_index])
pair = map_pair.MapPair(map1, map2,
noise_inv1, noise_inv2,
freq_list)
pair.lags = lags
pair.params = params
# Keep track of the names of maps in pairs so
# it knows what to save later.
pair.set_names(params['file_middles'][map1_index],
params['file_middles'][map2_index])
pairs.append(pair)
num_map_pairs = len(pairs)
print "%d map pairs created from %d maps." % (len(pairs), num_maps)
# Hold a reference in self.
self.pairs = pairs
# Get maps/ noise inv ready for running.
if params["convolve"]:
for pair in pairs:
pair.degrade_resolution()
if params['factorizable_noise']:
for pair in pairs:
pair.make_noise_factorizable()
if params['sub_weighted_mean']:
for pair in pairs:
pair.subtract_weighted_mean()
self.pairs = pairs
# Since correlating takes so long, if you already have the svds
# you can skip this first correlation [since that's all it's really
# for and it is the same no matter how many modes you want].
# Note: map_pairs will not have anything saved in 'fore_corr' if you
# skip this correlation.
if not params['skip_fore_corr']:
# Correlate the maps with multiprocessing. Note that the
# correlations are saved to file separately then loaded in
# together because that's (one way) how multiprocessing works.
runlist = [(pairs[pair_index],
params['output_root'],
pair_index, False) for
pair_index in range(0, num_map_pairs)]
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
pool.map(multiproc, runlist)
# Load the correlations and save them to each pair. The pairs that
# got passed to multiproc are not the same ones as ones in
# self.pairs, so this must be done to have actual values.
print "Loading map pairs back into program."
file_name = params['output_root']
file_name += "map_pair_for_freq_slices_fore_corr_"
fore_pairs = []
for count in range(0, num_map_pairs):
print "Loading correlation for pair %d" % (count)
pickle_handle = open(file_name + str(count) + ".pkl", "r")
correlate_results = cPickle.load(pickle_handle)
pairs[count].fore_corr = correlate_results[0]
pairs[count].fore_counts = correlate_results[1]
fore_pairs.append(pairs[count])
pickle_handle.close()
self.fore_pairs = copy.deepcopy(fore_pairs)
# With this, you do not need fore_pairs anymore.
self.pairs = copy.deepcopy(fore_pairs)
pairs = self.pairs
# Get foregrounds.
# svd_info_list keeps track of all of the modes of all maps in
# all pairs. This means if you want to subract a different number
# of modes for the same maps/noises/frequencies, you have the modes
# already saved and do not need to run the first correlation again.
svd_info_list = []
for pair in pairs:
vals, modes1, modes2 = cf.get_freq_svd_modes(pair.fore_corr,
len(freq_list))
pair.vals = vals
# Save ALL of the modes for reference.
pair.all_modes1 = modes1
pair.all_modes2 = modes2
svd_info = (vals, modes1, modes2)
svd_info_list.append(svd_info)
# Save only the modes you want to subtract.
n_modes = params['modes']
pair.modes1 = modes1[:n_modes]
pair.modes2 = modes2[:n_modes]
self.svd_info_list = svd_info_list
self.pairs = pairs
if params['save_svd_info']:
io_wrap.save_pickle(self.svd_info_list, params['svd_file'])
else:
# The first correlation and svd has been skipped.
# This means you already have the modes so you can just load
# them from file.
self.svd_info_list = io_wrap.load_pickle(params['svd_file'])
# Set the svd info to the pairs.
for i in range(0, len(pairs)):
svd_info = self.svd_info_list[i]
pairs[i].vals = svd_info[0]
pairs[i].all_modes1 = svd_info[1]
pairs[i].all_modes2 = svd_info[2]
n_modes = params['modes']
pairs[i].modes1 = svd_info[1][:n_modes]
pairs[i].modes2 = svd_info[2][:n_modes]
self.pairs = pairs
# Subtract foregrounds.
for pair_index in range(0, len(pairs)):
pairs[pair_index].subtract_frequency_modes(pairs[pair_index].modes1,
pairs[pair_index].modes2)
# Save cleaned clean maps, cleaned noises, and modes.
self.save_data(save_maps=params['save_maps'],
save_noises=params['save_noises'],
save_modes=params['save_modes'])
# Finish if this was just first pass.
if params['first_pass_only']:
self.pairs = pairs
return
# Correlate the cleaned maps.
# Here we could calculate the power spectrum instead eventually.
runlist = [(pairs[pair_index],
params['output_root'],
pair_index, True) for
pair_index in range(0, num_map_pairs)]
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
pool.map(multiproc, runlist)
print "Loading map pairs back into program."
file_name = params['output_root']
file_name += "map_pair_for_freq_slices_corr_"
temp_pair_list = []
for count in range(0, num_map_pairs):
print "Loading correlation for pair %d" % (count)
pickle_handle = open(file_name + str(count) + ".pkl", "r")
correlate_results = cPickle.load(pickle_handle)
pairs[count].corr = correlate_results[0]
pairs[count].counts = correlate_results[1]
temp_pair_list.append(pairs[count])
pickle_handle.close()
self.pairs = copy.deepcopy(temp_pair_list)
# Get the average correlation and its standard deviation.
corr_list = []
for pair in self.pairs:
corr_list.append(pair.corr)
self.corr_final, self.corr_std = cf.get_corr_and_std_3d(corr_list)
if params['pickle_slices']:
pickle_slices(self)
return
def load_pairs(self):
r"""load the set of map/noise pairs specified by keys handed to the
database. This sets up operations on the quadratic product
Q = map1^T noise_inv1 B noise_inv2 map2
"""
par = self.params
(self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'],
par['noise_inv1'],
par['noise_inv2'],
noise_inv_suffix=";noise_weight",
verbose=False,
tack_on=self.tack_on_input,
db_to_use=self.datapath_db)
for pairitem in self.pairlist:
pdict = pairdict[pairitem]
print "-" * 80
dp.print_dictionary(pdict, sys.stdout,
key_list=['map1', 'noise_inv1',
'map2', 'noise_inv2'])
map1 = algebra.make_vect(algebra.load(pdict['map1']))
map2 = algebra.make_vect(algebra.load(pdict['map2']))
if par['simfile'] is not None:
print "adding %s with multiplier %s" % (par['simfile'],
par['sim_multiplier'])
sim = algebra.make_vect(algebra.load(par['simfile']))
sim *= par['sim_multiplier']
print sim.shape, map1.shape
else:
sim = algebra.zeros_like(map1)
if not par['no_weights']:
noise_inv1 = wrap_find_weight(pdict['noise_inv1'],
regenerate=par['regenerate_noise_inv'])
noise_inv2 = wrap_find_weight(pdict['noise_inv2'],
regenerate=par['regenerate_noise_inv'])
else:
noise_inv1 = algebra.ones_like(map1)
noise_inv2 = algebra.ones_like(map2)
pair = map_pair.MapPair(map1 + sim, map2 + sim,
noise_inv1, noise_inv2,
self.freq_list)
pair.set_names(pdict['tag1'], pdict['tag2'])
pair.params = self.params
self.pairs[pairitem] = pair
if par['subtract_inputmap_from_sim'] or \
par['subtract_sim_from_inputmap']:
if par['subtract_inputmap_from_sim']:
pair_parallel_track = map_pair.MapPair(map1, map2,
noise_inv1, noise_inv2,
self.freq_list)
if par['subtract_sim_from_inputmap']:
pair_parallel_track = map_pair.MapPair(sim, sim,
noise_inv1, noise_inv2,
self.freq_list)
pair_parallel_track.set_names(pdict['tag1'], pdict['tag2'])
pair_parallel_track.params = self.params
self.pairs_parallel_track[pairitem] = pair_parallel_track
def calculate_mixing(weight_file1,
weight_file2,
bins,
xspec_fileout,
mixing_fileout,
unitless=False,
refinement=2,
pad=5,
order=1,
window='blackman',
zero_pad=False,
identity_test=False):
print "loading the weights and converting to physical coordinates"
weight1_obs = algebra.make_vect(algebra.load(weight_file1))
weight1 = bh.repackage_kiyo(
pg.physical_grid(weight1_obs,
refinement=refinement,
pad=pad,
order=order))
weight2_obs = algebra.make_vect(algebra.load(weight_file2))
weight2 = bh.repackage_kiyo(
pg.physical_grid(weight2_obs,
refinement=refinement,
pad=pad,
order=order))
if window:
window_function = fftutil.window_nd(weight1.shape, name=window)
weight1 *= window_function
weight2 *= window_function
print "calculating the cross-power of the spatial weighting functions"
arr1 = algebra.ones_like(weight1)
arr2 = algebra.ones_like(weight2)
# no window applied here (applied above)
xspec = pe.cross_power_est(weight1,
weight2,
arr1,
arr2,
window=None,
nonorm=True)
# for each point in the cube, find |k|, k_perp, k_parallel
# TODO: speed this up by using one direct numpy call (not limiting)
k_mag_arr = binning.radius_array(xspec)
k_perp_arr = binning.radius_array(xspec, zero_axes=[0])
k_parallel_arr = binning.radius_array(xspec, zero_axes=[1, 2])
if unitless:
xspec = pe.make_unitless(xspec, radius_arr=k_mag_arr)
# NOTE: assuming lowest k bin has only one point in 3D k-space
# could make this floor of dimensions divided by 2 also
center_3d = np.transpose(np.transpose(np.where(k_mag_arr == 0.))[0])
# In the estimator, we devide by 1/sum(w1 * w2) to get most of the effect
# of the weighing. The mixing matrix here can be thought of as a correction
# that that diagonal-only estimate.
leakage_ratio = xspec[center_3d[0], center_3d[1], center_3d[2]] / \
np.sum(weight1 * weight2)
print "power leakage ratio: %10.5g" % leakage_ratio
xspec /= np.sum(weight1 * weight2)
print "partitioning the 3D kspace up into the 2D k bins"
(kflat, ret_indices) = bin_indices_2d(k_perp_arr, k_parallel_arr, bins,
bins)
# perform a test where the window function is a delta function at the
# origin so that the mixing matrix is identity
if identity_test:
xspec = algebra.zeros_like(xspec)
xspec[center_3d[0], center_3d[1], center_3d[2]] = 1.
# now save the window cross-power for downstream pooled users
algebra.save(xspec_fileout, xspec)
runlist = []
for bin_index in range(kflat.shape[0]):
bin_3d = ret_indices[repr(bin_index)]
if bin_3d is not None:
runlist.append((xspec_fileout, bin_index, bins, bin_3d, center_3d))
pool = multiprocessing.Pool(processes=(multiprocessing.cpu_count() - 4))
# the longest runs get pushed to the end; randomize for better job packing
random.shuffle(runlist)
results = pool.map(sum_window, runlist)
#gnuplot_single_slice(runlist[0]) # for troubleshooting
# now save the results for post-processing
params = {
"unitless": unitless,
"refinement": refinement,
"pad": pad,
"order": order,
"window": window,
"zero_pad": zero_pad,
"identity_test": identity_test,
"weight_file1": weight_file1,
"weight_file2": weight_file2,
"bins": bins
}
outshelve = shelve.open(mixing_fileout, "n")
outshelve["params"] = params # parameters for this run
outshelve["weight1"] = weight1 # weight map 1
outshelve["weight2"] = weight2 # weight map 2
outshelve["xspec"] = xspec # copy of the weight spectra
outshelve["kflat"] = kflat # 2D k bin vector
outshelve["bins_3d"] = ret_indices # indices to k3d for a 2d k bin
outshelve["results"] = results # mixing matrix columns
outshelve.close()
def save_data(self, n_modes):
prodmap_list = []
weight_list = []
n_modes = "%dmodes" % n_modes
for pairitem in self.pairlist:
pair = self.pairs[pairitem]
(tag1, tag2) = (pair.map1_name, pair.map2_name)
clnoise = "cleaned_noise_inv"
map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag1, clnoise, tag2, n_modes)
modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
#if pair.map1.shape == pair.map2.shape:
map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag2, clnoise, tag1, n_modes)
modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
if self.params['subtract_inputmap_from_sim'] or \
self.params['subtract_sim_from_inputmap']:
map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1
map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2
elif self.params['subtract_realmap_from_sim']:
if not os.path.exists(self.params['realmap_dir']):
print "Error: Real map directory does not exists"
exit()
else:
realmap_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy"%\
(self.params['realmap_dir'], tag1, tag2, n_modes)
realmap = algebra.make_vect(algebra.load(realmap_file))
print "Subtract realmap from result"
map1 = copy.deepcopy(pair.map1) - realmap
map2 = copy.deepcopy(pair.map2)
if map2.shape == map1.shape:
map2 -= realmap
else:
map1 = copy.deepcopy(pair.map1)
map2 = copy.deepcopy(pair.map2)
prodmap_list.append(map1 * pair.noise_inv1)
prodmap_list.append(map2 * pair.noise_inv2)
weight_list.append(pair.noise_inv1)
weight_list.append(pair.noise_inv2)
if self.params['save_section']:
algebra.save(map1_file, map1)
algebra.save(noise_inv1_file, pair.noise_inv1)
algebra.save(modes1_file, pair.left_modes)
if pair.map1.shape == pair.map2.shape:
algebra.save(map2_file, map2)
algebra.save(noise_inv2_file, pair.noise_inv2)
algebra.save(modes2_file, pair.right_modes)
#if map2.shape[0] == 3*map1.shape[0]:
# #source_dict = {}
# #source_dict['map'] = map2_file
# #source_dict['weight'] = noise_inv2_file
# map_dict = {}
# map_dict['map'] = map2
# map_dict['weight'] = pair.noise_inv2
# target_dict = {}
# target_dict['imap'] = map2_file.replace('_'+tag2, '_'+tag2+'_I')
# target_dict['qmap'] = map2_file.replace('_'+tag2, '_'+tag2+'_Q')
# target_dict['umap'] = map2_file.replace('_'+tag2, '_'+tag2+'_U')
# target_dict['imap_weight'] =\
# noise_inv2_file.replace('_'+tag2, '_'+tag2+'_I')
# target_dict['qmap_weight'] =\
# noise_inv2_file.replace('_'+tag2, '_'+tag2+'_Q')
# target_dict['umap_weight'] =\
# noise_inv2_file.replace('_'+tag2, '_'+tag2+'_U')
# divide_iqu_map(map_dict=map_dict, target_dict=target_dict)
if map1.shape != map2.shape:
print "Shape of map1 and map2 are different, can not get combined map."
else:
cumulative_product = algebra.zeros_like(prodmap_list[0])
cumulative_weight = algebra.zeros_like(prodmap_list[0])
for mapind in range(0, len(prodmap_list)):
cumulative_product += prodmap_list[mapind]
cumulative_weight += weight_list[mapind]
algebra.compressed_array_summary(cumulative_weight, "weight map")
algebra.compressed_array_summary(cumulative_product, "product map")
cumulative_weight[cumulative_weight < 1.e-20] = 0.
cumulative_product[cumulative_weight < 1.e-20] = 0.
newmap = cumulative_product / cumulative_weight
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.
cumulative_product[nan_array] = 0.
cumulative_weight[nan_array] = 0.
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.
cumulative_product[inf_array] = 0.
cumulative_weight[inf_array] = 0.
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(cumulative_product,"final map * weight")
algebra.compressed_array_summary(cumulative_weight, "final weight map")
combined = "combined_clean"
combined_map_file = "%s/%s_map_%s.npy" % \
(self.output_root, combined, n_modes)
combined_weight_file = "%s/%s_weight_%s.npy" % \
(self.output_root, combined, n_modes)
combined_product_file = "%s/%s_product_%s.npy" % \
(self.output_root, combined, n_modes)
combined_ones_file = "%s/%s_ones_%s.npy" % \
(self.output_root, combined, n_modes)
algebra.save(combined_map_file, newmap)
algebra.save(combined_product_file, cumulative_product)
algebra.save(combined_weight_file, cumulative_weight)
algebra.save(combined_ones_file, algebra.ones_like(newmap))
def save_data(self, n_modes):
prodmap_list = []
weight_list = []
n_modes = "%dmodes" % n_modes
for pairitem in self.pairlist:
pair = self.pairs[pairitem]
(tag1, tag2) = (pair.map1_name, pair.map2_name)
clnoise = "cleaned_noise_inv"
map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag1, clnoise, tag2, n_modes)
noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \
(self.output_root, tag2, clnoise, tag1, n_modes)
modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag1, tag2, n_modes)
modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \
(self.output_root, tag2, tag1, n_modes)
if self.params['subtract_inputmap_from_sim'] or \
self.params['subtract_sim_from_inputmap']:
map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1
map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2
else:
map1 = copy.deepcopy(pair.map1)
map2 = copy.deepcopy(pair.map2)
prodmap_list.append(map1 * pair.noise_inv1)
prodmap_list.append(map2 * pair.noise_inv2)
weight_list.append(pair.noise_inv1)
weight_list.append(pair.noise_inv2)
algebra.save(map1_file, map1)
algebra.save(map2_file, map2)
algebra.save(noise_inv1_file, pair.noise_inv1)
algebra.save(noise_inv2_file, pair.noise_inv2)
algebra.save(modes1_file, pair.left_modes)
algebra.save(modes2_file, pair.right_modes)
cumulative_product = algebra.zeros_like(prodmap_list[0])
cumulative_weight = algebra.zeros_like(prodmap_list[0])
for mapind in range(0, len(prodmap_list)):
cumulative_product += prodmap_list[mapind]
cumulative_weight += weight_list[mapind]
algebra.compressed_array_summary(cumulative_weight, "weight map")
algebra.compressed_array_summary(cumulative_product, "product map")
cumulative_weight[cumulative_weight < 1.e-20] = 0.
cumulative_product[cumulative_weight < 1.e-20] = 0.
newmap = cumulative_product / cumulative_weight
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.
cumulative_product[nan_array] = 0.
cumulative_weight[nan_array] = 0.
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.
cumulative_product[inf_array] = 0.
cumulative_weight[inf_array] = 0.
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(cumulative_product,
"final map * weight")
algebra.compressed_array_summary(cumulative_weight, "final weight map")
combined = "combined_clean"
combined_map_file = "%s/%s_map_%s.npy" % \
(self.output_root, combined, n_modes)
combined_weight_file = "%s/%s_weight_%s.npy" % \
(self.output_root, combined, n_modes)
combined_product_file = "%s/%s_product_%s.npy" % \
(self.output_root, combined, n_modes)
combined_ones_file = "%s/%s_ones_%s.npy" % \
(self.output_root, combined, n_modes)
algebra.save(combined_map_file, newmap)
algebra.save(combined_product_file, cumulative_product)
algebra.save(combined_weight_file, cumulative_weight)
algebra.save(combined_ones_file, algebra.ones_like(newmap))
def combine_maps_driver(inputmap_dict, inputweight_dict, output_dict,
fullcov=False, datapath_db=None):
r"""Combine a list of weights, maps specified by their database keys
"""
if datapath_db is None:
datapath_db = data_paths.DataPath()
signal_list = []
weight_list = []
for mapkey in inputmap_dict:
signalfile = inputmap_dict[mapkey]
weightfile = inputweight_dict[mapkey]
print "loading pair: %s %s" % (signalfile, weightfile)
signal_list.append(algebra.make_vect(algebra.load(signalfile)))
if fullcov:
raw_weight = algebra.make_mat(
algebra.open_memmap(weightfile))
raw_weight = raw_weight.mat_diag()
else:
raw_weight = algebra.make_vect(algebra.load(weightfile))
# zero out any messy stuff
raw_weight[raw_weight < 1.e-20] = 0.
raw_weight[np.isnan(raw_weight)] = 0.
raw_weight[np.isinf(raw_weight)] = 0.
weight_list.append(raw_weight)
prodmap = []
for mapind in range(0, len(signal_list)):
prodmap.append(signal_list[mapind] * weight_list[mapind])
print "CHECK THESE: %d %d %d" % (len(signal_list), len(weight_list),
len(prodmap))
cumulative_product = algebra.zeros_like(prodmap[0])
cumulative_weight = algebra.zeros_like(prodmap[0])
for mapind in range(0, len(signal_list)):
cumulative_product += prodmap[mapind]
cumulative_weight += weight_list[mapind]
algebra.compressed_array_summary(cumulative_weight, "weight map")
algebra.compressed_array_summary(cumulative_product, "product map")
newmap = cumulative_product / cumulative_weight
cumulative_weight[cumulative_weight < 1.e-20] = 0.
cumulative_product[cumulative_weight < 1.e-20] = 0.
# if the new map is nan or inf, set it and the wieghts to zero
nan_array = np.isnan(newmap)
newmap[nan_array] = 0.
cumulative_product[nan_array] = 0.
cumulative_weight[nan_array] = 0.
inf_array = np.isinf(newmap)
newmap[inf_array] = 0.
cumulative_product[inf_array] = 0.
cumulative_weight[inf_array] = 0.
algebra.compressed_array_summary(newmap, "new map")
algebra.compressed_array_summary(cumulative_product, "final map * weight")
algebra.compressed_array_summary(cumulative_weight, "final weight map")
print output_dict
algebra.save(output_dict['map'], newmap)
algebra.save(output_dict['product'], cumulative_product)
algebra.save(output_dict['weight'], cumulative_weight)
algebra.save(output_dict['ones'], algebra.ones_like(newmap))
