def calculate_xspec_file(cube1_file, cube2_file, bins, weight1_file=None, weight2_file=None, truncate=False, window="blackman", return_3d=False, unitless=True): cube1 = algebra.make_vect(algebra.load(cube1_file)) cube2 = algebra.make_vect(algebra.load(cube2_file)) if weight1_file is None: weight1 = algebra.ones_like(cube1) else: weight1 = algebra.make_vect(algebra.load(weight1_file)) if weight2_file is None: weight2 = algebra.ones_like(cube2) else: weight2 = algebra.make_vect(algebra.load(weight2_file)) print cube1.shape, cube2.shape, weight1.shape, weight2.shape return calculate_xspec(cube1, cube2, weight1, weight2, bins=bins, window=window, unitless=unitless, truncate=truncate, return_3d=return_3d)
def __init__(self, map1, map2, noise_inv1, noise_inv2, freq, input_filenames=False, conv_factor=1.1): r""" arguments: map1, map2, noise_inv1, noise_inv2, freq conv_factor is the factor by which to multiply the largest beam in the convolution to a common resolution """ if input_filenames: self.map1 = algebra.make_vect(algebra.load(map1)) self.map2 = algebra.make_vect(algebra.load(map2)) if noise_inv1: print "loading noise1 file: " + noise_inv1 self.noise_inv1 = algebra.make_vect(algebra.load(noise_inv1)) else: print "WARNING: map1 has unity weight; no file given" self.noise_inv1 = algebra.ones_like(self.map1) if noise_inv2: print "loading noise2 file: " + noise_inv2 self.noise_inv2 = algebra.make_vect(algebra.load(noise_inv2)) else: print "WARNING: map2 has unity weight; no file given" self.noise_inv2 = algebra.ones_like(self.map2) else: self.map1 = map1 self.map2 = map2 self.noise_inv1 = noise_inv1 self.noise_inv2 = noise_inv2 self.freq = freq self.conv_factor = conv_factor # maps in physical coordinates (derived) self.phys_map1 = None self.phys_map2 = None self.phys_noise_inv1 = None self.phys_noise_inv2 = None # give infinite noise to masked bands self.sanitize() # Set attributes. self.left_modes = 0 self.right_modes = 0 # For saving, to keep track of each mapname. self.map1_name = '' self.map2_name = '' # Which section [A, B, C, D...] the maps is from. self.map1_code = '' self.map2_code = ''
def load_pairs(self, regenerate=True): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params (self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'], par['noise_inv1'], par['noise_inv2'], verbose=False) for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary( pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) map1 = algebra.make_vect(algebra.load(pdict['map1'])) map2 = algebra.make_vect(algebra.load(pdict['map2'])) sim = algebra.make_vect(algebra.load(par['simfile'])) if not par['no_weights']: noise_inv1 = self.process_noise_inv(pdict['noise_inv1'], regenerate=regenerate) noise_inv2 = self.process_noise_inv(pdict['noise_inv2'], regenerate=regenerate) else: noise_inv1 = algebra.ones_like(map1) noise_inv2 = algebra.ones_like(map2) pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1, noise_inv2, self.freq_list) pair.set_names(pdict['tag1'], pdict['tag2']) pair.lags = self.lags pair.params = self.params self.pairs[pairitem] = pair pair_nosim = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list) pair_nosim.set_names(pdict['tag1'], pdict['tag2']) pair_nosim.lags = self.lags pair_nosim.params = self.params self.pairs_nosim[pairitem] = pair_nosim
def test_with_random(unitless=True): """Test the power spectral estimator using a random noise cube""" delta = 1.33333 cube1 = algebra.make_vect(np.random.normal(0, 1, size=(257, 124, 68))) info = { 'axes': ["freq", "ra", "dec"], 'type': 'vect', 'freq_delta': delta / 3.78, 'freq_centre': 0., 'ra_delta': delta / 1.63, 'ra_centre': 0., 'dec_delta': delta, 'dec_centre': 0. } cube1.info = info cube2 = copy.deepcopy(cube1) weight1 = algebra.ones_like(cube1) weight2 = algebra.ones_like(cube2) bin_left, bin_center, bin_right, counts_histo, binavg = \ calculate_xspec(cube1, cube2, weight1, weight2, window="blackman", truncate=False, nbins=40, unitless=unitless, logbins=True) if unitless: pwrspec_input = bin_center**3. / 2. / math.pi / math.pi else: pwrspec_input = np.ones_like(bin_center) volume = 1. for axis_name in cube1.axes: axis_vector = cube1.get_axis(axis_name) volume *= abs(axis_vector[1] - axis_vector[0]) pwrspec_input *= volume for specdata in zip(bin_left, bin_center, bin_right, counts_histo, binavg, pwrspec_input): print("%10.15g " * 6) % specdata
def test_with_random(unitless=True): """Test the power spectral estimator using a random noise cube""" delta = 1.33333 cube1 = algebra.make_vect(np.random.normal(0, 1, size=(257, 124, 68))) info = {'axes': ["freq", "ra", "dec"], 'type': 'vect', 'freq_delta': delta / 3.78, 'freq_centre': 0., 'ra_delta': delta / 1.63, 'ra_centre': 0., 'dec_delta': delta, 'dec_centre': 0.} cube1.info = info cube2 = copy.deepcopy(cube1) weight1 = algebra.ones_like(cube1) weight2 = algebra.ones_like(cube2) bin_left, bin_center, bin_right, counts_histo, binavg = \ calculate_xspec(cube1, cube2, weight1, weight2, window="blackman", truncate=False, nbins=40, unitless=unitless, logbins=True) if unitless: pwrspec_input = bin_center ** 3. / 2. / math.pi / math.pi else: pwrspec_input = np.ones_like(bin_center) volume = 1. for axis_name in cube1.axes: axis_vector = cube1.get_axis(axis_name) volume *= abs(axis_vector[1] - axis_vector[0]) pwrspec_input *= volume for specdata in zip(bin_left, bin_center, bin_right, counts_histo, binavg, pwrspec_input): print ("%10.15g " * 6) % specdata
def load_pairs(self, regenerate=True): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params (self.pairlist, pairdict) = dp.cross_maps( par["map1"], par["map2"], par["noise_inv1"], par["noise_inv2"], verbose=False ) for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary(pdict, sys.stdout, key_list=["map1", "noise_inv1", "map2", "noise_inv2"]) map1 = algebra.make_vect(algebra.load(pdict["map1"])) map2 = algebra.make_vect(algebra.load(pdict["map2"])) sim = algebra.make_vect(algebra.load(par["simfile"])) if not par["no_weights"]: noise_inv1 = self.process_noise_inv(pdict["noise_inv1"], regenerate=regenerate) noise_inv2 = self.process_noise_inv(pdict["noise_inv2"], regenerate=regenerate) else: noise_inv1 = algebra.ones_like(map1) noise_inv2 = algebra.ones_like(map2) pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1, noise_inv2, self.freq_list) pair.set_names(pdict["tag1"], pdict["tag2"]) pair.lags = self.lags pair.params = self.params self.pairs[pairitem] = pair pair_nosim = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list) pair_nosim.set_names(pdict["tag1"], pdict["tag2"]) pair_nosim.lags = self.lags pair_nosim.params = self.params self.pairs_nosim[pairitem] = pair_nosim
def extend_iqu_map(source_dict=None, target_dict=None, map_dict=None): if source_dict != None: imap = algebra.make_vect(algebra.load(source_dict['imap'])) qmap = algebra.make_vect(algebra.load(source_dict['qmap'])) umap = algebra.make_vect(algebra.load(source_dict['umap'])) if source_dict.has_key('imap_weight'): imap_weight = algebra.make_vect(algebra.load(source_dict['imap_weight'])) qmap_weight = algebra.make_vect(algebra.load(source_dict['qmap_weight'])) umap_weight = algebra.make_vect(algebra.load(source_dict['umap_weight'])) elif source_dict.has_key('imap_inv'): imap_weight, info = find_weight_re_diagnal(source_dict['imap_inv']) qmap_weight, info = find_weight_re_diagnal(source_dict['qmap_inv']) umap_weight, info = find_weight_re_diagnal(source_dict['umap_inv']) else: print 'Warning: no weight' imap_weight = algebra.ones_like(imap) qmap_weight = algebra.ones_like(imap) umap_weight = algebra.ones_like(imap) elif map_dict != None: imap = map_dict['imap'] qmap = map_dict['qmap'] umap = map_dict['umap'] if 'imap_weight' in map_dict.keys(): imap_weight = map_dict['imap_weight'] qmap_weight = map_dict['qmap_weight'] umap_weight = map_dict['umap_weight'] else: print 'Warning: no weight' imap_weight = algebra.ones_like(imap) qmap_weight = algebra.ones_like(imap) umap_weight = algebra.ones_like(imap) else: print "Error: Can not find I Q U maps" exit() iqu = algebra.info_array(imap.tolist() + qmap.tolist() + umap.tolist()) iqu = algebra.make_vect(iqu) iqu.info = imap.info iqu.copy_axis_info(imap) iqu_weight = algebra.info_array(imap_weight.tolist() + qmap_weight.tolist() + umap_weight.tolist()) iqu_weight = algebra.make_vect(iqu_weight) iqu_weight.info = imap_weight.info iqu_weight.copy_axis_info(imap_weight) if target_dict != None: algebra.save(target_dict['map'], iqu) algebra.save(target_dict['weight'], iqu_weight) else: map_dict = {} map_dict['map'] = iqu map_dict['weight'] = iqu_weight return map_dict
def save_data(self, n_modes): prodmap_list = [] weight_list = [] n_modes = "%dmodes" % n_modes for pairitem in self.pairlist: pair = self.pairs[pairitem] (tag1, tag2) = (pair.map1_name, pair.map2_name) clnoise = "cleaned_noise_inv" map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag1, clnoise, tag2, n_modes) noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag2, clnoise, tag1, n_modes) modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) if self.params['subtract_inputmap_from_sim'] or \ self.params['subtract_sim_from_inputmap']: map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1 map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2 else: map1 = copy.deepcopy(pair.map1) map2 = copy.deepcopy(pair.map2) prodmap_list.append(map1 * pair.noise_inv1) prodmap_list.append(map2 * pair.noise_inv2) weight_list.append(pair.noise_inv1) weight_list.append(pair.noise_inv2) algebra.save(map1_file, map1) algebra.save(map2_file, map2) algebra.save(noise_inv1_file, pair.noise_inv1) algebra.save(noise_inv2_file, pair.noise_inv2) algebra.save(modes1_file, pair.left_modes) algebra.save(modes2_file, pair.right_modes) cumulative_product = algebra.zeros_like(prodmap_list[0]) cumulative_weight = algebra.zeros_like(prodmap_list[0]) for mapind in range(0, len(prodmap_list)): cumulative_product += prodmap_list[mapind] cumulative_weight += weight_list[mapind] algebra.compressed_array_summary(cumulative_weight, "weight map") algebra.compressed_array_summary(cumulative_product, "product map") cumulative_weight[cumulative_weight < 1.e-20] = 0. cumulative_product[cumulative_weight < 1.e-20] = 0. newmap = cumulative_product / cumulative_weight # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. cumulative_product[nan_array] = 0. cumulative_weight[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. cumulative_product[inf_array] = 0. cumulative_weight[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(cumulative_product, "final map * weight") algebra.compressed_array_summary(cumulative_weight, "final weight map") combined = "combined_clean" combined_map_file = "%s/%s_map_%s.npy" % \ (self.output_root, combined, n_modes) combined_weight_file = "%s/%s_weight_%s.npy" % \ (self.output_root, combined, n_modes) combined_product_file = "%s/%s_product_%s.npy" % \ (self.output_root, combined, n_modes) combined_ones_file = "%s/%s_ones_%s.npy" % \ (self.output_root, combined, n_modes) algebra.save(combined_map_file, newmap) algebra.save(combined_product_file, cumulative_product) algebra.save(combined_weight_file, cumulative_weight) algebra.save(combined_ones_file, algebra.ones_like(newmap))
def calculate_mixing(weight_file1, weight_file2, bins, xspec_fileout, mixing_fileout, unitless=False, refinement=2, pad=5, order=1, window='blackman', zero_pad=False, identity_test=False): print "loading the weights and converting to physical coordinates" weight1_obs = algebra.make_vect(algebra.load(weight_file1)) weight1 = bh.repackage_kiyo(pg.physical_grid( weight1_obs, refinement=refinement, pad=pad, order=order)) weight2_obs = algebra.make_vect(algebra.load(weight_file2)) weight2 = bh.repackage_kiyo(pg.physical_grid( weight2_obs, refinement=refinement, pad=pad, order=order)) if window: window_function = fftutil.window_nd(weight1.shape, name=window) weight1 *= window_function weight2 *= window_function print "calculating the cross-power of the spatial weighting functions" arr1 = algebra.ones_like(weight1) arr2 = algebra.ones_like(weight2) # no window applied here (applied above) xspec = pe.cross_power_est(weight1, weight2, arr1, arr2, window=None, nonorm=True) # for each point in the cube, find |k|, k_perp, k_parallel # TODO: speed this up by using one direct numpy call (not limiting) k_mag_arr = binning.radius_array(xspec) k_perp_arr = binning.radius_array(xspec, zero_axes=[0]) k_parallel_arr = binning.radius_array(xspec, zero_axes=[1, 2]) if unitless: xspec = pe.make_unitless(xspec, radius_arr=k_mag_arr) # NOTE: assuming lowest k bin has only one point in 3D k-space # could make this floor of dimensions divided by 2 also center_3d = np.transpose(np.transpose(np.where(k_mag_arr == 0.))[0]) # In the estimator, we devide by 1/sum(w1 * w2) to get most of the effect # of the weighing. The mixing matrix here can be thought of as a correction # that that diagonal-only estimate. leakage_ratio = xspec[center_3d[0], center_3d[1], center_3d[2]] / \ np.sum(weight1 * weight2) print "power leakage ratio: %10.5g" % leakage_ratio xspec /= np.sum(weight1 * weight2) print "partitioning the 3D kspace up into the 2D k bins" (kflat, ret_indices) = bin_indices_2d(k_perp_arr, k_parallel_arr, bins, bins) # perform a test where the window function is a delta function at the # origin so that the mixing matrix is identity if identity_test: xspec = algebra.zeros_like(xspec) xspec[center_3d[0], center_3d[1], center_3d[2]] = 1. # now save the window cross-power for downstream pooled users algebra.save(xspec_fileout, xspec) runlist = [] for bin_index in range(kflat.shape[0]): bin_3d = ret_indices[repr(bin_index)] if bin_3d is not None: runlist.append((xspec_fileout, bin_index, bins, bin_3d, center_3d)) pool = multiprocessing.Pool(processes=(multiprocessing.cpu_count() - 4)) # the longest runs get pushed to the end; randomize for better job packing random.shuffle(runlist) results = pool.map(sum_window, runlist) #gnuplot_single_slice(runlist[0]) # for troubleshooting # now save the results for post-processing params = {"unitless": unitless, "refinement": refinement, "pad": pad, "order": order, "window": window, "zero_pad": zero_pad, "identity_test": identity_test, "weight_file1": weight_file1, "weight_file2": weight_file2, "bins": bins} outshelve = shelve.open(mixing_fileout, "n") outshelve["params"] = params # parameters for this run outshelve["weight1"] = weight1 # weight map 1 outshelve["weight2"] = weight2 # weight map 2 outshelve["xspec"] = xspec # copy of the weight spectra outshelve["kflat"] = kflat # 2D k bin vector outshelve["bins_3d"] = ret_indices # indices to k3d for a 2d k bin outshelve["results"] = results # mixing matrix columns outshelve.close()
def execute(self): '''Clean the maps of foregrounds, save the results, and get the autocorrelation.''' params = self.params freq_list = sp.array(params['freq_list'], dtype=int) lags = sp.array(params['lags']) # Write parameter file. kiyopy.utils.mkparents(params['output_root']) parse_ini.write_params(params, params['output_root'] + 'params.ini', prefix=prefix) # Get the map data from file as well as the noise inverse. if len(params['file_middles']) == 1: fmid_name = params['file_middles'][0] params['file_middles'] = (fmid_name, fmid_name) if len(params['file_middles']) >= 2: # Deal with multiple files. num_maps = len(params['file_middles']) maps = [] noise_invs = [] # Load all maps and noises once. for map_index in range(0, num_maps): map_file = (params['input_root'] + params['file_middles'][map_index] + params['input_end_map']) print "Loading map %d of %d." % (map_index + 1, num_maps) map_in = algebra.make_vect(algebra.load(map_file)) maps.append(map_in) if not params["no_weights"]: noise_file = (params['input_root'] + params['file_middles'][map_index] + params['input_end_noise']) print "Loading noise %d of %d." % (map_index + 1, num_maps) noise_inv = algebra.make_mat( algebra.open_memmap(noise_file, mode='r')) noise_inv = noise_inv.mat_diag() else: noise_inv = algebra.ones_like(map_in) noise_invs.append(noise_inv) pairs = [] # Make pairs with deepcopies to not make mutability mistakes. for map1_index in range(0, num_maps): for map2_index in range(0, num_maps): if (map2_index > map1_index): map1 = copy.deepcopy(maps[map1_index]) map2 = copy.deepcopy(maps[map2_index]) noise_inv1 = copy.deepcopy(noise_invs[map1_index]) noise_inv2 = copy.deepcopy(noise_invs[map2_index]) pair = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, freq_list) pair.lags = lags pair.params = params # Keep track of the names of maps in pairs so # it knows what to save later. pair.set_names(params['file_middles'][map1_index], params['file_middles'][map2_index]) pairs.append(pair) num_map_pairs = len(pairs) print "%d map pairs created from %d maps." % (len(pairs), num_maps) # Hold a reference in self. self.pairs = pairs # Get maps/ noise inv ready for running. if params["convolve"]: for pair in pairs: pair.degrade_resolution() if params['factorizable_noise']: for pair in pairs: pair.make_noise_factorizable() if params['sub_weighted_mean']: for pair in pairs: pair.subtract_weighted_mean() self.pairs = pairs # Since correlating takes so long, if you already have the svds # you can skip this first correlation [since that's all it's really # for and it is the same no matter how many modes you want]. # Note: map_pairs will not have anything saved in 'fore_corr' if you # skip this correlation. if not params['skip_fore_corr']: # Correlate the maps with multiprocessing. Note that the # correlations are saved to file separately then loaded in # together because that's (one way) how multiprocessing works. fore_pairs = [] processes_list = [] for pair_index in range(0, num_map_pairs): # Calls 1 multiproc (which governs the correlating) for each # pair on a new CPU so you can have all pairs working at once. multi = multiprocessing.Process(target=multiproc, args=([ pairs[pair_index], params['output_root'], pair_index, False ])) processes_list.append(multi) multi.start() # Waits for all correlations to finish before continuing. while True in [multi.is_alive() for multi in processes_list]: print "processing" time.sleep(5) # just to be safe time.sleep(1) # more concise call, but multiprocessing does not behave well with # complex objects........... #runlist = [(pair_index, # params['output_root'], # False) for # pair_index in range(0, num_map_pairs)] #pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) #pool.map(self.multiproc, runlist) # Load the correlations and save them to each pair. The pairs that # got passed to multiproc are not the same ones as ones in # self.pairs, so this must be done to have actual values. print "Loading map pairs back into program." file_name = params['output_root'] file_name += "map_pair_for_freq_slices_fore_corr_" for count in range(0, num_map_pairs): print "Loading correlation for pair %d" % (count) pickle_handle = open(file_name + str(count) + ".pkl", "r") correlate_results = cPickle.load(pickle_handle) pairs[count].fore_corr = correlate_results[0] pairs[count].fore_counts = correlate_results[1] fore_pairs.append(pairs[count]) pickle_handle.close() self.fore_pairs = copy.deepcopy(fore_pairs) # With this, you do not need fore_pairs anymore. self.pairs = copy.deepcopy(fore_pairs) pairs = self.pairs # Get foregrounds. # svd_info_list keeps track of all of the modes of all maps in # all pairs. This means if you want to subract a different number # of modes for the same maps/noises/frequencies, you have the modes # already saved and do not need to run the first correlation again. svd_info_list = [] for pair in pairs: vals, modes1, modes2 = cf.get_freq_svd_modes( pair.fore_corr, len(freq_list)) pair.vals = vals # Save ALL of the modes for reference. pair.all_modes1 = modes1 pair.all_modes2 = modes2 svd_info = (vals, modes1, modes2) svd_info_list.append(svd_info) # Save only the modes you want to subtract. n_modes = params['modes'] pair.modes1 = modes1[:n_modes] pair.modes2 = modes2[:n_modes] self.svd_info_list = svd_info_list self.pairs = pairs if params['save_svd_info']: ft.save_pickle(self.svd_info_list, params['svd_file']) else: # The first correlation and svd has been skipped. # This means you already have the modes so you can just load # them from file. self.svd_info_list = ft.load_pickle(params['svd_file']) # Set the svd info to the pairs. for i in range(0, len(pairs)): svd_info = self.svd_info_list[i] pairs[i].vals = svd_info[0] pairs[i].all_modes1 = svd_info[1] pairs[i].all_modes2 = svd_info[2] n_modes = params['modes'] pairs[i].modes1 = svd_info[1][:n_modes] pairs[i].modes2 = svd_info[2][:n_modes] self.pairs = pairs # Subtract foregrounds. for pair_index in range(0, len(pairs)): pairs[pair_index].subtract_frequency_modes( pairs[pair_index].modes1, pairs[pair_index].modes2) # Save cleaned clean maps, cleaned noises, and modes. self.save_data(save_maps=params['save_maps'], save_noises=params['save_noises'], save_modes=params['save_modes']) # Finish if this was just first pass. if params['first_pass_only']: self.pairs = pairs return # Correlate the cleaned maps. # Here we could calculate the power spectrum instead eventually. temp_pair_list = [] processes_list = [] for pair_index in range(0, num_map_pairs): multi = multiprocessing.Process(target=multiproc, args=([ pairs[pair_index], params['output_root'], pair_index, True ])) processes_list.append(multi) multi.start() while True in [multi.is_alive() for multi in processes_list]: print "processing" time.sleep(5) # just to be safe time.sleep(1) # ugh, would really rathter use implementation below except multiprocessing # does not behave................. #runlist = [(pairs[pair_index], # params['output_root'], # pair_index, True) for # pair_index in range(0, num_map_pairs)] #pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) #pool.map(multiproc, runlist) print "Loading map pairs back into program." file_name = params['output_root'] file_name += "map_pair_for_freq_slices_corr_" for count in range(0, num_map_pairs): print "Loading correlation for pair %d" % (count) pickle_handle = open(file_name + str(count) + ".pkl", "r") correlate_results = cPickle.load(pickle_handle) pairs[count].corr = correlate_results[0] pairs[count].counts = correlate_results[1] temp_pair_list.append(pairs[count]) pickle_handle.close() self.pairs = copy.deepcopy(temp_pair_list) # Get the average correlation and its standard deviation. corr_list = [] for pair in self.pairs: corr_list.append(pair.corr) self.corr_final, self.corr_std = cf.get_corr_and_std_3d(corr_list) if params['pickle_slices']: ft.save_pickle(self, self.params['output_root'] + \ 'New_Slices_object.pkl') return
def load_pairs(self): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params if not par['pairlist']: if par['calc_diagnal']: noise_inv_suffix = ";noise_inv" else: noise_inv_suffix = ";noise_weight" (self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'], par['noise_inv1'], par['noise_inv2'], noise_inv_suffix=noise_inv_suffix, verbose=False, db_to_use=self.datapath_db) else: self.pairlist = par['pairlist'] pairdict = par['pairdict'] for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary(pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) # map1 & noise_inv1 map1 = algebra.make_vect(algebra.load(pdict['map1'])) if par['simfile1'] is not None: print "adding %s with multiplier %s" % (par['simfile1'], par['sim_multiplier']) sim1 = algebra.make_vect(algebra.load(par['simfile1'])) sim1 *= par['sim_multiplier'] else: sim1 = algebra.zeros_like(map1) if not par['no_weights']: noise_inv1 = wrap_find_weight(pdict['noise_inv1'], regenerate=par['regenerate_noise_inv'], calc_diagnal = par['calc_diagnal']) else: noise_inv1 = algebra.ones_like(map1) # map2 & noise_inv2 #if pairitem == 'I_with_E': if len(self.freq_list2) == 4*len(self.freq_list1): '''For IQUV case''' print 'Construct E map using I Q U V' iquvdict = {} iquvdict['imap'] = pdict['map2'].replace('_E', '_I') iquvdict['qmap'] = pdict['map2'].replace('_E', '_Q') iquvdict['umap'] = pdict['map2'].replace('_E', '_U') iquvdict['vmap'] = pdict['map2'].replace('_E', '_V') iquvdict['imap_weight'] = pdict['noise_inv2'].replace('_E', '_I') iquvdict['qmap_weight'] = pdict['noise_inv2'].replace('_E', '_Q') iquvdict['umap_weight'] = pdict['noise_inv2'].replace('_E', '_U') iquvdict['vmap_weight'] = pdict['noise_inv2'].replace('_E', '_V') map_dict = extend_iquv_map(source_dict=iquvdict) map2 = map_dict['map'] noise_inv2 = map_dict['weight'] sim2 = copy.deepcopy(sim1) map_dict = {} map_dict['imap'] = sim2 map_dict['qmap'] = algebra.zeros_like(sim1) map_dict['umap'] = algebra.zeros_like(sim1) map_dict['vmap'] = algebra.zeros_like(sim1) map_dict = extend_iquv_map(map_dict=map_dict) sim2 = map_dict['map'] elif len(self.freq_list2) == 3*len(self.freq_list1): '''For IQU case''' print 'Construct E map using I Q U' iquvdict = {} iquvdict['imap'] = pdict['map2'].replace('_E', '_I') iquvdict['qmap'] = pdict['map2'].replace('_E', '_Q') iquvdict['umap'] = pdict['map2'].replace('_E', '_U') iquvdict['imap_weight'] = pdict['noise_inv2'].replace('_E', '_I') iquvdict['qmap_weight'] = pdict['noise_inv2'].replace('_E', '_Q') iquvdict['umap_weight'] = pdict['noise_inv2'].replace('_E', '_U') map_dict = extend_iqu_map(source_dict=iquvdict) map2 = map_dict['map'] noise_inv2 = map_dict['weight'] sim2 = copy.deepcopy(sim1) map_dict = {} map_dict['imap'] = sim2 map_dict['qmap'] = algebra.zeros_like(sim1) map_dict['umap'] = algebra.zeros_like(sim1) map_dict = extend_iqu_map(map_dict=map_dict) sim2 = map_dict['map'] else: '''For common case''' map2 = algebra.make_vect(algebra.load(pdict['map2'])) if par['simfile2'] is not None: print "adding %s with multiplier %s" % (par['simfile2'], par['sim_multiplier']) sim2 = algebra.make_vect(algebra.load(par['simfile2'])) sim2 *= par['sim_multiplier'] else: sim2 = algebra.zeros_like(map2) if not par['no_weights']: noise_inv2 = wrap_find_weight(pdict['noise_inv2'], regenerate=par['regenerate_noise_inv'], calc_diagnal = par['calc_diagnal']) else: noise_inv2 = algebra.ones_like(map2) #if self.params['clip_weight_percent'] is not None: # print "Note: your are clipping the weight maps" # mask1 = self.define_weightmask(noise_inv1, # percentile=self.params['clip_weight_percent']) # mask2 = self.define_weightmask(noise_inv2, # percentile=self.params['clip_weight_percent']) # noise_inv1 = self.saturate_weight(noise_inv1, mask1) # noise_inv2 = self.saturate_weight(noise_inv2, mask2) pair = map_pair.MapPair(map1 + sim1, map2 + sim2, noise_inv1, noise_inv2, self.freq_list1, self.freq_list2) pair.set_names(pdict['tag1'], pdict['tag2']) pair.params = self.params self.pairs[pairitem] = pair if par['subtract_inputmap_from_sim'] or \ par['subtract_sim_from_inputmap']: if par['subtract_inputmap_from_sim']: pair_parallel_track = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list1, self.freq_list2) if par['subtract_sim_from_inputmap']: pair_parallel_track = map_pair.MapPair(sim1, sim2, noise_inv1, noise_inv2, self.freq_list1, self.freq_list2) pair_parallel_track.set_names(pdict['tag1'], pdict['tag2']) pair_parallel_track.params = self.params self.pairs_parallel_track[pairitem] = pair_parallel_track
def execute(self): '''Clean the maps of foregrounds, save the results, and get the autocorrelation.''' params = self.params freq_list = sp.array(params['freq_list'], dtype=int) lags = sp.array(params['lags']) # Write parameter file. kiyopy.utils.mkparents(params['output_root']) parse_ini.write_params(params, params['output_root'] + 'params.ini', prefix=prefix) # Get the map data from file as well as the noise inverse. if len(params['file_middles']) == 1: fmid_name = params['file_middles'][0] params['file_middles'] = (fmid_name, fmid_name) if len(params['file_middles']) >= 2: # Deal with multiple files. num_maps = len(params['file_middles']) maps = [] noise_invs = [] # Load all maps and noises once. for map_index in range(0, num_maps): map_file = (params['input_root'] + params['file_middles'][map_index] + params['input_end_map']) print "Loading map %d of %d." % (map_index + 1, num_maps) map_in = algebra.make_vect(algebra.load(map_file)) maps.append(map_in) if not params["no_weights"]: noise_file = (params['input_root'] + params['file_middles'][map_index] + params['input_end_noise']) print "Loading noise %d of %d." % (map_index + 1, num_maps) noise_inv = algebra.make_mat( algebra.open_memmap(noise_file, mode='r')) noise_inv = noise_inv.mat_diag() else: noise_inv = algebra.ones_like(map_in) noise_invs.append(noise_inv) pairs = [] # Make pairs with deepcopies to not make mutability mistakes. for map1_index in range(0, num_maps): for map2_index in range(0, num_maps): if (map2_index > map1_index): map1 = copy.deepcopy(maps[map1_index]) map2 = copy.deepcopy(maps[map2_index]) noise_inv1 = copy.deepcopy(noise_invs[map1_index]) noise_inv2 = copy.deepcopy(noise_invs[map2_index]) pair = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, freq_list) pair.lags = lags pair.params = params # Keep track of the names of maps in pairs so # it knows what to save later. pair.set_names(params['file_middles'][map1_index], params['file_middles'][map2_index]) pairs.append(pair) num_map_pairs = len(pairs) print "%d map pairs created from %d maps." % (len(pairs), num_maps) # Hold a reference in self. self.pairs = pairs # Get maps/ noise inv ready for running. if params["convolve"]: for pair in pairs: pair.degrade_resolution() if params['factorizable_noise']: for pair in pairs: pair.make_noise_factorizable() if params['sub_weighted_mean']: for pair in pairs: pair.subtract_weighted_mean() self.pairs = pairs # Since correlating takes so long, if you already have the svds # you can skip this first correlation [since that's all it's really # for and it is the same no matter how many modes you want]. # Note: map_pairs will not have anything saved in 'fore_corr' if you # skip this correlation. if not params['skip_fore_corr']: # Correlate the maps with multiprocessing. Note that the # correlations are saved to file separately then loaded in # together because that's (one way) how multiprocessing works. runlist = [(pairs[pair_index], params['output_root'], pair_index, False) for pair_index in range(0, num_map_pairs)] pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) pool.map(multiproc, runlist) # Load the correlations and save them to each pair. The pairs that # got passed to multiproc are not the same ones as ones in # self.pairs, so this must be done to have actual values. print "Loading map pairs back into program." file_name = params['output_root'] file_name += "map_pair_for_freq_slices_fore_corr_" fore_pairs = [] for count in range(0, num_map_pairs): print "Loading correlation for pair %d" % (count) pickle_handle = open(file_name + str(count) + ".pkl", "r") correlate_results = cPickle.load(pickle_handle) pairs[count].fore_corr = correlate_results[0] pairs[count].fore_counts = correlate_results[1] fore_pairs.append(pairs[count]) pickle_handle.close() self.fore_pairs = copy.deepcopy(fore_pairs) # With this, you do not need fore_pairs anymore. self.pairs = copy.deepcopy(fore_pairs) pairs = self.pairs # Get foregrounds. # svd_info_list keeps track of all of the modes of all maps in # all pairs. This means if you want to subract a different number # of modes for the same maps/noises/frequencies, you have the modes # already saved and do not need to run the first correlation again. svd_info_list = [] for pair in pairs: vals, modes1, modes2 = cf.get_freq_svd_modes(pair.fore_corr, len(freq_list)) pair.vals = vals # Save ALL of the modes for reference. pair.all_modes1 = modes1 pair.all_modes2 = modes2 svd_info = (vals, modes1, modes2) svd_info_list.append(svd_info) # Save only the modes you want to subtract. n_modes = params['modes'] pair.modes1 = modes1[:n_modes] pair.modes2 = modes2[:n_modes] self.svd_info_list = svd_info_list self.pairs = pairs if params['save_svd_info']: io_wrap.save_pickle(self.svd_info_list, params['svd_file']) else: # The first correlation and svd has been skipped. # This means you already have the modes so you can just load # them from file. self.svd_info_list = io_wrap.load_pickle(params['svd_file']) # Set the svd info to the pairs. for i in range(0, len(pairs)): svd_info = self.svd_info_list[i] pairs[i].vals = svd_info[0] pairs[i].all_modes1 = svd_info[1] pairs[i].all_modes2 = svd_info[2] n_modes = params['modes'] pairs[i].modes1 = svd_info[1][:n_modes] pairs[i].modes2 = svd_info[2][:n_modes] self.pairs = pairs # Subtract foregrounds. for pair_index in range(0, len(pairs)): pairs[pair_index].subtract_frequency_modes(pairs[pair_index].modes1, pairs[pair_index].modes2) # Save cleaned clean maps, cleaned noises, and modes. self.save_data(save_maps=params['save_maps'], save_noises=params['save_noises'], save_modes=params['save_modes']) # Finish if this was just first pass. if params['first_pass_only']: self.pairs = pairs return # Correlate the cleaned maps. # Here we could calculate the power spectrum instead eventually. runlist = [(pairs[pair_index], params['output_root'], pair_index, True) for pair_index in range(0, num_map_pairs)] pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) pool.map(multiproc, runlist) print "Loading map pairs back into program." file_name = params['output_root'] file_name += "map_pair_for_freq_slices_corr_" temp_pair_list = [] for count in range(0, num_map_pairs): print "Loading correlation for pair %d" % (count) pickle_handle = open(file_name + str(count) + ".pkl", "r") correlate_results = cPickle.load(pickle_handle) pairs[count].corr = correlate_results[0] pairs[count].counts = correlate_results[1] temp_pair_list.append(pairs[count]) pickle_handle.close() self.pairs = copy.deepcopy(temp_pair_list) # Get the average correlation and its standard deviation. corr_list = [] for pair in self.pairs: corr_list.append(pair.corr) self.corr_final, self.corr_std = cf.get_corr_and_std_3d(corr_list) if params['pickle_slices']: pickle_slices(self) return
def load_pairs(self): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params (self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'], par['noise_inv1'], par['noise_inv2'], noise_inv_suffix=";noise_weight", verbose=False, tack_on=self.tack_on_input, db_to_use=self.datapath_db) for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary(pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) map1 = algebra.make_vect(algebra.load(pdict['map1'])) map2 = algebra.make_vect(algebra.load(pdict['map2'])) if par['simfile'] is not None: print "adding %s with multiplier %s" % (par['simfile'], par['sim_multiplier']) sim = algebra.make_vect(algebra.load(par['simfile'])) sim *= par['sim_multiplier'] print sim.shape, map1.shape else: sim = algebra.zeros_like(map1) if not par['no_weights']: noise_inv1 = wrap_find_weight(pdict['noise_inv1'], regenerate=par['regenerate_noise_inv']) noise_inv2 = wrap_find_weight(pdict['noise_inv2'], regenerate=par['regenerate_noise_inv']) else: noise_inv1 = algebra.ones_like(map1) noise_inv2 = algebra.ones_like(map2) pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1, noise_inv2, self.freq_list) pair.set_names(pdict['tag1'], pdict['tag2']) pair.params = self.params self.pairs[pairitem] = pair if par['subtract_inputmap_from_sim'] or \ par['subtract_sim_from_inputmap']: if par['subtract_inputmap_from_sim']: pair_parallel_track = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list) if par['subtract_sim_from_inputmap']: pair_parallel_track = map_pair.MapPair(sim, sim, noise_inv1, noise_inv2, self.freq_list) pair_parallel_track.set_names(pdict['tag1'], pdict['tag2']) pair_parallel_track.params = self.params self.pairs_parallel_track[pairitem] = pair_parallel_track
def calculate_mixing(weight_file1, weight_file2, bins, xspec_fileout, mixing_fileout, unitless=False, refinement=2, pad=5, order=1, window='blackman', zero_pad=False, identity_test=False): print "loading the weights and converting to physical coordinates" weight1_obs = algebra.make_vect(algebra.load(weight_file1)) weight1 = bh.repackage_kiyo( pg.physical_grid(weight1_obs, refinement=refinement, pad=pad, order=order)) weight2_obs = algebra.make_vect(algebra.load(weight_file2)) weight2 = bh.repackage_kiyo( pg.physical_grid(weight2_obs, refinement=refinement, pad=pad, order=order)) if window: window_function = fftutil.window_nd(weight1.shape, name=window) weight1 *= window_function weight2 *= window_function print "calculating the cross-power of the spatial weighting functions" arr1 = algebra.ones_like(weight1) arr2 = algebra.ones_like(weight2) # no window applied here (applied above) xspec = pe.cross_power_est(weight1, weight2, arr1, arr2, window=None, nonorm=True) # for each point in the cube, find |k|, k_perp, k_parallel # TODO: speed this up by using one direct numpy call (not limiting) k_mag_arr = binning.radius_array(xspec) k_perp_arr = binning.radius_array(xspec, zero_axes=[0]) k_parallel_arr = binning.radius_array(xspec, zero_axes=[1, 2]) if unitless: xspec = pe.make_unitless(xspec, radius_arr=k_mag_arr) # NOTE: assuming lowest k bin has only one point in 3D k-space # could make this floor of dimensions divided by 2 also center_3d = np.transpose(np.transpose(np.where(k_mag_arr == 0.))[0]) # In the estimator, we devide by 1/sum(w1 * w2) to get most of the effect # of the weighing. The mixing matrix here can be thought of as a correction # that that diagonal-only estimate. leakage_ratio = xspec[center_3d[0], center_3d[1], center_3d[2]] / \ np.sum(weight1 * weight2) print "power leakage ratio: %10.5g" % leakage_ratio xspec /= np.sum(weight1 * weight2) print "partitioning the 3D kspace up into the 2D k bins" (kflat, ret_indices) = bin_indices_2d(k_perp_arr, k_parallel_arr, bins, bins) # perform a test where the window function is a delta function at the # origin so that the mixing matrix is identity if identity_test: xspec = algebra.zeros_like(xspec) xspec[center_3d[0], center_3d[1], center_3d[2]] = 1. # now save the window cross-power for downstream pooled users algebra.save(xspec_fileout, xspec) runlist = [] for bin_index in range(kflat.shape[0]): bin_3d = ret_indices[repr(bin_index)] if bin_3d is not None: runlist.append((xspec_fileout, bin_index, bins, bin_3d, center_3d)) pool = multiprocessing.Pool(processes=(multiprocessing.cpu_count() - 4)) # the longest runs get pushed to the end; randomize for better job packing random.shuffle(runlist) results = pool.map(sum_window, runlist) #gnuplot_single_slice(runlist[0]) # for troubleshooting # now save the results for post-processing params = { "unitless": unitless, "refinement": refinement, "pad": pad, "order": order, "window": window, "zero_pad": zero_pad, "identity_test": identity_test, "weight_file1": weight_file1, "weight_file2": weight_file2, "bins": bins } outshelve = shelve.open(mixing_fileout, "n") outshelve["params"] = params # parameters for this run outshelve["weight1"] = weight1 # weight map 1 outshelve["weight2"] = weight2 # weight map 2 outshelve["xspec"] = xspec # copy of the weight spectra outshelve["kflat"] = kflat # 2D k bin vector outshelve["bins_3d"] = ret_indices # indices to k3d for a 2d k bin outshelve["results"] = results # mixing matrix columns outshelve.close()
def save_data(self, n_modes): prodmap_list = [] weight_list = [] n_modes = "%dmodes" % n_modes for pairitem in self.pairlist: pair = self.pairs[pairitem] (tag1, tag2) = (pair.map1_name, pair.map2_name) clnoise = "cleaned_noise_inv" map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag1, clnoise, tag2, n_modes) modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) #if pair.map1.shape == pair.map2.shape: map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag2, clnoise, tag1, n_modes) modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) if self.params['subtract_inputmap_from_sim'] or \ self.params['subtract_sim_from_inputmap']: map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1 map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2 elif self.params['subtract_realmap_from_sim']: if not os.path.exists(self.params['realmap_dir']): print "Error: Real map directory does not exists" exit() else: realmap_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy"%\ (self.params['realmap_dir'], tag1, tag2, n_modes) realmap = algebra.make_vect(algebra.load(realmap_file)) print "Subtract realmap from result" map1 = copy.deepcopy(pair.map1) - realmap map2 = copy.deepcopy(pair.map2) if map2.shape == map1.shape: map2 -= realmap else: map1 = copy.deepcopy(pair.map1) map2 = copy.deepcopy(pair.map2) prodmap_list.append(map1 * pair.noise_inv1) prodmap_list.append(map2 * pair.noise_inv2) weight_list.append(pair.noise_inv1) weight_list.append(pair.noise_inv2) if self.params['save_section']: algebra.save(map1_file, map1) algebra.save(noise_inv1_file, pair.noise_inv1) algebra.save(modes1_file, pair.left_modes) if pair.map1.shape == pair.map2.shape: algebra.save(map2_file, map2) algebra.save(noise_inv2_file, pair.noise_inv2) algebra.save(modes2_file, pair.right_modes) #if map2.shape[0] == 3*map1.shape[0]: # #source_dict = {} # #source_dict['map'] = map2_file # #source_dict['weight'] = noise_inv2_file # map_dict = {} # map_dict['map'] = map2 # map_dict['weight'] = pair.noise_inv2 # target_dict = {} # target_dict['imap'] = map2_file.replace('_'+tag2, '_'+tag2+'_I') # target_dict['qmap'] = map2_file.replace('_'+tag2, '_'+tag2+'_Q') # target_dict['umap'] = map2_file.replace('_'+tag2, '_'+tag2+'_U') # target_dict['imap_weight'] =\ # noise_inv2_file.replace('_'+tag2, '_'+tag2+'_I') # target_dict['qmap_weight'] =\ # noise_inv2_file.replace('_'+tag2, '_'+tag2+'_Q') # target_dict['umap_weight'] =\ # noise_inv2_file.replace('_'+tag2, '_'+tag2+'_U') # divide_iqu_map(map_dict=map_dict, target_dict=target_dict) if map1.shape != map2.shape: print "Shape of map1 and map2 are different, can not get combined map." else: cumulative_product = algebra.zeros_like(prodmap_list[0]) cumulative_weight = algebra.zeros_like(prodmap_list[0]) for mapind in range(0, len(prodmap_list)): cumulative_product += prodmap_list[mapind] cumulative_weight += weight_list[mapind] algebra.compressed_array_summary(cumulative_weight, "weight map") algebra.compressed_array_summary(cumulative_product, "product map") cumulative_weight[cumulative_weight < 1.e-20] = 0. cumulative_product[cumulative_weight < 1.e-20] = 0. newmap = cumulative_product / cumulative_weight # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. cumulative_product[nan_array] = 0. cumulative_weight[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. cumulative_product[inf_array] = 0. cumulative_weight[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(cumulative_product,"final map * weight") algebra.compressed_array_summary(cumulative_weight, "final weight map") combined = "combined_clean" combined_map_file = "%s/%s_map_%s.npy" % \ (self.output_root, combined, n_modes) combined_weight_file = "%s/%s_weight_%s.npy" % \ (self.output_root, combined, n_modes) combined_product_file = "%s/%s_product_%s.npy" % \ (self.output_root, combined, n_modes) combined_ones_file = "%s/%s_ones_%s.npy" % \ (self.output_root, combined, n_modes) algebra.save(combined_map_file, newmap) algebra.save(combined_product_file, cumulative_product) algebra.save(combined_weight_file, cumulative_weight) algebra.save(combined_ones_file, algebra.ones_like(newmap))
def combine_maps_driver(inputmap_dict, inputweight_dict, output_dict, fullcov=False, datapath_db=None): r"""Combine a list of weights, maps specified by their database keys """ if datapath_db is None: datapath_db = data_paths.DataPath() signal_list = [] weight_list = [] for mapkey in inputmap_dict: signalfile = inputmap_dict[mapkey] weightfile = inputweight_dict[mapkey] print "loading pair: %s %s" % (signalfile, weightfile) signal_list.append(algebra.make_vect(algebra.load(signalfile))) if fullcov: raw_weight = algebra.make_mat( algebra.open_memmap(weightfile)) raw_weight = raw_weight.mat_diag() else: raw_weight = algebra.make_vect(algebra.load(weightfile)) # zero out any messy stuff raw_weight[raw_weight < 1.e-20] = 0. raw_weight[np.isnan(raw_weight)] = 0. raw_weight[np.isinf(raw_weight)] = 0. weight_list.append(raw_weight) prodmap = [] for mapind in range(0, len(signal_list)): prodmap.append(signal_list[mapind] * weight_list[mapind]) print "CHECK THESE: %d %d %d" % (len(signal_list), len(weight_list), len(prodmap)) cumulative_product = algebra.zeros_like(prodmap[0]) cumulative_weight = algebra.zeros_like(prodmap[0]) for mapind in range(0, len(signal_list)): cumulative_product += prodmap[mapind] cumulative_weight += weight_list[mapind] algebra.compressed_array_summary(cumulative_weight, "weight map") algebra.compressed_array_summary(cumulative_product, "product map") newmap = cumulative_product / cumulative_weight cumulative_weight[cumulative_weight < 1.e-20] = 0. cumulative_product[cumulative_weight < 1.e-20] = 0. # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. cumulative_product[nan_array] = 0. cumulative_weight[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. cumulative_product[inf_array] = 0. cumulative_weight[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(cumulative_product, "final map * weight") algebra.compressed_array_summary(cumulative_weight, "final weight map") print output_dict algebra.save(output_dict['map'], newmap) algebra.save(output_dict['product'], cumulative_product) algebra.save(output_dict['weight'], cumulative_weight) algebra.save(output_dict['ones'], algebra.ones_like(newmap))