def load_pairs(self): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params (self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'], par['noise_inv1'], par['noise_inv2'], noise_inv_suffix=";noise_weight", verbose=False, tack_on=self.tack_on_input, db_to_use=self.datapath_db) for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary(pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) map1 = algebra.make_vect(algebra.load(pdict['map1'])) map2 = algebra.make_vect(algebra.load(pdict['map2'])) if par['simfile'] is not None: print "adding %s with multiplier %s" % (par['simfile'], par['sim_multiplier']) sim = algebra.make_vect(algebra.load(par['simfile'])) sim *= par['sim_multiplier'] print sim.shape, map1.shape else: sim = algebra.zeros_like(map1) noise_inv1 = algebra.make_vect(algebra.load(pdict['noise_inv1'])) noise_inv2 = algebra.make_vect(algebra.load(pdict['noise_inv2'])) pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1, noise_inv2, self.freq_list) pair.set_names(pdict['tag1'], pdict['tag2']) pair.params = self.params self.pairs[pairitem] = pair if par['subtract_inputmap_from_sim'] or \ par['subtract_sim_from_inputmap']: if par['subtract_inputmap_from_sim']: pair_parallel_track = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list) if par['subtract_sim_from_inputmap']: pair_parallel_track = map_pair.MapPair(sim, sim, noise_inv1, noise_inv2, self.freq_list) pair_parallel_track.set_names(pdict['tag1'], pdict['tag2']) pair_parallel_track.params = self.params self.pairs_parallel_track[pairitem] = pair_parallel_track
def load_pairs(self): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params (self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'], par['noise_inv1'], par['noise_inv2'], noise_inv_suffix=";noise_weight", verbose=False, tack_on=self.tack_on_input, db_to_use=self.datapath_db) for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary( pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) map1 = algebra.make_vect(algebra.load(pdict['map1'])) map2 = algebra.make_vect(algebra.load(pdict['map2'])) if par['simfile'] is not None: print "adding %s with multiplier %s" % (par['simfile'], par['sim_multiplier']) sim = algebra.make_vect(algebra.load(par['simfile'])) sim *= par['sim_multiplier'] print sim.shape, map1.shape else: sim = algebra.zeros_like(map1) noise_inv1 = algebra.make_vect(algebra.load(pdict['noise_inv1'])) noise_inv2 = algebra.make_vect(algebra.load(pdict['noise_inv2'])) pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1, noise_inv2, self.freq_list) pair.set_names(pdict['tag1'], pdict['tag2']) pair.params = self.params self.pairs[pairitem] = pair if par['subtract_inputmap_from_sim'] or \ par['subtract_sim_from_inputmap']: if par['subtract_inputmap_from_sim']: pair_parallel_track = map_pair.MapPair( map1, map2, noise_inv1, noise_inv2, self.freq_list) if par['subtract_sim_from_inputmap']: pair_parallel_track = map_pair.MapPair( sim, sim, noise_inv1, noise_inv2, self.freq_list) pair_parallel_track.set_names(pdict['tag1'], pdict['tag2']) pair_parallel_track.params = self.params self.pairs_parallel_track[pairitem] = pair_parallel_track
def make_combined(self, map_list, targetroot, file): imap_temp = algebra.make_vect(algebra.load(map_list[0])) cumulative_product = algebra.zeros_like(imap_temp) cumulative_weight = algebra.zeros_like(imap_temp) for dir in map_list: imap = algebra.make_vect(algebra.load(dir)) wmap = algebra.make_vect(algebra.load(dir.replace('clean_map', 'noise_inv'))) cumulative_product += imap*wmap cumulative_weight += wmap algebra.compressed_array_summary(cumulative_weight, "weight map") algebra.compressed_array_summary(cumulative_product, "product map") cumulative_weight[cumulative_weight < 1.e-20] = 0. cumulative_product[cumulative_weight < 1.e-20] = 0. cumulative_weight[cumulative_weight == 0] = np.inf cumulative_weight[np.isnan(cumulative_weight)] = np.inf newmap = cumulative_product / cumulative_weight cumulative_weight[np.isinf(cumulative_weight)] = 0. # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. cumulative_product[nan_array] = 0. cumulative_weight[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. cumulative_product[inf_array] = 0. cumulative_weight[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(cumulative_product,"final map * weight") algebra.compressed_array_summary(cumulative_weight, "final weight map") combined_map_file = targetroot + file combined_weight_file = targetroot + file.replace('map', 'weight') print combined_map_file print combined_weight_file algebra.save(combined_map_file, newmap) algebra.save(combined_weight_file, cumulative_weight)
def sum_window(argt): """A given bin in 2D k-space (labelled by bin_index_2d) is the sum over a "washer" in 3D k-space, a band in k_parallel and an annulus in k_x, k_y. Let all of the 3D bins in k_space be indexed by bin_3d. The window function is centered at k_3d=0, and let these indices defining the center of the 3d volume be given in center_3d. TODO: replace this with NlogN convolution TODO: implement 0-padded roll instead of np.roll, algebra.roll_zeropad() """ (filename, bin_index_2d, k_2d, bin_3d, center_3d) = argt # load the cross-power of the weighting functions xspec = algebra.make_vect(algebra.load(filename)) windowsum = algebra.zeros_like(xspec) num_3dbins_in_2dbin = bin_3d.shape[0] print "%d: summing over %d bins" % (bin_index_2d, num_3dbins_in_2dbin) for bin_3dind in range(num_3dbins_in_2dbin): # TODO: is this sign right, does it matter? off = bin_3d[bin_3dind] - center_3d #print off windowsum += np.roll(np.roll(np.roll(xspec, off[0], axis=0), off[1], axis=1), off[2], axis=2) k_perp_arr = binning.radius_array(xspec, zero_axes=[0]) k_parallel_arr = binning.radius_array(xspec, zero_axes=[1, 2]) kx_2d = copy.deepcopy(k_2d) ky_2d = copy.deepcopy(k_2d) counts_histo_2d, binavg_2d = binning.bin_an_array_2d(windowsum, k_perp_arr, k_parallel_arr, kx_2d, ky_2d) return (bin_index_2d, counts_histo_2d, binavg_2d)
def sum_window(argt): """A given bin in 2D k-space (labelled by bin_index_2d) is the sum over a "washer" in 3D k-space, a band in k_parallel and an annulus in k_x, k_y. Let all of the 3D bins in k_space be indexed by bin_3d. The window function is centered at k_3d=0, and let these indices defining the center of the 3d volume be given in center_3d. TODO: replace this with NlogN convolution TODO: implement 0-padded roll instead of np.roll, algebra.roll_zeropad() """ (filename, bin_index_2d, k_2d, bin_3d, center_3d) = argt # load the cross-power of the weighting functions xspec = algebra.make_vect(algebra.load(filename)) windowsum = algebra.zeros_like(xspec) num_3dbins_in_2dbin = bin_3d.shape[0] print "%d: summing over %d bins" % (bin_index_2d, num_3dbins_in_2dbin) for bin_3dind in range(num_3dbins_in_2dbin): # TODO: is this sign right, does it matter? off = bin_3d[bin_3dind] - center_3d #print off windowsum += np.roll(np.roll(np.roll(xspec, off[0], axis=0), off[1], axis=1), off[2], axis=2) k_perp_arr = binning.radius_array(xspec, zero_axes=[0]) k_parallel_arr = binning.radius_array(xspec, zero_axes=[1, 2]) kx_2d = copy.deepcopy(k_2d) ky_2d = copy.deepcopy(k_2d) counts_histo_2d, binavg_2d = binning.bin_an_array_2d( windowsum, k_perp_arr, k_parallel_arr, kx_2d, ky_2d) return (bin_index_2d, counts_histo_2d, binavg_2d)
def calculate_mixing(weight_file1, weight_file2, bins, xspec_fileout, mixing_fileout, unitless=False, refinement=2, pad=5, order=1, window='blackman', zero_pad=False, identity_test=False): print "loading the weights and converting to physical coordinates" weight1_obs = algebra.make_vect(algebra.load(weight_file1)) weight1 = bh.repackage_kiyo(pg.physical_grid( weight1_obs, refinement=refinement, pad=pad, order=order)) weight2_obs = algebra.make_vect(algebra.load(weight_file2)) weight2 = bh.repackage_kiyo(pg.physical_grid( weight2_obs, refinement=refinement, pad=pad, order=order)) if window: window_function = fftutil.window_nd(weight1.shape, name=window) weight1 *= window_function weight2 *= window_function print "calculating the cross-power of the spatial weighting functions" arr1 = algebra.ones_like(weight1) arr2 = algebra.ones_like(weight2) # no window applied here (applied above) xspec = pe.cross_power_est(weight1, weight2, arr1, arr2, window=None, nonorm=True) # for each point in the cube, find |k|, k_perp, k_parallel # TODO: speed this up by using one direct numpy call (not limiting) k_mag_arr = binning.radius_array(xspec) k_perp_arr = binning.radius_array(xspec, zero_axes=[0]) k_parallel_arr = binning.radius_array(xspec, zero_axes=[1, 2]) if unitless: xspec = pe.make_unitless(xspec, radius_arr=k_mag_arr) # NOTE: assuming lowest k bin has only one point in 3D k-space # could make this floor of dimensions divided by 2 also center_3d = np.transpose(np.transpose(np.where(k_mag_arr == 0.))[0]) # In the estimator, we devide by 1/sum(w1 * w2) to get most of the effect # of the weighing. The mixing matrix here can be thought of as a correction # that that diagonal-only estimate. leakage_ratio = xspec[center_3d[0], center_3d[1], center_3d[2]] / \ np.sum(weight1 * weight2) print "power leakage ratio: %10.5g" % leakage_ratio xspec /= np.sum(weight1 * weight2) print "partitioning the 3D kspace up into the 2D k bins" (kflat, ret_indices) = bin_indices_2d(k_perp_arr, k_parallel_arr, bins, bins) # perform a test where the window function is a delta function at the # origin so that the mixing matrix is identity if identity_test: xspec = algebra.zeros_like(xspec) xspec[center_3d[0], center_3d[1], center_3d[2]] = 1. # now save the window cross-power for downstream pooled users algebra.save(xspec_fileout, xspec) runlist = [] for bin_index in range(kflat.shape[0]): bin_3d = ret_indices[repr(bin_index)] if bin_3d is not None: runlist.append((xspec_fileout, bin_index, bins, bin_3d, center_3d)) pool = multiprocessing.Pool(processes=(multiprocessing.cpu_count() - 4)) # the longest runs get pushed to the end; randomize for better job packing random.shuffle(runlist) results = pool.map(sum_window, runlist) #gnuplot_single_slice(runlist[0]) # for troubleshooting # now save the results for post-processing params = {"unitless": unitless, "refinement": refinement, "pad": pad, "order": order, "window": window, "zero_pad": zero_pad, "identity_test": identity_test, "weight_file1": weight_file1, "weight_file2": weight_file2, "bins": bins} outshelve = shelve.open(mixing_fileout, "n") outshelve["params"] = params # parameters for this run outshelve["weight1"] = weight1 # weight map 1 outshelve["weight2"] = weight2 # weight map 2 outshelve["xspec"] = xspec # copy of the weight spectra outshelve["kflat"] = kflat # 2D k bin vector outshelve["bins_3d"] = ret_indices # indices to k3d for a 2d k bin outshelve["results"] = results # mixing matrix columns outshelve.close()
def combine_maps_driver(inputmap_dict, inputweight_dict, output_dict, fullcov=False, datapath_db=None): r"""Combine a list of weights, maps specified by their database keys """ if datapath_db is None: datapath_db = data_paths.DataPath() signal_list = [] weight_list = [] for mapkey in inputmap_dict: signalfile = inputmap_dict[mapkey] weightfile = inputweight_dict[mapkey] print "loading pair: %s %s" % (signalfile, weightfile) signal_list.append(algebra.make_vect(algebra.load(signalfile))) if fullcov: raw_weight = algebra.make_mat( algebra.open_memmap(weightfile)) raw_weight = raw_weight.mat_diag() else: raw_weight = algebra.make_vect(algebra.load(weightfile)) # zero out any messy stuff raw_weight[raw_weight < 1.e-20] = 0. raw_weight[np.isnan(raw_weight)] = 0. raw_weight[np.isinf(raw_weight)] = 0. weight_list.append(raw_weight) prodmap = [] for mapind in range(0, len(signal_list)): prodmap.append(signal_list[mapind] * weight_list[mapind]) print "CHECK THESE: %d %d %d" % (len(signal_list), len(weight_list), len(prodmap)) cumulative_product = algebra.zeros_like(prodmap[0]) cumulative_weight = algebra.zeros_like(prodmap[0]) for mapind in range(0, len(signal_list)): cumulative_product += prodmap[mapind] cumulative_weight += weight_list[mapind] algebra.compressed_array_summary(cumulative_weight, "weight map") algebra.compressed_array_summary(cumulative_product, "product map") newmap = cumulative_product / cumulative_weight cumulative_weight[cumulative_weight < 1.e-20] = 0. cumulative_product[cumulative_weight < 1.e-20] = 0. # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. cumulative_product[nan_array] = 0. cumulative_weight[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. cumulative_product[inf_array] = 0. cumulative_weight[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(cumulative_product, "final map * weight") algebra.compressed_array_summary(cumulative_weight, "final weight map") print output_dict algebra.save(output_dict['map'], newmap) algebra.save(output_dict['product'], cumulative_product) algebra.save(output_dict['weight'], cumulative_weight) algebra.save(output_dict['ones'], algebra.ones_like(newmap))
def apply(self, alg_ob, wrap=False, cval=0, right_apply=False) : """Apply the beam, as a linear operator, to vector or matrix. This operation is equivalent to matrix multiplication by the beam matrix. The matrix multiplication can be performed with the beam matrix on either the left or the right. The beam matrix is a symetric matrix. Parameters ---------- alg_ob : `vect` or `mat` subclass. Object to which the beam will be applied. This object must live in map space. If it is a vect, alg_ob.axes must be ('freq', 'ra', 'dec'). If it is a mat, and `right_apply` is False, then alg_ob.row_names() must return ('freq', 'ra', 'dec'). If `right_apply` is True, then alg_ob.col_names() should return this tuple. Also, the meta data for these three axis must be set (see `algebra.alg_object.set_axis_info`). wrap : bool If tests True, use periodic boundary conditions for the convolution. Otherwise zero pad (default). right_apply : bool Whethar to apply the beam operator with the from the left (False, default) or from the right (True). If `alg_ob` is a vect subclass, this has no effect (because the beam matrix is symetric). Returns ------- out : `vect` or `mat` subclass same shape as `alg_ob`. Convolved map or matrix. Notes ----- The missing feature here is the ability to preallowcate memory or to be able to overwrite the input alg_ob in place. This will be important for large matricies that we can only hold in memory one at a time. Also this would be pretty easy to implement. """ if ((not 'freq' in alg_ob.axes) or (not 'ra' in alg_ob.axes) or (not 'dec' in alg_ob.axes)) : raise ce.DataError("Beam operation only works in frequncy, " "ra, dec, coords.") # allowcate memory for the output. out = algebra.zeros_like(alg_ob) # Figure out the pixel sizes (in real degrees). dfreq = abs(alg_ob.info['freq_delta']) dra = abs(alg_ob.info['ra_delta']) dra /= sp.cos(alg_ob.info['dec_centre']*sp.pi/180) ddec = abs(alg_ob.info['dec_delta']) # Figure out the convolution mode. if wrap : mode = 'wrap' else : mode = 'constant' # Loop over frequencies and do convolution one frequency at a time. freq_array = alg_ob.get_axis('freq') for ii, freq in enumerate(freq_array) : # How wide the kernal has to be. width = self.kernal_size(freq) # Make sure the dimensions are an odd number of pixels. nkx = width//abs(dra) if nkx%2 == 0 : nkx += 1 nky = width//abs(ddec) if nky%2 == 0 : nky += 1 # Calculate kernal lags. lagsx = (sp.arange(nkx, dtype=float) - (nkx - 1)//2)*dra lagsy = (sp.arange(nky, dtype=float) - (nky - 1)//2)*ddec lags_sq = lagsx[:, None]**2 + lagsy[None, :]**2 # Make gaussian beam profile. kernal = dra*ddec*self.beam_function(lags_sq, freq, squared_delta=True) # print kernal.shape, dra, nkx if isinstance(alg_ob, algebra.vect) : if alg_ob.axes != ('freq', 'ra', 'dec') : raise ce.DataError("Vector axis names must be exactly " "('freq', 'ra', 'dec')") # Do the convolution. convolve(alg_ob[ii, ...], kernal, out[ii], mode=mode, cval=cval) elif isinstance(alg_ob, algebra.mat) : # If applying from the left, loop over columns and convolve # over rows. If applying from the right, do the oposite. if right_apply : if alg_ob.col_names() != ('freq', 'ra', 'dec') : raise ce.DataError("Matrix column axis names must be " "exactly ('freq', 'ra', 'dec')") iterator = alg_ob.iter_row_index() else : if alg_ob.row_names() != ('freq', 'ra', 'dec') : raise ce.DataError("Matrix row axis names must be " "exactly ('freq', 'ra', 'dec')") iterator = alg_ob.iter_col_index() for index in iterator : sub_mat = alg_ob[index] # A view. # Pick out this frequency. sub_mat = sub_mat[ii, ...] # make a view of the ouput array. sub_out = out[index] sub_out = sub_out[ii, ...] # Do the convolution. convolve(sub_mat, kernal, sub_out, mode=mode, cval=cval) return out
def save_data(self, n_modes): prodmap_list = [] weight_list = [] n_modes = "%dmodes" % n_modes for pairitem in self.pairlist: pair = self.pairs[pairitem] (tag1, tag2) = (pair.map1_name, pair.map2_name) clnoise = "cleaned_noise_inv" map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag1, clnoise, tag2, n_modes) noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag2, clnoise, tag1, n_modes) modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) if self.params['subtract_inputmap_from_sim'] or \ self.params['subtract_sim_from_inputmap']: map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1 map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2 else: map1 = copy.deepcopy(pair.map1) map2 = copy.deepcopy(pair.map2) prodmap_list.append(map1 * pair.noise_inv1) prodmap_list.append(map2 * pair.noise_inv2) weight_list.append(pair.noise_inv1) weight_list.append(pair.noise_inv2) algebra.save(map1_file, map1) algebra.save(map2_file, map2) algebra.save(noise_inv1_file, pair.noise_inv1) algebra.save(noise_inv2_file, pair.noise_inv2) algebra.save(modes1_file, pair.left_modes) algebra.save(modes2_file, pair.right_modes) cumulative_product = algebra.zeros_like(prodmap_list[0]) cumulative_weight = algebra.zeros_like(prodmap_list[0]) for mapind in range(0, len(prodmap_list)): cumulative_product += prodmap_list[mapind] cumulative_weight += weight_list[mapind] algebra.compressed_array_summary(cumulative_weight, "weight map") algebra.compressed_array_summary(cumulative_product, "product map") cumulative_weight[cumulative_weight < 1.e-20] = 0. cumulative_product[cumulative_weight < 1.e-20] = 0. newmap = cumulative_product / cumulative_weight # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. cumulative_product[nan_array] = 0. cumulative_weight[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. cumulative_product[inf_array] = 0. cumulative_weight[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(cumulative_product, "final map * weight") algebra.compressed_array_summary(cumulative_weight, "final weight map") combined = "combined_clean" combined_map_file = "%s/%s_map_%s.npy" % \ (self.output_root, combined, n_modes) combined_weight_file = "%s/%s_weight_%s.npy" % \ (self.output_root, combined, n_modes) combined_product_file = "%s/%s_product_%s.npy" % \ (self.output_root, combined, n_modes) combined_ones_file = "%s/%s_ones_%s.npy" % \ (self.output_root, combined, n_modes) algebra.save(combined_map_file, newmap) algebra.save(combined_product_file, cumulative_product) algebra.save(combined_weight_file, cumulative_weight) algebra.save(combined_ones_file, algebra.ones_like(newmap))
def execute(self, nprocesses=1): """Worker funciton.""" params = self.params # Make parent directory and write parameter file. kiyopy.utils.mkparents(params['output_root']) parse_ini.write_params(params, params['output_root'] + 'params.ini', prefix=prefix) save_noise_diag = params['save_noise_diag'] in_root = params['input_root'] all_out_fname_list = [] all_in_fname_list = [] # Figure out what the band names are. bands = params['bands'] if not bands: map_files = glob.glob(in_root + 'dirty_map_' + pol_str + "_*.npy") bands = [] root_len = len(in_root + 'dirty_map_') for file_name in map_files: bands.append(file_name[root_len:-4]) # Loop over files to process. for pol_str in params['polarizations']: for band in bands: if band == -1: band_str = '' else: band_str = "_" + repr(band) dmap_fname = (in_root + 'dirty_map_' + pol_str + band_str + '.npy') all_in_fname_list.append( kiyopy.utils.abbreviate_file_path(dmap_fname)) # Load the dirty map and the noise matrix. dirty_map = algebra.load(dmap_fname) dirty_map = algebra.make_vect(dirty_map) if dirty_map.axes != ('freq', 'ra', 'dec'): msg = ("Expeced dirty map to have axes ('freq'," "'ra', 'dec'), but it has axes: " + str(dirty_map.axes)) raise ce.DataError(msg) shape = dirty_map.shape # Initialize the clean map. clean_map = algebra.info_array(sp.zeros(dirty_map.shape)) clean_map.info = dict(dirty_map.info) clean_map = algebra.make_vect(clean_map) # If needed, initialize a map for the noise diagonal. if save_noise_diag: noise_diag = algebra.zeros_like(clean_map) if params["from_eig"]: # Solving from eigen decomposition of the noise instead of # the noise itself. # Load in the decomposition. evects_fname = (in_root + 'noise_evects_' + pol_str + +band_str + '.npy') if self.feedback > 1: print "Using dirty map: " + dmap_fname print "Using eigenvectors: " + evects_fname evects = algebra.open_memmap(evects_fname, 'r') evects = algebra.make_mat(evects) evals_inv_fname = (in_root + 'noise_evalsinv_' + pol_str + "_" + repr(band) + '.npy') evals_inv = algebra.load(evals_inv_fname) evals_inv = algebra.make_mat(evals_inv) # Solve for the map. if params["save_noise_diag"]: clean_map, noise_diag = solve_from_eig( evals_inv, evects, dirty_map, True, self.feedback) else: clean_map = solve_from_eig(evals_inv, evects, dirty_map, False, self.feedback) # Delete the eigen vectors to recover memory. del evects else: # Solving from the noise. noise_fname = (in_root + 'noise_inv_' + pol_str + band_str + '.npy') if self.feedback > 1: print "Using dirty map: " + dmap_fname print "Using noise inverse: " + noise_fname all_in_fname_list.append( kiyopy.utils.abbreviate_file_path(noise_fname)) noise_inv = algebra.open_memmap(noise_fname, 'r') noise_inv = algebra.make_mat(noise_inv) # Two cases for the noise. If its the same shape as the map # then the noise is diagonal. Otherwise, it should be # block diagonal in frequency. if noise_inv.ndim == 3: if noise_inv.axes != ('freq', 'ra', 'dec'): msg = ("Expeced noise matrix to have axes " "('freq', 'ra', 'dec'), but it has: " + str(noise_inv.axes)) raise ce.DataError(msg) # Noise inverse can fit in memory, so copy it. noise_inv_memory = sp.array(noise_inv, copy=True) # Find the non-singular (covered) pixels. max_information = noise_inv_memory.max() good_data = noise_inv_memory < 1.0e-10 * max_information # Make the clean map. clean_map[good_data] = (dirty_map[good_data] / noise_inv_memory[good_data]) if save_noise_diag: noise_diag[good_data] = \ 1/noise_inv_memory[good_data] elif noise_inv.ndim == 5: if noise_inv.axes != ('freq', 'ra', 'dec', 'ra', 'dec'): msg = ("Expeced noise matrix to have axes " "('freq', 'ra', 'dec', 'ra', 'dec'), " "but it has: " + str(noise_inv.axes)) raise ce.DataError(msg) # Arrange the dirty map as a vector. dirty_map_vect = sp.array(dirty_map) # A view. dirty_map_vect.shape = (shape[0], shape[1] * shape[2]) frequencies = dirty_map.get_axis('freq') / 1.0e6 # Allowcate memory only once. noise_inv_freq = sp.empty( (shape[1], shape[2], shape[1], shape[2]), dtype=float) if self.feedback > 1: print "Inverting noise matrix." # Block diagonal in frequency so loop over frequencies. for ii in xrange(dirty_map.shape[0]): if self.feedback > 1: print "Frequency: ", "%5.1f" % ( frequencies[ii]), if self.feedback > 2: print ", start mmap read:", sys.stdout.flush() noise_inv_freq[...] = noise_inv[ii, ...] if self.feedback > 2: print "done, start eig:", sys.stdout.flush() noise_inv_freq.shape = (shape[1] * shape[2], shape[1] * shape[2]) # Solve the map making equation by diagonalization. noise_inv_diag, Rot = sp.linalg.eigh( noise_inv_freq, overwrite_a=True) if self.feedback > 2: print "done", map_rotated = sp.dot(Rot.T, dirty_map_vect[ii]) # Zero out infinite noise modes. bad_modes = (noise_inv_diag < 1.0e-5 * noise_inv_diag.max()) if self.feedback > 1: print ", discarded: ", print "%4.1f" % (100.0 * sp.sum(bad_modes) / bad_modes.size), print "% of modes", if self.feedback > 2: print ", start rotations:", sys.stdout.flush() map_rotated[bad_modes] = 0. noise_inv_diag[bad_modes] = 1.0 # Solve for the clean map and rotate back. map_rotated /= noise_inv_diag map = sp.dot(Rot, map_rotated) if self.feedback > 2: print "done", sys.stdout.flush() # Fill the clean array. map.shape = (shape[1], shape[2]) clean_map[ii, ...] = map if save_noise_diag: # Using C = R Lambda R^T # where Lambda = diag(1/noise_inv_diag). temp_noise_diag = 1 / noise_inv_diag temp_noise_diag[bad_modes] = 0 # Multiply R by the diagonal eigenvalue matrix. # Broadcasting does equivalent of mult by diag # matrix. temp_mat = Rot * temp_noise_diag # Multiply by R^T, but only calculate the # diagonal elements. for jj in range(shape[1] * shape[2]): temp_noise_diag[jj] = sp.dot( temp_mat[jj, :], Rot[jj, :]) temp_noise_diag.shape = (shape[1], shape[2]) noise_diag[ii, ...] = temp_noise_diag # Return workspace memory to origional shape. noise_inv_freq.shape = (shape[1], shape[2], shape[1], shape[2]) if self.feedback > 1: print "" sys.stdout.flush() elif noise_inv.ndim == 6: if save_noise_diag: # OLD WAY. #clean_map, noise_diag, chol = solve(noise_inv, # dirty_map, True, feedback=self.feedback) # NEW WAY. clean_map, noise_diag, noise_inv_diag, chol = \ solve(noise_fname, noise_inv, dirty_map, True, feedback=self.feedback) else: # OLD WAY. #clean_map, chol = solve(noise_inv, dirty_map, # False, feedback=self.feedback) # NEW WAY. clean_map, noise_inv_diag, chol = \ solve(noise_fname, noise_inv, dirty_map, False, feedback=self.feedback) if params['save_cholesky']: chol_fname = (params['output_root'] + 'chol_' + pol_str + band_str + '.npy') sp.save(chol_fname, chol) if params['save_noise_inv_diag']: noise_inv_diag_fname = (params['output_root'] + 'noise_inv_diag_' + pol_str + band_str + '.npy') algebra.save(noise_inv_diag_fname, noise_inv_diag) # Delete the cholesky to recover memory. del chol else: raise ce.DataError("Noise matrix has bad shape.") # In all cases delete the noise object to recover memeory. del noise_inv # Write the clean map to file. out_fname = (params['output_root'] + 'clean_map_' + pol_str + band_str + '.npy') if self.feedback > 1: print "Writing clean map to: " + out_fname algebra.save(out_fname, clean_map) all_out_fname_list.append( kiyopy.utils.abbreviate_file_path(out_fname)) if save_noise_diag: noise_diag_fname = (params['output_root'] + 'noise_diag_' + pol_str + band_str + '.npy') algebra.save(noise_diag_fname, noise_diag) all_out_fname_list.append( kiyopy.utils.abbreviate_file_path(noise_diag_fname)) # Check the clean map for faileur. if not sp.alltrue(sp.isfinite(clean_map)): n_bad = sp.sum(sp.logical_not(sp.isfinite(clean_map))) msg = ("Non finite entries found in clean map. Solve" " failed. %d out of %d entries bad" % (n_bad, clean_map.size)) raise RuntimeError(msg)
def save_data(self, n_modes): prodmap_list = [] weight_list = [] n_modes = "%dmodes" % n_modes for pairitem in self.pairlist: pair = self.pairs[pairitem] (tag1, tag2) = (pair.map1_name, pair.map2_name) clnoise = "cleaned_noise_inv" map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag1, clnoise, tag2, n_modes) modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) #if pair.map1.shape == pair.map2.shape: map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag2, clnoise, tag1, n_modes) modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) if self.params['subtract_inputmap_from_sim'] or \ self.params['subtract_sim_from_inputmap']: map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1 map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2 elif self.params['subtract_realmap_from_sim']: if not os.path.exists(self.params['realmap_dir']): print "Error: Real map directory does not exists" exit() else: realmap_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy"%\ (self.params['realmap_dir'], tag1, tag2, n_modes) realmap = algebra.make_vect(algebra.load(realmap_file)) print "Subtract realmap from result" map1 = copy.deepcopy(pair.map1) - realmap map2 = copy.deepcopy(pair.map2) if map2.shape == map1.shape: map2 -= realmap else: map1 = copy.deepcopy(pair.map1) map2 = copy.deepcopy(pair.map2) prodmap_list.append(map1 * pair.noise_inv1) prodmap_list.append(map2 * pair.noise_inv2) weight_list.append(pair.noise_inv1) weight_list.append(pair.noise_inv2) if self.params['save_section']: algebra.save(map1_file, map1) algebra.save(noise_inv1_file, pair.noise_inv1) algebra.save(modes1_file, pair.left_modes) if pair.map1.shape == pair.map2.shape: algebra.save(map2_file, map2) algebra.save(noise_inv2_file, pair.noise_inv2) algebra.save(modes2_file, pair.right_modes) #if map2.shape[0] == 3*map1.shape[0]: # #source_dict = {} # #source_dict['map'] = map2_file # #source_dict['weight'] = noise_inv2_file # map_dict = {} # map_dict['map'] = map2 # map_dict['weight'] = pair.noise_inv2 # target_dict = {} # target_dict['imap'] = map2_file.replace('_'+tag2, '_'+tag2+'_I') # target_dict['qmap'] = map2_file.replace('_'+tag2, '_'+tag2+'_Q') # target_dict['umap'] = map2_file.replace('_'+tag2, '_'+tag2+'_U') # target_dict['imap_weight'] =\ # noise_inv2_file.replace('_'+tag2, '_'+tag2+'_I') # target_dict['qmap_weight'] =\ # noise_inv2_file.replace('_'+tag2, '_'+tag2+'_Q') # target_dict['umap_weight'] =\ # noise_inv2_file.replace('_'+tag2, '_'+tag2+'_U') # divide_iqu_map(map_dict=map_dict, target_dict=target_dict) if map1.shape != map2.shape: print "Shape of map1 and map2 are different, can not get combined map." else: cumulative_product = algebra.zeros_like(prodmap_list[0]) cumulative_weight = algebra.zeros_like(prodmap_list[0]) for mapind in range(0, len(prodmap_list)): cumulative_product += prodmap_list[mapind] cumulative_weight += weight_list[mapind] algebra.compressed_array_summary(cumulative_weight, "weight map") algebra.compressed_array_summary(cumulative_product, "product map") cumulative_weight[cumulative_weight < 1.e-20] = 0. cumulative_product[cumulative_weight < 1.e-20] = 0. newmap = cumulative_product / cumulative_weight # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. cumulative_product[nan_array] = 0. cumulative_weight[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. cumulative_product[inf_array] = 0. cumulative_weight[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(cumulative_product,"final map * weight") algebra.compressed_array_summary(cumulative_weight, "final weight map") combined = "combined_clean" combined_map_file = "%s/%s_map_%s.npy" % \ (self.output_root, combined, n_modes) combined_weight_file = "%s/%s_weight_%s.npy" % \ (self.output_root, combined, n_modes) combined_product_file = "%s/%s_product_%s.npy" % \ (self.output_root, combined, n_modes) combined_ones_file = "%s/%s_ones_%s.npy" % \ (self.output_root, combined, n_modes) algebra.save(combined_map_file, newmap) algebra.save(combined_product_file, cumulative_product) algebra.save(combined_weight_file, cumulative_weight) algebra.save(combined_ones_file, algebra.ones_like(newmap))
def load_pairs(self): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params if not par['pairlist']: if par['calc_diagnal']: noise_inv_suffix = ";noise_inv" else: noise_inv_suffix = ";noise_weight" (self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'], par['noise_inv1'], par['noise_inv2'], noise_inv_suffix=noise_inv_suffix, verbose=False, db_to_use=self.datapath_db) else: self.pairlist = par['pairlist'] pairdict = par['pairdict'] for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary(pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) # map1 & noise_inv1 map1 = algebra.make_vect(algebra.load(pdict['map1'])) if par['simfile1'] is not None: print "adding %s with multiplier %s" % (par['simfile1'], par['sim_multiplier']) sim1 = algebra.make_vect(algebra.load(par['simfile1'])) sim1 *= par['sim_multiplier'] else: sim1 = algebra.zeros_like(map1) if not par['no_weights']: noise_inv1 = wrap_find_weight(pdict['noise_inv1'], regenerate=par['regenerate_noise_inv'], calc_diagnal = par['calc_diagnal']) else: noise_inv1 = algebra.ones_like(map1) # map2 & noise_inv2 #if pairitem == 'I_with_E': if len(self.freq_list2) == 4*len(self.freq_list1): '''For IQUV case''' print 'Construct E map using I Q U V' iquvdict = {} iquvdict['imap'] = pdict['map2'].replace('_E', '_I') iquvdict['qmap'] = pdict['map2'].replace('_E', '_Q') iquvdict['umap'] = pdict['map2'].replace('_E', '_U') iquvdict['vmap'] = pdict['map2'].replace('_E', '_V') iquvdict['imap_weight'] = pdict['noise_inv2'].replace('_E', '_I') iquvdict['qmap_weight'] = pdict['noise_inv2'].replace('_E', '_Q') iquvdict['umap_weight'] = pdict['noise_inv2'].replace('_E', '_U') iquvdict['vmap_weight'] = pdict['noise_inv2'].replace('_E', '_V') map_dict = extend_iquv_map(source_dict=iquvdict) map2 = map_dict['map'] noise_inv2 = map_dict['weight'] sim2 = copy.deepcopy(sim1) map_dict = {} map_dict['imap'] = sim2 map_dict['qmap'] = algebra.zeros_like(sim1) map_dict['umap'] = algebra.zeros_like(sim1) map_dict['vmap'] = algebra.zeros_like(sim1) map_dict = extend_iquv_map(map_dict=map_dict) sim2 = map_dict['map'] elif len(self.freq_list2) == 3*len(self.freq_list1): '''For IQU case''' print 'Construct E map using I Q U' iquvdict = {} iquvdict['imap'] = pdict['map2'].replace('_E', '_I') iquvdict['qmap'] = pdict['map2'].replace('_E', '_Q') iquvdict['umap'] = pdict['map2'].replace('_E', '_U') iquvdict['imap_weight'] = pdict['noise_inv2'].replace('_E', '_I') iquvdict['qmap_weight'] = pdict['noise_inv2'].replace('_E', '_Q') iquvdict['umap_weight'] = pdict['noise_inv2'].replace('_E', '_U') map_dict = extend_iqu_map(source_dict=iquvdict) map2 = map_dict['map'] noise_inv2 = map_dict['weight'] sim2 = copy.deepcopy(sim1) map_dict = {} map_dict['imap'] = sim2 map_dict['qmap'] = algebra.zeros_like(sim1) map_dict['umap'] = algebra.zeros_like(sim1) map_dict = extend_iqu_map(map_dict=map_dict) sim2 = map_dict['map'] else: '''For common case''' map2 = algebra.make_vect(algebra.load(pdict['map2'])) if par['simfile2'] is not None: print "adding %s with multiplier %s" % (par['simfile2'], par['sim_multiplier']) sim2 = algebra.make_vect(algebra.load(par['simfile2'])) sim2 *= par['sim_multiplier'] else: sim2 = algebra.zeros_like(map2) if not par['no_weights']: noise_inv2 = wrap_find_weight(pdict['noise_inv2'], regenerate=par['regenerate_noise_inv'], calc_diagnal = par['calc_diagnal']) else: noise_inv2 = algebra.ones_like(map2) #if self.params['clip_weight_percent'] is not None: # print "Note: your are clipping the weight maps" # mask1 = self.define_weightmask(noise_inv1, # percentile=self.params['clip_weight_percent']) # mask2 = self.define_weightmask(noise_inv2, # percentile=self.params['clip_weight_percent']) # noise_inv1 = self.saturate_weight(noise_inv1, mask1) # noise_inv2 = self.saturate_weight(noise_inv2, mask2) pair = map_pair.MapPair(map1 + sim1, map2 + sim2, noise_inv1, noise_inv2, self.freq_list1, self.freq_list2) pair.set_names(pdict['tag1'], pdict['tag2']) pair.params = self.params self.pairs[pairitem] = pair if par['subtract_inputmap_from_sim'] or \ par['subtract_sim_from_inputmap']: if par['subtract_inputmap_from_sim']: pair_parallel_track = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list1, self.freq_list2) if par['subtract_sim_from_inputmap']: pair_parallel_track = map_pair.MapPair(sim1, sim2, noise_inv1, noise_inv2, self.freq_list1, self.freq_list2) pair_parallel_track.set_names(pdict['tag1'], pdict['tag2']) pair_parallel_track.params = self.params self.pairs_parallel_track[pairitem] = pair_parallel_track
def execute(self, nprocesses=1) : """Worker funciton.""" params = self.params # Make parent directory and write parameter file. kiyopy.utils.mkparents(params['output_root']) parse_ini.write_params(params, params['output_root'] + 'params.ini', prefix='mm_') save_noise_diag = params['save_noise_diag'] in_root = params['input_root'] all_out_fname_list = [] all_in_fname_list = [] # Loop over files to process. for pol_str in params['polarizations']: dmap_fname = in_root + 'dirty_map_' + pol_str + '.npy' noise_fname = in_root + 'noise_inv_' + pol_str + '.npy' all_in_fname_list.append( kiyopy.utils.abbreviate_file_path(dmap_fname)) all_in_fname_list.append( kiyopy.utils.abbreviate_file_path(noise_fname)) # Load the dirty map and the noise matrix. dirty_map = algebra.load(dmap_fname) dirty_map = algebra.make_vect(dirty_map) if dirty_map.axes != ('freq', 'ra', 'dec') : raise ce.DataError("Expeced dirty map to have axes " "('freq', 'ra', 'dec'), but it has axes: " + str(dirty_map.axes)) shape = dirty_map.shape noise_inv = algebra.open_memmap(noise_fname, 'r') noise_inv = algebra.make_mat(noise_inv) # Initialize the clean map. clean_map = algebra.info_array(sp.zeros(dirty_map.shape)) clean_map.info = dict(dirty_map.info) clean_map = algebra.make_vect(clean_map) # If needed, initialize a map for the noise diagonal. if save_noise_diag : noise_diag = algebra.zeros_like(clean_map) # Two cases for the noise. If its the same shape as the map then # the noise is diagonal. Otherwise, it should be block diagonal in # frequency. if noise_inv.ndim == 3 : if noise_inv.axes != ('freq', 'ra', 'dec') : raise ce.DataError("Expeced noise matrix to have axes " "('freq', 'ra', 'dec'), but it has: " + str(noise_inv.axes)) # Noise inverse can fit in memory, so copy it. noise_inv_memory = sp.array(noise_inv, copy=True) # Find the non-singular (covered) pixels. max_information = noise_inv_memory.max() good_data = noise_inv_memory < 1.0e-10*max_information # Make the clean map. clean_map[good_data] = (dirty_map[good_data] / noise_inv_memory[good_data]) if save_noise_diag : noise_diag[good_data] = 1/noise_inv_memory[good_data] elif noise_inv.ndim == 5 : if noise_inv.axes != ('freq', 'ra', 'dec', 'ra', 'dec') : raise ce.DataError("Expeced noise matrix to have axes " "('freq', 'ra', 'dec', 'ra', 'dec'), " "but it has: " + str(noise_inv.axes)) # Arrange the dirty map as a vector. dirty_map_vect = sp.array(dirty_map) # A view. dirty_map_vect.shape = (shape[0], shape[1]*shape[2]) frequencies = dirty_map.get_axis('freq')/1.0e6 # Allowcate memory only once. noise_inv_freq = sp.empty((shape[1], shape[2], shape[1], shape[2]), dtype=float) if self.feedback > 1 : print "Inverting noise matrix." # Block diagonal in frequency so loop over frequencies. for ii in xrange(dirty_map.shape[0]) : if self.feedback > 1: print "Frequency: ", "%5.1f"%(frequencies[ii]), if self.feedback > 2: print ", start mmap read:", sys.stdout.flush() noise_inv_freq[...] = noise_inv[ii, ...] if self.feedback > 2: print "done, start eig:", sys.stdout.flush() noise_inv_freq.shape = (shape[1]*shape[2], shape[1]*shape[2]) # Solve the map making equation by diagonalization. noise_inv_diag, Rot = sp.linalg.eigh(noise_inv_freq, overwrite_a=True) if self.feedback > 2: print "done", map_rotated = sp.dot(Rot.T, dirty_map_vect[ii]) # Zero out infinite noise modes. bad_modes = noise_inv_diag < 1.0e-5*noise_inv_diag.max() if self.feedback > 1: print ", discarded: ", print "%4.1f"%(100.0*sp.sum(bad_modes)/bad_modes.size), print "% of modes", if self.feedback > 2: print ", start rotations:", sys.stdout.flush() map_rotated[bad_modes] = 0. noise_inv_diag[bad_modes] = 1.0 # Solve for the clean map and rotate back. map_rotated /= noise_inv_diag map = sp.dot(Rot, map_rotated) if self.feedback > 2: print "done", sys.stdout.flush() # Fill the clean array. map.shape = (shape[1], shape[2]) clean_map[ii, ...] = map if save_noise_diag : # Using C = R Lambda R^T # where Lambda = diag(1/noise_inv_diag). temp_noise_diag = 1/noise_inv_diag temp_noise_diag[bad_modes] = 0 # Multiply R by the diagonal eigenvalue matrix. # Broadcasting does equivalent of mult by diag matrix. temp_mat = Rot*temp_noise_diag # Multiply by R^T, but only calculate the diagonal # elements. for jj in range(shape[1]*shape[2]) : temp_noise_diag[jj] = sp.dot(temp_mat[jj,:], Rot[jj,:]) temp_noise_diag.shape = (shape[1], shape[2]) noise_diag[ii, ...] = temp_noise_diag # Return workspace memory to origional shape. noise_inv_freq.shape = (shape[1], shape[2], shape[1], shape[2]) if self.feedback > 1: print "" sys.stdout.flush() elif noise_inv.ndim == 6 : raise NotImplementedError("Full noise matrix not yet " "implemented. Best we can do is " "block diagonal in frequency.") else : raise ce.DataError("Noise matrix has bad shape.") # Write the clean map to file. out_fname = params['output_root'] + 'clean_map_' + pol_str + '.npy' algebra.save(out_fname, clean_map) all_out_fname_list.append( kiyopy.utils.abbreviate_file_path(out_fname)) if save_noise_diag : noise_diag_fname = (params['output_root'] + 'noise_diag_' + pol_str + '.npy') algebra.save(noise_diag_fname, noise_diag) all_out_fname_list.append( kiyopy.utils.abbreviate_file_path(noise_diag_fname)) # Finally update the history object. history = hist.read(in_root + 'history.hist') history.add('Read map and noise files:', all_in_fname_list) history.add('Converted dirty map to clean map.', all_out_fname_list) h_fname = params['output_root'] + "history.hist" history.write(h_fname)
def apply(self, alg_ob, mode="constant", cval=0, right_apply=False): """Apply the beam, as a linear operator, to vector or matrix. This operation is equivalent to matrix multiplication by the beam matrix. The matrix multiplication can be performed with the beam matrix on either the left or the right. The beam matrix is a symmetric matrix. Parameters ---------- alg_ob: `vect` or `mat` subclass. Object to which the beam will be applied. This object must live in map space. If it is a vect, alg_ob.axes must be ('freq', 'ra', 'dec'). If it is a mat, and `right_apply` is False, then alg_ob.row_names() must return ('freq', 'ra', 'dec'). If `right_apply` is True, then alg_ob.col_names() should return this tuple. Also, the meta data for these three axis must be set (see `algebra.alg_object.set_axis_info`). wrap: bool If tests True, use periodic boundary conditions for the convolution. Otherwise zero pad (default). right_apply: bool Whether to apply the beam operator with the from the left (False, default) or from the right (True). If `alg_ob` is a vect subclass, this has no effect (because the beam matrix is symmetric). Returns ------- out: `vect` or `mat` subclass same shape as `alg_ob`. Convolved map or matrix. Notes ----- The missing feature here is the ability to preallocate memory or to be able to overwrite the input alg_ob in place. This will be important for large matrices that we can only hold in memory one at a time. Also this would be pretty easy to implement. """ if ((not 'freq' in alg_ob.axes) or (not 'ra' in alg_ob.axes) or (not 'dec' in alg_ob.axes)): raise ce.DataError("Beam operation only works in frequency, " "ra, dec, coords.") out = algebra.zeros_like(alg_ob) # Figure out the pixel sizes (in real degrees). dra = abs(alg_ob.info['ra_delta']) dra /= sp.cos(alg_ob.info['dec_centre'] * sp.pi / 180.) ddec = abs(alg_ob.info['dec_delta']) # Loop over frequencies and do convolution one frequency at a time. freq_array = alg_ob.get_axis('freq') for ii, freq in enumerate(freq_array): width = self.kernel_size(freq) # Make sure the dimensions are an odd number of pixels. nkx = width // abs(dra) if nkx % 2 == 0: nkx += 1 nky = width // abs(ddec) if nky % 2 == 0: nky += 1 # Calculate kernel lags. lagsx = (sp.arange(nkx, dtype=float) - (nkx - 1) // 2) * dra lagsy = (sp.arange(nky, dtype=float) - (nky - 1) // 2) * ddec lags_sq = lagsx[:, None] ** 2. + lagsy[None, :] ** 2. kernel = dra * ddec * self.beam_function(lags_sq, freq, squared_delta=True) if isinstance(alg_ob, algebra.vect): if alg_ob.axes != ('freq', 'ra', 'dec'): raise ce.DataError("Vector axis names must be exactly " "('freq', 'ra', 'dec')") convolve(alg_ob[ii, ...], kernel, output=out[ii], mode=mode, cval=cval) elif isinstance(alg_ob, algebra.mat): # If applying from the left, loop over columns and convolve # over rows. If applying from the right, do the opposite. if right_apply: if alg_ob.col_names() != ('freq', 'ra', 'dec'): raise ce.DataError("Matrix column axis names must be " "exactly ('freq', 'ra', 'dec')") iterator = alg_ob.iter_row_index() else: if alg_ob.row_names() != ('freq', 'ra', 'dec'): raise ce.DataError("Matrix row axis names must be " "exactly ('freq', 'ra', 'dec')") iterator = alg_ob.iter_col_index() for index in iterator: sub_mat = alg_ob[index] # Pick out this frequency. sub_mat = sub_mat[ii, ...] # make a view of the output array. sub_out = out[index] sub_out = sub_out[ii, ...] convolve(sub_mat, kernel, sub_out, mode=mode, cval=cval) return out
def solve_from_eig(noise_evalsinv, noise_evects, dirty_map, return_noise_diag=False, feedback=0): """Converts a dirty map to a clean map using the eigen decomposition of the noise inverse. """ # Check the shapes. if noise_evects.ndim != 4: raise ValueError("Expected 4D array for 'noise_evects`.") if noise_evalsinv.shape != (noise_evects.shape[-1],): raise ValueError("Wrong number of eigenvalues.") if dirty_map.shape != noise_evects.shape[:-1]: raise ValueError("Dirty map and noise don't have matching dimensions.") if dirty_map.size != noise_evects.shape[-1]: raise ValueError("Eigen space not the same total size as map space.") n = noise_evalsinv.shape[0] nf = dirty_map.shape[0] nr = dirty_map.shape[1] nd = dirty_map.shape[2] # Copy the eigenvalues. noise_evalsinv = noise_evalsinv.copy() # Find poorly constrained modes and zero them out. bad_inds = noise_evalsinv < 1./constants.T_huge**2 n_bad = sp.sum(bad_inds) if feedback > 1: print ("Discarding %d modes of %d. %f percent." % (n_bad, n, 100. * n_bad / n)) noise_evalsinv[bad_inds] = 1. # Rotate the dirty map into the diagonal noise space. if feedback > 1: print "Rotating map to eigenspace." map_rot = sp.zeros(n, dtype=sp.float64) for ii in xrange(nf): for jj in xrange(nr): for kk in xrange(nd): tmp = noise_evects[ii,jj,kk,:].copy() map_rot += dirty_map[ii,jj,kk] * tmp # Multiply by the (diagonal) noise (inverse, inverse). Zero out any poorly # constrained modes. map_rot[bad_inds] = 0 # Take inverse and multiply. map_rot = map_rot / noise_evalsinv # Now rotate back to the origional space. if feedback > 1: print "Rotating back to map space." clean_map = algebra.zeros_like(dirty_map) for ii in xrange(nf): for jj in xrange(nr): for kk in xrange(nd): tmp = noise_evects[ii,jj,kk,:].copy() clean_map[ii,jj,kk] = sp.sum(map_rot * tmp) if return_noise_diag: if feedback > 1: print "Getting noise diagonal." noise_diag = algebra.zeros_like(dirty_map) noise_evals = 1. / noise_evalsinv noise_evals[bad_inds] = constants.T_huge**2 for ii in xrange(nf): for jj in xrange(nr): for kk in xrange(nd): tmp = noise_evects[ii,jj,kk,:].copy() noise_diag[ii,jj,kk] = sp.sum(tmp**2 * noise_evals) return clean_map, noise_diag else: return clean_map
def execute(self, nprocesses=1) : """Worker funciton.""" params = self.params # Make parent directory and write parameter file. kiyopy.utils.mkparents(params['output_root']) parse_ini.write_params(params, params['output_root'] + 'params.ini', prefix=prefix) save_noise_diag = params['save_noise_diag'] in_root = params['input_root'] all_out_fname_list = [] all_in_fname_list = [] # Figure out what the band names are. bands = params['bands'] if not bands: map_files = glob.glob(in_root + 'dirty_map_' + pol_str + "_*.npy") bands = [] root_len = len(in_root + 'dirty_map_') for file_name in map_files: bands.append(file_name[root_len:-4]) # Loop over files to process. for pol_str in params['polarizations']: for band in bands: if band == -1: band_str = '' else: band_str = "_" + repr(band) dmap_fname = (in_root + 'dirty_map_' + pol_str + band_str + '.npy') all_in_fname_list.append( kiyopy.utils.abbreviate_file_path(dmap_fname)) # Load the dirty map and the noise matrix. dirty_map = algebra.load(dmap_fname) dirty_map = algebra.make_vect(dirty_map) if dirty_map.axes != ('freq', 'ra', 'dec') : msg = ("Expeced dirty map to have axes ('freq'," "'ra', 'dec'), but it has axes: " + str(dirty_map.axes)) raise ce.DataError(msg) shape = dirty_map.shape # Initialize the clean map. clean_map = algebra.info_array(sp.zeros(dirty_map.shape)) clean_map.info = dict(dirty_map.info) clean_map = algebra.make_vect(clean_map) # If needed, initialize a map for the noise diagonal. if save_noise_diag : noise_diag = algebra.zeros_like(clean_map) if params["from_eig"]: # Solving from eigen decomposition of the noise instead of # the noise itself. # Load in the decomposition. evects_fname = (in_root + 'noise_evects_' + pol_str + + band_str + '.npy') if self.feedback > 1: print "Using dirty map: " + dmap_fname print "Using eigenvectors: " + evects_fname evects = algebra.open_memmap(evects_fname, 'r') evects = algebra.make_mat(evects) evals_inv_fname = (in_root + 'noise_evalsinv_' + pol_str + "_" + repr(band) + '.npy') evals_inv = algebra.load(evals_inv_fname) evals_inv = algebra.make_mat(evals_inv) # Solve for the map. if params["save_noise_diag"]: clean_map, noise_diag = solve_from_eig(evals_inv, evects, dirty_map, True, self.feedback) else: clean_map = solve_from_eig(evals_inv, evects, dirty_map, False, self.feedback) # Delete the eigen vectors to recover memory. del evects else: # Solving from the noise. noise_fname = (in_root + 'noise_inv_' + pol_str + band_str + '.npy') if self.feedback > 1: print "Using dirty map: " + dmap_fname print "Using noise inverse: " + noise_fname all_in_fname_list.append( kiyopy.utils.abbreviate_file_path(noise_fname)) noise_inv = algebra.open_memmap(noise_fname, 'r') noise_inv = algebra.make_mat(noise_inv) # Two cases for the noise. If its the same shape as the map # then the noise is diagonal. Otherwise, it should be # block diagonal in frequency. if noise_inv.ndim == 3 : if noise_inv.axes != ('freq', 'ra', 'dec') : msg = ("Expeced noise matrix to have axes " "('freq', 'ra', 'dec'), but it has: " + str(noise_inv.axes)) raise ce.DataError(msg) # Noise inverse can fit in memory, so copy it. noise_inv_memory = sp.array(noise_inv, copy=True) # Find the non-singular (covered) pixels. max_information = noise_inv_memory.max() good_data = noise_inv_memory < 1.0e-10*max_information # Make the clean map. clean_map[good_data] = (dirty_map[good_data] / noise_inv_memory[good_data]) if save_noise_diag : noise_diag[good_data] = \ 1/noise_inv_memory[good_data] elif noise_inv.ndim == 5 : if noise_inv.axes != ('freq', 'ra', 'dec', 'ra', 'dec'): msg = ("Expeced noise matrix to have axes " "('freq', 'ra', 'dec', 'ra', 'dec'), " "but it has: " + str(noise_inv.axes)) raise ce.DataError(msg) # Arrange the dirty map as a vector. dirty_map_vect = sp.array(dirty_map) # A view. dirty_map_vect.shape = (shape[0], shape[1]*shape[2]) frequencies = dirty_map.get_axis('freq')/1.0e6 # Allowcate memory only once. noise_inv_freq = sp.empty((shape[1], shape[2], shape[1], shape[2]), dtype=float) if self.feedback > 1 : print "Inverting noise matrix." # Block diagonal in frequency so loop over frequencies. for ii in xrange(dirty_map.shape[0]) : if self.feedback > 1: print "Frequency: ", "%5.1f"%(frequencies[ii]), if self.feedback > 2: print ", start mmap read:", sys.stdout.flush() noise_inv_freq[...] = noise_inv[ii, ...] if self.feedback > 2: print "done, start eig:", sys.stdout.flush() noise_inv_freq.shape = (shape[1]*shape[2], shape[1]*shape[2]) # Solve the map making equation by diagonalization. noise_inv_diag, Rot = sp.linalg.eigh( noise_inv_freq, overwrite_a=True) if self.feedback > 2: print "done", map_rotated = sp.dot(Rot.T, dirty_map_vect[ii]) # Zero out infinite noise modes. bad_modes = (noise_inv_diag < 1.0e-5 * noise_inv_diag.max()) if self.feedback > 1: print ", discarded: ", print "%4.1f" % (100.0 * sp.sum(bad_modes) / bad_modes.size), print "% of modes", if self.feedback > 2: print ", start rotations:", sys.stdout.flush() map_rotated[bad_modes] = 0. noise_inv_diag[bad_modes] = 1.0 # Solve for the clean map and rotate back. map_rotated /= noise_inv_diag map = sp.dot(Rot, map_rotated) if self.feedback > 2: print "done", sys.stdout.flush() # Fill the clean array. map.shape = (shape[1], shape[2]) clean_map[ii, ...] = map if save_noise_diag : # Using C = R Lambda R^T # where Lambda = diag(1/noise_inv_diag). temp_noise_diag = 1/noise_inv_diag temp_noise_diag[bad_modes] = 0 # Multiply R by the diagonal eigenvalue matrix. # Broadcasting does equivalent of mult by diag # matrix. temp_mat = Rot*temp_noise_diag # Multiply by R^T, but only calculate the # diagonal elements. for jj in range(shape[1]*shape[2]) : temp_noise_diag[jj] = sp.dot( temp_mat[jj,:], Rot[jj,:]) temp_noise_diag.shape = (shape[1], shape[2]) noise_diag[ii, ...] = temp_noise_diag # Return workspace memory to origional shape. noise_inv_freq.shape = (shape[1], shape[2], shape[1], shape[2]) if self.feedback > 1: print "" sys.stdout.flush() elif noise_inv.ndim == 6 : if save_noise_diag: # OLD WAY. #clean_map, noise_diag, chol = solve(noise_inv, # dirty_map, True, feedback=self.feedback) # NEW WAY. clean_map, noise_diag, noise_inv_diag, chol = \ solve(noise_fname, noise_inv, dirty_map, True, feedback=self.feedback) else: # OLD WAY. #clean_map, chol = solve(noise_inv, dirty_map, # False, feedback=self.feedback) # NEW WAY. clean_map, noise_inv_diag, chol = \ solve(noise_fname, noise_inv, dirty_map, False, feedback=self.feedback) if params['save_cholesky']: chol_fname = (params['output_root'] + 'chol_' + pol_str + band_str + '.npy') sp.save(chol_fname, chol) if params['save_noise_inv_diag']: noise_inv_diag_fname = (params['output_root'] + 'noise_inv_diag_' + pol_str + band_str + '.npy') algebra.save(noise_inv_diag_fname, noise_inv_diag) # Delete the cholesky to recover memory. del chol else : raise ce.DataError("Noise matrix has bad shape.") # In all cases delete the noise object to recover memeory. del noise_inv # Write the clean map to file. out_fname = (params['output_root'] + 'clean_map_' + pol_str + band_str + '.npy') if self.feedback > 1: print "Writing clean map to: " + out_fname algebra.save(out_fname, clean_map) all_out_fname_list.append( kiyopy.utils.abbreviate_file_path(out_fname)) if save_noise_diag : noise_diag_fname = (params['output_root'] + 'noise_diag_' + pol_str + band_str + '.npy') algebra.save(noise_diag_fname, noise_diag) all_out_fname_list.append( kiyopy.utils.abbreviate_file_path(noise_diag_fname)) # Check the clean map for faileur. if not sp.alltrue(sp.isfinite(clean_map)): n_bad = sp.sum(sp.logical_not(sp.isfinite(clean_map))) msg = ("Non finite entries found in clean map. Solve" " failed. %d out of %d entries bad" % (n_bad, clean_map.size)) raise RuntimeError(msg)
def calculate_mixing(weight_file1, weight_file2, bins, xspec_fileout, mixing_fileout, unitless=False, refinement=2, pad=5, order=1, window='blackman', zero_pad=False, identity_test=False): print "loading the weights and converting to physical coordinates" weight1_obs = algebra.make_vect(algebra.load(weight_file1)) weight1 = bh.repackage_kiyo( pg.physical_grid(weight1_obs, refinement=refinement, pad=pad, order=order)) weight2_obs = algebra.make_vect(algebra.load(weight_file2)) weight2 = bh.repackage_kiyo( pg.physical_grid(weight2_obs, refinement=refinement, pad=pad, order=order)) if window: window_function = fftutil.window_nd(weight1.shape, name=window) weight1 *= window_function weight2 *= window_function print "calculating the cross-power of the spatial weighting functions" arr1 = algebra.ones_like(weight1) arr2 = algebra.ones_like(weight2) # no window applied here (applied above) xspec = pe.cross_power_est(weight1, weight2, arr1, arr2, window=None, nonorm=True) # for each point in the cube, find |k|, k_perp, k_parallel # TODO: speed this up by using one direct numpy call (not limiting) k_mag_arr = binning.radius_array(xspec) k_perp_arr = binning.radius_array(xspec, zero_axes=[0]) k_parallel_arr = binning.radius_array(xspec, zero_axes=[1, 2]) if unitless: xspec = pe.make_unitless(xspec, radius_arr=k_mag_arr) # NOTE: assuming lowest k bin has only one point in 3D k-space # could make this floor of dimensions divided by 2 also center_3d = np.transpose(np.transpose(np.where(k_mag_arr == 0.))[0]) # In the estimator, we devide by 1/sum(w1 * w2) to get most of the effect # of the weighing. The mixing matrix here can be thought of as a correction # that that diagonal-only estimate. leakage_ratio = xspec[center_3d[0], center_3d[1], center_3d[2]] / \ np.sum(weight1 * weight2) print "power leakage ratio: %10.5g" % leakage_ratio xspec /= np.sum(weight1 * weight2) print "partitioning the 3D kspace up into the 2D k bins" (kflat, ret_indices) = bin_indices_2d(k_perp_arr, k_parallel_arr, bins, bins) # perform a test where the window function is a delta function at the # origin so that the mixing matrix is identity if identity_test: xspec = algebra.zeros_like(xspec) xspec[center_3d[0], center_3d[1], center_3d[2]] = 1. # now save the window cross-power for downstream pooled users algebra.save(xspec_fileout, xspec) runlist = [] for bin_index in range(kflat.shape[0]): bin_3d = ret_indices[repr(bin_index)] if bin_3d is not None: runlist.append((xspec_fileout, bin_index, bins, bin_3d, center_3d)) pool = multiprocessing.Pool(processes=(multiprocessing.cpu_count() - 4)) # the longest runs get pushed to the end; randomize for better job packing random.shuffle(runlist) results = pool.map(sum_window, runlist) #gnuplot_single_slice(runlist[0]) # for troubleshooting # now save the results for post-processing params = { "unitless": unitless, "refinement": refinement, "pad": pad, "order": order, "window": window, "zero_pad": zero_pad, "identity_test": identity_test, "weight_file1": weight_file1, "weight_file2": weight_file2, "bins": bins } outshelve = shelve.open(mixing_fileout, "n") outshelve["params"] = params # parameters for this run outshelve["weight1"] = weight1 # weight map 1 outshelve["weight2"] = weight2 # weight map 2 outshelve["xspec"] = xspec # copy of the weight spectra outshelve["kflat"] = kflat # 2D k bin vector outshelve["bins_3d"] = ret_indices # indices to k3d for a 2d k bin outshelve["results"] = results # mixing matrix columns outshelve.close()
def solve_from_eig(noise_evalsinv, noise_evects, dirty_map, return_noise_diag=False, feedback=0): """Converts a dirty map to a clean map using the eigen decomposition of the noise inverse. """ # Check the shapes. if noise_evects.ndim != 4: raise ValueError("Expected 4D array for 'noise_evects`.") if noise_evalsinv.shape != (noise_evects.shape[-1], ): raise ValueError("Wrong number of eigenvalues.") if dirty_map.shape != noise_evects.shape[:-1]: raise ValueError("Dirty map and noise don't have matching dimensions.") if dirty_map.size != noise_evects.shape[-1]: raise ValueError("Eigen space not the same total size as map space.") n = noise_evalsinv.shape[0] nf = dirty_map.shape[0] nr = dirty_map.shape[1] nd = dirty_map.shape[2] # Copy the eigenvalues. noise_evalsinv = noise_evalsinv.copy() # Find poorly constrained modes and zero them out. bad_inds = noise_evalsinv < 1. / constants.T_huge**2 n_bad = sp.sum(bad_inds) if feedback > 1: print("Discarding %d modes of %d. %f percent." % (n_bad, n, 100. * n_bad / n)) noise_evalsinv[bad_inds] = 1. # Rotate the dirty map into the diagonal noise space. if feedback > 1: print "Rotating map to eigenspace." map_rot = sp.zeros(n, dtype=sp.float64) for ii in xrange(nf): for jj in xrange(nr): for kk in xrange(nd): tmp = noise_evects[ii, jj, kk, :].copy() map_rot += dirty_map[ii, jj, kk] * tmp # Multiply by the (diagonal) noise (inverse, inverse). Zero out any poorly # constrained modes. map_rot[bad_inds] = 0 # Take inverse and multiply. map_rot = map_rot / noise_evalsinv # Now rotate back to the origional space. if feedback > 1: print "Rotating back to map space." clean_map = algebra.zeros_like(dirty_map) for ii in xrange(nf): for jj in xrange(nr): for kk in xrange(nd): tmp = noise_evects[ii, jj, kk, :].copy() clean_map[ii, jj, kk] = sp.sum(map_rot * tmp) if return_noise_diag: if feedback > 1: print "Getting noise diagonal." noise_diag = algebra.zeros_like(dirty_map) noise_evals = 1. / noise_evalsinv noise_evals[bad_inds] = constants.T_huge**2 for ii in xrange(nf): for jj in xrange(nr): for kk in xrange(nd): tmp = noise_evects[ii, jj, kk, :].copy() noise_diag[ii, jj, kk] = sp.sum(tmp**2 * noise_evals) return clean_map, noise_diag else: return clean_map