def add_manual_mask(source_key, cut_freq_list=None, signal_name='map', noise_inv_name='noise_inv', weight_name='weight', divider_token=";"): r""" `source_key` is the file db key for the maps to combine `signal_name` is the tag in the file db entry for the signal maps `noise_inv_name` is the tag in the file db entry for the N^-1 weights `weight_name` is the tag in the file db entry for the weights to write out `divider_token` is the token that divides the map section name from the data type e.g. "A_with_B;noise_inv" """ datapath_db = data_paths.DataPath() source_fdb = datapath_db.fetch(source_key, silent=True) source_fdict = source_fdb[1] # accumulate all the files to combine noise_inv_keys = {} weight_keys = {} signal_keys = {} for filekey in source_fdb[0]: if divider_token in filekey: data_type = filekey.split(divider_token)[1] map_section = filekey.split(divider_token)[0] if data_type == signal_name: signal_keys[map_section] = source_fdict[filekey] if data_type == noise_inv_name: noise_inv_keys[map_section] = source_fdict[filekey] if data_type == weight_name: weight_keys[map_section] = source_fdict[filekey] for mapkey in signal_keys: signal_file = signal_keys[mapkey] noise_inv_file = noise_inv_keys[mapkey] weight_file = weight_keys[mapkey] print "loading pair: %s %s -> %s" % \ (signal_file, noise_inv_file, weight_file) signal_map = algebra.make_vect(algebra.load(signal_file)) weightmap = algebra.make_vect(algebra.load(noise_inv_file)) # set the new weights to zero where the N^-1 is small # or the signal map is inf or nan weightmap[np.isnan(weightmap)] = 0. weightmap[np.isinf(weightmap)] = 0. weightmap[np.isnan(signal_map)] = 0. weightmap[np.isinf(signal_map)] = 0. weightmap[weightmap < 1.e-20] = 0. if cut_freq_list is not None: for cutindex in cut_freq_list: weightmap[cutindex, :, :] = 0. # could also determine the filename here, outside of the database #outputdir = datapath_db.fetch_parent(source_key, return_path=True) #weight_out = "%s/%s" % (outputdir, source_key) algebra.compressed_array_summary(weightmap, "new weight map") algebra.save(weight_file, weightmap)
def make_combined(self, map_list, targetroot, file): imap_temp = algebra.make_vect(algebra.load(map_list[0])) cumulative_product = algebra.zeros_like(imap_temp) cumulative_weight = algebra.zeros_like(imap_temp) for dir in map_list: imap = algebra.make_vect(algebra.load(dir)) wmap = algebra.make_vect(algebra.load(dir.replace('clean_map', 'noise_inv'))) cumulative_product += imap*wmap cumulative_weight += wmap algebra.compressed_array_summary(cumulative_weight, "weight map") algebra.compressed_array_summary(cumulative_product, "product map") cumulative_weight[cumulative_weight < 1.e-20] = 0. cumulative_product[cumulative_weight < 1.e-20] = 0. cumulative_weight[cumulative_weight == 0] = np.inf cumulative_weight[np.isnan(cumulative_weight)] = np.inf newmap = cumulative_product / cumulative_weight cumulative_weight[np.isinf(cumulative_weight)] = 0. # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. cumulative_product[nan_array] = 0. cumulative_weight[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. cumulative_product[inf_array] = 0. cumulative_weight[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(cumulative_product,"final map * weight") algebra.compressed_array_summary(cumulative_weight, "final weight map") combined_map_file = targetroot + file combined_weight_file = targetroot + file.replace('map', 'weight') print combined_map_file print combined_weight_file algebra.save(combined_map_file, newmap) algebra.save(combined_weight_file, cumulative_weight)
def save_data(self, n_modes): prodmap_list = [] weight_list = [] n_modes = "%dmodes" % n_modes for pairitem in self.pairlist: pair = self.pairs[pairitem] (tag1, tag2) = (pair.map1_name, pair.map2_name) clnoise = "cleaned_noise_inv" map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag1, clnoise, tag2, n_modes) noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag2, clnoise, tag1, n_modes) modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) if self.params['subtract_inputmap_from_sim'] or \ self.params['subtract_sim_from_inputmap']: map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1 map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2 else: map1 = copy.deepcopy(pair.map1) map2 = copy.deepcopy(pair.map2) prodmap_list.append(map1 * pair.noise_inv1) prodmap_list.append(map2 * pair.noise_inv2) weight_list.append(pair.noise_inv1) weight_list.append(pair.noise_inv2) algebra.save(map1_file, map1) algebra.save(map2_file, map2) algebra.save(noise_inv1_file, pair.noise_inv1) algebra.save(noise_inv2_file, pair.noise_inv2) algebra.save(modes1_file, pair.left_modes) algebra.save(modes2_file, pair.right_modes) cumulative_product = algebra.zeros_like(prodmap_list[0]) cumulative_weight = algebra.zeros_like(prodmap_list[0]) for mapind in range(0, len(prodmap_list)): cumulative_product += prodmap_list[mapind] cumulative_weight += weight_list[mapind] algebra.compressed_array_summary(cumulative_weight, "weight map") algebra.compressed_array_summary(cumulative_product, "product map") cumulative_weight[cumulative_weight < 1.e-20] = 0. cumulative_product[cumulative_weight < 1.e-20] = 0. newmap = cumulative_product / cumulative_weight # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. cumulative_product[nan_array] = 0. cumulative_weight[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. cumulative_product[inf_array] = 0. cumulative_weight[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(cumulative_product, "final map * weight") algebra.compressed_array_summary(cumulative_weight, "final weight map") combined = "combined_clean" combined_map_file = "%s/%s_map_%s.npy" % \ (self.output_root, combined, n_modes) combined_weight_file = "%s/%s_weight_%s.npy" % \ (self.output_root, combined, n_modes) combined_product_file = "%s/%s_product_%s.npy" % \ (self.output_root, combined, n_modes) combined_ones_file = "%s/%s_ones_%s.npy" % \ (self.output_root, combined, n_modes) algebra.save(combined_map_file, newmap) algebra.save(combined_product_file, cumulative_product) algebra.save(combined_weight_file, cumulative_weight) algebra.save(combined_ones_file, algebra.ones_like(newmap))
def wigglez_correlation(init_filename): """Perform the cross-correlation between the WiggleZ dataset and GBT """ np.set_printoptions(threshold=np.nan) print "starting wigglez correlation run with file: " + init_filename params = parse_ini.parse(init_filename, params_default, prefix=prefix, feedback=10) radio_file = params['radio_root'] + params['radio_data_file'] noiseinv_file = params['radio_root'] + params['radio_noiseinv_file'] optical_file = params['optical_root'] + params['optical_data_file'] optical_selection_file = params['optical_root'] + \ params['optical_selection_file'] map_radio = algebra.make_vect(algebra.load(radio_file)) #noiseinv_radio = np.abs(algebra.make_vect(algebra.load(noiseinv_file))) noiseinv_radio = algebra.make_vect(algebra.load(noiseinv_file)) map_opt = algebra.make_vect(algebra.load(optical_file)) map_nbar = algebra.make_vect(algebra.load(optical_selection_file)) algebra.compressed_array_summary(map_opt, "opt map as loaded") algebra.compressed_array_summary(map_nbar, "nbar map as loaded") algebra.compressed_array_summary(map_radio, "radio map as loaded") algebra.compressed_array_summary(noiseinv_radio, "radio N^-1 as loaded") noiseinv_radio[noiseinv_radio < 1.e-20] = 0. nan_array = np.isnan(noiseinv_radio) noiseinv_radio[nan_array] = 0. inf_array = np.isinf(noiseinv_radio) noiseinv_radio[inf_array] = 0. if params['convolve']: beam_data = sp.array([0.316148488246, 0.306805630985, 0.293729620792, 0.281176247549, 0.270856788455, 0.26745856078, 0.258910010848, 0.249188429031]) # also consider the beam model where everything is degraded to the # lowest resolution; e.g. in a MapPair beam_data_degrade = sp.zeros(8) + max(beam_data) freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905], dtype=float) freq_data *= 1.e6 psf = beam.GaussianBeam(beam_data_degrade, freq_data) # Convolve the optical data to the lowest radio resolution print "convolving the map" map_opt = psf.apply(map_opt) algebra.compressed_array_summary(map_opt, "opt after convolution") # how should the covariance be convolved? # nbar is N^-1; convolve B N B^T?, # or just convolve nbar? use?: #map_nbar[map_nbar<1.e-30] = 1.e-30 #map_nbar = 1./map_nbar #map_nbar = psf.apply(map_nbar, cval=1.e30) #map_nbar = 1./map_nbar #map_nbar[map_nbar<1.e-20] = 0 print "convolving the covariance/selection" map_nbar = psf.apply(map_nbar) algebra.compressed_array_summary(map_nbar, "nbar after convolution") # convert to delta-overdensity map_opt = map_opt / map_nbar - 1. #algebra.compressed_array_summary(map_opt, "opt after conversion to delta") # set the NaNs and infs to zero in data and weights nan_array = np.isnan(map_opt) map_opt[nan_array] = 0. map_nbar[nan_array] = 0. inf_array = np.isinf(map_opt) map_opt[inf_array] = 0. map_nbar[inf_array] = 0. freqlist = params['freq'] # full: range(map_radio.shape[0]) algebra.compressed_array_summary(map_opt[freqlist, :, :], "opt map as entering the correlation function") algebra.compressed_array_summary(map_nbar[freqlist, :, :], "nbar as entering the correlation function") algebra.compressed_array_summary(map_radio[freqlist, :, :], "radio map as entering the correlation function") algebra.compressed_array_summary(noiseinv_radio[freqlist, :, :], "radio map N^-1 as entering the correlation function") cross_pair = fs.MapPair(map_opt, map_radio, map_nbar, noiseinv_radio, freqlist) if params['subtract_mean']: cross_pair.subtract_weighted_mean() (corr, counts) = cross_pair.correlate(params['lags'], speedup=params['speedup']) corr_shelve = shelve.open(params['output_shelve_file']) corr_shelve["corr"] = corr corr_shelve["counts"] = counts corr_shelve["freq_axis"] = map_radio.get_axis('freq') corr_shelve["params"] = params corr_shelve.close()
def radio_correlation(init_filename): """Perform the cross-correlation between two mock GBT cubes """ np.set_printoptions(threshold=np.nan) print "starting wigglez correlation run with file: " + init_filename params = parse_ini.parse(init_filename, params_default, prefix=prefix, feedback=10) radio_file1 = params["radio_root1"] + params["radio_data_file1"] noiseinv_file1 = params["radio_noiseroot1"] + params["radio_noiseinv_file1"] radio_file2 = params["radio_root2"] + params["radio_data_file2"] noiseinv_file2 = params["radio_noiseroot2"] + params["radio_noiseinv_file2"] map_radio1 = algebra.make_vect(algebra.load(radio_file1)) noiseinv_radio1 = algebra.make_vect(algebra.load(noiseinv_file1)) map_radio2 = algebra.make_vect(algebra.load(radio_file2)) noiseinv_radio2 = algebra.make_vect(algebra.load(noiseinv_file2)) algebra.compressed_array_summary(map_radio1, "radio 1 map as loaded") algebra.compressed_array_summary(noiseinv_radio1, "radio 1 N^-1 as loaded") algebra.compressed_array_summary(map_radio2, "radio 2 map as loaded") algebra.compressed_array_summary(noiseinv_radio2, "radio 2 N^-1 as loaded") noiseinv_radio1[noiseinv_radio1 < 1.0e-20] = 0.0 nan_array = np.isnan(noiseinv_radio1) noiseinv_radio1[nan_array] = 0.0 inf_array = np.isinf(noiseinv_radio1) noiseinv_radio1[inf_array] = 0.0 noiseinv_radio2[noiseinv_radio2 < 1.0e-20] = 0.0 nan_array = np.isnan(noiseinv_radio2) noiseinv_radio2[nan_array] = 0.0 inf_array = np.isinf(noiseinv_radio2) noiseinv_radio2[inf_array] = 0.0 if params["convolve"]: print "ERROR: simulations should be convolved by the beam already!" freqlist = params["freq"] # full: range(map_radio.shape[0]) algebra.compressed_array_summary(map_radio1[freqlist, :, :], "radio map 1 as entering the correlation function") algebra.compressed_array_summary( noiseinv_radio1[freqlist, :, :], "radio map 1 N^-1 as entering the correlation function" ) algebra.compressed_array_summary(map_radio2[freqlist, :, :], "radio map 2 as entering the correlation function") algebra.compressed_array_summary( noiseinv_radio2[freqlist, :, :], "radio map 2 N^-1 as entering the correlation function" ) cross_pair = fs.MapPair(map_radio1, map_radio2, noiseinv_radio1, noiseinv_radio2, freqlist) if params["subtract_mean"]: cross_pair.subtract_weighted_mean() (corr, counts) = cross_pair.correlate(params["lags"], speedup=params["speedup"]) corr_shelve = shelve.open(params["output_shelve_file"]) corr_shelve["corr"] = corr corr_shelve["counts"] = counts corr_shelve["freq_axis"] = map_radio1.get_axis("freq") corr_shelve["params"] = params corr_shelve.close()
def combine_maps_driver(inputmap_dict, inputweight_dict, output_dict, fullcov=False, datapath_db=None): r"""Combine a list of weights, maps specified by their database keys """ if datapath_db is None: datapath_db = data_paths.DataPath() signal_list = [] weight_list = [] for mapkey in inputmap_dict: signalfile = inputmap_dict[mapkey] weightfile = inputweight_dict[mapkey] print "loading pair: %s %s" % (signalfile, weightfile) signal_list.append(algebra.make_vect(algebra.load(signalfile))) if fullcov: raw_weight = algebra.make_mat( algebra.open_memmap(weightfile)) raw_weight = raw_weight.mat_diag() else: raw_weight = algebra.make_vect(algebra.load(weightfile)) # zero out any messy stuff raw_weight[raw_weight < 1.e-20] = 0. raw_weight[np.isnan(raw_weight)] = 0. raw_weight[np.isinf(raw_weight)] = 0. weight_list.append(raw_weight) prodmap = [] for mapind in range(0, len(signal_list)): prodmap.append(signal_list[mapind] * weight_list[mapind]) print "CHECK THESE: %d %d %d" % (len(signal_list), len(weight_list), len(prodmap)) cumulative_product = algebra.zeros_like(prodmap[0]) cumulative_weight = algebra.zeros_like(prodmap[0]) for mapind in range(0, len(signal_list)): cumulative_product += prodmap[mapind] cumulative_weight += weight_list[mapind] algebra.compressed_array_summary(cumulative_weight, "weight map") algebra.compressed_array_summary(cumulative_product, "product map") newmap = cumulative_product / cumulative_weight cumulative_weight[cumulative_weight < 1.e-20] = 0. cumulative_product[cumulative_weight < 1.e-20] = 0. # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. cumulative_product[nan_array] = 0. cumulative_weight[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. cumulative_product[inf_array] = 0. cumulative_weight[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(cumulative_product, "final map * weight") algebra.compressed_array_summary(cumulative_weight, "final weight map") print output_dict algebra.save(output_dict['map'], newmap) algebra.save(output_dict['product'], cumulative_product) algebra.save(output_dict['weight'], cumulative_weight) algebra.save(output_dict['ones'], algebra.ones_like(newmap))
def radio_correlation(init_filename): """Perform the cross-correlation between two mock GBT cubes """ np.set_printoptions(threshold=np.nan) print "starting wigglez correlation run with file: " + init_filename params = parse_ini.parse(init_filename, params_default, prefix=prefix, feedback=10) radio_file1 = params['radio_root1'] + params['radio_data_file1'] noiseinv_file1 = params['radio_noiseroot1'] + params['radio_noiseinv_file1'] radio_file2 = params['radio_root2'] + params['radio_data_file2'] noiseinv_file2 = params['radio_noiseroot2'] + params['radio_noiseinv_file2'] map_radio1 = algebra.make_vect(algebra.load(radio_file1)) noiseinv_radio1 = algebra.make_vect(algebra.load(noiseinv_file1)) map_radio2 = algebra.make_vect(algebra.load(radio_file2)) noiseinv_radio2 = algebra.make_vect(algebra.load(noiseinv_file2)) algebra.compressed_array_summary(map_radio1, "radio 1 map as loaded") algebra.compressed_array_summary(noiseinv_radio1, "radio 1 N^-1 as loaded") algebra.compressed_array_summary(map_radio2, "radio 2 map as loaded") algebra.compressed_array_summary(noiseinv_radio2, "radio 2 N^-1 as loaded") noiseinv_radio1[noiseinv_radio1 < 1.e-20] = 0. nan_array = np.isnan(noiseinv_radio1) noiseinv_radio1[nan_array] = 0. inf_array = np.isinf(noiseinv_radio1) noiseinv_radio1[inf_array] = 0. noiseinv_radio2[noiseinv_radio2 < 1.e-20] = 0. nan_array = np.isnan(noiseinv_radio2) noiseinv_radio2[nan_array] = 0. inf_array = np.isinf(noiseinv_radio2) noiseinv_radio2[inf_array] = 0. freqlist = params['freq'] # full: range(map_radio.shape[0]) algebra.compressed_array_summary( map_radio1[freqlist, :, :], "radio map 1 as entering the correlation function") algebra.compressed_array_summary( noiseinv_radio1[freqlist, :, :], "radio map 1 N^-1 as entering the correlation function") algebra.compressed_array_summary( map_radio2[freqlist, :, :], "radio map 2 as entering the correlation function") algebra.compressed_array_summary( noiseinv_radio2[freqlist, :, :], "radio map 2 N^-1 as entering the correlation function") cross_pair = fs.MapPair(map_radio1, map_radio2, noiseinv_radio1, noiseinv_radio2, freqlist) if params['subtract_mean']: cross_pair.subtract_weighted_mean() if params['convolve']: print "WARNING: you are degrading the beams" cross_pair.degrade_resolution() (corr, counts) = cross_pair.correlate(params['lags'], speedup=params['speedup']) corr_shelve = shelve.open(params['output_shelve_file']) corr_shelve["corr"] = corr corr_shelve["counts"] = counts corr_shelve["freq_axis"] = map_radio1.get_axis('freq') corr_shelve["params"] = params corr_shelve.close()
def combine_maps(param_dict, fullcov=False, verbose=False): """combines a list of maps as a weighted mean using a specified list of inverse covariance weights fullcov indicates that it is not just the diagonal and should be squashed """ print param_dict covlist = param_dict["covlist"] try: mul_cov_list = zip(covlist, param_dict["multiply_cov"]) print "using user-specified covariance multipliers" + repr(param_dict["multiply_cov"]) except KeyError: mul_cov_list = zip(covlist, [1.0] * len(covlist)) maps = [] for tagname in param_dict["maplist"]: if verbose: print tagname maps.append(algebra.make_vect(algebra.load(param_dict["root_data"] + tagname + ".npy"))) weights = [] for cov_entry in mul_cov_list: if verbose: print cov_entry (tagname, multiplier) = cov_entry if fullcov: raw_weight = algebra.make_mat(algebra.open_memmap(param_dict["root_cov"] + tagname + ".npy", mode="r")) raw_weight = raw_weight.mat_diag() else: raw_weight = algebra.make_vect(algebra.load(param_dict["root_cov"] + tagname + ".npy")) # zero out any messy stuff raw_weight *= multiplier raw_weight[raw_weight < 1.0e-20] = 0.0 raw_weight[np.isnan(raw_weight)] = 0.0 raw_weight[np.isinf(raw_weight)] = 0.0 weights.append(raw_weight) prodmap = [] for mapind in range(0, len(maps)): prodmap.append(maps[mapind] * weights[mapind]) for mapind in range(1, len(maps)): prodmap[0] += prodmap[mapind] weights[0] += weights[mapind] algebra.compressed_array_summary(weights[0], "weight map") algebra.compressed_array_summary(prodmap[0], "product map") newmap = prodmap[0] / weights[0] newweights = weights[0] newweights[newweights < 1.0e-20] = 0.0 # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0.0 newweights[nan_array] = 0.0 inf_array = np.isinf(newmap) newmap[inf_array] = 0.0 newweights[inf_array] = 0.0 algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(newweights, "final weight map") return (newmap, newweights, prodmap[0])
def save_data(self, n_modes): prodmap_list = [] weight_list = [] n_modes = "%dmodes" % n_modes for pairitem in self.pairlist: pair = self.pairs[pairitem] (tag1, tag2) = (pair.map1_name, pair.map2_name) clnoise = "cleaned_noise_inv" map1_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) noise_inv1_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag1, clnoise, tag2, n_modes) modes1_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag1, tag2, n_modes) #if pair.map1.shape == pair.map2.shape: map2_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) noise_inv2_file = "%s/sec_%s_%s_I_with_%s_%s.npy" % \ (self.output_root, tag2, clnoise, tag1, n_modes) modes2_file = "%s/sec_%s_modes_clean_map_I_with_%s_%s.npy" % \ (self.output_root, tag2, tag1, n_modes) if self.params['subtract_inputmap_from_sim'] or \ self.params['subtract_sim_from_inputmap']: map1 = pair.map1 - self.pairs_parallel_track[pairitem].map1 map2 = pair.map2 - self.pairs_parallel_track[pairitem].map2 elif self.params['subtract_realmap_from_sim']: if not os.path.exists(self.params['realmap_dir']): print "Error: Real map directory does not exists" exit() else: realmap_file = "%s/sec_%s_cleaned_clean_map_I_with_%s_%s.npy"%\ (self.params['realmap_dir'], tag1, tag2, n_modes) realmap = algebra.make_vect(algebra.load(realmap_file)) print "Subtract realmap from result" map1 = copy.deepcopy(pair.map1) - realmap map2 = copy.deepcopy(pair.map2) if map2.shape == map1.shape: map2 -= realmap else: map1 = copy.deepcopy(pair.map1) map2 = copy.deepcopy(pair.map2) prodmap_list.append(map1 * pair.noise_inv1) prodmap_list.append(map2 * pair.noise_inv2) weight_list.append(pair.noise_inv1) weight_list.append(pair.noise_inv2) if self.params['save_section']: algebra.save(map1_file, map1) algebra.save(noise_inv1_file, pair.noise_inv1) algebra.save(modes1_file, pair.left_modes) if pair.map1.shape == pair.map2.shape: algebra.save(map2_file, map2) algebra.save(noise_inv2_file, pair.noise_inv2) algebra.save(modes2_file, pair.right_modes) #if map2.shape[0] == 3*map1.shape[0]: # #source_dict = {} # #source_dict['map'] = map2_file # #source_dict['weight'] = noise_inv2_file # map_dict = {} # map_dict['map'] = map2 # map_dict['weight'] = pair.noise_inv2 # target_dict = {} # target_dict['imap'] = map2_file.replace('_'+tag2, '_'+tag2+'_I') # target_dict['qmap'] = map2_file.replace('_'+tag2, '_'+tag2+'_Q') # target_dict['umap'] = map2_file.replace('_'+tag2, '_'+tag2+'_U') # target_dict['imap_weight'] =\ # noise_inv2_file.replace('_'+tag2, '_'+tag2+'_I') # target_dict['qmap_weight'] =\ # noise_inv2_file.replace('_'+tag2, '_'+tag2+'_Q') # target_dict['umap_weight'] =\ # noise_inv2_file.replace('_'+tag2, '_'+tag2+'_U') # divide_iqu_map(map_dict=map_dict, target_dict=target_dict) if map1.shape != map2.shape: print "Shape of map1 and map2 are different, can not get combined map." else: cumulative_product = algebra.zeros_like(prodmap_list[0]) cumulative_weight = algebra.zeros_like(prodmap_list[0]) for mapind in range(0, len(prodmap_list)): cumulative_product += prodmap_list[mapind] cumulative_weight += weight_list[mapind] algebra.compressed_array_summary(cumulative_weight, "weight map") algebra.compressed_array_summary(cumulative_product, "product map") cumulative_weight[cumulative_weight < 1.e-20] = 0. cumulative_product[cumulative_weight < 1.e-20] = 0. newmap = cumulative_product / cumulative_weight # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. cumulative_product[nan_array] = 0. cumulative_weight[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. cumulative_product[inf_array] = 0. cumulative_weight[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(cumulative_product,"final map * weight") algebra.compressed_array_summary(cumulative_weight, "final weight map") combined = "combined_clean" combined_map_file = "%s/%s_map_%s.npy" % \ (self.output_root, combined, n_modes) combined_weight_file = "%s/%s_weight_%s.npy" % \ (self.output_root, combined, n_modes) combined_product_file = "%s/%s_product_%s.npy" % \ (self.output_root, combined, n_modes) combined_ones_file = "%s/%s_ones_%s.npy" % \ (self.output_root, combined, n_modes) algebra.save(combined_map_file, newmap) algebra.save(combined_product_file, cumulative_product) algebra.save(combined_weight_file, cumulative_weight) algebra.save(combined_ones_file, algebra.ones_like(newmap))
def wigglez_correlation(init_filename): """Perform the cross-correlation between the WiggleZ dataset and GBT """ np.set_printoptions(threshold=np.nan) print "starting wigglez correlation run with file: " + init_filename params = parse_ini.parse(init_filename, params_default, prefix=prefix, feedback=10) optical_file1 = params['optical_root1'] + params['optical_data_file1'] optical_selection_file1 = params['optical_root1'] + \ params['optical_selection_file1'] optical_file2 = params['optical_root2'] + params['optical_data_file2'] optical_selection_file2 = params['optical_root2'] + \ params['optical_selection_file2'] map_opt1 = algebra.make_vect(algebra.load(optical_file1)) map_nbar1 = algebra.make_vect(algebra.load(optical_selection_file1)) map_opt2 = algebra.make_vect(algebra.load(optical_file2)) map_nbar2 = algebra.make_vect(algebra.load(optical_selection_file2)) algebra.compressed_array_summary(map_opt1, "opt map 1 as loaded") algebra.compressed_array_summary(map_nbar1, "nbar map 1 as loaded") algebra.compressed_array_summary(map_opt2, "opt map 2 as loaded") algebra.compressed_array_summary(map_nbar2, "nbar map 2 as loaded") if params['convolve']: beam_data = sp.array([0.316148488246, 0.306805630985, 0.293729620792, 0.281176247549, 0.270856788455, 0.26745856078, 0.258910010848, 0.249188429031]) # also consider the beam model where everything is degraded to the # lowest resolution; e.g. in a MapPair beam_data_degrade = sp.zeros(8) + max(beam_data) freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905], dtype=float) freq_data *= 1.e6 psf = beam.GaussianBeam(beam_data_degrade, freq_data) # Convolve the optical data to the lowest radio resolution print "convolving the first map" map_opt1 = psf.apply(map_opt1) algebra.compressed_array_summary(map_opt1, "opt 1 after convolution") print "convolving the second map" map_opt2 = psf.apply(map_opt2) algebra.compressed_array_summary(map_opt2, "opt 2 after convolution") # how should the covariance be convolved? # nbar is N^-1; convolve B N B^T?, # or just convolve nbar? use?: #map_nbar[map_nbar<1.e-30] = 1.e-30 #map_nbar = 1./map_nbar #map_nbar = psf.apply(map_nbar, cval=1.e30) #map_nbar = 1./map_nbar #map_nbar[map_nbar<1.e-20] = 0 print "convolving the first covariance/selection" map_nbar1 = psf.apply(map_nbar1) algebra.compressed_array_summary(map_nbar1, "nbar 1 after convolution") print "convolving the second covariance/selection" map_nbar2 = psf.apply(map_nbar2) algebra.compressed_array_summary(map_nbar2, "nbar 2 after convolution") # convert to delta-overdensity map_opt1 = map_opt1 / map_nbar1 - 1. map_opt2 = map_opt2 / map_nbar2 - 1. #algebra.compressed_array_summary(map_opt1, "opt 1 after conversion to delta") #algebra.compressed_array_summary(map_opt2, "opt 2 after conversion to delta") # set the NaNs and infs to zero in data and weights nan_array = np.isnan(map_opt1) map_opt1[nan_array] = 0. map_nbar1[nan_array] = 0. inf_array = np.isinf(map_opt1) map_opt1[inf_array] = 0. map_nbar1[inf_array] = 0. nan_array = np.isnan(map_opt2) map_opt2[nan_array] = 0. map_nbar2[nan_array] = 0. inf_array = np.isinf(map_opt2) map_opt2[inf_array] = 0. map_nbar2[inf_array] = 0. freqlist = params['freq'] # full: range(map_radio.shape[0]) algebra.compressed_array_summary(map_opt1[freqlist, :, :], "opt map 1 as entering the correlation function") algebra.compressed_array_summary(map_nbar1[freqlist, :, :], "nbar 1 as entering the correlation function") algebra.compressed_array_summary(map_opt2[freqlist, :, :], "opt map 2 as entering the correlation function") algebra.compressed_array_summary(map_nbar2[freqlist, :, :], "nbar 2 N^-1 as entering the correlation function") cross_pair = fs.MapPair(map_opt1, map_opt2, map_nbar1, map_nbar2, freqlist) if params['subtract_mean']: cross_pair.subtract_weighted_mean() (corr, counts) = cross_pair.correlate(params['lags'], speedup=params['speedup']) corr_shelve = shelve.open(params['output_shelve_file']) corr_shelve["corr"] = corr corr_shelve["counts"] = counts corr_shelve["freq_axis"] = map_opt1.get_axis('freq') corr_shelve["params"] = params corr_shelve.close()
def combine_maps(source_key, alt_weight=None, signal='map', weight='weight', divider_token=";", fullcov=False): r""" `source_key` is the file db key for the maps to combine `alt_weight` is an optional alternate weighting for the maps `signal` is the tag in the file db entry for the signal maps `weight` is the tag in the file db entry for the N^-1 weights `fullcov` uses a memory map for large N^-1 and pulls the diagonal `divider_token` is the token that divides the map section name from the data type e.g. "A_with_B;noise_inv" """ datapath_db = data_paths.DataPath() source_fdb = datapath_db.fetch(source_key, intend_read=True, silent=True) source_fdict = source_fdb[1] # accumulate all the files to combine weightkeys = {} signalkeys = {} for filekey in source_fdb[0]: if divider_token in filekey: data_type = filekey.split(divider_token)[1] map_section = filekey.split(divider_token)[0] if data_type == signal: signalkeys[map_section] = source_fdict[filekey] if data_type == weight: weightkeys[map_section] = source_fdict[filekey] signal_list = [] weight_list = [] for mapkey in signalkeys: signalfile = signalkeys[mapkey] weightfile = weightkeys[mapkey] print "loading pair: %s %s" % (signalfile, weightfile) signal_list.append(algebra.make_vect(algebra.load(signalfile))) if fullcov: raw_weight = algebra.make_mat( algebra.open_memmap(weightfile)) raw_weight = raw_weight.mat_diag() else: raw_weight = algebra.make_vect(algebra.load(weightfile)) # zero out any messy stuff (should have been done already) raw_weight[raw_weight < 1.e-20] = 0. raw_weight[np.isnan(raw_weight)] = 0. raw_weight[np.isinf(raw_weight)] = 0. weight_list.append(raw_weight) prodmap = [] for mapind in range(0, len(signal_list)): prodmap.append(signal_list[mapind] * weight_list[mapind]) for mapind in range(1, len(signal_list)): prodmap[0] += prodmap[mapind] weight_list[0] += weight_list[mapind] algebra.compressed_array_summary(weight_list[0], "weight map") algebra.compressed_array_summary(prodmap[0], "product map") newmap = prodmap[0] / weight_list[0] newweights = weight_list[0] prodmap = prodmap[0] newweights[newweights < 1.e-20] = 0. prodmap[newweights < 1.e-20] = 0. # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. prodmap[nan_array] = 0. newweights[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. prodmap[inf_array] = 0. newweights[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(prodmap, "final map * weight") algebra.compressed_array_summary(newweights, "final weight map") # write out the three maps (combined map, map * weight, weight) signal_out = source_fdict['combined_map'] prodmap_out = source_fdict['combined_mapxweight'] weight_out = source_fdict['combined_weight'] # alternately #signal_out = "%s/%s_signal.npy" % (outputdir, source_key) #prodmap_out = "%s/%s_product.npy" % (outputdir, source_key) #weight_out = "%s/%s_weight.npy" % (outputdir, source_key) algebra.save(signal_out, newmap) algebra.save(prodmap_out, prodmap) algebra.save(weight_out, newweights)
def combine_maps(param_dict, fullcov=False, verbose=False): """combines a list of maps as a weighted mean using a specified list of inverse covariance weights fullcov indicates that it is not just the diagonal and should be squashed """ print param_dict covlist = param_dict["covlist"] try: mul_cov_list = zip(covlist, param_dict["multiply_cov"]) print "using user-specified covariance multipliers" + \ repr(param_dict["multiply_cov"]) except KeyError: mul_cov_list = zip(covlist, [1.] * len(covlist)) maps = [] for tagname in param_dict["maplist"]: if verbose: print tagname maps.append(algebra.make_vect( algebra.load(param_dict["root_data"] + tagname + ".npy"))) weights = [] for cov_entry in mul_cov_list: if verbose: print cov_entry (tagname, multiplier) = cov_entry if fullcov: raw_weight = algebra.make_mat( algebra.open_memmap(param_dict["root_cov"] + \ tagname + ".npy", mode='r')) raw_weight = raw_weight.mat_diag() else: raw_weight = algebra.make_vect(algebra.load( param_dict["root_cov"] + tagname + ".npy")) # zero out any messy stuff raw_weight *= multiplier raw_weight[raw_weight < 1.e-20] = 0. raw_weight[np.isnan(raw_weight)] = 0. raw_weight[np.isinf(raw_weight)] = 0. weights.append(raw_weight) prodmap = [] for mapind in range(0, len(maps)): prodmap.append(maps[mapind] * weights[mapind]) for mapind in range(1, len(maps)): prodmap[0] += prodmap[mapind] weights[0] += weights[mapind] algebra.compressed_array_summary(weights[0], "weight map") algebra.compressed_array_summary(prodmap[0], "product map") newmap = prodmap[0] / weights[0] newweights = weights[0] newweights[newweights < 1.e-20] = 0. # if the new map is nan or inf, set it and the wieghts to zero nan_array = np.isnan(newmap) newmap[nan_array] = 0. newweights[nan_array] = 0. inf_array = np.isinf(newmap) newmap[inf_array] = 0. newweights[inf_array] = 0. algebra.compressed_array_summary(newmap, "new map") algebra.compressed_array_summary(newweights, "final weight map") return (newmap, newweights, prodmap[0])
def radio_correlation(init_filename): """Perform the cross-correlation between two mock GBT cubes """ np.set_printoptions(threshold=np.nan) print "starting wigglez correlation run with file: " + init_filename params = parse_ini.parse(init_filename, params_default, prefix=prefix, feedback=10) radio_file1 = params['radio_root1'] + params['radio_data_file1'] noiseinv_file1 = params['radio_noiseroot1'] + params['radio_noiseinv_file1'] radio_file2 = params['radio_root2'] + params['radio_data_file2'] noiseinv_file2 = params['radio_noiseroot2'] + params['radio_noiseinv_file2'] map_radio1 = algebra.make_vect(algebra.load(radio_file1)) noiseinv_radio1 = algebra.make_vect(algebra.load(noiseinv_file1)) map_radio2 = algebra.make_vect(algebra.load(radio_file2)) noiseinv_radio2 = algebra.make_vect(algebra.load(noiseinv_file2)) algebra.compressed_array_summary(map_radio1, "radio 1 map as loaded") algebra.compressed_array_summary(noiseinv_radio1, "radio 1 N^-1 as loaded") algebra.compressed_array_summary(map_radio2, "radio 2 map as loaded") algebra.compressed_array_summary(noiseinv_radio2, "radio 2 N^-1 as loaded") noiseinv_radio1[noiseinv_radio1 < 1.e-20] = 0. nan_array = np.isnan(noiseinv_radio1) noiseinv_radio1[nan_array] = 0. inf_array = np.isinf(noiseinv_radio1) noiseinv_radio1[inf_array] = 0. noiseinv_radio2[noiseinv_radio2 < 1.e-20] = 0. nan_array = np.isnan(noiseinv_radio2) noiseinv_radio2[nan_array] = 0. inf_array = np.isinf(noiseinv_radio2) noiseinv_radio2[inf_array] = 0. freqlist = params['freq'] # full: range(map_radio.shape[0]) algebra.compressed_array_summary(map_radio1[freqlist, :, :], "radio map 1 as entering the correlation function") algebra.compressed_array_summary(noiseinv_radio1[freqlist, :, :], "radio map 1 N^-1 as entering the correlation function") algebra.compressed_array_summary(map_radio2[freqlist, :, :], "radio map 2 as entering the correlation function") algebra.compressed_array_summary(noiseinv_radio2[freqlist, :, :], "radio map 2 N^-1 as entering the correlation function") cross_pair = fs.MapPair(map_radio1, map_radio2, noiseinv_radio1, noiseinv_radio2, freqlist) if params['subtract_mean']: cross_pair.subtract_weighted_mean() if params['convolve']: print "WARNING: you are degrading the beams" cross_pair.degrade_resolution() (corr, counts) = cross_pair.correlate(params['lags'], speedup=params['speedup']) corr_shelve = shelve.open(params['output_shelve_file']) corr_shelve["corr"] = corr corr_shelve["counts"] = counts corr_shelve["freq_axis"] = map_radio1.get_axis('freq') corr_shelve["params"] = params corr_shelve.close()