def load_pairs(self): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params (self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'], par['noise_inv1'], par['noise_inv2'], noise_inv_suffix=";noise_weight", verbose=False, tack_on=self.tack_on_input, db_to_use=self.datapath_db) for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary(pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) map1 = algebra.make_vect(algebra.load(pdict['map1'])) map2 = algebra.make_vect(algebra.load(pdict['map2'])) if par['simfile'] is not None: print "adding %s with multiplier %s" % (par['simfile'], par['sim_multiplier']) sim = algebra.make_vect(algebra.load(par['simfile'])) sim *= par['sim_multiplier'] print sim.shape, map1.shape else: sim = algebra.zeros_like(map1) noise_inv1 = algebra.make_vect(algebra.load(pdict['noise_inv1'])) noise_inv2 = algebra.make_vect(algebra.load(pdict['noise_inv2'])) pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1, noise_inv2, self.freq_list) pair.set_names(pdict['tag1'], pdict['tag2']) pair.params = self.params self.pairs[pairitem] = pair if par['subtract_inputmap_from_sim'] or \ par['subtract_sim_from_inputmap']: if par['subtract_inputmap_from_sim']: pair_parallel_track = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list) if par['subtract_sim_from_inputmap']: pair_parallel_track = map_pair.MapPair(sim, sim, noise_inv1, noise_inv2, self.freq_list) pair_parallel_track.set_names(pdict['tag1'], pdict['tag2']) pair_parallel_track.params = self.params self.pairs_parallel_track[pairitem] = pair_parallel_track
def load_pairs(self): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params (self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'], par['noise_inv1'], par['noise_inv2'], noise_inv_suffix=";noise_weight", verbose=False, tack_on=self.tack_on_input, db_to_use=self.datapath_db) for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary( pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) map1 = algebra.make_vect(algebra.load(pdict['map1'])) map2 = algebra.make_vect(algebra.load(pdict['map2'])) if par['simfile'] is not None: print "adding %s with multiplier %s" % (par['simfile'], par['sim_multiplier']) sim = algebra.make_vect(algebra.load(par['simfile'])) sim *= par['sim_multiplier'] print sim.shape, map1.shape else: sim = algebra.zeros_like(map1) noise_inv1 = algebra.make_vect(algebra.load(pdict['noise_inv1'])) noise_inv2 = algebra.make_vect(algebra.load(pdict['noise_inv2'])) pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1, noise_inv2, self.freq_list) pair.set_names(pdict['tag1'], pdict['tag2']) pair.params = self.params self.pairs[pairitem] = pair if par['subtract_inputmap_from_sim'] or \ par['subtract_sim_from_inputmap']: if par['subtract_inputmap_from_sim']: pair_parallel_track = map_pair.MapPair( map1, map2, noise_inv1, noise_inv2, self.freq_list) if par['subtract_sim_from_inputmap']: pair_parallel_track = map_pair.MapPair( sim, sim, noise_inv1, noise_inv2, self.freq_list) pair_parallel_track.set_names(pdict['tag1'], pdict['tag2']) pair_parallel_track.params = self.params self.pairs_parallel_track[pairitem] = pair_parallel_track
def load_pairs(self, regenerate=True): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params (self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'], par['noise_inv1'], par['noise_inv2'], verbose=False) for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary( pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) map1 = algebra.make_vect(algebra.load(pdict['map1'])) map2 = algebra.make_vect(algebra.load(pdict['map2'])) sim = algebra.make_vect(algebra.load(par['simfile'])) if not par['no_weights']: noise_inv1 = self.process_noise_inv(pdict['noise_inv1'], regenerate=regenerate) noise_inv2 = self.process_noise_inv(pdict['noise_inv2'], regenerate=regenerate) else: noise_inv1 = algebra.ones_like(map1) noise_inv2 = algebra.ones_like(map2) pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1, noise_inv2, self.freq_list) pair.set_names(pdict['tag1'], pdict['tag2']) pair.lags = self.lags pair.params = self.params self.pairs[pairitem] = pair pair_nosim = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list) pair_nosim.set_names(pdict['tag1'], pdict['tag2']) pair_nosim.lags = self.lags pair_nosim.params = self.params self.pairs_nosim[pairitem] = pair_nosim
def load_ext_pairs(self, index, map1name, map2name, noise1name, noise2name): r"""Load the external datasets (which improve cleaning) """ par = self.params (self.pairlist_ext, pairdict) = dp.cross_maps(map1name, map2name, noise1name, noise2name, noise_inv_suffix=";noise_weight", verbose=False, db_to_use=self.datapath_db) # probably not wanted for external maps: # tack_on=self.tack_on_input, self.pairs_ext[index] = {} for pairitem in self.pairlist_ext: pdict = pairdict[pairitem] print "-" * 80 print "loading ext %s pair %s" % (index, pairitem) dp.print_dictionary( pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) map1 = algebra.make_vect(algebra.load(pdict['map1'])) map2 = algebra.make_vect(algebra.load(pdict['map2'])) noise_inv1 = algebra.make_vect(algebra.load(pdict['noise_inv1'])) noise_inv2 = algebra.make_vect(algebra.load(pdict['noise_inv2'])) pair = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list, conv_factor=self.conv_factor) pair.set_names(pdict['tag1'], pdict['tag2']) pair.params = self.params self.pairs_ext[index][pairitem] = pair
def load_pairs(self, regenerate=True): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params (self.pairlist, pairdict) = dp.cross_maps( par["map1"], par["map2"], par["noise_inv1"], par["noise_inv2"], verbose=False ) for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary(pdict, sys.stdout, key_list=["map1", "noise_inv1", "map2", "noise_inv2"]) map1 = algebra.make_vect(algebra.load(pdict["map1"])) map2 = algebra.make_vect(algebra.load(pdict["map2"])) sim = algebra.make_vect(algebra.load(par["simfile"])) if not par["no_weights"]: noise_inv1 = self.process_noise_inv(pdict["noise_inv1"], regenerate=regenerate) noise_inv2 = self.process_noise_inv(pdict["noise_inv2"], regenerate=regenerate) else: noise_inv1 = algebra.ones_like(map1) noise_inv2 = algebra.ones_like(map2) pair = map_pair.MapPair(map1 + sim, map2 + sim, noise_inv1, noise_inv2, self.freq_list) pair.set_names(pdict["tag1"], pdict["tag2"]) pair.lags = self.lags pair.params = self.params self.pairs[pairitem] = pair pair_nosim = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list) pair_nosim.set_names(pdict["tag1"], pdict["tag2"]) pair_nosim.lags = self.lags pair_nosim.params = self.params self.pairs_nosim[pairitem] = pair_nosim
def load_ext_pairs(self, index, map1name, map2name, noise1name, noise2name): r"""Load the external datasets (which improve cleaning) """ par = self.params (self.pairlist_ext, pairdict) = dp.cross_maps(map1name, map2name, noise1name, noise2name, noise_inv_suffix=";noise_weight", verbose=False, db_to_use=self.datapath_db) # probably not wanted for external maps: # tack_on=self.tack_on_input, self.pairs_ext[index] = {} for pairitem in self.pairlist_ext: pdict = pairdict[pairitem] print "-" * 80 print "loading ext %s pair %s" % (index, pairitem) dp.print_dictionary(pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) map1 = algebra.make_vect(algebra.load(pdict['map1'])) map2 = algebra.make_vect(algebra.load(pdict['map2'])) noise_inv1 = algebra.make_vect(algebra.load(pdict['noise_inv1'])) noise_inv2 = algebra.make_vect(algebra.load(pdict['noise_inv2'])) pair = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list, conv_factor=self.conv_factor) pair.set_names(pdict['tag1'], pdict['tag2']) pair.params = self.params self.pairs_ext[index][pairitem] = pair
def load_pairs(self): r"""load the set of map/noise pairs specified by keys handed to the database. This sets up operations on the quadratic product Q = map1^T noise_inv1 B noise_inv2 map2 """ par = self.params if not par['pairlist']: if par['calc_diagnal']: noise_inv_suffix = ";noise_inv" else: noise_inv_suffix = ";noise_weight" (self.pairlist, pairdict) = dp.cross_maps(par['map1'], par['map2'], par['noise_inv1'], par['noise_inv2'], noise_inv_suffix=noise_inv_suffix, verbose=False, db_to_use=self.datapath_db) else: self.pairlist = par['pairlist'] pairdict = par['pairdict'] for pairitem in self.pairlist: pdict = pairdict[pairitem] print "-" * 80 dp.print_dictionary(pdict, sys.stdout, key_list=['map1', 'noise_inv1', 'map2', 'noise_inv2']) # map1 & noise_inv1 map1 = algebra.make_vect(algebra.load(pdict['map1'])) if par['simfile1'] is not None: print "adding %s with multiplier %s" % (par['simfile1'], par['sim_multiplier']) sim1 = algebra.make_vect(algebra.load(par['simfile1'])) sim1 *= par['sim_multiplier'] else: sim1 = algebra.zeros_like(map1) if not par['no_weights']: noise_inv1 = wrap_find_weight(pdict['noise_inv1'], regenerate=par['regenerate_noise_inv'], calc_diagnal = par['calc_diagnal']) else: noise_inv1 = algebra.ones_like(map1) # map2 & noise_inv2 #if pairitem == 'I_with_E': if len(self.freq_list2) == 4*len(self.freq_list1): '''For IQUV case''' print 'Construct E map using I Q U V' iquvdict = {} iquvdict['imap'] = pdict['map2'].replace('_E', '_I') iquvdict['qmap'] = pdict['map2'].replace('_E', '_Q') iquvdict['umap'] = pdict['map2'].replace('_E', '_U') iquvdict['vmap'] = pdict['map2'].replace('_E', '_V') iquvdict['imap_weight'] = pdict['noise_inv2'].replace('_E', '_I') iquvdict['qmap_weight'] = pdict['noise_inv2'].replace('_E', '_Q') iquvdict['umap_weight'] = pdict['noise_inv2'].replace('_E', '_U') iquvdict['vmap_weight'] = pdict['noise_inv2'].replace('_E', '_V') map_dict = extend_iquv_map(source_dict=iquvdict) map2 = map_dict['map'] noise_inv2 = map_dict['weight'] sim2 = copy.deepcopy(sim1) map_dict = {} map_dict['imap'] = sim2 map_dict['qmap'] = algebra.zeros_like(sim1) map_dict['umap'] = algebra.zeros_like(sim1) map_dict['vmap'] = algebra.zeros_like(sim1) map_dict = extend_iquv_map(map_dict=map_dict) sim2 = map_dict['map'] elif len(self.freq_list2) == 3*len(self.freq_list1): '''For IQU case''' print 'Construct E map using I Q U' iquvdict = {} iquvdict['imap'] = pdict['map2'].replace('_E', '_I') iquvdict['qmap'] = pdict['map2'].replace('_E', '_Q') iquvdict['umap'] = pdict['map2'].replace('_E', '_U') iquvdict['imap_weight'] = pdict['noise_inv2'].replace('_E', '_I') iquvdict['qmap_weight'] = pdict['noise_inv2'].replace('_E', '_Q') iquvdict['umap_weight'] = pdict['noise_inv2'].replace('_E', '_U') map_dict = extend_iqu_map(source_dict=iquvdict) map2 = map_dict['map'] noise_inv2 = map_dict['weight'] sim2 = copy.deepcopy(sim1) map_dict = {} map_dict['imap'] = sim2 map_dict['qmap'] = algebra.zeros_like(sim1) map_dict['umap'] = algebra.zeros_like(sim1) map_dict = extend_iqu_map(map_dict=map_dict) sim2 = map_dict['map'] else: '''For common case''' map2 = algebra.make_vect(algebra.load(pdict['map2'])) if par['simfile2'] is not None: print "adding %s with multiplier %s" % (par['simfile2'], par['sim_multiplier']) sim2 = algebra.make_vect(algebra.load(par['simfile2'])) sim2 *= par['sim_multiplier'] else: sim2 = algebra.zeros_like(map2) if not par['no_weights']: noise_inv2 = wrap_find_weight(pdict['noise_inv2'], regenerate=par['regenerate_noise_inv'], calc_diagnal = par['calc_diagnal']) else: noise_inv2 = algebra.ones_like(map2) #if self.params['clip_weight_percent'] is not None: # print "Note: your are clipping the weight maps" # mask1 = self.define_weightmask(noise_inv1, # percentile=self.params['clip_weight_percent']) # mask2 = self.define_weightmask(noise_inv2, # percentile=self.params['clip_weight_percent']) # noise_inv1 = self.saturate_weight(noise_inv1, mask1) # noise_inv2 = self.saturate_weight(noise_inv2, mask2) pair = map_pair.MapPair(map1 + sim1, map2 + sim2, noise_inv1, noise_inv2, self.freq_list1, self.freq_list2) pair.set_names(pdict['tag1'], pdict['tag2']) pair.params = self.params self.pairs[pairitem] = pair if par['subtract_inputmap_from_sim'] or \ par['subtract_sim_from_inputmap']: if par['subtract_inputmap_from_sim']: pair_parallel_track = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list1, self.freq_list2) if par['subtract_sim_from_inputmap']: pair_parallel_track = map_pair.MapPair(sim1, sim2, noise_inv1, noise_inv2, self.freq_list1, self.freq_list2) pair_parallel_track.set_names(pdict['tag1'], pdict['tag2']) pair_parallel_track.params = self.params self.pairs_parallel_track[pairitem] = pair_parallel_track