def generate_proc_sim(input_file, weightfile, output_file, meansub=False, degrade=False): r"""make the maps with various combinations of beam conv/meansub""" print "%s -> %s (beam, etc.)" % (input_file, output_file) simmap = algebra.make_vect(algebra.load(input_file)) if degrade: print "performing common resolution convolution" beam_data = sp.array([0.316148488246, 0.306805630985, 0.293729620792, 0.281176247549, 0.270856788455, 0.26745856078, 0.258910010848, 0.249188429031]) freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905], dtype=float) freq_data *= 1.0e6 beam_diff = sp.sqrt(max(1.1 * beam_data) ** 2 - (beam_data) ** 2) common_resolution = beam.GaussianBeam(beam_diff, freq_data) # Convolve to a common resolution. simmap = common_resolution.apply(simmap) if meansub: print "performing mean subtraction" noise_inv = algebra.make_vect(algebra.load(weightfile)) means = sp.sum(sp.sum(noise_inv * simmap, -1), -1) means /= sp.sum(sp.sum(noise_inv, -1), -1) means.shape += (1, 1) simmap -= means # the weights will be zero in some places simmap[noise_inv < 1.e-20] = 0. # extra sanity? simmap[np.isinf(simmap)] = 0. simmap[np.isnan(simmap)] = 0. print "saving to" + output_file algebra.save(output_file, simmap)
def process_noise_inv(self, filename, regenerate=True): r"""buffer reading the noise inverse files for speed and also save to a file in the intermediate output path. If the cached file exists as an intermediate product, load it else produce it. """ if filename not in self.noisefiledict: basename = filename.split("/")[-1].split(".npy")[0] filename_diag = "%s/%s_diag.npy" % (self.output_root, basename) exists = os.access(filename_diag, os.F_OK) if exists and not regenerate: print "loading pre-diagonalized noise: " + filename_diag self.noisefiledict[filename] = algebra.make_vect(algebra.load(filename_diag)) else: print "loading noise: " + filename # TODO: have this be smarter about reading various noise cov # inputs noise_inv = algebra.make_mat(algebra.open_memmap(filename, mode="r")) self.noisefiledict[filename] = noise_inv.mat_diag() # self.noisefiledict[filename] = algebra.make_vect( # algebra.load(filename)) algebra.save(filename_diag, self.noisefiledict[filename]) return copy.deepcopy(self.noisefiledict[filename])
def map_pair_cal(uncal_maplist, uncal_weightlist, calfactor_outlist, dirtymap_inlist, dirtymap_outlist, convolve=True, factorizable_noise=False, sub_weighted_mean=True, freq_list=range(256)): map1file = reference_clean weight1file = reference_weight #map1file = uncal_maplist.pop(0) #weight1file = uncal_weightlist.pop(0) #calfactor_outlist.pop(0) #dirtymap_out0 = dirtymap_outlist.pop(0) #dirtymap_in0 = dirtymap_inlist.pop(0) # do nothing to the reference map #ref_dirtymap = algebra.make_vect(algebra.load(dirtymap_in0)) #algebra.save(dirtymap_out0, ref_dirtymap) # load maps into pairs svdout = shelve.open("correlation_pairs_v2.shelve") for map2file, weight2file, calfactor_outfile, \ dirty_infile, dirty_outfile in zip(uncal_maplist, \ uncal_weightlist, calfactor_outlist, dirtymap_inlist, dirtymap_outlist): print map1file, weight1file, map2file, weight2file pair = map_pair.MapPair(map1file, map2file, weight1file, weight2file, freq_list, avoid_db=True) if factorizable_noise: pair.make_noise_factorizable() if sub_weighted_mean: pair.subtract_weighted_mean() if convolve: pair.degrade_resolution() (corr, counts) = pair.correlate() svd_info = ce.get_freq_svd_modes(corr, len(freq_list)) svdout[map2file] = svd_info # write out the left right and cal factors leftmode = svd_info[1][0] rightmode = svd_info[2][0] calfactor = leftmode/rightmode facout = open(calfactor_outfile, "w") for outvals in zip(leftmode, rightmode, calfactor): facout.write("%10.15g %10.15g %10.15g\n" % outvals) facout.close() newmap = algebra.make_vect(algebra.load(dirty_infile)) newmap[freq_list, :, :] *= calfactor[:,np.newaxis,np.newaxis] algebra.save(dirty_outfile, newmap) print dirty_outfile svdout.close()
def plot_difference(filename1, filename2, title, sigmarange=6., sigmacut=None, transverse=False, outputdir="./", multiplier=1000., logscale=False, fractional=False, ignore=None, diff_filename="./difference.npy"): """make movies of the difference of two maps (assuming same dimensions)""" map1 = algebra.make_vect(algebra.load(filename1)) map2 = algebra.make_vect(algebra.load(filename2)) if fractional: difftitle = "fractional diff." dmap = (map1 - map2) / map1 * 100. else: difftitle = "difference" dmap = map1 - map2 algebra.save(diff_filename, dmap) make_cube_movie(diff_filename, difftitle, cube_frame_dir, sigmarange=6., sigmacut=sigmacut, outputdir=outputdir, ignore=ignore, multiplier=multiplier, transverse=transverse, logscale=False) make_cube_movie(filename1, title, cube_frame_dir, sigmarange=sigmarange, sigmacut=sigmacut, outputdir=outputdir, ignore=ignore, multiplier=multiplier, transverse=transverse, logscale=logscale, filetag_suffix="_1") make_cube_movie(filename2, title, cube_frame_dir, sigmarange=sigmarange, sigmacut=sigmacut, outputdir=outputdir, ignore=ignore, multiplier=multiplier, transverse=transverse, logscale=logscale, filetag_suffix="_2")
def setUp(self) : # Read in just to fiugre out the band structure. this_test_file = 'testdata/testfile_guppi_rotated.fits' Reader = fitsGBT.Reader(this_test_file, feedback=0) Blocks = Reader.read((0,),()) bands = () for Data in Blocks: n_chan = Data.dims[3] Data.calc_freq() freq = Data.freq delta = abs(sp.mean(sp.diff(freq))) centre = freq[n_chan//2] band = int(centre/1e6) bands += (band,) map = sp.zeros((n_chan, 15, 11)) map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec')) map.set_axis_info('freq', centre, -delta) map.set_axis_info('ra', 218, -0.2) map.set_axis_info('dec', 2, 0.2) algebra.save('./testout_clean_map_I_' + str(band) + '.npy', map) self.params = {'sm_input_root' : 'testdata/', 'sm_file_middles' : ("testfile",), 'sm_input_end' : "_guppi_rotated.fits", 'sm_output_root' : "./testout_", 'sm_output_end' : "_sub.fits", 'sm_solve_for_gain' : True, 'sm_gain_output_end' : 'gain.pickle', 'sm_map_input_root' : './testout_', 'sm_map_type' : 'clean_map_', 'sm_map_polarizations' : ('I',), 'sm_map_bands' : bands }
def add_sim_radio(): """script: go through a list of simulations and add those to a selected map """ root_file = "/mnt/raid-project/gmrt/eswitzer/wiggleZ/" radio_file = root_file + "modetest_combined_maps/combined_41-73_cleaned_clean_15.npy" root_sim = "/mnt/raid-project/gmrt/calinliv/wiggleZ/simulations/test100/" root_out = root_file + "simulations_plus_data/" radio_data = algebra.make_vect(algebra.load(radio_file)) for simindex in range(1,101): simname = root_sim + "simulated_signal_map_" + \ repr(simindex)+"_with_beam.npy" filename = root_out + "simulated_signal_plusdata_map_" + \ repr(simindex)+"_with_beam.npy" simoutname = root_out + "simulated_signal_map_" + \ repr(simindex)+"_with_beam.npy" sim_data = algebra.make_vect(algebra.load(simname)) sim_data /= 1000. outmap = copy.deepcopy(radio_data) outmap += sim_data algebra.save(filename, outmap) algebra.save(simoutname, sim_data) print filename
def setUp(self): # Read in just to fiugre out the band structure. this_test_file = 'testdata/testfile_guppi_rotated.fits' Reader = fitsGBT.Reader(this_test_file, feedback=0) Blocks = Reader.read((0, ), ()) bands = () for Data in Blocks: n_chan = Data.dims[3] Data.calc_freq() freq = Data.freq delta = abs(sp.mean(sp.diff(freq))) centre = freq[n_chan // 2] band = int(centre / 1e6) bands += (band, ) map = sp.zeros((n_chan, 15, 11)) map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec')) map.set_axis_info('freq', centre, -delta) map.set_axis_info('ra', 218, -0.2) map.set_axis_info('dec', 2, 0.2) algebra.save('./testout_clean_map_I_' + str(band) + '.npy', map) self.params = { 'sm_input_root': 'testdata/', 'sm_file_middles': ("testfile", ), 'sm_input_end': "_guppi_rotated.fits", 'sm_output_root': "./testout_", 'sm_output_end': "_sub.fits", 'sm_solve_for_gain': True, 'sm_gain_output_end': 'gain.pickle', 'sm_map_input_root': './testout_', 'sm_map_type': 'clean_map_', 'sm_map_polarizations': ('I', ), 'sm_map_bands': bands }
def mktmp(rgn_i,rgn_j,rgn_k,srgn_i1,srgn_i2,srgn_j1,srgn_j2,srgn_k1,srgn_k2,outfilename): """Write to disk a file representing an empty matrix of given dimensions. Also write an identically shaped array of booleans, which are true if the index points to the subregion. rgn_i/j/k : the dimensions of the full region to be simulated srgn_i/j/k : the dimensions of the deep integration subregion outfilename: the name of the file to be created """ regiontype = np.zeros((rgn_i,rgn_j,rgn_k), bool) array = np.zeros((rgn_i,rgn_j,rgn_k)) for i in range(0,rgn_i): for j in range(0,rgn_j): for k in range(0,rgn_k): if (i>=(srgn_i1-1) and i<=(srgn_i2-1)): if (j>=(srgn_j1-1) and j<=(srgn_j2-1)): if (k>=(srgn_k1-1) and k<=(srgn_k2-1)): regiontype[i,j,k]=True else: regiontype[i,j,k]=False region=algebra.info_array(array) regiontypename = 'bool' + outfilename np.save(regiontypename, regiontype) algebra.save(outfilename,region) print "done" template_map = algebra.make_vect(algebra.load(outfilename))
def process_noise_inv(self, filename, regenerate=True): r"""buffer reading the noise inverse files for speed and also save to a file in the intermediate output path. If the cached file exists as an intermediate product, load it else produce it. """ if filename not in self.noisefiledict: basename = filename.split("/")[-1].split(".npy")[0] filename_diag = "%s/%s_diag.npy" % \ (self.output_root, basename) exists = os.access(filename_diag, os.F_OK) if exists and not regenerate: print "loading pre-diagonalized noise: " + filename_diag self.noisefiledict[filename] = algebra.make_vect( algebra.load(filename_diag)) else: print "loading noise: " + filename # TODO: have this be smarter about reading various noise cov # inputs noise_inv = algebra.make_mat( algebra.open_memmap(filename, mode='r')) self.noisefiledict[filename] = noise_inv.mat_diag() #self.noisefiledict[filename] = algebra.make_vect( # algebra.load(filename)) algebra.save(filename_diag, self.noisefiledict[filename]) return copy.deepcopy(self.noisefiledict[filename])
def mklink_for_mappair(source_dict, target): r"""to make a link from target direction to the source_dic direction. """ if not os.path.exists(target): os.mkdir(target) for map_pair in source_dict.keys(): for key in ['map1', 'map2']: fileroot = source_dict[map_pair][key] filename = fileroot.split('/')[-1] if not os.path.exists(target+filename): print target+filename os.symlink(fileroot, target + filename) os.symlink(fileroot+'.meta', target+filename+'.meta') source_dict[map_pair][key] = target + filename for key in ['noise_inv1', 'noise_inv2']: fileroot = source_dict[map_pair][key] filename = fileroot.split('/')[-1] filename = filename.replace('inv', 'weight') if not os.path.exists(target+filename): #os.symlink(fileroot, target + filename) noise_inv_diag, info = find_weight_re_diagnal(fileroot) algebra.save(target+filename, noise_inv_diag) source_dict[map_pair][key] = target + filename return source_dict
def noise_inv_to_weight(noiseinvlist_in, weightlist_in): for (noiseinv_item, weight_item) in zip(noiseinvlist_in, weightlist_in): print noiseinv_item, weight_item noise_inv = algebra.make_mat( algebra.open_memmap(noiseinv_item, mode='r')) noise_inv_diag = noise_inv.mat_diag() algebra.save(weight_item, noise_inv_diag)
def add_manual_mask(source_key, cut_freq_list=None, signal_name='map', noise_inv_name='noise_inv', weight_name='weight', divider_token=";"): r""" `source_key` is the file db key for the maps to combine `signal_name` is the tag in the file db entry for the signal maps `noise_inv_name` is the tag in the file db entry for the N^-1 weights `weight_name` is the tag in the file db entry for the weights to write out `divider_token` is the token that divides the map section name from the data type e.g. "A_with_B;noise_inv" """ datapath_db = data_paths.DataPath() source_fdb = datapath_db.fetch(source_key, silent=True) source_fdict = source_fdb[1] # accumulate all the files to combine noise_inv_keys = {} weight_keys = {} signal_keys = {} for filekey in source_fdb[0]: if divider_token in filekey: data_type = filekey.split(divider_token)[1] map_section = filekey.split(divider_token)[0] if data_type == signal_name: signal_keys[map_section] = source_fdict[filekey] if data_type == noise_inv_name: noise_inv_keys[map_section] = source_fdict[filekey] if data_type == weight_name: weight_keys[map_section] = source_fdict[filekey] for mapkey in signal_keys: signal_file = signal_keys[mapkey] noise_inv_file = noise_inv_keys[mapkey] weight_file = weight_keys[mapkey] print "loading pair: %s %s -> %s" % \ (signal_file, noise_inv_file, weight_file) signal_map = algebra.make_vect(algebra.load(signal_file)) weightmap = algebra.make_vect(algebra.load(noise_inv_file)) # set the new weights to zero where the N^-1 is small # or the signal map is inf or nan weightmap[np.isnan(weightmap)] = 0. weightmap[np.isinf(weightmap)] = 0. weightmap[np.isnan(signal_map)] = 0. weightmap[np.isinf(signal_map)] = 0. weightmap[weightmap < 1.e-20] = 0. if cut_freq_list is not None: for cutindex in cut_freq_list: weightmap[cutindex, :, :] = 0. # could also determine the filename here, outside of the database #outputdir = datapath_db.fetch_parent(source_key, return_path=True) #weight_out = "%s/%s" % (outputdir, source_key) algebra.compressed_array_summary(weightmap, "new weight map") algebra.save(weight_file, weightmap)
def extend_iqu_map(source_dict=None, target_dict=None, map_dict=None): if source_dict != None: imap = algebra.make_vect(algebra.load(source_dict['imap'])) qmap = algebra.make_vect(algebra.load(source_dict['qmap'])) umap = algebra.make_vect(algebra.load(source_dict['umap'])) if source_dict.has_key('imap_weight'): imap_weight = algebra.make_vect(algebra.load(source_dict['imap_weight'])) qmap_weight = algebra.make_vect(algebra.load(source_dict['qmap_weight'])) umap_weight = algebra.make_vect(algebra.load(source_dict['umap_weight'])) elif source_dict.has_key('imap_inv'): imap_weight, info = find_weight_re_diagnal(source_dict['imap_inv']) qmap_weight, info = find_weight_re_diagnal(source_dict['qmap_inv']) umap_weight, info = find_weight_re_diagnal(source_dict['umap_inv']) else: print 'Warning: no weight' imap_weight = algebra.ones_like(imap) qmap_weight = algebra.ones_like(imap) umap_weight = algebra.ones_like(imap) elif map_dict != None: imap = map_dict['imap'] qmap = map_dict['qmap'] umap = map_dict['umap'] if 'imap_weight' in map_dict.keys(): imap_weight = map_dict['imap_weight'] qmap_weight = map_dict['qmap_weight'] umap_weight = map_dict['umap_weight'] else: print 'Warning: no weight' imap_weight = algebra.ones_like(imap) qmap_weight = algebra.ones_like(imap) umap_weight = algebra.ones_like(imap) else: print "Error: Can not find I Q U maps" exit() iqu = algebra.info_array(imap.tolist() + qmap.tolist() + umap.tolist()) iqu = algebra.make_vect(iqu) iqu.info = imap.info iqu.copy_axis_info(imap) iqu_weight = algebra.info_array(imap_weight.tolist() + qmap_weight.tolist() + umap_weight.tolist()) iqu_weight = algebra.make_vect(iqu_weight) iqu_weight.info = imap_weight.info iqu_weight.copy_axis_info(imap_weight) if target_dict != None: algebra.save(target_dict['map'], iqu) algebra.save(target_dict['weight'], iqu_weight) else: map_dict = {} map_dict['map'] = iqu map_dict['weight'] = iqu_weight return map_dict
def process_map(self, imap_fname, nmap_fname, ii, mock_fname=None): params = self.params sigma = params['sigma'] mu = params['mu'] out_root = params['output_root'] in_root = params['input_root'] imap = algebra.load(in_root + imap_fname) imap = algebra.make_vect(imap) #print imap.flatten().mean() imap = imap - imap.flatten().mean() if imap.axes != ('freq', 'ra', 'dec') : raise ce.DataError('AXES ERROR!') print ' :: Set Noise to Gaussian' np.random.seed() nmap = algebra.info_array( sigma*np.random.randn(imap.shape[0],imap.shape[1], imap.shape[2])+mu) nmap.axes = imap.axes nmap = algebra.make_vect(nmap) nmap.info = imap.info if nmap.axes != ('freq', 'ra', 'dec') : raise ce.DataError('AXES ERROR!') ## add noise to map ## imap = imap + nmap non0 = nmap.nonzero() nmap[non0] = (1./sigma)**2 #if mock_fname != None: # mmap = algebra.info_array( # 2.*np.random.randn(imap.shape[0],imap.shape[1], imap.shape[2])-0.5) # mmap.axes = imap.axes # mmap = algebra.make_vect(mmap) # box, nbox, mbox = self.fill(imap, nmap, mmap) # pkrm_nfname = out_root + 'fftbox_' + mock_fname # algebra.save(pkrm_nfname, mbox) #else: # box, nbox = self.fill(imap, nmap) hr = params['hr'] mid = params['mid'] last = params['last'] pol_str = params['polarizations'][0] end = pol_str if len(last)!=0: end = end + last[ii] end = end + '_' + str(ii) imap_fname = hr[ii] + mid[0] + end + '.npy' nmap_fname = hr[ii] + mid[1] + end + '.npy' pkrm_fname = out_root + imap_fname algebra.save(pkrm_fname, imap) pkrm_nfname = out_root + nmap_fname algebra.save(pkrm_nfname, nmap)
def noise_inv_to_weight(noiseinvlist_in, weightlist_in): print reference_noise_inv, reference_weight noise_inv = algebra.make_mat(algebra.open_memmap(reference_noise_inv, mode='r')) noise_inv_diag = noise_inv.mat_diag() algebra.save(reference_weight, noise_inv_diag) print "done with reference weights" for (noiseinv_item, weight_item) in zip(noiseinvlist_in, weightlist_in): print noiseinv_item, weight_item noise_inv = algebra.make_mat(algebra.open_memmap(noiseinv_item, mode='r')) noise_inv_diag = noise_inv.mat_diag() algebra.save(weight_item, noise_inv_diag)
def test_scheme(template_file, sim_filename1, sim_filename2): r"""look at some differences between maps""" template_map = algebra.make_vect(algebra.load(template_file)) gbtsim1 = realize_simulation(template_map, scenario="streaming", seed=5489, refinement=1.0) gbtsim2 = realize_simulation(template_map, seed=5489, refinement=1.0) sim_map1 = algebra.make_vect(gbtsim1, axis_names=("freq", "ra", "dec")) sim_map2 = algebra.make_vect(gbtsim2, axis_names=("freq", "ra", "dec")) sim_map1.copy_axis_info(template_map) sim_map2.copy_axis_info(template_map) algebra.save(sim_filename1, sim_map1) algebra.save(sim_filename2, sim_map2)
def convert_noiseinv_to_weight(mapkey): datapath_db = dp.DataPath() filedb = datapath_db.fetch(mapkey)[1] map_cases = datapath_db.fileset_cases(mapkey, "section;maptype") for section in map_cases['section']: noiseinv_file = filedb[section + ";noise_inv"] noiseweight_file = filedb[section + ";noise_weight"] print noiseinv_file, noiseweight_file noise_inv = algebra.make_mat(algebra.open_memmap(noiseinv_file, mode='r')) noise_inv_diag = noise_inv.mat_diag() algebra.save(noiseweight_file, noise_inv_diag)
def realmap(self): """bin the real WiggleZ catalog""" self.realmap_binning = bin_catalog_file(self.infile_data, self.freq_axis, self.ra_axis, self.dec_axis, skip_header=1) map_wigglez = algebra.make_vect(self.realmap_binning, axis_names=('freq', 'ra', 'dec')) map_wigglez.copy_axis_info(self.template_map) algebra.save(self.outfile_data, map_wigglez) return
def map_pair_cal(uncal_maplist, uncal_weightlist, in_path, out_path, convolve=False, factorizable_noise=True, sub_weighted_mean=True): # load maps into pairs for mapname, noisename in zip(uncal_maplist, uncal_weightlist): pair = map_pair.MapPair(map1, map2, noise_inv1, noise_inv2, self.freq_list) pair.set_names(pdict['tag1'], pdict['tag2']) pair.lags = self.lags pair.params = self.params self.pairs[pairitem] = pair (corr, counts) = pair.correlate(pair.lags, speedup=True) svd_info = ce.get_freq_svd_modes(corr, len(self.freq_list)) leftmode = svd_info[1][:0] rightmode =svd_info[2][:0] # write out the maps and noise algebra.save(map1_file, pair.map1) algebra.save(map2_file, pair.map2) algebra.save(noise_inv1_file, pair.noise_inv1) algebra.save(noise_inv2_file, pair.noise_inv2)
def test_scheme(template_file, sim_filename1, sim_filename2): r"""look at some differences between maps""" template_map = algebra.make_vect(algebra.load(template_file)) gbtsim1 = realize_simulation(template_map, scenario='streaming', seed=5489, refinement=1.) gbtsim2 = realize_simulation(template_map, seed=5489, refinement=1.) sim_map1 = algebra.make_vect(gbtsim1, axis_names=('freq', 'ra', 'dec')) sim_map2 = algebra.make_vect(gbtsim2, axis_names=('freq', 'ra', 'dec')) sim_map1.copy_axis_info(template_map) sim_map2.copy_axis_info(template_map) algebra.save(sim_filename1, sim_map1) algebra.save(sim_filename2, sim_map2)
def generate_delta_sim(input_file, output_file): r"""make the map with the temperature divided out (delta)""" print "reading %s -> %s (dividing by T_b(z))" % (input_file, output_file) simmap = algebra.make_vect(algebra.load(input_file)) freq_axis = simmap.get_axis('freq') / 1.e6 z_axis = units.nu21 / freq_axis - 1.0 simobj = corr21cm.Corr21cm() T_b = simobj.T_b(z_axis)*1e-3 simmap /= T_b[:, np.newaxis, np.newaxis] print "saving to" + output_file algebra.save(output_file, simmap)
def diag_noise(source_dict, target): if not os.path.exists(target): os.mkdir(target) for map_pair in source_dict.keys(): fileroot = source_dict[map_pair] filename = fileroot.split('/')[-1] filename = filename.replace('inv', 'weight') if not os.path.exists(target+filename): #os.symlink(fileroot, target + filename) noise_inv_diag, info = find_weight_re_diagnal(fileroot) algebra.save(target+filename, noise_inv_diag) source_dict[map_pair] = target + filename return source_dict
def setUp(self): # Make a map that the Covariance gets it's axis info from map = sp.zeros((5, 10, 8), dtype=float) map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec')) map.set_axis_info("freq", 800.0e6, 2.0e6) dec = 10 map.set_axis_info("ra", 215., 0.1 / sp.cos(dec * sp.pi / 180)) map.set_axis_info("dec", dec, 0.1) algebra.save("tmp_mapfile.npy", map) # Set up some parameters to pass. self.params = { "cv_map_file": "tmp_mapfile.npy", "cv_unit_system": "deg-freq", "cv_out_file_name": "testout_covariance.npy" }
def setUp(self) : # Make a map that the Covariance gets it's axis info from map = sp.zeros((5, 10, 8), dtype=float) map = algebra.make_vect(map, axis_names=('freq', 'ra', 'dec')) map.set_axis_info("freq", 800.0e6, 2.0e6) dec = 10 map.set_axis_info("ra", 215., 0.1/sp.cos(dec*sp.pi/180)) map.set_axis_info("dec", dec, 0.1) algebra.save("tmp_mapfile.npy", map) # Set up some parameters to pass. self.params = { "cv_map_file" : "tmp_mapfile.npy", "cv_unit_system" : "deg-freq", "cv_out_file_name" : "testout_covariance.npy" }
def delta(self): """find the overdensity using a separable selection function""" delta_data = self.produce_delta_map(self.outfile_data, self.outfile_separable) algebra.save(self.outfile_delta_data, delta_data) for mockindex in self.outfile_mock[0]: print "mock delta", mockindex mockinfile = self.outfile_mock[1][mockindex] mockoutfile = self.outfile_delta_mock[1][mockindex] delta_mock = self.produce_delta_map(mockinfile, self.outfile_separable) algebra.save(mockoutfile, delta_mock)
def inv_diag_noise(source_dict, target): if not os.path.exists(target): os.mkdir(target) for map_pair in source_dict.keys(): fileroot = source_dict[map_pair] filename = fileroot.split('/')[-1] filename = filename.replace('diag', 'weight') noise_diag = algebra.make_vect(algebra.load(fileroot)) noise_diag[noise_diag==0] = np.inf noise_inv_diag = 1./noise_diag algebra.save(target+filename, noise_inv_diag) source_dict[map_pair] = target + filename return source_dict
def execute_assembledir(self): # link the weights through to the simulation directory for (weight_file_in, weight_file_out) in \ zip(self.input_weight_maps, self.output_weight_maps): os.symlink(weight_file_in, weight_file_out) os.symlink(weight_file_in + ".meta", weight_file_out + ".meta") signalfile = self.output_root + self.output_signal signalmap = algebra.make_vect(algebra.load(signalfile)) signalmap *= self.multiplier # now load the signal simulation add thermal noise and save for (thermal_file, mapfile) in \ zip(self.output_thermal, self.output_maps): thermalmap = algebra.make_vect(algebra.load(thermal_file)) algebra.save(mapfile, signalmap + thermalmap)
def execute(self, nprocesses): params = self.params # Make parent directory and write parameter file. kiyopy.utils.mkparents(params['output_root']) parse_ini.write_params(params, params['output_root'] + 'params.ini', prefix=prefix) in_root = params['input_root'] # Figure out what the band names are. bands = params['bands'] if not bands: map_files = glob.glob(in_root + pol_str + "_*.npy") bands = [] root_len = len(in_root) for file_name in map_files: bands.append(file_name[root_len:-4]) # Loop over polarizations. for pol_str in params['polarizations']: # Read in all the maps to be glued. maps = [] for band in bands: band_map_fname = (in_root + pol_str + "_" + repr(band) + '.npy') if self.feedback > 1: print "Read using map: " + band_map_fname if params['mat_diag']: if self.feedback > 1: print "Treating as a matrix, getting diagonal." band_map = al.open_memmap(band_map_fname, mode='r') band_map = al.make_mat(band_map) band_map = band_map.mat_diag() else: band_map = al.load(band_map_fname) band_map = al.make_vect(band_map) if band_map.axes != ('freq', 'ra', 'dec'): msg = ("Expeced maps to have axes ('freq'," "'ra', 'dec'), but it has axes: " + str(band_map.axes)) raise ce.DataError(msg) maps.append(band_map) # Now glue them together. out_map = glue(maps) out_fname = (params['output_root'] + pol_str + "_" + "all" + '.npy') if self.feedback > 1: print "Writing glued map to: " + out_fname al.save(out_fname, out_map)
def execute(self, nprocesses): params = self.params # Make parent directory and write parameter file. kiyopy.utils.mkparents(params['output_root']) parse_ini.write_params(params, params['output_root'] + 'params.ini', prefix=prefix) in_root = params['input_root'] # Figure out what the band names are. bands = params['bands'] if not bands: map_files = glob.glob(in_root + pol_str + "_*.npy") bands = [] root_len = len(in_root) for file_name in map_files: bands.append(file_name[root_len:-4]) # Loop over polarizations. for pol_str in params['polarizations']: # Read in all the maps to be glued. maps = [] for band in bands: band_map_fname = (in_root + pol_str + "_" + repr(band) + '.npy') if self.feedback > 1: print "Read using map: " + band_map_fname if params['mat_diag']: if self.feedback > 1: print "Treating as a matrix, getting diagonal." band_map = al.open_memmap(band_map_fname, mode='r') band_map = al.make_mat(band_map) band_map = band_map.mat_diag() else: band_map = al.load(band_map_fname) band_map = al.make_vect(band_map) if band_map.axes != ('freq', 'ra', 'dec') : msg = ("Expeced maps to have axes ('freq'," "'ra', 'dec'), but it has axes: " + str(band_map.axes)) raise ce.DataError(msg) maps.append(band_map) # Now glue them together. out_map = glue(maps) out_fname = (params['output_root'] + pol_str + "_" + "all" + '.npy') if self.feedback > 1: print "Writing glued map to: " + out_fname al.save(out_fname, out_map)
def map_pair_cal( uncal_maplist, uncal_weightlist, calfactor_outlist, dirtymap_inlist, dirtymap_outlist, reference_mapfile, reference_weightfile, sub_weighted_mean=True, freq_list=range(256), ): reference_map = algebra.make_vect(algebra.load(reference_mapfile)) reference_weight = algebra.make_vect(algebra.load(reference_weightfile)) reference_map = remove_mean(reference_map, reference_weight) # load maps into pairs for mapfile, weightfile, calfactor_outfile, dirty_infile, dirty_outfile in zip( uncal_maplist, uncal_weightlist, calfactor_outlist, dirtymap_inlist, dirtymap_outlist ): print mapfile, weightfile session_map = algebra.make_vect(algebra.load(mapfile)) session_weight = algebra.make_vect(algebra.load(weightfile)) session_map = remove_mean(session_map, session_weight) calfactor = template_fit(session_map, reference_map, session_weight) newmap = algebra.make_vect(algebra.load(dirty_infile)) newmap[freq_list, :, :] /= calfactor[:, np.newaxis, np.newaxis] algebra.save(dirty_outfile, newmap) print dirty_outfile # optional test by applying the factor to the maps # session_map[freq_list, :, :] /= calfactor[:, np.newaxis, np.newaxis] # calfactor = template_fit(session_map, reference_map, session_weight) facout = open(calfactor_outfile, "w") for outvals in calfactor: facout.write("%10.15g\n" % outvals) facout.close()
def execute(self, nprocesses=1): params = self.params # Make parent directory and write parameter file. kiyopy.utils.mkparents(params['output_root']) parse_ini.write_params(params, params['output_root']+'params.ini',prefix='pk_') in_root = params['input_root'] out_root = params['output_root'] mid = params['mid'] all_out_fname_list = [] all_in_fname_list = [] #### Process #### pol_str = params['polarizations'][0] #hr_str = params['hr'][0] for hr_str, ii in zip(params['hr'],range(len(params['hr']))): end = pol_str if len(last)!=0: end = end + last[ii] imap_fname = in_root + hr_str + mid[0] + end + '.npy' imap = algebra.load(imap_fname) imap = algebra.make_vect(imap) if imap.axes != ('freq', 'ra', 'dec') : raise ce.DataError('AXES ERROR!') nmap_fname = in_root + hr_str + mid[1] + end + '.npy' nmap = algebra.load(nmap_fname) nmap = algebra.make_vect(nmap) #invers noise weight print 'Inverse Noise Weight... Map:' + hr_str[:-1] self.weight(imap, nmap, out_root+hr_str+'wt_cleaned_clean_map_'+end+'.png') dmap_fname = out_root + 'wt_' + hr_str + mid[0] + end + '.npy' algebra.save(dmap_fname, imap) all_out_fname_list.append( kiyopy.utils.abbreviate_file_path(dmap_fname)) nmap_fname = out_root + 'wt_' + hr_str + mid[1] + end + '.npy' algebra.save(nmap_fname, nmap) all_out_fname_list.append( kiyopy.utils.abbreviate_file_path(nmap_fname)) return 0
def save_and_plot(array, template, filename): array = algebra.make_vect(array, axis_names=('freq', 'ra', 'dec')) array.copy_axis_info(template) beam_data = sp.array([0.316148488246, 0.306805630985, 0.293729620792, 0.281176247549, 0.270856788455, 0.26745856078, 0.258910010848, 0.249188429031]) freq_data = sp.array([695, 725, 755, 785, 815, 845, 875, 905], dtype=float) freq_data *= 1.0e6 beamobj = beam.GaussianBeam(beam_data, freq_data) array_beam = beamobj.apply(array) algebra.save(filename, array_beam) outputdir = "/cita/d/www/home/eswitzer/movies/" pc.make_cube_movie(filename, "Temperature (mK)", pc.cube_frame_dir, sigmarange=3., outputdir=outputdir, multiplier=1000., transverse=False, filetag_suffix="_trial")
def extract(in_dir, out_dir) : """Searches for noise_inv files, extracts the diagonal and writes it out. """ files = glob.glob(in_dir + '/*noise_inv*.npy') for file_path in files: if 'noise_inv_diag' in file_path: continue file_name = file_path[len(in_dir):] print file_name parts = file_name.split('noise_inv') if len(parts) != 2: raise RuntimeError("'noise_inv' appears in file name more than" " once. Wasn't prepared for this.") out_path = out_dir + '/' + parts[0] + 'noise_inv_diag' + parts[1] mat = al.open_memmap(file_path, 'r') mat = al.make_mat(mat) mat_diag = mat.mat_diag() al.save(out_path, mat_diag)
def add_sim_to_data(simkey, datakey, replace=False): datapath_db = data_paths.DataPath() mapA_file = datapath_db.fetch(datakey + ":A;clean_map", intend_read=True) mapB_file = datapath_db.fetch(datakey + ":B;clean_map", intend_read=True) mapC_file = datapath_db.fetch(datakey + ":C;clean_map", intend_read=True) mapD_file = datapath_db.fetch(datakey + ":D;clean_map", intend_read=True) simfile = datapath_db.fetch(simkey + ":1", intend_read=True) simmap = algebra.make_vect(algebra.load(simfile)) mapset = [mapA_file, mapB_file, mapC_file, mapD_file] for mapfile in mapset: print mapfile, simfile origmap = algebra.make_vect(algebra.load(mapfile)) if replace: algebra.save(mapfile, simmap) else: algebra.save(mapfile, origmap + simmap)
def separable(self): # now assume separability of the selection function spatial_selection = np.sum(self.selection_function, axis=0) freq_selection = np.apply_over_axes(np.sum, self.selection_function, [1, 2]) self.separable_selection = (freq_selection * spatial_selection) self.separable_selection /= np.sum(freq_selection.flatten()) map_wigglez_separable = algebra.make_vect(self.separable_selection, axis_names=('freq', 'ra', 'dec')) map_wigglez_separable.copy_axis_info(self.template_map) algebra.save(self.outfile_separable, map_wigglez_separable) return
def process_optical_to_delta(optical_file, optical_selection_file, outfile): print "-" * 80 print "in: " + optical_file print "nbar: " + optical_selection_file print "out: " + outfile map_opt = algebra.make_vect(algebra.load(optical_file)) map_nbar = algebra.make_vect(algebra.load(optical_selection_file)) # convert to delta-overdensity map_opt = map_opt / map_nbar - 1. #algebra.compressed_array_summary(map_opt, "opt after conversion to delta") # set the NaNs and infs to zero in data and weights nan_array = np.isnan(map_opt) map_opt[nan_array] = 0. map_nbar[nan_array] = 0. inf_array = np.isinf(map_opt) map_opt[inf_array] = 0. map_nbar[inf_array] = 0. algebra.save(outfile, map_opt)
def execute(self, processes): freq_list = self.weight_map.get_axis("freq") delta_freq = np.roll(freq_list, 1) - freq_list bandwidth = delta_freq[1] # this may be too picky assert bandwidth == np.mean(delta_freq[1:]), "bad freq. axis" print "assuming uniform bandwidth: ", bandwidth integration_time = self.total_integration * 3600. freq_weight = np.apply_over_axes(np.sum, self.weight_map, [1, 2]) best_freq = np.argmax(freq_weight.flatten()) print "best freq: ", freq_list[best_freq] # normalize such that sum of the best slice is the total integration # time norm = integration_time / np.sum(self.weight_map[best_freq, :, :]) self.weight_map *= norm sys_temp = self.noise_model(freq_list[best_freq] / 1.e6) print "system temp at best: ", sys_temp #print self.weight_map[best_freq, :, :] delta_temp_map = sys_temp / np.sqrt(bandwidth * self.weight_map) delta_temp_map[np.isinf(delta_temp_map)] = self.max_stdev # print the delta T for the best slice in mK #print delta_temp_map[best_freq, :, :] * 1000. algebra.save(self.delta_temp_file, delta_temp_map) thermal_noise = np.random.randn(*delta_temp_map.shape) thermal_noise *= delta_temp_map thermal_noise = algebra.make_vect(thermal_noise, axis_names=('freq', 'ra', 'dec')) thermal_noise.copy_axis_info(delta_temp_map) #print thermal_noise[best_freq, :, :] * 1000. algebra.save(self.output_file, thermal_noise)
def make_modetest_combined_sim(): """combine output simulated maps from a mode subtraction test""" modedir = "/mnt/raid-project/gmrt/eswitzer/wiggleZ/modetest/" outdir = "/mnt/raid-project/gmrt/eswitzer/wiggleZ/modetest_combined_maps_0_50/" dirprefix = "73_ABCD_all_" data_dirsuffix = "_modes_sim_maponly/" cov_dirsuffix = "_modes_real_maponly/" for run_index in range(0,55,5): fullpath_data = modedir + dirprefix + repr(run_index) + data_dirsuffix fullpath_cov = modedir + dirprefix + repr(run_index) + cov_dirsuffix print fullpath_data, fullpath_cov fourway_split = make_fourway_list(fullpath_data, fullpath_cov) (map_out, weights_out, prodmap_out) = combine_maps(fourway_split) filename = outdir + "combined_sim_41-73_cleaned_clean_" + \ repr(run_index) + ".npy" algebra.save(filename, map_out) filename = outdir + "combined_sim_41-73_cleaned_noise_inv_" + \ repr(run_index) + ".npy" algebra.save(filename, weights_out) filename = outdir + "combined_sim_41-73_cleaned_product_" + \ repr(run_index) + ".npy" algebra.save(filename, prodmap_out)
def generate_windows(window="blackman"): datapath_db = data_paths.DataPath() # first generate a window for the full physical volume filename = datapath_db.fetch('simideal_15hr_physical', intend_read=True, pick='1') print filename pcube = algebra.make_vect(algebra.load(filename)) pwindow = algebra.make_vect(fftutil.window_nd(pcube.shape, name=window), axis_names=('freq', 'ra', 'dec')) pwindow.copy_axis_info(pcube) print pwindow.shape algebra.save("physical_window.npy", pwindow) # now generate one for the observed region and project onto the physical # volume. filename = datapath_db.fetch('simideal_15hr_beam', intend_read=True, pick='1') print filename ocube = algebra.make_vect(algebra.load(filename)) owindow = algebra.make_vect(fftutil.window_nd(ocube.shape, name=window), axis_names=('freq', 'ra', 'dec')) owindow.copy_axis_info(ocube) print owindow.shape print owindow.axes algebra.save("observed_window.npy", owindow) pwindow = physical_gridding.physical_grid(owindow, refinement=2) print pwindow.shape algebra.save("observed_window_physreg.npy", pwindow)
def complete_wigglez_regions(target_sample=0.25, multiplier=16, search_start=16): """15hr: ([214.00001499999999, 222.99998500000001], [3.0000000000000001e-06, 3.9999989999999999], [676383691.31904757, 946937167.84666669]) 22hr: ([322.00003099999998, 329.99996900000002], [-1.9999990000000001, 1.9999979999999999], [676383691.31904757, 946937167.84666669]) 1hr ([9.0000020000000003, 15.999999000000001], [-1.9999979999999999, 1.999997], [676383691.31904757, 946937167.84666669]) """ #print "proposed 15hr field dimensions" template_15hr = find_map_region(223., 214., 0., 4, target_sample=target_sample, multiplier=multiplier, search_start=search_start, exact_freq=False, max_freq=676383691.31904757 / 1.e6, min_freq=946937167.84666669 / 1.e6, n_freq=512) find_map_dimensions(template_15hr) algebra.save("templates/wigglez_15hr_complete.npy", template_15hr) #print "proposed 22hr field dimensions" template_22hr = find_map_region(330., 322., -2., 2., target_sample=target_sample, multiplier=multiplier, search_start=search_start, exact_freq=False, max_freq=676383691.31904757 / 1.e6, min_freq=946937167.84666669 / 1.e6, n_freq=512) find_map_dimensions(template_22hr) algebra.save("templates/wigglez_22hr_complete.npy", template_22hr) #print "proposed 1hr field dimensions" template_1hr = find_map_region(16., 9., -2., 2., target_sample=target_sample, multiplier=multiplier, search_start=search_start, exact_freq=False, max_freq=676383691.31904757 / 1.e6, min_freq=946937167.84666669 / 1.e6, n_freq=512) find_map_dimensions(template_1hr) algebra.save("templates/wigglez_1hr_complete.npy", template_1hr)
def data_binning(h5py_file, num_bins): file = h5py.File(h5py_file) # reading data from h5py posx = file['Positions']['x'][:] posy = file['Positions']['y'][:] posz = file['Positions']['z'][:] HI_mass = file['HI_Masses']['HI_Masses'][:] # creating info for binning function xedges = np.linspace(0, posx.max(), num_bins) yedges = np.linspace(0, posy.max(), num_bins) zedges = np.linspace(0, posz.max(), num_bins) sample = np.zeros((len(posx), 4)) sample[:, 0] = posx sample[:, 1] = posy sample[:, 2] = posz sample[:, 3] = HI_mass # calling binning function z = binning.histogram3d_weights(sample, xedges, yedges, zedges) # info for make_cube_movie.py x_delta = posx.max()/num_bins y_delta = posy.max()/num_bins z_delta = posz.max()/num_bins x_centre = posx.max()/2. y_centre = posy.max()/2. z_centre = posz.max()/2. info = {'ra_delta': x_delta, 'dec_delta': y_delta, 'dec_centre': y_centre, 'axes': ('freq', 'ra', 'dec'), 'ra_centre': x_centre, 'freq_centre': z_centre, 'freq_delta': z_delta, 'type': 'vect'} # save data for make_cube_movie.py map = algebra.make_vect(z, axis_names=('ra', 'dec', 'freq')) map.info = info save_file = open("/tmp/mufma/data/HI_prelim_40_bins_weights.npy", "w") algebra.save(save_file, map) save_file.close()
def map_pair_cal(uncal_maplist, uncal_weightlist, calfactor_outlist, dirtymap_inlist, dirtymap_outlist, reference_mapfile, reference_weightfile, sub_weighted_mean=True, freq_list=range(256)): reference_map = algebra.make_vect(algebra.load(reference_mapfile)) reference_weight = algebra.make_vect(algebra.load(reference_weightfile)) reference_map = remove_mean(reference_map, reference_weight) # load maps into pairs for mapfile, weightfile, calfactor_outfile, \ dirty_infile, dirty_outfile in zip(uncal_maplist, \ uncal_weightlist, calfactor_outlist, dirtymap_inlist, dirtymap_outlist): print mapfile, weightfile session_map = algebra.make_vect(algebra.load(mapfile)) session_weight = algebra.make_vect(algebra.load(weightfile)) session_map = remove_mean(session_map, session_weight) calfactor = template_fit(session_map, reference_map, session_weight) newmap = algebra.make_vect(algebra.load(dirty_infile)) newmap[freq_list, :, :] /= calfactor[:, np.newaxis, np.newaxis] algebra.save(dirty_outfile, newmap) print dirty_outfile # optional test by applying the factor to the maps #session_map[freq_list, :, :] /= calfactor[:, np.newaxis, np.newaxis] #calfactor = template_fit(session_map, reference_map, session_weight) facout = open(calfactor_outfile, "w") for outvals in calfactor: facout.write("%10.15g\n" % outvals) facout.close()
def selection(self): """bin the mock catalogs""" self.selection_function = np.zeros(self.template_map.shape) for mockindex in self.infile_mock[0]: print mockindex mockfile = self.infile_mock[1][mockindex] mock_binning = bin_catalog_file(mockfile, self.freq_axis, self.ra_axis, self.dec_axis, skip_header=1, mock=True) self.selection_function += mock_binning # if this binned mock catalog should be saved if mockindex in self.outfile_mock[0]: print "mock", self.outfile_mock[1][mockindex] map_wigglez_mock = algebra.make_vect(mock_binning, axis_names=('freq', 'ra', 'dec')) map_wigglez_mock.copy_axis_info(self.template_map) algebra.save(self.outfile_mock[1][mockindex], map_wigglez_mock) # adding the real map back to the selection function is a kludge which # ensures the selection function is not zero where there is real data # (limit of insufficient mocks) self.selection_function += self.realmap_binning self.selection_function /= float(len(self.infile_mock[0]) + 1) print np.mean(self.selection_function) map_wigglez_selection = algebra.make_vect(self.selection_function, axis_names=('freq', 'ra', 'dec')) map_wigglez_selection.copy_axis_info(self.template_map) algebra.save(self.outfile_selection, map_wigglez_selection) return
def make_individual(): fourway_split = make_fourway_list("/mnt/raid-project/gmrt/calinliv/wiggleZ/corr/84_ABCD_all_15_modes/", "/mnt/raid-project/gmrt/calinliv/wiggleZ/corr/84_ABCD_all_15_modes/", map_middle = "_22hr_41-84_cleaned_clean_map_I_with_", cov_middle = "_22hr_41-84_cleaned_noise_inv_I_with_") (map_out, weights_out, prodmap_out) = combine_maps(fourway_split) algebra.save("combined_22hr_41-84_cleaned_clean.npy", map_out) algebra.save("combined_22hr_41-84_cleaned_noise_inv.npy", weights_out) algebra.save("combined_22hr_41-84_cleaned_product.npy", prodmap_out)
def execute(self, processes): file_list_1 = self.datapath_db.fetch(self.params['map_key_1'], tack_on=self.params["tack_on_1"], silent=True) file_list_2 = self.datapath_db.fetch(self.params['map_key_2'], tack_on=self.params["tack_on_2"], silent=True) file_list_out = self.datapath_db.fetch( self.params['map_key_out'], tack_on=self.params["tack_on_out"], silent=True) for file_key in file_list_1[0]: infile = file_list_1[1][file_key] subfile = file_list_2[1][file_key] outfile = file_list_out[1][file_key] rootdir = "/".join(outfile.split("/")[0:-1]) if len(rootdir) > 0 and rootdir != ".": if not os.path.isdir(rootdir): print "print_multicolumn: making dir " + rootdir os.mkdir(rootdir) print "input: ", infile print "out: ", outfile if "map" in file_key: print "minus: ", subfile inmap = algebra.make_vect(algebra.load(infile)) submap = algebra.make_vect(algebra.load(subfile)) print inmap.shape, submap.shape algebra.save(outfile, inmap - submap) else: shutil.copy2(infile, outfile) shutil.copy2(infile + ".meta", outfile + ".meta")
def get_cached_physical(filename, refinement=2, pad=5, order=1): basename = filename.split(".")[0] phys_cachename = basename + "_physical.npy" chksum_cachename = basename + ".md5" print phys_cachename, chksum_cachename curr_chksum = ft.hashfile(filename) # ALSO CHECK IF THE PARAMS CHANGED! # try to get an existing checksum try: chkfile = open(chksum_cachename, "r") old_chksum = chkfile.read() chkfile.close() if old_chksum == curr_chksum: chksum_not_changed = True except IOError as e: chksum_not_changed = False if os.path.isfile(phys_cachename) and chksum_not_changed: print "using the cached file: " + phys_cachename ret_data = algebra.make_vect(algebra.load(chksum_cachename)) else: print "writing a physical cache for: " + filename # calculate the physical coordinate box obs_map = algebra.make_vect(algebra.load(filename)) ret_data = bh.repackage_kiyo(pg.physical_grid(obs_map, refinement=refinement, pad=pad, order=order)) algebra.save(chksum_cachename, ret_data) # save the new checksum chkfile = open(chksum_cachename, "w") chkfile.write(curr_chksum) chkfile.close() return ret_data