Mexw = lambda Wx, txt: np.load(cmb_dir+'cfht/W%i_M%s_nocut.npy'%(Wx,txt)) Me1_arr = [Mexw(Wx, 'e1w') for Wx in range(1,5)] Me2_arr = [Mexw(Wx, 'e2w') for Wx in range(1,5)] Mwm_arr = [Mexw(Wx, 'wm') for Wx in range(1,5)] def randmap (iseedWx): iseed, Wx = iseedWx Me1, Me2 = Me1_arr[Wx-1], Me2_arr[Wx-1] Mwm = Mwm_arr[Wx-1] Me1rnd, Me2rnd = WLanalysis.rndrot(Me1, Me2, iseed=iseed) Me1smooth = WLanalysis.weighted_smooth(Me1rnd, Mwm) Me2smooth = WLanalysis.weighted_smooth(Me2rnd, Mwm) kmap_rand = WLanalysis.KSvw(Me1smooth, Me2smooth) print Wx, iseed, kmap_rand.shape np.save(bmap_fn(Wx, iseed), kmap_rand) p.map(randmap, [[iseed, Wx] for iseed in arange(500) for Wx in range(1,5)]) print 'done creating 500 noise KS maps' if cross_correlate_cmbl_noise: cmblmap_arr = map(cmblGen, range(1,5)) mask_arr = map(maskGen, range(1,5)) masked_cmbl_arr = [cmblmap_arr[i]*mask_arr[i] for i in range(4)] def cmblxNoise(iinput): '''iinput = (Wx, iseed) return the cross power between cmbl and convergence maps, both with smoothed mask. ''' Wx, iseed = iinput print 'cmblxNoise', Wx, iseed bmap = bmapGen(Wx, iseed)*mask_arr[Wx-1] cmblmap = masked_cmbl_arr[Wx-1] edges = edgesGen(Wx)
############################ ## test on one map ############################ # map_stats (cosmo_tomo_cone_arr[1]) ######################################## ################## mass production block ######################################## pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) process_arr = bias_tomo_cone_arr + cov_tomo_cone_arr + cosmo_tomo_cone_arr out = pool.map(map_stats, process_arr) # out=pool.map(map_stats, bias_tomo_cone_arr) # out=pool.map(map_stats, cov_tomo_cone_arr) pool.close() ######################################## ########## redo the problematic one ######################################## # cosmo_tomo_cone_arr_fix = [[cosmos[0], tomo, 2] # for tomo in range(1,6)] # # out=map(map_stats, cosmo_tomo_cone_arr) # for icosmo in cosmo_tomo_cone_arr_fix: # print (icosmo) # map_stats(icosmo)
L @ ((Smat * H) * H[:, np.newaxis] + np.diag(N * (100 - 99 * H))) @ np.conjugate(L.T)), (nf2 + nf2 // 4, nt2 + nt2 // 4))[i * nf2 - f_start:(i + 1) * nf2 - f_start, j * nt2 - t_start:(j + 1) * nt2 - t_start] print('Rank %s (%s,%s): %s' % (rank, i, j, MPI.Wtime() - ts), flush=True) return (i, j, filt, err) comm.Barrier() pool = MPIPool(loadbalance=True) if not pool.is_master(): pool.wait() else: print('Filter Intensity', flush=True) results = pool.map(I_filter, tasks) I_filt = np.zeros((nf, nt)) I_filt_err = np.zeros((nf, nt)) for i, j, arr, err in results: I_filt[i * nf2:(i + 1) * nf2, j * nt2:(j + 1) * nt2] = arr.real I_filt_err[i * nf2:(i + 1) * nf2, j * nt2:(j + 1) * nt2] = err.real np.save('%s/DB_b0834_mfilt_I_%s.npy' % (dirname, specs), I_filt) np.save('%s/DB_b0834_mfilt_I_err_%s.npy' % (dirname, specs), I_filt_err) pool.close() comm.Barrier() if rank > 0: ##GBAR Power Spectrum fft_1[:] = np.copy(GBAR) fft_object_F()
theta = sqrt((x_fore-x_back)**2+(y_fore-y_back)**2) print '%i\t%s\t%.2f\t%.3f\t%.3f\t%.4f\t%.6f'%(i, jj,log10(jMvir), z_fore, z_back, rad2arcmin(theta), kappa_temp) return ikappa #a=map(kappa_individual_gal, randint(0,len(idx_back)-1,5)) step=2e3 def temp (ix): print ix temp_fn = obsPK_dir+'temp/kappa_proj%i_%07d.npy'%(Wx, ix) if not os.path.isfile(temp_fn): kappa_all = map(kappa_individual_gal, arange(ix, amin([len(idx_back), ix+step]))) np.save(temp_fn,kappa_all) pool = MPIPool() ix_arr = arange(0, len(idx_back), step) pool.map(temp, ix_arr) all_kappa_proj = concatenate([np.load(obsPK_dir+'temp/kappa_proj%i_%07d.npy'%(Wx, ix)) for ix in ix_arr]) np.save(obsPK_dir+'kappa_predict_W%i.npy'%(Wx), all_kappa_proj) ######################################################### ####################### plotting correlation ############ ######################################################### make_predict_maps = 0 plot_predict_maps = 0 peak_proj_vs_lensing = 1 cross_correlate = 0 #kmap_predict_Gen = lambda Wx, sigmaG: np.load(obsPK_dir+'maps/r20arcmin_varyingcNFW_VO06/kmap_W%i_predict_sigmaG%02d.npy'%(Wx, sigmaG*10)) kmap_predict_Gen = lambda Wx, sigmaG: np.load(obsPK_dir+'maps/kmap_W%i_predict_sigmaG%02d.npy'%(Wx, sigmaG*10))
import WLanalysis import numpy as np from scipy import * from emcee.utils import MPIPool import sys CMBlensing_dir = '/work/02977/jialiu/CMBnonGaussian/' PPA = 2048.0/(sqrt(12.25)*60.0) sigmaG = PPA*1.0#9.7523809523809533 kmapGen = lambda r: WLanalysis.readFits('/scratch/02977/jialiu/CMB_hopper/CMB_batch_storage/Om0.296_Ol0.704_w-1.000_si0.786/1024b600/Maps/WLconv_z1100.00_%04dr.fits'%(r)) def k3Gen(r): print r kmap=kmapGen(r) kmap_smoothed=WLanalysis.smooth(kmap,sigmaG) k3=mean(kmap_smoothed**3) return k3 pool=MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) k3_arr = array(pool.map(k3Gen, range(1,10241))) save(CMBlensing_dir+'k3_arr.npy',k3_arr) pool.close() print 'done-done-done'
while jj< len(y): print 'icount, jj',icount,jj iipix_mask,iipix = WLanalysis.coords2grid(x[jj:jj+istep], y[jj:jj+istep], idata.flatten().reshape(1,-1)[:,jj:jj+istep], size=sizes[Wx-1]) ipix_mask += iipix_mask ipix += iipix jj+=istep print icount,'W%i done coords2grid %s'%(Wx,icount)#,time.strftime("%Y-%m-%d %H:%M") save(mask_dir+'smaller/weight0_W%i_%i_numpix'%(Wx,icount), ipix) save(mask_dir+'smaller/weight0_W%i_%i_nummask'%(Wx,icount), ipix_mask) #ipix is the num. of pixels fall in that big pix, ipix_mask is the mask return ipix, ipix_mask p = MPIPool() if not p.is_master(): p.wait() sys.exit(0) #p.map(partialdata2grid, range(63)) ismall_map=p.map(partialdata2grid, range(63)) small_map = sum(array(ismall_map),axis=0) save(mask_dir+'weight0_W%i_smaller_mask.npy'%(Wx),small_map) weight=1-small_map[1]/small_map[0] weight[isnan(weight)]=0 save(mask_dir+'ludoweight_weight0_W%i.npy'%Wx, weight) mask=weight/weight mask[isnan(mask)]=0 save(mask_dir+'ludomask_weight0_W%i.npy'%Wx, mask) p.close()
###### create new gadget snapshot ############ halo_snap = Gadget2Snapshot() hg = Gadget2Snapshot.open(snap_fn_arr[0]).header #header_gadget halo_snap.setPositions(array(halo_position)*halo_ID_position[0][1].unit) halo_snap.setHeaderInfo(Om0=hg['Om0'], Ode0=hg['Ode0'], w0=hg['w0'], wa=hg['wa'], h=hg['h'], redshift=hg['redshift'], box_size=hg['box_size']) ###### write the new snapshot to file ########## halo_snap.write(new_snap_fn, files = len(snap_fn_arr)) print "WROTE", new_snap_fn ### test on laptop #halo_snap.write('snapshots_amiga/snapshot_%03d'%(snap_id), files = len(snap_fn_arr)) print 'start job' pool.map(halo_particles, [[ID, snap_id] for ID in ID_arr for snap_id in snap_id_arr]) ################ there're problems if only 8 amiga files, so re-write into 16 files ############ def snapposition (fn): snap = Gadget2Snapshot.open(fn) return snap.getPositions() def reorganize_snaps (IDsnap_id): ID, snap_id = IDsnap_id cosmo_id,geometry_id, ic_id = ID.split("|") new_snap_fn = os.path.join(storage, cosmo_id, geometry_id, ic_id, 'snapshots_amiga/snapshot_%03d'%(snap_id)) os.system('mkdir -p %s'%(os.path.join(storage, cosmo_id, geometry_id, ic_id, 'snapshots_amiga'))) amiga8_arr = glob.glob(os.path.join(storage, cosmo_id, geometry_id, ic_id, 'snapshots_amiga8/snapshot_%03d.*'%(snap_id))) halo_position = concatenate(array(map(snapposition, amiga8_arr)),axis=0)
cosmo_jia = cosmo_jia_arr[i] print cosmo_jia foldername = '/global/cscratch1/sd/jialiu/convergence_6redshifts/convergence_6redshifts_'+cosmo_jia tarname = '/global/cscratch1/sd/jialiu/convergence_6redshifts/convergence_CMB_'+cosmo_jia+'.tar' bash_tar_maps = '''mkdir -pv {0} tar -xvf {1} -C {0} echo done untar {0}'''.format(foldername,tarname) bash_chmod = ''' chmod 755 {0}/Maps11000 chmod 644 {0}/Maps11000/* done echo done-done-done '''.format(foldername) os.system(bash_tar_maps) os.system(bash_chmod) pool=MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) pool.map(create_targz, range(101)) #pool.map(create_targz, [0,]) #pool.map(unzip, range(101)) #pool.map(unzip_maps_edison, range(101)) #pool.map(unzip_CMBmaps_edison, range(101)) pool.close() print 'done-done-done'
# This is a short script to test the MPI implementation with the pattern used # by Prospector. However, try/except blocks are minimized to enable more # useful error messages, and the code is simple enough to *only* test the MPI # implementation. # Invoke with: # mpirun -np 4 python mpi_hello_world.py import numpy as np import sys from mpi4py import MPI def speak(i): print("I am core {} with task {}".format(pool.rank, i)) return i, pool.rank from emcee.utils import MPIPool pool = MPIPool(debug=False, loadbalance=True) if not pool.is_master(): # Wait for instructions from the master process. pool.wait() sys.exit(0) if __name__ == "__main__": result = pool.map(speak, np.arange(10)) print(result) pool.close()
from scipy import * import numpy as np from emcee.utils import MPIPool ############# organize files from text to npy with only cols needed col_names = ['ID', 'DescID', 'Mvir', 'Vmax', 'Vrms', 'Rvir', 'Rs', 'Np', 'X', 'Y', 'Z', 'VX', 'VY', 'VZ', 'JX', 'JY', 'JZ', 'Spin', 'rs_klypin', 'Mvir_all', 'M200b', 'M200c', 'M500c', 'M2500c', 'Xoff', 'Voff', 'spin_bullock', 'b_to_a', 'c_to_a', 'A[x]', 'A[y]', 'A[z]', 'b_to_a(500c)', 'c_to_a(500c)', 'A[x](500c)', 'A[y](500c)', 'A[z](500c)', 'T/|U|', 'M_pe_Behroozi', 'M_pe_Diemer', 'Halfmass_Radius', 'PID'] #usecols = ID, Mvir, Vmax, Rvir, Rs, x, y, z, Spin, PID usecols=[0, 2, 3, 5, 6, 8, 9, 10, 17, 41] dir_ayanna='/tigress/ayannam/' dir_cat='/tigress/jialiu/NUassemblybias/cat/' def compressfile (mnuN): mnu, N = mnuN print mnu, N out=loadtxt(dir_ayanna+'mnv_%.1f/out_%02d_parents.list'%(mnu, N), usecols=usecols) save(dir_cat+'mnv_%.1f_%02d_parents.npy'%(mnu, N), out) mnuN_arr = [[mnu, N] for mnu in (0, 0.1, 0.6) for N in 37,42,47,56,66] # operation pool=MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) pool.map(compressfile, mnuN_arr) print "DONE COMPRESSING FILES" ##########
zarr = arange(0.1, zhi, 0.1) params_arr = array([[iM, iz] for iM in Marr for iz in zarr]) def sampling_MPI(params): print params log10M, zlens = params fn = 'sampling/%s/M%.1f_z%.1f.npy' % (folder_name, log10M, zlens) if not os.path.isfile(fn): out = [sampling(log10M, zlens, iseed=i) for i in xrange(100)] save(fn, out) #else: #out=load(fn) #return out ############ MPI ################## MPIsampling = 1 if MPIsampling: os.system('mkdir -p sampling/' + folder_name) from emcee.utils import MPIPool pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) pool.map(sampling_MPI, params_arr) pool.close()
############ end: functions ############## ############ begein: calculate ############ #1. process maps if processmaps: ## Make sure the thread we're running on is the master #if not pool.is_master(): #pool.wait() #sys.exit(0) ## logger.debug("Running with MPI...") iRcosmo_pk = [[i, sigmaG, zg, bins, cosmo] for i in i_arr for sigmaG in sigmaG_arr for zg in zg_arr for bins in bins_arr for cosmo in cosmo_arr] iRcosmo_ps = [[i, sigmaG, zg, 0, cosmo] for i in i_arr for sigmaG in sigmaG_arr for zg in zg_arr for cosmo in cosmo_arr] pool = MPIPool() pool.map(Pmat, iRcosmo_ps+iRcosmo_pk) pool.close() #2. cosmology model #2.1 covariance matrix # (1.8 arcmin, 25bins)[3:17] # 14 bins # (3.5 arcmin, 25bins)[5:12] # 7 bins ## build array of sigmaG, bins, start, end, to prepare for cov, fisher mat # config_2_21 = array([[1.8, 25, 3, 17], #[3.5, 25, 5, 12]]) if fitcosmo: ## no need to do fit with ASTRO, can do it on my laptop bintol = int(sum(config[:,-1]-config[:,-2])) # total bins used in cosmo model cosmo_mat = zeros((4, Rtol, bintol))
###### generate GRF else: ps=0 random.seed(r) GRF = (WLanalysis.GRF_Gen(kmap)).newGRF() #save(CMBlensing_dir+'GRF_fidu/'+'GRF_fidu_%04dr.npy'%(r), GRF) #GRF = load(CMBlensing_dir+'GRF_fidu/'+'GRF_fidu_%04dr.npy'%(r)) GRF_smoothed = [WLanalysis.smooth(GRF, sigmaP) for sigmaP in sigmaP_arr] PDF = [PDFGen(GRF_smoothed[i], PDFbin_arr[i]) for i in i_arr] peaks = [peaksGen(GRF_smoothed[i], peak_bins_arr[i]) for i in i_arr] ############# return [ps,], PDF, peaks#, PDF_GRF, peaks_GRF pool=MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) a=pool.map(compute_GRF_PDF_ps_pk, range(start*1024+1, (start+1)*1024+1)) ##save(CMBlensing_dir+'%s_PDF_pk_600b_GRF'%(cosmo), a) if doGRF: save(CMBlensing_dir+'GRF_%s_ps_PDF_pk_z1100_%i.npy'%(cosmo,start), a) else: save(CMBlensing_dir+'kappa_%s_ps_PDF_pk_z1100_%i.npy'%(cosmo,start), a) pool.close() print '---DONE---DONE---'
sizedeg=12.25, size=2048, bins=50) return ells, powspec_1, powspec_1gauss, powspec_2, powspec_5, powspec_10 image_range = np.arange(1, 1001) from emcee.utils import MPIPool pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) results = np.array(pool.map(toPowspec, image_range)) pool.close() ells = results[0, 0] powspecs_1 = np.array([r[1] for r in results]) powermean_1 = np.mean(powspecs_1, axis=0) powspecs_1gauss = np.array([r[2] for r in results]) powermean_1gauss = np.mean(powspecs_1gauss, axis=0) powspecs_2 = np.array([r[3] for r in results]) powermean_2 = np.mean(powspecs_2, axis=0) powspecs_5 = np.array([r[4] for r in results]) powermean_5 = np.mean(powspecs_5, axis=0) powspecs_10 = np.array([r[5] for r in results]) powermean_10 = np.mean(powspecs_10, axis=0)
#!~/anaconda/bin/python # Jia Liu 2014/5/21 # What the code does: create mass maps for 100 cosmologies, for the CFHT emulator project # Cluster: XSEDE Stampede from emcee.utils import MPIPool import numpy as np from scipy import * from multiprocessing import Pool #from scoop import futures print 'start' i_arr=arange(1,14) def testMPIPool(i): savetxt('/home1/02977/jialiu/test%i'%(i),zeros(5)) pool = MPIPool() #pool = Pool(len(i_arr)) pool.map(testMPIPool, i_arr) #pool.close() #futures.map(testMPIPool, i_arr) print 'DONE-DONE-DONE'
mu_max=mu_max, om_max=om_max, beta_max=beta_max) return (mu_max, om_max, beta_max, dens, rat) pool = MPIPool(loadbalance=True) if not pool.is_master(): pool.wait() sys.exit(0) tasks = list() for i in range(len(filelist)): tasks.append((direct, float(filelist[i][2:-4]), 'Under')) vals = pool.map(dspec_find, tasks) pool.close() mu_max = np.zeros((len(vals), widths.shape[0])) om_max = np.zeros((len(vals), widths.shape[0])) beta_max = np.zeros((len(vals), widths.shape[0])) rats = np.zeros(len(vals)) dens = np.empty(len(vals), dtype='<U10') for i in range(len(vals)): mu_max[i, :] = vals[i][0] om_max[i, :] = vals[i][1] beta_max[i, :] = vals[i][2] rats[i] = vals[i][4] dens[i] = vals[i][3]
KS_fn = KS_dir+'CFHT_KS_sigma%02d_subfield%02d.fits'%(sigmaG*10,i) mask_fn = '/scratch/02977/jialiu/KSsim/mask/CFHT_mask_ngal%i_sigma%02d_subfield%02d.fits'%(ngal_arcmin,sigmaG*10,i) if WLanalysis.TestComplete((KS_fn,mask_fn),rm=True): kmap = WLanalysis.readFits(KS_fn) Mmask = WLanalysis.readFits(mask_fn) else: Me1_smooth = WLanalysis.weighted_smooth(Me1, Mw, PPA=PPA512, sigmaG=sigmaG) Me2_smooth = WLanalysis.weighted_smooth(Me2, Mw, PPA=PPA512, sigmaG=sigmaG) galn_smooth = snd.filters.gaussian_filter(galn.astype(float),sigmaG*PPA512, mode='constant') ## KS kmap = WLanalysis.KSvw(Me1_smooth, Me2_smooth) ## mask maskidx = where(galn_smooth < ngal_cut) #cut at ngal=5 Mmask = ones(shape=galn.shape) Mmask[maskidx]=0 WLanalysis.writeFits(kmap, KS_fn) WLanalysis.writeFits(Mmask, mask_fn) # Initialize the MPI pool pool = MPIPool() # Make sure the thread we're running on is the master if not pool.is_master(): pool.wait() sys.exit(0) # logger.debug("Running with MPI...") pool.map(KSmap, arange(1,14)) savetxt(KS_dir+'done.ls',zeros(5))
xiplus_arr [ibin] += sum(w[i]*w[j] * real(ei * conj(ej))) ximinus_arr[ibin] += sum(w[i]*w[j] * real(ei * ej)) norm_arr [ibin] += sum(w[i]*w[j]*(1+m[i])*(1+m[j])) ipair += step K = array([xiplus_arr, ximinus_arr, norm_arr]) savetxt(fn,K) return K #xiplus_arr[1:] -= xiplus_arr[:-1] #ximinus_arr[1:] -= ximinus_arr[:-1] #norm_arr[1:] -= norm_arr[:-1] pool = MPIPool() pool.map(twoPCF, range(1,14)) print 'done-done-done: calculating 2pcf, now sum up all subfields' # -------------------- grab everything --------------------- bigmatrix = zeros(shape=(13, 3, len(bins))) for isf in range(1,14): #fn = twoPCF_dir+'twoPCF_subfield%i'%(isf) #bigmatrix [isf-1] = genfromtxt(fn) bigmatrix [isf-1] = twoPCF(isf) # cumulative sum to sum in each bin - no need anymore, since got rid of this step in query_pairs #bigmatrix[:, :, 1:] -= bigmatrix[:, :, :-1] sum13fields = sum(bigmatrix, axis=0) xip = sum13fields[0]/sum13fields[-1] xim = sum13fields[1]/sum13fields[-1]
save(fn_cl, [ell, clkk]) return iseed, om, si8, S8, A_se9 ####### compute ###seed(94720) #S8_arr = (rand(20000)-0.5)*0.4+0.77 #om_arr = (rand(20000) - 0.5)*0.4 + 0.27 ##A_se9_find = lambda om, S8: -2.1+7.915*S8/sqrt(om/0.3) #A_se9_arr = A_se9_find(om_arr, S8_arr) #si8B_arr = S8_arr/sqrt(om_arr/0.3) ### savetxt('GRF_params.txt',array([om_arr, S8_arr, A_se9_arr]).T, header='om\tS8\tAs*1e9') om_arr, S8_arr, A_se9_arr = loadtxt('GRF_params.txt').T inputs = array([om_arr, S8_arr, arange(len(om_arr))]).T ##omS8seed #### test on laptop - pass # out=map(GRF_from_PS, inputs[:3]) ##### MPIPool pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) out = pool.map(GRF_from_PS, inputs) pool.close() savetxt('GRF_params_output.txt', out, header='seed\tom\tsi8\tS8\tAs*1e9') print 'done-done-done'
import os from emcee.utils import MPIPool inDIR = '/scratch/02977/jialiu/CMB_hopper/CMB_batch_storage/' outDIR = '/scratch/02977/jialiu/ranch_archive/CMB_batch_storage/' os.system("lfs setstripe -c 4 %s"%(outDIR)) def create_tarball (FNs): inFN, outFN = FNs os.system("tar cfv %s.tar %s"%(outFN, inFN)) pool = MPIPool() cosmo_arr = os.listdir(inDIR) FNs_arr = [['%s%s'%(inDIR, cosmo_arr[j]),'%s%s'%(outDIR, cosmo_arr[j])] for j in range(len(cosmo_arr))] pool.map(create_tarball, FNs_arr)
import os, sys import time # Third-party import numpy as np from emcee.utils import MPIPool def f(x): time.sleep(0.1) return np.sqrt(x) # Initialize the MPI pool pool = MPIPool() # Make sure the thread we're running on is the master if not pool.is_master(): pool.wait() sys.exit(0) v = np.random.random(size=1000) a = time.time() results = pool.map(f, v) pool.close() print(time.time() - a, "MPI map") # now try in serial a = time.time() map(f, v) print(time.time() - a, "map")
create 500 noise maps by randomly rotate galaxies, also include weights and (1+m) correction. all maps are smoothed over 1 arcmin. ''' #Wx=int(sys.argv[1]) p = MPIPool() bmap_fn = lambda Wx, iseed: kSZ_dir+'CFHT/Noise/W%i_Noise_sigmaG10_%04d.npy'%(Wx, iseed) Mexw = lambda Wx, txt: WLanalysis.readFits(kSZ_dir+'CFHT/Me_Mw_galn/W%i_M%s_1.3_lo.fit'%(Wx,txt)) Me1, Me2, Mwm = Mexw(Wx, 'e1w'), Mexw(Wx, 'e2w'), Mexw(Wx, 'wm') def randmap (iseed, Wx=Wx): Me1rnd, Me2rnd = WLanalysis.rndrot(Me1, Me2, iseed=iseed) Me1smooth = WLanalysis.weighted_smooth(Me1rnd, Mwm) Me2smooth = WLanalysis.weighted_smooth(Me2rnd, Mwm) kmap_rand = WLanalysis.KSvw(Me1smooth, Me2smooth) print Wx, iseed, kmap_rand.shape np.save(bmap_fn(Wx, iseed), kmap_rand) p.map(randmap, arange(500)) print 'done creating 500 noise KS maps' if cross_correlate_kSZ_noise: kSZmap_arr = map(kSZmapGen, range(1,5)) mask_arr = map(maskGen, range(1,5)) masked_kSZ_arr = [kSZmap_arr[i]*mask_arr[i] for i in range(4)] def kSZxNoise(iinput): '''iinput = (Wx, iseed) return the cross power between kSZ and convergence maps, both with smoothed mask. ''' Wx, iseed = iinput print 'kSZxNoise', Wx, iseed bmap = bmapGen(Wx, iseed)*mask_arr[Wx-1] kSZmap = masked_kSZ_arr[Wx-1] edges = edgesGen(Wx)
cosmo_dir = '/scratch/02977/jialiu/temp/' + cosmo nsnaps = nsnaps_arr[jjj] jjj += 1 for isnap in arange(nsnaps): INPUT_FILENAME = cosmo_dir + '/snapshots/snapshot_%03d' % (isnap) OUTPUT_DIR = cosmo_dir + '/snapshots_subsample/' all_snaps.append([INPUT_FILENAME, OUTPUT_DIR, int(isnap)]) def subsample_bysnap(input3): INPUT_FILENAME, OUTPUT_DIR, isnap = input3 out_fn = OUTPUT_DIR + 'snapshot_%03d_idmod_101_0.hdf5' % (isnap) if not os.path.isfile(out_fn): print 'subsampling:::', out_fn os.system( 'python /work/02977/jialiu/neutrino-batch/subsample_gadget_snapshot.py %s %s' % (INPUT_FILENAME, OUTPUT_DIR)) else: print 'skip:::', out_fn pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) pool.map(subsample_bysnap, all_snaps[::-1]) pool.close() print 'done-done-done'
] for idx in idx_arr]) return SFR_arr, Mhalo_arr, halo_means, SFR_Mstarbin, halo_means_massweight, halo_means_SFRweight if operation: NN = len(files_arr) if machine == 'local': map(createfile, range(NN)) else: pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) statsfolder = 'stats_Suto_Weinberg' #_smoothedge all_points = pool.map(createfile, range(NN)) save(statsfolder + '/SFhalos_SFR_lambda%s.npy' % (prefix), [all_points[i][0] for i in range(NN)]) save(statsfolder + '/SFhalos_mvir_lambda%s.npy' % (prefix), [all_points[i][1] for i in range(NN)]) save(statsfolder + '/SFhalos_stats_lambda%s.npy' % (prefix), [all_points[i][2] for i in range(NN)]) save(statsfolder + '/SFhalos_SFRmstarbins_lambda%s.npy' % (prefix), [all_points[i][3] for i in range(NN)]) save(statsfolder + '/SFhalos_stats_Mweighted_lambda%s.npy' % (prefix), [all_points[i][4] for i in range(NN)]) save( statsfolder + '/SFhalos_stats_SFRweighted_lambda%s.npy' % (prefix), [all_points[i][5] for i in range(NN)]) pool.close()
hi_m='mQ3-512b240_Om0.290_Ol0.710_w-1.000_ns0.960_si0.800' cosmo_arr=(fidu,hi_m,hi_w,hi_s) def galn_gen(i): print i y, x, z = WLanalysis.readFits(emucat_dir+'emulator_subfield%i_zcut0213.fit'%(i)).T Mz, galn = WLanalysis.coords2grid(x, y, np.array([z,])) WLanalysis.writeFits(galn, galn_fn) # map(galn_gen,range(1,14)) # power spectrum for all of the rz1, 4 cosmo def ps (R): Mk = WLanalysis.readFits(Mk_fn(i, cosmo, R)) galn = WLanalysis.readFits(galn_fn(i)) Mk_smooth = WLanalysis.weighted_smooth(Mk, galn, sigmaG) ps = WLanalysis.PowerSpectrum(Mk_smooth) return ps[1] for i in range(1, 14): for cosmo in cosmo_arr: print i, cosmo p = MPIPool() pmat = np.array(p.map(ps, arange(1,1001))) pmat_fn = KS_dir + 'powspec_Mk/SIM_powspec_sigma05_subfield%i_rz1_%s_1000R.fit'%(i,cosmo) p.close() try: WLanalysis.writeFits(pmat,pmat_fn) except Exception: pass print 'done'
#def test_corrupte (iRcosmo): #cosmo, R = iRcosmo #if WLanalysis.TestFitsComplete(SIMfn(i, cosmo, R))==False: #print SIMfn(i, cosmo, R) #return 1 #else: #return 0 #Rcosmo = [[ cosmo, R] for R in R_arr for cosmo in cosmo_arr] #badfiles = array(pool.map(test_corrupte, Rcosmo)) #save(KS_dir+'badfiles.npy',badfiles) ###################################################### ### (1)create KS map, uncomment next 4 lines ######### ###################################################### iRcosmo = [[i, R, cosmo] for cosmo in cosmo_arr for R in R_arr] pool.map(KSmap_massproduce, iRcosmo) ## (cov 1) this block is for covariance cosmology #cosmo='WL-only_cfhtcov-512b240_Om0.260_Ol0.740_w-1.000_ns0.960_si0.800' #iRcosmo = [[i, R, cosmo] for i in range(1,14)[::-1] for R in R_arr]# for cosmo in cosmo_arr] #cosmo_arr = [cosmo,] ####################################################### #### (2) power spectrum for 0.5 smoothing scale only### ####################################################### #for i in i_arr: #mask_bad_arr = array([Mask_bad_fn(i, sigmaG) for sigmaG in sigmaG_arr]) #mask_all_arr = array([Mask_all_fn(i, sigmaG) for sigmaG in sigmaG_arr]) #for cosmo in cosmo_arr: #ps_all_fn = powspec_sum_sf_fn(cosmo, 0.5, i, 'ALL')
Exists = False for i in range(len(dirlist)): if np.all(params == np.load('./%s/Params.npy' % dirlist[i])): Exists = True par_number = dirlist[i][5:] if not Exists: par_number = len(dirlist) + 1 while not Exists: try: os.mkdir('./Sims-%s' % par_number) np.save('./Sims-%s/Params.npy' % par_number, params) Exists = True except: if np.all(params == np.load('./Sims-%s/Params.npy' % par_number)): Exists = True else: par_number += 1 filelistall = os.listdir('./Sims-%s/' % par_number) filelist = list(filter(lambda x: x.startswith('x-'), filelistall)) Amps = np.zeros(len(filelist)) for i in range(Amps.shape[0]): Amps[i] = float(filelist[i][2:-4]) rats = rats[np.isin(rats, Amps, invert=True)] tasks = list((rats[i], par_number) for i in range(rats.shape[0])) vals = pool.map(Lens_Calc, tasks) pool.close()
# Third-party import numpy as np from emcee.utils import MPIPool def f(x): time.sleep(0.1) return np.sqrt(x) # Initialize the MPI pool pool = MPIPool() # Make sure the thread we're running on is the master if not pool.is_master(): pool.wait() sys.exit(0) v = np.random.random(size=1000) a = time.time() results = pool.map(f, v) pool.close() print(time.time() - a, "MPI map") # now try in serial a = time.time() map(f, v) print(time.time() - a, "map")
k, s1, s2 = kappaGen(r)[:3] A, galn = WLanalysis.coords2grid(x, y, array([s1,s2 ])) Ms1,Ms2 = A s1_smooth = WLanalysis.weighted_smooth(Ms1, galn, PPA=PPA512, sigmaG=sigmaG) s2_smooth = WLanalysis.weighted_smooth(Ms1, galn, PPA=PPA512, sigmaG=sigmaG) kmap = WLanalysis.KSvw(s1_smooth, s2_smooth) pk = WLanalysis.peaks_mask_hist(kmap, mask, bins=25, kmin = -0.04, kmax = 0.12) return pk def kmapPk_1sim (r): print r,'1sim' k, s1, s2 = kappaGen_1sim(r)[:3] A, galn = WLanalysis.coords2grid(x, y, array([s1,s2])) Ms1,Ms2 = A s1_smooth = WLanalysis.weighted_smooth(Ms1, galn, PPA=PPA512, sigmaG=sigmaG) s2_smooth = WLanalysis.weighted_smooth(Ms1, galn, PPA=PPA512, sigmaG=sigmaG) kmap = WLanalysis.KSvw(s1_smooth, s2_smooth) pk = WLanalysis.peaks_mask_hist(kmap, mask, bins=25, kmin = -0.04, kmax = 0.12) return pk pool = MPIPool() #ps_mat = pool.map(kmapPs, range(1,1001)) #WLanalysis.writeFits(ps_mat,KS_dir+'ps_mat_sf1_fidu_shear_noiseless_mask.fit') #pk_mat = pool.map(kmapPk, range(1,1001)) #np.save(KS_dir+'cfhtcov-512b240_Om0.260_Ol0.740_w-1.000_ns0.960_si0.800/pk_sigmaG10_sf1_cov.npy',pk_mat) pk_mat1sim = pool.map(kmapPk_1sim, range(1,1001)) np.save(KS_dir+'cfhtcov-512b240_Om0.260_Ol0.740_w-1.000_ns0.960_si0.800/pk_sigmaG10_sf1_1sim.npy',pk_mat1sim) print 'Done'
def gen_comp(zs): f = bigfile.File(folder + 'wlen_jliu/WL-%.2f-N4096' % (zs)) nside = f['kappa'].attrs['nside'][0] zmin = f['kappa'].attrs['zlmin'][0] zmax = f['kappa'].attrs['zlmax'][0] #zstep = f['kappa'].attrs['zstep'][0] zs = f['kappa'].attrs['zs'][0] print('nside = ', nside) print('redshifts = ', zs) lmax = min([5000, nside]) #5000 ell_sim = np.arange(lmax + 1) print(f['kappa'][:].shape) fn_cl = folder + '/clkk/kappa_cl_z%.2f.npz' % (zs) if not os.path.isfile(fn_cl): cl = hp.anafast(f['kappa'][:], lmax=lmax) np.savez(fn_cl, ell=ell_sim, cl=cl) zs = np.arange(0.1, 2.21, 0.1) ##### MPIPool pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) out = pool.map(gen_comp, zs) pool.close()
rndz=histogram(rndz,bins=edges)[0]/float(len(rndz)) all_PDF=m[:,range(89-70,89)] all_PDF/=sum(all_PDF,axis=1)[:,newaxis] average_PDF=average(all_PDF,axis=0) savetxt(full_dir+'avgPDF_arr%s'%(i),average_PDF) savetxt(full_dir+'pks_arr%s'%(i),pk) savetxt(full_dir+'rnds_arr%s'%(i),rndz) # for CFHT each bin has 3.5/70 probability, for SIM, each bin has 3.5/67 height return all_PDF, pk, rndz, len(all_PDF) avgPDF_arr = zeros(shape=(13,70)) pks_arr = zeros(shape=(13,67)) rnds_arr = zeros(shape=(13,67)) lena_arr = zeros(13) p = MPIPool() x = p.map(pdfs,range(1,14)) for i in range(1,14): aa, peaks, rnds, lena=x[i-1] avgPDF_arr_arr[i-1]=aa pks_arr[i-1]=peaks rnds_arr[i-1]=rnds lena_arr[i-1]=lena print i, lena savetxt('/direct/astro+astronfs01/workarea/jia/CFHT/full_subfields/avgPDF_arr.ls',avgPDF_arr) savetxt('/direct/astro+astronfs01/workarea/jia/CFHT/full_subfields/pks_arr.ls',pks_arr) savetxt('/direct/astro+astronfs01/workarea/jia/CFHT/full_subfields/rnds_arr.ls',rnds_arr) savetxt('/direct/astro+astronfs01/workarea/jia/CFHT/full_subfields/lena_arr.ls',lena_arr) print 'done-done-done'
#igaln = x.newGRF()*mask_arr[Wx-1] ######## # use Planck sim map, and CFHT GRF map Pkmap = PlanckSim15Gen(Wx, i)*mask_arr[Wx-1] #Ckmap = CFHTx.newGRF() Ckmap = load('/work/02977/jialiu/kSZ/CFHT/Noise/W%i_Noise_sigmaG10_%04d.npy'%(Wx, i)) ############# CCP = WLanalysis.CrossCorrelate(Pkmap, igaln, edges = edges_arr[Wx-1], sigmaG1=1.0, sigmaG2=1.0)[1]/fmask2_arr[Wx-1]/factor CCC = WLanalysis.CrossCorrelate(Ckmap, igaln, edges = edges_arr[Wx-1], sigmaG1=1.0, sigmaG2=1.0)[1]/fmask2_arr[Wx-1]/factor return CCP, CCC if not p.is_master(): p.wait() sys.exit(0) CCsim_err_arr = array(p.map(iCC, range(100))) ####### referee #save(main_dir+'powspec/CC_Plancksim_CFHTrot_ludomask_cut%i_W%i.npy'%(cut, Wx), CCsim_err_arr) save(main_dir+'referee/CC_Plancksim_CFHTrot_ludomask_cut%i_W%i.npy'%(cut, Wx), CCsim_err_arr) p.close() ############# finish compute sim error ##################### ############ calculate theory ################# if compute_model: for cut in (22,23,24):#cut=22 print cut from scipy.integrate import quad z_center= arange(0.025, 3.5, 0.05)
### skips if files do not exist for some reason print 'Warning: file does not exist, cosmo, snap' return if os.path.isfile(out_fn): ###### in case the code breaks return ######### read rockstar files print 'Opening rockstar files:', rockstar_fn reader = sim_manager.TabularAsciiReader(rockstar_fn, columns_to_keep_dict) rock_arr = reader.read_ascii() logM = log10(rock_arr['halo_mvir']) rock_arr = 0 ## release memory hmf = histogram(logM, bins=hist_bins)[0] save(out_fn, hmf) all_snaps = [] for i in range(len(cosmo_arr)): for isnap in arange(30, nsnaps_arr[i]): all_snaps.append([cosmo_arr[i], int(isnap)]) pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) pool.map(Phm_gen, all_snaps) #pool.map(Phh_gen, all_snaps) #pool.map(hmf_gen, all_snaps) pool.close()
#### smooth function, for all smoothing bins def smooth_map (r, mnu=imnu, zidx=izidx): imap = mapgen(z_arr[zidx], r, mnu) imap_smooth = array([smooth(imap, thetaG) for thetaG in thetaG_arr]) hist_arr = array([histogram(imap_smooth[i], bins=binedges[zidx, i])[0] for i in range(len(thetaG_arr))]) return hist_arr def std_map (r, mnu=imnu, zidx=izidx): imap = mapgen(z_arr[zidx], r, mnu) imap_smooth = array([smooth(imap, thetaG) for thetaG in thetaG_arr]) std_arr = [std(imap) for imap in imap_smooth] return std_arr pool=MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) print 'Mnu, z:', imnu, z_arr[izidx] out = array(pool.map(smooth_map, range(1,10001))) for j in range(len(thetaG_arr)): print out.shape save(out_dir+'PDFs_Mnu0.%i_z%.1f_smooth%02d.npy'%(imnu, z_arr[izidx], thetaG_arcmin[j]), out[:,j,:]) #out_std = array(pool.map(std_map, range(1,10001))) #print out_std.shape #save(out_dir+'PDFs_Mnu0.%i_z%.1f.npy'%(imnu, z_arr[izidx]), out_std) pool.close() print 'DONE DONE DONE'