def initial_pool(prior_obj, eps0, N_particles, N_threads=1): """ Initial Pool """ args_list = [[i, prior_obj, eps0, N_particles] for i in xrange(N_particles)] if N_threads > 1: pool = InterruptiblePool(processes = N_threads) mapfn = pool.map results = mapfn(initial_pool_sampling, args_list) pool.close() pool.terminate() pool.join() else: results = [] for arg in args_list: results.append(initial_pool_sampling(arg)) results = np.array(results).T theta_t = results[1:prior_obj.n_params+1,:] w_t = results[prior_obj.n_params+1,:] rhos = results[prior_obj.n_params+2,:] sig_t = covariance(theta_t , w_t) return theta_t, w_t, rhos, sig_t
def func_grad_lnX_Nthreads(self, params): """ Use multiprocessing to calculate negative-log-likelihood and gradinets w.r.t lnX, plus the terms coming from th regularization terms. """ n_samples = self.N self.lnX = params #self.fl, self.f, self.g, self.H, Nthreads = args Pool = InterruptiblePool(Nthreads) mapfn = Pool.map Nchunk = np.ceil(1. / Nthreads * n_samples).astype(np.int) arglist = [None] * Nthreads for i in range(Nthreads): s = int(i * Nchunk) e = int(s + Nchunk) arglist[i] = (self.lnX, self.F, self.B, self.fl, self.f, self.g, self.H, s, e) result = list(mapfn(fg, [ars for ars in arglist])) nll, grad = result[0] a = time.time() for i in range(1, Nthreads): nll += result[i][0] grad += result[i][1] #print "adding up nll's from individual threads", time.time() - a Pool.close() Pool.terminate() Pool.join() #computing the regularization term and its derivative w.r.t lnX reg_func, reg_grad = self.reg_func_grad_lnX() return nll + reg_func, grad + reg_grad
def log_multivariate_gaussian_Nthreads(x, mu, V, xcov, Nthreads=1): """ Use multiprocessing to calculate log likelihoods. """ n_samples = x.shape[0] pool = InterruptiblePool(Nthreads) mapfn = pool.map Nchunk = np.ceil(1. / Nthreads * n_samples).astype(np.int) arglist = [None] * Nthreads for i in range(Nthreads): s = i * Nchunk e = s + Nchunk arglist[i] = (x[s:e], mu, V, xcov[s:e]) result = list(mapfn(lmg, [args for args in arglist])) logls = result[0] for i in range(1, Nthreads): logls = np.vstack((logls, result[i])) pool.close() pool.terminate() pool.join() return logls
def initial_pool(prior_obj, eps0, N_particles, N_threads=1): """ Initial Pool """ args_list = [[i, prior_obj, eps0, N_particles] for i in xrange(N_particles)] if N_threads > 1: pool = InterruptiblePool(processes=N_threads) mapfn = pool.map results = mapfn(initial_pool_sampling, args_list) pool.close() pool.terminate() pool.join() else: results = [] for arg in args_list: results.append(initial_pool_sampling(arg)) results = np.array(results).T theta_t = results[1:prior_obj.n_params + 1, :] w_t = results[prior_obj.n_params + 1, :] rhos = results[prior_obj.n_params + 2, :] sig_t = covariance(theta_t, w_t) return theta_t, w_t, rhos, sig_t
def pmc_abc(N_threads=N_threads): # initial pool theta_t, w_t, rhos, sig_t = initial_pool() t = 0 # iternation number plot_thetas(theta_t, w_t, t) plt.savefig("/home/mj/public_html/scatter_hod_gaussian_t" + str(t) + ".png") plt.close() while t < N_iter: eps_t = np.percentile(rhos, 75) print "New Distance Threshold Eps_t = ", eps_t theta_t_1 = theta_t.copy() w_t_1 = w_t.copy() sig_t_1 = sig_t.copy() """these lines are borrowed from initial sampling to double-check multiprocessing""" # pool = InterruptiblePool(processes = N_threads) # mapfn = pool.map # args_list = [i for i in xrange(N_particles)] # results = mapfn(initial_pool_sampling, args_list) # pool.close() # pool.terminate() # pool.join() pool = InterruptiblePool(processes=N_threads) mapfn = pool.map args_list = [[i, theta_t_1, w_t_1, sig_t_1, eps_t] for i in xrange(N_particles)] # results = [] # for args in args_list: # pool_sample = importance_pool_sampling(args) # results.append( pool_sample ) results = mapfn(importance_pool_sampling, args_list) pool.close() pool.terminate() pool.join() sig_t = np.cov(theta_t) results = np.array(results).T theta_t = results[1 : n_params + 1, :] w_t = results[n_params + 1, :] rhos = results[n_params + 2, :] sig_t = np.cov(theta_t) t += 1 plot_thetas(theta_t, w_t, t) plt.savefig("/home/mj/public_html/scatter_hod_gaussian_t" + str(t) + ".png") plt.close()
def pmc_abc(N_threads = N_threads): # initial pool theta_t, w_t, rhos, sig_t = initial_pool() t = 0 # iternation number plot_thetas(theta_t , w_t, t) plt.savefig("/home/mj/public_html/scatter_hod_gaussian_t"+str(t)+".png") plt.close() while t < N_iter: eps_t = np.percentile(rhos, 75) print 'New Distance Threshold Eps_t = ', eps_t theta_t_1 = theta_t.copy() w_t_1 = w_t.copy() sig_t_1 = sig_t.copy() """these lines are borrowed from initial sampling to double-check multiprocessing""" #pool = InterruptiblePool(processes = N_threads) #mapfn = pool.map #args_list = [i for i in xrange(N_particles)] #results = mapfn(initial_pool_sampling, args_list) #pool.close() #pool.terminate() #pool.join() pool = InterruptiblePool(processes = N_threads) mapfn = pool.map args_list = [[i, theta_t_1, w_t_1, sig_t_1, eps_t] for i in xrange(N_particles)] #results = [] #for args in args_list: # pool_sample = importance_pool_sampling(args) # results.append( pool_sample ) results = mapfn(importance_pool_sampling, args_list) pool.close() pool.terminate() pool.join() sig_t = np.cov(theta_t) results = np.array(results).T theta_t = results[1:n_params+1,:] w_t = results[n_params+1,:] rhos = results[n_params+2,:] sig_t = np.cov(theta_t) t += 1 plot_thetas(theta_t, w_t , t) plt.savefig("/home/mj/public_html/scatter_hod_gaussian_t"+str(t)+".png") plt.close()
def pmc_abc(prior_dict, N_particles=100, N_iter=30, eps0=20.0, N_threads=1): """ """ prior_obj = Prior(prior_dict) # initial pool theta_t, w_t, rhos, sig_t = initial_pool(prior_obj, eps0, N_particles, N_threads=N_threads) t = 0 # iternation number #plot_thetas(theta_t , w_t, prior_dict, t) while t < N_iter: eps_t = np.percentile(rhos, 75) print 'New Distance Threshold Eps_t = ', eps_t theta_t_1 = theta_t.copy() w_t_1 = w_t.copy() sig_t_1 = sig_t.copy() args_list = [[i, prior_obj, theta_t_1, w_t_1, sig_t_1, eps_t] for i in xrange(N_particles)] if N_threads > 1: pool = InterruptiblePool(processes=N_threads) mapfn = pool.map results = mapfn(importance_pool_sampling, args_list) pool.close() pool.terminate() pool.join() else: results = [] for args in args_list: pool_sample = importance_pool_sampling(args) results.append(pool_sample) results = np.array(results).T theta_t = results[1:prior_obj.n_params + 1, :] w_t = results[prior_obj.n_params + 1, :] rhos = results[prior_obj.n_params + 2, :] sig_t = covariance(theta_t, w_t) t += 1 plot_thetas(theta_t, w_t, prior_dict, t)
def pmc_abc(N_threads = N_threads): # initial pool theta_t, w_t, rhos, sig_t = initial_pool() w_t = w_t/np.sum(w_t) t = 0 # iternation number plot_thetas(theta_t , w_t, t) while t < N_iter: if t < 4 : eps_t = np.percentile(np.atleast_2d(rhos), 20, axis=1) else: eps_t = np.percentile(np.atleast_2d(rhos), 50, axis=1) print 'New Distance Threshold Eps_t = ', eps_t , "t=" , t theta_t_1 = theta_t.copy() w_t_1 = w_t.copy() sig_t_1 = sig_t.copy() args_list = [[i, theta_t_1, w_t_1, sig_t_1, eps_t] for i in xrange(N_particles)] """serial""" results = [] #for args in args_list: # pool_sample = importance_pool_sampling(args) # results.append( pool_sample ) """parallel""" pool = InterruptiblePool(processes = N_threads) mapfn = pool.map results = mapfn(importance_pool_sampling, args_list) pool.close() pool.terminate() pool.join() results = np.array(results).T theta_t = results[1:n_params+1,:] w_t = results[n_params+1,:] w_t = w_t/np.sum(w_t) rhos = results[n_params+2:,:] #sig_t = knn_sigma(theta_t , k = 10) sig_t = 2. * covariance(theta_t , w_t) t += 1 plot_thetas(theta_t, w_t , t)
def pmc_abc(prior_dict, N_particles=100, N_iter=30, eps0=20.0, N_threads = 1): """ """ prior_obj = Prior(prior_dict) # initial pool theta_t, w_t, rhos, sig_t = initial_pool(prior_obj, eps0, N_particles, N_threads=N_threads) t = 0 # iternation number #plot_thetas(theta_t , w_t, prior_dict, t) while t < N_iter: eps_t = np.percentile(rhos, 75) print 'New Distance Threshold Eps_t = ', eps_t theta_t_1 = theta_t.copy() w_t_1 = w_t.copy() sig_t_1 = sig_t.copy() args_list = [[i, prior_obj, theta_t_1, w_t_1, sig_t_1, eps_t] for i in xrange(N_particles)] if N_threads > 1: pool = InterruptiblePool(processes = N_threads) mapfn = pool.map results = mapfn(importance_pool_sampling, args_list) pool.close() pool.terminate() pool.join() else: results = [] for args in args_list: pool_sample = importance_pool_sampling(args) results.append( pool_sample ) results = np.array(results).T theta_t = results[1:prior_obj.n_params+1,:] w_t = results[prior_obj.n_params+1,:] rhos = results[prior_obj.n_params+2,:] sig_t = covariance(theta_t , w_t) t += 1 plot_thetas(theta_t , w_t, prior_dict, t)
def pmc_abc(N_threads=N_threads): # initial pool theta_t, w_t, rhos, sig_t = initial_pool() w_t = w_t / np.sum(w_t) t = 0 # iternation number plot_thetas(theta_t, w_t, t) while t < N_iter: if t < 4: eps_t = np.percentile(np.atleast_2d(rhos), 20, axis=1) else: eps_t = np.percentile(np.atleast_2d(rhos), 50, axis=1) print 'New Distance Threshold Eps_t = ', eps_t, "t=", t theta_t_1 = theta_t.copy() w_t_1 = w_t.copy() sig_t_1 = sig_t.copy() args_list = [[i, theta_t_1, w_t_1, sig_t_1, eps_t] for i in xrange(N_particles)] """serial""" results = [] #for args in args_list: # pool_sample = importance_pool_sampling(args) # results.append( pool_sample ) """parallel""" pool = InterruptiblePool(processes=N_threads) mapfn = pool.map results = mapfn(importance_pool_sampling, args_list) pool.close() pool.terminate() pool.join() results = np.array(results).T theta_t = results[1:n_params + 1, :] w_t = results[n_params + 1, :] w_t = w_t / np.sum(w_t) rhos = results[n_params + 2:, :] #sig_t = knn_sigma(theta_t , k = 10) sig_t = 2. * covariance(theta_t, w_t) t += 1 plot_thetas(theta_t, w_t, t)
def initial_pool(): pool = InterruptiblePool(processes = N_threads) mapfn = pool.map args_list = [i for i in xrange(N_particles)] results = mapfn(initial_pool_sampling, args_list) pool.close() pool.terminate() pool.join() results = np.array(results).T theta_t = results[1:n_params+1,:] w_t = results[n_params+1,:] rhos = results[n_params+2,:] sig_t = np.cov(theta_t) return theta_t, w_t, rhos, sig_t
def initial_pool(): pool = InterruptiblePool(processes=N_threads) mapfn = pool.map args_list = [i for i in xrange(N_particles)] results = mapfn(initial_pool_sampling, args_list) pool.close() pool.terminate() pool.join() results = np.array(results).T theta_t = results[1 : n_params + 1, :] w_t = results[n_params + 1, :] rhos = results[n_params + 2, :] sig_t = np.cov(theta_t) return theta_t, w_t, rhos, sig_t
def initial_pool(): args_list = np.arange(N_particles) """serial""" #results = [] #for arg in args_list: # results.append(initial_pool_sampling(arg)) """parallel""" pool = InterruptiblePool(processes = N_threads) mapfn = pool.map results = mapfn(initial_pool_sampling, args_list) pool.close() pool.terminate() pool.join() results = np.array(results).T theta_t = results[1:n_params+1,:] w_t = results[n_params+1,:] w_t = w_t / np.sum(w_t) rhos = results[n_params+2:,:] sig_t = covariance(theta_t , w_t) return theta_t, w_t, rhos, sig_t
def initial_pool(): args_list = np.arange(N_particles) """serial""" #results = [] #for arg in args_list: # results.append(initial_pool_sampling(arg)) """parallel""" pool = InterruptiblePool(processes=N_threads) mapfn = pool.map results = mapfn(initial_pool_sampling, args_list) pool.close() pool.terminate() pool.join() results = np.array(results).T theta_t = results[1:n_params + 1, :] w_t = results[n_params + 1, :] w_t = w_t / np.sum(w_t) rhos = results[n_params + 2:, :] sig_t = covariance(theta_t, w_t) return theta_t, w_t, rhos, sig_t
def pmc_abc(self): """ """ self.rhos = self.initial_pool() while self.t < self.T: self.eps_t = np.percentile(self.rhos, 75) print 'Epsilon t', self.eps_t self.theta_t_1 = self.theta_t.copy() self.w_t_1 = self.w_t.copy() self.sig_t_1 = self.sig_t.copy() pool = InterruptiblePool(self.Nthreads) mapfn = pool.map args_list = [ i for i in xrange(self.N) ] results = mapfn(unwrap_self_importance_sampling, zip([self]*len(args_list), args_list)) pool.close() pool.terminate() pool.join() pars = np.array(results).T self.theta_t = pars[1:self.n_params+1,:].copy() self.w_t = pars[self.n_params+1,:].copy() self.rhos = pars[self.n_params+2,:].copy() self.sig_t = 2.0 * np.cov(self.theta_t) self.t += 1 self.writeout() self.plotout() return None
def build_multipro(type, catalog_name, corr_name, n_mocks, Nthreads=8, ell=2, Ngrid=360, **kwargs): ''' Calculate dLOS for catalogs in parallel using interruptible pool, which is multiprocessing pool that allows for interrputions Parameters ---------- catalog_name : Name of catalog corr_name : Name of correction n_mocks : Number of mock catalogs to calculate Nthreads : Number of CPUs to use ''' if isinstance(n_mocks, list): n_mock_list = n_mocks else: n_mock_list = range(1, n_mocks + 1) corrdict = {} if catalog_name == 'nseries': if isinstance(corr_name, dict): corrdict = corr_name else: corrdict['name'] = corr_name if 'dlospeak' in corr_name: # hardcoded values for bestfit dlos peak # parameters corrdict['fit'] = 'gauss' corrdict['sigma'] = 3.9 corrdict['fpeak'] = 0.68 if 'env' in corr_name: # hardcoded values for galaxy environment # parameters corrdict['n_NN'] = 5 if 'photoz' in corr_name: corrdict['d_photoz_tail_cut'] = 15 if corr_name == 'fourier_tophat': corrdict['fs'] = 1.0 corrdict['rc'] = 0.43 corrdict['k_fit'] = 0.7 corrdict['k_fixed'] = 0.84 if type == 'bk': arglist = [ [{ 'catalog': {'name': catalog_name, 'n_mock': i_mock}, 'spec': {'P0': 20000, 'Lbox': 3600, 'Ngrid': Ngrid} }, kwargs] for i_mock in n_mock_list] else: arglist = [ [{ 'catalog': {'name': catalog_name, 'n_mock': i_mock}, 'correction': corrdict, 'spec': { 'P0': 20000, #P0 'Lbox': 3600, 'Ngrid': Ngrid, 'ell': ell } }, ell, kwargs] for i_mock in n_mock_list ] if Nthreads > 1: pool = Pewl(processes=Nthreads) mapfn = pool.map if type == 'data': mapfn( build_corrdata_wrapper, [arg for arg in arglist]) elif type == 'pk': mapfn( build_pk_wrapper, [arg for arg in arglist]) elif type == 'bk': mapfn( build_bk_wrapper, [arg for arg in arglist]) pool.close() pool.terminate() pool.join() else: for arg in arglist: if type == 'data': build_corrdata_wrapper(arg) elif type == 'pk': build_pk_wrapper(arg) elif type == 'bk': build_bk_wrapper(arg) return None
zstr = Sys.argv[6] if zstr == 'z': zbool = True elif zstr == 'real': zbool = False else: raise ValueError nthreads = int(Sys.argv[7]) if nthreads > 1: args_list = [(mneut, ireal, nzbin, zbool) for ireal in np.arange(nreal_i, nreal_f+1)] pool = Pewl(processes=nthreads) mapfn = pool.map results = mapfn(_NeutHalo_pre3PCF, args_list) pool.close() pool.terminate() pool.join() else: for ireal in range(nreal_i, nreal_f+1): NeutHalo_pre3PCF(mneut, ireal, nzbin, zspace=zbool) elif arg1 == 'plk': nreal = int(Sys.argv[3]) nzbin = int(Sys.argv[4]) zstr = Sys.argv[5] if zstr == 'z': zbool = True elif zstr == 'real': zbool = False else: raise ValueError NeutHalo_Plk(mneut, nreal, nzbin, zspace=zbool)