def __init__(self, likelihood_module, prior_type='uniform', prior_means=None, prior_sigmas=None, width_scale=1, sigma_scale=1, bound='multi', sample='auto', use_mpi=False, use_pool=None): """ :param likelihood_module: likelihood_module like in likelihood.py (should be callable) :param prior_type: 'uniform' of 'gaussian', for converting the unit hypercube to param cube :param prior_means: if prior_type is 'gaussian', mean for each param :param prior_sigmas: if prior_type is 'gaussian', std dev for each param :param width_scale: scale the widths of the parameters space by this factor :param sigma_scale: if prior_type is 'gaussian', scale the gaussian sigma by this factor :param bound: specific to Dynesty, see https://dynesty.readthedocs.io :param sample: specific to Dynesty, see https://dynesty.readthedocs.io :param use_mpi: Use MPI computing if `True` :param use_pool: specific to Dynesty, see https://dynesty.readthedocs.io """ self._check_install() super(DynestySampler, self).__init__(likelihood_module, prior_type, prior_means, prior_sigmas, width_scale, sigma_scale) # create the Dynesty sampler if use_mpi: from schwimmbad import MPIPool import sys pool = MPIPool( use_dill=True ) # use_dill=True not supported for some versions of schwimmbad if not pool.is_master(): pool.wait() sys.exit(0) self._sampler = self._dynesty.DynamicNestedSampler( self.log_likelihood, self.prior, self.n_dims, bound=bound, sample=sample, pool=pool, use_pool=use_pool) else: self._sampler = self._dynesty.DynamicNestedSampler( self.log_likelihood, self.prior, self.n_dims, bound=bound, sample=sample) self._has_warned = False
def __init__(self, func, low, high, particleCount=25, threads=1): self.threads = threads pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) ParticleSwarmOptimizer.__init__(self, func, low, high, particleCount=particleCount, pool=pool)
def run_MCMC_mpi(self, Nwalker, Nsteps, outfile_MCMC=None, save_step_size=2): Ndim = len(self.active_par_key) starting_point = [self.par_fid[item] for item in self.active_par_key] std = [self.par_std[item] for item in self.active_par_key] p0_walkers = emcee.utils.sample_ball(starting_point, std, size=Nwalker) with MPIPool() as pool: if not pool.is_master(): pool.wait() sys.exit(0) sampler = emcee.EnsembleSampler(Nwalker, Ndim, self.cal_loglike, a=2.0, pool=pool) step_size = save_step_size steps_taken = 0 ##### start long sampling Tstart = time.time() while steps_taken < Nsteps: posInfo = sampler.run_mcmc(p0_walkers, step_size, progress=True) p0_walkers = posInfo.coords steps_taken+=step_size print("steps_taken", steps_taken) self.chainInfo = self.save_chain(sampler=sampler, outfile_MCMC=outfile_MCMC) Time_MCMC = (time.time()-Tstart)/60. print("Total MCMC time (mins):", Time_MCMC) return self.chainInfo
def pool(request): multimode = 'None' # multimode = 'Serial' # multimode = 'Multi' # multimode = 'MPI' # setup code pool = None if multimode == 'Serial': from schwimmbad import SerialPool pool = SerialPool() elif multimode == 'Multi': from schwimmbad import MultiPool pool = MultiPool() elif multimode == 'MPI': from schwimmbad import MPIPool pool = MPIPool() if not pool.is_master(): pool.wait() import sys sys.exit(0) # inject class variables request.cls.pool = pool yield # tear down if multimode == 'Multi' or multimode == 'MPI': pool.close()
def mcmc_std(mapa,mpi=False): def lnlike(theta, x0, y0, N): n1,n2,kc = theta x1,y1=sigma_resolution(Noise(N,n1,n2,kc)) y1=y1/x1**2 y1+=np.mean(y0)-np.mean(y1) error=np.sum((y1-y0)**2) plt.plot(x0,y0) plt.plot(x1,y1) plt.title(str(error)) plt.xscale('log') plt.savefig('Temp_Figures/n1_%04.2f' %n1+'_n2_%04.2f' %n2+'_kc_%04d' %int(kc)+'.png') plt.close() return -0.5*error def lnprior(theta, N): n1, n2, kc = theta if 0 < n1 < 5 and n1 < n2 < 9.0 and 1 < kc < N: return 0.0 return -1e32 def lnprob(theta, x0, y0, N): lp = lnprior(theta, N) if not np.isfinite(lp): return -1e32 return lp + lnlike(theta, x0,y0,N) new_map=mapa/np.std(mapa) x0,y0=sigma_resolution(new_map) y0=y0/x0**2 N=len(new_map) if mpi: with MPIPool() as pool: if not pool.is_master(): pool.wait() sys.exit(0) ndim, nwalkers = 3, 10 pos = [[1.33,4,N/4] + np.random.randn(ndim) for i in range(nwalkers)] nsteps = 10 sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, args=(x0,y0,N),pool=pool) start = time.time() sampler.run_mcmc(pos, nsteps) end = time.time() return sampler else: ndim, nwalkers = 3, 20 pos = [[1.33,4,N] + np.random.randn(ndim) for i in range(nwalkers)] for p in pos: p[2]*=np.random.uniform(0,1) print(pos) nsteps = 100 sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, args=(x0,y0,N)) start = time.time() sampler.run_mcmc(pos, nsteps) end = time.time() return sampler
def mcmc_emcee(self, n_walkers, n_run, n_burn, mean_start, sigma_start, mpi=False, progress=False, threadCount=1): numParam, _ = self.chain.param.num_param() p0 = sampling_util.sample_ball(mean_start, sigma_start, n_walkers) time_start = time.time() if mpi is True: pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) is_master_pool = pool.is_master() sampler = emcee.EnsembleSampler(n_walkers, numParam, self.chain.logL, pool=pool) else: is_master_pool = True if threadCount > 1 : pool = Pool(processes=threadCount) else : pool = None sampler = emcee.EnsembleSampler(n_walkers, numParam, self.chain.likelihood, pool=pool) sampler.run_mcmc(p0, n_burn + n_run, progress=progress) flat_samples = sampler.get_chain(discard=n_burn, thin=1, flat=True) dist = sampler.get_log_prob(flat=True, discard=n_burn, thin=1) if is_master_pool: print('Computing the MCMC...') print('Number of walkers = ', n_walkers) print('Burn-in iterations: ', n_burn) print('Sampling iterations:', n_run) time_end = time.time() print(time_end - time_start, 'time taken for MCMC sampling') return flat_samples, dist
def sample(self, verbosity=0, use_mpi=False): import emcee if use_mpi: from schwimmbad import MPIPool pool = MPIPool() print("Using MPI") pool_use = pool else: pool = DumPool() print("Not using MPI") pool_use = None if not pool.is_master(): pool.wait() sys.exit(0) fname_chain = self.prefix_out + "chain" found_file = os.path.isfile(fname_chain + '.txt') if (not found_file) or self.rerun: pos_ini = (np.array(self.p0)[None, :] + 0.001 * np.random.randn(self.nwalkers, self.ndim)) nsteps_use = self.nsteps else: print("Restarting from previous run") old_chain = np.loadtxt(fname_chain + '.txt') if np.ndim(old_chain) == 1: old_chain = np.atleast_2d(old_chain).T pos_ini = old_chain[-self.nwalkers:, :] nsteps_use = max(self.nsteps - len(old_chain) // self.nwalkers, 0) print(self.nsteps - len(old_chain) // self.nwalkers) chain_file = SampleFileUtil(self.prefix_out + "chain", rerun=self.rerun) sampler = emcee.EnsembleSampler(self.nwalkers, self.ndim, self.lnprob, pool=pool_use) counter = 1 for pos, prob, _ in sampler.sample(pos_ini, iterations=nsteps_use): if pool.is_master(): chain_file.persistSamplingValues(pos, prob) if counter % 10 == 0: print(f"Finished sample {counter}") counter += 1 pool.close() return sampler
def postprocess_run(jobdir, savename, exp_type, fields, save_beta=False, comm=None, return_dframe=True): # Distribute postprocessing across ranks, if desired if comm is not None: # Collect all .h5 files if comm.rank == 0: data_files = grab_files(jobdir, '*.dat', exp_type) data_files = [(d) for d in data_files] print(len(data_files)) else: data_files = None rank = comm.rank size = comm.size # print('Rank %d' % rank) master = StreamWorker(savename) worker = PostprocessWorker(jobdir, fields, rank, size) pool = MPIPool(comm) pool.map(worker, data_files, callback=master.stream, track_results=False) if not pool.is_master(): pool.wait() sys.exit(0) pool.close() if rank == 0: master.close() else: worker = PostprocessWorker(jobdir, fields, savename) for i, data_file in enumerate(data_files): t0 = time.time() result = worker(data_file) worker.extend(result) print(time.time() - t0) dframe = worker.save(save_beta) if return_dframe: return dframe
def test_MPI(self): if MPIPool.enabled(): # mpi should work mpi_pool = choose_pool('mpi') self.assertTrue(isinstance(mpi_pool, MPIPool)) elif find_spec('mpi4py') is None: # mpi4py not installed self.assertRaises(ImportError, choose_pool, 'mpi') else: # mpi4py installed but only one process available self.assertRaises(ValueError, choose_pool, 'mpi')
def MPIpool_evaluate(representations, ndim, fid, iids, num_reps, budget=None): from schwimmbad import MPIPool budget = Config.ES_budget_factor * ndim if budget is None else budget run_es = partial(runCustomizedES, ndim=ndim, fid=fid, budget=budget) func = partial(helper, func=run_es) tasks = product(representations, iids, range(num_reps)) with MPIPool() as pool: results = pool.map(func, tasks) return results
def main_fit(): pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) # Create an initial point psize = pos_lim_max - pos_lim_min p0 = [pos_lim_min + psize * np.random.rand(ndim) for i in range(nwalkers)] # Set a sample sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, pool=pool) # Run MCMC sampler.run_mcmc(p0, burnin) # Flat sample and discard 10% sample flat_samples = sampler.get_chain(discard=int(0.1 * burnin), flat=True) pool.close() return flat_samples
def megafit_emcee(self): nsamples = 300 atm = [] for j in range(len(self.names)): prod_name = os.path.join(parameters.PROD_DIRECTORY, parameters.PROD_NAME) atmgrid = AtmosphereGrid( filename=(prod_name + '/' + self.names[j].split('/')[-1]).replace('sim', 'reduc').replace( 'spectrum.txt', 'atmsim.fits')) atm.append(atmgrid) def matrice_data(): y = self.data - self.order2 nb_spectre = len(self.names) nb_bin = len(self.data) D = np.zeros(nb_bin * nb_spectre) for j in range(nb_spectre): D[j * nb_bin: (j + 1) * nb_bin] = y[:, j] return D def Atm(atm, ozone, eau, aerosols): nb_spectre = len(self.names) nb_bin = len(self.data) M = np.zeros((nb_spectre, nb_bin, nb_bin)) M_p = np.zeros((nb_spectre * nb_bin, nb_bin)) for j in range(nb_spectre): Atmo = np.zeros(len(self.new_lambda)) Lambdas = np.arange(self.Bin[0],self.Bin[-1],0.2) Atm = atm[j].simulate(ozone, eau, aerosols)(Lambdas) for i in range(len(self.new_lambda)): Atmo[i] = np.mean(Atm[i*int(self.binwidths/0.2):(i+1)*int(self.binwidths/0.2)]) a = np.diagflat(Atmo) M[j, :, :] = a M_p[nb_bin * j:nb_bin * (j+1),:] = a return M, M_p def log_likelihood(params_fit, atm): nb_spectre = len(self.names) nb_bin = len(self.data) ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1] D = matrice_data() M, M_p = Atm(atm, ozone, eau, aerosols) prod = np.zeros((nb_bin, nb_spectre * nb_bin)) for spec in range(nb_spectre): prod[:,spec * nb_bin : (spec+1) * nb_bin] = M[spec] @ self.INVCOV[spec] COV = inv(prod @ M_p) A = COV @ prod @ D chi2 = 0 for spec in range(nb_spectre): mat = D[spec * nb_bin : (spec+1) * nb_bin] - M[spec] @ A chi2 += mat @ self.INVCOV[spec] @ mat n = np.random.randint(0, 100) if n > 97: print(chi2 / (nb_spectre * nb_bin)) print(ozone, eau, aerosols) return -0.5 * chi2 def log_prior(params_fit): ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1] if 100 < ozone < 700 and 0 < eau < 10 and 0 < aerosols < 0.1: return 0 else: return -np.inf def log_probability(params_fit, atm): lp = log_prior(params_fit) if not np.isfinite(lp): return -np.inf return lp + log_likelihood(params_fit, atm) if self.sim: filename = "sps/" + self.disperseur + "_"+ "sim_"+ parameters.PROD_NUM + "_emcee.h5" else: filename = "sps/" + self.disperseur + "_" + "reduc_" + parameters.PROD_NUM + "_emcee.h5" p_ozone = 300 p_eau = 5 p_aerosols = 0.03 p0 = np.array([p_ozone, p_eau, p_aerosols]) walker = 10 init_ozone = p0[0] + p0[0] / 5 * np.random.randn(walker) init_eau = p0[1] + p0[1] / 5 * np.random.randn(walker) init_aerosols = p0[2] + p0[2] / 5 * np.random.randn(walker) p0 = np.array([[init_ozone[i], init_eau[i], init_aerosols[i]] for i in range(walker)]) nwalkers, ndim = p0.shape backend = emcee.backends.HDFBackend(filename) try: pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) sampler = emcee.EnsembleSampler(nwalkers, ndim, log_probability, args=(atm,), pool=pool, backend=backend) if backend.iteration > 0: p0 = backend.get_last_sample() if nsamples - backend.iteration > 0: sampler.run_mcmc(p0, nsteps=max(0, nsamples - backend.iteration), progress=True) pool.close() except ValueError: sampler = emcee.EnsembleSampler(nwalkers, ndim, log_probability, args=(atm,), threads=multiprocessing.cpu_count(), backend=backend) if backend.iteration > 0: p0 = sampler.get_last_sample() for _ in sampler.sample(p0, iterations=max(0, nsamples - backend.iteration), progress=True, store=True): continue flat_samples = sampler.get_chain(discard=100, thin=1, flat=True) ozone, d_ozone = np.mean(flat_samples[:, -3]), np.std(flat_samples[:, -3]) eau, d_eau = np.mean(flat_samples[:, -2]), np.std(flat_samples[:, -2]) aerosols, d_aerosols = np.mean(flat_samples[:, -1]), np.std(flat_samples[:, -1]) print(ozone, d_ozone) print(eau, d_eau) print(aerosols, d_aerosols) self.params_atmo = np.array([ozone, eau, aerosols]) self.err_params_atmo = np.array([d_ozone, d_eau, d_aerosols]) nb_spectre = len(self.names) nb_bin = len(self.data) M, M_p = Atm(atm, ozone, eau, aerosols) prod = np.zeros((nb_bin, nb_spectre * nb_bin)) chi2 = 0 for spec in range(nb_spectre): prod[:, spec * nb_bin: (spec + 1) * nb_bin] = M[spec] @ self.INVCOV[spec] COV = inv(prod @ M_p) D = matrice_data() Tinst = COV @ prod @ D Tinst_err = np.array([np.sqrt(COV[i,i]) for i in range(len(Tinst))]) if self.disperseur == 'HoloAmAg' and self.sim == False: a, b = np.argmin(abs(self.new_lambda - 537.5)), np.argmin(abs(self.new_lambda - 542.5)) Tinst_err[a], Tinst_err[b] = Tinst_err[a-1], Tinst_err[b+1] for spec in range(nb_spectre): mat = D[spec * nb_bin: (spec + 1) * nb_bin] - M[spec] @ Tinst chi2 += mat @ self.INVCOV[spec] @ mat print(chi2 / (nb_spectre * nb_bin)) err = np.zeros_like(D) for j in range(len(self.names)): for i in range(len(self.data)): if self.disperseur == 'HoloAmAg' and self.sim == False: if self.new_lambda[i] == 537.5 or self.new_lambda[i] == 542.5: err[j * len(self.data) + i] = 1 else: err[j * len(self.data) + i] = np.sqrt(self.cov[j][i, i]) else: err[j * len(self.data) + i] = np.sqrt(self.cov[j][i, i]) model = M_p @ Tinst Err = (D - model) / err if parameters.plot_residuals: self.Plot_residuals(model, D, Err, COV) return Tinst, Tinst_err
def main(path2config, time_likelihood): # Read params from yaml config = yaml.load(open(path2config)) mode = config['modus_operandi'] default_params = config['default_params'] power_params = config['power_params'] template_params = config['template_params'] cl_params = config['cl_params'] ch_config_params = config['ch_config_params'] fit_params = config['fit_params'] # parameters to fit nparams = len(fit_params.keys()) param_mapping = {} params = np.zeros((nparams, 4)) for key in fit_params.keys(): param_mapping[key] = fit_params[key][0] params[fit_params[key][0], :] = fit_params[key][1:] # Cosmology if set(COSMO_PARAM_KEYS) == set(default_params.keys()): print("We are NOT varying the cosmology") #cosmo_dict = {} #for key in COSMO_PARAM_KEYS: #cosmo_dict[key] = default_params[key] cosmo = ccl.Cosmology(**default_params) else: print("We ARE varying the cosmology") cosmo = None if power_params['lmax'] == 'kmax': lmax = kmax2lmax(power_params['kmax'], power_params['z'], cosmo) power_params['lmax'] = lmax if power_params['lmin'] == 'kmax': lmin = 0. power_params['lmin'] = lmin # read data parameters Data = PowerData(mode, power_params) Data.setup() # read theory parameters Theory = PowerTheory(mode, Data.x, Data.z, template_params, cl_params, power_params, default_params, param_mapping) Theory.setup() # initialize the Cl templates if mode == 'Cl' and cosmo != None: Theory.init_cl_ij(cosmo) # Make path to output if not os.path.isdir(os.path.expanduser(ch_config_params['path2output'])): try: os.makedirs(os.path.expanduser(ch_config_params['path2output'])) except: pass # MPI option if ch_config_params['use_mpi']: from schwimmbad import MPIPool pool = MPIPool() print("Using MPI") pool_use = pool else: pool = DumPool() print("Not using MPI") pool_use = None if not pool.is_master(): pool.wait() sys.exit(0) # just time the likelihood calculation if time_likelihood: time_lnprob(params, Data, Theory) return # emcee parameters nwalkers = nparams * ch_config_params['walkersRatio'] nsteps = ch_config_params['burninIterations'] + ch_config_params[ 'sampleIterations'] # where to record prefix_chain = os.path.join( os.path.expanduser(ch_config_params['path2output']), ch_config_params['chainsPrefix']) # fix initial conditions found_file = os.path.isfile(prefix_chain + '.txt') if (not found_file) or (not ch_config_params['rerun']): p_initial = params[:, 0] + np.random.normal( size=(nwalkers, nparams)) * params[:, 3][None, :] nsteps_use = nsteps else: print("Restarting from a previous run") old_chain = np.loadtxt(prefix_chain + '.txt') p_initial = old_chain[-nwalkers:, :] nsteps_use = max(nsteps - len(old_chain) // nwalkers, 0) # initializing sampler chain_file = SampleFileUtil(prefix_chain, carry_on=ch_config_params['rerun']) sampler = emcee.EnsembleSampler(nwalkers, nparams, lnprob, args=(params, Data, Theory), pool=pool_use) start = time.time() print("Running %d samples" % nsteps_use) # record every iteration counter = 1 for pos, prob, _ in sampler.sample(p_initial, iterations=nsteps_use): if pool.is_master(): print('Iteration done. Persisting.') chain_file.persistSamplingValues(pos, prob) if counter % 10: print(f"Finished sample {counter}") counter += 1 pool.close() end = time.time() print("Took ", (end - start), " seconds")
def ensemble(likelihood, prior, MPI = True, **kwargs): """ Initialise `emcee <http://dfm.io/emcee/current/>`_ and sample. :param likelihood: An instance of :class:`~.Likelihood.Likelihood`. :param prior: An instance of :class:`~.Prior.Prior`. :param bool MPI: Parallelise with MPI? If calling script not lauched with an MPI directive, sampling will not commence because there is only one process. Default is ``True`` since only in testing is it justifiable to use a single process. :param kwargs: Passed to initialisers of appropriate classes. * boolean to resume, under keyword :obj:`resume`; * number of steps, under keyword :obj:`nsteps`; * number of walkers, under keyword :obj:`nwalkers`; * moments of initial walker multivariate Gaussian distribution, under keyword :obj:`walker_dist_moments` (can be ``None``); * root directory for output, under keyword :obj:`root_dir`; The above objects are used to instantiate :class:`~.Posterior.Posterior`. :return: An instance of :class:`emcee.backends.HDFBackend`. """ # use the globally scoped variable global posterior from xpsi.Posterior import Posterior # Callable instance of the posterior posterior = Posterior(likelihood, prior, **kwargs) if MPI or global_imports._size > 1: try: from schwimmbad import MPIPool except ImportError: raise ImportError('Check your schwimmbad package installation.') # Initialize the MPI-based pool used for parallelisation. with MPIPool() as pool: if not pool.is_master(): # Wait for instructions from the master process. pool.wait() _sys.exit(0) from xpsi.EnsembleSampler import EnsembleSampler # Initialise emcee sampler sampler = EnsembleSampler(ndims = len(likelihood), posterior = func, pool = pool, **kwargs) # Commence emcee sampling process sampler() else: from xpsi.EnsembleSampler import EnsembleSampler # Initialise emcee sampler sampler = EnsembleSampler(ndims = len(likelihood), posterior = posterior, pool = None, **kwargs) # Commence emcee sampling process sampler() return sampler.backend
n = dt.now() print(f'starting lag {tau}, time now: {n.strftime("%Y %b %d %H:%M:%S")}') cf = dict(zip(sorted(dims),fin.apply(lambda col: col.autocorr(tau)))) print(f'computed lag {tau}, seconds taken: {(dt.now()-n).total_seconds():.2f}') return cf ## make up a function for batch computing autocorrelations def batch_autocorrelation(taus): n = dt.now() print(f'starting batch of lags {taus}, time now: {n.strftime("%Y %b %d %H:%M:%S")}') cf = [dict(zip(sorted(dims),fin.apply(lambda col: col.autocorr(t)))) for t in taus] print(f'computed batch of lags {taus}, seconds taken: {(dt.now()-n).total_seconds():.2f}', flush=True) return cf ## shut down all processes except the master one that will map tasks to others pool = MPIPool() if not pool.is_master(): print('one worker on standby') pool.wait() sys.exit(0) print('MPI master proceeding to map lags to workers...') print(f'There are {len(lag_batches)} total batches for parallelization') ## do the actual parallel computation of the autocorrelation function print(f'{dt.now().strftime("%Y %b %d %H:%M:%S")}, computing from {options["file"]}') if 'batch' in list(options.keys()) : acf = flatten(pool.map(batch_autocorrelation, lag_batches)) else : acf = pool.map(one_autocorrelation, lags) print(f'done with parallel computation, {dt.now().strftime("%Y %b %d %H:%M:%S")}') ## convert to dataframe
initcosmo("halofit") initbins(25, 30.0, 15000.0, 4000.0, 21.0, 10, 10) initsurvey("WFIRST") initgalaxies(file_source_z, file_lens_z, "gaussian", "gaussian", "source") initclusters() initia("none", "none") initpriors("none", "none", "PhotoBAO", "none") # test also with #initpriors("none","none","none","Planck") #initpriors("none","none","none","random") initprobes("all_2pt") initdatainv(cov_file, data_file) #sample_params=sample_LCDM_only() #sample_params= sample_cosmology_only() #sample_params = sample_cosmology_shear_nuisance(get_N_tomo_shear()) sample_params = sample_cosmology_2pt_shear_nuisance(get_N_tomo_shear(), get_N_tomo_clustering()) #sample_params = sample_cosmology_2pt_nuisance(get_N_tomo_shear(),get_N_tomo_clustering()) #sample_params = sample_cosmology_2pt_nuisance_IA_marg(get_N_tomo_shear(),get_N_tomo_clustering()) #sample_params = sample_cosmology_2pt_cluster_nuisance(get_N_tomo_shear(),get_N_tomo_clustering()) sample_main(sample_params, 1000, 560, 1, chain_file, blind=False, pool=MPIPool())
#nbunch = 25 nstep = 50000 # (total_number_of_cores - 1)*2, this should be equal to ndim x integer nwalkers = (size - 1) * 2 #nwalkers = 30 if nwalkers < ndim: nwalkers = ndim * 2 coords = np.random.randn(nwalkers, ndim) pos = [pinit + 1e-4 * np.random.randn(ndim) for j in range(nwalkers)] # run MCMC with MPIPool() as pool: if not pool.is_master(): pool.wait() sys.exit(0) filename_backend = os.path.join(dirname, "backend.h5") backend = emcee.backends.HDFBackend(filename_backend) backend.reset(nwalkers, ndim) sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob_global, pool=pool, backend=backend) index = 0 autocorr = np.empty(nstep)
def runMPI(runFunction, arguments): from schwimmbad import MPIPool with MPIPool() as pool: results = pool.map(runFunction, arguments) return results
else: if anlz.pmt1 and anlz.pmt2: saveName = ("/N/slate/frangonz/outputSing"+str(runListAll[0])+".txt") elif anlz.pmt1: saveName = ("/N/slate/frangonz/outputPMT1"+str(runListAll[0])+".txt") else: saveName = ("/N/slate/frangonz/outputPMT2"+str(runListAll[0])+".txt") #saveName = ("/N/u/frangonz/BigRed3/New_MCA_Analysis/outputs/outputSing"+str(iniRun)+".txt") if len(ctsTrunc) == 0: ctsTrunc = ctsSing saveName = "outputDebug.txt" nwalkers,ndim,walker_init = optimized_likelihood_fit(runListAll,ctsTrunc,nMon,bkgsH,bkgsT,anlz) try: # I've moved this down here so that we pickle the updated global variables pool = MPIPool() # This is the master MPI check if not pool.is_master(): # All the pools will get called after the master loads pool.wait() # Wait if you're not the master sys.exit(0) # Now let's do some quick writeouts for debug print("Environment:") print(" Config:",os.environ['ANALYSIS_CONFIG']) print(" Data:",os.environ['DATA_DIR']) print("RUNS:",runListAll) if anlz.sing: if anlz.pmt1 and anlz.pmt2: print("PMT 1+2") elif anlz.pmt1: print("PMT 1") else:
def MCMC(self, nwalkers=50, nburn=200, nMCMC=1000, use_MPI=False, chain_file='chain.dat', fig_name='./MCMC_corner.png', plot_corner=False, **kwargs): # The function to carry out MCMC. # parameters: # nwalkers: int, optional # the number of walkers in MCMC, which must be even. # default: 50 # nburn: int, optional # the number of burn-in steps in MCMC. # default: 200 # nMCMC: int, optional # the number of final MCMC steps in MCMC. # default: 1000 # use_MPI: Boolean, optional # whether to use MPI. # default: False # returns: # p_best: array_like # best fitting parameter set. # Initialize the walkers with a set of initial points, p0. E0 = np.random.normal(0.5, 0.3, size=nwalkers) T0 = np.random.normal(0.5, 0.3, size=nwalkers) a0 = np.random.normal(0, 0.7, size=nwalkers) covEE = truncnorm.rvs(0, 1, loc=0.3, scale=0.1, size=nwalkers) covTT = truncnorm.rvs(0 ,1, loc=0.3, scale=0.1, size=nwalkers) covaa = truncnorm.rvs(0, 1, loc=0.1, scale=0.1, size=nwalkers) covEa = truncnorm.rvs(0, 1, loc=0.1, scale=0.1, size=nwalkers) p0 = [[E0[i], T0[i], a0[i], covEE[i], covTT[i], covaa[i], covEa[i]] for i in range(nwalkers)] print 'start MCMC.' if not use_MPI: sampler = EnsembleSampler(nwalkers, self.ndim, lnprob, **kwargs) # sampler = EnsembleSampler(nwalkers, self.ndim, lnprob, \ # args=(self.E_grid, self.T_grid, self.a_grid, self.ba_lgSMA_bins, self.bin_obs, self.num_obs), **kwargs) # When using MPI, we differentiate between different processes. else: pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) # sampler = EnsembleSampler(nwalkers, self.ndim, lnprob, \ # args=(self.E_grid, self.T_grid, self.a_grid, self.ba_lgSMA_bins, self.bin_obs, self.num_obs), pool=pool, **kwargs) sampler = EnsembleSampler(nwalkers, self.ndim, lnprob, pool=pool, **kwargs) # burn-in phase pos, prob, state = sampler.run_mcmc(p0, nburn, chain_file=chain_file) sampler.reset() # MCMC phase sampler.run_mcmc(pos, nMCMC, chain_file=chain_file) if use_MPI: pool.close() # If we want to make classic corner plots... if plot_corner: samples = sampler.chain[:, nMCMC / 2:, :].reshape((-1, self.ndim)) fig = corner.corner(samples, labels=['E', 'T', 'a', 'covEE', 'covTT', 'covaa', 'covEa']) fig.savefig(fig_name) # Get the best fitting parameters. We take the median parameter value for the ensemble # of steps with log-probabilities within the largest 30% among the whole ensemble as the # best parameters. samples = sampler.flatchain lnp = sampler.flatlnprobability crit_lnp = np.percentile(lnp, 70) good = np.where(lnp > crit_lnp) p_best = [np.median(samples[good, i]) for i in range(self.ndim)] return np.array(p_best)
def sample(self, carry_on=False, verbosity=0, use_mpi=False): """ Sample the posterior distribution Args: carry_on (bool): if True, the sampler will restart from its last iteration. verbosity (int): if >0, progress will be reported. use_mpi (bool): set to True to parallelize with MPI Returns: :obj:`emcee.EnsembleSampler`: sampler with chain. """ import emcee if use_mpi: from schwimmbad import MPIPool pool = MPIPool() print("Using MPI") pool_use = pool else: pool = DumPool() print("Not using MPI") pool_use = None if not pool.is_master(): pool.wait() sys.exit(0) fname_chain = self.prefix_out+"chain" found_file = os.path.isfile(fname_chain+'.txt') counter = 1 if (not found_file) or (not carry_on): pos_ini = (np.array(self.p0)[None, :] + 0.001 * np.random.randn(self.nwalkers, self.ndim)) nsteps_use = self.nsteps else: print("Restarting from previous run") with warnings.catch_warnings(): warnings.simplefilter("ignore") old_chain = np.loadtxt(fname_chain+'.txt') if old_chain.size != 0: pos_ini = old_chain[-self.nwalkers:, :] nsteps_use = max(self.nsteps-len(old_chain) // self.nwalkers, 0) counter = len(old_chain) // self.nwalkers # print(self.nsteps - len(old_chain) // self.nwalkers) else: pos_ini = (np.array(self.p0)[None, :] + 0.001 * np.random.randn(self.nwalkers, self.ndim)) nsteps_use = self.nsteps chain_file = SampleFileUtil(self.prefix_out+"chain", carry_on=carry_on) sampler = emcee.EnsembleSampler(self.nwalkers, self.ndim, self.lnprob, pool=pool_use) for pos, prob, _ in sampler.sample(pos_ini, iterations=nsteps_use): if pool.is_master(): if verbosity > 0: print('Iteration done. Persisting.') chain_file.persistSamplingValues(pos, prob) if (counter % 10) == 0: print(f"Finished sample {counter}") counter += 1 pool.close() return sampler
flux_df = pd.read_csv('../'+args.fluxes) filter_df = pd.read_csv('../filters.csv') #Read in and zip up dataframes sCM20_df = pd.read_hdf('models.h5','sCM20') lCM20_df = pd.read_hdf('models.h5','lCM20') aSilM5_df = pd.read_hdf('models.h5','aSilM5') wavelength_df = pd.read_hdf('models.h5','wavelength') pandas_dfs = [sCM20_df,lCM20_df, aSilM5_df,wavelength_df] if args.mpi: mpi_pool = MPIPool() if not mpi_pool.is_master(): mpi_pool.wait() sys.exit(0) mpi_pool.map( main, range(len(flux_df)) ) mpi_pool.close() else: for gal_row in range(len(flux_df)): main(gal_row)
totsum = np.sum(chunksum, axis=0) weightsum = np.sum(chunkweightsum, axis=0) finalstack = totsum / weightsum finalstack.dump('%s.npy' % outname) print(time.time() - starttime) if __name__ == "__main__": import sys from schwimmbad import MPIPool from astropy.io import fits import glob # set up schwimmbad MPI pool pool = MPIPool() # if current instantiation is not the master, wait for tasks if not pool.is_master(): pool.wait() sys.exit(0) sample_name = 'xdqso_specz' imsize = 160 reso = 1.5 stack_maps = False stack_noise = False nbootstacks = 10 nrandoms = 0 temperature = False
def MCMC(self,pool): """ The MCMC method. Construct starting points of the chains around the best solution found by the 'DE' method. The objective function is :func:`chichi_MCMC`. Telescope flux (fs and g), can be optimized thanks to MCMC if flux_estimation_MCMC is 'MCMC', either they are derived through np.polyfit. Based on the emcee python package : " emcee: The MCMC Hammer" (Foreman-Mackey et al. 2013). Have a look here : http://dan.iel.fm/emcee/current/ :return: a tuple containing (MCMC_chains, MCMC_probabilities) :rtype: tuple **WARNING** : nwalkers is set to 4 times the len of pazynski_parameters nlinks is set to 4000 5*nwalkers*nlinks MCMC steps in total """ # start = python_time.time() if len(self.model.parameters_guess) == 0: differential_evolution_estimation = self.differential_evolution(pool)[0] self.DE_population_size = 10 self.guess = differential_evolution_estimation else: self.guess = list(self.model.parameters_guess) self.guess += self.find_fluxes(self.guess, self.model) # Best solution if self.fluxes_MCMC_method != 'MCMC': limit_parameters = len(self.model.parameters_boundaries) best_solution = self.guess[:limit_parameters] else: limit_parameters = len(self.guess) best_solution = self.guess nwalkers = 2*len(best_solution) nlinks = 5*1000 # Initialize the population of MCMC population = [] count_walkers = 0 while count_walkers < nwalkers: # Construct an individual of the population around the best solution. individual = [] for parameter_key in list(self.model.model_dictionnary.keys())[:limit_parameters]: parameter_trial = microlguess.MCMC_parameters_initialization(parameter_key, self.model.model_dictionnary, best_solution) if parameter_trial: for parameter in parameter_trial: individual.append(parameter) #if self.fluxes_MCMC_method == 'MCMC': # fluxes = self.find_fluxes(individual, self.model) # individual += (fluxes*np.random.uniform(0.99,1.01,len(fluxes))+np.random.uni).tolist() chichi = self.chichi_MCMC(individual) if chichi != -np.inf: # np.array(individual) # print count_walkers population.append(np.array(individual)) count_walkers += 1 print('pre MCMC done') number_of_parameters = len(individual) try: # create a new MPI pool from schwimmbad import MPIPool pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) except: pass sampler = emcee.EnsembleSampler(nwalkers, number_of_parameters, self.chichi_MCMC, a=2.0, pool= pool) # First estimation using population as a starting points. #final_positions, final_probabilities, state = sampler.run_mcmc(population, nlinks, progress=True) #print('MCMC preburn done') #sampler.reset() sampler.run_mcmc(population, nlinks, progress=True) MCMC_chains = np.c_[sampler.get_chain().reshape(nlinks*nwalkers,number_of_parameters),sampler.get_log_prob().reshape(nlinks*nwalkers)] # Final estimation using the previous output. #for positions, probabilities, states in sampler.sample(final_positions, iterations= nlinks, # storechain=True): # chains = np.c_[positions, probabilities] # if MCMC_chains is not None: # MCMC_chains = np.r_[MCMC_chains, chains] # else: # MCMC_chains = chains print(sys._getframe().f_code.co_name, ' : MCMC fit SUCCESS') return MCMC_chains
file_source_z = os.path.join(dirname, "zdistris/zdistribution_DESY1_source") file_lens_z = os.path.join(dirname, "zdistris/zdistribution_DESY1_lens") data_file = os.path.join(dirname, "datav/DES_all_2pt_fid_opti") cov_file = os.path.join(dirname, "cov/DES_cov_3x2pt_inv") chain_file = os.path.join(dirname, "like/like_DES_ocelote_3x2pt_LCDM") initcosmo("halofit") initbins(25,30.0,15000.0,4000.0,21.0,4,5) initpriors("photo_opti","shear_opti","none","none") initsurvey("WFIRST") initgalaxies(file_source_z,file_lens_z,"gaussian","gaussian","DES_Y1") initclusters() initia("none","none") # test also with #initpriors("none","none","none","Planck") #initpriors("none","none","none","random") initprobes("all_2pt") initdatainv(cov_file ,data_file) sample_params=sample_LCDM_only() #sample_params=sample_LCDM_2pt_nuisance() #sample_params= sample_cosmology_only() #sample_params = sample_cosmology_shear_nuisance(get_N_tomo_shear()) #sample_params = sample_cosmology_2pt_nuisance(get_N_tomo_shear(),get_N_tomo_clustering()) #sample_params = sample_cosmology_2pt_nuisance_IA_marg(get_N_tomo_shear(),get_N_tomo_clustering()) #sample_params = sample_cosmology_2pt_cluster_nuisance(get_N_tomo_shear(),get_N_tomo_clustering()) sample_main(sample_params,2000,560,1,chain_file, blind=False, pool=MPIPool())
def main(path2config, time_likelihood): # load the yaml parameters config = yaml.load(open(path2config)) sim_params = config['sim_params'] HOD_params = config['HOD_params'] clustering_params = config['clustering_params'] data_params = config['data_params'] ch_config_params = config['ch_config_params'] fit_params = config['fit_params'] # create a new abacushod object and load the subsamples newBall = AbacusHOD(sim_params, HOD_params, clustering_params) # read data parameters newData = PowerData(data_params, HOD_params) # parameters to fit nparams = len(fit_params.keys()) param_mapping = {} param_tracer = {} params = np.zeros((nparams, 4)) for key in fit_params.keys(): mapping_idx = fit_params[key][0] tracer_type = fit_params[key][-1] param_mapping[key] = mapping_idx param_tracer[key] = tracer_type params[mapping_idx, :] = fit_params[key][1:-1] # Make path to output if not os.path.isdir(os.path.expanduser(ch_config_params['path2output'])): try: os.makedirs(os.path.expanduser(ch_config_params['path2output'])) except: pass # MPI option if ch_config_params['use_mpi']: from schwimmbad import MPIPool pool = MPIPool() print("Using MPI") pool_use = pool else: pool = DumPool() print("Not using MPI") pool_use = None if not pool.is_master(): pool.wait() sys.exit(0) # just time the likelihood calculation if time_likelihood: time_lnprob(params, param_mapping, param_tracer, newData, newBall) return # emcee parameters nwalkers = nparams * ch_config_params['walkersRatio'] nsteps = ch_config_params['burninIterations'] + ch_config_params[ 'sampleIterations'] # where to record prefix_chain = os.path.join( os.path.expanduser(ch_config_params['path2output']), ch_config_params['chainsPrefix']) # fix initial conditions found_file = os.path.isfile(prefix_chain + '.txt') if (not found_file) or (not ch_config_params['rerun']): p_initial = params[:, 0] + np.random.normal( size=(nwalkers, nparams)) * params[:, 3][None, :] nsteps_use = nsteps else: print("Restarting from a previous run") old_chain = np.loadtxt(prefix_chain + '.txt') p_initial = old_chain[-nwalkers:, :] nsteps_use = max(nsteps - len(old_chain) // nwalkers, 0) # initializing sampler chain_file = SampleFileUtil(prefix_chain, carry_on=ch_config_params['rerun']) sampler = emcee.EnsembleSampler(nwalkers, nparams, lnprob, args=(params, param_mapping, param_tracer, newData, newBall), pool=pool_use) start = time.time() print("Running %d samples" % nsteps_use) # record every iteration counter = 1 for pos, prob, _ in sampler.sample(p_initial, iterations=nsteps_use): if pool.is_master(): print('Iteration done. Persisting.') chain_file.persistSamplingValues(pos, prob) if counter % 10: print(f"Finished sample {counter}") counter += 1 pool.close() end = time.time() print("Took ", (end - start), " seconds")
def fit_events(self, events=[], models=[], max_time='', time_list=[], time_unit=None, band_list=[], band_systems=[], band_instruments=[], band_bandsets=[], band_sampling_points=17, iterations=10000, num_walkers=None, num_temps=1, parameter_paths=['parameters.json'], fracking=True, frack_step=50, burn=None, post_burn=None, gibbs=False, smooth_times=-1, extrapolate_time=0.0, limit_fitting_mjds=False, exclude_bands=[], exclude_instruments=[], exclude_systems=[], exclude_sources=[], exclude_kinds=[], output_path='', suffix='', upload=False, write=False, upload_token='', check_upload_quality=False, variance_for_each=[], user_fixed_parameters=[], user_released_parameters=[], convergence_type=None, convergence_criteria=None, save_full_chain=False, draw_above_likelihood=False, maximum_walltime=False, start_time=False, print_trees=False, maximum_memory=np.inf, speak=False, return_fits=True, extra_outputs=None, walker_paths=[], catalogs=[], exit_on_prompt=False, download_recommended_data=False, local_data_only=False, method=None, seed=None, **kwargs): """Fit a list of events with a list of models.""" global model if start_time is False: start_time = time.time() self._seed = seed if seed is not None: np.random.seed(seed) self._start_time = start_time self._maximum_walltime = maximum_walltime self._maximum_memory = maximum_memory self._debug = False self._speak = speak self._download_recommended_data = download_recommended_data self._local_data_only = local_data_only self._draw_above_likelihood = draw_above_likelihood prt = self._printer event_list = listify(events) model_list = listify(models) if len(model_list) and not len(event_list): event_list = [''] # Exclude catalogs not included in catalog list. self._fetcher.add_excluded_catalogs(catalogs) if not len(event_list) and not len(model_list): prt.message('no_events_models', warning=True) # If the input is not a JSON file, assume it is either a list of # transients or that it is the data from a single transient in tabular # form. Try to guess the format first, and if that fails ask the user. self._converter = Converter(prt, require_source=upload) event_list = self._converter.generate_event_list(event_list) event_list = [x.replace('‑', '-') for x in event_list] entries = [[] for x in range(len(event_list))] ps = [[] for x in range(len(event_list))] lnprobs = [[] for x in range(len(event_list))] # Load walker data if provided a list of walker paths. walker_data = [] if len(walker_paths): try: pool = MPIPool() except (ImportError, ValueError): pool = SerialPool() if pool.is_master(): prt.message('walker_file') wfi = 0 for walker_path in walker_paths: if os.path.exists(walker_path): prt.prt(' {}'.format(walker_path)) with codecs.open(walker_path, 'r', encoding='utf-8') as f: all_walker_data = json.load( f, object_pairs_hook=OrderedDict) # Support both the format where all data stored in a # single-item dictionary (the OAC format) and the older # MOSFiT format where the data was stored in the # top-level dictionary. if ENTRY.NAME not in all_walker_data: all_walker_data = all_walker_data[list( all_walker_data.keys())[0]] models = all_walker_data.get(ENTRY.MODELS, []) choice = None if len(models) > 1: model_opts = [ '{}-{}-{}'.format(x['code'], x['name'], x['date']) for x in models ] choice = prt.prompt('select_model_walkers', kind='select', message=True, options=model_opts) choice = model_opts.index(choice) elif len(models) == 1: choice = 0 if choice is not None: walker_data.extend([[ wfi, x[REALIZATION.PARAMETERS], x.get(REALIZATION.WEIGHT) ] for x in models[choice][MODEL.REALIZATIONS]]) for i in range(len(walker_data)): if walker_data[i][2] is not None: walker_data[i][2] = float(walker_data[i][2]) if not len(walker_data): prt.message('no_walker_data') else: prt.message('no_walker_data') if self._offline: prt.message('omit_offline') raise RuntimeError wfi = wfi + 1 for rank in range(1, pool.size + 1): pool.comm.send(walker_data, dest=rank, tag=3) else: walker_data = pool.comm.recv(source=0, tag=3) pool.wait() if pool.is_master(): pool.close() self._event_name = 'Batch' self._event_path = '' self._event_data = {} try: pool = MPIPool() except (ImportError, ValueError): pool = SerialPool() if pool.is_master(): fetched_events = self._fetcher.fetch( event_list, offline=self._offline, prefer_cache=self._prefer_cache) for rank in range(1, pool.size + 1): pool.comm.send(fetched_events, dest=rank, tag=0) pool.close() else: fetched_events = pool.comm.recv(source=0, tag=0) pool.wait() for ei, event in enumerate(fetched_events): if event is not None: self._event_name = event.get('name', 'Batch') self._event_path = event.get('path', '') if not self._event_path: continue self._event_data = self._fetcher.load_data(event) if not self._event_data: continue if model_list: lmodel_list = model_list else: lmodel_list = [''] entries[ei] = [None for y in range(len(lmodel_list))] ps[ei] = [None for y in range(len(lmodel_list))] lnprobs[ei] = [None for y in range(len(lmodel_list))] if (event is not None and (not self._event_data or ENTRY.PHOTOMETRY not in self._event_data[list(self._event_data.keys())[0]])): prt.message('no_photometry', [self._event_name]) continue for mi, mod_name in enumerate(lmodel_list): for parameter_path in parameter_paths: try: pool = MPIPool() except (ImportError, ValueError): pool = SerialPool() self._model = Model(model=mod_name, data=self._event_data, parameter_path=parameter_path, output_path=output_path, wrap_length=self._wrap_length, test=self._test, printer=prt, fitter=self, pool=pool, print_trees=print_trees) if not self._model._model_name: prt.message('no_models_avail', [self._event_name], warning=True) continue if not event: prt.message('gen_dummy') self._event_name = mod_name gen_args = { 'name': mod_name, 'max_time': max_time, 'time_list': time_list, 'band_list': band_list, 'band_systems': band_systems, 'band_instruments': band_instruments, 'band_bandsets': band_bandsets } self._event_data = self.generate_dummy_data(**gen_args) success = False alt_name = None while not success: self._model.reset_unset_recommended_keys() success = self._model.load_data( self._event_data, event_name=self._event_name, smooth_times=smooth_times, extrapolate_time=extrapolate_time, limit_fitting_mjds=limit_fitting_mjds, exclude_bands=exclude_bands, exclude_instruments=exclude_instruments, exclude_systems=exclude_systems, exclude_sources=exclude_sources, exclude_kinds=exclude_kinds, time_list=time_list, time_unit=time_unit, band_list=band_list, band_systems=band_systems, band_instruments=band_instruments, band_bandsets=band_bandsets, band_sampling_points=band_sampling_points, variance_for_each=variance_for_each, user_fixed_parameters=user_fixed_parameters, user_released_parameters=user_released_parameters, pool=pool) if not success: break if self._local_data_only: break # If our data is missing recommended keys, offer the # user option to pull the missing data from online and # merge with existing data. urk = self._model.get_unset_recommended_keys() ptxt = prt.text('acquire_recommended', [', '.join(list(urk))]) while event and len(urk) and ( alt_name or self._download_recommended_data or prt.prompt(ptxt, [', '.join(urk)], kind='bool')): try: pool = MPIPool() except (ImportError, ValueError): pool = SerialPool() if pool.is_master(): en = (alt_name if alt_name else self._event_name) extra_event = self._fetcher.fetch( en, offline=self._offline, prefer_cache=self._prefer_cache)[0] extra_data = self._fetcher.load_data( extra_event) for rank in range(1, pool.size + 1): pool.comm.send(extra_data, dest=rank, tag=4) pool.close() else: extra_data = pool.comm.recv(source=0, tag=4) pool.wait() if extra_data is not None: extra_data = extra_data[list( extra_data.keys())[0]] for key in urk: new_val = extra_data.get(key) self._event_data[list( self._event_data.keys()) [0]][key] = new_val if new_val is not None and len(new_val): prt.message('extra_value', [ key, str(new_val[0].get(QUANTITY.VALUE)) ]) success = False prt.message('reloading_merged') break else: text = prt.text('extra_not_found', [self._event_name]) alt_name = prt.prompt(text, kind='string') if not alt_name: break if success: self._walker_data = walker_data entry, p, lnprob = self.fit_data( event_name=self._event_name, method=method, iterations=iterations, num_walkers=num_walkers, num_temps=num_temps, burn=burn, post_burn=post_burn, fracking=fracking, frack_step=frack_step, gibbs=gibbs, pool=pool, output_path=output_path, suffix=suffix, write=write, upload=upload, upload_token=upload_token, check_upload_quality=check_upload_quality, convergence_type=convergence_type, convergence_criteria=convergence_criteria, save_full_chain=save_full_chain, extra_outputs=extra_outputs) if return_fits: entries[ei][mi] = deepcopy(entry) ps[ei][mi] = deepcopy(p) lnprobs[ei][mi] = deepcopy(lnprob) if pool.is_master(): pool.close() # Remove global model variable and garbage collect. try: model except NameError: pass else: del (model) del (self._model) gc.collect() return (entries, ps, lnprobs)
parser.add_argument('savepath') args = parser.parse_args() n = args.n S_ = args.S savepath = args.savepath p = 250 comm = MPI.COMM_WORLD rank = comm.rank numproc = comm.Get_size() # Keep sigma and gamma squared equal to each other sigma_sq = 1 gamma_sq = 0.1 F = np.linspace(0, 2 * np.log(n), 10) T = np.linspace(1, p, 25, dtype=int) pool = MPIPool(comm) worker = Worker(savepath, gamma_sq, sigma_sq, n, S_) # Split F and T into tasks tasks = itertools.product(T, F) pool.map(worker, tasks, callback=worker.save) pool.close()
# update epsilon based on median thresholding eps.eps = np.median(pool.dists, axis=0) print('eps%i' % pool.t, eps.eps) print('----------------------------------------') #if pool.ratio <0.2: break abcpmc_sampler.close() return None if __name__ == "__main__": if machine == 'siro': # run with MPI from mpi4py import MPI from schwimmbad import MPIPool pewl = MPIPool() if not pewl.is_master(): pewl.wait() sys.exit(0) else: pewl = None name = sys.argv[7] # name of ABC run niter = int(sys.argv[8]) # number of iterations restart = (sys.argv[9] == 'True') print('Runnin ABC with ...') print('%s simulation' % sim) print('%s DEM' % dem) print('%s distance' % distance_method) print('%s summary statistic' % statistic)
return beam, tune, pol = cFrame.parseID() if tune == 0: tune += 1 aStand = 2 * (tune - 1) + pol data[aStand, count[aStand] * 4096:(count[aStand] + 1) * 4096] = cFrame.data.iq count[aStand] += 1 # Calculate the spectra for this block of data, in the unit of intensity masterSpectra[i, 0, :] = ( (numpy.fft.fftshift(numpy.abs(numpy.fft.fft2( data[:2, :]))[:, 1:])[:, Lfcl:Lfch])**2.).mean(0) / LFFT / 2. masterSpectra[i, 1, :] = ( (numpy.fft.fftshift(numpy.abs(numpy.fft.fft2( data[2:, :]))[:, 1:])[:, Hfcl:Hfch])**2.).mean(0) / LFFT / 2. # Save the results to the various master arrays outname = "%s_%i_fft_offset_%.9i_frames" % (filename, beam, offset) log("Writing %s" % outname) numpy.save(outname, masterSpectra) if __name__ == "__main__": from schwimmbad import MPIPool pool = MPIPool() if not pool.is_master(): # Workers wait here pool.wait() sys.exit(0) main(pool)