def get_model(n_obs=50, true_params=None, seed_obs=None): """Return a complete Gaussian noise model. Parameters ---------- n_obs : int, optional the number of observations true_params : list, optional true_params[0] corresponds to the mean, true_params[1] corresponds to the standard deviation seed_obs : int, optional seed for the observed data generation Returns ------- m : elfi.ElfiModel """ if true_params is None: true_params = [10, 2] y_obs = Gauss(*true_params, n_obs=n_obs, random_state=np.random.RandomState(seed_obs)) sim_fn = partial(Gauss, n_obs=n_obs) m = elfi.ElfiModel() elfi.Prior('uniform', -10, 50, model=m, name='mu') elfi.Prior('truncnorm', 0.01, 5, model=m, name='sigma') elfi.Simulator(sim_fn, m['mu'], m['sigma'], observed=y_obs, name='Gauss') elfi.Summary(ss_mean, m['Gauss'], name='S1') elfi.Summary(ss_var, m['Gauss'], name='S2') elfi.Distance('euclidean', m['S1'], m['S2'], name='d') return m
def get_model(self, n_obs=100, true_params=None, seed_obs=None): """Return a complete MA2 model in inference task. Parameters ---------- n_obs : int, optional observation length of the MA2 process true_params : list, optional parameters with which the observed data is generated seed_obs : int, optional seed for the observed data generation Returns ------- m : elfi.ElfiModel """ if true_params is None: true_params = [60] y_obs = self.func(*true_params, random_state=np.random.RandomState(seed_obs)) m = elfi.ElfiModel() elfi.Prior(ss.uniform, 0, 100, model=m, name='t1') elfi.Simulator(self.func, m['t1'], observed=y_obs, name='sim') elfi.Distance('euclidean', m['sim'], name='dist') return m
def create_model(shell_command): m = elfi.ElfiModel(name='nfds') #creates model # shell_command = './updated_nfds_model/freqDepSelect -f input_files/mass/mass.input -c rmlB -v {0} -u 0.95 -n 50000 -s {1} -g 73 -t 1 -l 0.05 -i {2} -p f -o temp_output | perl -lane "print $F[2];"' nfds_sim = elfi.tools.external_operation( shell_command, stdout=True) #wraps command in a python function nfds_sim_vector = elfi.tools.vectorize(nfds_sim) # vectorizes command # m = elfi.ElfiModel(name = 'nfds') #creates model #Defines 3 priors for the model elfi.Prior('uniform', 0, 1, model=m, name='t1') elfi.Prior('uniform', 0, 0.05, model=m, name='t2') elfi.Prior('uniform', 0, 1, model=m, name='t3') # create central node that runs the model, I think nfds_simulator = elfi.Simulator(nfds_sim_vector, m['t1'], m['t2'], m['t3'], name='nfds', observed=0.0) return (nfds_simulator)
def test_list_output(): vsim = elfi.tools.vectorize(lsimulator) vsum = elfi.tools.vectorize(lsummary) v = vsim(np.array([[.2, .8], [.3, .7]])) assert is_array(v) assert not isinstance(v[0], list) vsim = elfi.tools.vectorize(lsimulator, dtype=False) v = vsim(np.array([[.2, .8], [.3, .7]])) assert is_array(v) assert isinstance(v[0], list) obs = lsimulator([.2, .8]) elfi.ElfiModel() p = elfi.Prior('dirichlet', [2, 2]) sim = elfi.Simulator(vsim, p, observed=obs) S = elfi.Summary(vsum, sim) d = elfi.Distance('euclidean', S) pool = elfi.OutputPool(['sim']) rej = elfi.Rejection(d, batch_size=100, pool=pool, output_names=['sim']) sample = rej.sample(100, n_sim=1000) mean = np.mean(sample.samples['p'], axis=0) # Crude test assert mean[1] - mean[0] > .2
def get_model(n_obs=100, true_params=None, seed_obs=None): """Returns a complete MA2 model in inference task Parameters ---------- n_obs : int observation length of the MA2 process true_params : list parameters with which the observed data is generated seed_obs : None, int seed for the observed data generation Returns ------- m : elfi.ElfiModel """ if true_params is None: true_params = [.6, .2] y = MA2(*true_params, n_obs=n_obs, random_state=np.random.RandomState(seed_obs)) sim_fn = partial(MA2, n_obs=n_obs) m = elfi.ElfiModel(set_current=False) elfi.Prior(CustomPrior1, 2, model=m, name='t1') elfi.Prior(CustomPrior2, m['t1'], 1, name='t2') elfi.Simulator(sim_fn, m['t1'], m['t2'], observed=y, name='MA2') elfi.Summary(autocov, m['MA2'], name='S1') elfi.Summary(autocov, m['MA2'], 2, name='S2') elfi.Distance('euclidean', m['S1'], m['S2'], name='d') return m
def get_model(self, n_obs=100, true_params=None, seed_obs=None): """Return a complete model in inference task. Parameters ---------- n_obs : int, optional observation length of the MA2 process true_params : list, optional parameters with which the observed data is generated seed_obs : int, optional seed for the observed data generation Returns ------- m : elfi.ElfiModel """ m = elfi.ElfiModel() if true_params is None: elfi.Prior('uniform', -20.0, 20.0, model=m, name='white') elfi.Prior('uniform', -20.0, 20.0, model=m, name='yellow') elfi.Prior('uniform', -20.0, 20.0, model=m, name='red') elfi.Prior('uniform', -20.0, 20.0, model=m, name='green') elfi.Prior('uniform', -20.0, 20.0, model=m, name='purple') params = [ m['white'], m['yellow'], m['red'], m['green'], m['purple'] ] y_obs = self.get_observed_data() elfi.Simulator(self.func, *params, observed=y_obs, name='sim') elfi.Distance('euclidean', m['sim'], name='dist') # elfi.Distance(self.discrepancyII, m['DGP'], name='d') return m
def test_single_parameter_linear_adjustment(): """A regression test against values obtained in the notebook.""" seed = 20170616 n_obs = 50 batch_size = 100 mu, sigma = (5, 1) # Hyperparameters mu0, sigma0 = (10, 100) y_obs = gauss.Gauss( mu, sigma, n_obs=n_obs, batch_size=1, random_state=np.random.RandomState(seed)) sim_fn = partial(gauss.Gauss, sigma=sigma, n_obs=n_obs) # Posterior n = y_obs.shape[1] mu1 = (mu0 / sigma0**2 + y_obs.sum() / sigma**2) / (1 / sigma0**2 + n / sigma**2) sigma1 = (1 / sigma0**2 + n / sigma**2)**(-0.5) # Model m = elfi.ElfiModel() elfi.Prior('norm', mu0, sigma0, model=m, name='mu') elfi.Simulator(sim_fn, m['mu'], observed=y_obs, name='Gauss') elfi.Summary(lambda x: x.mean(axis=1), m['Gauss'], name='S1') elfi.Distance('euclidean', m['S1'], name='d') res = elfi.Rejection(m['d'], output_names=['S1'], seed=seed).sample(1000, threshold=1) adj = elfi.adjust_posterior(model=m, sample=res, parameter_names=['mu'], summary_names=['S1']) assert np.allclose(_statistics(adj.outputs['mu']), (4.9772879640569778, 0.02058680115402544))
def get_model(n_obs=100, true_params=None, seed_obs=None, n_lags=5): """Return a complete ARCH(1) model. Parameters ---------- n_obs: int Observation length of the ARCH(1) process. true_params: list, optinal Parameters with which the observed data are generated. seed_obs: int, optional Seed for the observed data generation. n_lags: int, optional Number of lags in summary statistics. Returns ------- elfi.ElfiModel """ if true_params is None: true_params = [0.3, 0.7] logger.info( f'true_params were not given. Now using [t1, t2] = {true_params}.') # elfi model m = elfi.ElfiModel() # priors t1 = elfi.Prior('uniform', -1, 2, model=m) t2 = elfi.Prior('uniform', 0, 1, model=m) priors = [t1, t2] # observations y_obs = arch(*true_params, n_obs=n_obs, random_state=np.random.RandomState(seed_obs)) # simulator Y = elfi.Simulator(arch, *priors, observed=y_obs) # summary statistics ss = [] ss.append(elfi.Summary(sample_mean, Y, name='MU', model=m)) ss.append(elfi.Summary(sample_variance, Y, name='VAR', model=m)) for i in range(1, n_lags + 1): ss.append(elfi.Summary(autocorr, Y, i, name=f'AC_{i}', model=m)) for i, j in combinations(range(1, n_lags + 1), 2): ss.append( elfi.Summary(pairwise_autocorr, Y, i, j, name=f'PW_{i}_{j}', model=m)) # distance elfi.Distance('euclidean', *ss, name='d', model=m) return m
def get_model(n_obs=50, true_params=None, seed_obs=None, stochastic=True): """Returns a complete Ricker model in inference task. This is a simplified example that achieves reasonable predictions. For more extensive treatment and description using 13 summary statistics, see: Wood, S. N. (2010) Statistical inference for noisy nonlinear ecological dynamic systems, Nature 466, 1102–1107. Parameters ---------- n_obs : int, optional Number of observations. true_params : list, optional Parameters with which the observed data is generated. seed_obs : None, int, optional Seed for the observed data generation. stochastic : bool, optional Whether to use the stochastic or deterministic Ricker model. Returns ------- m : elfi.ElfiModel """ if stochastic: simulator = partial(stochastic_ricker, n_obs=n_obs) if true_params is None: true_params = [3.8, 0.3, 10.] else: simulator = partial(ricker, n_obs=n_obs) if true_params is None: true_params = [3.8] m = elfi.ElfiModel() y_obs = simulator(*true_params, n_obs=n_obs, random_state=np.random.RandomState(seed_obs)) sim_fn = partial(simulator, n_obs=n_obs) sumstats = [] if stochastic: elfi.Prior(ss.expon, np.e, 2, model=m, name='t1') elfi.Prior(ss.truncnorm, 0, 5, model=m, name='t2') elfi.Prior(ss.uniform, 0, 100, model=m, name='t3') elfi.Simulator(sim_fn, m['t1'], m['t2'], m['t3'], observed=y_obs, name='Ricker') sumstats.append(elfi.Summary(partial(np.mean, axis=1), m['Ricker'], name='Mean')) sumstats.append(elfi.Summary(partial(np.var, axis=1), m['Ricker'], name='Var')) sumstats.append(elfi.Summary(num_zeros, m['Ricker'], name='#0')) elfi.Discrepancy(chi_squared, *sumstats, name='d') else: # very simple deterministic case elfi.Prior(ss.expon, np.e, model=m, name='t1') elfi.Simulator(sim_fn, m['t1'], observed=y_obs, name='Ricker') sumstats.append(elfi.Summary(partial(np.mean, axis=1), m['Ricker'], name='Mean')) elfi.Distance('euclidean', *sumstats, name='d') return m
def test_numerical_grad_logpdf(self): # Test gradient with a normal distribution loc = 2.2 scale = 1.1 x = np.random.rand() analytical_grad_logpdf = -(x - loc) / scale**2 prior_node = elfi.Prior('normal', loc, scale, model=elfi.ElfiModel()) num_grad = ModelPrior(prior_node.model).gradient_logpdf(x) assert np.isclose(num_grad, analytical_grad_logpdf, atol=0.01)
def get_model(alpha=0.2, delta=0, tau=0.198, N=20, seed_obs=None): """Returns the example model used in Lintusaari et al. 2016. Here we infer alpha using the summary statistic T1. We expect the executable `bdm` be available in the working directory. Parameters ---------- alpha : float birth rate delta : float death rate tau : float mutation rate N : int size of the population seed_obs : None, int Seed for the observed data generation. None gives the same data as in Lintusaari et al. 2016 Returns ------- m : elfi.ElfiModel """ if seed_obs is None and N == 20: y = np.zeros(N, dtype='int16') data = np.array([6, 3, 2, 2, 1, 1, 1, 1, 1, 1, 1], dtype='int16') y[0:len(data)] = data else: y = BDM(alpha, delta, tau, N, random_state=np.random.RandomState(seed_obs)) m = elfi.ElfiModel(name='bdm') elfi.Prior('uniform', .005, 2, model=m, name='alpha') elfi.Simulator(BDM, m['alpha'], delta, tau, N, observed=y, name='BDM') elfi.Summary(T1, m['BDM'], name='T1') elfi.Distance('minkowski', m['T1'], p=1, name='d') m['BDM'].uses_meta = True # Warn the user if the executable is not present if not os.path.isfile('bdm') and not os.path.isfile('bdm.exe'): cpp_path = get_sources_path() warnings.warn( "This model uses an external simulator `bdm` implemented in C++ " "that needs to be compiled and copied to your working directory. " "We could not find it from your current working directory. Please" "copy the folder `{}` to your working directory " "and compile the source.".format(cpp_path), RuntimeWarning) return m
def test_basics(self, ma2, distribution_test): # A 1D case normal = elfi.Prior('normal', 5, model=elfi.ElfiModel()) normal_prior = ModelPrior(normal.model) distribution_test(normal_prior) # A 2D case prior = ModelPrior(ma2) distribution_test(prior)
def get_model(true_params=None, seed_obs=None, **kwargs): """Return a complete ELFI graph ready for inference. Selection of true values, priors etc. follows the approach in Numminen, E., Cheng, L., Gyllenberg, M. and Corander, J.: Estimating the transmission dynamics of Streptococcus pneumoniae from strain prevalence data, Biometrics, 69, 748-757, 2013. and Gutmann M U, Corander J (2016). Bayesian Optimization for Likelihood-Free Inference of Simulator-Based Statistical Models. JMLR 17(125):1−47, 2016. Parameters ---------- true_params : list, optional Parameters with which the observed data is generated. seed_obs : int, optional Seed for the observed data generation. Returns ------- m : elfi.ElfiModel """ logger = logging.getLogger() if true_params is None: true_params = [3.6, 0.6, 0.1] m = elfi.ElfiModel() y_obs = daycare(*true_params, random_state=np.random.RandomState(seed_obs), **kwargs) sim_fn = partial(daycare, **kwargs) priors = [] sumstats = [] priors.append(elfi.Prior('uniform', 0, 11, model=m, name='t1')) priors.append(elfi.Prior('uniform', 0, 2, model=m, name='t2')) priors.append(elfi.Prior('uniform', 0, 1, model=m, name='t3')) elfi.Simulator(sim_fn, *priors, observed=y_obs, name='DCC') sumstats.append(elfi.Summary(ss_shannon, m['DCC'], name='Shannon')) sumstats.append(elfi.Summary(ss_strains, m['DCC'], name='n_strains')) sumstats.append(elfi.Summary(ss_prevalence, m['DCC'], name='prevalence')) sumstats.append(elfi.Summary(ss_prevalence_multi, m['DCC'], name='multi')) elfi.Discrepancy(distance, *sumstats, name='d') logger.info( "Generated observations with true parameters " "t1: %.1f, t2: %.3f, t3: %.1f, ", *true_params) return m
def get_model(n_obs=50, true_params=None, stats_summary=None, seed_obs=None): """Return an initialised univariate g-and-k model. Parameters ---------- n_obs : int, optional The number of the observed points. true_params : array_like, optional The parameters defining the model. stats_summary : array_like, optional The chosen summary statistics, expressed as a list of strings. Options: ['ss_order'], ['ss_robust'], ['ss_octile']. seed_obs : np.random.RandomState, optional Returns ------- elfi.ElfiModel """ m = elfi.ElfiModel() # Initialising the default parameter settings as given in [2]. if true_params is None: true_params = [3, 1, 2, .5] if stats_summary is None: stats_summary = ['ss_order'] # Initialising the default prior settings as given in [2]. elfi.Prior('uniform', 0, 10, model=m, name='a') elfi.Prior('uniform', 0, 10, model=m, name='b') elfi.Prior('uniform', 0, 10, model=m, name='g') elfi.Prior('uniform', 0, 10, model=m, name='k') # Generating the observations. y_obs = GNK(*true_params, n_obs=n_obs, random_state=np.random.RandomState(seed_obs)) # Defining the simulator. fn_sim = partial(GNK, n_obs=n_obs) elfi.Simulator(fn_sim, m['a'], m['b'], m['g'], m['k'], observed=y_obs, name='GNK') # Initialising the chosen summary statistics. fns_summary_all = [ss_order, ss_robust, ss_octile] fns_summary_chosen = [] for fn_summary in fns_summary_all: if fn_summary.__name__ in stats_summary: summary = elfi.Summary(fn_summary, m['GNK'], name=fn_summary.__name__) fns_summary_chosen.append(summary) elfi.Discrepancy(euclidean_multidim, *fns_summary_chosen, name='d') return m
def multivariate_model(request): ndim = request.param def fun(x, batch_size, random_state): return np.sum(x, keepdims=True, axis=1) m = elfi.ElfiModel() elfi.Prior(ss.multivariate_normal, [0] * ndim, model=m, name='t1') elfi.Simulator(fun, m['t1'], observed=np.array([[0]]), model=m, name='sim') elfi.Distance('euclidean', m['sim'], model=m, name='d') return m
def multivariate_model(request): ndim = request.param m = elfi.ElfiModel() elfi.Prior(ss.multivariate_normal, [0] * ndim, model=m, name='t1') elfi.Simulator(rowsummer, m['t1'], observed=np.array([[0]]), model=m, name='sim') elfi.Distance('euclidean', m['sim'], model=m, name='d') return m
def sleep_model(request): """The true param will be half of the given sleep time.""" ub_sec = request.param or .5 m = elfi.ElfiModel() elfi.Constant(ub_sec, model=m, name='ub') elfi.Prior('uniform', 0, m['ub'], model=m, name='sec') elfi.Simulator(sleeper, m['sec'], model=m, name='slept') elfi.Summary(no_op, m['slept'], model=m, name='summary') elfi.Distance('euclidean', m['summary'], model=m, name='d') m.observed['slept'] = ub_sec / 2 return m
def get_model(p): y_obs = np.zeros(p) y_obs[0] = 10 y_obs = y_obs[None, :] sim = Simulator(p=p) m = elfi.ElfiModel() mu = elfi.Prior(TwistedNormal(p=p), model=m, name='mu') simulator = elfi.Simulator(sim, mu, observed=y_obs, name='Gauss') summary = elfi.Summary(identity, simulator, name='summary') return m
def get_model(n_obs=50, true_params=None, seed_obs=None, **kwargs): """Return a complete Lotka-Volterra model in inference task. Parameters ---------- n_obs : int, optional Number of observations. true_params : list, optional Parameters with which the observed data is generated. seed_obs : int, optional Seed for the observed data generation. Returns ------- m : elfi.ElfiModel """ logger = logging.getLogger() if true_params is None: true_params = [1.0, 0.005, 0.6, 50, 100, 10.] kwargs['n_obs'] = n_obs y_obs = lotka_volterra(*true_params, random_state=np.random.RandomState(seed_obs), **kwargs) m = elfi.ElfiModel() sim_fn = partial(lotka_volterra, **kwargs) priors = [] sumstats = [] priors.append(elfi.Prior(ExpUniform, -6., 2., model=m, name='r1')) priors.append(elfi.Prior(ExpUniform, -6., 2., model=m, name='r2')) # easily kills populations priors.append(elfi.Prior(ExpUniform, -6., 2., model=m, name='r3')) priors.append(elfi.Prior('poisson', 50, model=m, name='prey0')) priors.append(elfi.Prior('poisson', 100, model=m, name='predator0')) priors.append( elfi.Prior(ExpUniform, np.log(0.5), np.log(50), model=m, name='sigma')) elfi.Simulator(sim_fn, *priors, observed=y_obs, name='LV') sumstats.append( elfi.Summary(partial(pick_stock, species=0), m['LV'], name='prey')) sumstats.append( elfi.Summary(partial(pick_stock, species=1), m['LV'], name='predator')) elfi.Distance('sqeuclidean', *sumstats, name='d') logger.info( "Generated %i observations with true parameters r1: %.1f, r2: %.3f, r3: %.1f, " "prey0: %i, predator0: %i, sigma: %.1f.", n_obs, *true_params) return m
def simple_gaussian_model(true_param, seed, n_summaries=10): """The simple gaussian model that has been used as a toy example in the LFIRE paper.""" def power(x, y): return x**y m = elfi.ElfiModel() mu = elfi.Prior('uniform', -5, 10, model=m, name='mu') y = elfi.Simulator(gauss, *[mu], observed=gauss(true_param, seed=seed), name='y') for i in range(n_summaries): elfi.Summary(power, y, i, model=m, name=f'power_{i}') return m
def test_batch_index_value(ma2): bi = lambda meta: meta['batch_index'] # Test the correct batch_index value m = elfi.ElfiModel() op = elfi.Operation(bi, model=m, name='op') op['_uses_meta'] = True client = elfi.get_client() c = elfi.ComputationContext() compiled_net = client.compile(m.source_net, m.nodes) loaded_net = client.load_data(compiled_net, c, batch_index=3) res = client.compute(loaded_net) assert res['op'] == 3
def get_model(self, n_obs=100, true_params=None, seed_obs=None): """Return a complete model in inference task. Parameters ---------- n_obs : int, optional observation length of the MA2 process true_params : list, optional parameters with which the observed data is generated seed_obs : int, optional seed for the observed data generation Returns ------- m : elfi.ElfiModel """ m = elfi.ElfiModel() if true_params is None and self.env_name == "CartPole-v0": # gravity, masscart, masspole, length, force_mag true_params = [[9.8], [1.0], [0.1], [0.5], [10.0]] elfi.Prior('uniform', 5.0, 10.0, model=m, name='gravity') elfi.Prior('uniform', 0.1, 5, model=m, name='masscart') elfi.Prior('uniform', 0.1, 5, model=m, name='masspole') elfi.Prior('uniform', 0.1, 5, model=m, name='length') elfi.Prior('uniform', 5.0, 10.0, model=m, name='force_mag') params = [ m['gravity'], m['masscart'], m['masspole'], m['length'], m['force_mag'] ] elif true_params is None and self.env_name == "MountainCar-v0": # goal_position, goal_velocity, force, gravity true_params = [0.5, 0, 0.001, 0.0025] elfi.Prior('uniform', -1.2, 0.6, model=m, name='goal_position') elfi.Prior('uniform', 0, 5.0, model=m, name='goal_velocity') elfi.Prior('uniform', 0.0001, 0.01, model=m, name='force') elfi.Prior('uniform', 0.0001, 0.01, model=m, name='gravity') params = [ m['goal_position'], m['goal_velocity'], m['force'], m['gravity'] ] y_obs = self.func(*true_params, random_state=np.random.RandomState(seed_obs)) elfi.Simulator(self.func, *params, observed=y_obs, name='DGP') elfi.Distance('euclidean', m['DGP'], name='d') return m
def test_dict_output(): vsim = elfi.tools.vectorize(simulator) vsum = elfi.tools.vectorize(summary) obs = simulator([.2, .8]) elfi.ElfiModel() p = elfi.Prior('dirichlet', [2, 2]) sim = elfi.Simulator(vsim, p, observed=obs) S = elfi.Summary(vsum, sim) d = elfi.Distance('euclidean', S) pool = elfi.OutputPool(['sim']) rej = elfi.Rejection(d, batch_size=100, pool=pool, output_names=['sim']) sample = rej.sample(100, n_sim=1000) mean = np.mean(sample.samples['p'], axis=0) # Crude test assert mean[1] - mean[0] > .2
def get_model(true_params=None, seed_obs=None, initial_state=None, n_obs=40, f=10., phi=0.984, total_duration=4): """Return a complete Lorenz model in inference task. This is a simplified example that achieves reasonable predictions. Hakkarainen, J., Ilin, A., Solonen, A., Laine, M., Haario, H., Tamminen, J., Oja, E., and Järvinen, H. (2012). On closure parameter estimation in chaotic systems. Nonlinear Processes in Geophysics, 19(1), 127–143. Parameters ---------- true_params : list, optional Parameters with which the observed data is generated. seed_obs : int, optional Seed for the observed data generation. initial_state : ndarray Initial state value of the time-series. n_obs : int, optional Number of observed variables f : float, optional Force term phi : float, optional This value is used to express stochastic forcing term. It should be configured according to force term and eventually impacts to the result of eta. More details in Wilks (2005) et al. total_duration : float, optional Returns ------- m : elfi.ElfiModel """ simulator = partial(forecast_lorenz, initial_state=initial_state, f=f, n_obs=n_obs, phi=phi, total_duration=total_duration) if not true_params: true_params = [2.0, 0.1] m = elfi.ElfiModel() y_obs = simulator(*true_params, random_state=np.random.RandomState(seed_obs)) sumstats = [] elfi.Prior('uniform', 0.5, 3., model=m, name='theta1') elfi.Prior('uniform', 0, 0.3, model=m, name='theta2') elfi.Simulator(simulator, m['theta1'], m['theta2'], observed=y_obs, name='Lorenz') sumstats.append(elfi.Summary(mean, m['Lorenz'], name='Mean')) sumstats.append(elfi.Summary(var, m['Lorenz'], name='Var')) sumstats.append(elfi.Summary(autocov, m['Lorenz'], name='Autocov')) sumstats.append(elfi.Summary(cov, m['Lorenz'], name='Cov')) sumstats.append(elfi.Summary(xcov, m['Lorenz'], True, name='CrosscovPrev')) sumstats.append(elfi.Summary(xcov, m['Lorenz'], False, name='CrosscovNext')) elfi.Distance('euclidean', *sumstats, name='d') return m
def run( task: Task, num_samples: int, num_simulations: int, num_observation: Optional[int] = None, observation: Optional[torch.Tensor] = None, num_chains: int = 10, num_warmup: int = 1000, ) -> (torch.Tensor, int, Optional[torch.Tensor]): """Runs BOLFI from elfi package Args: task: Task instance num_samples: Number of samples to generate from posterior num_simulations: Simulation budget num_observation: Observation number to load, alternative to `observation` observation: Observation, alternative to `num_observation` num_chains: Number of chains num_warmup: Warmup steps Returns: Samples from posterior, number of simulator calls, log probability of true params if computable """ assert not (num_observation is None and observation is None) assert not (num_observation is not None and observation is not None) logging.basicConfig(level=logging.INFO) log = logging.getLogger(__name__) log.warn("ELFI is not fully supported yet!") # Initialize model object m = elfi.ElfiModel() # Prior bounds = build_prior(task=task, model=m) # Observation if observation is None: observation = task.get_observation(num_observation) observation = observation.numpy() # Simulator simulator = task.get_simulator(max_calls=num_simulations) elfi.Simulator( Simulator(simulator), *[m[f"parameter_{dim}"] for dim in range(task.dim_parameters)], observed=observation, name=task.name, ) # Euclidean distance elfi.Distance("euclidean", m[task.name], name="distance") # Log distance elfi.Operation(np.log, m["distance"], name="log_distance") # Inference num_samples_per_chain = ceil(num_samples / num_chains) tic = time.time() bolfi = elfi.BOLFI(model=m, target_name="log_distance", bounds=bounds) bolfi.fit(n_evidence=num_simulations) result_BOLFI = bolfi.sample( num_samples_per_chain + num_warmup, warmup=num_warmup, n_chains=num_chains, info_freq=int(100), ) toc = time.time() samples = torch.from_numpy(result_BOLFI.samples_array.astype( np.float32)).reshape(-1, task.dim_parameters)[:num_samples, :] assert samples.shape[0] == num_samples # TODO: return log prob of true parameters return samples, simulator.num_simulations, None
def simple_model(): m = elfi.ElfiModel() elfi.Constant(10, model=m, name='tau') elfi.Prior('uniform', 0, m['tau'], size=1, model=m, name='k1') elfi.Prior('normal', m['k1'], size=3, model=m, name='k2') return m
# OR load global marginal data #save_gm_name='GM_syn_DI5_15_320.npz' #save_gm_name='GM_syn_DI15_25_320.npz' #saved_marginal = np.load(save_gm_name) #global_marginal = saved_marginal['global_marginal'] # Synthetic data: # Use pre-generated data with DI = 10 or 20, R0=2, sigma = 1 (rng=1) # y_obs = np.array([[5.8847,1.8307,26.0000,3.8889,9.0000,9.0000,29.0527,4.4326,1.8000,1.7000,-0.1976]]) # DI = 10yr y_obs = np.array([[10.2160571656385, 2.54025816073426, 22, 6, 12,14, 33.4501948260742, 2.63235705270612, 1.83333333333333, 1.76666666666667, -0.213817505572094]]) # DI = 20 yr # OR regenerate synthetic data: # random_state_obs = np.random.RandomState(0) # y_obs = GASmodel2_local(p_1_true, p_2_true, p_3_true, random_state=random_state_obs) # Build an elfi model m = elfi.ElfiModel() # Priors p_1 = elfi.Prior('uniform', 0, 5, model=m) #p_2 = elfi.Prior('uniform', 5, 10, model=m) p_2 = elfi.Prior('uniform', 15, 10, model=m) p_3 = elfi.Prior('uniform', 0.5, 1, model=m) priors = [p_1, p_2, p_3] # Simulator Y = elfi.Simulator(GASmodel2, *priors, observed=y_obs) # Summary statistics sumstats = []
current_time = get_time() logger.debug('Started at {}'.format(current_time)) args = process_inputs() prefix = args['p'] with open('cog_categories_dict.pkl', 'rb') as a: cog_catergories_dict = pickle.load(a) elfi.set_client('multiprocessing') shell_command = create_shell_command(args) #builds shell command m = elfi.ElfiModel(name='nfds') #creates model nfds_simulator = create_model(shell_command, m, cog_catergories_dict) #creates model # Set an arbitrary global seed to keep the randomly generated quantities the same seed = 1 np.random.seed(seed) ###################################### # rejection initial_evidence = 5 rej = elfi.Rejection(nfds_simulator, batch_size=1) n_rejection_samples = 2 * initial_evidence rej_result = rej.sample(n_samples=n_rejection_samples, quantile=1) update_interval = 1
def main(base, output, number_processes): # generate null (should only execute once) use = 'BOLFI' #use = 'SINGLE' if use != 'SINGLE': click.secho('Generating null dataset', fg='yellow') subprocess.check_call( ["snakemake", "null", "--profile", "elfi_profile"]) click.secho('Creating environments', fg='yellow') subprocess.check_call( ["snakemake", "--profile", "elfi_profile", "--create-envs-only"]) click.secho('Starting BOLFI', fg='yellow') if base: global base_output base_output = base if output: global summary_output summary_output = output # setup priors elfi.set_client(multiprocessing.Client(num_processes=number_processes)) model = elfi.ElfiModel(name='msprime') elfi.Prior('uniform', 0, 0.5, model=model, name='n1') elfi.Prior('uniform', 0, 0.5, model=model, name='n2') elfi.Prior('uniform', 0, 1, model=model, name='split_prop') elfi.Prior('uniform', 1, 1861, model=model, name='split_size') snake_sim = elfi.tools.external_operation(command, prepare_inputs=prepare_inputs, process_result=process_result, stdout=False) vec_snake = elfi.tools.vectorize(snake_sim) empirical = np.array([ 0, # desert MSE 0.291761, 0.328705, 0.335145, # pi 0.12985, 0.156539, 0.0921547, # fst 0.023, 0.019 ]) # admix snake_node = elfi.Simulator(vec_snake, model['n1'], model['n2'], model['split_prop'], model['split_size'], observed=empirical, name='snake_sim') snake_node.uses_meta = True distance = elfi.Distance('euclidean', snake_node, name='dist') if use == 'SINGLE': print( snake_sim(0.1, 0.1, 0.1, 100, seed=2, meta={ 'model_name': 'test', 'batch_index': 1, 'submission_index': 2 })) return elif use == 'BOLFI': pool = elfi.ArrayPool( ['n1', 'n2', 'split_prop', 'split_size', 'snake_sim'], name='bolfi_pool') bolfi = elfi.BOLFI(distance, batch_size=1, initial_evidence=20, update_interval=3, bounds={ 'n1': (0, 0.5), 'n2': (0, 0.5), 'split_prop': (0, 1), 'split_size': (1, 1861) }, pool=pool) post = bolfi.fit(n_evidence=10, bar=False) click.secho('Saving results', fg='yellow') pool.save() post.save() click.secho('Done', fg='green') return
c = elfi.client.get_client() num_cores = c.num_cores if time.time() > timeout: raise TimeoutError("Could not find any cores after 600 seconds") time.sleep(10) logging.info(f"Ipyparallel client started with {num_cores} cores.") with open("../output/priors.pkl", "rb") as f: # Priors specified in priors.py priors = pickle.load(f) with open("../data/e3_phased.pkl", "rb") as f: y_obs = np.atleast_2d(pickle.load(f)) # Set up elfi model m = elfi.ElfiModel("m", observed={"sim": y_obs}) elfi.Constant(seq_length, name="length", model=m) elfi.Constant(1.8e-8, name="recombination_rate", model=m) elfi.Constant(6e-8, name="mutation_rate", model=m) for prior_name, prior in priors.items(): elfi.Prior(prior.sampling, name=prior_name, model=m) prior_args = [m[name] for name in m.parameter_names] elfi.Simulator(elfi_sim, *prior_args, m["length"], m["recombination_rate"], m["mutation_rate"],