def test_multiprocessing_kwargs(simple_model):
m = simple_model
num_proc = 2
elfi.set_client('multiprocessing', num_processes=num_proc)
rej = elfi.Rejection(m['k1'])
assert rej.client.num_cores == num_proc
elfi.set_client('multiprocessing', processes=num_proc)
rej = elfi.Rejection(m['k1'])
assert rej.client.num_cores == num_proc
logging.basicConfig()
logging.getLogger().setLevel(logging.INFO)
seq_length = 44648284 # E3 length is 44648284
train_scaler_n_sim = 32
smc_n_samples = 100
smc_schedule = [17, 16]
logging.info(f"Seq length is set to {seq_length}")
num_cores = 0
timeout = time.time() + 600 # seconds
while num_cores == 0:
elfi.set_client("ipyparallel",
profile="pbs") # Assumes ipython profile named pbs
c = elfi.client.get_client()
num_cores = c.num_cores
if time.time() > timeout:
raise TimeoutError("Could not find any cores after 600 seconds")
time.sleep(10)
logging.info(f"Ipyparallel client started with {num_cores} cores.")
with open("../output/priors.pkl", "rb") as f: # Priors specified in priors.py
priors = pickle.load(f)
with open("../data/e3_phased.pkl", "rb") as f:
y_obs = np.atleast_2d(pickle.load(f))
# Set up elfi model
def main(base, output, number_processes):
# generate null (should only execute once)
use = 'BOLFI'
#use = 'SINGLE'
if use != 'SINGLE':
click.secho('Generating null dataset', fg='yellow')
subprocess.check_call(
["snakemake", "null", "--profile", "elfi_profile"])
click.secho('Creating environments', fg='yellow')
subprocess.check_call(
["snakemake", "--profile", "elfi_profile", "--create-envs-only"])
click.secho('Starting BOLFI', fg='yellow')
if base:
global base_output
base_output = base
if output:
global summary_output
summary_output = output
# setup priors
elfi.set_client(multiprocessing.Client(num_processes=number_processes))
model = elfi.ElfiModel(name='msprime')
elfi.Prior('uniform', 0, 0.5, model=model, name='n1')
elfi.Prior('uniform', 0, 0.5, model=model, name='n2')
elfi.Prior('uniform', 0, 1, model=model, name='split_prop')
elfi.Prior('uniform', 1, 1861, model=model, name='split_size')
snake_sim = elfi.tools.external_operation(command,
prepare_inputs=prepare_inputs,
process_result=process_result,
stdout=False)
vec_snake = elfi.tools.vectorize(snake_sim)
empirical = np.array([
0, # desert MSE
0.291761,
0.328705,
0.335145, # pi
0.12985,
0.156539,
0.0921547, # fst
0.023,
0.019
]) # admix
snake_node = elfi.Simulator(vec_snake,
model['n1'],
model['n2'],
model['split_prop'],
model['split_size'],
observed=empirical,
name='snake_sim')
snake_node.uses_meta = True
distance = elfi.Distance('euclidean', snake_node, name='dist')
if use == 'SINGLE':
print(
snake_sim(0.1,
0.1,
0.1,
100,
seed=2,
meta={
'model_name': 'test',
'batch_index': 1,
'submission_index': 2
}))
return
elif use == 'BOLFI':
pool = elfi.ArrayPool(
['n1', 'n2', 'split_prop', 'split_size', 'snake_sim'],
name='bolfi_pool')
bolfi = elfi.BOLFI(distance,
batch_size=1,
initial_evidence=20,
update_interval=3,
bounds={
'n1': (0, 0.5),
'n2': (0, 0.5),
'split_prop': (0, 1),
'split_size': (1, 1861)
},
pool=pool)
post = bolfi.fit(n_evidence=10, bar=False)
click.secho('Saving results', fg='yellow')
pool.save()
post.save()
click.secho('Done', fg='green')
return
if __name__ == '__main__':
fitting_steps = 200
samples = 100000
current_time = get_time()
logger.debug('Started at {}'.format(current_time))
args = process_inputs()
prefix = args['p']
with open('cog_categories_dict.pkl', 'rb') as a:
cog_catergories_dict = pickle.load(a)
elfi.set_client('multiprocessing')
shell_command = create_shell_command(args) #builds shell command
m = elfi.ElfiModel(name='nfds') #creates model
nfds_simulator = create_model(shell_command, m,
cog_catergories_dict) #creates model
# Set an arbitrary global seed to keep the randomly generated quantities the same
seed = 1
np.random.seed(seed)
######################################
# rejection
initial_evidence = 5
rej = elfi.Rejection(nfds_simulator, batch_size=1)
# This file is drawing samples from the approximated posterior distribution by utilising BOLFI.
# Written by: Anttoni Ojanperä, Nitin Williams
# Based on code written by: Antti Karvanen
num_processes = 2
seed = 23112021
np.random.seed(seed)
# Loading observed summary statistics
mat_model = loadmat('WCnet_noisy_300s_MEG_simulatednetwork_sub1.mat')
observed_model = mat_model['wPLImags_vals']
# Set up multiprocessing
elfi.set_client(elfi.clients.multiprocessing.Client(num_processes=num_processes))
# Specifying prior distributions
wee = elfi.Prior('norm',20,5)
wei = elfi.Prior('norm',18,6)
wie = elfi.Prior('norm',18,6)
wii = elfi.Prior('norm',1,0.2)
be = elfi.Prior('norm',3,1)
bi = elfi.Prior('norm',5,1)
taue = elfi.Prior('norm',18.6,3.8)
taui = elfi.Prior('norm',15.1,4.7)
k = elfi.Prior('norm',1.5,0.5)
IH = elfi.Prior('norm',2.5,0.5)
psi_sigma = elfi.Prior('norm',0.15,0.05)
import elfi
import os
from datetime import datetime
import csv
import shutil
from subprocess import *
import multiprocessing
currentDT = datetime.now()
currentDT = currentDT.strftime("/%Y-%m-%d %H:%M:%S")
shutil.rmtree("results")
os.mkdir("results")
shutil.rmtree("prop")
os.mkdir("prop")
elfi.set_client('multiprocessing')
iter = multiprocessing.Value('i', 1)
global start
start = 800
global end
end = 850
global threshold
threshold = 0
#def readData1():
# file = open("target_data.csv","r")
# data = file.read()
# print(data)
def readData():