def run_synth_test():
""" Run a test with synthetic data and MCMC inference
"""
options, popn, data, popn_true, x_true = initialize_test_harness()
results_file = os.path.join(options.resultsDir, 'results.pkl')
N_samples = 100
if os.path.exists(results_file):
print "Results found. Loading from file."
with open(results_file) as f:
x_smpls = cPickle.load(f)
N_samples = len(x_smpls)
# TODO: Check that the results are from the same model?
else:
print "Results not found. Running MCMC inference."
# If x0 specified, load x0 from file
x0 = None
if options.x0_file is not None:
with open(options.x0_file, 'r') as f:
print "Initializing with state from: %s" % options.x0_file
mle_x0 = cPickle.load(f)
# HACK: We're assuming x0 came from a standard GLM
mle_model = make_model('standard_glm', N=data['N'])
mle_popn = Population(mle_model)
mle_popn.set_data(data)
x0 = popn.sample()
x0 = convert_model(mle_popn, mle_model, mle_x0, popn,
popn.model, x0)
# Prepare for online plotting
plt.ion()
plotters = initialize_plotting(popn_true, x_true, popn)
plt.show()
cbk = lambda x: plot_sample_callback(x, plotters)
# Perform inference
raw_input('Press any key to begin inference...\n')
x_smpls = gibbs_sample(popn,
data,
x0=x0,
N_samples=N_samples,
init_from_mle=False,
callback=cbk)
# Save results
print "Saving results to %s" % results_file
with open(results_file, 'w') as f:
cPickle.dump(x_smpls, f, protocol=-1)
# Plot average of last 20% of samples
smpl_frac = 0.2
plot_results(popn,
x_smpls[-1 * int(smpl_frac * N_samples):],
popn_true=popn_true,
x_true=x_true,
resdir=options.resultsDir)
def run_synth_test():
""" Run a test with synthetic data and MCMC inference
"""
options, popn, data, popn_true, x_true = initialize_test_harness()
results_file = os.path.join(options.resultsDir, 'results.pkl')
N_samples = 100
if os.path.exists(results_file):
print "Results found. Loading from file."
with open(results_file) as f:
x_smpls = cPickle.load(f)
N_samples = len(x_smpls)
# TODO: Check that the results are from the same model?
else:
print "Results not found. Running MCMC inference."
# If x0 specified, load x0 from file
x0 = None
if options.x0_file is not None:
with open(options.x0_file, 'r') as f:
print "Initializing with state from: %s" % options.x0_file
mle_x0 = cPickle.load(f)
# HACK: We're assuming x0 came from a standard GLM
mle_model = make_model('standard_glm', N=data['N'])
mle_popn = Population(mle_model)
mle_popn.set_data(data)
x0 = popn.sample()
x0 = convert_model(mle_popn, mle_model, mle_x0, popn, popn.model, x0)
# Prepare for online plotting
plt.ion()
plotters = initialize_plotting(popn_true, x_true, popn)
plt.show()
cbk = lambda x: plot_sample_callback(x, plotters)
# Perform inference
raw_input('Press any key to begin inference...\n')
x_smpls = gibbs_sample(popn, data, x0=x0, N_samples=N_samples,
init_from_mle=False,
callback=cbk)
# Save results
print "Saving results to %s" % results_file
with open(results_file, 'w') as f:
cPickle.dump(x_smpls, f, protocol=-1)
# Plot average of last 20% of samples
smpl_frac = 0.2
plot_results(popn,
x_smpls[-1*int(smpl_frac*N_samples):],
popn_true=popn_true,
x_true=x_true,
resdir=options.resultsDir)
def run_synth_test():
""" Run a test with synthetic data and MAP inference via
parallel coordinate descent.
"""
options, popn, data, client, popn_true, x_true = initialize_parallel_test_harness()
print "Performing parallel inference"
x_inf = parallel_coord_descent(client, data['N'], maxiter=1)
ll_inf = popn.compute_log_p(x_inf)
print "LL_inf: %f" % ll_inf
# Save results
with open(os.path.join(options.resultsDir, 'results.pkl'),'w') as f:
cPickle.dump(x_inf,f, protocol=-1)
# Plot results
plot_results(popn, x_inf,
popn_true, x_true,
do_plot_imp_responses=False,
resdir=options.resultsDir)
def run_synth_test():
""" Run a test with synthetic data and MAP inference via
parallel coordinate descent.
"""
options, popn, data, client, popn_true, x_true = initialize_parallel_test_harness(
)
print "Performing parallel inference"
x_inf = parallel_coord_descent(client, data['N'], maxiter=1)
ll_inf = popn.compute_log_p(x_inf)
print "LL_inf: %f" % ll_inf
# Save results
with open(os.path.join(options.resultsDir, 'results.pkl'), 'w') as f:
cPickle.dump(x_inf, f, protocol=-1)
# Plot results
plot_results(popn,
x_inf,
popn_true,
x_true,
do_plot_imp_responses=False,
resdir=options.resultsDir)
def run_synth_test():
""" Run a test with synthetic data and MCMC inference
"""
options, popn, data, popn_true, x_true = initialize_test_harness()
# Sample random initial state
x0 = popn.sample()
ll0 = popn.compute_log_p(x0)
print "LL0: %f" % ll0
# Perform inference
x_inf = coord_descent(popn, x0=x0, maxiter=1)
ll_inf = popn.compute_log_p(x_inf)
print "LL_inf: %f" % ll_inf
# Save results
results_file = os.path.join(options.resultsDir, 'results.pkl')
print "Saving results to %s" % results_file
with open(results_file, 'w') as f:
cPickle.dump(x_inf, f, protocol=-1)
# Plot results
plot_results(popn, x_inf, popn_true, x_true, resdir=options.resultsDir)
def run_synth_test():
""" Run a test with synthetic data and MCMC inference
"""
options, popn, data, popn_true, x_true = initialize_test_harness()
# Sample random initial state
x0 = popn.sample()
ll0 = popn.compute_log_p(x0)
print "LL0: %f" % ll0
# Perform inference
x_inf = coord_descent(popn, x0=x0, maxiter=1)
ll_inf = popn.compute_log_p(x_inf)
print "LL_inf: %f" % ll_inf
# Save results
results_file = os.path.join(options.resultsDir, 'results.pkl')
print "Saving results to %s" % results_file
with open(results_file, 'w') as f:
cPickle.dump(x_inf, f, protocol=-1)
# Plot results
plot_results(popn, x_inf, popn_true, x_true, resdir=options.resultsDir)
def run_synth_test():
""" Run a test with synthetic data and MAP inference with cross validation
"""
options, popn, data, popn_true, x_true = initialize_test_harness()
# Get the list of models for cross validation
base_model = make_model(options.model, N=data['N'], dt=0.001)
models = get_xv_models(base_model)
# TODO Segment data into training and cross validation sets
train_frac = 0.75
T_split = data['T'] * train_frac
train_data = segment_data(data, (0, T_split))
xv_data = segment_data(data, (T_split, data['T']))
# Preprocess the data sequences
train_data = popn.preprocess_data(train_data)
xv_data = popn.preprocess_data(xv_data)
# Sample random initial state
x0 = popn.sample()
# Track the best model and parameters
best_ind = -1
best_xv_ll = -np.Inf
best_x = x0
best_model = None
# Fit each model using the optimum of the previous models
train_lls = np.zeros(len(models))
xv_lls = np.zeros(len(models))
total_lls = np.zeros(len(models))
for (i, model) in enumerate(models):
print "Training model %d" % i
x0 = copy.deepcopy(best_x)
popn.set_hyperparameters(model)
popn.set_data(train_data)
ll0 = popn.compute_log_p(x0)
print "Training LL0: %f" % ll0
# Perform inference
x_inf = coord_descent(popn, x0=x0, maxiter=1)
ll_train = popn.compute_log_p(x_inf)
print "Training LP_inf: %f" % ll_train
train_lls[i] = ll_train
# Compute log lkhd on xv data
popn.set_data(xv_data)
ll_xv = popn.compute_ll(x_inf)
print "Cross Validation LL: %f" % ll_xv
xv_lls[i] = ll_xv
# Compute log lkhd on total dataset
popn.set_data(data)
ll_total = popn.compute_ll(x_inf)
print "Total LL: %f" % ll_total
total_lls[i] = ll_total
# Update best model
if ll_xv > best_xv_ll:
best_ind = i
best_xv_ll = ll_xv
best_x = copy.deepcopy(x_inf)
best_model = copy.deepcopy(model)
# Create a population with the best model
popn.set_hyperparameters(best_model)
popn.set_data(data)
# Fit the best model on the full training data
best_x = coord_descent(popn,
data,
x0=x0,
maxiter=1,
use_hessian=False,
use_rop=False)
# Print results summary
for i in np.arange(len(models)):
print "Model %d:\tTrain LL: %.1f\tXV LL: %.1f\tTotal LL: %.1f" % (
i, train_lls[i], xv_lls[i], total_lls[i])
print "Best model: %d" % best_ind
print "Best Total LL: %f" % popn.compute_ll(best_x)
print "True LL: %f" % popn_true.compute_ll(x_true)
# Save results
results_file = os.path.join(options.resultsDir, 'results.pkl')
print "Saving results to %s" % results_file
with open(results_file, 'w') as f:
cPickle.dump(best_x, f)
# Plot results
plot_results(popn, best_x, popn_true, x_true, resdir=options.resultsDir)
def run_gen_synth_data():
""" Run a test with synthetic data and MCMC inference
"""
options, args = parse_cmd_line_args()
# Create the model
dt = 0.001
model = make_model(options.model, N=options.N, dt=dt)
# Set the sparsity level to minimize the risk of unstable networks
stabilize_sparsity(model)
print "Creating master population object"
popn = Population(model)
# Sample random parameters from the model
x_true = popn.sample()
# Check stability of matrix
assert check_stability(model, x_true,
options.N), "ERROR: Sampled network is unstable!"
# Save the model so it can be loaded alongside the data
fname_model = os.path.join(options.resultsDir, 'model.pkl')
print "Saving data to %s" % fname_model
with open(fname_model, 'w') as f:
cPickle.dump(model, f, protocol=-1)
print "Generating synthetic data with %d neurons and %.2f seconds." % \
(options.N, options.T_stop)
# Set simulation parametrs
dt_stim = 0.1
D_stim = (5, 5)
# D_stim = model['bkgd']['D_stim'] if 'D_stim' in model['bkgd'] else 0
if isinstance(D_stim, int):
D_stim = [D_stim]
stim = np.random.randn(options.T_stop / dt_stim, *D_stim)
data = gen_synth_data(options.N, options.T_stop, popn, x_true, dt, dt_stim,
D_stim, stim)
# Set the data so that the population state can be evaluated
popn.add_data(data)
# DEBUG Evaluate the firing rate and the simulated firing rate
state = popn.eval_state(x_true)
for n in np.arange(options.N):
lam_true = state['glms'][n]['lam']
lam_sim = popn.glm.nlin_model.f_nlin(data['X'][:, n])
assert np.allclose(lam_true, lam_sim)
# Pickle the data so we can open it more easily
fname_pkl = os.path.join(options.resultsDir, 'data.pkl')
print "Saving data to %s" % fname_pkl
with open(fname_pkl, 'w') as f:
cPickle.dump(data, f, protocol=-1)
# Plot firing rates, stimulus responses, etc
do_plot_imp_resonses = int(options.N) <= 16
plot_results(popn,
data['vars'],
resdir=options.resultsDir,
do_plot_stim_resp=True,
do_plot_imp_responses=do_plot_imp_resonses)
def run_gen_synth_data():
""" Run a test with synthetic data and MCMC inference
"""
options, args = parse_cmd_line_args()
# Create the model
dt = 0.001
model = make_model(options.model, N=options.N, dt=dt)
# Set the sparsity level to minimize the risk of unstable networks
stabilize_sparsity(model)
print "Creating master population object"
popn = Population(model)
# Sample random parameters from the model
x_true = popn.sample()
# Check stability of matrix
assert check_stability(model, x_true, options.N), "ERROR: Sampled network is unstable!"
# Save the model so it can be loaded alongside the data
fname_model = os.path.join(options.resultsDir, 'model.pkl')
print "Saving data to %s" % fname_model
with open(fname_model,'w') as f:
cPickle.dump(model, f, protocol=-1)
print "Generating synthetic data with %d neurons and %.2f seconds." % \
(options.N, options.T_stop)
# Set simulation parametrs
dt_stim = 0.1
D_stim = (5,5)
# D_stim = model['bkgd']['D_stim'] if 'D_stim' in model['bkgd'] else 0
if isinstance(D_stim, int):
D_stim = [D_stim]
stim = np.random.randn(options.T_stop/dt_stim, *D_stim)
data = gen_synth_data(options.N, options.T_stop, popn, x_true, dt, dt_stim, D_stim, stim)
# Set the data so that the population state can be evaluated
popn.add_data(data)
# DEBUG Evaluate the firing rate and the simulated firing rate
state = popn.eval_state(x_true)
for n in np.arange(options.N):
lam_true = state['glms'][n]['lam']
lam_sim = popn.glm.nlin_model.f_nlin(data['X'][:,n])
assert np.allclose(lam_true, lam_sim)
# Pickle the data so we can open it more easily
fname_pkl = os.path.join(options.resultsDir, 'data.pkl')
print "Saving data to %s" % fname_pkl
with open(fname_pkl,'w') as f:
cPickle.dump(data, f, protocol=-1)
# Plot firing rates, stimulus responses, etc
do_plot_imp_resonses = int(options.N) <= 16
plot_results(popn, data['vars'],
resdir=options.resultsDir,
do_plot_stim_resp=True,
do_plot_imp_responses=do_plot_imp_resonses)
def run_synth_test():
""" Run a test with synthetic data and MAP inference with cross validation
"""
options, popn, data, popn_true, x_true = initialize_test_harness()
# Get the list of models for cross validation
base_model = make_model(options.model, N=data['N'], dt=0.001)
models = get_xv_models(base_model)
# TODO Segment data into training and cross validation sets
train_frac = 0.75
T_split = data['T'] * train_frac
train_data = segment_data(data, (0,T_split))
xv_data = segment_data(data, (T_split,data['T']))
# Preprocess the data sequences
train_data = popn.preprocess_data(train_data)
xv_data = popn.preprocess_data(xv_data)
# Sample random initial state
x0 = popn.sample()
# Track the best model and parameters
best_ind = -1
best_xv_ll = -np.Inf
best_x = x0
best_model = None
# Fit each model using the optimum of the previous models
train_lls = np.zeros(len(models))
xv_lls = np.zeros(len(models))
total_lls = np.zeros(len(models))
for (i,model) in enumerate(models):
print "Training model %d" % i
x0 = copy.deepcopy(best_x)
popn.set_hyperparameters(model)
popn.set_data(train_data)
ll0 = popn.compute_log_p(x0)
print "Training LL0: %f" % ll0
# Perform inference
x_inf = coord_descent(popn, x0=x0, maxiter=1)
ll_train = popn.compute_log_p(x_inf)
print "Training LP_inf: %f" % ll_train
train_lls[i] = ll_train
# Compute log lkhd on xv data
popn.set_data(xv_data)
ll_xv = popn.compute_ll(x_inf)
print "Cross Validation LL: %f" % ll_xv
xv_lls[i] = ll_xv
# Compute log lkhd on total dataset
popn.set_data(data)
ll_total = popn.compute_ll(x_inf)
print "Total LL: %f" % ll_total
total_lls[i] = ll_total
# Update best model
if ll_xv > best_xv_ll:
best_ind = i
best_xv_ll = ll_xv
best_x = copy.deepcopy(x_inf)
best_model = copy.deepcopy(model)
# Create a population with the best model
popn.set_hyperparameters(best_model)
popn.set_data(data)
# Fit the best model on the full training data
best_x = coord_descent(popn, data, x0=x0, maxiter=1,
use_hessian=False,
use_rop=False)
# Print results summary
for i in np.arange(len(models)):
print "Model %d:\tTrain LL: %.1f\tXV LL: %.1f\tTotal LL: %.1f" % (i, train_lls[i], xv_lls[i], total_lls[i])
print "Best model: %d" % best_ind
print "Best Total LL: %f" % popn.compute_ll(best_x)
print "True LL: %f" % popn_true.compute_ll(x_true)
# Save results
results_file = os.path.join(options.resultsDir, 'results.pkl')
print "Saving results to %s" % results_file
with open(results_file, 'w') as f:
cPickle.dump(best_x, f)
# Plot results
plot_results(popn, best_x, popn_true, x_true, resdir=options.resultsDir)
