def run_synth_test(): """ Run a test with synthetic data and MCMC inference """ options, popn, data, popn_true, x_true = initialize_test_harness() results_file = os.path.join(options.resultsDir, 'results.pkl') N_samples = 100 if os.path.exists(results_file): print "Results found. Loading from file." with open(results_file) as f: x_smpls = cPickle.load(f) N_samples = len(x_smpls) # TODO: Check that the results are from the same model? else: print "Results not found. Running MCMC inference." # If x0 specified, load x0 from file x0 = None if options.x0_file is not None: with open(options.x0_file, 'r') as f: print "Initializing with state from: %s" % options.x0_file mle_x0 = cPickle.load(f) # HACK: We're assuming x0 came from a standard GLM mle_model = make_model('standard_glm', N=data['N']) mle_popn = Population(mle_model) mle_popn.set_data(data) x0 = popn.sample() x0 = convert_model(mle_popn, mle_model, mle_x0, popn, popn.model, x0) # Prepare for online plotting plt.ion() plotters = initialize_plotting(popn_true, x_true, popn) plt.show() cbk = lambda x: plot_sample_callback(x, plotters) # Perform inference raw_input('Press any key to begin inference...\n') x_smpls = gibbs_sample(popn, data, x0=x0, N_samples=N_samples, init_from_mle=False, callback=cbk) # Save results print "Saving results to %s" % results_file with open(results_file, 'w') as f: cPickle.dump(x_smpls, f, protocol=-1) # Plot average of last 20% of samples smpl_frac = 0.2 plot_results(popn, x_smpls[-1 * int(smpl_frac * N_samples):], popn_true=popn_true, x_true=x_true, resdir=options.resultsDir)
def run_synth_test(): """ Run a test with synthetic data and MCMC inference """ options, popn, data, popn_true, x_true = initialize_test_harness() results_file = os.path.join(options.resultsDir, 'results.pkl') N_samples = 100 if os.path.exists(results_file): print "Results found. Loading from file." with open(results_file) as f: x_smpls = cPickle.load(f) N_samples = len(x_smpls) # TODO: Check that the results are from the same model? else: print "Results not found. Running MCMC inference." # If x0 specified, load x0 from file x0 = None if options.x0_file is not None: with open(options.x0_file, 'r') as f: print "Initializing with state from: %s" % options.x0_file mle_x0 = cPickle.load(f) # HACK: We're assuming x0 came from a standard GLM mle_model = make_model('standard_glm', N=data['N']) mle_popn = Population(mle_model) mle_popn.set_data(data) x0 = popn.sample() x0 = convert_model(mle_popn, mle_model, mle_x0, popn, popn.model, x0) # Prepare for online plotting plt.ion() plotters = initialize_plotting(popn_true, x_true, popn) plt.show() cbk = lambda x: plot_sample_callback(x, plotters) # Perform inference raw_input('Press any key to begin inference...\n') x_smpls = gibbs_sample(popn, data, x0=x0, N_samples=N_samples, init_from_mle=False, callback=cbk) # Save results print "Saving results to %s" % results_file with open(results_file, 'w') as f: cPickle.dump(x_smpls, f, protocol=-1) # Plot average of last 20% of samples smpl_frac = 0.2 plot_results(popn, x_smpls[-1*int(smpl_frac*N_samples):], popn_true=popn_true, x_true=x_true, resdir=options.resultsDir)
def run_synth_test(): """ Run a test with synthetic data and MAP inference via parallel coordinate descent. """ options, popn, data, client, popn_true, x_true = initialize_parallel_test_harness() print "Performing parallel inference" x_inf = parallel_coord_descent(client, data['N'], maxiter=1) ll_inf = popn.compute_log_p(x_inf) print "LL_inf: %f" % ll_inf # Save results with open(os.path.join(options.resultsDir, 'results.pkl'),'w') as f: cPickle.dump(x_inf,f, protocol=-1) # Plot results plot_results(popn, x_inf, popn_true, x_true, do_plot_imp_responses=False, resdir=options.resultsDir)
def run_synth_test(): """ Run a test with synthetic data and MAP inference via parallel coordinate descent. """ options, popn, data, client, popn_true, x_true = initialize_parallel_test_harness( ) print "Performing parallel inference" x_inf = parallel_coord_descent(client, data['N'], maxiter=1) ll_inf = popn.compute_log_p(x_inf) print "LL_inf: %f" % ll_inf # Save results with open(os.path.join(options.resultsDir, 'results.pkl'), 'w') as f: cPickle.dump(x_inf, f, protocol=-1) # Plot results plot_results(popn, x_inf, popn_true, x_true, do_plot_imp_responses=False, resdir=options.resultsDir)
def run_synth_test(): """ Run a test with synthetic data and MCMC inference """ options, popn, data, popn_true, x_true = initialize_test_harness() # Sample random initial state x0 = popn.sample() ll0 = popn.compute_log_p(x0) print "LL0: %f" % ll0 # Perform inference x_inf = coord_descent(popn, x0=x0, maxiter=1) ll_inf = popn.compute_log_p(x_inf) print "LL_inf: %f" % ll_inf # Save results results_file = os.path.join(options.resultsDir, 'results.pkl') print "Saving results to %s" % results_file with open(results_file, 'w') as f: cPickle.dump(x_inf, f, protocol=-1) # Plot results plot_results(popn, x_inf, popn_true, x_true, resdir=options.resultsDir)
def run_synth_test(): """ Run a test with synthetic data and MAP inference with cross validation """ options, popn, data, popn_true, x_true = initialize_test_harness() # Get the list of models for cross validation base_model = make_model(options.model, N=data['N'], dt=0.001) models = get_xv_models(base_model) # TODO Segment data into training and cross validation sets train_frac = 0.75 T_split = data['T'] * train_frac train_data = segment_data(data, (0, T_split)) xv_data = segment_data(data, (T_split, data['T'])) # Preprocess the data sequences train_data = popn.preprocess_data(train_data) xv_data = popn.preprocess_data(xv_data) # Sample random initial state x0 = popn.sample() # Track the best model and parameters best_ind = -1 best_xv_ll = -np.Inf best_x = x0 best_model = None # Fit each model using the optimum of the previous models train_lls = np.zeros(len(models)) xv_lls = np.zeros(len(models)) total_lls = np.zeros(len(models)) for (i, model) in enumerate(models): print "Training model %d" % i x0 = copy.deepcopy(best_x) popn.set_hyperparameters(model) popn.set_data(train_data) ll0 = popn.compute_log_p(x0) print "Training LL0: %f" % ll0 # Perform inference x_inf = coord_descent(popn, x0=x0, maxiter=1) ll_train = popn.compute_log_p(x_inf) print "Training LP_inf: %f" % ll_train train_lls[i] = ll_train # Compute log lkhd on xv data popn.set_data(xv_data) ll_xv = popn.compute_ll(x_inf) print "Cross Validation LL: %f" % ll_xv xv_lls[i] = ll_xv # Compute log lkhd on total dataset popn.set_data(data) ll_total = popn.compute_ll(x_inf) print "Total LL: %f" % ll_total total_lls[i] = ll_total # Update best model if ll_xv > best_xv_ll: best_ind = i best_xv_ll = ll_xv best_x = copy.deepcopy(x_inf) best_model = copy.deepcopy(model) # Create a population with the best model popn.set_hyperparameters(best_model) popn.set_data(data) # Fit the best model on the full training data best_x = coord_descent(popn, data, x0=x0, maxiter=1, use_hessian=False, use_rop=False) # Print results summary for i in np.arange(len(models)): print "Model %d:\tTrain LL: %.1f\tXV LL: %.1f\tTotal LL: %.1f" % ( i, train_lls[i], xv_lls[i], total_lls[i]) print "Best model: %d" % best_ind print "Best Total LL: %f" % popn.compute_ll(best_x) print "True LL: %f" % popn_true.compute_ll(x_true) # Save results results_file = os.path.join(options.resultsDir, 'results.pkl') print "Saving results to %s" % results_file with open(results_file, 'w') as f: cPickle.dump(best_x, f) # Plot results plot_results(popn, best_x, popn_true, x_true, resdir=options.resultsDir)
def run_gen_synth_data(): """ Run a test with synthetic data and MCMC inference """ options, args = parse_cmd_line_args() # Create the model dt = 0.001 model = make_model(options.model, N=options.N, dt=dt) # Set the sparsity level to minimize the risk of unstable networks stabilize_sparsity(model) print "Creating master population object" popn = Population(model) # Sample random parameters from the model x_true = popn.sample() # Check stability of matrix assert check_stability(model, x_true, options.N), "ERROR: Sampled network is unstable!" # Save the model so it can be loaded alongside the data fname_model = os.path.join(options.resultsDir, 'model.pkl') print "Saving data to %s" % fname_model with open(fname_model, 'w') as f: cPickle.dump(model, f, protocol=-1) print "Generating synthetic data with %d neurons and %.2f seconds." % \ (options.N, options.T_stop) # Set simulation parametrs dt_stim = 0.1 D_stim = (5, 5) # D_stim = model['bkgd']['D_stim'] if 'D_stim' in model['bkgd'] else 0 if isinstance(D_stim, int): D_stim = [D_stim] stim = np.random.randn(options.T_stop / dt_stim, *D_stim) data = gen_synth_data(options.N, options.T_stop, popn, x_true, dt, dt_stim, D_stim, stim) # Set the data so that the population state can be evaluated popn.add_data(data) # DEBUG Evaluate the firing rate and the simulated firing rate state = popn.eval_state(x_true) for n in np.arange(options.N): lam_true = state['glms'][n]['lam'] lam_sim = popn.glm.nlin_model.f_nlin(data['X'][:, n]) assert np.allclose(lam_true, lam_sim) # Pickle the data so we can open it more easily fname_pkl = os.path.join(options.resultsDir, 'data.pkl') print "Saving data to %s" % fname_pkl with open(fname_pkl, 'w') as f: cPickle.dump(data, f, protocol=-1) # Plot firing rates, stimulus responses, etc do_plot_imp_resonses = int(options.N) <= 16 plot_results(popn, data['vars'], resdir=options.resultsDir, do_plot_stim_resp=True, do_plot_imp_responses=do_plot_imp_resonses)
def run_gen_synth_data(): """ Run a test with synthetic data and MCMC inference """ options, args = parse_cmd_line_args() # Create the model dt = 0.001 model = make_model(options.model, N=options.N, dt=dt) # Set the sparsity level to minimize the risk of unstable networks stabilize_sparsity(model) print "Creating master population object" popn = Population(model) # Sample random parameters from the model x_true = popn.sample() # Check stability of matrix assert check_stability(model, x_true, options.N), "ERROR: Sampled network is unstable!" # Save the model so it can be loaded alongside the data fname_model = os.path.join(options.resultsDir, 'model.pkl') print "Saving data to %s" % fname_model with open(fname_model,'w') as f: cPickle.dump(model, f, protocol=-1) print "Generating synthetic data with %d neurons and %.2f seconds." % \ (options.N, options.T_stop) # Set simulation parametrs dt_stim = 0.1 D_stim = (5,5) # D_stim = model['bkgd']['D_stim'] if 'D_stim' in model['bkgd'] else 0 if isinstance(D_stim, int): D_stim = [D_stim] stim = np.random.randn(options.T_stop/dt_stim, *D_stim) data = gen_synth_data(options.N, options.T_stop, popn, x_true, dt, dt_stim, D_stim, stim) # Set the data so that the population state can be evaluated popn.add_data(data) # DEBUG Evaluate the firing rate and the simulated firing rate state = popn.eval_state(x_true) for n in np.arange(options.N): lam_true = state['glms'][n]['lam'] lam_sim = popn.glm.nlin_model.f_nlin(data['X'][:,n]) assert np.allclose(lam_true, lam_sim) # Pickle the data so we can open it more easily fname_pkl = os.path.join(options.resultsDir, 'data.pkl') print "Saving data to %s" % fname_pkl with open(fname_pkl,'w') as f: cPickle.dump(data, f, protocol=-1) # Plot firing rates, stimulus responses, etc do_plot_imp_resonses = int(options.N) <= 16 plot_results(popn, data['vars'], resdir=options.resultsDir, do_plot_stim_resp=True, do_plot_imp_responses=do_plot_imp_resonses)
def run_synth_test(): """ Run a test with synthetic data and MAP inference with cross validation """ options, popn, data, popn_true, x_true = initialize_test_harness() # Get the list of models for cross validation base_model = make_model(options.model, N=data['N'], dt=0.001) models = get_xv_models(base_model) # TODO Segment data into training and cross validation sets train_frac = 0.75 T_split = data['T'] * train_frac train_data = segment_data(data, (0,T_split)) xv_data = segment_data(data, (T_split,data['T'])) # Preprocess the data sequences train_data = popn.preprocess_data(train_data) xv_data = popn.preprocess_data(xv_data) # Sample random initial state x0 = popn.sample() # Track the best model and parameters best_ind = -1 best_xv_ll = -np.Inf best_x = x0 best_model = None # Fit each model using the optimum of the previous models train_lls = np.zeros(len(models)) xv_lls = np.zeros(len(models)) total_lls = np.zeros(len(models)) for (i,model) in enumerate(models): print "Training model %d" % i x0 = copy.deepcopy(best_x) popn.set_hyperparameters(model) popn.set_data(train_data) ll0 = popn.compute_log_p(x0) print "Training LL0: %f" % ll0 # Perform inference x_inf = coord_descent(popn, x0=x0, maxiter=1) ll_train = popn.compute_log_p(x_inf) print "Training LP_inf: %f" % ll_train train_lls[i] = ll_train # Compute log lkhd on xv data popn.set_data(xv_data) ll_xv = popn.compute_ll(x_inf) print "Cross Validation LL: %f" % ll_xv xv_lls[i] = ll_xv # Compute log lkhd on total dataset popn.set_data(data) ll_total = popn.compute_ll(x_inf) print "Total LL: %f" % ll_total total_lls[i] = ll_total # Update best model if ll_xv > best_xv_ll: best_ind = i best_xv_ll = ll_xv best_x = copy.deepcopy(x_inf) best_model = copy.deepcopy(model) # Create a population with the best model popn.set_hyperparameters(best_model) popn.set_data(data) # Fit the best model on the full training data best_x = coord_descent(popn, data, x0=x0, maxiter=1, use_hessian=False, use_rop=False) # Print results summary for i in np.arange(len(models)): print "Model %d:\tTrain LL: %.1f\tXV LL: %.1f\tTotal LL: %.1f" % (i, train_lls[i], xv_lls[i], total_lls[i]) print "Best model: %d" % best_ind print "Best Total LL: %f" % popn.compute_ll(best_x) print "True LL: %f" % popn_true.compute_ll(x_true) # Save results results_file = os.path.join(options.resultsDir, 'results.pkl') print "Saving results to %s" % results_file with open(results_file, 'w') as f: cPickle.dump(best_x, f) # Plot results plot_results(popn, best_x, popn_true, x_true, resdir=options.resultsDir)