def load_experiments(self): """ Load a specific human experiment. """ fit_mixture_model = self.parameters.get("fit_mixturemodel", False) # Load each experimental dataset for experiment_id in self.experiment_ids: if experiment_id == "bays09": self.experimental_datasets[experiment_id] = load_experimental_data.load_data_bays09( data_dir=self.data_dir, fit_mixture_model=fit_mixture_model ) if self.debug: print "Loading Bays09 dataset" elif experiment_id == "dualrecall": self.experimental_datasets[experiment_id] = load_experimental_data.load_data_dualrecall( data_dir=self.data_dir ) if self.debug: print "Loading double Recall dataset" elif experiment_id == "gorgo11": self.experimental_datasets[experiment_id] = load_experimental_data.load_data_simult( data_dir=self.data_dir, fit_mixture_model=fit_mixture_model ) if self.debug: print "Loading Gorgo11 simult dataset"
def launcher_do_fit_mixturemodel_dualrecall(args): ''' Run the model for T items, trying to fit the DualRecall dataset, which has two conditions. Get: - Precision - EM mixture model fits - Theoretical Fisher Information - EM Mixture model distances ''' print "Doing a piece of work for launcher_do_fit_mixturemodel_dualrecall" all_parameters = utils.argparse_2_dict(args) print all_parameters if all_parameters['burn_samples'] + all_parameters['num_samples'] < 200: print "WARNING> you do not have enough samples I think!", all_parameters['burn_samples'] + all_parameters['num_samples'] # Create DataIO # (complete label with current variable state) dataio = DataIO.DataIO(output_folder=all_parameters['output_directory'], label=all_parameters['label'].format(**all_parameters)) save_every = 1 run_counter = 0 # Load datasets to compare against data_dualrecall = load_experimental_data.load_data_dualrecall(data_dir=all_parameters['experiment_data_dir'], fit_mixture_model=True) dualrecall_T_space = data_dualrecall['data_to_fit']['n_items'] dualrecall_experimental_angle_emfits_mean = data_dualrecall['em_fits_angle_nitems_arrays']['mean'] dualrecall_experimental_colour_emfits_mean = data_dualrecall['em_fits_colour_nitems_arrays']['mean'] # Parameters to vary repetitions_axis = -1 # Result arrays result_all_precisions = np.nan*np.empty((all_parameters['num_repetitions'])) result_fi_theo = np.nan*np.empty((all_parameters['num_repetitions'])) result_fi_theocov = np.nan*np.empty((all_parameters['num_repetitions'])) result_em_fits = np.nan*np.empty((6, all_parameters['num_repetitions'])) # kappa, mixt_target, mixt_nontarget, mixt_random, ll, bic result_dist_dualrecall_angle = np.nan*np.empty((4, all_parameters['num_repetitions'])) # kappa, mixt_target, mixt_nontarget, mixt_random result_dist_dualrecall_angle_emmixt_KL = np.nan*np.empty((all_parameters['num_repetitions'])) result_dist_dualrecall_colour = np.nan*np.empty((4, all_parameters['num_repetitions'])) # kappa, mixt_target, mixt_nontarget, mixt_random result_dist_dualrecall_colour_emmixt_KL = np.nan*np.empty((all_parameters['num_repetitions'])) # If desired, will automatically save all Model responses. if all_parameters['collect_responses']: print "--- Collecting all responses..." result_responses = np.nan*np.ones((all_parameters['N'], all_parameters['num_repetitions'])) result_target = np.nan*np.ones((all_parameters['N'], all_parameters['num_repetitions'])) result_nontargets = np.nan*np.ones((all_parameters['N'], all_parameters['T'] - 1, all_parameters['num_repetitions'])) search_progress = progress.Progress(all_parameters['num_repetitions']) for repet_i in xrange(all_parameters['num_repetitions']): print "%.2f%%, %s left - %s" % (search_progress.percentage(), search_progress.time_remaining_str(), search_progress.eta_str()) print "Fit for T=%d, %d/%d" % (all_parameters['T'], repet_i+1, all_parameters['num_repetitions']) ## Update parameter ### WORK WORK WORK work? ### # Instantiate (_, _, _, sampler) = launchers.init_everything(all_parameters) # Sample sampler.run_inference(all_parameters) # Compute precision print "get precision..." result_all_precisions[repet_i] = sampler.get_precision() # Fit mixture model print "fit mixture model..." curr_params_fit = sampler.fit_mixture_model(use_all_targets=False) # curr_params_fit['mixt_nontargets_sum'] = np.sum(curr_params_fit['mixt_nontargets']) result_em_fits[:, repet_i] = [curr_params_fit[key] for key in ['kappa', 'mixt_target', 'mixt_nontargets_sum', 'mixt_random', 'train_LL', 'bic']] # Compute fisher info print "compute fisher info" result_fi_theo[repet_i] = sampler.estimate_fisher_info_theocov(use_theoretical_cov=False) result_fi_theocov[repet_i] = sampler.estimate_fisher_info_theocov(use_theoretical_cov=True) # Compute distances to datasets if all_parameters['T'] in dualrecall_T_space: # Angle trials result_dist_dualrecall_angle[:, repet_i] = (dualrecall_experimental_angle_emfits_mean[:, dualrecall_T_space == all_parameters['T']].flatten() - result_em_fits[:4, repet_i])**2. result_dist_dualrecall_angle_emmixt_KL[repet_i] = utils.KL_div(result_em_fits[1:4, repet_i], dualrecall_experimental_angle_emfits_mean[1:, dualrecall_T_space==all_parameters['T']].flatten()) # Colour trials result_dist_dualrecall_colour[:, repet_i] = (dualrecall_experimental_colour_emfits_mean[:, dualrecall_T_space == all_parameters['T']].flatten() - result_em_fits[:4, repet_i])**2. result_dist_dualrecall_colour_emmixt_KL[repet_i] = utils.KL_div(result_em_fits[1:4, repet_i], dualrecall_experimental_colour_emfits_mean[1:, dualrecall_T_space==all_parameters['T']].flatten()) # If needed, store responses if all_parameters['collect_responses']: (responses, target, nontarget) = sampler.collect_responses() result_responses[:, repet_i] = responses result_target[:, repet_i] = target result_nontargets[..., repet_i] = nontarget print "collected responses" print "CURRENT RESULTS:\n", result_all_precisions[repet_i], curr_params_fit, result_fi_theo[repet_i], result_fi_theocov[repet_i], np.sum(result_dist_dualrecall_angle[:, repet_i]), np.sum(result_dist_dualrecall_colour[:, repet_i]), "\n" ### /Work ### search_progress.increment() if run_counter % save_every == 0 or search_progress.done(): dataio.save_variables_default(locals()) run_counter += 1 # Finished dataio.save_variables_default(locals()) print "All finished" return locals()
def postprocess_dualrecall_fitmixturemodel(data_pbs, generator_module=None): ''' Reload runs from PBS To be plotted in Ipython later ''' #### SETUP # savedata = True colormap = None # or 'cubehelix' plt.rcParams['font.size'] = 16 # #### /SETUP print "Order parameters: ", data_pbs.dataset_infos['parameters'] # parameters: M, ratio_conj, sigmax # Extract data result_em_fits = np.array(data_pbs.dict_arrays['result_em_fits']['results_flat']) result_dist_dualrecall_angle = np.array(data_pbs.dict_arrays['result_dist_dualrecall_angle']['results_flat']) result_dist_dualrecall_angle_emmixt_KL = np.array(data_pbs.dict_arrays['result_dist_dualrecall_angle_emmixt_KL']['results_flat']) result_dist_dualrecall_colour = np.array(data_pbs.dict_arrays['result_dist_dualrecall_colour']['results_flat']) result_dist_dualrecall_colour_emmixt_KL = np.array(data_pbs.dict_arrays['result_dist_dualrecall_colour_emmixt_KL']['results_flat']) result_parameters_flat = np.array(data_pbs.dict_arrays['result_em_fits']['parameters_flat']) all_repeats_completed = data_pbs.dict_arrays['result_em_fits']['repeats_completed'] all_args_arr = np.array(data_pbs.loaded_data['args_list']) M_space = data_pbs.loaded_data['parameters_uniques']['M'] ratio_conj_space = data_pbs.loaded_data['parameters_uniques']['ratio_conj'] sigmax_space = data_pbs.loaded_data['parameters_uniques']['sigmax'] num_repetitions = generator_module.num_repetitions parameter_names_sorted = data_pbs.dataset_infos['parameters'] dataio = DataIO(output_folder=generator_module.pbs_submission_infos['simul_out_dir'] + '/outputs/', label='global_' + dataset_infos['save_output_filename']) # Load ground truth data_dualrecall = load_experimental_data.load_data_dualrecall(fit_mixture_model=True) ## Filter everything with repeats_completed == num_repet filter_data = all_repeats_completed == num_repetitions - 1 result_parameters_flat = result_parameters_flat[filter_data] result_em_fits = result_em_fits[filter_data] result_dist_dualrecall_angle = result_dist_dualrecall_angle[filter_data] result_dist_dualrecall_angle_emmixt_KL = result_dist_dualrecall_angle_emmixt_KL[filter_data] result_dist_dualrecall_colour = result_dist_dualrecall_colour[filter_data] result_dist_dualrecall_colour_emmixt_KL = result_dist_dualrecall_colour_emmixt_KL[filter_data] all_args_arr = all_args_arr[filter_data] all_repeats_completed = all_repeats_completed[filter_data] print "Size post-filter: ", result_parameters_flat.shape[0] # Compute lots of averages over the repetitions result_em_fits_avg = utils.nanmean(result_em_fits, axis=-1) result_dist_dualrecall_angle_avg = utils.nanmean(result_dist_dualrecall_angle, axis=-1) result_dist_dualrecall_angle_emmixt_KL_avg = utils.nanmean(result_dist_dualrecall_angle_emmixt_KL, axis=-1) result_dist_dualrecall_colour_avg = utils.nanmean(result_dist_dualrecall_colour, axis=-1) result_dist_dualrecall_colour_emmixt_KL_avg = utils.nanmean(result_dist_dualrecall_colour_emmixt_KL, axis=-1) # all_args = data_pbs.loaded_data['args_list'] variables_to_save = ['parameter_names_sorted', 'all_args_arr', 'all_repeats_completed', 'filter_data'] if savedata: dataio.save_variables_default(locals(), variables_to_save) dataio.make_link_output_to_dropbox(dropbox_current_experiment_folder='dualrecall_fitmixturemodel') plt.show() return locals()