def plots_fitting_experiments_random(data_pbs, generator_module=None):
    '''
        Reload 2D volume runs from PBS and plot them

    '''

    #### SETUP
    #
    savefigs = True
    savedata = True
    savemovies = False

    do_bays09 = True
    do_gorgo11 = True

    scatter3d_sumT = False
    plots_flat_sorted_performance = False
    plots_memorycurves_fits_best = True

    nb_best_points = 20
    nb_best_points_per_T = nb_best_points/6
    size_normal_points = 8
    size_best_points = 50
    downsampling = 2


    # do_relaunch_bestparams_pbs = True

    colormap = None  # or 'cubehelix'
    plt.rcParams['font.size'] = 16
    #
    #### /SETUP

    print "Order parameters: ", generator_module.dict_parameters_range.keys()
    # parameters: ratio_conj, sigmax, T

    # Extract data
    result_fitexperiments_flat = np.array(data_pbs.dict_arrays['result_fitexperiments']['results_flat'])
    result_fitexperiments_all_flat = np.array(data_pbs.dict_arrays['result_fitexperiments_all']['results_flat'])
    result_fitexperiments_noiseconv_flat = np.array(data_pbs.dict_arrays['result_fitexperiments_noiseconv']['results_flat'])
    result_fitexperiments_noiseconv_all_flat = np.array(data_pbs.dict_arrays['result_fitexperiments_noiseconv_all']['results_flat'])
    result_parameters_flat = np.array(data_pbs.dict_arrays['result_fitexperiments']['parameters_flat'])

    all_repeats_completed = data_pbs.dict_arrays['result_fitexperiments']['repeats_completed']
    all_args = data_pbs.loaded_data['args_list']
    all_args_arr = np.array(all_args)
    num_repetitions = generator_module.num_repetitions

    # Extract order of datasets
    experiment_ids = data_pbs.loaded_data['datasets_list'][0]['fitexperiment_parameters']['experiment_ids']
    parameter_names_sorted = data_pbs.dataset_infos['parameters']

    T_space = data_pbs.loaded_data['datasets_list'][0]['T_space']

    dataio = DataIO(output_folder=generator_module.pbs_submission_infos['simul_out_dir'] + '/outputs/', label='global_' + dataset_infos['save_output_filename'])

    # filter_data = (result_parameters_flat[:, -1] < 1.0) & (all_repeats_completed == num_repetitions - 1)
    # filter_data = (all_repeats_completed == num_repetitions - 1)
    # result_fitexperiments_flat = result_fitexperiments_flat[filter_data]
    # result_fitexperiments_all_flat = result_fitexperiments_all_flat[filter_data]
    # result_fitexperiments_noiseconv_flat = result_fitexperiments_noiseconv_flat[filter_data]
    # result_fitexperiments_noiseconv_all_flat = result_fitexperiments_noiseconv_all_flat[filter_data]
    # result_parameters_flat = result_parameters_flat[filter_data]

    # Compute some stuff
    # Data is summed over all experiments for _flat, contains bic, ll and ll90.
    # for _all_flat, contains bic, ll and ll90 per experiment. Given that Gorgo11 and Bays09 are incompatible, shouldn't really use the combined version directly!
    result_fitexperiments_noiseconv_bic_avg_allT = utils.nanmean(result_fitexperiments_noiseconv_flat, axis=-1)[..., 0]
    result_fitexperiments_noiseconv_allexp_bic_avg_allT = utils.nanmean(result_fitexperiments_noiseconv_all_flat, axis=-1)[:, :, 0]
    result_fitexperiments_noiseconv_allexp_ll90_avg_allT = -utils.nanmean(result_fitexperiments_noiseconv_all_flat, axis=-1)[:, :, -1]

    ### BIC
    # result_fitexperiments_noiseconv_allexp_bic_avg_allT: N x T x exp
    result_fitexperiments_noiseconv_bays09_bic_avg_allT = result_fitexperiments_noiseconv_allexp_bic_avg_allT[..., 0]
    result_fitexperiments_noiseconv_gorgo11_bic_avg_allT = result_fitexperiments_noiseconv_allexp_bic_avg_allT[..., 1]
    result_fitexperiments_noiseconv_dualrecall_bic_avg_allT = result_fitexperiments_noiseconv_allexp_bic_avg_allT[..., 2]
    # Summed T
    result_fitexperiments_noiseconv_bays09_bic_avg_sumT = np.nansum(result_fitexperiments_noiseconv_bays09_bic_avg_allT, axis=-1)
    result_fitexperiments_noiseconv_gorgo11_bic_avg_sumT = np.nansum(result_fitexperiments_noiseconv_gorgo11_bic_avg_allT, axis=-1)
    result_fitexperiments_noiseconv_dualrecall_bic_avg_sumT = np.nansum(result_fitexperiments_noiseconv_dualrecall_bic_avg_allT, axis=-1)

    ### LL90
    # N x T x exp
    result_fitexperiments_noiseconv_bays09_ll90_avg_allT = result_fitexperiments_noiseconv_allexp_ll90_avg_allT[..., 0]
    result_fitexperiments_noiseconv_gorgo11_ll90_avg_allT = result_fitexperiments_noiseconv_allexp_ll90_avg_allT[..., 1]
    result_fitexperiments_noiseconv_dualrecall_ll90_avg_allT = result_fitexperiments_noiseconv_allexp_ll90_avg_allT[..., 2]
    # Summed T
    result_fitexperiments_noiseconv_bays09_ll90_avg_sumT = np.nansum(result_fitexperiments_noiseconv_bays09_ll90_avg_allT, axis=-1)
    result_fitexperiments_noiseconv_gorgo11_ll90_avg_sumT = np.nansum(result_fitexperiments_noiseconv_gorgo11_ll90_avg_allT, axis=-1)
    result_fitexperiments_noiseconv_dualrecall_ll90_avg_sumT = np.nansum(result_fitexperiments_noiseconv_dualrecall_ll90_avg_allT, axis=-1)

    def mask_outliers_array(result_dist_to_use, sigma_outlier=3):
        '''
            Mask outlier datapoints.
            Compute the mean of the results and assume that points with:
              result > mean + sigma_outlier*std
            are outliers.

            As we want the minimum values, do not mask small values
        '''
        return np.ma.masked_greater(result_dist_to_use, np.mean(result_dist_to_use) + sigma_outlier*np.std(result_dist_to_use))

    def best_points_allT(result_dist_to_use):
        '''
            Best points for all T
        '''
        return np.argsort(result_dist_to_use)[:nb_best_points]

    def str_best_params(best_i, result_dist_to_use):
        return ' '.join(["%s %.4f" % (parameter_names_sorted[param_i], result_parameters_flat[best_i, param_i]) for param_i in xrange(len(parameter_names_sorted))]) + ' >> %f' % result_dist_to_use[best_i]

    def plot_scatter(all_vars, result_dist_to_use_name, title='', log_color=True, downsampling=1, label_file='', mask_outliers=True):

        result_dist_to_use = all_vars[result_dist_to_use_name]
        result_parameters_flat = all_vars['result_parameters_flat']

        # Filter if downsampling
        filter_downsampling = np.arange(0, result_dist_to_use.size, downsampling)
        result_dist_to_use = result_dist_to_use[filter_downsampling]
        result_parameters_flat = result_parameters_flat[filter_downsampling]

        if mask_outliers:
            result_dist_to_use = mask_outliers_array(result_dist_to_use)

        best_points_result_dist_to_use = np.argsort(result_dist_to_use)[:nb_best_points]

        # Construct all permutations of 3 parameters, for 3D scatters
        params_permutations = set([tuple(np.sort(np.random.choice(result_parameters_flat.shape[-1], 3, replace=False)).tolist()) for i in xrange(1000)])

        for param_permut in params_permutations:
            fig = plt.figure()
            ax = Axes3D(fig)

            # One plot per parameter permutation
            if log_color:
                color_points = np.log(result_dist_to_use)
            else:
                color_points = result_dist_to_use

            utils.scatter3d(result_parameters_flat[:, param_permut[0]], result_parameters_flat[:, param_permut[1]], result_parameters_flat[:, param_permut[2]], s=size_normal_points, c=color_points, xlabel=parameter_names_sorted[param_permut[0]], ylabel=parameter_names_sorted[param_permut[1]], zlabel=parameter_names_sorted[param_permut[2]], title=title, ax_handle=ax)

            utils.scatter3d(result_parameters_flat[best_points_result_dist_to_use, param_permut[0]], result_parameters_flat[best_points_result_dist_to_use, param_permut[1]], result_parameters_flat[best_points_result_dist_to_use, param_permut[2]], c='r', s=size_best_points, ax_handle=ax)

            if savefigs:
                dataio.save_current_figure('scatter3d_%s_%s%s_{label}_{unique_id}.pdf' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file))

            if savemovies:
                try:
                    utils.rotate_plot3d(ax, dataio.create_formatted_filename('scatter3d_%s_%s%s_{label}_{unique_id}.mp4' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file)), bitrate=8000, min_duration=8)
                    utils.rotate_plot3d(ax, dataio.create_formatted_filename('scatter3d_%s_%s%s_{label}_{unique_id}.gif' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file)), nb_frames=30, min_duration=8)
                except Exception:
                    # Most likely wrong aggregator...
                    print "failed when creating movies for ", result_dist_to_use_name


            if False and savefigs:
                ax.view_init(azim=90, elev=10)
                dataio.save_current_figure('scatter3d_view2_%s_%s%s_{label}_{unique_id}.pdf' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file))

            # plt.close('all')

        print "Parameters: %s" % ', '.join(parameter_names_sorted)
        print "Best points, %s:" % title
        print '\n'.join([str_best_params(best_i, result_dist_to_use) for best_i in best_points_result_dist_to_use])



    if scatter3d_sumT:

        plot_scatter(locals(), 'result_fitexperiments_noiseconv_bays09_bic_avg_sumT', 'BIC Bays09')
        plot_scatter(locals(), 'result_fitexperiments_noiseconv_bays09_ll90_avg_sumT', 'LL90 Bays09')

        plot_scatter(locals(), 'result_fitexperiments_noiseconv_gorgo11_bic_avg_sumT', 'BIC Gorgo11')
        plot_scatter(locals(), 'result_fitexperiments_noiseconv_gorgo11_ll90_avg_sumT', 'LL90 Gorgo11')

        plot_scatter(locals(), 'result_fitexperiments_noiseconv_dualrecall_bic_avg_sumT', 'BIC Dual recall')
        plot_scatter(locals(), 'result_fitexperiments_noiseconv_dualrecall_ll90_avg_sumT', 'LL90 Dual recall')


    if plots_flat_sorted_performance:
        result_dist_to_try = []
        if do_bays09:
            result_dist_to_try.extend(['result_fitexperiments_noiseconv_bays09_bic_avg_sumT', 'result_fitexperiments_noiseconv_bays09_ll90_avg_sumT'])
        if do_gorgo11:
            result_dist_to_try.extend(['result_fitexperiments_noiseconv_gorgo11_bic_avg_sumT', 'result_fitexperiments_noiseconv_gorgo11_ll90_avg_sumT'])

        for result_dist in result_dist_to_try:
            order_indices = np.argsort(locals()[result_dist])[::-1]

            f, axes = plt.subplots(2, 1)
            axes[0].plot(np.arange(4) + result_parameters_flat[order_indices]/np.max(result_parameters_flat[order_indices], axis=0))
            axes[0].legend(parameter_names_sorted, loc='upper left')
            axes[0].set_ylabel('Parameters')
            axes[1].plot(locals()[result_dist][order_indices])
            axes[1].set_ylabel(result_dist.split('result_dist_')[-1])
            axes[0].set_title('Distance ordered ' + result_dist.split('result_dist_')[-1])
            f.canvas.draw()

            if savefigs:
                dataio.save_current_figure('plot_sortedperf_full_%s_{label}_{unique_id}.pdf' % (result_dist))

    if plots_memorycurves_fits_best:
        # Alright, will actually reload the data from another set of runs, and find the closest parameter set to the ones found here.
        data = utils.load_npy('normalisedsigmaxsigmaoutput_random_fitmixturemodels_sigmaxMratiosigmaoutput_repetitions3_280814/outputs/global_plots_fitmixtmodel_random_sigmaoutsigmaxnormMratio-plots_fit_mixturemodels_random-75eb9c74-72e0-4165-8014-92c1ef446f0a.npy')
        result_em_fits_flat_fitmixture = data['result_em_fits_flat']
        result_parameters_flat_fitmixture = data['result_parameters_flat']
        all_args_arr_fitmixture = data['all_args_arr']

        data_dir = None
        if not os.environ.get('WORKDIR_DROP'):
            data_dir = '../experimental_data/'

        plotting_parameters = launchers_memorycurves_marginal_fi.load_prepare_datasets()

        def plot_memorycurves_fits_fromexternal(all_vars, result_dist_to_use_name, nb_best_points=10):
            result_dist_to_use = all_vars[result_dist_to_use_name]

            result_em_fits_flat_fitmixture = all_vars['result_em_fits_flat_fitmixture']
            result_parameters_flat_fitmixture = all_vars['result_parameters_flat_fitmixture']
            all_args_arr_fitmixture = all_vars['all_args_arr_fitmixture']

            best_point_indices_result_dist = np.argsort(result_dist_to_use)[:nb_best_points]

            for best_point_index in best_point_indices_result_dist:
                print "extended plot desired for: " + str_best_params(best_point_index, result_dist_to_use)

                dist_best_points_fitmixture = np.abs(result_parameters_flat_fitmixture - result_parameters_flat[best_point_index])
                dist_best_points_fitmixture -= np.min(dist_best_points_fitmixture, axis=0)
                dist_best_points_fitmixture /= np.max(dist_best_points_fitmixture, axis=0)

                best_point_index_fitmixture = np.argmax(np.prod(1-dist_best_points_fitmixture, axis=-1))

                print "found closest: " + ' '.join(["%s %.4f" % (parameter_names_sorted[param_i], result_parameters_flat_fitmixture[best_point_index_fitmixture, param_i]) for param_i in xrange(len(parameter_names_sorted))])

                # Update arguments
                all_args_arr_fitmixture[best_point_index_fitmixture].update(dict(zip(parameter_names_sorted, result_parameters_flat_fitmixture[best_point_index_fitmixture])))
                packed_data = dict(T_space=T_space, result_em_fits=result_em_fits_flat_fitmixture[best_point_index_fitmixture], all_parameters=all_args_arr_fitmixture[best_point_index_fitmixture])

                plotting_parameters['suptitle'] = result_dist_to_use_name
                plotting_parameters['reuse_axes'] = False
                if savefigs:
                    packed_data['dataio'] = dataio

                launchers_memorycurves_marginal_fi.do_memory_plots(packed_data, plotting_parameters)


        plot_memorycurves_fits_fromexternal(locals(), 'result_external_fitexperiments_noiseconv_bays09_ll90_avg_sumT', nb_best_points=3)

        plot_memorycurves_fits_fromexternal(locals(), 'result_external_fitexperiments_noiseconv_gorgo11_ll90_avg_sumT', nb_best_points=3)

        plot_memorycurves_fits_fromexternal(locals(), 'result_external_fitexperiments_noiseconv_dualrecall_ll90_avg_sumT', nb_best_points=3)



    all_args = data_pbs.loaded_data['args_list']
    variables_to_save = ['experiment_ids', 'parameter_names_sorted', 'T_space', 'all_args_arr', 'all_repeats_completed', 'filter_data']

    if savedata:
        dataio.save_variables_default(locals(), variables_to_save)
        dataio.make_link_output_to_dropbox(dropbox_current_experiment_folder='sigmaoutput_normalisedsigmax_random')


    plt.show()

    return locals()
示例#2
0
def plots_fitmixtmodel_rcscale_effect(data_pbs, generator_module=None):
    '''
        Reload runs from PBS
    '''

    #### SETUP
    #
    savefigs = True
    savedata = True

    plots_all_T = True
    plots_per_T = True

    # do_relaunch_bestparams_pbs = True

    colormap = None  # or 'cubehelix'
    plt.rcParams['font.size'] = 16
    #
    #### /SETUP

    print "Order parameters: ", data_pbs.dataset_infos['parameters']
    # parameters: M, ratio_conj, sigmax

    # Extract data
    T_space = data_pbs.loaded_data['datasets_list'][0]['T_space']

    result_em_fits_flat = np.array(data_pbs.dict_arrays['result_em_fits']['results_flat'])
    result_precisions_flat = np.array(data_pbs.dict_arrays['result_all_precisions']['results_flat'])
    result_dist_bays09_flat = np.array(data_pbs.dict_arrays['result_dist_bays09']['results_flat'])
    result_dist_gorgo11_flat = np.array(data_pbs.dict_arrays['result_dist_gorgo11']['results_flat'])
    result_dist_bays09_emmixt_KL = np.array(data_pbs.dict_arrays['result_dist_bays09_emmixt_KL']['results_flat'])
    result_dist_gorgo11_emmixt_KL = np.array(data_pbs.dict_arrays['result_dist_gorgo11_emmixt_KL']['results_flat'])
    result_parameters_flat = np.array(data_pbs.dict_arrays['result_em_fits']['parameters_flat'])

    rc_scale_space = data_pbs.loaded_data['parameters_uniques']['rc_scale']
    num_repetitions = generator_module.num_repetitions
    parameter_names_sorted = data_pbs.dataset_infos['parameters']

    dataio = DataIO(output_folder=generator_module.pbs_submission_infos['simul_out_dir'] + '/outputs/', label='global_' + dataset_infos['save_output_filename'])

    # Load bays09
    data_bays09 = load_experimental_data.load_data_bays09(fit_mixture_model=True)
    bays09_nitems = data_bays09['data_to_fit']['n_items']
    bays09_em_target = np.nan*np.empty((bays09_nitems.max(), 4))  #kappa, prob_target, prob_nontarget, prob_random
    bays09_em_target[bays09_nitems - 1] = data_bays09['em_fits_nitems_arrays']['mean'].T
    bays09_emmixt_target = bays09_em_target[:, 1:]


    ## Compute some stuff
    result_parameters_flat = result_parameters_flat.flatten()

    result_em_fits_all_avg = utils.nanmean(result_em_fits_flat, axis=-1)
    result_em_kappa_allT = result_em_fits_all_avg[..., 0]
    result_em_emmixt_allT = result_em_fits_all_avg[..., 1:4]

    result_precisions_all_avg = utils.nanmean(result_precisions_flat, axis=-1)

    # Square distance to kappa
    result_dist_bays09_allT_avg = utils.nanmean(result_dist_bays09_flat, axis=-1)
    result_dist_bays09_emmixt_KL_allT_avg = utils.nanmean(result_dist_bays09_emmixt_KL, axis=-1)

    result_dist_bays09_kappa_allT = result_dist_bays09_allT_avg[..., 0]

    # result_dist_bays09_allT_avg = utils.nanmean((result_em_fits_flat[:, :, :4] - bays09_em_target[np.newaxis, :, :, np.newaxis])**2, axis=-1)
    # result_dist_bays09_kappa_sum = np.nansum(result_dist_bays09_allT_avg[:, :, 0], axis=-1)

    # result_dist_bays09_kappa_T1_sum = result_dist_bays09_allT_avg[:, 0, 0]
    # result_dist_bays09_kappa_T25_sum = np.nansum(result_dist_bays09_allT_avg[:, 1:, 0], axis=-1)

    # # Square and KL distance for EM Mixtures
    # result_dist_bays09_emmixt_sum = np.nansum(np.nansum(result_dist_bays09_allT_avg[:, :, 1:], axis=-1), axis=-1)
    # result_dist_bays09_emmixt_T1_sum = np.nansum(result_dist_bays09_allT_avg[:, 0, 1:], axis=-1)
    # result_dist_bays09_emmixt_T25_sum = np.nansum(np.nansum(result_dist_bays09_allT_avg[:, 1:, 1:], axis=-1), axis=-1)


    # result_dist_bays09_emmixt_KL = utils.nanmean(utils.KL_div(result_em_fits_flat[:, :, 1:4], bays09_emmixt_target[np.newaxis, :, :, np.newaxis], axis=-2), axis=-1)   # KL over dimension of mixtures, then mean over repetitions
    # result_dist_bays09_emmixt_KL_sum = np.nansum(result_dist_bays09_emmixt_KL, axis=-1)  # sum over T
    # result_dist_bays09_emmixt_KL_T1_sum = result_dist_bays09_emmixt_KL[:, 0]
    # result_dist_bays09_emmixt_KL_T25_sum = np.nansum(result_dist_bays09_emmixt_KL[:, 1:], axis=-1)


    # result_dist_bays09_both_normalised = result_dist_bays09_emmixt_sum/np.max(result_dist_bays09_emmixt_sum) + result_dist_bays09_kappa_sum/np.max(result_dist_bays09_kappa_sum)

    # # Mask kappa for performance too bad
    # result_dist_bays09_kappa_sum_masked = np.ma.masked_greater(result_dist_bays09_kappa_sum, 2*np.median(result_dist_bays09_kappa_sum))
    # result_dist_bays09_emmixt_KL_sum_masked = np.ma.masked_greater(result_dist_bays09_emmixt_KL_sum, 2*np.median(result_dist_bays09_emmixt_KL_sum))
    # result_dist_bays09_both_normalised_mult_masked = 1-(1. - result_dist_bays09_emmixt_KL_sum/np.max(result_dist_bays09_emmixt_KL_sum))*(1. - result_dist_bays09_kappa_sum_masked/np.max(result_dist_bays09_kappa_sum_masked))

    # Compute optimal rc_scale
    all_args = data_pbs.loaded_data['args_list']
    specific_arg = all_args[0]
    specific_arg['autoset_parameters'] = True
    (_, _, _, sampler) = launchers.init_everything(specific_arg)
    optimal_rc_scale = sampler.random_network.rc_scale[0]

    if plots_all_T:
        # Show Kappa evolution wrt rc_scale
        f, ax = plt.subplots()
        # utils.plot_mean_std_from_samples(result_parameters_flat, np.nansum(result_em_kappa_allT, axis=-1), bins=60, bins_y=150, xlabel='rc_scale', ylabel='EM kappa', title='Kappa, summed T',  ax_handle=ax, show_scatter=False)
        utils.plot_mean_std_from_samples_rolling(result_parameters_flat, np.nansum(result_em_kappa_allT, axis=-1), window=35, xlabel='rc_scale', ylabel='EM kappa', title='Kappa, summed T',  ax_handle=ax, show_scatter=False)
        ax.axvline(x=optimal_rc_scale, color='g', linewidth=2)
        ax.axvline(x=2*optimal_rc_scale, color='r', linewidth=2)
        f.canvas.draw()

        if savefigs:
            dataio.save_current_figure('rcscaleeffect_kappa_summedT_{label}_{unique_id}.pdf')

        # Show Mixt proportions
        f, ax = plt.subplots()
        for i in xrange(3):
            # utils.plot_mean_std_from_samples(result_parameters_flat, np.nansum(result_em_emmixt_allT[..., i], axis=-1), bins=60, bins_y=100, xlabel='rc_scale', ylabel='EM mixt proportions', title='EM mixtures, summed T',  ax_handle=ax, show_scatter=False)
            utils.plot_mean_std_from_samples_rolling(result_parameters_flat, np.nansum(result_em_emmixt_allT[..., i], axis=-1), window=35, xlabel='rc_scale', ylabel='EM mixt proportions', title='EM mixtures, summed T',  ax_handle=ax, show_scatter=False)
        ax.axvline(x=optimal_rc_scale, color='g', linewidth=2)
        ax.axvline(x=2*optimal_rc_scale, color='r', linewidth=2)
        f.canvas.draw()

        if savefigs:
            dataio.save_current_figure('rcscaleeffect_mixtprop_summedT_{label}_{unique_id}.pdf')

        # Show Precision
        f, ax = plt.subplots()
        # utils.plot_mean_std_from_samples(result_parameters_flat, np.nansum(result_precisions_all_avg, axis=-1), bins=60, bins_y=150, xlabel='rc_scale', ylabel='Precision', title='Precision, summed T',  ax_handle=ax, show_scatter=False)
        utils.plot_mean_std_from_samples_rolling(result_parameters_flat, np.nansum(result_precisions_all_avg, axis=-1), window=35, xlabel='rc_scale', ylabel='Precision', title='Precision, summed T',  ax_handle=ax, show_scatter=False)
        ax.axvline(x=optimal_rc_scale, color='g', linewidth=2)
        ax.axvline(x=2*optimal_rc_scale, color='r', linewidth=2)
        f.canvas.draw()

        if savefigs:
            dataio.save_current_figure('rcscaleeffect_precision_summedT_{label}_{unique_id}.pdf')


        plt.close('all')


    if plots_per_T:
        for T_i, T in enumerate(T_space):
            # Show Kappa evolution wrt rc_scale
            f, ax = plt.subplots()
            # utils.plot_mean_std_from_samples(result_parameters_flat, result_em_kappa_allT[:, T_i], bins=40, bins_y=100, xlabel='rc_scale', ylabel='EM kappa', title='Kappa, T %d' % T,  ax_handle=ax, show_scatter=False)
            utils.plot_mean_std_from_samples_rolling(result_parameters_flat, result_em_kappa_allT[:, T_i], window=35, xlabel='rc_scale', ylabel='EM kappa', title='Kappa, T %d' % T,  ax_handle=ax, show_scatter=False)
            ax.axvline(x=optimal_rc_scale, color='g', linewidth=2)
            ax.axvline(x=2*optimal_rc_scale, color='r', linewidth=2)
            f.canvas.draw()

            if savefigs:
                dataio.save_current_figure('rcscaleeffect_kappa_T%d_{label}_{unique_id}.pdf' % T)

            # Show Mixt proportions
            f, ax = plt.subplots()
            for i in xrange(3):
                # utils.plot_mean_std_from_samples(result_parameters_flat, result_em_emmixt_allT[:, T_i, i], bins=40, bins_y=100, xlabel='rc_scale', ylabel='EM mixt proportions', title='EM mixtures, T %d' % T,  ax_handle=ax, show_scatter=False)
                utils.plot_mean_std_from_samples_rolling(result_parameters_flat, result_em_emmixt_allT[:, T_i, i], window=35, xlabel='rc_scale', ylabel='EM mixt proportions', title='EM mixtures, T %d' % T,  ax_handle=ax, show_scatter=False)
            ax.axvline(x=optimal_rc_scale, color='g', linewidth=2)
            ax.axvline(x=2*optimal_rc_scale, color='r', linewidth=2)
            f.canvas.draw()

            if savefigs:
                dataio.save_current_figure('rcscaleeffect_mixtprop_T%d_{label}_{unique_id}.pdf' % T)

            # Show Precision
            f, ax = plt.subplots()
            # utils.plot_mean_std_from_samples(result_parameters_flat, result_precisions_all_avg[:, T_i], bins=40, bins_y=100, xlabel='rc_scale', ylabel='Precision', title='Precision, T %d' % T,  ax_handle=ax, show_scatter=False)
            utils.plot_mean_std_from_samples_rolling(result_parameters_flat, result_precisions_all_avg[:, T_i], window=35, xlabel='rc_scale', ylabel='Precision', title='Precision, T %d' % T,  ax_handle=ax, show_scatter=False)
            ax.axvline(x=optimal_rc_scale, color='g', linewidth=2)
            ax.axvline(x=2*optimal_rc_scale, color='r', linewidth=2)
            f.canvas.draw()

            if savefigs:
                dataio.save_current_figure('rcscaleeffect_precision_T%d_{label}_{unique_id}.pdf' % T)

            plt.close('all')





    # # Interpolate
    # if plots_interpolate:

    #     sigmax_target = 0.9

    #     M_interp_space = np.arange(6, 625, 5)
    #     ratio_interp_space = np.linspace(0.01, 1.0, 50)
    #     # sigmax_interp_space = np.linspace(0.01, 1.0, 50)
    #     sigmax_interp_space = np.array([sigmax_target])
    #     params_crossspace = np.array(utils.cross(M_interp_space, ratio_interp_space, sigmax_interp_space))

    #     interpolated_data = rbf_interpolator(params_crossspace[:, 0], params_crossspace[:, 1], params_crossspace[:, 2]).reshape((M_interp_space.size, ratio_interp_space.size))

    #     utils.pcolor_2d_data(interpolated_data, M_interp_space, ratio_interp_space, 'M', 'ratio', 'interpolated, fixing sigmax= %.2f' % sigmax_target)

    #     points_closeby = ((result_parameters_flat[:, 2] - sigmax_target)**2)< 0.01
    #     plt.figure()
    #     # plt.imshow(interpolated_data, extent=(M_interp_space.min(), M_interp_space.max(), ratio_interp_space.min(), ratio_interp_space.max()))
    #     plt.imshow(interpolated_data)
    #     plt.scatter(result_parameters_flat[points_closeby, 0], result_parameters_flat[points_closeby, 1], s=100, c=result_fitexperiments_bic_avg[points_closeby], marker='o')


    # if plot_per_ratio:
    #     # Plot the evolution of loglike as a function of sigmax, with std shown
    #     for ratio_conj_i, ratio_conj in enumerate(ratio_space):
    #         ax = utils.plot_mean_std_area(sigmax_space, result_log_posterior_mean[ratio_conj_i], result_log_posterior_std[ratio_conj_i])

    #         ax.get_figure().canvas.draw()

    #         if savefigs:
    #             dataio.save_current_figure('results_fitexp_%s_loglike_ratioconj%.2f_{label}_global_{unique_id}.pdf' % (exp_dataset, ratio_conj))



    all_args = data_pbs.loaded_data['args_list']
    variables_to_save = ['parameter_names_sorted']

    if savedata:
        dataio.save_variables_default(locals(), variables_to_save)
        dataio.make_link_output_to_dropbox(dropbox_current_experiment_folder='rcscale_characterisation')


    plt.show()

    return locals()
def plots_fit_mixturemodels_random(data_pbs, generator_module=None):
    '''
        Reload runs from PBS
    '''

    #### SETUP
    #
    savefigs = True
    savedata = True
    savemovies = True

    plots_dist_bays09 = True
    plots_per_T = True
    plots_interpolate = False

    # do_relaunch_bestparams_pbs = True

    colormap = None  # or 'cubehelix'
    plt.rcParams['font.size'] = 16
    #
    #### /SETUP

    print "Order parameters: ", data_pbs.dataset_infos['parameters']
    # parameters: M, ratio_conj, sigmax

    # Extract data
    T_space = data_pbs.loaded_data['datasets_list'][0]['T_space']

    result_em_fits_flat = np.array(data_pbs.dict_arrays['result_em_fits']['results_flat'])
    result_dist_bays09_flat = np.array(data_pbs.dict_arrays['result_dist_bays09']['results_flat'])
    result_dist_gorgo11_flat = np.array(data_pbs.dict_arrays['result_dist_gorgo11']['results_flat'])
    result_parameters_flat = np.array(data_pbs.dict_arrays['result_em_fits']['parameters_flat'])

    sigmaoutput_space = data_pbs.loaded_data['parameters_uniques']['sigma_output']
    sigmax_space = data_pbs.loaded_data['parameters_uniques']['sigmax']
    ratio_space = data_pbs.loaded_data['parameters_uniques']['sigmax']
    num_repetitions = generator_module.num_repetitions
    parameter_names_sorted = data_pbs.dataset_infos['parameters']

    dataio = DataIO(output_folder=generator_module.pbs_submission_infos['simul_out_dir'] + '/outputs/', label='global_' + dataset_infos['save_output_filename'])

    # Load bays09
    data_bays09 = load_experimental_data.load_data_bays09(fit_mixture_model=True)
    bays09_nitems = data_bays09['data_to_fit']['n_items']
    bays09_em_target = np.nan*np.empty((bays09_nitems.max(), 4))  #kappa, prob_target, prob_nontarget, prob_random
    bays09_em_target[bays09_nitems - 1] = data_bays09['em_fits_nitems_arrays']['mean'].T
    bays09_emmixt_target = bays09_em_target[:, 1:]


    ## Compute some stuff

    # result_dist_bays09_kappa_T1_avg = utils.nanmean(result_dist_bays09_flat[:, 0, 0], axis=-1)
    # result_dist_bays09_kappa_allT_avg = np.nansum(utils.nanmean(result_dist_bays09_flat[:, :, 0], axis=-1), axis=1)

    # Square distance to kappa
    result_dist_bays09_allT_avg = utils.nanmean((result_em_fits_flat[:, :, :4] - bays09_em_target[np.newaxis, :, :, np.newaxis])**2, axis=-1)
    result_dist_bays09_kappa_sum = np.nansum(result_dist_bays09_allT_avg[:, :, 0], axis=-1)

    result_dist_bays09_kappa_T1_sum = result_dist_bays09_allT_avg[:, 0, 0]
    result_dist_bays09_kappa_T25_sum = np.nansum(result_dist_bays09_allT_avg[:, 1:, 0], axis=-1)

    # Square and KL distance for EM Mixtures
    result_dist_bays09_emmixt_sum = np.nansum(np.nansum(result_dist_bays09_allT_avg[:, :, 1:], axis=-1), axis=-1)
    result_dist_bays09_emmixt_T1_sum = np.nansum(result_dist_bays09_allT_avg[:, 0, 1:], axis=-1)
    result_dist_bays09_emmixt_T25_sum = np.nansum(np.nansum(result_dist_bays09_allT_avg[:, 1:, 1:], axis=-1), axis=-1)


    result_dist_bays09_emmixt_KL = utils.nanmean(utils.KL_div(result_em_fits_flat[:, :, 1:4], bays09_emmixt_target[np.newaxis, :, :, np.newaxis], axis=-2), axis=-1)   # KL over dimension of mixtures, then mean over repetitions
    result_dist_bays09_emmixt_KL_sum = np.nansum(result_dist_bays09_emmixt_KL, axis=-1)  # sum over T
    result_dist_bays09_emmixt_KL_T1_sum = result_dist_bays09_emmixt_KL[:, 0]
    result_dist_bays09_emmixt_KL_T25_sum = np.nansum(result_dist_bays09_emmixt_KL[:, 1:], axis=-1)


    result_dist_bays09_both_normalised = result_dist_bays09_emmixt_sum/np.max(result_dist_bays09_emmixt_sum) + result_dist_bays09_kappa_sum/np.max(result_dist_bays09_kappa_sum)

    if plots_dist_bays09:
        nb_best_points = 30
        size_normal_points = 8
        size_best_points = 50

        def plot_scatter(all_vars, result_dist_to_use_name, title='', log_color=True, downsampling=1, label_file=''):

            fig = plt.figure()
            ax = Axes3D(fig)

            result_dist_to_use = all_vars[result_dist_to_use_name]
            if not log_color:
                result_dist_to_use = np.exp(result_dist_to_use)

            utils.scatter3d(result_parameters_flat[:, 0], result_parameters_flat[:, 1], result_parameters_flat[:, 2], s=size_normal_points, c=np.log(result_dist_to_use), xlabel=parameter_names_sorted[0], ylabel=parameter_names_sorted[1], zlabel=parameter_names_sorted[2], title=title, ax_handle=ax)
            best_points_result_dist_to_use = np.argsort(result_dist_to_use)[:nb_best_points]
            utils.scatter3d(result_parameters_flat[best_points_result_dist_to_use, 0], result_parameters_flat[best_points_result_dist_to_use, 1], result_parameters_flat[best_points_result_dist_to_use, 2], c='r', s=size_best_points, ax_handle=ax)
            print "Best points, %s:" % title
            print '\n'.join(['sigma output %.2f, ratio %.2f, sigmax %.2f:  %f' % (result_parameters_flat[i, 0], result_parameters_flat[i, 1], result_parameters_flat[i, 2], result_dist_to_use[i]) for i in best_points_result_dist_to_use])

            if savefigs:
                dataio.save_current_figure('scatter3d_%s%s_{label}_{unique_id}.pdf' % (result_dist_to_use_name, label_file))

                if savemovies:
                    try:
                        utils.rotate_plot3d(ax, dataio.create_formatted_filename('scatter3d_%s%s_{label}_{unique_id}.mp4' % (result_dist_to_use_name, label_file)), bitrate=8000, min_duration=8)
                        utils.rotate_plot3d(ax, dataio.create_formatted_filename('scatter3d_%s%s_{label}_{unique_id}.gif' % (result_dist_to_use_name, label_file)), nb_frames=30, min_duration=8)
                    except Exception:
                        # Most likely wrong aggregator...
                        print "failed when creating movies for ", result_dist_to_use_name

                ax.view_init(azim=90, elev=10)
                dataio.save_current_figure('scatter3d_view2_%s%s_{label}_{unique_id}.pdf' % (result_dist_to_use_name, label_file))

            return ax

        # Distance for kappa, all T
        plot_scatter(locals(), 'result_dist_bays09_kappa_sum', 'kappa all T')

        # Distance for em fits, all T, Squared distance
        plot_scatter(locals(), 'result_dist_bays09_emmixt_sum', 'em fits, all T')

        # Distance for em fits, all T, KL distance
        plot_scatter(locals(), 'result_dist_bays09_emmixt_KL_sum', 'em fits, all T, KL')

        # Distance for sum of normalised em fits + normalised kappa, all T
        plot_scatter(locals(), 'result_dist_bays09_both_normalised', 'summed normalised em mixt + kappa')

        # Distance kappa T = 1
        plot_scatter(locals(), 'result_dist_bays09_kappa_T1_sum', 'Kappa T=1')

        # Distance kappa T = 2...5
        plot_scatter(locals(), 'result_dist_bays09_kappa_T25_sum', 'Kappa T=2/5')

        # Distance em fits T = 1
        plot_scatter(locals(), 'result_dist_bays09_emmixt_T1_sum', 'em fits T=1')

        # Distance em fits T = 2...5
        plot_scatter(locals(), 'result_dist_bays09_emmixt_T25_sum', 'em fits T=2/5')

        # Distance em fits T = 1, KL
        plot_scatter(locals(), 'result_dist_bays09_emmixt_KL_T1_sum', 'em fits T=1, KL')

        # Distance em fits T = 2...5, KL
        plot_scatter(locals(), 'result_dist_bays09_emmixt_KL_T25_sum', 'em fits T=2/5, KL')



    if plots_per_T:
        for T_i, T in enumerate(T_space):

            # Kappa per T, fit to Bays09
            result_dist_bays09_kappa_currT = result_dist_bays09_allT_avg[:, T_i, 0]
            result_dist_bays09_kappa_currT_masked = mask_outliers(result_dist_bays09_kappa_currT)

            plot_scatter(locals(), 'result_dist_bays09_kappa_currT_masked', 'kappa T %d masked' % T, label_file="T{}".format(T))

            # EM Mixt per T, fit to Bays09
            result_dist_bays09_emmixt_sum_currT = np.nansum(result_dist_bays09_allT_avg[:, T_i, 1:], axis=-1)
            result_dist_bays09_emmixt_sum_currT_masked = mask_outliers(result_dist_bays09_emmixt_sum_currT)

            plot_scatter(locals(), 'result_dist_bays09_emmixt_sum_currT_masked', 'EM mixt T %d masked' % T, label_file="T{}".format(T))

            # EM Mixt per T, fit to Bays09 KL divergence
            result_dist_bays09_emmixt_KL_sum_currT = result_dist_bays09_emmixt_KL[:, T_i]
            plot_scatter(locals(), 'result_dist_bays09_emmixt_KL_sum_currT', 'KL EM mixt T %d masked' % T, label_file="T{}".format(T))




    # # Interpolate
    # if plots_interpolate:

    #     sigmax_target = 0.9

    #     M_interp_space = np.arange(6, 625, 5)
    #     ratio_interp_space = np.linspace(0.01, 1.0, 50)
    #     # sigmax_interp_space = np.linspace(0.01, 1.0, 50)
    #     sigmax_interp_space = np.array([sigmax_target])
    #     params_crossspace = np.array(utils.cross(M_interp_space, ratio_interp_space, sigmax_interp_space))

    #     interpolated_data = rbf_interpolator(params_crossspace[:, 0], params_crossspace[:, 1], params_crossspace[:, 2]).reshape((M_interp_space.size, ratio_interp_space.size))

    #     utils.pcolor_2d_data(interpolated_data, M_interp_space, ratio_interp_space, 'M', 'ratio', 'interpolated, fixing sigmax= %.2f' % sigmax_target)

    #     points_closeby = ((result_parameters_flat[:, 2] - sigmax_target)**2)< 0.01
    #     plt.figure()
    #     # plt.imshow(interpolated_data, extent=(M_interp_space.min(), M_interp_space.max(), ratio_interp_space.min(), ratio_interp_space.max()))
    #     plt.imshow(interpolated_data)
    #     plt.scatter(result_parameters_flat[points_closeby, 0], result_parameters_flat[points_closeby, 1], s=100, c=result_fitexperiments_bic_avg[points_closeby], marker='o')


    # if plot_per_ratio:
    #     # Plot the evolution of loglike as a function of sigmax, with std shown
    #     for ratio_conj_i, ratio_conj in enumerate(ratio_space):
    #         ax = utils.plot_mean_std_area(sigmax_space, result_log_posterior_mean[ratio_conj_i], result_log_posterior_std[ratio_conj_i])

    #         ax.get_figure().canvas.draw()

    #         if savefigs:
    #             dataio.save_current_figure('results_fitexp_%s_loglike_ratioconj%.2f_{label}_global_{unique_id}.pdf' % (exp_dataset, ratio_conj))



    all_args = data_pbs.loaded_data['args_list']
    variables_to_save = ['parameter_names_sorted']

    if savedata:
        dataio.save_variables_default(locals(), variables_to_save)
        dataio.make_link_output_to_dropbox(dropbox_current_experiment_folder='output_noise')


    plt.show()

    return locals()
def plots_fit_mixturemodels_random(data_pbs, generator_module=None):
    '''
        Reload runs from PBS
    '''

    #### SETUP
    #
    savefigs = True
    savedata = True
    savemovies = False

    plots_dist_bays09 = True

    do_scatters_3d = True
    do_best_points_extended_plots = True
    # do_relaunch_bestparams_pbs = True

    colormap = None  # or 'cubehelix'
    plt.rcParams['font.size'] = 16
    #
    #### /SETUP

    print "Order parameters: ", data_pbs.dataset_infos['parameters']
    # parameters: M, ratio_conj, sigmax

    # Extract data
    result_em_fits_flat = np.array(data_pbs.dict_arrays['result_em_fits']['results_flat'])
    result_dist_bays09_flat = np.array(data_pbs.dict_arrays['result_dist_bays09']['results_flat'])
    result_dist_gorgo11_flat = np.array(data_pbs.dict_arrays['result_dist_gorgo11']['results_flat'])
    result_parameters_flat = np.array(data_pbs.dict_arrays['result_em_fits']['parameters_flat'])

    M_space = data_pbs.loaded_data['parameters_uniques']['M']
    sigmax_space = data_pbs.loaded_data['parameters_uniques']['sigmax']
    ratio_space = data_pbs.loaded_data['parameters_uniques']['sigmax']
    num_repetitions = generator_module.num_repetitions
    parameter_names_sorted = data_pbs.dataset_infos['parameters']
    T_space = data_pbs.loaded_data['datasets_list'][0]['T_space']
    all_args = data_pbs.loaded_data['args_list']
    all_repeats_completed = data_pbs.dict_arrays['result_em_fits']['repeats_completed']

    dataio = DataIO(output_folder=generator_module.pbs_submission_infos['simul_out_dir'] + '/outputs/', label='global_' + dataset_infos['save_output_filename'])

    # Load bays09
    data_bays09 = load_experimental_data.load_data_bays09(fit_mixture_model=True)
    bays09_nitems = data_bays09['data_to_fit']['n_items']
    bays09_em_target = np.nan*np.empty((bays09_nitems.max(), 4))  #kappa, prob_target, prob_nontarget, prob_random
    bays09_em_target[bays09_nitems - 1] = data_bays09['em_fits_nitems_arrays']['mean'].T
    bays09_emmixt_target = bays09_em_target[:, 1:]

    # All parameters info
    plotting_parameters = launcher_memorycurve.load_prepare_datasets()

    ## Compute some stuff

    # result_dist_bays09_kappa_T1_avg = utils.nanmean(result_dist_bays09_flat[:, 0, 0], axis=-1)
    # result_dist_bays09_kappa_allT_avg = np.nansum(utils.nanmean(result_dist_bays09_flat[:, :, 0], axis=-1), axis=1)

    # Square distance to kappa
    result_dist_bays09_allT_avg = utils.nanmean((result_em_fits_flat[:, :, :4] - bays09_em_target[np.newaxis, :, :, np.newaxis])**2, axis=-1)
    result_dist_bays09_kappa_sum = np.nansum(result_dist_bays09_allT_avg[:, :, 0], axis=-1)
    result_dist_bays09_kappa_sum_masked = np.ma.masked_greater(result_dist_bays09_kappa_sum, 1e8)

    result_dist_bays09_kappa_T1_sum = result_dist_bays09_allT_avg[:, 0, 0]
    result_dist_bays09_kappa_T25_sum = np.nansum(result_dist_bays09_allT_avg[:, 1:, 0], axis=-1)

    # Square and KL distance for EM Mixtures
    result_dist_bays09_emmixt_sum = np.nansum(np.nansum(result_dist_bays09_allT_avg[:, :, 1:], axis=-1), axis=-1)
    result_dist_bays09_emmixt_T1_sum = np.nansum(result_dist_bays09_allT_avg[:, 0, 1:], axis=-1)
    result_dist_bays09_emmixt_T25_sum = np.nansum(np.nansum(result_dist_bays09_allT_avg[:, 1:, 1:], axis=-1), axis=-1)


    result_dist_bays09_emmixt_KL = utils.nanmean(utils.KL_div(result_em_fits_flat[:, :, 1:4], bays09_emmixt_target[np.newaxis, :, :, np.newaxis], axis=-2), axis=-1)   # KL over dimension of mixtures, then mean over repetitions
    result_dist_bays09_emmixt_KL_sum = np.nansum(result_dist_bays09_emmixt_KL, axis=-1)  # sum over T
    result_dist_bays09_emmixt_KL_T1_sum = result_dist_bays09_emmixt_KL[:, 0]
    result_dist_bays09_emmixt_KL_T25_sum = np.nansum(result_dist_bays09_emmixt_KL[:, 1:], axis=-1)

    result_dist_bays09_both_normalised = result_dist_bays09_emmixt_sum/np.max(result_dist_bays09_emmixt_sum) + result_dist_bays09_kappa_sum/np.max(result_dist_bays09_kappa_sum)
    result_dist_bays09_kappaKL_normalised_summed = result_dist_bays09_emmixt_KL_sum/np.max(result_dist_bays09_emmixt_KL_sum) + result_dist_bays09_kappa_sum/np.max(result_dist_bays09_kappa_sum)


    if plots_dist_bays09:
        nb_best_points = 30
        size_normal_points = 8
        size_best_points = 50
        nb_best_points_extended_plots = 3


        def plot_memorycurve(result_em_fits, args_used, suptitle=''):

            packed_data = dict(T_space=T_space, result_em_fits=result_em_fits, all_parameters=args_used)
            if suptitle:
                plotting_parameters['suptitle'] = suptitle
            if savefigs:
                packed_data['dataio'] = dataio

            plotting_parameters['reuse_axes'] = False

            launcher_memorycurve.do_memory_plots(packed_data, plotting_parameters)


        def plot_scatter(result_dist_to_use, best_points_result_dist_to_use, result_dist_to_use_name='', title=''):

            fig = plt.figure()
            ax = Axes3D(fig)

            utils.scatter3d(result_parameters_flat[:, 0], result_parameters_flat[:, 1], result_parameters_flat[:, 2], s=size_normal_points, c=np.log(result_dist_to_use), xlabel=parameter_names_sorted[0], ylabel=parameter_names_sorted[1], zlabel=parameter_names_sorted[2], title=title, ax_handle=ax)
            utils.scatter3d(result_parameters_flat[best_points_result_dist_to_use, 0], result_parameters_flat[best_points_result_dist_to_use, 1], result_parameters_flat[best_points_result_dist_to_use, 2], c='r', s=size_best_points, ax_handle=ax)
            print "Best points, %s:" % title
            print '\n'.join(['M %d, ratio %.2f, sigmax %.2f:  %f' % (result_parameters_flat[i, 0], result_parameters_flat[i, 1], result_parameters_flat[i, 2], result_dist_to_use[i]) for i in best_points_result_dist_to_use])

            if savefigs:
                dataio.save_current_figure('scatter3d_%s_{label}_{unique_id}.pdf' % result_dist_to_use_name)

                if savemovies:
                    try:
                        utils.rotate_plot3d(ax, dataio.create_formatted_filename('scatter3d_%s_{label}_{unique_id}.mp4' % result_dist_to_use_name), bitrate=8000, min_duration=8)
                        utils.rotate_plot3d(ax, dataio.create_formatted_filename('scatter3d_%s_{label}_{unique_id}.gif' % result_dist_to_use_name), nb_frames=30, min_duration=8)
                    except Exception:
                        # Most likely wrong aggregator...
                        print "failed when creating movies for ", result_dist_to_use_name

                ax.view_init(azim=90, elev=10)
                dataio.save_current_figure('scatter3d_%s_view2_{label}_{unique_id}.pdf' % result_dist_to_use_name)

            return ax

        def plots_redirects(all_vars, result_dist_to_use_name, log_color=True, title='', avoid_incomplete_repeats=True):

            result_dist_to_use = all_vars[result_dist_to_use_name]

            if avoid_incomplete_repeats:
                result_dist_to_use = np.ma.masked_where(~(all_repeats_completed == num_repetitions-1), result_dist_to_use)

            if not log_color:
                result_dist_to_use = np.exp(result_dist_to_use)
            best_points_result_dist_to_use = np.argsort(result_dist_to_use)[:nb_best_points]

            # Scatter
            if do_scatters_3d:
                plot_scatter(result_dist_to_use, best_points_result_dist_to_use, result_dist_to_use_name, title=title)

            # Now do the additional plots if required
            if do_best_points_extended_plots:
                for best_point_index in best_points_result_dist_to_use[:nb_best_points_extended_plots]:
                    print "extended plot for M %d, ratio %.2f, sigmax %.2f:  score %f" % (result_parameters_flat[best_point_index, 0], result_parameters_flat[best_point_index, 1], result_parameters_flat[best_point_index, 2], result_dist_to_use[best_point_index])

                    plot_memorycurve(result_em_fits_flat[best_point_index], all_args[best_point_index], suptitle=result_dist_to_use_name)

        # Distance for kappa, all T
        plots_redirects(locals(), 'result_dist_bays09_kappa_sum', title='kappa all T')

        # Distance for em fits, all T, Squared distance
        plots_redirects(locals(), 'result_dist_bays09_emmixt_sum', title='em fits, all T')

        # Distance for em fits, all T, KL distance
        plots_redirects(locals(), 'result_dist_bays09_emmixt_KL_sum', title='em fits, all T, KL')

        # Distance for sum of normalised em fits + normalised kappa, all T
        plots_redirects(locals(), 'result_dist_bays09_both_normalised', title='summed normalised em mixt + kappa')

        # Distance kappa T = 1
        plots_redirects(locals(), 'result_dist_bays09_kappa_T1_sum', title='Kappa T=1')

        # Distance kappa T = 2...5
        plots_redirects(locals(), 'result_dist_bays09_kappa_T25_sum', title='Kappa T=2/5')

        # Distance em fits T = 1
        plots_redirects(locals(), 'result_dist_bays09_emmixt_T1_sum', title='em fits T=1')

        # Distance em fits T = 2...5
        plots_redirects(locals(), 'result_dist_bays09_emmixt_T25_sum', title='em fits T=2/5')

        # Distance em fits T = 1, KL
        plots_redirects(locals(), 'result_dist_bays09_emmixt_KL_T1_sum', title='em fits T=1, KL')

        # Distance em fits T = 2...5, KL
        plots_redirects(locals(), 'result_dist_bays09_emmixt_KL_T25_sum', title='em fits T=2/5, KL')

    # if plots_per_T:
    #     for T in T_space:
    #         currT_indices = result_parameters_flat[:, 2] == T

    #         utils.contourf_interpolate_data_interactive_maxvalue(result_parameters_flat[currT_indices][..., :2], result_fitexperiments_bic_avg[currT_indices], xlabel='Ratio_conj', ylabel='sigma x', title='BIC, T %d' % T, interpolation_numpoints=200, interpolation_method='nearest', log_scale=False)

    # # Interpolate
    # if plots_interpolate:

    #     sigmax_target = 0.9

    #     M_interp_space = np.arange(6, 625, 5)
    #     ratio_interp_space = np.linspace(0.01, 1.0, 50)
    #     # sigmax_interp_space = np.linspace(0.01, 1.0, 50)
    #     sigmax_interp_space = np.array([sigmax_target])
    #     params_crossspace = np.array(utils.cross(M_interp_space, ratio_interp_space, sigmax_interp_space))

    #     interpolated_data = rbf_interpolator(params_crossspace[:, 0], params_crossspace[:, 1], params_crossspace[:, 2]).reshape((M_interp_space.size, ratio_interp_space.size))

    #     utils.pcolor_2d_data(interpolated_data, M_interp_space, ratio_interp_space, 'M', 'ratio', 'interpolated, fixing sigmax= %.2f' % sigmax_target)

    #     points_closeby = ((result_parameters_flat[:, 2] - sigmax_target)**2)< 0.01
    #     plt.figure()
    #     # plt.imshow(interpolated_data, extent=(M_interp_space.min(), M_interp_space.max(), ratio_interp_space.min(), ratio_interp_space.max()))
    #     plt.imshow(interpolated_data)
    #     plt.scatter(result_parameters_flat[points_closeby, 0], result_parameters_flat[points_closeby, 1], s=100, c=result_fitexperiments_bic_avg[points_closeby], marker='o')


    # if plot_per_ratio:
    #     # Plot the evolution of loglike as a function of sigmax, with std shown
    #     for ratio_conj_i, ratio_conj in enumerate(ratio_space):
    #         ax = utils.plot_mean_std_area(sigmax_space, result_log_posterior_mean[ratio_conj_i], result_log_posterior_std[ratio_conj_i])

    #         ax.get_figure().canvas.draw()

    #         if savefigs:
    #             dataio.save_current_figure('results_fitexp_%s_loglike_ratioconj%.2f_{label}_global_{unique_id}.pdf' % (exp_dataset, ratio_conj))


    variables_to_save = ['parameter_names_sorted', 'all_repeats_completed', 'T_space']

    if savedata:
        dataio.save_variables_default(locals(), variables_to_save)
        dataio.make_link_output_to_dropbox(dropbox_current_experiment_folder='fit_mixturemodels')


    plt.show()

    return locals()
def postprocess_dualrecall_fitmixturemodel(data_pbs, generator_module=None):
    '''
        Reload runs from PBS

        To be plotted in Ipython later
    '''

    #### SETUP
    #
    savedata = True

    colormap = None  # or 'cubehelix'
    plt.rcParams['font.size'] = 16
    #
    #### /SETUP

    print "Order parameters: ", data_pbs.dataset_infos['parameters']
    # parameters: M, ratio_conj, sigmax

    # Extract data
    result_em_fits = np.array(data_pbs.dict_arrays['result_em_fits']['results_flat'])
    result_dist_dualrecall_angle = np.array(data_pbs.dict_arrays['result_dist_dualrecall_angle']['results_flat'])
    result_dist_dualrecall_angle_emmixt_KL = np.array(data_pbs.dict_arrays['result_dist_dualrecall_angle_emmixt_KL']['results_flat'])
    result_dist_dualrecall_colour = np.array(data_pbs.dict_arrays['result_dist_dualrecall_colour']['results_flat'])
    result_dist_dualrecall_colour_emmixt_KL = np.array(data_pbs.dict_arrays['result_dist_dualrecall_colour_emmixt_KL']['results_flat'])


    result_parameters_flat = np.array(data_pbs.dict_arrays['result_em_fits']['parameters_flat'])
    all_repeats_completed = data_pbs.dict_arrays['result_em_fits']['repeats_completed']

    all_args_arr = np.array(data_pbs.loaded_data['args_list'])

    M_space = data_pbs.loaded_data['parameters_uniques']['M']
    ratio_conj_space = data_pbs.loaded_data['parameters_uniques']['ratio_conj']
    sigmax_space = data_pbs.loaded_data['parameters_uniques']['sigmax']

    num_repetitions = generator_module.num_repetitions
    parameter_names_sorted = data_pbs.dataset_infos['parameters']

    dataio = DataIO(output_folder=generator_module.pbs_submission_infos['simul_out_dir'] + '/outputs/', label='global_' + dataset_infos['save_output_filename'])

    # Load ground truth
    data_dualrecall = load_experimental_data.load_data_dualrecall(fit_mixture_model=True)

    ## Filter everything with repeats_completed == num_repet
    filter_data = all_repeats_completed == num_repetitions - 1
    result_parameters_flat = result_parameters_flat[filter_data]

    result_em_fits = result_em_fits[filter_data]
    result_dist_dualrecall_angle = result_dist_dualrecall_angle[filter_data]
    result_dist_dualrecall_angle_emmixt_KL = result_dist_dualrecall_angle_emmixt_KL[filter_data]
    result_dist_dualrecall_colour = result_dist_dualrecall_colour[filter_data]
    result_dist_dualrecall_colour_emmixt_KL = result_dist_dualrecall_colour_emmixt_KL[filter_data]

    all_args_arr = all_args_arr[filter_data]
    all_repeats_completed = all_repeats_completed[filter_data]

    print "Size post-filter: ", result_parameters_flat.shape[0]

    # Compute lots of averages over the repetitions
    result_em_fits_avg = utils.nanmean(result_em_fits, axis=-1)
    result_dist_dualrecall_angle_avg = utils.nanmean(result_dist_dualrecall_angle, axis=-1)
    result_dist_dualrecall_angle_emmixt_KL_avg = utils.nanmean(result_dist_dualrecall_angle_emmixt_KL, axis=-1)
    result_dist_dualrecall_colour_avg = utils.nanmean(result_dist_dualrecall_colour, axis=-1)
    result_dist_dualrecall_colour_emmixt_KL_avg = utils.nanmean(result_dist_dualrecall_colour_emmixt_KL, axis=-1)

    # all_args = data_pbs.loaded_data['args_list']
    variables_to_save = ['parameter_names_sorted', 'all_args_arr', 'all_repeats_completed', 'filter_data']

    if savedata:
        dataio.save_variables_default(locals(), variables_to_save)
        dataio.make_link_output_to_dropbox(dropbox_current_experiment_folder='dualrecall_fitmixturemodel')


    plt.show()

    return locals()
def plots_fit_collapsedmixturemodels_random(data_pbs, generator_module=None):
    '''
        Reload runs from PBS

        Sequential data analysis.
    '''

    #### SETUP
    #
    plots_bestfits = True
    plots_scatter3d = False

    savefigs = True
    savedata = True

    colormap = None  # or 'cubehelix'
    plt.rcParams['font.size'] = 16
    #
    #### /SETUP

    print "Order parameters: ", data_pbs.dataset_infos['parameters']
    # parameters: M, ratio_conj, sigmax

    # Extract data
    T_space = data_pbs.loaded_data['datasets_list'][0]['T_space']

    result_em_fits_collapsed_tr_flat = np.array(data_pbs.dict_arrays['result_em_fits_collapsed_tr']['results_flat'])
    result_em_fits_collapsed_summary_flat = np.array(data_pbs.dict_arrays['result_em_fits_collapsed_summary']['results_flat'])
    result_dist_gorgo11_sequ_collapsed_flat = np.array(data_pbs.dict_arrays['result_dist_gorgo11_sequ_collapsed']['results_flat'])
    result_dist_gorgo11_sequ_collapsed_emmixt_KL_flat = np.array(data_pbs.dict_arrays['result_dist_gorgo11_sequ_collapsed_emmixt_KL']['results_flat'])

    result_parameters_flat = np.array(data_pbs.dict_arrays['result_em_fits_collapsed_tr']['parameters_flat'])
    all_repeats_completed = data_pbs.dict_arrays['result_em_fits_collapsed_tr']['repeats_completed']

    all_args_arr = np.array(data_pbs.loaded_data['args_list'])

    M_space = data_pbs.loaded_data['parameters_uniques']['M']
    ratio_conj_space = data_pbs.loaded_data['parameters_uniques']['ratio_conj']
    sigmax_space = data_pbs.loaded_data['parameters_uniques']['sigmax']
    alpha_space = data_pbs.loaded_data['parameters_uniques']['alpha']

    num_repetitions = generator_module.num_repetitions
    parameter_names_sorted = data_pbs.dataset_infos['parameters']

    dataio = DataIO(output_folder=generator_module.pbs_submission_infos['simul_out_dir'] + '/outputs/', label='global_' + dataset_infos['save_output_filename'])

    # Load ground truth
    data_gorgo11_sequ = load_experimental_data.load_data_gorgo11_sequential(fit_mixture_model=True)

    ## Filter everything with repeats_completed == num_repet
    filter_data = all_repeats_completed == num_repetitions - 1
    result_parameters_flat = result_parameters_flat[filter_data]

    result_em_fits_collapsed_tr_flat = result_em_fits_collapsed_tr_flat[filter_data]
    result_em_fits_collapsed_summary_flat = result_em_fits_collapsed_summary_flat[filter_data]
    result_dist_gorgo11_sequ_collapsed_flat = result_dist_gorgo11_sequ_collapsed_flat[filter_data]
    result_dist_gorgo11_sequ_collapsed_emmixt_KL_flat = result_dist_gorgo11_sequ_collapsed_emmixt_KL_flat[filter_data]

    all_args_arr = all_args_arr[filter_data]
    all_repeats_completed = all_repeats_completed[filter_data]

    print "Size post-filter: ", result_parameters_flat.shape[0]

    # Compute lots of averages over the repetitions
    result_em_fits_collapsed_tr_flat_avg = utils.nanmean(result_em_fits_collapsed_tr_flat, axis=-1)
    result_em_fits_collapsed_summary_flat_avg = utils.nanmean(result_em_fits_collapsed_summary_flat, axis=-1)
    result_dist_gorgo11_sequ_collapsed_flat_avg = utils.nanmean(result_dist_gorgo11_sequ_collapsed_flat, axis=-1)
    result_dist_gorgo11_sequ_collapsed_emmixt_KL_flat_avg = utils.nanmean(result_dist_gorgo11_sequ_collapsed_emmixt_KL_flat, axis=-1)

    result_dist_gorgo11_sequ_collapsed_flat_avg_overall = np.nansum(np.nansum(np.nansum(result_dist_gorgo11_sequ_collapsed_flat_avg, axis=-1), axis=-1), axis=-1)
    # We will now grid some of the parameters, to have a 2D/3D surface back.
    # Let's fix the ratio_conj, as we know that the other models need around
    # ratio =0.8 to fit data well.

    def str_best_params(best_i, result_dist_to_use):
        return ' '.join(["%s %.4f" % (parameter_names_sorted[param_i], result_parameters_flat[best_i, param_i]) for param_i in xrange(len(parameter_names_sorted))]) + ' >> %f' % result_dist_to_use[best_i]

    ###### Best fitting points
    if plots_bestfits:
        nb_best_points = 5

        def plot_collapsed_modelfits(T_space, curr_result_emfits_collapsed_tr, labelplot='', dataio=None):
            f, ax = plt.subplots()
            for nitems_i, nitems in enumerate(T_space):
                ax = plots_experimental_data.plot_kappa_mean_error(T_space[:nitems], curr_result_emfits_collapsed_tr[..., 0][nitems_i, :nitems], 0.0*curr_result_emfits_collapsed_tr[..., 0][nitems_i, :nitems], title='model fit fig7 %s' % labelplot , ax=ax, label='%d items' % nitems, xlabel='T_recall')

            if dataio is not None:
                dataio.save_current_figure('bestfit_doublepowerlaw_%s_kappa_{label}_{unique_id}.pdf' % labelplot)

            _, ax_target = plt.subplots()
            _, ax_nontarget = plt.subplots()
            _, ax_random = plt.subplots()
            for nitems_i, nitems in enumerate(T_space):
                ax_target = plots_experimental_data.plot_emmixture_mean_error(T_space[:nitems], curr_result_emfits_collapsed_tr[..., 1][nitems_i, :nitems], curr_result_emfits_collapsed_tr[..., 1][nitems_i, :nitems]*0.0, title='Target model fit %s' % labelplot, ax=ax_target, label='%d items' % nitems, xlabel='T_recall')
                ax_nontarget = plots_experimental_data.plot_emmixture_mean_error(T_space[:nitems], curr_result_emfits_collapsed_tr[..., 2][nitems_i, :nitems], curr_result_emfits_collapsed_tr[..., 2][nitems_i, :nitems]*0.0, title='Nontarget model fit %s' % labelplot, ax=ax_nontarget, label='%d items' % nitems, xlabel='T_recall')
                ax_random = plots_experimental_data.plot_emmixture_mean_error(T_space[:nitems], curr_result_emfits_collapsed_tr[..., 3][nitems_i, :nitems], curr_result_emfits_collapsed_tr[..., 3][nitems_i, :nitems]*0.0, title='Random model fit %s' % labelplot, ax=ax_random, label='%d items' % nitems, xlabel='T_recall')

            if dataio is not None:
                plt.figure(ax_target.get_figure().number)
                dataio.save_current_figure('bestfit_doublepowerlaw_%s_mixttarget_{label}_{unique_id}.pdf' % labelplot)

                plt.figure(ax_nontarget.get_figure().number)
                dataio.save_current_figure('bestfit_doublepowerlaw_%s_mixtnontarget_{label}_{unique_id}.pdf' % labelplot)

                plt.figure(ax_random.get_figure().number)
                dataio.save_current_figure('bestfit_doublepowerlaw_%s_mixtrandom_{label}_{unique_id}.pdf' % labelplot)


        best_points_result_dist_gorgo11seq_all = np.argsort(result_dist_gorgo11_sequ_collapsed_flat_avg_overall)[:nb_best_points]

        for best_point_i in best_points_result_dist_gorgo11seq_all:
            plot_collapsed_modelfits(T_space, result_em_fits_collapsed_tr_flat_avg[best_point_i], labelplot='%.1f' % result_dist_gorgo11_sequ_collapsed_flat_avg_overall[best_point_i], dataio=dataio)


    ###### 3D scatter plots
    if plots_scatter3d:
        nb_best_points = 30
        size_normal_points = 8
        size_best_points = 50

        def plot_scatter(all_vars, result_dist_to_use_name, title='', log_color=True, downsampling=1, label_file=''):

            result_dist_to_use = all_vars[result_dist_to_use_name]
            result_parameters_flat_3d = all_vars['result_parameters_flat_3d']

            # Filter if downsampling
            filter_downsampling = np.arange(0, result_dist_to_use.size, downsampling)
            result_dist_to_use = result_dist_to_use[filter_downsampling]
            result_parameters_flat_3d = result_parameters_flat_3d[filter_downsampling]

            best_points_result_dist_to_use = np.argsort(result_dist_to_use)[:nb_best_points]

            # Construct all permutations of 3 parameters, for 3D scatters
            params_permutations = set([tuple(np.sort(np.random.choice(result_parameters_flat_3d.shape[-1], 3, replace=False)).tolist()) for i in xrange(1000)])

            for param_permut in params_permutations:
                fig = plt.figure()
                ax = Axes3D(fig)

                # One plot per parameter permutation
                if log_color:
                    color_points = np.log(result_dist_to_use)
                else:
                    color_points = result_dist_to_use

                utils.scatter3d(result_parameters_flat_3d[:, param_permut[0]], result_parameters_flat_3d[:, param_permut[1]], result_parameters_flat_3d[:, param_permut[2]], s=size_normal_points, c=color_points, xlabel=parameter_names_sorted[param_permut[0]], ylabel=parameter_names_sorted[param_permut[1]], zlabel=parameter_names_sorted[param_permut[2]], title=title, ax_handle=ax)

                utils.scatter3d(result_parameters_flat_3d[best_points_result_dist_to_use, param_permut[0]], result_parameters_flat_3d[best_points_result_dist_to_use, param_permut[1]], result_parameters_flat_3d[best_points_result_dist_to_use, param_permut[2]], c='r', s=size_best_points, ax_handle=ax)

                if savefigs:
                    dataio.save_current_figure('scatter3d_%s_%s%s_{label}_{unique_id}.pdf' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file))

                if savemovies:
                    try:
                        utils.rotate_plot3d(ax, dataio.create_formatted_filename('scatter3d_%s_%s%s_{label}_{unique_id}.mp4' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file)), bitrate=8000, min_duration=8)
                        utils.rotate_plot3d(ax, dataio.create_formatted_filename('scatter3d_%s_%s%s_{label}_{unique_id}.gif' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file)), nb_frames=30, min_duration=8)
                    except Exception:
                        # Most likely wrong aggregator...
                        print "failed when creating movies for ", result_dist_to_use_name


                if False and savefigs:
                    ax.view_init(azim=90, elev=10)
                    dataio.save_current_figure('scatter3d_view2_%s_%s%s_{label}_{unique_id}.pdf' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file))

                # plt.close('all')

            print "Parameters: %s" % ', '.join(parameter_names_sorted)
            print "Best points, %s:" % title
            print '\n'.join([str_best_params(best_i, result_dist_to_use) for best_i in best_points_result_dist_to_use])


    # all_args = data_pbs.loaded_data['args_list']
    variables_to_save = ['parameter_names_sorted', 'all_args_arr', 'all_repeats_completed', 'filter_data']

    if savedata:
        dataio.save_variables_default(locals(), variables_to_save)
        dataio.make_link_output_to_dropbox(dropbox_current_experiment_folder='gorgo11_sequential_fitmixturemodel')


    plt.show()

    return locals()
def plots_fit_mixturemodels_random(data_pbs, generator_module=None):
    """
        Reload runs from PBS
    """

    #### SETUP
    #
    savefigs = True
    savedata = True

    colormap = None  # or 'cubehelix'
    plt.rcParams["font.size"] = 16
    #
    #### /SETUP

    print "Order parameters: ", data_pbs.dataset_infos["parameters"]
    # parameters: M, ratio_conj, sigmax

    # Extract data
    T_space = data_pbs.loaded_data["datasets_list"][0]["T_space"]

    result_responses_flat = np.array(data_pbs.dict_arrays["result_responses"]["results_flat"])
    result_targets_flat = np.array(data_pbs.dict_arrays["result_target"]["results_flat"])
    result_nontargets_flat = np.array(data_pbs.dict_arrays["result_nontargets"]["results_flat"])

    result_parameters_flat = np.array(data_pbs.dict_arrays["result_responses"]["parameters_flat"])
    all_repeats_completed = data_pbs.dict_arrays["result_responses"]["repeats_completed"]

    all_args_arr = np.array(data_pbs.loaded_data["args_list"])

    M_space = data_pbs.loaded_data["parameters_uniques"]["M"]
    ratio_conj_space = data_pbs.loaded_data["parameters_uniques"]["ratio_conj"]
    sigmax_space = data_pbs.loaded_data["parameters_uniques"]["sigmax"]
    alpha_space = data_pbs.loaded_data["parameters_uniques"]["alpha"]
    trecall_space = data_pbs.loaded_data["parameters_uniques"]["fixed_cued_feature_time"]

    num_repetitions = generator_module.num_repetitions
    parameter_names_sorted = data_pbs.dataset_infos["parameters"]

    dataio = DataIO(
        output_folder=generator_module.pbs_submission_infos["simul_out_dir"] + "/outputs/",
        label="global_" + dataset_infos["save_output_filename"],
    )

    ##### Because of lazyness, the responses are weird.
    # Each run is for a given trecall. But we run N items= 1 .. Nmax anyway
    # so if trecall > N, you have np.nan
    # => Need to reconstruct the thing properly, to have lower triangle of Nitem x Trecall filled
    # Also, trecall is the actual Time. Hence we need to change its meaning to be Tmax- (trecall + 1) or whatever.

    # Load ground truth
    data_gorgo11_sequ = load_experimental_data.load_data_gorgo11_sequential(fit_mixture_model=True)

    ## Filter everything with repeats_completed == num_repet and trecall=last
    filter_data = (result_parameters_flat[:, 0] == (T_space.max() - 1)) & (all_repeats_completed == num_repetitions - 1)
    result_parameters_flat = result_parameters_flat[filter_data]
    result_responses_flat = result_responses_flat[filter_data]
    result_targets_flat = result_targets_flat[filter_data]
    result_nontargets_flat = result_nontargets_flat[filter_data]
    all_args_arr = all_args_arr[filter_data]
    all_repeats_completed = all_repeats_completed[filter_data]

    print "Size post-filter: ", result_parameters_flat.shape[0]

    def str_best_params(best_i, result_dist_to_use):
        return (
            " ".join(
                [
                    "%s %.4f" % (parameter_names_sorted[param_i], result_parameters_flat[best_i, param_i])
                    for param_i in xrange(len(parameter_names_sorted))
                ]
            )
            + " >> %f" % result_dist_to_use[best_i]
        )

    # all_args = data_pbs.loaded_data['args_list']
    variables_to_save = ["parameter_names_sorted", "all_args_arr", "all_repeats_completed", "filter_data"]

    if savedata:
        dataio.save_variables_default(locals(), variables_to_save)
        dataio.make_link_output_to_dropbox(dropbox_current_experiment_folder="gorgo11_sequential_fitmixturemodel")

    plt.show()

    return locals()
def plots_fit_mixturemodels_random(data_pbs, generator_module=None):
    '''
        Reload runs from PBS

        !!!
        IMPORTANT LOOK AT ME
        !!!
    '''

    #### SETUP
    #
    savefigs = True
    savedata = True
    savemovies = False

    do_bays09 = True
    do_gorgo11 = True

    plots_scatter3d = True
    plots_scatter_per_T = False
    plots_flat_sorted_performance = False
    plots_memorycurves_fits_best = True

    # do_relaunch_bestparams_pbs = True

    colormap = None  # or 'cubehelix'
    plt.rcParams['font.size'] = 16
    #
    #### /SETUP

    print "Order parameters: ", data_pbs.dataset_infos['parameters']
    # parameters: M, ratio_conj, sigmax

    # Extract data
    T_space = data_pbs.loaded_data['datasets_list'][0]['T_space']

    result_em_fits_flat = np.array(data_pbs.dict_arrays['result_em_fits']['results_flat'])
    result_precisions_flat = np.array(data_pbs.dict_arrays['result_all_precisions']['results_flat'])
    result_dist_bays09_flat = np.array(data_pbs.dict_arrays['result_dist_bays09']['results_flat'])
    result_dist_gorgo11_flat = np.array(data_pbs.dict_arrays['result_dist_gorgo11']['results_flat'])
    result_dist_bays09_emmixt_KL = np.array(data_pbs.dict_arrays['result_dist_bays09_emmixt_KL']['results_flat'])
    result_dist_gorgo11_emmixt_KL = np.array(data_pbs.dict_arrays['result_dist_gorgo11_emmixt_KL']['results_flat'])
    result_parameters_flat = np.array(data_pbs.dict_arrays['result_em_fits']['parameters_flat'])
    all_repeats_completed = data_pbs.dict_arrays['result_em_fits']['repeats_completed']

    all_args = data_pbs.loaded_data['args_list']
    all_args_arr = np.array(all_args)

    sigmaoutput_space = data_pbs.loaded_data['parameters_uniques']['sigma_output']
    sigmax_space = data_pbs.loaded_data['parameters_uniques']['sigmax']
    ratio_space = data_pbs.loaded_data['parameters_uniques']['sigmax']
    num_repetitions = generator_module.num_repetitions
    parameter_names_sorted = data_pbs.dataset_infos['parameters']

    dataio = DataIO(output_folder=generator_module.pbs_submission_infos['simul_out_dir'] + '/outputs/', label='global_' + dataset_infos['save_output_filename'])

    # Load bays09
    # data_bays09 = load_experimental_data.load_data_bays09(fit_mixture_model=True)
    # bays09_nitems = data_bays09['data_to_fit']['n_items']
    # bays09_em_target = np.nan*np.empty((bays09_nitems.max(), 4))  #kappa, prob_target, prob_nontarget, prob_random
    # bays09_em_target[bays09_nitems - 1] = data_bays09['em_fits_nitems_arrays']['mean'].T
    # bays09_emmixt_target = bays09_em_target[:, 1:]

    ## Filter everything with sigma_output > 1.0 and repeats_completed == num_repet
    filter_data = (result_parameters_flat[:, -1] < 1.0) & (all_repeats_completed == num_repetitions - 1)
    result_em_fits_flat = result_em_fits_flat[filter_data]
    result_precisions_flat = result_precisions_flat[filter_data]
    result_dist_bays09_flat = result_dist_bays09_flat[filter_data]
    result_dist_gorgo11_flat = result_dist_gorgo11_flat[filter_data]
    result_dist_bays09_emmixt_KL = result_dist_bays09_emmixt_KL[filter_data]
    result_dist_gorgo11_emmixt_KL = result_dist_gorgo11_emmixt_KL[filter_data]
    result_parameters_flat = result_parameters_flat[filter_data]
    all_args_arr = all_args_arr[filter_data]
    all_repeats_completed = all_repeats_completed[filter_data]

    # for _result_dist in ['result_em_fits_flat', 'result_precisions_flat', 'result_dist_bays09_flat', 'result_dist_gorgo11_flat', 'result_dist_bays09_emmixt_KL', 'result_dist_gorgo11_emmixt_KL', 'result_parameters_flat']:
    #     locals()[key] = locals()[key][filter_sigmaout]
    #     # exec("%s = %s[%s]" % (_result_dist, _result_dist, 'filter_sigmaout'))

    ## Compute some stuff

    result_em_fits_all_avg = utils.nanmean(result_em_fits_flat, axis=-1)
    result_em_kappa_allT = result_em_fits_all_avg[..., 0]
    result_em_emmixt_allT = result_em_fits_all_avg[..., 1:4]

    result_precisions_all_avg = utils.nanmean(result_precisions_flat, axis=-1)

    ##### Distance to Bays09
    result_dist_bays09_allT_avg = utils.nanmean(result_dist_bays09_flat, axis=-1)
    result_dist_bays09_emmixt_KL_allT_avg = utils.nanmean(result_dist_bays09_emmixt_KL, axis=-1)
    result_dist_bays09_kappa_allT = result_dist_bays09_allT_avg[..., 0]

    result_dist_bays09_kappa_sumT = np.nansum(result_dist_bays09_kappa_allT, axis=-1)
    result_dist_bays09_logkappa_sumT = np.log(result_dist_bays09_kappa_sumT)
    result_dist_bays09_emmixt_KL_sumT = np.nansum(result_dist_bays09_emmixt_KL_allT_avg, axis=-1)

    # combined versions
    result_dist_bays09_both_normalised = result_dist_bays09_emmixt_KL_sumT/np.max(result_dist_bays09_emmixt_KL_sumT) + result_dist_bays09_kappa_sumT/np.max(result_dist_bays09_kappa_sumT)
    result_dist_bays09_logkappamixtKL = result_dist_bays09_logkappa_sumT + result_dist_bays09_emmixt_KL_sumT
    result_dist_bays09_logkappamixtKL_normalised = result_dist_bays09_logkappa_sumT/np.max(result_dist_bays09_logkappa_sumT) + result_dist_bays09_emmixt_KL_sumT/np.max(result_dist_bays09_emmixt_KL_sumT)

    result_dist_bays09_logkappa_sumT_forand = result_dist_bays09_logkappa_sumT - np.min(result_dist_bays09_logkappa_sumT)*np.sign(np.min(result_dist_bays09_logkappa_sumT))
    result_dist_bays09_logkappa_sumT_forand /= np.max(result_dist_bays09_logkappa_sumT_forand)

    result_dist_bays09_emmixt_KL_sumT_forand = result_dist_bays09_emmixt_KL_sumT - np.min(result_dist_bays09_emmixt_KL_sumT)*np.sign(np.min(result_dist_bays09_emmixt_KL_sumT))
    result_dist_bays09_emmixt_KL_sumT_forand /= np.max(result_dist_bays09_emmixt_KL_sumT_forand)

    result_dist_bays09_logkappamixtKL_AND = 1. - (1. - result_dist_bays09_logkappa_sumT_forand)*(1. - result_dist_bays09_emmixt_KL_sumT_forand)

    # Mask kappa for bad performance
    # result_dist_bays09_kappa_sumT_masked = np.ma.masked_greater(result_dist_bays09_kappa_sumT, 2*np.median(result_dist_bays09_kappa_sumT))
    # result_dist_bays09_logkappa_sumT_masked = np.ma.masked_greater(result_dist_bays09_logkappa_sumT, 2*np.median(result_dist_bays09_logkappa_sumT))
    # result_dist_bays09_emmixt_KL_sumT_masked = np.ma.masked_greater(result_dist_bays09_emmixt_KL_sumT, 2*np.median(result_dist_bays09_emmixt_KL_sumT))
    # result_dist_bays09_both_normalised_mult_masked = 1-(1. - result_dist_bays09_emmixt_KL_sumT_masked/np.max(result_dist_bays09_emmixt_KL_sumT_masked))*(1. - result_dist_bays09_kappa_sumT_masked/np.max(result_dist_bays09_kappa_sumT_masked))

    ##### Distance to Gorgo11
    result_dist_gorgo11_allT_avg = utils.nanmean(result_dist_gorgo11_flat, axis=-1)
    result_dist_gorgo11_emmixt_KL_allT_avg = utils.nanmean(result_dist_gorgo11_emmixt_KL, axis=-1)
    result_dist_gorgo11_kappa_allT = result_dist_gorgo11_allT_avg[..., 0]

    result_dist_gorgo11_kappa_sumT = np.nansum(result_dist_gorgo11_kappa_allT, axis=-1)
    result_dist_gorgo11_logkappa_sumT = np.log(result_dist_gorgo11_kappa_sumT)
    result_dist_gorgo11_emmixt_KL_sumT = np.nansum(result_dist_gorgo11_emmixt_KL_allT_avg, axis=-1)
    result_dist_gorgo11_emmixt_KL_sumT25 = np.nansum(result_dist_gorgo11_emmixt_KL_allT_avg[:, 1:], axis=-1)
    result_dist_gorgo11_logkappa_sumT25 = np.log(np.nansum(result_dist_gorgo11_kappa_allT[..., 1:], axis=-1))

    # combined versions
    result_dist_gorgo11_both_normalised = result_dist_gorgo11_emmixt_KL_sumT/np.max(result_dist_gorgo11_emmixt_KL_sumT) + result_dist_gorgo11_kappa_sumT/np.max(result_dist_gorgo11_kappa_sumT)
    result_dist_gorgo11_logkappamixtKL = result_dist_gorgo11_logkappa_sumT + result_dist_gorgo11_emmixt_KL_sumT
    result_dist_gorgo11_logkappamixtKL_normalised = result_dist_gorgo11_logkappa_sumT/np.max(result_dist_gorgo11_logkappa_sumT) + result_dist_gorgo11_emmixt_KL_sumT/np.max(result_dist_gorgo11_emmixt_KL_sumT)

    result_dist_gorgo11_logkappa_sumT_forand = result_dist_gorgo11_logkappa_sumT - np.min(result_dist_gorgo11_logkappa_sumT)*np.sign(np.min(result_dist_gorgo11_logkappa_sumT))
    result_dist_gorgo11_logkappa_sumT_forand /= np.max(result_dist_gorgo11_logkappa_sumT_forand)


    result_dist_gorgo11_logkappa_sumT25_forand = result_dist_gorgo11_logkappa_sumT25 - np.min(result_dist_gorgo11_logkappa_sumT25)*np.sign(np.min(result_dist_gorgo11_logkappa_sumT25))
    result_dist_gorgo11_logkappa_sumT25_forand /= np.max(result_dist_gorgo11_logkappa_sumT25_forand)

    result_dist_gorgo11_emmixt_KL_sumT_forand = result_dist_gorgo11_emmixt_KL_sumT - np.min(result_dist_gorgo11_emmixt_KL_sumT)*np.sign(np.min(result_dist_gorgo11_emmixt_KL_sumT))
    result_dist_gorgo11_emmixt_KL_sumT_forand /= np.max(result_dist_gorgo11_emmixt_KL_sumT_forand)

    result_dist_gorgo11_emmixt_KL_sumT25_forand = result_dist_gorgo11_emmixt_KL_sumT25 - np.min(result_dist_gorgo11_emmixt_KL_sumT25)*np.sign(np.min(result_dist_gorgo11_emmixt_KL_sumT25))
    result_dist_gorgo11_emmixt_KL_sumT25_forand /= np.max(result_dist_gorgo11_emmixt_KL_sumT25_forand)

    result_dist_gorgo11_logkappamixtKL_AND = 1. - (1. - result_dist_gorgo11_logkappa_sumT_forand)*(1. - result_dist_gorgo11_emmixt_KL_sumT_forand)

    result_dist_gorgo11_logkappa25mixtKL_AND = 1. - (1. - result_dist_gorgo11_logkappa_sumT25_forand)*(1. - result_dist_gorgo11_emmixt_KL_sumT25_forand)

    def str_best_params(best_i, result_dist_to_use):
        return ' '.join(["%s %.4f" % (parameter_names_sorted[param_i], result_parameters_flat[best_i, param_i]) for param_i in xrange(len(parameter_names_sorted))]) + ' >> %f' % result_dist_to_use[best_i]

    if plots_scatter3d:
        nb_best_points = 30
        size_normal_points = 8
        size_best_points = 50

        def plot_scatter(all_vars, result_dist_to_use_name, title='', log_color=True, downsampling=1, label_file=''):

            result_dist_to_use = all_vars[result_dist_to_use_name]
            result_parameters_flat = all_vars['result_parameters_flat']

            # Filter if downsampling
            filter_downsampling = np.arange(0, result_dist_to_use.size, downsampling)
            result_dist_to_use = result_dist_to_use[filter_downsampling]
            result_parameters_flat = result_parameters_flat[filter_downsampling]

            best_points_result_dist_to_use = np.argsort(result_dist_to_use)[:nb_best_points]

            # Construct all permutations of 3 parameters, for 3D scatters
            params_permutations = set([tuple(np.sort(np.random.choice(result_parameters_flat.shape[-1], 3, replace=False)).tolist()) for i in xrange(1000)])

            for param_permut in params_permutations:
                fig = plt.figure()
                ax = Axes3D(fig)

                # One plot per parameter permutation
                if log_color:
                    color_points = np.log(result_dist_to_use)
                else:
                    color_points = result_dist_to_use

                utils.scatter3d(result_parameters_flat[:, param_permut[0]], result_parameters_flat[:, param_permut[1]], result_parameters_flat[:, param_permut[2]], s=size_normal_points, c=color_points, xlabel=parameter_names_sorted[param_permut[0]], ylabel=parameter_names_sorted[param_permut[1]], zlabel=parameter_names_sorted[param_permut[2]], title=title, ax_handle=ax)

                utils.scatter3d(result_parameters_flat[best_points_result_dist_to_use, param_permut[0]], result_parameters_flat[best_points_result_dist_to_use, param_permut[1]], result_parameters_flat[best_points_result_dist_to_use, param_permut[2]], c='r', s=size_best_points, ax_handle=ax)

                if savefigs:
                    dataio.save_current_figure('scatter3d_%s_%s%s_{label}_{unique_id}.pdf' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file))

                if savemovies:
                    try:
                        utils.rotate_plot3d(ax, dataio.create_formatted_filename('scatter3d_%s_%s%s_{label}_{unique_id}.mp4' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file)), bitrate=8000, min_duration=8)
                        utils.rotate_plot3d(ax, dataio.create_formatted_filename('scatter3d_%s_%s%s_{label}_{unique_id}.gif' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file)), nb_frames=30, min_duration=8)
                    except Exception:
                        # Most likely wrong aggregator...
                        print "failed when creating movies for ", result_dist_to_use_name


                if False and savefigs:
                    ax.view_init(azim=90, elev=10)
                    dataio.save_current_figure('scatter3d_view2_%s_%s%s_{label}_{unique_id}.pdf' % (result_dist_to_use_name, '_'.join([parameter_names_sorted[i] for i in param_permut]), label_file))

                # plt.close('all')

            print "Parameters: %s" % ', '.join(parameter_names_sorted)
            print "Best points, %s:" % title
            print '\n'.join([str_best_params(best_i, result_dist_to_use) for best_i in best_points_result_dist_to_use])



        #### BAYS 09
        if do_bays09:
            # Distance for log kappa, all T
            plot_scatter(locals(), 'result_dist_bays09_logkappa_sumT', 'Bays09 kappa all T', log_color=False)

            # # Distance for em fits, all T, KL distance
            plot_scatter(locals(), 'result_dist_bays09_emmixt_KL_sumT', 'Bays09 em fits, sum T, KL', log_color=False)

            # Distance for product of normalised em fits KL + normalised log kappa, all T
            plot_scatter(locals(), 'result_dist_bays09_logkappamixtKL', 'Bays09 em fits KL, log kappa')

            # Distance for AND normalised em fits KL + log kappa
            plot_scatter(locals(), 'result_dist_bays09_logkappamixtKL_AND', 'Bays09 em fits KL AND log kappa')

        #### Gorgo 11
        if do_gorgo11:
            # Distance for product of normalised em fits KL + normalised log kappa, all T
            plot_scatter(locals(), 'result_dist_gorgo11_logkappamixtKL', 'Gorgo11 em fits KL, log kappa')

            # Distance for AND normalised em fits KL + log kappa
            plot_scatter(locals(), 'result_dist_gorgo11_logkappamixtKL_AND', 'Gorgo11 em fits KL AND log kappa')

            # Distance for logkappa
            plot_scatter(locals(), 'result_dist_gorgo11_logkappa_sumT', 'Gorgo11 log kappa all T', log_color=False)

            # Distance for EM mixture proportions
            plot_scatter(locals(), 'result_dist_gorgo11_emmixt_KL_sumT', 'Gorgo11 em fits, sum T, KL', log_color=False)



    if plots_flat_sorted_performance:
        result_dist_to_try = []
        if do_bays09:
            result_dist_to_try.extend(['result_dist_bays09_logkappamixtKL_AND', 'result_dist_bays09_logkappamixtKL'])
        if do_gorgo11:
            result_dist_to_try.extend(['result_dist_gorgo11_logkappamixtKL_AND', 'result_dist_gorgo11_logkappamixtKL'])

        for result_dist in result_dist_to_try:
            order_indices = np.argsort(locals()[result_dist])[::-1]

            f, axes = plt.subplots(2, 1)
            axes[0].plot(np.arange(4) + result_parameters_flat[order_indices]/np.max(result_parameters_flat[order_indices], axis=0))
            axes[0].legend(parameter_names_sorted, loc='upper left')
            axes[0].set_ylabel('Parameters')
            axes[1].plot(locals()[result_dist][order_indices])
            axes[1].set_ylabel(result_dist.split('result_dist_')[-1])
            axes[0].set_title('Distance ordered ' + result_dist.split('result_dist_')[-1])
            f.canvas.draw()

            if savefigs:
                dataio.save_current_figure('plot_sortedperf_full_%s_{label}_{unique_id}.pdf' % (result_dist))

        ## Extra plot for logkappamixtKL_AND, it seems well behaved

        def plot_flat_best(all_vars, result_name, order_indices_filter, filter_goodAND, ordering='fitness'):
            f = plt.figure()
            axp1 = plt.subplot2grid((3, 2), (0, 0))
            axp2 = plt.subplot2grid((3, 2), (0, 1))
            axp3 = plt.subplot2grid((3, 2), (1, 0))
            axp4 = plt.subplot2grid((3, 2), (1, 1))
            axfit = plt.subplot2grid((3, 2), (2, 0), colspan=2)

            axp1.plot(result_parameters_flat[filter_goodAND][order_indices_filter, 0])
            axp1.set_title(parameter_names_sorted[0])
            axp2.plot(result_parameters_flat[filter_goodAND][order_indices_filter, 1], 'g')
            axp2.set_title(parameter_names_sorted[1])
            axp3.plot(result_parameters_flat[filter_goodAND][order_indices_filter, 2], 'r')
            axp3.set_title(parameter_names_sorted[2])
            axp4.plot(result_parameters_flat[filter_goodAND][order_indices_filter, 3], 'k')
            axp4.set_title(parameter_names_sorted[3])

            axfit.plot(all_vars[result_name][filter_goodAND][order_indices_filter])
            axfit.set_ylabel('bays09_logkappamixtKL_AND')
            plt.suptitle('Distance ordered bays09_logkappamixtKL_AND')

            if savefigs:
                dataio.save_current_figure('plot_sortedperf_best_%s_%s_{label}_{unique_id}.pdf' % (result_name, ordering))

        if do_bays09:
            filter_goodAND = result_dist_bays09_logkappamixtKL_AND < 0.2

            # First order them by fitness
            order_indices_filter = np.argsort(result_dist_bays09_logkappamixtKL_AND[filter_goodAND])[::-1]
            plot_flat_best(locals(), 'result_dist_bays09_logkappamixtKL_AND', order_indices_filter, filter_goodAND, 'fitness')

            # Then by M, to see if there is some structure
            order_indices_filter = np.argsort(result_parameters_flat[filter_goodAND, 0])
            plot_flat_best(locals(), 'result_dist_bays09_logkappamixtKL_AND', order_indices_filter, filter_goodAND, 'M')

        if do_gorgo11:
            filter_goodAND = result_dist_gorgo11_logkappamixtKL_AND < 0.5

            # First order them by fitness
            order_indices_filter = np.argsort(result_dist_gorgo11_logkappamixtKL_AND[filter_goodAND])[::-1]
            plot_flat_best(locals(), 'result_dist_gorgo11_logkappamixtKL_AND', order_indices_filter, filter_goodAND, 'fitness')

            # Then by M, to see if there is some structure
            order_indices_filter = np.argsort(result_parameters_flat[filter_goodAND, 0])
            plot_flat_best(locals(), 'result_dist_gorgo11_logkappamixtKL_AND', order_indices_filter, filter_goodAND, 'M')

            # dist_cmaes_result = np.sum((result_parameters_flat - np.array([75, 1.0, 0.1537, 0.2724]))**2., axis=-1)
            # filter_close_cmaes_result = np.argsort(dist_cmaes_result)[:20]
            # order_indices_filter = np.argsort(result_dist_gorgo11_logkappamixtKL_AND[filter_close_cmaes_result])[::-1]
            # plot_flat_best(locals(), 'result_dist_gorgo11_logkappamixtKL_AND', order_indices_filter, filter_close_cmaes_result, 'Like current CMA/ES run')


    if plots_scatter_per_T:
        for T_i, T in enumerate(T_space):

            # Kappa per T, fit to Bays09
            result_dist_bays09_kappa_currT = result_dist_bays09_kappa_allT[:, T_i]
            result_dist_bays09_kappa_currT_masked = mask_outliers(result_dist_bays09_kappa_currT)

            plot_scatter(locals(), 'result_dist_bays09_kappa_currT_masked', 'kappa T %d masked' % T, label_file="T{}".format(T))

            # EM Mixt per T, fit to Bays09
            result_dist_bays09_emmixt_KL_currT = result_dist_bays09_emmixt_KL_allT_avg[:, T_i]
            result_dist_bays09_emmixt_KL_currT_masked = mask_outliers(result_dist_bays09_emmixt_KL_currT)

            plot_scatter(locals(), 'result_dist_bays09_emmixt_KL_currT_masked', 'KL EM mixt T %d masked' % T, label_file="T{}".format(T), log_color=False)


    if plots_memorycurves_fits_best:

        data_dir = None
        if not os.environ.get('WORKDIR_DROP'):
            data_dir = '../experimental_data/'

        plotting_parameters = launchers_memorycurves_marginal_fi.load_prepare_datasets(data_dir = data_dir)

        def plot_memorycurves_fits(all_vars, result_dist_to_use_name, nb_best_points=10):
            result_dist_to_use = all_vars[result_dist_to_use_name]

            best_points_result_dist_to_use = np.argsort(result_dist_to_use)[:nb_best_points]

            for best_point_index in best_points_result_dist_to_use:
                print "extended plot for: " + str_best_params(best_point_index, result_dist_to_use)

                # Update arguments
                all_args_arr[best_point_index].update(dict(zip(parameter_names_sorted, result_parameters_flat[best_point_index])))
                packed_data = dict(T_space=T_space, result_em_fits=result_em_fits_flat[best_point_index], all_parameters=all_args_arr[best_point_index])

                plotting_parameters['suptitle'] = result_dist_to_use_name
                plotting_parameters['reuse_axes'] = False
                if savefigs:
                    packed_data['dataio'] = dataio

                launchers_memorycurves_marginal_fi.do_memory_plots(packed_data, plotting_parameters)


        plot_memorycurves_fits(locals(), 'result_dist_bays09_logkappamixtKL_AND', nb_best_points=3)

        plot_memorycurves_fits(locals(), 'result_dist_gorgo11_logkappamixtKL_AND', nb_best_points=3)
        # plot_memorycurves_fits(locals(), 'result_dist_gorgo11_logkappamixtKL', nb_best_points=3)

        plot_memorycurves_fits(locals(), 'result_dist_gorgo11_logkappa25mixtKL_AND', nb_best_points=3)

        # plot_memorycurves_fits(locals(), 'result_dist_gorgo11_logkappa_sumT', nb_best_points=3)




    # # Interpolate
    # if plots_interpolate:

    #     sigmax_target = 0.9

    #     M_interp_space = np.arange(6, 625, 5)
    #     ratio_interp_space = np.linspace(0.01, 1.0, 50)
    #     # sigmax_interp_space = np.linspace(0.01, 1.0, 50)
    #     sigmax_interp_space = np.array([sigmax_target])
    #     params_crossspace = np.array(utils.cross(M_interp_space, ratio_interp_space, sigmax_interp_space))

    #     interpolated_data = rbf_interpolator(params_crossspace[:, 0], params_crossspace[:, 1], params_crossspace[:, 2]).reshape((M_interp_space.size, ratio_interp_space.size))

    #     utils.pcolor_2d_data(interpolated_data, M_interp_space, ratio_interp_space, 'M', 'ratio', 'interpolated, fixing sigmax= %.2f' % sigmax_target)

    #     points_closeby = ((result_parameters_flat[:, 2] - sigmax_target)**2)< 0.01
    #     plt.figure()
    #     # plt.imshow(interpolated_data, extent=(M_interp_space.min(), M_interp_space.max(), ratio_interp_space.min(), ratio_interp_space.max()))
    #     plt.imshow(interpolated_data)
    #     plt.scatter(result_parameters_flat[points_closeby, 0], result_parameters_flat[points_closeby, 1], s=100, c=result_fitexperiments_bic_avg[points_closeby], marker='o')


    # if plot_per_ratio:
    #     # Plot the evolution of loglike as a function of sigmax, with std shown
    #     for ratio_conj_i, ratio_conj in enumerate(ratio_space):
    #         ax = utils.plot_mean_std_area(sigmax_space, result_log_posterior_mean[ratio_conj_i], result_log_posterior_std[ratio_conj_i])

    #         ax.get_figure().canvas.draw()

    #         if savefigs:
    #             dataio.save_current_figure('results_fitexp_%s_loglike_ratioconj%.2f_{label}_global_{unique_id}.pdf' % (exp_dataset, ratio_conj))



    # all_args = data_pbs.loaded_data['args_list']
    variables_to_save = ['parameter_names_sorted', 'all_args_arr', 'all_repeats_completed', 'filter_data']

    if savedata:
        dataio.save_variables_default(locals(), variables_to_save)
        dataio.make_link_output_to_dropbox(dropbox_current_experiment_folder='sigmaoutput_normalisedsigmax_random')


    plt.show()

    return locals()