def plot_std(distribution): from mjhmc.samplers.markov_jump_hmc import MarkovJumpHMC, ControlHMC from mjhmc.misc.autocor import calculate_autocorrelation # change this!!! steps = int(1E4) plt.clf() c_ac_1 = calculate_autocorrelation(ControlHMC, distribution, num_steps=steps, sample_steps=1, half_window=True, **std_param1) m_ac_1 = calculate_autocorrelation(MarkovJumpHMC, distribution, num_steps=steps, sample_steps=1, half_window=True, **std_param1) c_ac_2 = calculate_autocorrelation(ControlHMC, distribution, num_steps=steps, sample_steps=1, half_window=True, **std_param2) m_ac_2 = calculate_autocorrelation(MarkovJumpHMC, distribution, num_steps=steps, sample_steps=1, half_window=True, **std_param2) # c1_trunc = c_ac_1.loc[:, 'autocorrelation'] < 0.0 # c2_trunc = c_ac_2.loc[:, 'autocorrelation'] < 0.0 # m1_trunc = m_ac_1.loc[:, 'autocorrelation'] < 0.0 # m2_trunc = c_ac_2.loc[:, 'autocorrelation'] < 0.0 # trunc_idx = max(c1_trunc[c1_trunc].index[0], # c2_trunc[c2_trunc].index[0], # m1_trunc[m1_trunc].index[0], # m2_trunc[m2_trunc].index[0]) # c_ac_1 = c_ac_1.loc[:trunc_idx] # c_ac_2 = c_ac_2.loc[:trunc_idx] # m_ac_1 = m_ac_1.loc[:trunc_idx] # m_ac_2 = m_ac_2.loc[:trunc_idx] c_ac_1.index = c_ac_1['num grad'] c_ac_2.index = c_ac_1['num grad'] m_ac_1.index = c_ac_1['num grad'] m_ac_2.index = c_ac_1['num grad'] c_ac_1.autocorrelation.plot(label=r"Control HMC $\beta = 0.1$") m_ac_1.autocorrelation.plot(label=r"Markov Jump HMC $\beta = 0.1$") c_ac_2.autocorrelation.plot(label=r"Control HMC $\beta = 0.8$") m_ac_2.autocorrelation.plot(label=r"Markov Jump HMC $\beta = 0.8$") plt.xlabel("Gradient Evaluations") plt.ylabel("Autocorrelation") plt.title("{}D {}".format(distribution.ndims, type(distribution).__name__)) plt.legend() plt.savefig("{}_std.pdf".format(type(distribution).__name__))
def plot_std(distribution): # change this!!! steps = int(1E4) plt.clf() c_ac_1 = calculate_autocorrelation(ControlHMC, distribution, num_steps=steps, sample_steps=1, half_window=True, **std_param1) m_ac_1 = calculate_autocorrelation(MarkovJumpHMC, distribution, num_steps=steps, sample_steps=1, half_window=True, **std_param1) c_ac_2 = calculate_autocorrelation(ControlHMC, distribution, num_steps=steps, sample_steps=1, half_window=True, **std_param2) m_ac_2 = calculate_autocorrelation(MarkovJumpHMC, distribution, num_steps=steps, sample_steps=1, half_window=True, **std_param2) # c1_trunc = c_ac_1.loc[:, 'autocorrelation'] < 0.0 # c2_trunc = c_ac_2.loc[:, 'autocorrelation'] < 0.0 # m1_trunc = m_ac_1.loc[:, 'autocorrelation'] < 0.0 # m2_trunc = c_ac_2.loc[:, 'autocorrelation'] < 0.0 # trunc_idx = max(c1_trunc[c1_trunc].index[0], # c2_trunc[c2_trunc].index[0], # m1_trunc[m1_trunc].index[0], # m2_trunc[m2_trunc].index[0]) # c_ac_1 = c_ac_1.loc[:trunc_idx] # c_ac_2 = c_ac_2.loc[:trunc_idx] # m_ac_1 = m_ac_1.loc[:trunc_idx] # m_ac_2 = m_ac_2.loc[:trunc_idx] c_ac_1.index = c_ac_1['num grad'] c_ac_2.index = c_ac_1['num grad'] m_ac_1.index = c_ac_1['num grad'] m_ac_2.index = c_ac_1['num grad'] c_ac_1.autocorrelation.plot(label=r"Control HMC $\beta = 0.1$") m_ac_1.autocorrelation.plot(label=r"Markov Jump HMC $\beta = 0.1$") c_ac_2.autocorrelation.plot(label=r"Control HMC $\beta = 0.8$") m_ac_2.autocorrelation.plot(label=r"Markov Jump HMC $\beta = 0.8$") plt.xlabel("Gradient Evaluations") plt.ylabel("Autocorrelation") plt.title("{}D {}".format(distribution.ndims, type(distribution).__name__)) plt.legend() plt.savefig("{}_std.pdf".format(type(distribution).__name__))
def obj_func(sampler, distr, job_id, **kwargs): num_target_grad_evals = grad_evals[type(distr).__name__] default_args = { "num_grad_steps": num_target_grad_evals, "sample_steps": 1, "num_steps": None, "half_window": True } kwargs = unpack_params(kwargs) kwargs.update(default_args) ac_df = calculate_autocorrelation(sampler, distr, **kwargs) n = ac_df['num grad'].values.astype(int) # necessary to keep curve_fit from borking normed_n = n / (0.5 * num_target_grad_evals) y = ac_df['autocorrelation'].values if use_exp: # wtf fit is mutating input somehow r1 = fit(normed_n.copy(), y.copy())[0] if debug: plot_fit(normed_n, y, r1, job_id, kwargs) return -r1 else: if np.isnan(np.sum(ac_df.autocorrelation.values)): return 11 * num_target_grad_evals for trial, target in enumerate(np.arange(0, 1, 0.1)): score = min_idx(ac_df, target) if score is not None: score += trial * num_target_grad_evals * 0.5 break if debug: plot_search_ac(ac_df['num grad'].values.astype(int), ac_df.autocorrelation.values, job_id, kwargs, score) return score or 5 * num_target_grad_evals
def benchmark_ac_batch_size_scaling(sampler_cls=MarkovJumpHMC, distribution_cls=Gaussian, ndims=2, n_samples=1000, n_itrs=10): """ Plots run time of sampling and calculating the autocorrelation of n_samples (fixed) against batch size normalizes to time per sample :param sampler_cls: sampler to test :param distribution_cls: distribution to test on :param ndims: dimension of the space, passed to distribution_cls upon instantiation :param n_samples: number of samples to generate :param n_itrs: number of trials to average :returns: None, makes a plot :rtype: None """ run_time = [] batch_sizes = np.arange(1, 500, 25) for n_batch in batch_sizes: distribution = distribution_cls(ndims=ndims, nbatch=n_batch) print "now doing batch size {}".format(n_batch) times = np.zeros(n_itrs) for idx in range(n_itrs): t_i = time.time() _ = calculate_autocorrelation(sampler_cls, distribution, num_steps=n_samples, half_window=True) t_f = time.time() times[idx] = t_f - t_i run_time.append(np.mean(times) / (n_samples * n_batch)) fig = plt.figure() axis = fig.add_subplot(111) axis.plot(batch_sizes, run_time) axis.set_yscale('log') axis.set_ylabel("log seconds per sample") axis.set_xlabel("batch size") axis.set_title("{} autocorrelation: compute time per sample versus batch size".format(sampler_cls.__name__))
def obj_func_helper(sampler, distr, unpack, kwargs): """ Helper function for the objective function :param sampler: sampler being tested. instance of mjhmc.samplers.markov_jump_hmc.HMCBase :param distr: distribution being used. instance of mjhmc.misc.distributions.Distribution :param unpack: boolean flag of whether to unpack params or not. :param kwargs: dictionary of kwargs passed from parent function :returns: parameters of fitted curves, graph data for computed autocorrelation, kwargs :rtype: tuple """ num_target_grad_evals = grad_evals[type(distr).__name__] default_args = { "num_grad_steps": num_target_grad_evals, "sample_steps": 1, "num_steps": None, "half_window": True, "use_cached_var": True } if unpack: kwargs = unpack_params(kwargs) if sampler.__name__ == 'MarkovJumpHMC': default_args["resample"] = False kwargs.update(default_args) print "Calculating autocorrelation for {} grad evals".format(num_target_grad_evals) ac_df = calculate_autocorrelation(sampler, distr, **kwargs) n_grad_evals = ac_df['num grad'].values.astype(int) # necessary to keep curve_fit from borking: THIS IS VERY IMPORTANT normed_n_grad_evals = n_grad_evals / (0.5 * num_target_grad_evals) autocor = ac_df['autocorrelation'].values print "Fitting curve" exp_coef, cos_coef = fit(normed_n_grad_evals.copy(), autocor.copy()) return cos_coef, normed_n_grad_evals, exp_coef, autocor, kwargs
def obj_func_helper(sampler, distr, unpack, kwargs): """ Helper function for the objective function :param sampler: sampler being tested. instance of mjhmc.samplers.markov_jump_hmc.HMCBase :param distr: distribution being used. instance of mjhmc.misc.distributions.Distribution :param unpack: boolean flag of whether to unpack params or not. :param kwargs: dictionary of kwargs passed from parent function :returns: parameters of fitted curves, graph data for computed autocorrelation, kwargs :rtype: tuple """ num_target_grad_evals = grad_evals[type(distr).__name__] default_args = { "num_grad_steps": num_target_grad_evals, "sample_steps": 1, "num_steps": None, "half_window": True, "use_cached_var": True } if unpack: kwargs = unpack_params(kwargs) if sampler.__name__ == 'MarkovJumpHMC': default_args["resample"] = False kwargs.update(default_args) print "Calculating autocorrelation for {} grad evals".format(num_target_grad_evals) # grad evals was previously cast to int, why? autocor, _, n_grad_evals = calculate_autocorrelation(sampler, distr, **kwargs) # necessary to keep curve_fit from borking: THIS IS VERY IMPORTANT normed_n_grad_evals = n_grad_evals / (0.5 * num_target_grad_evals) print "Fitting curve" exp_coef, cos_coef = tf_fit(normed_n_grad_evals.copy(), autocor.copy()) if SAVE_TRACE: formatted_time = time.strftime("%Y%m%d-%H%M%S") trace_name = '{}_{}'.format(type(distr).__name__, formatted_time) save_trace(normed_n_grad_evals, autocor, exp_coef, cos_coef, trace_name) return cos_coef, normed_n_grad_evals, exp_coef, autocor, kwargs
def autocorrelation_vs_n_samples(sampler_cls=MarkovJumpHMC, distribution_cls=Gaussian, ndims=2, n_batch=200, n_itrs=10): """ Plots run time of sampling and calculating the autocorrelation against n_samples, fixing batch size plots total time - not time per sample :param sampler_cls: sampler to test :param distribution_cls: distribution to test on :param ndims: dimension of the space, passed to distribution_cls upon instantiation :param n_samples: number of samples to generate :param n_itrs: number of trials to average :returns: None, makes a plot :rtype: None """ run_time = [] sample_sizes = np.arange(100, 10000, 250) for n_samples in izes: distribution = distribution_cls(ndims=ndims, nbatch=n_batch) print "now running for {} samples".format(n_samples) times = np.zeros(n_itrs) for idx in range(n_itrs): t_i = time.time() _ = calculate_autocorrelation(sampler_cls, distribution, num_steps=n_samples, half_window=True) t_f = time.time() times[idx] = t_f - t_i print "last run took time {}".format(times[-1]) run_time.append(np.mean(times)) fig = plt.figure() axis = fig.add_subplot(111) axis.plot(sample_sizes, run_time) axis.set_yscale('log') axis.set_ylabel("log seconds per sample") axis.set_xlabel("batch size") axis.set_title("{}: autocorrelation compute time by number of samples".format(sampler_cls.__name__))
from mjhmc.samplers.markov_jump_hmc import MarkovJumpHMC from mjhmc.misc.autocor import calculate_autocorrelation from mjhmc.misc.distributions import ProductOfT import numpy as np import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt from scipy.sparse import rand np.random.seed(2015) BURN_IN_STEPS = int(1E+3) BATCH_SIZE = int(1E+1) nbasis = 36 ndims = 36 W_half = rand(ndims,nbasis/2,density=0.35) W = np.concatenate((W_half.todense(),-W_half.todense()),axis=1) #Not doing this makes it bork crazily. Poor implementation of inheritance PoT = ProductOfT(nbasis=nbasis,ndims=ndims,W=W) ac_df = calculate_autocorrelation(MarkovJumpHMC,PoT,num_steps=1000) import IPython; IPython.embed() plt.plot(ac_df['autocorrelation']) plt.title('Random_Sparse_W_AC') plt.savefig('ac_rand_sparse_poe_w.png')
def plot_ac(distribution, control_params, mjhmc_params, lahmc_params, max_steps=3000, sample_steps=1, truncate=False, truncate_at=0.0, nuts=False, truncate_idx=None): """ distribution is an instantiated distribution object runs the sampler for max steps and then truncates the output to autocorrelation 0.5 throws an error if ac 0.5 is not reached """ from mjhmc.samplers.markov_jump_hmc import MarkovJumpHMC, ControlHMC from mjhmc.misc.autocor import calculate_autocorrelation, autocorrelation from mjhmc.misc.nutshell import sample_nuts_to_df # TODO: bring up to speed of DF-less calc autocor plt.clf() print('Calculating AutoCorrelation for ControlHMC') control_ac = calculate_autocorrelation(ControlHMC, distribution, num_steps=max_steps, sample_steps=sample_steps, half_window=True, use_cached_var=True, **control_params) print('Calculating AutoCorrelation for MJHMC') mjhmc_ac = calculate_autocorrelation(MarkovJumpHMC, distribution, num_steps=max_steps, sample_steps=sample_steps, half_window=True, resample=False, use_cached_var=True, **mjhmc_params) # lahmc_ac = calculate_autocorrelation(LAHMC, distribution, # num_steps=max_steps, # sample_steps=sample_steps, # half_window=True, # **lahmc_params) if nuts: print('Calculating AutoCorrelation for NUTS') nuts_df = sample_nuts_to_df(distribution, 100000, n_burn_in=10000) nuts_ac = autocorrelation(nuts_df, half_window=True) # find idx with autocorrelation closest to truncate_at if truncate: control_trunc = control_ac.loc[:, 'autocorrelation'] < truncate_at mjhmc_trunc = mjhmc_ac.loc[:, 'autocorrelation'] < truncate_at if truncate_idx is None: if nuts: nuts_trunc = nuts_ac.loc[:, 'autocorrelation'] < truncate_at trunc_idx = max(control_trunc[control_trunc].index[0], mjhmc_trunc[mjhmc_trunc].index[0]) # nuts_trunc[nuts_trunc].index[0]) nuts_ac = nuts_ac.loc[:trunc_idx] else: trunc_idx = max(control_trunc[control_trunc].index[0], mjhmc_trunc[mjhmc_trunc].index[0]) else: trunc_idx = truncate_idx control_ac = control_ac.loc[:trunc_idx] mjhmc_ac = mjhmc_ac.loc[:trunc_idx] control_ac.index = control_ac['num grad'] mjhmc_ac.index = control_ac['num grad'] if nuts: nuts_ac.index = nuts_ac['num grad'] nuts_ac['autocorrelation'].plot(label='NUTS') # control_ac['autocorrelation'].plot(label='Control HMC;\n {}'.format(str(control_params))) # mjhmc_ac['autocorrelation'].plot(label='Markov Jump HMC;\n {}'.format(str(mjhmc_params))) control_ac['autocorrelation'].plot(label='Control HMC') mjhmc_ac['autocorrelation'].plot(label='Markov Jump HMC') plt.xlabel("Gradient Evaluations") plt.ylabel("Autocorrelation") plt.title("{}D {}".format(distribution.ndims, type(distribution).__name__)) plt.legend() plt.show() plt.savefig("{}_{}_dim_ac_{}_steps.pdf".format( type(distribution).__name__, distribution.ndims, max_steps)) plt.show()
def plot_ac(distribution, control_params, mjhmc_params, lahmc_params, max_steps=3000, sample_steps=1, truncate=False, truncate_at=0.0, nuts=False, truncate_idx=None): """ distribution is an instantiated distribution object runs the sampler for max steps and then truncates the output to autocorrelation 0.5 throws an error if ac 0.5 is not reached """ plt.clf() print('Calculating AutoCorrelation for ControlHMC') control_ac = calculate_autocorrelation(ControlHMC, distribution, num_steps=max_steps, sample_steps=sample_steps, half_window=True, use_cached_var=True, **control_params) print('Calculating AutoCorrelation for MJHMC') mjhmc_ac = calculate_autocorrelation(MarkovJumpHMC, distribution, num_steps=max_steps, sample_steps=sample_steps, half_window=True, resample=False, use_cached_var=True, **mjhmc_params) # lahmc_ac = calculate_autocorrelation(LAHMC, distribution, # num_steps=max_steps, # sample_steps=sample_steps, # half_window=True, # **lahmc_params) if nuts: print('Calculating AutoCorrelation for NUTS') nuts_df = sample_nuts_to_df(distribution, 100000, n_burn_in=10000) nuts_ac = autocorrelation(nuts_df, half_window=True) # find idx with autocorrelation closest to truncate_at if truncate: control_trunc = control_ac.loc[:, 'autocorrelation'] < truncate_at mjhmc_trunc = mjhmc_ac.loc[:, 'autocorrelation'] < truncate_at if truncate_idx is None: if nuts: nuts_trunc = nuts_ac.loc[:, 'autocorrelation'] < truncate_at trunc_idx = max(control_trunc[control_trunc].index[0], mjhmc_trunc[mjhmc_trunc].index[0]) # nuts_trunc[nuts_trunc].index[0]) nuts_ac = nuts_ac.loc[:trunc_idx] else: trunc_idx = max(control_trunc[control_trunc].index[0], mjhmc_trunc[mjhmc_trunc].index[0]) else: trunc_idx = truncate_idx control_ac = control_ac.loc[:trunc_idx] mjhmc_ac = mjhmc_ac.loc[:trunc_idx] control_ac.index = control_ac['num grad'] mjhmc_ac.index = control_ac['num grad'] if nuts: nuts_ac.index = nuts_ac['num grad'] nuts_ac['autocorrelation'].plot(label='NUTS') # control_ac['autocorrelation'].plot(label='Control HMC;\n {}'.format(str(control_params))) # mjhmc_ac['autocorrelation'].plot(label='Markov Jump HMC;\n {}'.format(str(mjhmc_params))) control_ac['autocorrelation'].plot(label='Control HMC') mjhmc_ac['autocorrelation'].plot(label='Markov Jump HMC') plt.xlabel("Gradient Evaluations") plt.ylabel("Autocorrelation") plt.title("{}D {}".format(distribution.ndims, type(distribution).__name__)) plt.legend() plt.show() plt.savefig("{}_{}_dim_ac_{}_steps.pdf".format(type(distribution).__name__, distribution.ndims, max_steps)) plt.show()