def plot_std(distribution):
from mjhmc.samplers.markov_jump_hmc import MarkovJumpHMC, ControlHMC
from mjhmc.misc.autocor import calculate_autocorrelation
# change this!!!
steps = int(1E4)
plt.clf()
c_ac_1 = calculate_autocorrelation(ControlHMC,
distribution,
num_steps=steps,
sample_steps=1,
half_window=True,
**std_param1)
m_ac_1 = calculate_autocorrelation(MarkovJumpHMC,
distribution,
num_steps=steps,
sample_steps=1,
half_window=True,
**std_param1)
c_ac_2 = calculate_autocorrelation(ControlHMC,
distribution,
num_steps=steps,
sample_steps=1,
half_window=True,
**std_param2)
m_ac_2 = calculate_autocorrelation(MarkovJumpHMC,
distribution,
num_steps=steps,
sample_steps=1,
half_window=True,
**std_param2)
# c1_trunc = c_ac_1.loc[:, 'autocorrelation'] < 0.0
# c2_trunc = c_ac_2.loc[:, 'autocorrelation'] < 0.0
# m1_trunc = m_ac_1.loc[:, 'autocorrelation'] < 0.0
# m2_trunc = c_ac_2.loc[:, 'autocorrelation'] < 0.0
# trunc_idx = max(c1_trunc[c1_trunc].index[0],
# c2_trunc[c2_trunc].index[0],
# m1_trunc[m1_trunc].index[0],
# m2_trunc[m2_trunc].index[0])
# c_ac_1 = c_ac_1.loc[:trunc_idx]
# c_ac_2 = c_ac_2.loc[:trunc_idx]
# m_ac_1 = m_ac_1.loc[:trunc_idx]
# m_ac_2 = m_ac_2.loc[:trunc_idx]
c_ac_1.index = c_ac_1['num grad']
c_ac_2.index = c_ac_1['num grad']
m_ac_1.index = c_ac_1['num grad']
m_ac_2.index = c_ac_1['num grad']
c_ac_1.autocorrelation.plot(label=r"Control HMC $\beta = 0.1$")
m_ac_1.autocorrelation.plot(label=r"Markov Jump HMC $\beta = 0.1$")
c_ac_2.autocorrelation.plot(label=r"Control HMC $\beta = 0.8$")
m_ac_2.autocorrelation.plot(label=r"Markov Jump HMC $\beta = 0.8$")
plt.xlabel("Gradient Evaluations")
plt.ylabel("Autocorrelation")
plt.title("{}D {}".format(distribution.ndims, type(distribution).__name__))
plt.legend()
plt.savefig("{}_std.pdf".format(type(distribution).__name__))
def plot_std(distribution):
# change this!!!
steps = int(1E4)
plt.clf()
c_ac_1 = calculate_autocorrelation(ControlHMC, distribution,
num_steps=steps,
sample_steps=1,
half_window=True,
**std_param1)
m_ac_1 = calculate_autocorrelation(MarkovJumpHMC, distribution,
num_steps=steps,
sample_steps=1,
half_window=True,
**std_param1)
c_ac_2 = calculate_autocorrelation(ControlHMC, distribution,
num_steps=steps,
sample_steps=1,
half_window=True,
**std_param2)
m_ac_2 = calculate_autocorrelation(MarkovJumpHMC, distribution,
num_steps=steps,
sample_steps=1,
half_window=True,
**std_param2)
# c1_trunc = c_ac_1.loc[:, 'autocorrelation'] < 0.0
# c2_trunc = c_ac_2.loc[:, 'autocorrelation'] < 0.0
# m1_trunc = m_ac_1.loc[:, 'autocorrelation'] < 0.0
# m2_trunc = c_ac_2.loc[:, 'autocorrelation'] < 0.0
# trunc_idx = max(c1_trunc[c1_trunc].index[0],
# c2_trunc[c2_trunc].index[0],
# m1_trunc[m1_trunc].index[0],
# m2_trunc[m2_trunc].index[0])
# c_ac_1 = c_ac_1.loc[:trunc_idx]
# c_ac_2 = c_ac_2.loc[:trunc_idx]
# m_ac_1 = m_ac_1.loc[:trunc_idx]
# m_ac_2 = m_ac_2.loc[:trunc_idx]
c_ac_1.index = c_ac_1['num grad']
c_ac_2.index = c_ac_1['num grad']
m_ac_1.index = c_ac_1['num grad']
m_ac_2.index = c_ac_1['num grad']
c_ac_1.autocorrelation.plot(label=r"Control HMC $\beta = 0.1$")
m_ac_1.autocorrelation.plot(label=r"Markov Jump HMC $\beta = 0.1$")
c_ac_2.autocorrelation.plot(label=r"Control HMC $\beta = 0.8$")
m_ac_2.autocorrelation.plot(label=r"Markov Jump HMC $\beta = 0.8$")
plt.xlabel("Gradient Evaluations")
plt.ylabel("Autocorrelation")
plt.title("{}D {}".format(distribution.ndims, type(distribution).__name__))
plt.legend()
plt.savefig("{}_std.pdf".format(type(distribution).__name__))
def obj_func(sampler, distr, job_id, **kwargs):
num_target_grad_evals = grad_evals[type(distr).__name__]
default_args = {
"num_grad_steps": num_target_grad_evals,
"sample_steps": 1,
"num_steps": None,
"half_window": True
}
kwargs = unpack_params(kwargs)
kwargs.update(default_args)
ac_df = calculate_autocorrelation(sampler, distr, **kwargs)
n = ac_df['num grad'].values.astype(int)
# necessary to keep curve_fit from borking
normed_n = n / (0.5 * num_target_grad_evals)
y = ac_df['autocorrelation'].values
if use_exp:
# wtf fit is mutating input somehow
r1 = fit(normed_n.copy(), y.copy())[0]
if debug:
plot_fit(normed_n, y, r1, job_id, kwargs)
return -r1
else:
if np.isnan(np.sum(ac_df.autocorrelation.values)):
return 11 * num_target_grad_evals
for trial, target in enumerate(np.arange(0, 1, 0.1)):
score = min_idx(ac_df, target)
if score is not None:
score += trial * num_target_grad_evals * 0.5
break
if debug:
plot_search_ac(ac_df['num grad'].values.astype(int),
ac_df.autocorrelation.values,
job_id,
kwargs, score)
return score or 5 * num_target_grad_evals
def benchmark_ac_batch_size_scaling(sampler_cls=MarkovJumpHMC,
distribution_cls=Gaussian, ndims=2,
n_samples=1000, n_itrs=10):
""" Plots run time of sampling and calculating the autocorrelation of n_samples (fixed)
against batch size
normalizes to time per sample
:param sampler_cls: sampler to test
:param distribution_cls: distribution to test on
:param ndims: dimension of the space, passed to distribution_cls upon instantiation
:param n_samples: number of samples to generate
:param n_itrs: number of trials to average
:returns: None, makes a plot
:rtype: None
"""
run_time = []
batch_sizes = np.arange(1, 500, 25)
for n_batch in batch_sizes:
distribution = distribution_cls(ndims=ndims, nbatch=n_batch)
print "now doing batch size {}".format(n_batch)
times = np.zeros(n_itrs)
for idx in range(n_itrs):
t_i = time.time()
_ = calculate_autocorrelation(sampler_cls, distribution, num_steps=n_samples, half_window=True)
t_f = time.time()
times[idx] = t_f - t_i
run_time.append(np.mean(times) / (n_samples * n_batch))
fig = plt.figure()
axis = fig.add_subplot(111)
axis.plot(batch_sizes, run_time)
axis.set_yscale('log')
axis.set_ylabel("log seconds per sample")
axis.set_xlabel("batch size")
axis.set_title("{} autocorrelation: compute time per sample versus batch size".format(sampler_cls.__name__))
def obj_func_helper(sampler, distr, unpack, kwargs):
""" Helper function for the objective function
:param sampler: sampler being tested. instance of mjhmc.samplers.markov_jump_hmc.HMCBase
:param distr: distribution being used. instance of mjhmc.misc.distributions.Distribution
:param unpack: boolean flag of whether to unpack params or not.
:param kwargs: dictionary of kwargs passed from parent function
:returns: parameters of fitted curves, graph data for computed autocorrelation, kwargs
:rtype: tuple
"""
num_target_grad_evals = grad_evals[type(distr).__name__]
default_args = {
"num_grad_steps": num_target_grad_evals,
"sample_steps": 1,
"num_steps": None,
"half_window": True,
"use_cached_var": True
}
if unpack:
kwargs = unpack_params(kwargs)
if sampler.__name__ == 'MarkovJumpHMC':
default_args["resample"] = False
kwargs.update(default_args)
print "Calculating autocorrelation for {} grad evals".format(num_target_grad_evals)
ac_df = calculate_autocorrelation(sampler, distr, **kwargs)
n_grad_evals = ac_df['num grad'].values.astype(int)
# necessary to keep curve_fit from borking: THIS IS VERY IMPORTANT
normed_n_grad_evals = n_grad_evals / (0.5 * num_target_grad_evals)
autocor = ac_df['autocorrelation'].values
print "Fitting curve"
exp_coef, cos_coef = fit(normed_n_grad_evals.copy(), autocor.copy())
return cos_coef, normed_n_grad_evals, exp_coef, autocor, kwargs
def obj_func_helper(sampler, distr, unpack, kwargs):
""" Helper function for the objective function
:param sampler: sampler being tested. instance of mjhmc.samplers.markov_jump_hmc.HMCBase
:param distr: distribution being used. instance of mjhmc.misc.distributions.Distribution
:param unpack: boolean flag of whether to unpack params or not.
:param kwargs: dictionary of kwargs passed from parent function
:returns: parameters of fitted curves, graph data for computed autocorrelation, kwargs
:rtype: tuple
"""
num_target_grad_evals = grad_evals[type(distr).__name__]
default_args = {
"num_grad_steps": num_target_grad_evals,
"sample_steps": 1,
"num_steps": None,
"half_window": True,
"use_cached_var": True
}
if unpack:
kwargs = unpack_params(kwargs)
if sampler.__name__ == 'MarkovJumpHMC':
default_args["resample"] = False
kwargs.update(default_args)
print "Calculating autocorrelation for {} grad evals".format(num_target_grad_evals)
# grad evals was previously cast to int, why?
autocor, _, n_grad_evals = calculate_autocorrelation(sampler, distr, **kwargs)
# necessary to keep curve_fit from borking: THIS IS VERY IMPORTANT
normed_n_grad_evals = n_grad_evals / (0.5 * num_target_grad_evals)
print "Fitting curve"
exp_coef, cos_coef = tf_fit(normed_n_grad_evals.copy(), autocor.copy())
if SAVE_TRACE:
formatted_time = time.strftime("%Y%m%d-%H%M%S")
trace_name = '{}_{}'.format(type(distr).__name__, formatted_time)
save_trace(normed_n_grad_evals, autocor, exp_coef, cos_coef, trace_name)
return cos_coef, normed_n_grad_evals, exp_coef, autocor, kwargs
def autocorrelation_vs_n_samples(sampler_cls=MarkovJumpHMC,
distribution_cls=Gaussian, ndims=2,
n_batch=200, n_itrs=10):
""" Plots run time of sampling and calculating the autocorrelation against n_samples,
fixing batch size
plots total time - not time per sample
:param sampler_cls: sampler to test
:param distribution_cls: distribution to test on
:param ndims: dimension of the space, passed to distribution_cls upon instantiation
:param n_samples: number of samples to generate
:param n_itrs: number of trials to average
:returns: None, makes a plot
:rtype: None
"""
run_time = []
sample_sizes = np.arange(100, 10000, 250)
for n_samples in izes:
distribution = distribution_cls(ndims=ndims, nbatch=n_batch)
print "now running for {} samples".format(n_samples)
times = np.zeros(n_itrs)
for idx in range(n_itrs):
t_i = time.time()
_ = calculate_autocorrelation(sampler_cls, distribution, num_steps=n_samples, half_window=True)
t_f = time.time()
times[idx] = t_f - t_i
print "last run took time {}".format(times[-1])
run_time.append(np.mean(times))
fig = plt.figure()
axis = fig.add_subplot(111)
axis.plot(sample_sizes, run_time)
axis.set_yscale('log')
axis.set_ylabel("log seconds per sample")
axis.set_xlabel("batch size")
axis.set_title("{}: autocorrelation compute time by number of samples".format(sampler_cls.__name__))
from mjhmc.samplers.markov_jump_hmc import MarkovJumpHMC
from mjhmc.misc.autocor import calculate_autocorrelation
from mjhmc.misc.distributions import ProductOfT
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from scipy.sparse import rand
np.random.seed(2015)
BURN_IN_STEPS = int(1E+3)
BATCH_SIZE = int(1E+1)
nbasis = 36
ndims = 36
W_half = rand(ndims,nbasis/2,density=0.35)
W = np.concatenate((W_half.todense(),-W_half.todense()),axis=1)
#Not doing this makes it bork crazily. Poor implementation of inheritance
PoT = ProductOfT(nbasis=nbasis,ndims=ndims,W=W)
ac_df = calculate_autocorrelation(MarkovJumpHMC,PoT,num_steps=1000)
import IPython; IPython.embed()
plt.plot(ac_df['autocorrelation'])
plt.title('Random_Sparse_W_AC')
plt.savefig('ac_rand_sparse_poe_w.png')
def plot_ac(distribution,
control_params,
mjhmc_params,
lahmc_params,
max_steps=3000,
sample_steps=1,
truncate=False,
truncate_at=0.0,
nuts=False,
truncate_idx=None):
"""
distribution is an instantiated distribution object
runs the sampler for max steps and then truncates the output to autocorrelation 0.5
throws an error if ac 0.5 is not reached
"""
from mjhmc.samplers.markov_jump_hmc import MarkovJumpHMC, ControlHMC
from mjhmc.misc.autocor import calculate_autocorrelation, autocorrelation
from mjhmc.misc.nutshell import sample_nuts_to_df
# TODO: bring up to speed of DF-less calc autocor
plt.clf()
print('Calculating AutoCorrelation for ControlHMC')
control_ac = calculate_autocorrelation(ControlHMC,
distribution,
num_steps=max_steps,
sample_steps=sample_steps,
half_window=True,
use_cached_var=True,
**control_params)
print('Calculating AutoCorrelation for MJHMC')
mjhmc_ac = calculate_autocorrelation(MarkovJumpHMC,
distribution,
num_steps=max_steps,
sample_steps=sample_steps,
half_window=True,
resample=False,
use_cached_var=True,
**mjhmc_params)
# lahmc_ac = calculate_autocorrelation(LAHMC, distribution,
# num_steps=max_steps,
# sample_steps=sample_steps,
# half_window=True,
# **lahmc_params)
if nuts:
print('Calculating AutoCorrelation for NUTS')
nuts_df = sample_nuts_to_df(distribution, 100000, n_burn_in=10000)
nuts_ac = autocorrelation(nuts_df, half_window=True)
# find idx with autocorrelation closest to truncate_at
if truncate:
control_trunc = control_ac.loc[:, 'autocorrelation'] < truncate_at
mjhmc_trunc = mjhmc_ac.loc[:, 'autocorrelation'] < truncate_at
if truncate_idx is None:
if nuts:
nuts_trunc = nuts_ac.loc[:, 'autocorrelation'] < truncate_at
trunc_idx = max(control_trunc[control_trunc].index[0],
mjhmc_trunc[mjhmc_trunc].index[0])
# nuts_trunc[nuts_trunc].index[0])
nuts_ac = nuts_ac.loc[:trunc_idx]
else:
trunc_idx = max(control_trunc[control_trunc].index[0],
mjhmc_trunc[mjhmc_trunc].index[0])
else:
trunc_idx = truncate_idx
control_ac = control_ac.loc[:trunc_idx]
mjhmc_ac = mjhmc_ac.loc[:trunc_idx]
control_ac.index = control_ac['num grad']
mjhmc_ac.index = control_ac['num grad']
if nuts:
nuts_ac.index = nuts_ac['num grad']
nuts_ac['autocorrelation'].plot(label='NUTS')
# control_ac['autocorrelation'].plot(label='Control HMC;\n {}'.format(str(control_params)))
# mjhmc_ac['autocorrelation'].plot(label='Markov Jump HMC;\n {}'.format(str(mjhmc_params)))
control_ac['autocorrelation'].plot(label='Control HMC')
mjhmc_ac['autocorrelation'].plot(label='Markov Jump HMC')
plt.xlabel("Gradient Evaluations")
plt.ylabel("Autocorrelation")
plt.title("{}D {}".format(distribution.ndims, type(distribution).__name__))
plt.legend()
plt.show()
plt.savefig("{}_{}_dim_ac_{}_steps.pdf".format(
type(distribution).__name__, distribution.ndims, max_steps))
plt.show()
def plot_ac(distribution, control_params, mjhmc_params, lahmc_params, max_steps=3000,
sample_steps=1, truncate=False, truncate_at=0.0, nuts=False, truncate_idx=None):
"""
distribution is an instantiated distribution object
runs the sampler for max steps and then truncates the output to autocorrelation 0.5
throws an error if ac 0.5 is not reached
"""
plt.clf()
print('Calculating AutoCorrelation for ControlHMC')
control_ac = calculate_autocorrelation(ControlHMC, distribution,
num_steps=max_steps,
sample_steps=sample_steps,
half_window=True,
use_cached_var=True,
**control_params)
print('Calculating AutoCorrelation for MJHMC')
mjhmc_ac = calculate_autocorrelation(MarkovJumpHMC, distribution,
num_steps=max_steps,
sample_steps=sample_steps,
half_window=True,
resample=False,
use_cached_var=True,
**mjhmc_params)
# lahmc_ac = calculate_autocorrelation(LAHMC, distribution,
# num_steps=max_steps,
# sample_steps=sample_steps,
# half_window=True,
# **lahmc_params)
if nuts:
print('Calculating AutoCorrelation for NUTS')
nuts_df = sample_nuts_to_df(distribution, 100000, n_burn_in=10000)
nuts_ac = autocorrelation(nuts_df, half_window=True)
# find idx with autocorrelation closest to truncate_at
if truncate:
control_trunc = control_ac.loc[:, 'autocorrelation'] < truncate_at
mjhmc_trunc = mjhmc_ac.loc[:, 'autocorrelation'] < truncate_at
if truncate_idx is None:
if nuts:
nuts_trunc = nuts_ac.loc[:, 'autocorrelation'] < truncate_at
trunc_idx = max(control_trunc[control_trunc].index[0],
mjhmc_trunc[mjhmc_trunc].index[0])
# nuts_trunc[nuts_trunc].index[0])
nuts_ac = nuts_ac.loc[:trunc_idx]
else:
trunc_idx = max(control_trunc[control_trunc].index[0],
mjhmc_trunc[mjhmc_trunc].index[0])
else:
trunc_idx = truncate_idx
control_ac = control_ac.loc[:trunc_idx]
mjhmc_ac = mjhmc_ac.loc[:trunc_idx]
control_ac.index = control_ac['num grad']
mjhmc_ac.index = control_ac['num grad']
if nuts:
nuts_ac.index = nuts_ac['num grad']
nuts_ac['autocorrelation'].plot(label='NUTS')
# control_ac['autocorrelation'].plot(label='Control HMC;\n {}'.format(str(control_params)))
# mjhmc_ac['autocorrelation'].plot(label='Markov Jump HMC;\n {}'.format(str(mjhmc_params)))
control_ac['autocorrelation'].plot(label='Control HMC')
mjhmc_ac['autocorrelation'].plot(label='Markov Jump HMC')
plt.xlabel("Gradient Evaluations")
plt.ylabel("Autocorrelation")
plt.title("{}D {}".format(distribution.ndims, type(distribution).__name__))
plt.legend()
plt.show()
plt.savefig("{}_{}_dim_ac_{}_steps.pdf".format(type(distribution).__name__, distribution.ndims, max_steps))
plt.show()
