def test_gp(plot=False, method='full'):
"""
Compares model prediction with an exact GP (without optimisation)
"""
# note that this test fails without latent noise in the case of full Gaussian
np.random.seed(111)
num_input_samples = 10
num_samples = 10000
gaussian_sigma = .2
X, Y, kernel = DataSource.normal_generate_samples(num_input_samples, gaussian_sigma, 1)
kernel = [GPy.kern.RBF(1, variance=1., lengthscale=np.array((1.,)))]
if method == 'full':
m = SAVIGP_SingleComponent(X, Y, num_input_samples, UnivariateGaussian(np.array(gaussian_sigma)),
kernel, num_samples, None, 0.001, True, True)
if method == 'diag':
m = SAVIGP_Diag(X, Y, num_input_samples, 1, UnivariateGaussian(np.array(gaussian_sigma)),
kernel, num_samples, None, 0.001, True, True)
# update model using optimal parameters
# gp = SAVIGP_Test.gpy_prediction(X, Y, gaussian_sigma, kernel[0])
# gp_mean, gp_var = gp.predict(X, full_cov=True)
# m.MoG.m[0,0] = gp_mean[:,0]
# m.MoG.update_covariance(0, gp_var - gaussian_sigma * np.eye(10))
try:
folder_name = 'test' + '_' + ModelLearn.get_ID()
logger = ModelLearn.get_logger(folder_name, logging.DEBUG)
Optimizer.optimize_model(m, 10000, logger, ['mog'])
except KeyboardInterrupt:
pass
sa_mean, sa_var = m.predict(X)
gp = SAVIGP_Test.gpy_prediction(X, Y, gaussian_sigma, deepcopy(kernel[0]))
gp_mean, gp_var = gp.predict(X)
mean_error = (np.abs(sa_mean - gp_mean)).sum() / sa_mean.shape[0]
var_error = (np.abs(sa_var - gp_var)).sum() / gp_var.T.shape[0]
if mean_error < 0.1:
print bcolors.OKBLUE, "passed: mean gp prediction ", mean_error
else:
print bcolors.WARNING, "failed: mean gp prediction ", mean_error
print bcolors.ENDC
if var_error < 0.1:
print bcolors.OKBLUE, "passed: var gp prediction ", var_error
else:
print bcolors.WARNING, "failed: var gp prediction ", var_error
print bcolors.ENDC
if plot:
plot_fit(m)
gp.plot()
show(block=True)
def test_model_learn(config):
"""
Compares the model output with exact GP
"""
method = config['method']
sparsify_factor = config['sparse_factor']
np.random.seed(12000)
names = []
num_input_samples = 20
gaussian_sigma = .2
X, Y, kernel = DataSource.normal_generate_samples(num_input_samples, gaussian_sigma)
train_n = int(0.5 * num_input_samples)
Xtrain = X[:train_n, :]
Ytrain = Y[:train_n, :]
Xtest = X[train_n:, :]
Ytest = Y[train_n:, :]
kernel1 = ModelLearn.get_kernels(Xtrain.shape[1], 1, True)
kernel2 = ModelLearn.get_kernels(Xtrain.shape[1], 1, True)
gaussian_sigma = 1.0
#number of inducing points
num_inducing = int(Xtrain.shape[0] * sparsify_factor)
num_samples = 10000
cond_ll = UnivariateGaussian(np.array(gaussian_sigma))
n1, _ = ModelLearn.run_model(Xtest, Xtrain, Ytest, Ytrain, cond_ll, kernel1, method,
'test_' + ModelLearn.get_ID(), 'test', num_inducing,
num_samples, sparsify_factor, ['mog', 'll', 'hyp'], IdentityTransformation, True,
logging.DEBUG, True)
n2, _ =ModelLearn.run_model(Xtest, Xtrain, Ytest, Ytrain, cond_ll, kernel2, 'gp',
'test_' + ModelLearn.get_ID(), 'test', num_inducing,
num_samples, sparsify_factor, ['mog', 'll', 'hyp'], IdentityTransformation)
PlotOutput.plot_output('test', ModelLearn.get_output_path(), [n1, n2], None, False)
# plots for an specific experiment
# PlotOutput.plot_output_all('abalone_graph', Experiments.get_output_path(),
# lambda x: x['experiment'] == 'abalone', False)
def run_config(config):
try:
logger.info('started config: ' + str(config))
getattr(ExperimentSetup, config['method_to_run'])(config)
logger.info('finished config: ' + str(config))
except Exception as e:
logger.exception(config)
if __name__ == '__main__':
logger = ModelLearn.get_logger(ModelLearn.get_logger_path(), 'general_' + ModelLearn.get_ID(), logging.DEBUG)
# uncomment to run experiments in parallel
# ExperimentRunner.run_parallel(3)
# runs an individual configuration
# ExperimentRunner.boston_experiment()
# ExperimentRunner.wisconsin_breast_experiment()
# ExperimentRunner.USPS_experiment()
# ExperimentRunner.mining_experiment()
# ExperimentRunner.abalone_experiment()
# ExperimentRunner.mnist_binary_inducing_experiment()
ExperimentRunner.mnist_binary_experiment()
# ExperimentRunner.sarcos_all_joins_experiment()
# ExperimentRunner.sarcos_experiment()
# PlotOutput.plot_output_all('abalone_graph', Experiments.get_output_path(),
# lambda x: x['experiment'] == 'abalone', False)
def run_config(config):
try:
logger.info('started config: ' + str(config))
getattr(ExperimentSetup, config['method_to_run'])(config)
logger.info('finished config: ' + str(config))
except Exception as e:
logger.exception(config)
if __name__ == '__main__':
logger = ModelLearn.get_logger(ModelLearn.get_logger_path(),
'general_' + ModelLearn.get_ID(),
logging.DEBUG)
# uncomment to run experiments in parallel
# ExperimentRunner.run_parallel(3)
# runs an individual configuration
ExperimentRunner.boston_experiment()
# ExperimentRunner.wisconsin_breast_experiment()
# ExperimentRunner.USPS_experiment()
# ExperimentRunner.mining_experiment()
# ExperimentRunner.abalone_experiment()
# ExperimentRunner.mnist_binary_inducing_experiment()
# ExperimentRunner.mnist_binary_experiment()
# ExperimentRunner.sarcos_all_joins_experiment()
# ExperimentRunner.sarcos_experiment()
# plots for an specific experiment
# PlotOutput.plot_output_all('abalone_graph', Experiments.get_output_path(),
# lambda x: x['experiment'] == 'abalone', False)
def run_config(config):
try:
logger.info('started config: ' + str(config))
getattr(ModelLearn, config['method_to_run'])(config)
logger.info('finished config: ' + str(config))
except Exception as e:
logger.exception(config)
if __name__ == '__main__':
logger = ModelLearn.get_logger('general_' + ModelLearn.get_ID(), logging.DEBUG)
# uncomment to run experiments in parallel
# ExperimentRunner.run_parallel(3)
# runs an individual configuration
# ExperimentRunner.boston_experiment()
# ExperimentRunner.wisconsin_breast_experiment()
# ExperimentRunner.USPS_experiment()
# ExperimentRunner.mining_experiment()
# ExperimentRunner.abalone_experiment()
# ExperimentRunner.mnist_binary_inducing_experiment()
ExperimentRunner.mnist_binary_experiment()
# ExperimentRunner.sarcos_all_joins_experiment()
# ExperimentRunner.sarcos_experiment()
