def test_gp(plot=False, method='full'):
"""
Compares model prediction with an exact GP (without optimisation)
"""
# note that this test fails without latent noise in the case of full Gaussian
np.random.seed(111)
num_input_samples = 10
num_samples = 10000
gaussian_sigma = .2
X, Y, kernel = DataSource.normal_generate_samples(num_input_samples, gaussian_sigma, 1)
kernel = [GPy.kern.RBF(1, variance=1., lengthscale=np.array((1.,)))]
if method == 'full':
m = SAVIGP_SingleComponent(X, Y, num_input_samples, UnivariateGaussian(np.array(gaussian_sigma)),
kernel, num_samples, None, 0.001, True, True)
if method == 'diag':
m = SAVIGP_Diag(X, Y, num_input_samples, 1, UnivariateGaussian(np.array(gaussian_sigma)),
kernel, num_samples, None, 0.001, True, True)
# update model using optimal parameters
# gp = SAVIGP_Test.gpy_prediction(X, Y, gaussian_sigma, kernel[0])
# gp_mean, gp_var = gp.predict(X, full_cov=True)
# m.MoG.m[0,0] = gp_mean[:,0]
# m.MoG.update_covariance(0, gp_var - gaussian_sigma * np.eye(10))
try:
folder_name = 'test' + '_' + ModelLearn.get_ID()
logger = ModelLearn.get_logger(folder_name, logging.DEBUG)
Optimizer.optimize_model(m, 10000, logger, ['mog'])
except KeyboardInterrupt:
pass
sa_mean, sa_var = m.predict(X)
gp = SAVIGP_Test.gpy_prediction(X, Y, gaussian_sigma, deepcopy(kernel[0]))
gp_mean, gp_var = gp.predict(X)
mean_error = (np.abs(sa_mean - gp_mean)).sum() / sa_mean.shape[0]
var_error = (np.abs(sa_var - gp_var)).sum() / gp_var.T.shape[0]
if mean_error < 0.1:
print bcolors.OKBLUE, "passed: mean gp prediction ", mean_error
else:
print bcolors.WARNING, "failed: mean gp prediction ", mean_error
print bcolors.ENDC
if var_error < 0.1:
print bcolors.OKBLUE, "passed: var gp prediction ", var_error
else:
print bcolors.WARNING, "failed: var gp prediction ", var_error
print bcolors.ENDC
if plot:
plot_fit(m)
gp.plot()
show(block=True)
def test_grad_single(config, verbose, sparse, likelihood_type):
num_input_samples = 3
num_samples = 100000
cov, gaussian_sigma, ll, num_process = SAVIGP_Test.get_cond_ll(likelihood_type)
np.random.seed(111)
if sparse:
num_inducing = num_input_samples - 1
else:
num_inducing = num_input_samples
X, Y, kernel = DataSource.normal_generate_samples(num_input_samples, cov)
s1 = SAVIGP_SingleComponent(X, Y, num_inducing, ll,
[deepcopy(kernel) for j in range(num_process)], num_samples, config, 0, True, True)
s1.rand_init_mog()
def f(x):
s1.set_params(x)
return s1.objective_function()
def f_grad(x):
s1.set_params(x)
return s1.objective_function_gradients()
return GradChecker.check(f, f_grad, s1.get_params(), s1.get_param_names(), verbose=verbose)
def run_model(Xtest,
Xtrain,
Ytest,
Ytrain,
cond_ll,
kernel,
method,
name,
run_id,
num_inducing,
num_samples,
sparsify_factor,
to_optimize,
trans_class,
random_Z,
logging_level,
export_X,
latent_noise=0.001,
opt_per_iter=None,
max_iter=200,
n_threads=1,
model_image_file=None,
xtol=1e-3,
ftol=1e-5,
partition_size=3000):
"""
Fits a model to the data (Xtrain, Ytrain) using the method provided by 'method', and makes predictions on
'Xtest' and 'Ytest', and exports the result to csv files.
Parameters
----------
Xtest : ndarray
X of test points
Xtrain : ndarray
X of training points
Ytest : ndarray
Y of test points
Ytrain : ndarray
Y of traiing points
cond_ll : subclass of likelihood/Likelihood
Conditional log likelihood function used to build the model.
kernel : list
The kernel that the model uses. It should be an array, and size of the array should be same as the
number of latent processes. Each element should provide interface similar to ``ExtRBF`` class
method : string
The method to use to learns the model. It can be 'full', 'mix1', and 'mix2'
name : string
The name that will be used for logger file names, and results files names
run_id : object
ID of the experiment, which can be anything, and it will be included in the configuration file. It can provdie
for example a number referring to a particular test and train partition.
num_inducing : integer
Number of inducing points
num_samples : integer
Number of samples for estimating objective function and gradients
sparsify_factor : float
Can be any number and will be included in the configuration file. It will not determine
the number of inducing points
to_optimize : list
The set of parameters to optimize. It should be a list, and it can include 'll', 'mog', 'hyp', 'inducing' e.g.,
it can be ['ll', 'mog'] in which case posterior and ll will be optimised.
trans_class : subclass of DataTransformation
The class which will be used to transform data.
random_Z : boolean
Whether to initialise inducing points randomly on the training data. If False, inducing points
will be placed using k-means (or mini-batch k-mean) clustering. If True, inducing points will be placed randomly
on the training data.
logging_level : string
The logging level to use.
export_X : boolean
Whether to export X to csv files.
latent_noise : integer
The amount of latent noise to add to the kernel. A white noise of amount latent_noise will be
added to the kernel.
opt_per_iter: integer
Number of update of each subset of parameters in each iteration, e.g., {'mog': 15000, 'hyp': 25, 'll': 25}
max_iter: integer
Maximum of global iterations used on optimization.
n_threads: integer
Maximum number of threads used.
model_image_file: string
The image file from the which the model will be initialized.
xtol: float
Tolerance of 'X' below which the optimization is determined as converged.
ftol: float
Tolerance of 'f' below which the optimization is determined as converged.
partition_size: integer
The size which is used to partition training data (This is not the partition used for SGD).
Training data will be split to the partitions of size ``partition_size`` and calculations will be done on each
partition separately. This aim of this partitioning of data is to make algorithm memory efficient.
Returns
-------
folder : string
the name of the folder in which results are stored
model : model
the fitted model itself.
"""
if opt_per_iter is None:
opt_per_iter = {'mog': 40, 'hyp': 40, 'll': 40}
folder_name = name + '_' + ModelLearn.get_ID()
logger = ModelLearn.get_logger(folder_name, logging_level)
transformer = trans_class.get_transformation(Ytrain, Xtrain)
Ytrain = transformer.transform_Y(Ytrain)
Ytest = transformer.transform_Y(Ytest)
Xtrain = transformer.transform_X(Xtrain)
Xtest = transformer.transform_X(Xtest)
opt_max_fun_evals = None
total_time = None
timer_per_iter = None
tracker = None
export_model = False
git_hash, git_branch = get_git()
properties = {
'method': method,
'sparsify_factor': sparsify_factor,
'sample_num': num_samples,
'll': cond_ll.__class__.__name__,
'opt_max_evals': opt_max_fun_evals,
'opt_per_iter': opt_per_iter,
'xtol': xtol,
'ftol': ftol,
'run_id': run_id,
'experiment': name,
'max_iter': max_iter,
'git_hash': git_hash,
'git_branch': git_branch,
'random_Z': random_Z,
'latent_noise:': latent_noise,
'model_init': model_image_file
}
logger.info('experiment started for:' + str(properties))
model_image = None
current_iter = None
if model_image_file is not None:
model_image = pickle.load(open(model_image_file + 'model.dump'))
opt_params = pickle.load(open(model_image_file + 'opt.dump'))
current_iter = opt_params['current_iter']
if model_image:
logger.info('loaded model - iteration started from: ' +
str(opt_params['current_iter']) + ' Obj fun: ' +
str(opt_params['obj_fun']) + ' fun evals: ' +
str(opt_params['total_evals']))
if method == 'full':
m = SAVIGP_SingleComponent(Xtrain,
Ytrain,
num_inducing,
cond_ll,
kernel,
num_samples,
None,
latent_noise,
False,
random_Z,
n_threads=n_threads,
image=model_image,
partition_size=partition_size)
_, timer_per_iter, total_time, tracker, total_evals = \
Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol,
ModelLearn.opt_callback(folder_name), current_iter)
if method == 'mix1':
m = SAVIGP_Diag(Xtrain,
Ytrain,
num_inducing,
1,
cond_ll,
kernel,
num_samples,
None,
latent_noise,
False,
random_Z,
n_threads=n_threads,
image=model_image,
partition_size=partition_size)
_, timer_per_iter, total_time, tracker, total_evals = \
Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol,
ModelLearn.opt_callback(folder_name), current_iter)
if method == 'mix2':
m = SAVIGP_Diag(Xtrain,
Ytrain,
num_inducing,
2,
cond_ll,
kernel,
num_samples,
None,
latent_noise,
False,
random_Z,
n_threads=n_threads,
image=model_image,
partition_size=partition_size)
_, timer_per_iter, total_time, tracker, total_evals = \
Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol,
ModelLearn.opt_callback(folder_name), current_iter)
if method == 'gp':
m = GPy.models.GPRegression(Xtrain, Ytrain, kernel[0])
if 'll' in to_optimize and 'hyp' in to_optimize:
m.optimize('bfgs')
y_pred, var_pred, nlpd = m.predict(Xtest, Ytest)
if not (tracker is None):
ModelLearn.export_track(folder_name, tracker)
ModelLearn.export_train(folder_name, transformer.untransform_X(Xtrain),
transformer.untransform_Y(Ytrain), export_X)
ModelLearn.export_test(folder_name, transformer.untransform_X(Xtest),
transformer.untransform_Y(Ytest),
[transformer.untransform_Y(y_pred)],
[transformer.untransform_Y_var(var_pred)],
transformer.untransform_NLPD(nlpd), [''],
export_X)
if export_model and isinstance(m, SAVIGP):
ModelLearn.export_model(m, folder_name)
properties['total_time'] = total_time
properties['time_per_iter'] = timer_per_iter
properties['total_evals'] = total_evals
ModelLearn.export_configuration(folder_name, properties)
return folder_name, m
def run_model(Xtest, Xtrain, Ytest, Ytrain, cond_ll, kernel, method, name, run_id, num_inducing, num_samples,
sparsify_factor, to_optimize, trans_class, random_Z, logging_level, export_X,
latent_noise=0.001, opt_per_iter=None, max_iter=200, n_threads=1, model_image_file=None,
xtol=1e-3, ftol=1e-5, partition_size=3000):
"""
Fits a model to the data (Xtrain, Ytrain) using the method provided by 'method', and makes predictions on
'Xtest' and 'Ytest', and exports the result to csv files.
Parameters
----------
Xtest : ndarray
X of test points
Xtrain : ndarray
X of training points
Ytest : ndarray
Y of test points
Ytrain : ndarray
Y of traiing points
cond_ll : subclass of likelihood/Likelihood
Conditional log likelihood function used to build the model.
kernel : list
The kernel that the model uses. It should be an array, and size of the array should be same as the
number of latent processes. Each element should provide interface similar to ``ExtRBF`` class
method : string
The method to use to learns the model. It can be 'full', 'mix1', and 'mix2'
name : string
The name that will be used for logger file names, and results files names
run_id : object
ID of the experiment, which can be anything, and it will be included in the configuration file. It can provdie
for example a number referring to a particular test and train partition.
num_inducing : integer
Number of inducing points
num_samples : integer
Number of samples for estimating objective function and gradients
sparsify_factor : float
Can be any number and will be included in the configuration file. It will not determine
the number of inducing points
to_optimize : list
The set of parameters to optimize. It should be a list, and it can include 'll', 'mog', 'hyp', 'inducing' e.g.,
it can be ['ll', 'mog'] in which case posterior and ll will be optimised.
trans_class : subclass of DataTransformation
The class which will be used to transform data.
random_Z : boolean
Whether to initialise inducing points randomly on the training data. If False, inducing points
will be placed using k-means (or mini-batch k-mean) clustering. If True, inducing points will be placed randomly
on the training data.
logging_level : string
The logging level to use.
export_X : boolean
Whether to export X to csv files.
latent_noise : integer
The amount of latent noise to add to the kernel. A white noise of amount latent_noise will be
added to the kernel.
opt_per_iter: integer
Number of update of each subset of parameters in each iteration, e.g., {'mog': 15000, 'hyp': 25, 'll': 25}
max_iter: integer
Maximum of global iterations used on optimization.
n_threads: integer
Maximum number of threads used.
model_image_file: string
The image file from the which the model will be initialized.
xtol: float
Tolerance of 'X' below which the optimization is determined as converged.
ftol: float
Tolerance of 'f' below which the optimization is determined as converged.
partition_size: integer
The size which is used to partition training data (This is not the partition used for SGD).
Training data will be split to the partitions of size ``partition_size`` and calculations will be done on each
partition separately. This aim of this partitioning of data is to make algorithm memory efficient.
Returns
-------
folder : string
the name of the folder in which results are stored
model : model
the fitted model itself.
"""
if opt_per_iter is None:
opt_per_iter = {'mog': 40, 'hyp': 40, 'll': 40}
folder_name = name + '_' + ModelLearn.get_ID()
transformer = trans_class.get_transformation(Ytrain, Xtrain)
Ytrain = transformer.transform_Y(Ytrain)
Ytest = transformer.transform_Y(Ytest)
Xtrain = transformer.transform_X(Xtrain)
Xtest = transformer.transform_X(Xtest)
opt_max_fun_evals = None
total_time = None
timer_per_iter = None
tracker = None
export_model = False
git_hash, git_branch = get_git()
properties = {'method': method,
'sparsify_factor': sparsify_factor,
'sample_num': num_samples,
'll': cond_ll.__class__.__name__,
'opt_max_evals': opt_max_fun_evals,
'opt_per_iter': opt_per_iter,
'xtol': xtol,
'ftol': ftol,
'run_id': run_id,
'experiment': name,
'max_iter': max_iter,
'git_hash': git_hash,
'git_branch': git_branch,
'random_Z': random_Z,
'latent_noise:': latent_noise,
'model_init': model_image_file
}
logger = ModelLearn.get_logger(ModelLearn.get_output_path() + folder_name, folder_name, logging_level)
logger.info('experiment started for:' + str(properties))
model_image = None
current_iter = None
if model_image_file is not None:
model_image = pickle.load(open(model_image_file + 'model.dump'))
opt_params = pickle.load(open(model_image_file + 'opt.dump'))
current_iter = opt_params['current_iter']
if model_image:
logger.info('loaded model - iteration started from: ' + str(opt_params['current_iter']) +
' Obj fun: ' + str(opt_params['obj_fun']) + ' fun evals: ' + str(opt_params['total_evals']))
if method == 'full':
m = SAVIGP_SingleComponent(Xtrain, Ytrain, num_inducing, cond_ll,
kernel, num_samples, None, latent_noise, False, random_Z, n_threads=n_threads,
image=model_image, partition_size=partition_size)
_, timer_per_iter, total_time, tracker, total_evals = \
Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol,
ModelLearn.opt_callback(folder_name), current_iter)
if method == 'mix1':
m = SAVIGP_Diag(Xtrain, Ytrain, num_inducing, 1, cond_ll,
kernel, num_samples, None, latent_noise, False, random_Z, n_threads=n_threads,
image=model_image, partition_size=partition_size)
_, timer_per_iter, total_time, tracker, total_evals = \
Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol,
ModelLearn.opt_callback(folder_name), current_iter)
if method == 'mix2':
m = SAVIGP_Diag(Xtrain, Ytrain, num_inducing, 2, cond_ll,
kernel, num_samples, None, latent_noise, False, random_Z, n_threads=n_threads,
image=model_image, partition_size=partition_size)
_, timer_per_iter, total_time, tracker, total_evals = \
Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol,
ModelLearn.opt_callback(folder_name), current_iter)
if method == 'gp':
m = GPy.models.GPRegression(Xtrain, Ytrain, kernel[0])
if 'll' in to_optimize and 'hyp' in to_optimize:
m.optimize('bfgs')
logger.debug("prediction started...")
y_pred, var_pred, nlpd = m.predict(Xtest, Ytest)
logger.debug("prediction finished")
if not (tracker is None):
ModelLearn.export_track(folder_name, tracker)
ModelLearn.export_train(folder_name, transformer.untransform_X(Xtrain), transformer.untransform_Y(Ytrain),
export_X)
ModelLearn.export_test(folder_name,
transformer.untransform_X(Xtest),
transformer.untransform_Y(Ytest),
[transformer.untransform_Y(y_pred)],
[transformer.untransform_Y_var(var_pred)],
transformer.untransform_NLPD(nlpd),
[''], export_X)
if export_model and isinstance(m, SAVIGP):
ModelLearn.export_model(m, folder_name)
properties['total_time'] = total_time
properties['time_per_iter'] = timer_per_iter
properties['total_evals'] = total_evals
ModelLearn.export_configuration(folder_name, properties)
return folder_name, m
