def test_gp(plot=False, method='full'): """ Compares model prediction with an exact GP (without optimisation) """ # note that this test fails without latent noise in the case of full Gaussian np.random.seed(111) num_input_samples = 10 num_samples = 10000 gaussian_sigma = .2 X, Y, kernel = DataSource.normal_generate_samples(num_input_samples, gaussian_sigma, 1) kernel = [GPy.kern.RBF(1, variance=1., lengthscale=np.array((1.,)))] if method == 'full': m = SAVIGP_SingleComponent(X, Y, num_input_samples, UnivariateGaussian(np.array(gaussian_sigma)), kernel, num_samples, None, 0.001, True, True) if method == 'diag': m = SAVIGP_Diag(X, Y, num_input_samples, 1, UnivariateGaussian(np.array(gaussian_sigma)), kernel, num_samples, None, 0.001, True, True) # update model using optimal parameters # gp = SAVIGP_Test.gpy_prediction(X, Y, gaussian_sigma, kernel[0]) # gp_mean, gp_var = gp.predict(X, full_cov=True) # m.MoG.m[0,0] = gp_mean[:,0] # m.MoG.update_covariance(0, gp_var - gaussian_sigma * np.eye(10)) try: folder_name = 'test' + '_' + ModelLearn.get_ID() logger = ModelLearn.get_logger(folder_name, logging.DEBUG) Optimizer.optimize_model(m, 10000, logger, ['mog']) except KeyboardInterrupt: pass sa_mean, sa_var = m.predict(X) gp = SAVIGP_Test.gpy_prediction(X, Y, gaussian_sigma, deepcopy(kernel[0])) gp_mean, gp_var = gp.predict(X) mean_error = (np.abs(sa_mean - gp_mean)).sum() / sa_mean.shape[0] var_error = (np.abs(sa_var - gp_var)).sum() / gp_var.T.shape[0] if mean_error < 0.1: print bcolors.OKBLUE, "passed: mean gp prediction ", mean_error else: print bcolors.WARNING, "failed: mean gp prediction ", mean_error print bcolors.ENDC if var_error < 0.1: print bcolors.OKBLUE, "passed: var gp prediction ", var_error else: print bcolors.WARNING, "failed: var gp prediction ", var_error print bcolors.ENDC if plot: plot_fit(m) gp.plot() show(block=True)
def test_grad_single(config, verbose, sparse, likelihood_type): num_input_samples = 3 num_samples = 100000 cov, gaussian_sigma, ll, num_process = SAVIGP_Test.get_cond_ll(likelihood_type) np.random.seed(111) if sparse: num_inducing = num_input_samples - 1 else: num_inducing = num_input_samples X, Y, kernel = DataSource.normal_generate_samples(num_input_samples, cov) s1 = SAVIGP_SingleComponent(X, Y, num_inducing, ll, [deepcopy(kernel) for j in range(num_process)], num_samples, config, 0, True, True) s1.rand_init_mog() def f(x): s1.set_params(x) return s1.objective_function() def f_grad(x): s1.set_params(x) return s1.objective_function_gradients() return GradChecker.check(f, f_grad, s1.get_params(), s1.get_param_names(), verbose=verbose)
def run_model(Xtest, Xtrain, Ytest, Ytrain, cond_ll, kernel, method, name, run_id, num_inducing, num_samples, sparsify_factor, to_optimize, trans_class, random_Z, logging_level, export_X, latent_noise=0.001, opt_per_iter=None, max_iter=200, n_threads=1, model_image_file=None, xtol=1e-3, ftol=1e-5, partition_size=3000): """ Fits a model to the data (Xtrain, Ytrain) using the method provided by 'method', and makes predictions on 'Xtest' and 'Ytest', and exports the result to csv files. Parameters ---------- Xtest : ndarray X of test points Xtrain : ndarray X of training points Ytest : ndarray Y of test points Ytrain : ndarray Y of traiing points cond_ll : subclass of likelihood/Likelihood Conditional log likelihood function used to build the model. kernel : list The kernel that the model uses. It should be an array, and size of the array should be same as the number of latent processes. Each element should provide interface similar to ``ExtRBF`` class method : string The method to use to learns the model. It can be 'full', 'mix1', and 'mix2' name : string The name that will be used for logger file names, and results files names run_id : object ID of the experiment, which can be anything, and it will be included in the configuration file. It can provdie for example a number referring to a particular test and train partition. num_inducing : integer Number of inducing points num_samples : integer Number of samples for estimating objective function and gradients sparsify_factor : float Can be any number and will be included in the configuration file. It will not determine the number of inducing points to_optimize : list The set of parameters to optimize. It should be a list, and it can include 'll', 'mog', 'hyp', 'inducing' e.g., it can be ['ll', 'mog'] in which case posterior and ll will be optimised. trans_class : subclass of DataTransformation The class which will be used to transform data. random_Z : boolean Whether to initialise inducing points randomly on the training data. If False, inducing points will be placed using k-means (or mini-batch k-mean) clustering. If True, inducing points will be placed randomly on the training data. logging_level : string The logging level to use. export_X : boolean Whether to export X to csv files. latent_noise : integer The amount of latent noise to add to the kernel. A white noise of amount latent_noise will be added to the kernel. opt_per_iter: integer Number of update of each subset of parameters in each iteration, e.g., {'mog': 15000, 'hyp': 25, 'll': 25} max_iter: integer Maximum of global iterations used on optimization. n_threads: integer Maximum number of threads used. model_image_file: string The image file from the which the model will be initialized. xtol: float Tolerance of 'X' below which the optimization is determined as converged. ftol: float Tolerance of 'f' below which the optimization is determined as converged. partition_size: integer The size which is used to partition training data (This is not the partition used for SGD). Training data will be split to the partitions of size ``partition_size`` and calculations will be done on each partition separately. This aim of this partitioning of data is to make algorithm memory efficient. Returns ------- folder : string the name of the folder in which results are stored model : model the fitted model itself. """ if opt_per_iter is None: opt_per_iter = {'mog': 40, 'hyp': 40, 'll': 40} folder_name = name + '_' + ModelLearn.get_ID() logger = ModelLearn.get_logger(folder_name, logging_level) transformer = trans_class.get_transformation(Ytrain, Xtrain) Ytrain = transformer.transform_Y(Ytrain) Ytest = transformer.transform_Y(Ytest) Xtrain = transformer.transform_X(Xtrain) Xtest = transformer.transform_X(Xtest) opt_max_fun_evals = None total_time = None timer_per_iter = None tracker = None export_model = False git_hash, git_branch = get_git() properties = { 'method': method, 'sparsify_factor': sparsify_factor, 'sample_num': num_samples, 'll': cond_ll.__class__.__name__, 'opt_max_evals': opt_max_fun_evals, 'opt_per_iter': opt_per_iter, 'xtol': xtol, 'ftol': ftol, 'run_id': run_id, 'experiment': name, 'max_iter': max_iter, 'git_hash': git_hash, 'git_branch': git_branch, 'random_Z': random_Z, 'latent_noise:': latent_noise, 'model_init': model_image_file } logger.info('experiment started for:' + str(properties)) model_image = None current_iter = None if model_image_file is not None: model_image = pickle.load(open(model_image_file + 'model.dump')) opt_params = pickle.load(open(model_image_file + 'opt.dump')) current_iter = opt_params['current_iter'] if model_image: logger.info('loaded model - iteration started from: ' + str(opt_params['current_iter']) + ' Obj fun: ' + str(opt_params['obj_fun']) + ' fun evals: ' + str(opt_params['total_evals'])) if method == 'full': m = SAVIGP_SingleComponent(Xtrain, Ytrain, num_inducing, cond_ll, kernel, num_samples, None, latent_noise, False, random_Z, n_threads=n_threads, image=model_image, partition_size=partition_size) _, timer_per_iter, total_time, tracker, total_evals = \ Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol, ModelLearn.opt_callback(folder_name), current_iter) if method == 'mix1': m = SAVIGP_Diag(Xtrain, Ytrain, num_inducing, 1, cond_ll, kernel, num_samples, None, latent_noise, False, random_Z, n_threads=n_threads, image=model_image, partition_size=partition_size) _, timer_per_iter, total_time, tracker, total_evals = \ Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol, ModelLearn.opt_callback(folder_name), current_iter) if method == 'mix2': m = SAVIGP_Diag(Xtrain, Ytrain, num_inducing, 2, cond_ll, kernel, num_samples, None, latent_noise, False, random_Z, n_threads=n_threads, image=model_image, partition_size=partition_size) _, timer_per_iter, total_time, tracker, total_evals = \ Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol, ModelLearn.opt_callback(folder_name), current_iter) if method == 'gp': m = GPy.models.GPRegression(Xtrain, Ytrain, kernel[0]) if 'll' in to_optimize and 'hyp' in to_optimize: m.optimize('bfgs') y_pred, var_pred, nlpd = m.predict(Xtest, Ytest) if not (tracker is None): ModelLearn.export_track(folder_name, tracker) ModelLearn.export_train(folder_name, transformer.untransform_X(Xtrain), transformer.untransform_Y(Ytrain), export_X) ModelLearn.export_test(folder_name, transformer.untransform_X(Xtest), transformer.untransform_Y(Ytest), [transformer.untransform_Y(y_pred)], [transformer.untransform_Y_var(var_pred)], transformer.untransform_NLPD(nlpd), [''], export_X) if export_model and isinstance(m, SAVIGP): ModelLearn.export_model(m, folder_name) properties['total_time'] = total_time properties['time_per_iter'] = timer_per_iter properties['total_evals'] = total_evals ModelLearn.export_configuration(folder_name, properties) return folder_name, m
def run_model(Xtest, Xtrain, Ytest, Ytrain, cond_ll, kernel, method, name, run_id, num_inducing, num_samples, sparsify_factor, to_optimize, trans_class, random_Z, logging_level, export_X, latent_noise=0.001, opt_per_iter=None, max_iter=200, n_threads=1, model_image_file=None, xtol=1e-3, ftol=1e-5, partition_size=3000): """ Fits a model to the data (Xtrain, Ytrain) using the method provided by 'method', and makes predictions on 'Xtest' and 'Ytest', and exports the result to csv files. Parameters ---------- Xtest : ndarray X of test points Xtrain : ndarray X of training points Ytest : ndarray Y of test points Ytrain : ndarray Y of traiing points cond_ll : subclass of likelihood/Likelihood Conditional log likelihood function used to build the model. kernel : list The kernel that the model uses. It should be an array, and size of the array should be same as the number of latent processes. Each element should provide interface similar to ``ExtRBF`` class method : string The method to use to learns the model. It can be 'full', 'mix1', and 'mix2' name : string The name that will be used for logger file names, and results files names run_id : object ID of the experiment, which can be anything, and it will be included in the configuration file. It can provdie for example a number referring to a particular test and train partition. num_inducing : integer Number of inducing points num_samples : integer Number of samples for estimating objective function and gradients sparsify_factor : float Can be any number and will be included in the configuration file. It will not determine the number of inducing points to_optimize : list The set of parameters to optimize. It should be a list, and it can include 'll', 'mog', 'hyp', 'inducing' e.g., it can be ['ll', 'mog'] in which case posterior and ll will be optimised. trans_class : subclass of DataTransformation The class which will be used to transform data. random_Z : boolean Whether to initialise inducing points randomly on the training data. If False, inducing points will be placed using k-means (or mini-batch k-mean) clustering. If True, inducing points will be placed randomly on the training data. logging_level : string The logging level to use. export_X : boolean Whether to export X to csv files. latent_noise : integer The amount of latent noise to add to the kernel. A white noise of amount latent_noise will be added to the kernel. opt_per_iter: integer Number of update of each subset of parameters in each iteration, e.g., {'mog': 15000, 'hyp': 25, 'll': 25} max_iter: integer Maximum of global iterations used on optimization. n_threads: integer Maximum number of threads used. model_image_file: string The image file from the which the model will be initialized. xtol: float Tolerance of 'X' below which the optimization is determined as converged. ftol: float Tolerance of 'f' below which the optimization is determined as converged. partition_size: integer The size which is used to partition training data (This is not the partition used for SGD). Training data will be split to the partitions of size ``partition_size`` and calculations will be done on each partition separately. This aim of this partitioning of data is to make algorithm memory efficient. Returns ------- folder : string the name of the folder in which results are stored model : model the fitted model itself. """ if opt_per_iter is None: opt_per_iter = {'mog': 40, 'hyp': 40, 'll': 40} folder_name = name + '_' + ModelLearn.get_ID() transformer = trans_class.get_transformation(Ytrain, Xtrain) Ytrain = transformer.transform_Y(Ytrain) Ytest = transformer.transform_Y(Ytest) Xtrain = transformer.transform_X(Xtrain) Xtest = transformer.transform_X(Xtest) opt_max_fun_evals = None total_time = None timer_per_iter = None tracker = None export_model = False git_hash, git_branch = get_git() properties = {'method': method, 'sparsify_factor': sparsify_factor, 'sample_num': num_samples, 'll': cond_ll.__class__.__name__, 'opt_max_evals': opt_max_fun_evals, 'opt_per_iter': opt_per_iter, 'xtol': xtol, 'ftol': ftol, 'run_id': run_id, 'experiment': name, 'max_iter': max_iter, 'git_hash': git_hash, 'git_branch': git_branch, 'random_Z': random_Z, 'latent_noise:': latent_noise, 'model_init': model_image_file } logger = ModelLearn.get_logger(ModelLearn.get_output_path() + folder_name, folder_name, logging_level) logger.info('experiment started for:' + str(properties)) model_image = None current_iter = None if model_image_file is not None: model_image = pickle.load(open(model_image_file + 'model.dump')) opt_params = pickle.load(open(model_image_file + 'opt.dump')) current_iter = opt_params['current_iter'] if model_image: logger.info('loaded model - iteration started from: ' + str(opt_params['current_iter']) + ' Obj fun: ' + str(opt_params['obj_fun']) + ' fun evals: ' + str(opt_params['total_evals'])) if method == 'full': m = SAVIGP_SingleComponent(Xtrain, Ytrain, num_inducing, cond_ll, kernel, num_samples, None, latent_noise, False, random_Z, n_threads=n_threads, image=model_image, partition_size=partition_size) _, timer_per_iter, total_time, tracker, total_evals = \ Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol, ModelLearn.opt_callback(folder_name), current_iter) if method == 'mix1': m = SAVIGP_Diag(Xtrain, Ytrain, num_inducing, 1, cond_ll, kernel, num_samples, None, latent_noise, False, random_Z, n_threads=n_threads, image=model_image, partition_size=partition_size) _, timer_per_iter, total_time, tracker, total_evals = \ Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol, ModelLearn.opt_callback(folder_name), current_iter) if method == 'mix2': m = SAVIGP_Diag(Xtrain, Ytrain, num_inducing, 2, cond_ll, kernel, num_samples, None, latent_noise, False, random_Z, n_threads=n_threads, image=model_image, partition_size=partition_size) _, timer_per_iter, total_time, tracker, total_evals = \ Optimizer.optimize_model(m, opt_max_fun_evals, logger, to_optimize, xtol, opt_per_iter, max_iter, ftol, ModelLearn.opt_callback(folder_name), current_iter) if method == 'gp': m = GPy.models.GPRegression(Xtrain, Ytrain, kernel[0]) if 'll' in to_optimize and 'hyp' in to_optimize: m.optimize('bfgs') logger.debug("prediction started...") y_pred, var_pred, nlpd = m.predict(Xtest, Ytest) logger.debug("prediction finished") if not (tracker is None): ModelLearn.export_track(folder_name, tracker) ModelLearn.export_train(folder_name, transformer.untransform_X(Xtrain), transformer.untransform_Y(Ytrain), export_X) ModelLearn.export_test(folder_name, transformer.untransform_X(Xtest), transformer.untransform_Y(Ytest), [transformer.untransform_Y(y_pred)], [transformer.untransform_Y_var(var_pred)], transformer.untransform_NLPD(nlpd), [''], export_X) if export_model and isinstance(m, SAVIGP): ModelLearn.export_model(m, folder_name) properties['total_time'] = total_time properties['time_per_iter'] = timer_per_iter properties['total_evals'] = total_evals ModelLearn.export_configuration(folder_name, properties) return folder_name, m