def train_exact_gp_model_average(trainX, trainY, testX, testY, kind, model_kwargs, train_kwargs,
devices=('cpu',), skip_posterior_variances=False, evaluate_on_train=True,
output_device=None, record_pred_unc=False):
from fitting.sampling import ModelAverage
model_kwargs = copy.deepcopy(model_kwargs)
train_kwargs = copy.deepcopy(train_kwargs)
if len(devices) > 1:
raise ValueError("CGP not implemented for multi GPUs (yet?)")
if str(devices[0]) != 'cpu':
torch.cuda.set_device(torch.device(devices[0]))
devices = [torch.device(device) for device in devices]
if output_device is None:
output_device = devices[0]
else:
output_device = torch.device(output_device)
trainX = trainX.to(output_device)
trainY = trainY.to(output_device)
testX = testX.to(output_device)
testY = testY.to(output_device)
predictions, log_mlls = [], []
varying_params = model_kwargs.pop('varying_params')
k = list(varying_params.keys())[0]
for i in range(len(varying_params[k])):
for k, v in varying_params.items():
model_kwargs[k] = v[i]
metrics, pred_mean, model = train_exact_gp(trainX, trainY, testX, testY, kind, model_kwargs, train_kwargs, devices=devices,
skip_posterior_variances=skip_posterior_variances, skip_random_restart=True,
evaluate_on_train=False)
log_mlls.append(-metrics['prior_train_nmll'])
model.eval()
predictions.append(model(testX))
test_outputs = ModelAverage(predictions, log_mlls)
model_metrics = dict()
with torch.no_grad():
if not skip_posterior_variances:
model_metrics['test_nll'] = -test_outputs.log_prob(testY).item()
# TODO: implement confidence region method for model average object.
model_metrics['sampled_mean_mse'] = mean_squared_error(test_outputs.sample_mean(), testY)
model_metrics['normal_mean_mse'] = mean_squared_error(test_outputs.mean(), testY)
# model_metrics['state_dict_file'] = _save_state_dict(model)
return model_metrics, test_outputs.mean().to('cpu'), None
def train_compressed_gp(trainX, trainY, testX, testY, model_kwargs, train_kwargs, devices=('cpu',),
skip_posterior_variances=False, evaluate_on_train=True,
output_device=None, record_pred_unc=False):
from fitting.sampling import CGPSampler
d = trainX.shape[-1]
if len(devices) > 1:
raise ValueError("CGP not implemented for multi GPUs (yet?)")
if str(devices[0]) != 'cpu':
torch.cuda.set_device(torch.device(devices[0]))
devices = [torch.device(device) for device in devices]
if output_device is None:
output_device = devices[0]
else:
output_device = torch.device(output_device)
trainX = trainX.to(output_device)
trainY = trainY.to(output_device)
testX = testX.to(output_device)
testY = testY.to(output_device)
# Pack all of the kwargs into one object... maybe not the best idea.
model = CGPSampler(trainX, trainY, **model_kwargs, **train_kwargs)
model_metrics = dict()
with torch.no_grad():
if evaluate_on_train:
train_outputs = model.pred(trainX)
model_metrics['train_mse'] = mean_squared_error(train_outputs.mean(), trainY)
test_outputs = model.pred(testX)
if not skip_posterior_variances:
if evaluate_on_train:
model_metrics['train_nll'] = -train_outputs.log_prob(trainY).item()
model_metrics['test_nll'] = -test_outputs.log_prob(testY).item()
# TODO: implement confidence region method for model average object.
model_metrics['sampled_mean_mse'] = mean_squared_error(test_outputs.sample_mean(), testY)
model_metrics['normal_mean_mse'] = mean_squared_error(test_outputs.mean(), testY)
# model_metrics['state_dict_file'] = _save_state_dict(model)
return model_metrics, test_outputs.mean().to('cpu'), model
def train_ppr_gp(trainX, trainY, testX, testY, model_kwargs, train_kwargs, device='cpu',
skip_posterior_variances=False):
model_kwargs = copy.copy(model_kwargs)
train_kwargs = copy.copy(train_kwargs)
d = trainX.shape[-1]
device = torch.device(device)
trainX = trainX.to(device)
trainY = trainY.to(device)
testX = testX.to(device)
testY = testY.to(device)
kernel_type = model_kwargs.pop('kernel_type', 'RBF')
if kernel_type == 'RBF':
kernels = [RBFKernel() for _ in range(10)]
elif kernel_type == 'Matern':
kernels = [MaternKernel(nu=1.5) for _ in range(10)]
else:
raise ValueError("Unknown kernel type")
if len(model_kwargs) > 0:
warnings.warn("Not all model kwargs are used: {}".format(list(model_kwargs.keys())))
optimizer_ = _map_to_optim(train_kwargs.pop('optimizer'))
model = learn_projections(kernels, trainX, trainY, optimizer=optimizer_, **train_kwargs)
likelihood = model.likelihood
mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model)
model.eval()
likelihood.eval()
mll.eval()
model_metrics = dict()
with torch.no_grad():
model.train() # consider prior for evaluation on train dataset
likelihood.train()
train_outputs = model(trainX)
model_metrics['prior_train_nmll'] = -mll(train_outputs, trainY).item()
with gpytorch.settings.skip_posterior_variances(skip_posterior_variances):
model.eval() # Now consider posterior distributions
likelihood.eval()
train_outputs = model(trainX)
test_outputs = model(testX)
if not skip_posterior_variances:
model_metrics['train_nll'] = -likelihood(train_outputs).log_prob(
trainY).item()
model_metrics['test_nll'] = -likelihood(test_outputs).log_prob(
testY).item()
model_metrics['train_mse'] = mean_squared_error(train_outputs.mean,
trainY)
model_metrics['state_dict_file'] = _save_state_dict(model)
return model_metrics, test_outputs.mean.to('cpu'), model
def benchmark_on_n_pts(n_pts,
create_model_func,
target_func,
ho_x,
ho_y,
fit=True,
repeats=3,
max_iter=1000,
return_model=False,
verbose=0,
checkpoint=True,
print_freq=1,
use_chol=False,
**kwargs):
dims = ho_x.shape[1]
# if n_pts > 20:
# ho_x = ho_x.to(device)
# ho_y = ho_y.to(device)
rep_mses = []
models = []
mlls = []
for i in range(repeats):
# Don't edit the master copies fo the hold-out dataset
test_ho_x = torch.empty_like(ho_x).copy_(ho_x)
test_ho_y = torch.empty_like(ho_y).copy_(ho_y)
# test_ho_x = ho_x.copy_()
# test_ho_y = ho_y.copy_()
# Create the data.
data = torch.rand(n_pts, dims) * 4 - 2
y = target_func(data) + torch.randn(n_pts) * 0.01
# Normalize by TEST in this case for all methods for more accurate comparison
m = ho_x.mean(dim=0)
s = ho_x.std(dim=0)
data = (data - m) / s
test_ho_x = (test_ho_x - m) / s
# Do the same for Ys.
m = ho_y.mean()
s = ho_y.std()
y = (y - m) / s
test_ho_y = (test_ho_y - m) / s
# Create the model now.
model = create_model_func(data, y, **kwargs)
# Put things on the GPU if necessary
if n_pts > 20:
test_ho_x = test_ho_x.to(device)
test_ho_y = test_ho_y.to(device)
model = model.to(device)
data = data.to(device)
y = y.to(device)
fast = not use_chol
with gp_set.fast_computations(fast, fast,
fast), gp_set.max_cg_iterations(10_000):
with gp_set.cg_tolerance(0.001), gp_set.eval_cg_tolerance(
0.0005), gp_set.memory_efficient(True):
if fit:
mll = ExactMarginalLogLikelihood(model.likelihood, model)
train_to_convergence(model,
data,
y,
torch.optim.Adam,
objective=mll,
checkpoint=checkpoint,
max_iter=max_iter,
print_freq=print_freq,
verbose=verbose)
model.eval()
with torch.no_grad():
mse = mean_squared_error(model(test_ho_x).mean, test_ho_y)
print(i, mse)
rep_mses.append(mse)
if return_model:
models.append(model)
mlls.append(mll)
else:
del mll
del model
del data
del y
def run_experiment(training_routine: Callable,
training_options: Dict,
dataset: Union[str, pd.DataFrame],
split: float,
cv: bool,
addl_metrics: Dict = {},
repeats=1,
error_repeats=10,
normalize_using_train=True,
chosen_fold=0,
print_to_console=True):
"""Main function to run a model on a dataset.
This function is intended to take a training routine (implicitly instantiates
a model and trains it),
options for said training routine, any metrics that should be run
in addition to negative log likelihood (if applicable) and mean squared
error given in {'name': function} format, a dataset name/dataframe,
a fraction determining the train/test split of the data, a bool that
determines whether full CV is used (or whether a single train/test split
is used), and finally how many times to repeat each fitting/evaluating
step in each fold.
The output is a dataframe of the results.
Note that if the dataset if provided, it is assumed to be sufficiently
shuffled already.
:param training_routine:
:param training_options:
:param addl_metrics:
:param dataset:
:param split:
:param cv:
:return: results_list
"""
if isinstance(dataset, str):
dataset = load_dataset(dataset)
cols = list(dataset.columns)
features = [x for x in cols if (x != 'target' and x.lower() != 'index')]
fold_starts = _determine_folds(split, dataset)
n_folds = len(fold_starts) - 1
results_list = []
t0 = time.time()
for fold in range(n_folds):
if not cv and fold != chosen_fold: # only do one fold if you're not doing CV
continue
train, test = _access_fold(dataset, fold_starts, fold)
if normalize_using_train:
train, test = _normalize_by_train(train, test)
succeed = False
n_errors = 0
while not succeed and n_errors < error_repeats:
try:
trainX = torch.tensor(train[features].values,
dtype=torch.float).contiguous()
trainY = torch.tensor(train['target'].values,
dtype=torch.float).contiguous()
testX = torch.tensor(test[features].values,
dtype=torch.float).contiguous()
testY = torch.tensor(test['target'].values,
dtype=torch.float).contiguous()
for repeat in range(repeats):
result_dict = {
'fold': fold,
'repeat': repeat,
'n': len(dataset),
'd': len(features)
}
start = time.perf_counter()
ret = training_routine(trainX, trainY, testX, testY,
**training_options)
model_metrics = ret[0]
ypred = ret[1]
end = time.perf_counter()
result_dict['mse'] = mean_squared_error(ypred, testY)
result_dict['rmse'] = np.sqrt(result_dict['mse'])
result_dict['train_time'] = end - start
# e.g. -ll, -mll
for name, value in model_metrics.items():
result_dict[name] = value
# e.g. mae, ...
for name, fxn in addl_metrics.items():
result_dict[name] = fxn(ypred, testY)
results_list.append(result_dict)
succeed = True
t = time.time()
elapsed = t - t0
num_finished = fold * repeats + repeat + 1
num_remaining = n_folds * repeats - num_finished
eta = datetime.timedelta(seconds=elapsed / num_finished *
num_remaining)
if print_to_console:
print('{}, fold={}, rep={}, eta={} \n{}'.format(
datetime.datetime.now(), fold, repeat,
format_timedelta(eta), result_dict))
# print("succeed: ", succeed)
except Exception:
result_dict = dict(error=traceback.format_exc(),
fold=fold,
n=len(dataset),
d=len(features) - 2,
mse=np.nan,
rmse=np.nan)
print(result_dict)
traceback.print_exc()
results_list.append(result_dict)
n_errors += 1
print('errors: ', n_errors)
results = pd.DataFrame(results_list)
print('Mean RMSE = {}'.format(results['rmse'].mean()))
return results
def train_exact_gp(trainX, trainY, testX, testY, kind, model_kwargs, train_kwargs, devices=('cpu',),
skip_posterior_variances=False, skip_random_restart=False, evaluate_on_train=True,
output_device=None, record_pred_unc=False, double=False):
"""Create and train an exact GP with the given options"""
model_kwargs = copy.copy(model_kwargs)
train_kwargs = copy.copy(train_kwargs)
d = trainX.shape[-1]
devices = [torch.device(device) for device in devices]
if output_device is None:
output_device = devices[0]
else:
output_device = torch.device(output_device)
type_ = torch.double if double else torch.float
trainX = trainX.to(output_device, type_)
trainY = trainY.to(output_device, type_)
testX = testX.to(output_device, type_)
testY = testY.to(output_device, type_)
# replace with value from dataset for convenience
for k, v in list(model_kwargs.items()):
if isinstance(v, str) and v == 'd':
model_kwargs[k] = d
# Change some options just for initial training with random restarts.
random_restarts = train_kwargs.pop('random_restarts', 1)
init_iters = train_kwargs.pop('init_iters', 20)
optimizer_ = _map_to_optim(train_kwargs.pop('optimizer'))
rr_check_conv = train_kwargs.pop('rr_check_conv', False)
initial_train_kwargs = copy.copy(train_kwargs)
initial_train_kwargs['max_iter'] = init_iters
initial_train_kwargs['check_conv'] = rr_check_conv
# initial_train_kwargs['verbose'] = 0 # don't shout about it
best_model, best_likelihood, best_mll = None, None, None
best_loss = np.inf
# TODO: move random restarts code to a train_to_convergence-like function
if not skip_random_restart:
# Do some number of random restarts, keeping the best one after a truncated training.
for restart in range(random_restarts):
# TODO: log somehow what's happening in the restarts.
model, likelihood = create_exact_gp(trainX, trainY, kind, devices=devices, **model_kwargs)
model = model.to(output_device, type_)
# regular marginal log likelihood
mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model)
_ = train_to_convergence(model, trainX, trainY, optimizer=optimizer_,
objective=mll, isloss=False, **initial_train_kwargs)
model.train()
output = model(trainX)
loss = -mll(output, trainY).item()
if loss < best_loss:
best_loss = loss
best_model = model
best_likelihood = likelihood
best_mll = mll
model = best_model
likelihood = best_likelihood
mll = best_mll
else:
model, likelihood = create_exact_gp(trainX, trainY, kind, devices=devices, **model_kwargs)
model = model.to(output_device, type_)
mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model)
# fit GP
with warnings.catch_warnings(record=True) as w:
trained_epochs = train_to_convergence(model, trainX, trainY, optimizer=optimizer_,
objective=mll, isloss=False, **train_kwargs)
model.eval()
likelihood.eval()
mll.eval()
model_metrics = dict()
model_metrics['trained_epochs'] = trained_epochs
with torch.no_grad():
model.train() # consider prior for evaluation on train dataset
likelihood.train()
train_outputs = model(trainX)
model_metrics['prior_train_nmll'] = -mll(train_outputs, trainY).item()
with gpytorch.settings.skip_posterior_variances(skip_posterior_variances):
model.eval() # Now consider posterior distributions
likelihood.eval()
if evaluate_on_train:
train_outputs = model(trainX)
model_metrics['train_mse'] = mean_squared_error(train_outputs.mean, trainY)
with warnings.catch_warnings(record=True) as w2:
test_outputs = model(testX)
pred_mean = test_outputs.mean
if not skip_posterior_variances:
# model_metrics['train_nll'] = -likelihood(train_outputs).log_prob(
# trainY).item()
# model_metrics['test_nll'] = -likelihood(test_outputs).log_prob(
# testY).item()
if evaluate_on_train:
model_metrics['train_nll'] = -mll(train_outputs, trainY).item()
model_metrics['test_nll'] = -mll(test_outputs, testY).item()
distro = likelihood(test_outputs)
lower, upper = distro.confidence_region()
frac = ((testY > lower) * (testY < upper)).to(torch.float).mean().item()
model_metrics['test_pred_frac_in_cr'] = frac
if record_pred_unc:
model_metrics['test_pred_z_score'] = (testY - distro.mean) / distro.stddev
# model_metrics['test_pred_var'] = distro.variance.tolist()
# model_metrics['test_pred_mean'] = distro.mean.tolist()
model_metrics['training_warnings'] = len(w)
model_metrics['testing_warning'] = '' if len(w2) == 0 else w2[-1].message
model_metrics['state_dict_file'] = _save_state_dict(model)
return model_metrics, pred_mean.to('cpu', torch.float), model