def test_output_shapes_generate_train(self):
rds = np.random.RandomState(123)
dataset = MNISTRegressionDataset(random_state=rds)
for n_tasks in [24, 2]:
for n_samples in [1, 85]:
data_test = dataset.generate_meta_train_data(
n_tasks=n_tasks, n_samples=n_samples)
assert len(data_test) == n_tasks
for (x_train, t_train) in data_test:
assert x_train.shape[0] == t_train.shape[0]
assert x_train.shape[1] == 2
def test_seed_reproducability(self):
rds = np.random.RandomState(55)
dataset = MNISTRegressionDataset(random_state=rds)
data_test_1 = dataset.generate_meta_test_data(n_tasks=2,
n_samples_context=5,
n_samples_test=10)
data_train_1 = dataset.generate_meta_train_data(n_tasks=5,
n_samples=20)
rds = np.random.RandomState(55)
dataset = MNISTRegressionDataset(random_state=rds)
data_test_2 = dataset.generate_meta_test_data(n_tasks=2,
n_samples_context=5,
n_samples_test=10)
data_train_2 = dataset.generate_meta_train_data(n_tasks=5,
n_samples=20)
for test_tuple_1, test_tuple_2 in zip(data_test_1, data_test_2):
for data_array_1, data_array_2 in zip(test_tuple_1, test_tuple_2):
assert np.array_equal(data_array_1, data_array_2)
for train_tuple_1, train_tuple_2 in zip(data_train_1, data_train_2):
for data_array_1, data_array_2 in zip(train_tuple_1,
train_tuple_2):
assert np.array_equal(data_array_1, data_array_2)
def main(argv):
# setup logging
logger, exp_dir = setup_exp_doc(FLAGS.exp_name)
if FLAGS.dataset == 'swissfel':
data_train, _, data_test = provide_data(dataset='swissfel')
else:
if FLAGS.dataset == 'sin-nonstat':
dataset = SinusoidNonstationaryDataset(random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'sin':
dataset = SinusoidDataset(random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'cauchy':
dataset = CauchyDataset(random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'mnist':
dataset = MNISTRegressionDataset(random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'physionet':
dataset = PhysionetDataset(random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'gp-funcs':
dataset = GPFunctionsDataset(random_state=np.random.RandomState(FLAGS.seed + 1))
else:
raise NotImplementedError('Does not recognize dataset flag')
data_train = dataset.generate_meta_train_data(n_tasks=FLAGS.n_train_tasks, n_samples=FLAGS.n_train_samples)
data_test = dataset.generate_meta_test_data(n_tasks=FLAGS.n_test_tasks, n_samples_context=FLAGS.n_context_samples,
n_samples_test=FLAGS.n_test_samples)
nn_layers = tuple([FLAGS.layer_size for _ in range(FLAGS.num_layers)])
torch.set_num_threads(FLAGS.n_threads)
gp_meta = GPRegressionMetaLearned(data_train,
learning_mode=FLAGS.learning_mode,
num_iter_fit=FLAGS.n_iter_fit,
covar_module=FLAGS.covar_module,
mean_module=FLAGS.mean_module,
kernel_nn_layers=nn_layers,
mean_nn_layers=nn_layers,
weight_decay=FLAGS.weight_decay,
lr_params=FLAGS.lr,
lr_decay=FLAGS.lr_decay,
random_seed=FLAGS.seed,
task_batch_size=FLAGS.batch_size,
optimizer=FLAGS.optimizer,
normalize_data=FLAGS.normalize_data
)
gp_meta.meta_fit(valid_tuples=data_test[:100], log_period=2000)
test_ll, rmse, calib_err = gp_meta.eval_datasets(data_test)
# save results
results_dict = {
'test_ll': test_ll,
'test_rmse': rmse,
'calib_err': calib_err
}
print(results_dict)
save_results(results_dict, exp_dir, log=True)
def test_output_shapes_generate_test(self):
rds = np.random.RandomState(123)
dataset = MNISTRegressionDataset(random_state=rds)
for n_tasks in [1, 5]:
for n_samples_context in [1, 85]:
for n_samples_test in [-1, 23]:
data_test = dataset.generate_meta_test_data(
n_tasks=n_tasks,
n_samples_context=n_samples_context,
n_samples_test=n_samples_test)
assert len(data_test) == n_tasks
for (x_context, t_context, x_test, t_test) in data_test:
assert x_context.shape[0] == t_context.shape[0]
assert x_context.shape[1] == x_test.shape[1] == 2
if n_samples_test == -1:
assert x_context.shape[0] + x_test.shape[0] == 28**2
def main(argv):
# setup logging
logger, exp_dir = setup_exp_doc(FLAGS.exp_name)
if FLAGS.dataset == 'swissfel':
raise NotImplementedError
else:
if FLAGS.dataset == 'sin-nonstat':
dataset = SinusoidNonstationaryDataset(
random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'sin':
dataset = SinusoidDataset(
random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'cauchy':
dataset = CauchyDataset(
random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'mnist':
dataset = MNISTRegressionDataset(
random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'physionet':
dataset = PhysionetDataset(
random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'gp-funcs':
dataset = GPFunctionsDataset(
random_state=np.random.RandomState(FLAGS.seed + 1))
else:
raise NotImplementedError('Does not recognize dataset flag')
meta_train_data = dataset.generate_meta_test_data(
n_tasks=1024,
n_samples_context=FLAGS.n_context_samples,
n_samples_test=FLAGS.n_test_samples)
meta_test_data = dataset.generate_meta_test_data(
n_tasks=FLAGS.n_test_tasks,
n_samples_context=FLAGS.n_context_samples,
n_samples_test=FLAGS.n_test_samples)
nn_layers = tuple([FLAGS.layer_size for _ in range(FLAGS.num_layers)])
torch.set_num_threads(FLAGS.n_threads)
# only take meta-train context for training
meta_train_data = meta_train_data[:FLAGS.n_train_tasks]
data_train = [(context_x, context_y)
for context_x, context_y, _, _ in meta_train_data]
assert len(data_train) == FLAGS.n_train_tasks
gp_meta = GPRegressionMetaLearned(data_train,
learning_mode=FLAGS.learning_mode,
num_iter_fit=FLAGS.n_iter_fit,
covar_module=FLAGS.covar_module,
mean_module=FLAGS.mean_module,
kernel_nn_layers=nn_layers,
mean_nn_layers=nn_layers,
weight_decay=FLAGS.weight_decay,
lr_params=FLAGS.lr,
lr_decay=FLAGS.lr_decay,
random_seed=FLAGS.seed,
task_batch_size=FLAGS.batch_size,
optimizer=FLAGS.optimizer,
normalize_data=FLAGS.normalize_data)
gp_meta.meta_fit(log_period=1000)
test_ll_meta_train, test_rmse_meta_train, calib_err_meta_train = gp_meta.eval_datasets(
meta_train_data)
test_ll_meta_test, test_rmse_meta_test, calib_err_test = gp_meta.eval_datasets(
meta_test_data)
# save results
results_dict = {
'test_ll_meta_train': test_ll_meta_train,
'test_ll_meta_test': test_ll_meta_test,
'test_rmse_meta_train': test_rmse_meta_train,
'test_rmse_meta_test': test_rmse_meta_test,
'calib_err_meta_train': calib_err_meta_train,
'calib_err_test': calib_err_test
}
pprint(results_dict)
save_results(results_dict, exp_dir, log=True)
def main(argv):
# setup logging
logger, exp_dir = setup_exp_doc(FLAGS.exp_name)
if FLAGS.dataset == 'sin-nonstat':
dataset = SinusoidNonstationaryDataset(
random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'sin':
dataset = SinusoidDataset(
random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'cauchy':
dataset = CauchyDataset(random_state=np.random.RandomState(FLAGS.seed +
1))
elif FLAGS.dataset == 'mnist':
dataset = MNISTRegressionDataset(
random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'physionet':
dataset = PhysionetDataset(
random_state=np.random.RandomState(FLAGS.seed + 1))
elif FLAGS.dataset == 'gp-funcs':
dataset = GPFunctionsDataset(
random_state=np.random.RandomState(FLAGS.seed + 1))
else:
raise NotImplementedError('Does not recognize dataset flag')
data_train = dataset.generate_meta_train_data(
n_tasks=FLAGS.n_train_tasks, n_samples=FLAGS.n_train_samples)
data_test = dataset.generate_meta_test_data(
n_tasks=FLAGS.n_test_tasks,
n_samples_context=FLAGS.n_context_samples,
n_samples_test=FLAGS.n_test_samples)
nn_layers = tuple([FLAGS.layer_size for _ in range(FLAGS.num_layers)])
torch.set_num_threads(FLAGS.n_threads)
gp_meta = GPRegressionMetaLearnedVI(
data_train,
weight_prior_std=FLAGS.weight_prior_std,
prior_factor=FLAGS.prior_factor,
covar_module=FLAGS.covar_module,
mean_module=FLAGS.mean_module,
kernel_nn_layers=nn_layers,
mean_nn_layers=nn_layers,
random_seed=FLAGS.seed,
optimizer=FLAGS.optimizer,
lr=FLAGS.lr,
lr_decay=FLAGS.lr_decay,
num_iter_fit=FLAGS.n_iter_fit,
svi_batch_size=FLAGS.svi_batch_size,
normalize_data=FLAGS.normalize_data,
cov_type=FLAGS.cov_type,
task_batch_size=FLAGS.task_batch_size)
gp_meta.meta_fit(valid_tuples=data_test[:100], log_period=1000)
test_ll_bayes, rmse_bayes, calib_err_bayes = gp_meta.eval_datasets(
data_test, mode='Bayes')
test_ll_map, rmse_map, calib_err_map = gp_meta.eval_datasets(data_test,
mode='MAP')
# save results
results_dict = {
'test_ll_bayes': test_ll_bayes,
'test_rmse_bayes': rmse_bayes,
'calib_err_bayes': calib_err_bayes,
'test_ll_map': test_ll_map,
'rmse_map': rmse_map,
'calib_err_map': calib_err_map
}
print(results_dict)
save_results(results_dict, exp_dir, log=True)