def run(ARGS, is_test):
data = get_regression_data(ARGS.dataset, split=ARGS.split, prop=1.)
ind = np.zeros(data.X_train.shape[0]).astype(bool)
ind[:ARGS.num_initial_points] = True
X, Y = data.X_train, data.Y_train
Model = non_bayesian_model(ARGS.model, 'regression') or\
import_module('bayesian_benchmarks.models.{}.models'.format(ARGS.model)).RegressionModel
model = Model(is_test=is_test, seed=ARGS.seed)
test_ll = []
train_ll = []
all_ll = []
test_rmse = []
train_rmse = []
all_rmse = []
for _ in range(min(ARGS.iterations, X.shape[0] - ARGS.num_initial_points)):
model.fit(X[ind], Y[ind])
m, v = model.predict(X) # ND
vars = v.copy()
# set the seen ones to -inf so we don't choose them
vars[ind] = -np.inf
# choose the highest variance point
i = np.argmax(vars)
ind[i] = True
logp = norm.logpdf(Y, loc=m, scale=v**0.5) # N
d2 = (Y - m)**2
test_ll.append(np.average(logp[np.invert(ind)]))
train_ll.append(np.average(logp[ind]))
all_ll.append(np.average(logp))
test_rmse.append(np.average(d2[np.invert(ind)])**0.5)
train_rmse.append(np.average(d2[ind])**0.5)
all_rmse.append(np.average(d2)**0.5)
# save
res = {
'test_loglik': np.array(test_ll),
'train_loglik': np.array(train_ll),
'total_loglik': np.array(all_ll),
'test_rmse': np.array(test_rmse),
'train_rmse': np.array(train_rmse),
'total_rmse': np.array(all_rmse),
}
res.update(ARGS.__dict__)
if not is_test: # pragma: no cover
with Database() as db:
db.write('active_learning_continuous', res)
def get_regression_model(name):
assert name in all_regression_models
return non_bayesian_model(name, 'regression') or \
import_module('bayesian_benchmarks.models.{}.models'.format(name)).RegressionModel
def get_classification_model(name):
assert name in all_classification_models
return non_bayesian_model(name, 'classification') or \
import_module('bayesian_benchmarks.models.{}.models'.format(name)).ClassificationModel
def get_classification_model(name):
assert name in all_classification_models
return non_bayesian_model(name, 'classification') or \
expert_models_class(name, 'classification')
def get_regression_model(name):
assert name in all_regression_models
return non_bayesian_model(name, 'regression') or \
expert_models_reg(name, 'regression') or \
gp_models(name, 'regression')