def run(ARGS, data=None, model=None, is_test=False):
data = data or get_classification_data(ARGS.dataset, split=ARGS.split)
model = model or get_classification_model(ARGS.model)(data.K, is_test=is_test, seed=ARGS.seed)
def onehot(Y, K):
return np.eye(K)[Y.flatten().astype(int)].reshape(Y.shape[:-1] + (K,))
Y_oh = onehot(data.Y_test, data.K)[None, :, :] # [1 x N_test x K]
model.fit(data.X_train, data.Y_train)
p = model.predict(data.X_test) # [N_test x K] or [samples x N_test x K]
assert p.ndim in {2, 3} # 3-dim in case of approximate predictions (multiple samples per each X)
# clip very large and small probs
eps = 1e-12
p = np.clip(p, eps, 1 - eps)
p = p / np.expand_dims(np.sum(p, -1), -1)
assert np.all(p >= 0.0) and np.all(p <= 1.0)
# evaluation metrics
res = {}
if p.ndim == 2: # keep analysis as in the original code in case 2-dim predictions
logp = multinomial.logpmf(Y_oh, n=1, p=p) # [N_test]
res['test_loglik'] = np.average(logp)
pred = np.argmax(p, axis=-1)
else: # compute metrics in case of 3-dim predictions
res['test_loglik'] = []
for n in range(p.shape[0]): # iterate through samples
logp = multinomial.logpmf(Y_oh, n=1, p=p[n]) # [N_test]
res['test_loglik'].append(logp)
# Mixture test likelihood (mean over per data point evaluations)
res['test_loglik'] = meanlogsumexp(res['test_loglik'])
p = np.mean(p, axis=0)
pred = np.argmax(p, axis=-1)
res['test_acc'] = np.average(np.array(pred == data.Y_test.flatten()).astype(float))
if not is_test:
res.update(ARGS.__dict__)
if not is_test: # pragma: no cover
with Database(ARGS.database_path) as db:
db.write('classification', res)
return res
def test_models_classification(name, K, N):
S, Ns, D = 4, 3, 2
model = get_classification_model(name)(K, is_test=True)
# make sure the multiclass can cope with a missing class in the train data
model.fit(np.random.randn(N, D),
np.random.choice(range(2), size=(N, 1)).astype(float))
model.fit(np.random.randn(N, D),
np.random.choice(range(K), size=(N, 1)).astype(float))
p = model.predict(np.random.randn(Ns, D))
assert p.shape == (Ns, K)
assert_allclose(np.sum(p, 1), np.ones(Ns), atol=0.001)
def run(ARGS, data=None, model=None, is_test=False):
data = data or get_classification_data(ARGS.dataset, split=ARGS.split)
model = model or get_classification_model(ARGS.model)(
data.K, is_test=is_test, seed=ARGS.seed)
def onehot(Y, K):
return np.eye(K)[Y.flatten().astype(int)].reshape(Y.shape[:-1] + (K, ))
Y_oh = onehot(data.Y_test, data.K)[None, :, :] # 1, N_test, K
model.fit(data.X_train, data.Y_train)
p = model.predict(data.X_test) # N_test, K
# clip very large and small probs
eps = 1e-12
p = np.clip(p, eps, 1 - eps)
p = p / np.expand_dims(np.sum(p, -1), -1)
# evaluation metrics
res = {}
logp = multinomial.logpmf(Y_oh, n=1, p=p)
res['test_loglik'] = np.average(logp)
pred = np.argmax(p, axis=-1)
res['test_acc'] = np.average(
np.array(pred == data.Y_test.flatten()).astype(float))
res['Y_test'] = data.Y_test
res['p_test'] = p
res.update(ARGS.__dict__)
if not is_test: # pragma: no cover
with Database(ARGS.database_path) as db:
db.write('classification', res)
return res
def run(ARGS, is_test=False):
data = get_classification_data(ARGS.dataset, split=ARGS.split, prop=1.)
ind = np.zeros(data.X_train.shape[0]).astype(bool)
ind[:ARGS.num_initial_points] = True
X, Y = data.X_train, data.Y_train
def onehot(Y, K):
return np.eye(K)[Y.flatten().astype(int)].reshape(Y.shape[:-1] + (K, ))
Y_oh = onehot(Y, data.K)
Model = get_classification_model(ARGS.model)
model = Model(data.K, is_test=is_test, seed=ARGS.seed)
test_ll = []
train_ll = []
all_ll = []
test_acc = []
train_acc = []
all_acc = []
for _ in range(min(ARGS.iterations, X.shape[0] - ARGS.num_initial_points)):
model.fit(X[ind], Y[ind])
p = model.predict(X) # NK
# clip very large and small probs
eps = 1e-12
p = np.clip(p, eps, 1 - eps)
p = p / np.expand_dims(np.sum(p, -1), -1)
# entropy of predictions at all points
ent = multinomial.entropy(n=1, p=p)
# set the seen ones to -inf so we don't choose them
ent[ind] = -np.inf
# choose the highest entropy point to see next
i = np.argmax(ent)
ind[i] = True
logp = multinomial.logpmf(Y_oh, n=1, p=p) # N
is_correct = (np.argmax(p, 1) == Y.flatten()) # N
test_ll.append(np.average(logp[np.invert(ind)]))
train_ll.append(np.average(logp[ind]))
all_ll.append(np.average(logp))
test_acc.append(np.average(is_correct[np.invert(ind)]))
train_acc.append(np.average(is_correct[ind]))
all_acc.append(np.average(is_correct))
res = {
'test_loglik': np.array(test_ll),
'train_loglik': np.array(train_ll),
'total_loglik': np.array(all_ll),
'test_acc': np.array(test_acc),
'train_acc': np.array(train_acc),
'total_acc': np.array(all_acc),
}
res.update(ARGS.__dict__)
if not is_test: # pragma: no cover
with Database(ARGS.database_path) as db:
db.write('active_learning_discrete', res)