def compute_acc_and_nmi_conflicted_data(self, x, y, centers_emb, beta1, beta2):
features = self.predict_encoder(x)
unconf_indices, conf_indices = self.generate_unconflicted_data_index(x, centers_emb, beta1, beta2)
if unconf_indices.size == 0:
print(' '*8 + "Empty list of unconflicted data")
acc_unconf = 0
nmi_unconf = 0
else:
x_emb_unconf = self.predict_encoder(x[unconf_indices])
y_unconf = y[unconf_indices]
y_pred_unconf = q_mat(x_emb_unconf, centers_emb, alpha=1.0).argmax(axis=1)
acc_unconf = metrics.acc(y_unconf, y_pred_unconf)
nmi_unconf = metrics.nmi(y_unconf, y_pred_unconf)
print(' '*8 + '|==> acc unconflicted data: %.4f, nmi unconflicted data: %.4f <==|'% (acc_unconf, nmi_unconf))
if conf_indices.size == 0:
print(' '*8 + "Empty list of conflicted data")
acc_conf = 0
nmi_conf = 0
else:
x_emb_conf = self.predict_encoder(x[conf_indices])
y_conf = y[conf_indices]
y_pred_conf = q_mat(x_emb_conf, centers_emb, alpha=1.0).argmax(axis=1)
acc_conf = metrics.acc(y_conf, y_pred_conf)
nmi_conf = metrics.nmi(y_conf, y_pred_conf)
print(' '*8 + '|==> acc conflicted data: %.4f, nmi conflicted data: %.4f <==|'% (metrics.acc(y_conf, y_pred_conf), metrics.nmi(y_conf, y_pred_conf)))
return acc_unconf, nmi_unconf, acc_conf, nmi_conf
def test_accuracy(classifier, Xt, Yt, Xv, Yv, filename=None):
# Apprently, arrays don't work here as they try to access second dimension size...
Yv = mat(Yv).transpose()
Yt = mat(Yt).transpose()
predictions = classifier.predict(Xt)
print "Neural Net Train Accuracy:", acc(Yt, predictions), "%"
predictions = classifier.predict(Xv)
print "Neural Net Test Accuracy:", acc(Yv, predictions), "%"
def test_accuracy(classifier, Xt, Yt, Xv, Yv, filename=None):
# Apprently, arrays don't work here as they try to access second dimension size...
Yv = mat(Yv).transpose()
Yt = mat(Yt).transpose()
predictions = classifier.predict(Xt)
print "Neural Net Train Accuracy:",acc(Yt, predictions),"%"
predictions = classifier.predict(Xv)
print "Neural Net Test Accuracy:",acc(Yv, predictions),"%"
def train_model(net,
optimizer,
criterion,
trainloader,
num_ens=1,
beta_type=0.1,
epoch=None,
num_epochs=None,
layer_type='bbb'):
net.train()
training_loss = 0.0
accs = []
kl_list = []
for i, (inputs, labels) in enumerate(trainloader, 1):
optimizer.zero_grad()
inputs, labels = inputs.to(device), labels.to(device)
outputs = torch.zeros(inputs.shape[0], net.num_classes,
num_ens).to(device)
if layer_type == 'mgp':
log_priors = torch.zeros(num_ens).to(device)
log_variational_posteriors = torch.zeros(num_ens).to(device)
for j in range(num_ens):
net_out, log_prior, log_variational_posterior = net(inputs)
outputs[:, :, j] = F.log_softmax(net_out, dim=1)
log_priors[j] = log_prior
log_variational_posteriors[j] = log_variational_posterior
log_prior = log_priors.mean()
log_variational_posterior = log_variational_posteriors.mean()
kl_list.append((log_variational_posterior - log_prior).item())
log_outputs = utils.logmeanexp(outputs, dim=2)
beta = metrics.get_beta(i - 1, len(trainloader), beta_type, epoch,
num_epochs)
loss = criterion(log_outputs, labels, log_prior,
log_variational_posterior, beta)
else:
kl = 0.0
for j in range(num_ens):
net_out, _kl = net(inputs)
kl += _kl
outputs[:, :, j] = F.log_softmax(net_out, dim=1)
kl = kl / num_ens
kl_list.append(kl.item())
log_outputs = utils.logmeanexp(outputs, dim=2)
beta = metrics.get_beta(i - 1, len(trainloader), beta_type, epoch,
num_epochs)
loss = criterion(log_outputs, labels, kl, beta)
loss.backward()
optimizer.step()
accs.append(metrics.acc(log_outputs.data, labels))
training_loss += loss.cpu().data.numpy()
return training_loss / len(trainloader), np.mean(accs), np.mean(kl_list)
def match(y,cl):
cl=np.array(cl)
y=np.array(y)
acc = np.round(metrics.acc(y, cl), 5)
nmi = np.round(metrics.nmi(y, cl), 5)
ari = np.round(metrics.ari(y, cl), 5)
return acc,nmi,ari
def train_and_validate(train_paths, val_paths):
############################## Training Process ###########################################
X_train, Y_train = load_split(train_paths[0], train_paths[1])
print(X_train.var())
print("Started Training")
clf = svm.SVC(gamma='scale',
kernel="rbf",
C=1.,
degree=2,
verbose=False,
probability=True)
#clf = svm.SVC(gamma="scale", kernel="rbf", C=2., verbose=False, probability=True)
clf.fit(X_train, Y_train)
joblib.dump(clf, 'data/model_repo/svm_test.pkl')
# ############################## Validation Process ###########################################
print("Preparing validation documents")
X_val, Y_val = load_split(val_paths[0], val_paths[1])
predictions = clf.predict(X_val)
np.savetxt("data/errors/system_predictions_bow.txt",
clf.predict_proba(X_val))
np.savetxt("data/errors/gt.txt", Y_val)
val_accuracy = metrics.acc(predictions, Y_val)
return val_accuracy
def validate(self, validate_dl):
iterator = tqdm(validate_dl, leave=True, dynamic_ncols=True)
iter_len = len(iterator)
errs = []
accs = []
for i, (_, data, label) in enumerate(iterator):
iterator.set_description(
f'validate:[{self.epoch}/{self.opt.epochs}|{self.global_steps}]'
)
err, pred = self.validate_step(data, label)
pred = self.get_pred_number(pred)
label = self.get_pred_number(label.to(self.opt.device))
acc = metrics.acc(pred, label)
accs.append(acc)
errs.append(err)
curr_acc = torch.tensor(accs).mean().item()
self.writer.add_scalar('validate/error',
torch.tensor(errs).mean().item(),
self.global_steps)
self.writer.add_scalar('validate/acc', curr_acc, self.global_steps)
if curr_acc > self.best_acc:
self.best_acc = curr_acc
self.test()
def validate_model(net,
criterion,
validloader,
num_ens=1,
beta_type=0.1,
epoch=None,
num_epochs=None):
"""Calculate ensemble accuracy and NLL Loss"""
net.train()
valid_loss = 0.0
accs = []
for i, (inputs, labels) in enumerate(validloader):
inputs, labels = inputs.to(device), labels.to(device)
outputs = torch.zeros(inputs.shape[0], net.num_classes,
num_ens).to(device)
kl = 0.0
for j in range(num_ens):
net_out, _kl = net(inputs)
kl += _kl
outputs[:, :, j] = F.log_softmax(net_out, dim=1).data
log_outputs = utils.logmeanexp(outputs, dim=2)
beta = metrics.get_beta(i - 1, len(validloader), beta_type, epoch,
num_epochs)
valid_loss += criterion(log_outputs, labels, kl, beta).item()
accs.append(metrics.acc(log_outputs, labels))
return valid_loss / len(validloader), np.mean(accs)
def train_model(net, optimizer, criterion, trainloader, num_ens=1):
net.train()
training_loss = 0.0
accs = []
kl_list = []
for i, (inputs, labels) in enumerate(trainloader, 0):
optimizer.zero_grad()
inputs, labels = inputs.to(device), labels.to(device)
outputs = torch.zeros(inputs.shape[0], net.num_classes,
num_ens).to(device)
kl = 0.0
for j in range(num_ens):
net_out, _kl = net(inputs)
kl += _kl
outputs[:, :, j] = F.log_softmax(net_out, dim=1)
kl = kl / num_ens
kl_list.append(kl.item())
log_outputs = utils.logmeanexp(outputs, dim=2)
loss = criterion(log_outputs, labels, kl)
loss.backward()
optimizer.step()
accs.append(metrics.acc(log_outputs.data, labels))
training_loss += loss.cpu().data.numpy()
return training_loss / len(trainloader), np.mean(accs), np.mean(kl_list)
def metric(self, y, y_pred):
acc = np.round(metrics.acc(y, y_pred), 5)
nmi = np.round(metrics.nmi(y, y_pred), 5)
ari = np.round(metrics.ari(y, y_pred), 5)
print('acc:', acc)
print('nmi:', nmi)
print('ari:', ari)
def fit(self,
x,
y=None,
maxiter=2e4,
batch_size=256,
tol=1e-3,
update_interval=140,
save_dir='./results/temp',
rand_seed=None):
print('Update interval', update_interval)
save_interval = int(x.shape[0] / batch_size) * 5 # 5 epochs
print('Save interval', save_interval)
# Step 1: initialize cluster centers using k-means
print('Initializing cluster centers with k-means.')
kmeans = KMeans(n_clusters=self.n_clusters, n_init=100)
y_pred = kmeans.fit_predict(self.encoder.predict(x))
y_pred_last = np.copy(y_pred)
self.model.get_layer(name='clustering').set_weights(
[kmeans.cluster_centers_])
loss = 0
index = 0
index_array = np.arange(x.shape[0])
for ite in range(int(maxiter)):
if ite % update_interval == 0:
q = self.model.predict(x, verbose=0)
p = self.target_distribution(q)
y_pred = q.argmax(1)
if y is not None:
acc = np.round(metrics.acc(y, y_pred), 5)
nmi = np.round(metrics.nmi(y, y_pred), 5)
loss = np.round(loss, 5)
print('Iter %d: acc = %.5f, nmi = %.5f' % (ite, acc, nmi),
' ; loss=', loss)
# check stop criterion
delta_label = np.sum(y_pred != y_pred_last).astype(
np.float32) / y_pred.shape[0]
y_pred_last = np.copy(y_pred)
if ite > 0 and delta_label < tol:
print('delta_label ', delta_label, '< tol ', tol)
print('Reached tolerance threshold. Stopping training.')
break
idx = index_array[index * batch_size:min((index + 1) *
batch_size, x.shape[0])]
loss = self.model.train_on_batch(x=x[idx], y=p[idx])
index = index + 1 if (index + 1) * batch_size <= x.shape[0] else 0
ite += 1
# save the trained model
print('saving model to:', save_dir + 'STC_model_final.h5')
self.model.save_weights(save_dir + 'STC_model_final.h5')
return y_pred
def significance_test_holdout():
system_a_pred = np.loadtxt(
"data/trained_models/predictions_old/no_fold/unigram_false_bigram_true_laplace_true.txt"
)
system_b_pred = np.loadtxt(
"data/trained_models/predictions_old/no_fold/unigram_false_bigram_true_laplace_true.txt"
)
gt = [1] * 100 + [0] * 100
sign_test_result = metrics.sign_test_precision(system_a_pred,
system_b_pred, gt)
system_a_acc = metrics.acc(system_a_pred, gt)
system_b_acc = metrics.acc(system_b_pred, gt)
print("The accuracy of system A is: ", system_a_acc)
print("The accuracy of system B is: ", system_b_acc)
print("The result of the sign test is: ", sign_test_result)
def train(args):
# get data and model
(x, y), model = _get_data_and_model(args)
# split train validation data
if y is None:
x_train, x_val = train_test_split(x, test_size=0.1)
y_val = None
y_train = None
else:
x_train, x_val, y_train, y_val = train_test_split(x,
y,
stratify=y,
test_size=0.1)
model.model.summary()
# pretraining
t0 = time()
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
if args.pretrained_weights is not None and os.path.exists(
args.pretrained_weights): # load pretrained weights
model.autoencoder.load_weights(args.pretrained_weights)
else: # train
pretrain_optimizer = SGD(1.0, 0.9) if args.method in [
'FcDEC', 'FcIDEC', 'FcDEC-DA', 'FcIDEC-DA'
] else 'adam'
model.pretrain(x_train,
y_train,
x_val,
y_val,
optimizer=pretrain_optimizer,
epochs=args.pretrain_epochs,
batch_size=args.batch_size,
save_dir=args.save_dir,
verbose=args.verbose,
aug_pretrain=args.aug_pretrain)
t1 = time()
print("Time for pretraining: %ds" % (t1 - t0))
# clustering
y_pred = model.fit(x,
y,
maxiter=args.maxiter,
batch_size=args.batch_size,
update_interval=args.update_interval,
save_dir=args.save_dir,
aug_cluster=args.aug_cluster)
if y is not None:
print('Final: acc=%.4f, nmi=%.4f, ari=%.4f' % (metrics.acc(
y, y_pred), metrics.nmi(y, y_pred), metrics.ari(y, y_pred)))
t2 = time()
print("Time for pretaining, clustering and total: (%ds, %ds, %ds)" %
(t1 - t0, t2 - t1, t2 - t0))
print('=' * 60)
def validate_model(net, criterion, valid_loader):
valid_loss = 0.0
net.eval()
accs = []
for data, target in valid_loader:
data, target = data.to(device), target.to(device)
output = net(data)
loss = criterion(output, target)
valid_loss += loss.item() * data.size(0)
accs.append(metrics.acc(output.detach(), target))
return valid_loss, np.mean(accs)
def on_epoch_end(self, epoch, logs=None):
if int(epochs / 10) != 0 and epoch % int(epochs / 10) != 0:
return
feature_model = tf.keras.models.Model(self.model.input,
self.model.get_layer('encoder_3').output)
features = feature_model.predict(self.x)
km = KMeans(n_clusters=len(np.unique(self.y)), n_init=20, n_jobs=4)
y_pred = km.fit_predict(features)
# print()
print(' ' * 8 + '|==> acc: %.4f, nmi: %.4f <==|'
% (metrics.acc(self.y, y_pred), metrics.nmi(self.y, y_pred)))
def test(args):
assert args.weights is not None
(x, y), model = _get_data_and_model(args)
model.model.summary()
print('Begin testing:', '-' * 60)
model.load_weights(args.weights)
y_pred = model.predict_labels(x)
print('acc=%.4f, nmi=%.4f, ari=%.4f' % (metrics.acc(
y, y_pred), metrics.nmi(y, y_pred), metrics.ari(y, y_pred)))
print('End testing:', '-' * 60)
def predict_regular(net, validloader, bayesian=True, num_ens=10):
"""
For both bayesian and Frequentist models
"""
net.eval()
accs = []
for i, (inputs, labels) in enumerate(validloader):
inputs, labels = inputs.to(device), labels.to(device)
if bayesian:
outputs = torch.zeros(inputs.shape[0], net.num_classes, num_ens).to(device)
for j in range(num_ens):
net_out, _ = net(inputs)
outputs[:, :, j] = F.log_softmax(net_out, dim=1).data
log_outputs = utils.logmeanexp(outputs, dim=2)
accs.append(metrics.acc(log_outputs, labels))
else:
output = net(inputs)
accs.append(metrics.acc(output.detach(), labels))
return np.mean(accs)
def kmeans_():
# use features for clustering
from sklearn.cluster import KMeans
km = KMeans(n_clusters=N, init='k-means++')
#features = np.reshape(x_train, newshape=(features.shape[0], -1))
km_trans = km.fit_transform(x_train)
pred = km.predict(x_train)
print pred.shape
print('acc=', met.acc(y_train, pred), 'nmi=', met.nmi(y_train,
pred), 'ari=',
met.ari(y_train, pred))
return km_trans, pred
def fit(self, x, y=None, save_dir='./results/temp'):
# print('Begin training:', '-' * 60)
t1 = time()
print(
'******************** Use Denpeak to Cluster ************************'
)
features = self.encoder.predict(x)
print("features shape:", features.shape)
features = TSNE(n_components=2).fit_transform(features)
# np.savetxt("features.txt", features)
print("features shape:", features.shape)
y_pred, y_border, center_num, dc_percent, dc = DenPeakCluster(features)
print('saving picture to:', save_dir + '/2D.png')
plt.cla()
plt.scatter(features[:, 0], features[:, 1], c=y_pred, s=0.5, alpha=0.5)
plt.savefig(save_dir + '/2D.png')
np.savetxt(save_dir + '/dc_coeff.txt', [dc_percent, dc])
# logging file
import csv, os
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logfile = open(save_dir + '/log.csv', 'w')
logwriter = csv.DictWriter(
logfile,
fieldnames=['iter', 'acc', 'nmi', 'ari', 'loss', 'center_num'])
logwriter.writeheader()
acc = np.round(metrics.acc(y, y_pred), 5)
nmi = np.round(metrics.nmi(y, y_pred), 5)
ari = np.round(metrics.ari(y, y_pred), 5)
# if acc>=0.95:
np.savetxt(save_dir + '/features.txt', features)
np.savetxt(save_dir + '/labels.txt', y_pred)
np.savetxt(save_dir + '/border.txt', y_border)
from Draw_border import draw
draw(save_dir)
logdict = dict(iter=0,
acc=acc,
nmi=nmi,
ari=ari,
center_num=center_num)
logwriter.writerow(logdict)
logfile.flush()
print('Iter %d: acc=%.5f, nmi=%.5f, ari=%.5f; center_num=%d' %
(0, acc, nmi, ari, center_num))
logfile.close()
return y_pred
def init_center_with_kmeans(n_clusters, X, dims, y_real=None, device="cpu"):
model = Encoder(dims)
enc_dec_model = {k[2:]: v for k, v in torch.load("enc_dec_model").items()}
model.load_state_dict(enc_dec_model, strict=False)
model = model.to(device)
model.eval()
with torch.no_grad():
X_encoded = model(torch.from_numpy(X).to(device))
kmeans = KMeans(n_clusters=n_clusters, n_init=20)
y_pred = kmeans.fit_predict(X_encoded.cpu().numpy())
if y_real is not None:
print("kmeans acc: {}".format(metrics.acc(y_real, y_pred)))
np.save("cluster_centers.npy", kmeans.cluster_centers_)
def train_model(net, optimizer, criterion, train_loader):
train_loss = 0.0
net.train()
accs = []
for data, target in train_loader:
data, target = data.to(device), target.to(device)
optimizer.zero_grad()
output = net(data)
loss = criterion(output, target)
loss.backward()
optimizer.step()
train_loss += loss.item() * data.size(0)
accs.append(metrics.acc(output.detach(), target))
return train_loss, np.mean(accs)
def train_feature(net1, train_data):
map_dict = read_pkl()
if torch.cuda.is_available():
net1 = torch.nn.DataParallel(net1, device_ids=[0])
net1 = net1.cuda()
prev_time = datetime.now()
for i_dir in range(classnum):
if not os.path.isdir('./data/' + str(i_dir)):
os.makedirs('./data/' + str(i_dir))
label_np = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0] * 10).reshape(10, 10)
# label_np = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] * 20).reshape(20, 20)
label2 = []
idx2 = []
for im, label in tqdm(train_data, desc="Processing train data: "):
im = im.cuda()
feat = net1(im)
for i in range(feat.size(0)):
distance_list = list()
for ui_50D_label in map_dict.values():
distance = sum(sum((ui_50D_label.float().cuda() - feat[i])**2))
distance_list.append(distance.item())
idx = distance_list.index(min(distance_list))
save_image(
inver_transform2(im[i]), './data/' + str(idx) + '/' +
str(random.randint(1, 10000000)) + '.png')
label_np[idx][label[i].item()] += 1
label2.append(idx)
label1 = label.numpy()
# for _,i in enumerate(label):
# idx2.append(i)
for i in label1:
idx2.append(i)
t2 = np.array(idx2)
t1 = np.array(label2)
# print(t2.shape)
# t2 = t2.reshape([t1.size,-1]).squeeze(0)
print('acc=%.4f, nmi=%.4f, ari=%.4f' %
(metrics.acc(t1, t2), metrics.nmi(t1, t2), metrics.ari(t1, t2)))
corr_num = 0
for item in label_np:
corr_num += item.max()
corr = corr_num / label_np.sum()
print(corr)
np.save('./model/MNIST/feature/' + str(feat.size(1)) + '_' + '.npy',
label_np)
def load_pretrain_weights(self, vade, weights_path, dataset, inputs, Y):
vade.load_weights(weights_path + dataset + '.h5')
sample = self.sample_output.predict(inputs, batch_size=self.batch_size)
kmeans = KMeans(n_clusters=self.n_centroid, n_init=20)
kmeans.fit(sample)
self.u_p.set_value(self.floatX(kmeans.cluster_centers_.T))
y_pred = kmeans.predict(sample)
gam = self.gamma_output.predict(inputs, batch_size=batch_size)
gam_acc = metrics.acc(np.argmax(gam, axis=1), Y)
print('pretrain weights loaded!')
print('Initial_acc:', gam_acc)
return vade
def gmm_kmeans_cluster(self, dataloader):
use_cuda = torch.cuda.is_available()
if use_cuda:
self.cuda()
self.eval()
data = []
Y = []
for batch_idx, (inputs, y) in enumerate(dataloader):
inputs = inputs.view(inputs.size(0), -1).float()
if use_cuda:
inputs = inputs.cuda()
inputs = Variable(inputs)
_, _, _, mu, _ = self.forward(inputs)
data.append(mu.data.cpu().numpy())
Y.append(y.numpy())
data = np.concatenate(data)
Y = np.concatenate(Y)
gmm = GaussianMixture(n_components=self.n_centroids,
covariance_type='full')
gmm.fit(data)
y_pred_gmm = gmm.predict(data)
acc = np.round(metrics.acc(Y, y_pred_gmm), 5)
nmi = np.round(metrics.nmi(Y, y_pred_gmm), 5)
ari = np.round(metrics.ari(Y, y_pred_gmm), 5)
print(
'GMM fit of AutoEncoder embedding: acc = %.5f, nmi = %.5f, ari = %.5f'
% (acc, nmi, ari))
km = KMeans(n_clusters=self.n_centroids, n_init=20)
y_pred_kmeans = km.fit_predict(data)
acc = np.round(metrics.acc(Y, y_pred_kmeans), 5)
nmi = np.round(metrics.nmi(Y, y_pred_kmeans), 5)
ari = np.round(metrics.ari(Y, y_pred_kmeans), 5)
print(
'Kmeans clustering of AutoEncoder embedding: acc = %.5f, nmi = %.5f, ari = %.5f'
% (acc, nmi, ari))
def on_epoch_end(self, epoch, logs=None):
if epochs < 10 or epoch % int(epochs / 10) != 0:
return
feature_model = Model(
self.model.input,
self.model.get_layer(
index=int(len(self.model.layers) / 2)).output)
features = feature_model.predict(self.x)
km = KMeans(n_clusters=len(np.unique(self.y)),
n_init=20,
n_jobs=4)
y_pred = km.fit_predict(features)
print(' ' * 8 + '|==> acc: %.4f, nmi: %.4f <==|' %
(metrics.acc(self.y, y_pred),
metrics.nmi(self.y, y_pred)))
def generate_predictions(vocab_path, vocab_pos_freq_path, vocab_neg_freq_path,
prior_pos_path, prior_neg_path, pos_test, neg_test):
with open(vocab_path, 'r') as f:
vocabulary = f.read().splitlines()
with open(vocab_pos_freq_path, 'r') as f:
vocab_pos_freq = []
pos_freq = f.read().splitlines()
for item in pos_freq:
vocab_pos_freq.append(float(item))
with open(vocab_neg_freq_path, 'r') as f:
vocab_neg_freq = []
neg_freq = f.read().splitlines()
for item in neg_freq:
vocab_neg_freq.append(float(item))
with open(prior_pos_path, 'r') as f:
prior_pos = float(f.read().splitlines()[0])
with open(prior_neg_path, 'r') as f:
prior_neg = float(f.read().splitlines()[0])
# Generate the predictions by using a saved model
m = multiprocessing.Manager()
preds = m.list()
with multiprocessing.Pool(processes=multiprocessing.cpu_count() -
45) as pool:
pool.map(
partial(classifiers.apply_multinomial_NB, vocabulary, prior_pos,
prior_neg, vocab_pos_freq, vocab_neg_freq, 1, preds),
pos_test)
with multiprocessing.Pool(processes=multiprocessing.cpu_count() -
45) as pool:
pool.map(
partial(classifiers.apply_multinomial_NB, vocabulary, prior_pos,
prior_neg, vocab_pos_freq, vocab_neg_freq, 0, preds),
neg_test)
all_gt = np.array(preds)[:, 0]
all_preds = np.array(preds)[:, 1]
overall_accuracy = metrics.acc(all_preds, all_gt)
print("The overall accuracy of the model is: ", overall_accuracy)
def predict_using_uncertainty_separate_models(
net1, net2, valid_loader, uncertainty_type='epistemic_softmax', T=25):
"""
For Bayesian models
"""
accs = []
total_u1 = 0.0
total_u2 = 0.0
set1_selected = 0
set2_selected = 0
epi_or_ale, soft_or_norm = uncertainty_type.split('_')
soft_or_norm = True if soft_or_norm == 'normalized' else False
for i, (inputs, labels) in enumerate(valid_loader):
inputs, labels = inputs.to(device), labels.to(device)
pred1, epi1, ale1 = ue.get_uncertainty_per_batch(
net1, inputs, T=T, normalized=soft_or_norm)
pred2, epi2, ale2 = ue.get_uncertainty_per_batch(
net2, inputs, T=T, normalized=soft_or_norm)
if epi_or_ale == 'epistemic':
u1 = np.sum(epi1, axis=1)
u2 = np.sum(epi2, axis=1)
elif epi_or_ale == 'aleatoric':
u1 = np.sum(ale1, axis=1)
u2 = np.sum(ale2, axis=1)
elif epi_or_ale == 'both':
u1 = np.sum(epi1, axis=1) + np.sum(ale1, axis=1)
u2 = np.sum(epi2, axis=1) + np.sum(ale2, axis=1)
else:
raise ValueError("Not correct uncertainty type")
total_u1 += np.sum(u1).item()
total_u2 += np.sum(u2).item()
set1_preferred = u2 > u1 # idx where set1 has less uncertainty
set1_preferred = np.expand_dims(set1_preferred, 1)
preds = np.where(set1_preferred, pred1, pred2)
set1_selected += np.sum(set1_preferred)
set2_selected += np.sum(~set1_preferred)
accs.append(metrics.acc(torch.tensor(preds), labels))
return np.mean(accs), set1_selected/(set1_selected + set2_selected), \
set2_selected/(set1_selected + set2_selected), total_u1, total_u2
def train_model(net,
optimizer,
criterion,
trainloader,
num_ens=1,
beta_type=0.1):
net.train()
training_loss = 0.0
accs = []
kl_list = []
freq = cfg.recording_freq_per_epoch
freq = len(trainloader) // freq
for i, (inputs, labels) in enumerate(trainloader, 1):
cfg.curr_batch_no = i
if i % freq == 0:
cfg.record_now = True
else:
cfg.record_now = False
optimizer.zero_grad()
inputs, labels = inputs.to(device), labels.to(device)
outputs = torch.zeros(inputs.shape[0], net.num_classes,
num_ens).to(device)
kl = 0.0
for j in range(num_ens):
net_out, _kl = net(inputs)
kl += _kl
outputs[:, :, j] = F.log_softmax(net_out, dim=1)
kl = kl / num_ens
kl_list.append(kl.item())
log_outputs = utils.logmeanexp(outputs, dim=2)
beta = metrics.get_beta(i - 1, len(trainloader), beta_type)
loss = criterion(log_outputs, labels, kl, beta)
loss.backward()
optimizer.step()
accs.append(metrics.acc(log_outputs.data, labels))
training_loss += loss.cpu().data.numpy()
return training_loss / len(trainloader), np.mean(accs), np.mean(kl_list)
def test_model(net, criterion, testloader, num_ens=10):
"""Calculate ensemble accuracy and NLL Loss"""
net.eval()
test_loss = 0.0
accs = []
for i, (inputs, labels) in enumerate(testloader):
inputs, labels = inputs.to(device), labels.to(device)
outputs = torch.zeros(inputs.shape[0], net.num_classes,
num_ens).to(device)
kl = 0.0
for j in range(num_ens):
net_out, _kl = net(inputs)
kl += _kl
outputs[:, :, j] = F.log_softmax(net_out, dim=1).data
log_outputs = utils.logmeanexp(outputs, dim=2)
test_loss += criterion(log_outputs, labels, kl).item()
accs.append(metrics.acc(log_outputs, labels))
return test_loss / len(testloader), np.mean(accs)
def fold_acc():
gt = [1] * 100 + [0] * 100
#system_a_pred = np.loadtxt("data/trained_models/predictions/cross_fold/0/unigram_false_bigram_true_laplace_false_stopwords_false.txt")
#system_b_pred = np.loadtxt("data/trained_models/predictions/cross_fold/0/unigram_true_bigram_false_laplace_false_stopwords_false.txt")
overall_acc = []
for i in range(0, 10):
system_a_pred = np.loadtxt(
"data/trained_models_new/predictions/cross_fold/" + str(i) +
"/unigram_true_bigram_true_laplace_true_stopwords_false.txt")
fold_acc = metrics.acc(system_a_pred, gt)
print("Accuracy for fold: ", str(i), " is ", fold_acc)
overall_acc.append(fold_acc)
all_acc = sum(overall_acc) / 10.
print("The overall accuracy is: ", all_acc)
print("The mean is: ", np.mean(all_acc))
print("The variance is: ",
math.sqrt(np.mean(abs(overall_acc - np.mean(overall_acc))**2)))
def test(net1, test_data):
#
if torch.cuda.is_available():
net1 = torch.nn.DataParallel(net1, device_ids=[0])
net1 = net1.cuda()
#
label2 = []
idx2 = []
for im, label in tqdm(test_data, desc="Processing train data: "):
im = im.cuda()
_, feat = net1(im)
for i in range(feat.size(0)):
distance = feat[i].cpu().numpy().tolist()
idx = distance.index(max(distance))
label2.append(idx)
label1 = label.numpy()
for i in label1:
idx2.append(i)
t2 = np.array(idx2)
t1 = np.array(label2)
return metrics.acc(t2, t1), metrics.nmi(t2, t1)
def score(self, X, y):
"""Returns a score of how well the classifier is trained to predict the data."""
return acc(y, self.trainer.testOnClassData(dataset=convert_to_pybrain_dataset(X,y)))
classifiers = [
adaboost.train(Xt1, Yt1),
extra_randomized_trees.train(Xt1, Yt1),
gradient_boost.train(Xt1, Yt1),
random_forest.train(Xt1, Yt1),
logistic_regression.train(Xt1, Yt1),
]
# Train another classifier on the ensembles output training predictions
# for each sample in the training data
training_predictions = np.mat([[c.predict(sample)[0] for c in classifiers] for sample in Xt1])
meta_classifier = logistic_regression.train(training_predictions, Yt1)
# Check results on training data
print "Accuracy for individual classifiers:", [acc(Yt2, c.predict(Xt2)) for c in classifiers]
predictions = np.mat([c.predict(Xt2) for c in classifiers]).transpose()
print "Accuracy for ensemble classifier:", acc(Yt2, meta_classifier.predict(predictions))
else:
# Now, we train each classifier on the training data
classifiers = [
adaboost.train(Xt, Yt),
extra_randomized_trees.train(Xt, Yt),
gradient_boost.train(Xt, Yt),
random_forest.train(Xt, Yt),
logistic_regression.train(Xt, Yt),
]
# Train another classifier on the ensembles output training predictions
# for each sample in the training data
training_predictions = np.mat([[c.predict(sample)[0] for c in classifiers] for sample in Xt])
from sklearn import tree
def classify(Xtrain, Ytrain):
""" Use entirety of provided X, Y to predict
Arguments
Xtrain -- Training data
Ytrain -- Training prediction
Returns
ready_tree -- a tree fitted to Xtrain and Ytrain
"""
ready_tree = tree.DecisionTreeClassifier()
ready_tree.fit(Xtrain, Ytrain)
return ready_tree
if __name__ == "__main__":
# Let's take our training data and train a decision tree
# on a subset. Scikit-learn provides a good module for cross-
# validation.
if len(sys.argv) < 2:
print "Usage: $ python decision-tree.py /path/to/data/file/"
else:
training = sys.argv[1]
X,Y,n,f = load_data(training)
Xt, Xv, Yt, Yv = shuffle_split(X,Y)
tree = classify(Xt, Yt)
print "Decision Tree Accuracy:",acc(Yv, tree.predict(Xv)),"%"
