def eval_ood(src_classifier, src_data_loader, tgt_data_loader_eval):
mean, inv, m_mean, m_std = get_distribution(src_data_loader)
"""Evaluate classifier for source domain."""
# set eval state for Dropout and BN layers
src_classifier.eval()
acc = 0
f1 = 0
ys_true = []
ys_pred = []
for (images, labels) in tgt_data_loader_eval:
images = make_variable(images, volatile=True)
labels = make_variable(labels).detach().cpu().numpy()
preds = src_classifier(images)
for pred, image, label in zip(preds, images, labels):
if is_in_distribution(image.detach().cpu().numpy(), mean, inv,
m_mean, m_std):
ys_true.append(label)
ys_pred.append(np.argmax(pred.detach().cpu().numpy()))
else:
continue
acc = accuracy_score(ys_true, ys_pred)
f1 = get_f1(ys_pred, ys_true, 'macro')
print(" F1 = {:2%}, accuracy = {:2%}".format(f1, acc))
def eval_tgt(encoder, classifier, data_loader):
"""Evaluation for target encoder by source classifier on target dataset."""
# set eval state for Dropout and BN layers
encoder.eval()
classifier.eval()
# init loss and accuracy
loss = 0
acc = 0
f1 = 0
# set loss function
criterion = nn.CrossEntropyLoss()
flag = False
# evaluate network
for (images, labels) in data_loader:
images = make_variable(images, volatile=True)
labels = make_variable(labels).squeeze_()
preds = classifier(encoder(images))
loss += criterion(preds,
labels).data # criterion is cross entropy loss
pred_cls = preds.data.max(1)[1]
#f1 += get_f1(pred_cls, labels.data, average='macro')
acc += pred_cls.eq(labels.data).cpu().sum()
if not flag:
ys_pred = pred_cls
ys_true = labels
flag = True
else:
ys_pred = torch.cat((ys_pred, pred_cls), 0)
ys_true = torch.cat((ys_true, labels), 0)
loss = loss.float()
acc = acc.float()
loss /= len(data_loader)
acc /= len(data_loader.dataset)
#f1 /= len(data_loader.dataset)
f1 = get_f1(ys_pred, ys_true, 'macro')
f1_weighted = get_f1(ys_pred, ys_true, 'weighted')
print("Avg Loss = {}, F1 = {:2%}, Weighted F1 = {:2%}".format(
loss, f1, f1_weighted))
def eval_tgt_with_probe(encoder, critic, src_classifier, tgt_classifier,
data_loader):
"""Evaluation for target encoder by source classifier on target dataset."""
# set eval state for Dropout and BN layers
encoder.eval()
src_classifier.eval()
tgt_classifier.eval()
# init loss and accuracy
loss = 0
acc = 0
f1 = 0
ys_pred = []
ys_true = []
# set loss function
criterion = nn.CrossEntropyLoss()
flag = False
# evaluate network
for (images, labels) in data_loader:
images = make_variable(images, volatile=True)
labels = make_variable(labels).squeeze_()
probeds = critic(encoder(images))
for image, label, probed in zip(images, labels, probeds):
if torch.argmax(probed) == 1:
pred = torch.argmax(
src_classifier(encoder(torch.unsqueeze(
image, 0)))).detach().cpu().numpy()
else:
pred = torch.argmax(
tgt_classifier(encoder(torch.unsqueeze(
image, 0)))).detach().cpu().numpy()
ys_pred.append(np.squeeze(pred))
ys_true.append(np.squeeze(label.detach().cpu().numpy()))
loss /= len(data_loader)
acc /= len(data_loader.dataset)
#f1 /= len(data_loader.dataset)
f1 = get_f1(ys_pred, ys_true, 'macro')
f1_weighted = get_f1(ys_pred, ys_true, 'weighted')
print("Avg Loss = {}, F1 = {:2%}, Weighted F1 = {:2%}".format(
loss, f1, f1_weighted))
def eval(classifier, data_loader):
"""Evaluate classifier for source domain."""
# set eval state for Dropout and BN layers
classifier.eval()
# init loss and accuracy
loss = 0
acc = 0
f1 = 0
# set loss function
criterion = nn.CrossEntropyLoss()
flag = False
# evaluate network
for (images, labels) in data_loader:
images = make_variable(images, volatile=True)
labels = make_variable(labels)
preds = classifier(images)
loss += criterion(preds, labels).data
pred_cls = preds.data.max(1)[1]
acc += pred_cls.eq(labels.data).cpu().sum()
if not flag:
ys_pred = pred_cls
ys_true = labels
flag = True
else:
ys_pred = torch.cat((ys_pred, pred_cls), 0)
ys_true = torch.cat((ys_true, labels), 0)
loss = loss.float()
acc = acc.float()
loss /= len(data_loader)
acc /= len(data_loader.dataset)
#f1 /= len(data_loader.dataset)
f1 = get_f1(ys_pred, ys_true, 'weighted')
#f1_weighted = get_f1(ys_pred, ys_true, 'weighted')
print("Avg Loss = {}, F1 = {:2%}, accuracy = {:2%}".format(loss, f1, acc))
def main():
logging.basicConfig(filename='logs/train.log', level=logging.DEBUG)
# saved model path
save_path = 'history/trained_model'
# input file
#filename = 'data/train_and_test.csv'
filename = 'data/golden_400.csv'
embedding_size = 300 # 128 for torch embeddings, 300 for pre-trained
hidden_size = 24
batch_size = 64
nb_epochs = 200
lr = 1e-4
max_norm = 5
folds = 3
# Dataset
ds = ClaimsDataset(filename)
vocab_size = ds.vocab.__len__()
pad_id = ds.vocab.token2id.get('<pad>')
test_len = val_len = math.ceil(ds.__len__() * .10)
train_len = ds.__len__() - (val_len + test_len)
print("\nTrain size: {}\tValidate size: {}\tTest Size: {}".format(
train_len, val_len, test_len))
# randomly split dataset into tr, te, & val sizes
d_tr, d_val, d_te = torch.utils.data.dataset.random_split(
ds, [train_len, val_len, test_len])
# data loaders
dl_tr = torch.utils.data.DataLoader(d_tr, batch_size=batch_size)
dl_val = torch.utils.data.DataLoader(d_val, batch_size=batch_size)
dl_test = torch.utils.data.DataLoader(d_te, batch_size=batch_size)
model = RNN(vocab_size, embedding_size, hidden_size, pad_id, ds)
model = utils.cuda(model)
model.zero_grad()
parameters = list([
parameter for parameter in model.parameters()
if parameter.requires_grad
])
#parameters = list(model.parameters()) # comment out when using pre-trained embeddings
optim = torch.optim.Adam(parameters,
lr=lr,
weight_decay=35e-3,
amsgrad=True) # optimizer
criterion = nn.NLLLoss(weight=torch.Tensor([1.0, 2.2]).cuda())
losses = defaultdict(list)
print("\nTraining started: {}\n".format(utils.get_time()))
phases, loaders = ['train', 'val'], [dl_tr, dl_val]
tr_acc, v_acc = [], []
for epoch in range(nb_epochs):
for phase, loader in zip(phases, loaders):
if phase == 'train':
model.train()
else:
model.eval()
ep_loss, out_list, label_list = [], [], []
for i, inputs in enumerate(loader):
optim.zero_grad()
claim, labels = inputs
labels = utils.variable(labels)
out = model(claim)
out_list.append(utils.normalize_out(
out)) # collect output from every epoch
label_list.append(labels)
out = torch.log(out)
# criterion.weight = get_weights(labels)
loss = criterion(out, labels)
# back propagate, for training only
if phase == 'train':
loss.backward()
torch.nn.utils.clip_grad_norm_(
parameters,
max_norm=max_norm) # exploding gradients? say no more!
optim.step()
ep_loss.append(loss.item())
losses[phase].append(
np.mean(ep_loss)
) # record average losses from every phase at each epoch
acc = utils.get_accuracy(label_list, out_list)
if phase == 'train':
tr_acc.append(acc)
else:
v_acc.append(acc)
print("Epoch: {} \t Phase: {} \t Loss: {:.4f} \t Accuracy: {:.3f}".
format(epoch, phase, loss, acc))
print("\nTime finished: {}\n".format(utils.get_time()))
utils.plot_loss(losses['train'], losses['val'], tr_acc, v_acc, filename,
-1)
logging.info("\nTrain file=> " + filename +
"\nParameters=> \nBatch size: " + str(batch_size) +
"\nHidden size: " + str(hidden_size) + "\nMax_norm: " +
str(max_norm) + "\nL2 Reg/weight decay: " +
str(optim.param_groups[0]['weight_decay']) +
"\nLoss function: \n" + str(criterion))
logging.info('Final train accuracy: ' + str(tr_acc[-1]))
logging.info('Final validation accuracy: ' + str(v_acc[-1]))
# Save the model
torch.save(model.state_dict(), save_path)
#test(model, batch_size)
# predict
f1_test, acc_test = [], []
for i, inputs in enumerate(dl_test):
claim, label = inputs
label = utils.variable(label.float())
out = model(claim)
y_pred = utils.normalize_out(out)
#print("\n\t\tF1 score: {}\n\n".format(get_f1(label, y_pred))) # f1 score
f1_test.append(utils.get_f1(label, y_pred))
acc_test.append(metrics.accuracy_score(label, y_pred))
print("\t\tF1: {:.3f}\tAccuracy: {:.3f}".format(np.mean(f1_test),
np.mean(acc_test)))
logging.info('\nTest f1: ' + str(np.mean(f1_test)) + '\nTest Accuracy: ' +
str(np.mean(acc_test)))
def main():
# saved model path
save_path = 'history/model_fold_'
test_file = 'data/test120.csv'
# create dataset
#filename = 'data/golden_400.csv'
#filename = 'data/golden_train_and_val.csv'
filename = 'data/train_val120.csv'
ds = ClaimsDataset(filename)
vocab_size = ds.vocab.__len__()
pad_id = ds.vocab.token2id.get('<pad>')
embedding_size = 128 # 128 for torch embeddings, 300 for pre-trained
hidden_size = 24
batch_size = 64
nb_epochs = 150
lr = 1e-4
max_norm = 5
folds = 10
criterion = nn.NLLLoss(weight=torch.Tensor([1.0, 2.2]).cuda())
# For testing phase
fold_scores = {}
test_set = ClaimsDataset(test_file)
dl_test = torch_data.DataLoader(test_set,
batch_size=batch_size,
shuffle=True)
mean = [] # holds the mean validation accuracy of every fold
print("\nTraining\n")
logger.info(utils.get_time())
for i in range(folds):
print("\nFold: {}\n".format(i))
losses = defaultdict(list)
train, val = utils.split_dataset(ds, i)
print("Train size: {} \t Validate size: {}".format(
len(train), len(val)))
dl_train = torch_data.DataLoader(train,
batch_size=batch_size,
shuffle=True)
dl_val = torch_data.DataLoader(val,
batch_size=batch_size,
shuffle=True)
model = RNN(vocab_size, embedding_size, hidden_size, pad_id, ds)
model = utils.cuda(model)
model.zero_grad()
# When using pre-trained embeddings, uncomment below otherwise, use the second statement
#parameters = list([parameter for parameter in model.parameters()
# if parameter.requires_grad])
parameters = list(model.parameters())
optim = torch.optim.Adam(parameters,
lr=lr,
weight_decay=35e-3,
amsgrad=True)
phases, loaders = ['train', 'val'], [dl_train, dl_val]
tr_acc, v_acc = [], []
for epoch in range(nb_epochs):
for p, loader in zip(phases, loaders):
if p == 'train':
model.train()
else:
model.eval()
ep_loss, out_list, label_list = [], [], []
for _, inputs in enumerate(loader):
optim.zero_grad()
claim, labels = inputs
labels = utils.variable(labels)
out = model(claim)
out_list.append(utils.normalize_out(out))
label_list.append(labels)
out = torch.log(out)
loss = criterion(out, labels)
if p == 'train':
loss.backward()
torch.nn.utils.clip_grad_norm_(parameters,
max_norm=max_norm)
optim.step()
ep_loss.append(loss.item())
losses[p].append(np.mean(ep_loss))
acc = utils.get_accuracy(label_list, out_list)
if p == 'train':
tr_acc.append(acc)
else:
v_acc.append(acc)
print(
"Epoch: {} \t Phase: {} \t Loss: {:.4f} \t Accuracy: {:.3f}"
.format(epoch, p, loss, acc))
utils.plot_loss(losses['train'], losses['val'], tr_acc, v_acc,
filename, i)
mean.append(np.mean(v_acc))
logger.info("\n Fold: " + str(i))
logger.info("Train file=> " + filename +
"\nParameters=> \nBatch size: " + str(batch_size) +
"\nHidden size: " + str(hidden_size) + "\nMax_norm: " +
str(max_norm) + "\nL2 Reg/weight decay: " +
str(optim.param_groups[0]['weight_decay']) +
"\nLoss function: " + str(criterion))
logger.info('Final train accuracy: ' + str(tr_acc[-1]))
logger.info('Final validation accuracy: ' + str(v_acc[-1]))
# Save model for current fold
torch.save(model.state_dict(), save_path + str(i))
test_f1, test_acc = [], []
for _, inp in enumerate(dl_test):
claim, label = inp
label = utils.variable(label)
model.eval()
out = model(claim)
y_pred = utils.normalize_out(out)
test_f1.append(utils.get_f1(label, y_pred))
test_acc.append(metrics.accuracy_score(label, y_pred))
t_f1, t_acc = np.mean(test_f1), np.mean(test_acc)
fold_scores[i] = dict([('F1', t_f1), ('Accuracy', t_acc)])
print("\tf1: {:.3f} \t accuracy: {:.3f}".format(t_f1, t_acc))
#logger.info('\nTest f1: '+str(t_f1)+'\nTest Accuracy: '+str(t_acc))
logger.info('Mean accuracy over 10 folds: \t' + str(np.mean(mean)))
logger.info(fold_scores)
def train(epochs=1, batchSize=2, lr=0.0001, device='cpu', accumulate=True, a_step=16, load_saved=False, file_path='./saved_best.pt', use_dtp=False, pretrained_model='./bert_pretrain_model/', tokenizer_model='bert-base-chinese', weighted_loss=False):
device = device
tokenizer = load_tokenizer(tokenizer_model)
my_net = torch.load(file_path) if load_saved else Net(load_pretrained_model(pretrained_model))
my_net.to(device, non_blocking=True)
label_dict = dict()
with open('./tianchi_datasets/label.json') as f:
for line in f:
label_dict = json.loads(line)
break
label_weights_dict = dict()
with open('./tianchi_datasets/label_weights.json') as f:
for line in f:
label_weights_dict = json.loads(line)
break
ocnli_train = dict()
with open('./tianchi_datasets/OCNLI/train.json') as f:
for line in f:
ocnli_train = json.loads(line)
break
ocnli_dev = dict()
with open('./tianchi_datasets/OCNLI/dev.json') as f:
for line in f:
ocnli_dev = json.loads(line)
break
ocemotion_train = dict()
with open('./tianchi_datasets/OCEMOTION/train.json') as f:
for line in f:
ocemotion_train = json.loads(line)
break
ocemotion_dev = dict()
with open('./tianchi_datasets/OCEMOTION/dev.json') as f:
for line in f:
ocemotion_dev = json.loads(line)
break
tnews_train = dict()
with open('./tianchi_datasets/TNEWS/train.json') as f:
for line in f:
tnews_train = json.loads(line)
break
tnews_dev = dict()
with open('./tianchi_datasets/TNEWS/dev.json') as f:
for line in f:
tnews_dev = json.loads(line)
break
train_data_generator = Data_generator(ocnli_train, ocemotion_train, tnews_train, label_dict, device, tokenizer)
dev_data_generator = Data_generator(ocnli_dev, ocemotion_dev, tnews_dev, label_dict, device, tokenizer)
tnews_weights = torch.tensor(label_weights_dict['TNEWS']).to(device, non_blocking=True)
ocnli_weights = torch.tensor(label_weights_dict['OCNLI']).to(device, non_blocking=True)
ocemotion_weights = torch.tensor(label_weights_dict['OCEMOTION']).to(device, non_blocking=True)
loss_object = Calculate_loss(label_dict, weighted=weighted_loss, tnews_weights=tnews_weights, ocnli_weights=ocnli_weights, ocemotion_weights=ocemotion_weights)
optimizer=torch.optim.Adam(my_net.parameters(), lr=lr)
best_dev_f1 = 0.0
best_epoch = -1
for epoch in range(epochs):
my_net.train()
train_loss = 0.0
train_total = 0
train_correct = 0
train_ocnli_correct = 0
train_ocemotion_correct = 0
train_tnews_correct = 0
train_ocnli_pred_list = []
train_ocnli_gold_list = []
train_ocemotion_pred_list = []
train_ocemotion_gold_list = []
train_tnews_pred_list = []
train_tnews_gold_list = []
cnt_train = 0
while True:
raw_data = train_data_generator.get_next_batch(batchSize)
if raw_data == None:
break
data = dict()
data['input_ids'] = raw_data['input_ids']
data['token_type_ids'] = raw_data['token_type_ids']
data['attention_mask'] = raw_data['attention_mask']
data['ocnli_ids'] = raw_data['ocnli_ids']
data['ocemotion_ids'] = raw_data['ocemotion_ids']
data['tnews_ids'] = raw_data['tnews_ids']
tnews_gold = raw_data['tnews_gold']
ocnli_gold = raw_data['ocnli_gold']
ocemotion_gold = raw_data['ocemotion_gold']
if not accumulate:
optimizer.zero_grad()
ocnli_pred, ocemotion_pred, tnews_pred = my_net(**data)
if use_dtp:
tnews_kpi = 0.1 if len(train_tnews_pred_list) == 0 else train_tnews_correct / len(train_tnews_pred_list)
ocnli_kpi = 0.1 if len(train_ocnli_pred_list) == 0 else train_ocnli_correct / len(train_ocnli_pred_list)
ocemotion_kpi = 0.1 if len(train_ocemotion_pred_list) == 0 else train_ocemotion_correct / len(train_ocemotion_pred_list)
current_loss = loss_object.compute_dtp(tnews_pred, ocnli_pred, ocemotion_pred, tnews_gold, ocnli_gold,
ocemotion_gold, tnews_kpi, ocnli_kpi, ocemotion_kpi)
else:
current_loss = loss_object.compute(tnews_pred, ocnli_pred, ocemotion_pred, tnews_gold, ocnli_gold, ocemotion_gold)
train_loss += current_loss.item()
current_loss.backward()
if accumulate and (cnt_train + 1) % a_step == 0:
optimizer.step()
optimizer.zero_grad()
if not accumulate:
optimizer.step()
if use_dtp:
good_tnews_nb, good_ocnli_nb, good_ocemotion_nb, total_tnews_nb, total_ocnli_nb, total_ocemotion_nb = loss_object.correct_cnt_each(tnews_pred, ocnli_pred, ocemotion_pred, tnews_gold, ocnli_gold, ocemotion_gold)
tmp_good = sum([good_tnews_nb, good_ocnli_nb, good_ocemotion_nb])
tmp_total = sum([total_tnews_nb, total_ocnli_nb, total_ocemotion_nb])
train_ocemotion_correct += good_ocemotion_nb
train_ocnli_correct += good_ocnli_nb
train_tnews_correct += good_tnews_nb
else:
tmp_good, tmp_total = loss_object.correct_cnt(tnews_pred, ocnli_pred, ocemotion_pred, tnews_gold, ocnli_gold, ocemotion_gold)
train_correct += tmp_good
train_total += tmp_total
p, g = loss_object.collect_pred_and_gold(ocnli_pred, ocnli_gold)
train_ocnli_pred_list += p
train_ocnli_gold_list += g
p, g = loss_object.collect_pred_and_gold(ocemotion_pred, ocemotion_gold)
train_ocemotion_pred_list += p
train_ocemotion_gold_list += g
p, g = loss_object.collect_pred_and_gold(tnews_pred, tnews_gold)
train_tnews_pred_list += p
train_tnews_gold_list += g
cnt_train += 1
#torch.cuda.empty_cache()
if (cnt_train + 1) % 1000 == 0:
print('[', cnt_train + 1, '- th batch : train acc is:', train_correct / train_total, '; train loss is:', train_loss / cnt_train, ']')
if accumulate:
optimizer.step()
optimizer.zero_grad()
train_ocnli_f1 = get_f1(train_ocnli_gold_list, train_ocnli_pred_list)
train_ocemotion_f1 = get_f1(train_ocemotion_gold_list, train_ocemotion_pred_list)
train_tnews_f1 = get_f1(train_tnews_gold_list, train_tnews_pred_list)
train_avg_f1 = (train_ocnli_f1 + train_ocemotion_f1 + train_tnews_f1) / 3
print(epoch, 'th epoch train average f1 is:', train_avg_f1)
print(epoch, 'th epoch train ocnli is below:')
print_result(train_ocnli_gold_list, train_ocnli_pred_list)
print(epoch, 'th epoch train ocemotion is below:')
print_result(train_ocemotion_gold_list, train_ocemotion_pred_list)
print(epoch, 'th epoch train tnews is below:')
print_result(train_tnews_gold_list, train_tnews_pred_list)
train_data_generator.reset()
my_net.eval()
dev_loss = 0.0
dev_total = 0
dev_correct = 0
dev_ocnli_correct = 0
dev_ocemotion_correct = 0
dev_tnews_correct = 0
dev_ocnli_pred_list = []
dev_ocnli_gold_list = []
dev_ocemotion_pred_list = []
dev_ocemotion_gold_list = []
dev_tnews_pred_list = []
dev_tnews_gold_list = []
cnt_dev = 0
with torch.no_grad():
while True:
raw_data = dev_data_generator.get_next_batch(batchSize)
if raw_data == None:
break
data = dict()
data['input_ids'] = raw_data['input_ids']
data['token_type_ids'] = raw_data['token_type_ids']
data['attention_mask'] = raw_data['attention_mask']
data['ocnli_ids'] = raw_data['ocnli_ids']
data['ocemotion_ids'] = raw_data['ocemotion_ids']
data['tnews_ids'] = raw_data['tnews_ids']
tnews_gold = raw_data['tnews_gold']
ocnli_gold = raw_data['ocnli_gold']
ocemotion_gold = raw_data['ocemotion_gold']
ocnli_pred, ocemotion_pred, tnews_pred = my_net(**data)
if use_dtp:
tnews_kpi = 0.1 if len(dev_tnews_pred_list) == 0 else dev_tnews_correct / len(
dev_tnews_pred_list)
ocnli_kpi = 0.1 if len(dev_ocnli_pred_list) == 0 else dev_ocnli_correct / len(
dev_ocnli_pred_list)
ocemotion_kpi = 0.1 if len(dev_ocemotion_pred_list) == 0 else dev_ocemotion_correct / len(
dev_ocemotion_pred_list)
current_loss = loss_object.compute_dtp(tnews_pred, ocnli_pred, ocemotion_pred, tnews_gold,
ocnli_gold,
ocemotion_gold, tnews_kpi, ocnli_kpi, ocemotion_kpi)
else:
current_loss = loss_object.compute(tnews_pred, ocnli_pred, ocemotion_pred, tnews_gold, ocnli_gold, ocemotion_gold)
dev_loss += current_loss.item()
if use_dtp:
good_tnews_nb, good_ocnli_nb, good_ocemotion_nb, total_tnews_nb, total_ocnli_nb, total_ocemotion_nb = loss_object.correct_cnt_each(
tnews_pred, ocnli_pred, ocemotion_pred, tnews_gold, ocnli_gold, ocemotion_gold)
tmp_good += sum([good_tnews_nb, good_ocnli_nb, good_ocemotion_nb])
tmp_total += sum([total_tnews_nb, total_ocnli_nb, total_ocemotion_nb])
dev_ocemotion_correct += good_ocemotion_nb
dev_ocnli_correct += good_ocnli_nb
dev_tnews_correct += good_tnews_nb
else:
tmp_good, tmp_total = loss_object.correct_cnt(tnews_pred, ocnli_pred, ocemotion_pred, tnews_gold, ocnli_gold, ocemotion_gold)
dev_correct += tmp_good
dev_total += tmp_total
p, g = loss_object.collect_pred_and_gold(ocnli_pred, ocnli_gold)
dev_ocnli_pred_list += p
dev_ocnli_gold_list += g
p, g = loss_object.collect_pred_and_gold(ocemotion_pred, ocemotion_gold)
dev_ocemotion_pred_list += p
dev_ocemotion_gold_list += g
p, g = loss_object.collect_pred_and_gold(tnews_pred, tnews_gold)
dev_tnews_pred_list += p
dev_tnews_gold_list += g
cnt_dev += 1
#torch.cuda.empty_cache()
#if (cnt_dev + 1) % 1000 == 0:
# print('[', cnt_dev + 1, '- th batch : dev acc is:', dev_correct / dev_total, '; dev loss is:', dev_loss / cnt_dev, ']')
dev_ocnli_f1 = get_f1(dev_ocnli_gold_list, dev_ocnli_pred_list)
dev_ocemotion_f1 = get_f1(dev_ocemotion_gold_list, dev_ocemotion_pred_list)
dev_tnews_f1 = get_f1(dev_tnews_gold_list, dev_tnews_pred_list)
dev_avg_f1 = (dev_ocnli_f1 + dev_ocemotion_f1 + dev_tnews_f1) / 3
print(epoch, 'th epoch dev average f1 is:', dev_avg_f1)
print(epoch, 'th epoch dev ocnli is below:')
print_result(dev_ocnli_gold_list, dev_ocnli_pred_list)
print(epoch, 'th epoch dev ocemotion is below:')
print_result(dev_ocemotion_gold_list, dev_ocemotion_pred_list)
print(epoch, 'th epoch dev tnews is below:')
print_result(dev_tnews_gold_list, dev_tnews_pred_list)
dev_data_generator.reset()
if dev_avg_f1 > best_dev_f1:
best_dev_f1 = dev_avg_f1
best_epoch = epoch
torch.save(my_net, file_path)
print('best epoch is:', best_epoch, '; with best f1 is:', best_dev_f1)
def test_single_embed(zs, rrefs, times, cutoff):
# Load train data into workers
ld_args = get_work_units(
len(rrefs),
times['te_start'],
times['te_end'],
times['delta'],
True
)
jobs = [
_remote_method_async(
DTGAE_Encoder.run_arbitrary_fn,
rrefs[i],
ld.load_edge_list_dist, W_THREADS*2, ld_args[i]['start'], ld_args[i]['end']
)
for i in range(len(rrefs))
]
probs = []
guesses = []
labels = []
for i in range(len(jobs)):
ei, y = jobs[i].wait()
# Decode the list of edges using just the last zs
# generated by the model
prob = _remote_method(
DTGAE_Encoder.decode,
rrefs[i],
ei, zs
)
guess = torch.zeros(prob.size())
guess[prob <= cutoff] = 1
probs.append(prob)
guesses.append(guess)
labels.append(y)
# Using the same var names as in the test() method
# so I can do a little ctrl+c, ctrl+v action here
scores = torch.cat(probs)
classified = torch.cat(guesses)
labels = torch.cat(labels)
anoms = scores[labels==1].sort()[0]
# Copied from test()
tpr = classified[labels==1].mean() * 100
fpr = classified[labels==0].mean() * 100
tp = classified[labels==1].sum()
fp = classified[labels==0].sum()
f1 = get_f1(classified, labels)
auc,ap = get_score(scores[labels==0], scores[labels==1])
print("Learned Cutoff %0.4f" % cutoff)
print("TPR: %0.2f, FPR: %0.2f" % (tpr, fpr))
print("TP: %d FP: %d" % (tp, fp))
print("F1: %0.8f" % f1)
print("AUC: %0.4f AP: %0.4f\n" % (auc,ap))
print("Top anom scored %0.04f" % anoms[0].item())
print("Lowest anom scored %0.4f" % anoms[-1].item())
print("Mean anomaly score: %0.4f" % anoms.mean().item())
return {
'TPR':tpr.item(),
'FPR':fpr.item(),
'TP':tp.item(),
'FP':fp.item(),
'F1':f1.item(),
'AUC':auc,
'AP': ap,
'FwdTime': 0
}
def test(model, h0, times, rrefs, manual=False):
# For whatever reason, it doesn't know what to do if you call
# the parent object's methods. Kind of defeats the purpose of
# using OOP at all IMO, but whatever
Encoder = StaticEncoder if isinstance(model, StaticRecurrent) \
else DynamicEncoder
# Load train data into workers
ld_args = get_work_units(
len(rrefs),
times['te_start'],
times['te_end'],
times['delta'],
True
)
print("Loading test data")
futs = [
_remote_method_async(
Encoder.load_new_data,
rrefs[i],
LOAD_FN,
ld_args[i]
) for i in range(len(rrefs))
]
# Wait until all workers have finished
[f.wait() for f in futs]
with torch.no_grad():
model.eval()
s = time.time()
zs = model.forward(TData.TEST, h0=h0, no_grad=True)
ctime = time.time()-s
# Scores all edges and matches them with name/timestamp
print("Scoring")
scores, labels = model.score_all(zs)
if manual:
labels = [
_remote_method_async(
Encoder.get_repr,
rrefs[i],
scores[i],
delta=times['delta']
)
for i in range(len(rrefs))
]
labels = sum([l.wait() for l in labels], [])
labels.sort(key=lambda x : x[0])
with open(SCORE_FILE, 'w+') as f:
cutoff_hit = False
for l in labels:
f.write(str(l[0].item()))
f.write('\t')
f.write(l[1])
f.write('\n')
if l[0] >= model.cutoff and not cutoff_hit:
f.write('-'*100 + '\n')
cutoff_hit = True
return {}
scores = torch.cat(sum(scores, []), dim=0)
labels = torch.cat(sum(labels, []), dim=0)
anoms = scores[labels==1].sort()[0]
# Classify using cutoff from earlier
classified = torch.zeros(labels.size())
classified[scores <= model.cutoff] = 1
#default = torch.zeros(labels.size())
#default[scores <= 0.5] = 1
tpr = classified[labels==1].mean() * 100
fpr = classified[labels==0].mean() * 100
tp = classified[labels==1].sum()
fp = classified[labels==0].sum()
f1 = get_f1(classified, labels)
auc,ap = get_score(scores[labels==0], scores[labels==1])
print("Learned Cutoff %0.4f" % model.cutoff)
print("TPR: %0.2f, FPR: %0.2f" % (tpr, fpr))
print("TP: %d FP: %d" % (tp, fp))
print("F1: %0.8f" % f1)
print("AUC: %0.4f AP: %0.4f\n" % (auc,ap))
print("Top anom scored %0.04f" % anoms[0].item())
print("Lowest anom scored %0.4f" % anoms[-1].item())
print("Mean anomaly score: %0.4f" % anoms.mean().item())
return {
'TPR':tpr.item(),
'FPR':fpr.item(),
'TP':tp.item(),
'FP':fp.item(),
'F1':f1.item(),
'AUC':auc,
'AP': ap,
'FwdTime':ctime
}
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
required=True,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument("--embeddings_file",
default=None,
type=str,
required=True)
parser.add_argument("--output_dir", default=None, type=str, required=True)
parser.add_argument("--train_language",
default=None,
type=str,
required=True)
parser.add_argument("--train_steps", default=-1, type=int, required=True)
parser.add_argument("--eval_steps", default=-1, type=int, required=True)
parser.add_argument(
"--load_word2vec",
action='store_true',
help=
'if true, load word2vec file for the first time; if false, load generated word-vector csv file'
)
parser.add_argument("--generate_word2vec_csv",
action='store_true',
help='if true, generate word2vec csv file')
## normal parameters
parser.add_argument("--embedding_size", default=300, type=int)
parser.add_argument("--query_maxlen", default=30, type=int)
parser.add_argument("--hidden_size", default=300, type=int)
parser.add_argument("--learning_rate",
default=5e-4,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_classes", default=2, type=int)
parser.add_argument("--dropout", default=0.2, type=float)
parser.add_argument("--do_test",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_eval_train",
action='store_true',
help="Whether to run eval on the train set.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--per_gpu_eval_batch_size", default=10, type=int)
parser.add_argument("--per_gpu_train_batch_size", default=10, type=int)
parser.add_argument("--seed", default=1, type=int)
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--gradient_accumulation_steps", default=1, type=int)
args = parser.parse_args()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
args.n_gpu = torch.cuda.device_count()
# device = torch.device("cpu")
args.device = device
# Set seed
set_seed(args)
logger.info("Training/evaluation parameters %s", args)
args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
# Training
if args.do_train:
# build model
logger.info("*** building model ***")
embeddings = load_embeddings(args)
model = ESIM(args.hidden_size,
embeddings=embeddings,
dropout=args.dropout,
num_classes=args.num_classes,
device=args.device)
model.to(args.device)
if args.n_gpu > 1:
model = torch.nn.DataParallel(model)
args.train_batch_size = args.per_gpu_train_batch_size * max(
1, args.n_gpu)
logger.info("*** Loading training data ***")
train_data = ATEC_Dataset(os.path.join(args.data_dir, 'train.csv'),
os.path.join(args.data_dir, 'vocab.csv'),
args.query_maxlen)
train_loader = DataLoader(train_data,
shuffle=True,
batch_size=args.train_batch_size)
logger.info("*** Loading validation data ***")
dev_data = ATEC_Dataset(os.path.join(args.data_dir, 'dev.csv'),
os.path.join(args.data_dir, 'vocab.csv'),
args.query_maxlen)
dev_loader = DataLoader(dev_data,
shuffle=False,
batch_size=args.eval_batch_size)
num_train_optimization_steps = args.train_steps
# 过滤出需要梯度更新的参数
parameters = filter(lambda p: p.requires_grad, model.parameters())
# optimizer = optim.Adadelta(parameters, params["LEARNING_RATE"])
optimizer = torch.optim.Adam(parameters, lr=args.learning_rate)
# optimizer = torch.optim.Adam(model.parameters(), lr=lr)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode="max",
factor=0.85,
patience=0)
criterion = nn.CrossEntropyLoss()
global_step = 0
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_data))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Gradient Accumulation steps = %d",
args.gradient_accumulation_steps)
logger.info(" Num steps = %d", num_train_optimization_steps)
best_acc = 0
model.train()
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
bar = tqdm(range(num_train_optimization_steps),
total=num_train_optimization_steps)
train_loader = cycle(train_loader)
output_dir = args.output_dir + "eval_results_{}_{}_{}_{}_{}_{}".format(
'ESIM', str(args.query_maxlen), str(args.learning_rate),
str(args.train_batch_size), str(args.train_language),
str(args.train_steps))
try:
os.makedirs(output_dir)
except:
pass
output_eval_file = os.path.join(output_dir, 'eval_result.txt')
with open(output_eval_file, "w") as writer:
writer.write('*' * 80 + '\n')
for step in bar:
batch = next(train_loader)
batch = tuple(t.to(device) for t in batch)
q1, q1_lens, q2, q2_lens, labels = batch
# 正常训练
optimizer.zero_grad()
logits, probs = model(q1, q1_lens, q2, q2_lens)
loss = criterion(logits, labels)
if args.n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
tr_loss += loss.item()
train_loss = round(
tr_loss * args.gradient_accumulation_steps / (nb_tr_steps + 1),
4)
bar.set_description("loss {}".format(train_loss))
nb_tr_examples += q1.size(0)
nb_tr_steps += 1
loss.backward()
# 对抗训练
# fgm.attack() # 在embedding上添加对抗扰动
# loss_adv = model(input_ids=input_ids, token_type_ids=segment_ids, attention_mask=input_mask, labels=label_ids)
# if args.n_gpu > 1:
# loss_adv = loss_adv.mean() # mean() to average on multi-gpu.
# if args.gradient_accumulation_steps > 1:
# loss_adv = loss_adv / args.gradient_accumulation_steps
# loss_adv.backward() # 反向传播，并在正常的grad基础上，累加对抗训练的梯度
# fgm.restore() # 恢复embedding参数
if (nb_tr_steps + 1) % args.gradient_accumulation_steps == 0:
# scheduler.step()
optimizer.step()
global_step += 1
if (step + 1) % (args.eval_steps *
args.gradient_accumulation_steps) == 0:
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
logger.info("***** Report result *****")
logger.info(" %s = %s", 'global_step', str(global_step))
logger.info(" %s = %s", 'train loss', str(train_loss))
if args.do_eval and (step + 1) % (
args.eval_steps * args.gradient_accumulation_steps) == 0:
if args.do_eval_train:
file_list = ['train.csv', 'dev.csv']
else:
file_list = ['dev.csv']
for file in file_list:
inference_labels = []
gold_labels = []
inference_logits = []
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(dev_data))
logger.info(" Batch size = %d", args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for q1, q1_lens, q2, q2_lens, labels in tqdm(dev_loader):
with torch.no_grad():
logits, probs = model(q1, q1_lens, q2, q2_lens)
probs = probs.detach().cpu().numpy()
# print(logits.shape, probs.shape)
# label_ids = labels.to('cpu').numpy()
inference_labels.append(np.argmax(probs, 1))
gold_labels.append(labels)
# eval_loss += tmp_eval_loss.mean().item()
nb_eval_examples += logits.size(0)
nb_eval_steps += 1
gold_labels = np.concatenate(gold_labels, 0)
inference_labels = np.concatenate(inference_labels, 0)
model.train()
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = get_f1(inference_labels, gold_labels)
result = {
# 'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'train_loss': train_loss
}
if 'dev' in file:
with open(output_eval_file, "a") as writer:
writer.write(file + '\n')
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" %
(key, str(result[key])))
writer.write('*' * 80)
writer.write('\n')
if eval_accuracy > best_acc and 'dev' in file:
print("=" * 80)
print("Best ACC", eval_accuracy)
print("Saving Model......")
best_acc = eval_accuracy
# Save a trained model
model_to_save = model.module if hasattr(
model,
'module') else model # Only save the model it-self
output_model_file = os.path.join(
output_dir, "pytorch_model.bin")
torch.save(model_to_save.state_dict(),
output_model_file)
print("=" * 80)
else:
print("=" * 80)
with open(output_eval_file, "a") as writer:
writer.write('bert_acc: %f' % best_acc)
if args.do_test:
if args.do_train == False:
output_dir = args.output_dir
# build model
logger.info("*** building model ***")
embeddings = load_embeddings(args)
model = ESIM(args.hidden_size,
embeddings=embeddings,
dropout=args.dropout,
num_classes=args.num_classes,
device=args.device)
model.load_state_dict(
torch.load(os.path.join(output_dir, 'pytorch_model.bin')))
model.to(args.device)
if args.n_gpu > 1:
model = torch.nn.DataParallel(model)
inference_labels = []
gold_labels = []
logger.info("*** Loading testing data ***")
dev_data = ATEC_Dataset(os.path.join(args.data_dir, 'test.csv'),
os.path.join(args.data_dir, 'vocab.csv'),
args.query_maxlen)
dev_loader = DataLoader(dev_data,
shuffle=False,
batch_size=args.eval_batch_size)
logger.info(" *** Run Prediction ***")
logger.info(" Num examples = %d", len(dev_data))
logger.info(" Batch size = %d", args.eval_batch_size)
model.eval()
for q1, q1_lens, q2, q2_lens, labels in tqdm(dev_loader):
with torch.no_grad():
logits, probs = model(q1, q1_lens, q2, q2_lens)
probs = probs.detach().cpu().numpy()
inference_labels.append(np.argmax(probs, 1))
gold_labels.append(labels)
gold_labels = np.concatenate(gold_labels, 0)
logits = np.concatenate(inference_labels, 0)
test_f1 = get_f1(logits, gold_labels)
logger.info('predict f1:{}'.format(str(test_f1)))
def forward(self,
images,
captions,
lengths,
img_lengths,
img_txts,
img_spans,
txt_spans,
labels,
ids=None,
epoch=None,
*args):
self.niter += 1
self.logger.update('Eit', self.niter)
self.logger.update('lr', self.optimizer.param_groups[0]['lr'])
img_lengths = torch.tensor(img_lengths).long() if isinstance(
img_lengths, list) else img_lengths
lengths = torch.tensor(lengths).long() if isinstance(lengths,
list) else lengths
if torch.cuda.is_available():
images = images.cuda()
captions = captions.cuda()
lengths = lengths.cuda()
img_lengths = img_lengths.cuda()
bsize = captions.size(0)
img_emb, nll_img, kl_img, span_margs_img, argmax_spans_img, trees_img, lprobs_img = self.forward_img_parser(
images, img_lengths)
ll_loss_img = nll_img.sum()
kl_loss_img = kl_img.sum()
txt_emb, nll_txt, kl_txt, span_margs_txt, argmax_spans_txt, trees_txt, lprobs_txt = self.forward_txt_parser(
captions, lengths)
ll_loss_txt = nll_txt.sum()
kl_loss_txt = kl_txt.sum()
contrastive_loss = self.forward_loss(img_emb, txt_emb, img_lengths,
lengths, argmax_spans_img,
argmax_spans_txt, span_margs_img,
span_margs_txt)
mt_loss = contrastive_loss.sum()
loss_img = self.vse_lm_alpha * (ll_loss_img + kl_loss_img) / bsize
loss_txt = self.vse_lm_alpha * (ll_loss_txt + kl_loss_txt) / bsize
loss_mt = self.vse_mt_alpha * mt_loss / bsize
loss = loss_img + loss_txt + loss_mt
self.optimizer.zero_grad()
loss.backward()
if self.grad_clip > 0:
clip_grad_norm_(self.all_params, self.grad_clip)
self.optimizer.step()
self.logger.update('Loss_img', loss_img.item(), bsize)
self.logger.update('Loss_txt', loss_txt.item(), bsize)
self.logger.update('KL-Loss_img', kl_loss_img.item() / bsize, bsize)
self.logger.update('KL-Loss_txt', kl_loss_txt.item() / bsize, bsize)
self.logger.update('LL-Loss_img', ll_loss_img.item() / bsize, bsize)
self.logger.update('LL-Loss_txt', ll_loss_txt.item() / bsize, bsize)
self.n_word_img += (img_lengths + 1).sum().item()
self.n_word_txt += (lengths + 1).sum().item()
self.n_sent += bsize
for b in range(bsize):
max_img_len = img_lengths[b].item()
pred_img = [(a[0], a[1]) for a in argmax_spans_img[b]
if a[0] != a[1]]
pred_set_img = set(pred_img[:-1])
gold_img = [(img_spans[b][i][0].item(), img_spans[b][i][1].item())
for i in range(max_img_len - 1)]
gold_set_img = set(gold_img[:-1])
utils.update_stats(pred_set_img, [gold_set_img],
self.all_stats_img)
max_txt_len = lengths[b].item()
pred_txt = [(a[0], a[1]) for a in argmax_spans_txt[b]
if a[0] != a[1]]
pred_set_txt = set(pred_txt[:-1])
gold_txt = [(txt_spans[b][i][0].item(), txt_spans[b][i][1].item())
for i in range(max_txt_len - 1)]
gold_set_txt = set(gold_txt[:-1])
utils.update_stats(pred_set_txt, [gold_set_txt],
self.all_stats_txt)
# if self.niter % self.log_step == 0:
p_norm, g_norm = self.norms()
all_f1_img = utils.get_f1(self.all_stats_img)
all_f1_txt = utils.get_f1(self.all_stats_txt)
train_kl_img = self.logger.meters["KL-Loss_img"].sum
train_ll_img = self.logger.meters["LL-Loss_img"].sum
train_kl_txt = self.logger.meters["KL-Loss_txt"].sum
train_ll_txt = self.logger.meters["LL-Loss_txt"].sum
info = '|Pnorm|: {:.6f}, |Gnorm|: {:.2f}, ReconPPL-Img: {:.2f}, KL-Img: {:.2f}, ' + \
'PPLBound-Img: {:.2f}, CorpusF1-Img: {:.2f}, ' + \
'ReconPPL-Txt: {:.2f}, KL-Txt: {:.2f}, ' + \
'PPLBound-Txt: {:.2f}, CorpusF1-Txt: {:.2f}, ' + \
'Speed: {:.2f} sents/sec'
info = info.format(
p_norm, g_norm, np.exp(train_ll_img / self.n_word_img),
train_kl_img / self.n_sent,
np.exp((train_ll_img + train_kl_img) / self.n_word_img),
all_f1_img[0], np.exp(train_ll_txt / self.n_word_txt),
train_kl_txt / self.n_sent,
np.exp((train_ll_txt + train_kl_txt) / self.n_word_txt),
all_f1_txt[0], self.n_sent / (time.time() - self.s_time))
pred_action_img = utils.get_actions(trees_img[0])
sent_s_img = img_txts[0]
pred_t_img = utils.get_tree(pred_action_img, sent_s_img)
gold_t_img = utils.span_to_tree(img_spans[0].tolist(),
img_lengths[0].item())
gold_action_img = utils.get_actions(gold_t_img)
gold_t_img = utils.get_tree(gold_action_img, sent_s_img)
info += "\nPred T Image: {}\nGold T Image: {}".format(
pred_t_img, gold_t_img)
pred_action_txt = utils.get_actions(trees_txt[0])
sent_s_txt = [
self.vocab.idx2word[wid] for wid in captions[0].cpu().tolist()
]
pred_t_txt = utils.get_tree(pred_action_txt, sent_s_txt)
gold_t_txt = utils.span_to_tree(txt_spans[0].tolist(),
lengths[0].item())
gold_action_txt = utils.get_actions(gold_t_txt)
gold_t_txt = utils.get_tree(gold_action_txt, sent_s_txt)
info += "\nPred T Text: {}\nGold T Text: {}".format(
pred_t_txt, gold_t_txt)
return info
def eval_tgt_ood(src_encoder, tgt_encoder, src_classifier, tgt_classifier,
src_data_loader, tgt_data_loader, data_loader):
src_inv, src_mean, src_mahalanobis_mean, src_mahalanobis_std = \
get_distribution(src_encoder, tgt_encoder, src_data_loader)
tgt_inv, tgt_mean, tgt_mahalanobis_mean, tgt_mahalanobis_std = \
get_distribution(src_encoder, tgt_encoder, tgt_data_loader)
"""Evaluation for target encoder by source classifier on target dataset."""
# set eval state for Dropout and BN layers
src_encoder.eval()
tgt_encoder.eval()
src_classifier.eval()
tgt_classifier.eval()
# init loss and accuracy
loss = 0
acc = 0
f1 = 0
ys_pred = []
ys_true = []
# evaluate network
for (images, labels) in data_loader:
images = make_variable(images, volatile=True)
labels = make_variable(labels).squeeze_()
for image, label in zip(images, labels):
#image = image.detach().cpu().numpy()
label = label.detach().cpu().numpy()
if not is_in_distribution(image, tgt_inv, tgt_mean, tgt_mahalanobis_mean, tgt_mahalanobis_std) and \
not is_in_distribution(image, src_inv, src_mean, src_mahalanobis_mean, src_mahalanobis_std):
continue
elif is_in_distribution(image, tgt_inv, tgt_mean,
tgt_mahalanobis_mean, tgt_mahalanobis_std):
y_pred = np.argmax(
np.squeeze(
src_classifier(
tgt_encoder(torch.unsqueeze(
image, dim=0))).detach().cpu().numpy()))
else:
y_pred = np.argmax(
np.squeeze(
tgt_classifier(
src_encoder(torch.unsqueeze(
image, dim=0))).detach().cpu().numpy()))
ys_pred.append(y_pred)
ys_true.append(label)
loss /= len(data_loader)
acc /= len(data_loader.dataset)
#f1 /= len(data_loader.dataset)
f1 = get_f1(ys_pred, ys_true, 'macro')
f1_weighted = get_f1(ys_pred, ys_true, 'weighted')
print("Avg Loss = {}, F1 = {:2%}, Weighted F1 = {:2%}".format(
loss, f1, f1_weighted))