def evaluate_cnn(model, validation_generator, criterion, epoch, print_every=25):
model.eval()
losses = AverageMeter()
accuracies = AverageMeter()
aucs = AverageMeter()
num_iter_per_epoch = len(validation_generator)
y_true, y_pred, pos_scores = [], [], []
progress_bar = tqdm(enumerate(validation_generator),
total=num_iter_per_epoch)
for iter, batch in progress_bar:
samples, labels = batch
labels = labels.squeeze(1)
if use_cuda:
samples = samples.cuda()
labels = labels.cuda()
with torch.no_grad():
logits = model(samples)
loss = criterion(logits, labels)
## validation metrics
validation_metrics = get_evaluation(labels.cpu().numpy(),
logits.cpu().detach().numpy(), metrics_list=['accuracy'])
acc = validation_metrics['accuracy']
losses.update(loss.data, samples.size(0))
accuracies.update(acc, samples.size(0))
y_true += labels.cpu().numpy().tolist()
y_pred += torch.max(logits, 1)[1].cpu().numpy().tolist()
pos_scores += logits.cpu().detach().numpy()[:,1].tolist()
if (iter % print_every == 0) and (iter > 0):
print_str = "[Validation - Epoch: {}], Iteration: {}/{}, Loss: {}, Accuracy: {}"
print(print_str.format(
epoch + 1,
iter,
num_iter_per_epoch,
losses.avg,
accuracies.avg,
))
try:
vali_auc = metrics.roc_auc_score(y_true, pos_scores)
except ValueError as e:
vali_auc = 0
exit()
print("Avg loss: {}, Acc: {}, AUC: {}".format(losses.avg.item(), accuracies.avg.item(), vali_auc))
return losses.avg.item(), accuracies.avg.item(), vali_auc
for n_iter, batch in tqdm(enumerate(test_generator), total=num_iter_per_epoch):
features, labels = batch
labels.sub_(1)
if torch.cuda.is_available():
features = features.cuda()
labels = labels.cuda()
with torch.no_grad():
predictions = model(features)
loss = criterion(predictions, labels)
y_true += labels.cpu().numpy().tolist()
y_pred += torch.max(predictions, 1)[1].cpu().numpy().tolist()
validation_metrics = utils.get_evaluation(labels.cpu().numpy(),
predictions.cpu().detach().numpy(),
list_metrics=["accuracy", "f1_weighted", "f1_micro", "f1_macro"])
accuracy = validation_metrics['accuracy']
f1_weighted = validation_metrics['f1_weighted']
f1_micro = validation_metrics['f1_micro']
f1_macro = validation_metrics["f1_macro"]
losses.update(loss.data, features.size(0))
accuracies.update(validation_metrics["accuracy"], features.size(0))
writer.add_scalar('Test/Loss',
loss.item(),
n_iter)
writer.add_scalar('Test/Accuracy',
accuracy,
def train_cnn(model,
training_generator,
optimizer,
criterion,
epoch,
print_every=25):
model.train()
losses = AverageMeter()
accuracies = AverageMeter()
num_iter_per_epoch = len(training_generator)
y_true, y_pred, pos_scores = [], [], []
progress_bar = tqdm(enumerate(training_generator),
total=num_iter_per_epoch)
for iter, batch in progress_bar:
samples, labels = batch
labels = labels.squeeze(1)
if use_cuda:
samples = samples.cuda()
labels = labels.cuda()
optimizer.zero_grad()
logits = model(samples)
loss = criterion(logits, labels)
loss.backward()
optimizer.step()
## training metrics
training_metrics = get_evaluation(
labels.cpu().numpy(),
logits.cpu().detach().numpy(),
metrics_list=["accuracy"],
)
acc = training_metrics["accuracy"]
losses.update(loss.data, samples.size(0))
accuracies.update(acc, samples.size(0))
y_true += labels.cpu().numpy().tolist()
y_pred += torch.max(logits, 1)[1].cpu().numpy().tolist()
pos_scores += logits.cpu().detach().numpy()[:, 1].tolist()
lr = optimizer.state_dict()["param_groups"][0]["lr"]
if (iter % print_every == 0) and (iter > 0):
print_str = "[Training - Epoch: {}], LR: {}, Iteration: {}/{}, Loss: {}, Accuracy: {}"
print(
print_str.format(
epoch + 1,
lr,
iter,
num_iter_per_epoch,
losses.avg,
accuracies.avg,
))
try:
train_auc = metrics.roc_auc_score(y_true, pos_scores)
except ValueError as e:
print("y_prob = ", y_prob)
print("y_true = ", y_true)
train_auc = 0
exit()
print("Avg loss: {}, Acc: {}, AUC: {}".format(losses.avg.item(),
accuracies.avg.item(),
train_auc))
return losses.avg.item(), accuracies.avg.item(), train_auc
def train(model,
training_generator,
optimizer,
criterion,
epoch,
writer,
log_file,
scheduler,
class_names,
args,
print_every=25):
model.train()
losses = utils.AverageMeter()
accuracies = utils.AverageMeter()
num_iter_per_epoch = len(training_generator)
progress_bar = tqdm(enumerate(training_generator),
total=num_iter_per_epoch)
y_true = []
y_pred = []
for iter, batch in progress_bar:
features, labels = batch
if torch.cuda.is_available():
features = features.cuda()
labels = labels.cuda()
optimizer.zero_grad()
predictions = model(features)
y_true += labels.cpu().numpy().tolist()
y_pred += torch.max(predictions, 1)[1].cpu().numpy().tolist()
loss = criterion(predictions, labels)
loss.backward()
if args.scheduler == 'clr':
scheduler.step()
optimizer.step()
training_metrics = utils.get_evaluation(
labels.cpu().numpy(),
predictions.cpu().detach().numpy(),
list_metrics=["accuracy", "f1"])
losses.update(loss.data, features.size(0))
accuracies.update(training_metrics["accuracy"], features.size(0))
f1 = training_metrics['f1']
writer.add_scalar('Train/Loss', loss.item(),
epoch * num_iter_per_epoch + iter)
writer.add_scalar('Train/Accuracy', training_metrics['accuracy'],
epoch * num_iter_per_epoch + iter)
writer.add_scalar('Train/f1', f1, epoch * num_iter_per_epoch + iter)
lr = optimizer.state_dict()["param_groups"][0]["lr"]
if (iter % print_every == 0) and (iter > 0):
print(
"[Training - Epoch: {}], LR: {} , Iteration: {}/{} , Loss: {}, Accuracy: {}"
.format(epoch + 1, lr, iter, num_iter_per_epoch, losses.avg,
accuracies.avg))
if bool(args.log_f1):
intermediate_report = classification_report(y_true,
y_pred,
output_dict=True)
f1_by_class = 'F1 Scores by class: '
for class_name in class_names:
f1_by_class += f"{class_name} : {np.round(intermediate_report[class_name]['f1-score'], 4)} |"
print(f1_by_class)
f1_train = f1_score(y_true, y_pred, average='weighted')
writer.add_scalar('Train/loss/epoch', losses.avg, epoch + iter)
writer.add_scalar('Train/acc/epoch', accuracies.avg, epoch + iter)
writer.add_scalar('Train/f1/epoch', f1_train, epoch + iter)
report = classification_report(y_true, y_pred)
print(report)
with open(log_file, 'a') as f:
f.write(f'Training on Epoch {epoch} \n')
f.write(f'Average loss: {losses.avg.item()} \n')
f.write(f'Average accuracy: {accuracies.avg.item()} \n')
f.write(f'F1 score: {f1_train} \n\n')
f.write(report)
f.write('*' * 25)
f.write('\n')
return losses.avg.item(), accuracies.avg.item(), f1_train
def evaluate(model,
validation_generator,
criterion,
epoch,
writer,
log_file,
print_every=25):
model.eval()
losses = utils.AverageMeter()
accuracies = utils.AverageMeter()
num_iter_per_epoch = len(validation_generator)
y_true = []
y_pred = []
for iter, batch in tqdm(enumerate(validation_generator),
total=num_iter_per_epoch):
features, labels = batch
if torch.cuda.is_available():
features = features.cuda()
labels = labels.cuda()
with torch.no_grad():
predictions = model(features)
loss = criterion(predictions, labels)
y_true += labels.cpu().numpy().tolist()
y_pred += torch.max(predictions, 1)[1].cpu().numpy().tolist()
validation_metrics = utils.get_evaluation(
labels.cpu().numpy(),
predictions.cpu().detach().numpy(),
list_metrics=["accuracy", "f1"])
accuracy = validation_metrics['accuracy']
f1 = validation_metrics['f1']
losses.update(loss.data, features.size(0))
accuracies.update(validation_metrics["accuracy"], features.size(0))
writer.add_scalar('Test/Loss', loss.item(),
epoch * num_iter_per_epoch + iter)
writer.add_scalar('Test/Accuracy', accuracy,
epoch * num_iter_per_epoch + iter)
writer.add_scalar('Test/f1', f1, epoch * num_iter_per_epoch + iter)
if (iter % print_every == 0) and (iter > 0):
print(
"[Validation - Epoch: {}] , Iteration: {}/{} , Loss: {}, Accuracy: {}"
.format(epoch + 1, iter, num_iter_per_epoch, losses.avg,
accuracies.avg))
f1_test = f1_score(y_true, y_pred, average='weighted')
writer.add_scalar('Test/loss/epoch', losses.avg, epoch + iter)
writer.add_scalar('Test/acc/epoch', accuracies.avg, epoch + iter)
writer.add_scalar('Test/f1/epoch', f1_test, epoch + iter)
report = classification_report(y_true, y_pred)
print(report)
with open(log_file, 'a') as f:
f.write(f'Validation on Epoch {epoch} \n')
f.write(f'Average loss: {losses.avg.item()} \n')
f.write(f'Average accuracy: {accuracies.avg.item()} \n')
f.write(f'F1 score {f1_test} \n\n')
f.write(report)
f.write('=' * 50)
f.write('\n')
return losses.avg.item(), accuracies.avg.item(), f1_test
def train(opt):
dataset_train, dataset_test = fetch_dataset(opt.dataset)
dataset_train = CharacterDataset(dataset_train, opt.max_length)
dataset_test = CharacterDataset(dataset_test, opt.max_length)
train_generator = DataLoader(dataset_train,
batch_size=opt.batch_size,
shuffle=True)
test_generator = DataLoader(dataset_test,
batch_size=opt.batch_size,
shuffle=False)
if opt.feature == "small":
model = CharacterLevelCNN(input_length=opt.max_length,
n_classes=dataset_train.num_classes,
input_dim=len(opt.alphabet),
n_conv_filters=256,
n_fc_neurons=1024,
init_std=0.05)
elif opt.feature == "large":
model = CharacterLevelCNN(input_length=opt.max_length,
n_classes=dataset_test.num_classes,
input_dim=len(opt.alphabet),
n_conv_filters=1024,
n_fc_neurons=2048,
init_std=0.02)
else:
sys.exit("Invalid feature mode!")
log_path = "{}_{}_{}".format(opt.log_path, opt.feature, opt.dataset)
if os.path.isdir(log_path):
shutil.rmtree(log_path)
os.makedirs(log_path)
writer = SummaryWriter(log_path)
if torch.cuda.is_available():
model.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
best_loss = 1e5
best_epoch = 0
model.train()
num_iter_per_epoch = len(train_generator)
for epoch in range(opt.num_epochs):
for iter, batch in enumerate(train_generator):
feature, label = batch
label = label.long()
if torch.cuda.is_available():
feature = feature.cuda()
label = label.cuda()
optimizer.zero_grad()
pred = model(feature)
loss = criterion(pred, label)
loss.backward()
optimizer.step()
training_metrics = get_evaluation(label.cpu().numpy(),
pred.cpu().detach().numpy(),
list_metrics=["accuracy"])
print(
"Epoch: %d/%d, Iteration: %d/%d, lr: %.6f, loss: %.6f, acc: %.6f"
% (epoch + 1, opt.num_epochs, iter + 1,
num_iter_per_epoch, optimizer.param_groups[0]['lr'],
loss.item(), training_metrics["accuracy"]))
writer.add_scalar('Train/Loss', loss,
epoch * num_iter_per_epoch + iter)
writer.add_scalar('Train/Accuracy', training_metrics["accuracy"],
epoch * num_iter_per_epoch + iter)
model.eval()
loss_ls = []
te_label_ls = []
te_pred_ls = []
for batch in test_generator:
with torch.no_grad():
te_feature, te_label = batch
te_label = te_label.long()
num_sample = len(te_label)
if torch.cuda.is_available():
te_feature = te_feature.cuda()
te_label = te_label.cuda()
te_predictions = model(te_feature)
te_loss = criterion(te_predictions, te_label)
loss_ls.append(te_loss.item() * num_sample)
te_label_ls.extend(te_label.clone().cpu())
te_pred_ls.append(te_predictions.clone().cpu())
te_loss = sum(loss_ls) / len(dataset_test)
te_pred = torch.cat(te_pred_ls, 0)
te_label = np.array(te_label_ls)
test_metrics = get_evaluation(
te_label,
te_pred.numpy(),
list_metrics=["accuracy", "confusion_matrix"])
print("Epoch: %d/%d, loss: %.6f, acc: %.6f,confusion matrix: \n%s" %
(epoch + 1, opt.num_epochs, te_loss, test_metrics["accuracy"],
test_metrics["confusion_matrix"]))
writer.add_scalar('Test/Loss', te_loss, epoch)
writer.add_scalar('Test/Accuracy', test_metrics["accuracy"], epoch)
model.train()
if te_loss + opt.es_min_delta < best_loss:
best_loss = te_loss
best_epoch = epoch
torch.save(
model.state_dict(), "./weights/char-cnn_%s_%s_%d.pth" %
(opt.dataset, opt.feature, epoch))
# Early stopping
if epoch - best_epoch > opt.es_patience > 0:
print(
"Stop training at epoch %d. The lowest loss achieved is %.6f at epoch %d"
% (epoch, best_loss, best_epoch))
break
def train(model, training_generator, optimizer, criterion, epoch, start_iter, writer, log_file, scheduler, class_names,
args, print_every=10):
model.train()
losses = utils.AverageMeter()
accuracies = utils.AverageMeter()
num_iter_per_epoch = len(training_generator)
progress_bar = tqdm(enumerate(training_generator, start=start_iter),
total=num_iter_per_epoch)
y_true = []
y_pred = []
for iter, batch in progress_bar:
features, labels = batch
labels.sub_(1)
if torch.cuda.is_available():
features = features.cuda()
labels = labels.cuda()
optimizer.zero_grad()
predictions = model(features)
y_true += labels.cpu().numpy().tolist()
y_pred += torch.max(predictions, 1)[1].cpu().numpy().tolist()
loss = criterion(predictions, labels)
loss.backward()
if args.get('Train', 'scheduler') == 'clr':
scheduler.step()
optimizer.step()
training_metrics = utils.get_evaluation(labels.cpu().numpy(),
predictions.cpu().detach().numpy(),
list_metrics=["accuracy", "f1_weighted", "f1_micro", "f1_macro"])
losses.update(loss.data, features.size(0))
accuracies.update(training_metrics["accuracy"], features.size(0))
f1_weighted = training_metrics['f1_weighted']
f1_micro = training_metrics['f1_micro']
f1_macro = training_metrics["f1_macro"]
writer.add_scalar('Train/Loss',
loss.item(),
epoch * num_iter_per_epoch + iter)
writer.add_scalar('Train/Accuracy',
training_metrics['accuracy'],
epoch * num_iter_per_epoch + iter)
writer.add_scalar('Train/f1-weighted',
f1_weighted,
epoch * num_iter_per_epoch + iter)
writer.add_scalar('Train/f1-micro',
f1_micro,
epoch * num_iter_per_epoch + iter)
writer.add_scalar('Train/f1-macro',
f1_macro,
epoch * num_iter_per_epoch + iter)
#f1_scalars = {'f1_weighted': f1_weighted, 'f1_micro': f1_micro, 'f1_macro': f1_macro}
#writer.add_scalars('Train/all-f1-scores',
# f1_scalars,
# epoch * num_iter_per_epoch + iter)
for name, param in model.named_parameters():
writer.add_histogram(name, param, iter)
lr = optimizer.state_dict()["param_groups"][0]["lr"]
if (iter % print_every == 0) and (iter > 0):
print("\n[Training - Epoch: {}], LR: {} , Iteration: {}/{} , Loss: {}, Accuracy: {}".format(
epoch + 1,
lr,
iter,
num_iter_per_epoch,
losses.avg,
accuracies.avg
))
if bool(args.getboolean('Log', 'log_f1')):
intermediate_report = classification_report(
y_true, y_pred, output_dict=True)
print(intermediate_report)
# f1_by_class = 'F1 Scores by class: '
# for class_name in class_names:
# f1_by_class += f"{class_name} : {np.round(intermediate_report[class_name]['f1-score'], 4)} |"
# print(f1_by_class)
f1_train_weighted = f1_score(y_true, y_pred, average='weighted')
f1_train_micro = f1_score(y_true, y_pred, average='micro')
f1_train_macro = f1_score(y_true, y_pred, average='macro')
writer.add_scalar('Train/loss/epoch', losses.avg, epoch + iter)
writer.add_scalar('Train/acc/epoch', accuracies.avg, epoch + iter)
writer.add_scalar('Train/f1-weighted/epoch', f1_train_weighted, epoch + iter)
writer.add_scalar('Train/f1-micro/epoch', f1_train_micro, epoch + iter)
writer.add_scalar('Train/f1-macro/epoch', f1_train_macro, epoch + iter)
#f1_scalars = {'f1_weighted': f1_train_weighted, 'f1_micro': f1_train_micro, 'f1_macro': f1_train_macro}
#writer.add_scalars('Train/all-f1-scores/epoch',
# f1_scalars,
# epoch * num_iter_per_epoch + iter)
report = classification_report(y_true, y_pred)
print(report)
with open(log_file, 'a') as f:
f.write(f'Training on Epoch {epoch} \n')
f.write(f'Average loss: {losses.avg.item()} \n')
f.write(f'Average accuracy: {accuracies.avg.item()} \n')
f.write(f'F1 Weighted score: {f1_train_weighted} \n\n')
f.write(f'F1 Micro score: {f1_train_micro} \n\n')
f.write(f'F1 Macro score: {f1_train_macro} \n\n')
f.write(report)
f.write('*' * 25)
f.write('\n')
return losses.avg.item(), accuracies.avg.item(), f1_train_weighted
def train(
opt,
task,
data_path=None,
fed_data=None,
k_fold=0,
cols_to_train=['tweets', 'judgments'],
subsample=1,
overwrite=True,
config=''):
"""
Runs the HAN with the given parameters
Args:
opt (dict): Configuration for the model
task (str): Name of the task
data_path (str, optional): Path to the data. Defaults to None.
fed_data (pd.DataFrame, optional): [description]. Defaults to None.
k_fold (int, optional): [description]. Defaults to 0.
cols_to_train (list, optional): [description]. Defaults to ['tweets', 'judgments'].
subsample (int, optional): [description]. Defaults to 1.
overwrite (bool, optional): [description]. Defaults to True.
config (str, optional): [description]. Defaults to ''.
Returns:
[type]: [description]
"""
if torch.cuda.is_available():
torch.cuda.manual_seed(2019)
else:
torch.manual_seed(2019)
output_file = open(opt['saved_path'] + "logs.txt", "w")
output_file.write("Model's parameters: {}".format(opt.items()))
training_params = {"batch_size": opt['batch_size'],
"shuffle": False,
"drop_last": True}
test_params = {"batch_size": opt['batch_size'],
"shuffle": False,
"drop_last": False}
max_word_length, max_sent_length = get_max_lengths(
opt['train_set'], fed_data=fed_data)
if fed_data is None:
df_data = pd.read_csv(data_path, encoding='utf8', sep=',')
else:
df_train = pd.DataFrame(fed_data[0], columns=cols_to_train),
df_test = pd.DataFrame(fed_data[1], columns=cols_to_train)
kf = model_selection.KFold(n_splits=5)
predicted_all_folds = []
true_all_folds = []
counter = 0
accuracies_all_folds = []
precision_all_folds = []
recall_all_folds = []
f1_all_folds = []
f1_macro_all_folds = []
f1_micro_all_folds = []
if fed_data is None:
kf = model_selection.KFold(n_splits=5)
iterator = kf.split(df_data)
else:
iterator = [(0, 0)]
for train_index, test_index in iterator:
counter += 1
print()
print("CV fold: ", counter)
print()
if os.path.exists(opt['vocab_path']):
os.remove(opt['vocab_path'])
if fed_data is None:
df_train, df_test = df_data.iloc[train_index], df_data.iloc[test_index]
sep_idx = int(df_test.shape[0] / 2)
df_valid = df_test[:sep_idx]
df_test = df_test[sep_idx:]
print(
"Train size: ",
df_train.shape,
"Valid size: ",
df_valid.shape,
"Test size: ",
df_test.shape)
training_set = HANData(
df_train,
opt['vocab_path'],
task,
max_sent_length,
max_word_length)
training_generator = DataLoader(training_set, **training_params)
test_set = HANData(
df_test,
opt['vocab_path'],
task,
max_sent_length,
max_word_length)
test_generator = DataLoader(test_set, **test_params)
valid_set = HANData(
df_valid,
opt['vocab_path'],
task,
max_sent_length,
max_word_length)
valid_generator = DataLoader(valid_set, **test_params)
model = HierAttNet(
opt['word_hidden_size'],
opt['sent_hidden_size'],
opt['batch_size'],
training_set.num_classes,
opt['vocab_path'],
max_sent_length,
max_word_length)
if torch.cuda.is_available():
model.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=opt['lr'])
best_loss = 1e5
best_epoch = 0
num_iter_per_epoch = len(training_generator)
for epoch in range(opt['num_epochs']):
model.train()
for iter, (feature, label) in enumerate(training_generator):
if torch.cuda.is_available():
feature = feature.cuda()
label = label.cuda()
optimizer.zero_grad()
model._init_hidden_state()
predictions = model(feature)
print(predictions.shape)
loss = criterion(predictions, label)
loss.backward()
optimizer.step()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm_(model.parameters(), 0.25)
training_metrics = get_evaluation(
label.cpu().numpy(),
predictions.cpu().detach().numpy(),
list_metrics=["accuracy"])
print(
"Epoch: {}/{}, Iteration: {}/{}, Lr: {}, Loss: {}, Accuracy: {}".format(
epoch + 1,
opt['num_epochs'],
iter + 1,
num_iter_per_epoch,
optimizer.param_groups[0]['lr'],
loss,
training_metrics["accuracy"]))
if epoch % opt['test_interval'] == 0:
model.eval()
loss_ls = []
te_label_ls = []
te_pred_ls = []
for te_feature, te_label in valid_generator:
num_sample = len(te_label)
if torch.cuda.is_available():
te_feature = te_feature.cuda()
te_label = te_label.cuda()
with torch.no_grad():
model._init_hidden_state(num_sample)
te_predictions = model(te_feature)
te_loss = criterion(te_predictions, te_label)
loss_ls.append(te_loss * num_sample)
te_label_ls.extend(te_label.clone().cpu())
te_pred_ls.append(te_predictions.clone().cpu())
te_loss = sum(loss_ls) / test_set.__len__()
te_pred = torch.cat(te_pred_ls, 0)
te_label = np.array(te_label_ls)
test_metrics = get_evaluation(
te_label, te_pred.numpy(), list_metrics=[
"accuracy", "confusion_matrix"])
print("Epoch: {}/{}, Lr: {}, Loss: {}, Accuracy: {}".format(
epoch + 1,
opt['num_epochs'],
optimizer.param_groups[0]['lr'],
te_loss, test_metrics["accuracy"]))
if te_loss + opt['es_min_delta'] < best_loss:
best_loss = te_loss
best_epoch = epoch
print('Saving model')
torch.save(
model,
opt['saved_path'] +
os.sep +
f"{epoch}_{iter+1}_subsample_{subsample}_han.bin")
# Early stopping
if epoch - best_epoch > opt['es_patience'] > 0:
print(
"Stop training at epoch {}. The lowest loss achieved is {}".format(
epoch, best_loss))
break
print()
print('Evaluation: ')
print()
torch.save(
model,
opt['saved_path'] +
os.sep +
"whole_model_han.pt")
model.eval()
model = torch.load(opt['saved_path'] + os.sep + "whole_model_han.pt")
loss_ls = []
te_label_ls = []
te_pred_ls = []
for te_feature, te_label in test_generator:
num_sample = len(te_label)
if torch.cuda.is_available():
te_feature = te_feature.cuda()
te_label = te_label.cuda()
with torch.no_grad():
model._init_hidden_state(num_sample)
te_predictions = model(te_feature)
te_loss = criterion(te_predictions, te_label)
loss_ls.append(te_loss * num_sample)
te_label_ls.extend(te_label.clone().cpu())
te_pred_ls.append(te_predictions.clone().cpu())
te_pred = torch.cat(te_pred_ls, 0)
te_label = np.array(te_label_ls)
test_metrics = get_evaluation(
te_label,
te_pred.numpy(),
list_metrics=[
"accuracy",
"precision",
"recall",
"weighted f1",
"macro f1",
"f1",
"confusion_matrix"])
true = te_label
preds = np.argmax(te_pred.numpy(), -1)
predicted_all_folds.extend(preds)
true_all_folds.extend(true)
f1 = test_metrics['f1']
macro_f1 = test_metrics['macro_f1']
micro_f1 = test_metrics['micro_f1']
rmse = test_metrics['rsme']
# print("Test set accuracy: {}".format(test_metrics["accuracy"]))
# print("Test set precision: {}".format(test_metrics["precision"]))
# print("Test set recall: {}".format(test_metrics["recall"]))
# print("Test set weighted f1: {}".format(test_metrics["f1"]))
# print("Test set macro f1: {}".format(test_metrics["macro_f1"]))
# print("Test set cm: {}".format(test_metrics["confusion_matrix"]))
for (metric, val) in test_metrics.items():
print("Test set {0}: {1}".format(metric, val))
accuracies_all_folds.append(test_metrics["accuracy"])
precision_all_folds.append(test_metrics["precision"])
recall_all_folds.append(test_metrics["recall"])
f1_all_folds.append(test_metrics["f1"])
f1_macro_all_folds.append(macro_f1)
f1_micro_all_folds.append(micro_f1)
print()
print()
print("Task: ", task)
print("Accuracy: ", accuracy_score(true_all_folds, predicted_all_folds))
print(
'Confusion matrix: ',
confusion_matrix(
true_all_folds,
predicted_all_folds))
print('All folds accuracy: ', accuracies_all_folds)
print('All folds precision: ', precision_all_folds)
print('All folds recall: ', recall_all_folds)
print('All folds f1: ', f1_all_folds)
print('All folds macro - f1: ', f1_macro_all_folds)
print('All folds micro - f1: ', f1_micro_all_folds)
if fed_data is not None:
class FakeMagpie:
def predict_from_text(self, text: str, return_float=False):
df_single = pd.DataFrame(
[[text, -1]], columns=[
cols_to_train[0], cols_to_train[1]])
el_set = HANData(
df_single,
opt['vocab_path'],
task,
max_sent_length,
max_word_length)
el_generator = DataLoader(el_set, **test_params)
for te_feature, te_label in el_generator:
num_sample = len(te_label)
if torch.cuda.is_available():
te_feature = te_feature.cuda()
te_label = te_label.cuda()
with torch.no_grad():
model._init_hidden_state(num_sample)
te_predictions = model(te_feature)
if return_float:
return float(te_predictions.clone().cpu())
else:
return int(te_predictions.clone().cpu())
return FakeMagpie(), f1, macro_f1, micro_f1, rmse
def train(opt):
device = torch.device("cuda" if (torch.cuda.is_available()) else "cpu")
if torch.cuda.is_available():
torch.cuda.manual_seed(123)
else:
torch.manual_seed(123)
output_file = open(opt.saved_path + os.sep + "logs.txt", "w")
output_file.write("Model's parameters: {}".format(vars(opt)))
training_params = {
"batch_size": opt.batch_size,
"shuffle": True,
"drop_last": True
}
test_params = {
"batch_size": opt.batch_size,
"shuffle": False,
"drop_last": False
}
# max_word_length, max_sent_length = get_max_lengths(opt.train_set)
max_word_length, max_sent_length = 13, 24
vocab = read_vocab('data/yelp_review_full_csv/train.csv.txt')
emb, word_to_ix = get_pretrained_word_embedding(opt.word2vec_path, vocab)
df = pd.read_csv(opt.train_set, names=['label', 'text'])
texts = np.array(df['text'])
labels = np.array(df['label'])
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)
for train_index, test_index in sss.split(texts, labels):
X_train, X_valid = texts[train_index], texts[test_index]
y_train, y_valid = labels[train_index], labels[test_index]
training_set = Custom_Dataset(X_train, y_train, word_to_ix,
max_sent_length, max_word_length)
valid_set = Custom_Dataset(X_valid, y_valid, word_to_ix, max_sent_length,
max_word_length)
training_generator = DataLoader(training_set,
num_workers=32,
**training_params)
valid_generator = DataLoader(valid_set, num_workers=32, **training_params)
df_test = pd.read_csv(opt.test_set, names=['label', 'text'])
test_texts = np.array(df_test['text'])
test_labels = np.array(df_test['label'])
test_set = Custom_Dataset(test_texts, test_labels, word_to_ix,
max_sent_length, max_word_length)
test_generator = DataLoader(test_set, num_workers=32, **test_params)
model = nn.DataParallel(
HierarchicalAttention(opt.word_hidden_size, opt.sent_hidden_size,
opt.batch_size, training_set.num_classes, emb,
max_sent_length, max_word_length))
if os.path.isdir(opt.log_path):
shutil.rmtree(opt.log_path)
os.makedirs(opt.log_path)
if torch.cuda.is_available():
model.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=opt.lr)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='max',
factor=0.1,
patience=5,
verbose=True,
min_lr=1e-8)
best_acc = 0.
best_epoch = 0
model.train()
num_iter_per_epoch = len(training_generator)
for epoch in range(opt.num_epoches):
print("Epoch " + str(epoch))
for feature, label, doc_len, sent_len in training_generator:
if torch.cuda.is_available():
sent_len = torch.stack(sent_len, dim=1).to(device)
doc_len = doc_len.to(device)
feature = feature.to(device)
label = label.to(device)
optimizer.zero_grad()
predictions = model(feature, sent_len, doc_len)
loss = criterion(predictions, label)
loss.backward()
optimizer.step()
training_metrics = get_evaluation(label.cpu().numpy(),
predictions.cpu().detach().numpy(),
list_metrics=["accuracy"])
print("Epoch: {}/{}, Lr: {}, Loss: {}, Accuracy: {}".format(
epoch + 1, opt.num_epoches, optimizer.param_groups[0]['lr'], loss,
training_metrics["accuracy"]))
if epoch % opt.test_interval == 0:
model.eval()
loss_ls = []
te_label_ls = []
te_pred_ls = []
for te_feature, te_label, te_doc_len, te_sent_len in test_generator:
num_sample = len(te_label)
if torch.cuda.is_available():
te_sent_len = torch.stack(te_sent_len, dim=1).to(device)
te_doc_len = te_doc_len.to(device)
te_feature = te_feature.to(device)
te_label = te_label.to(device)
with torch.no_grad():
te_predictions = model(te_feature, te_sent_len, te_doc_len)
te_loss = criterion(te_predictions, te_label)
loss_ls.append(te_loss * num_sample)
te_label_ls.extend(te_label.clone().cpu())
te_pred_ls.append(te_predictions.clone().cpu())
te_loss = sum(loss_ls) / test_set.__len__()
te_pred = torch.cat(te_pred_ls, 0)
te_label = np.array(te_label_ls)
test_metrics = get_evaluation(
te_label,
te_pred.numpy(),
list_metrics=["accuracy", "confusion_matrix"])
vl_loss_ls = []
vl_label_ls = []
vl_pred_ls = []
for vl_feature, vl_label, vl_doc_len, vl_sent_len in valid_generator:
num_sample = len(vl_label)
if torch.cuda.is_available():
vl_sent_len = torch.stack(vl_sent_len, dim=1).to(device)
vl_doc_len = vl_doc_len.to(device)
vl_feature = vl_feature.to(device)
vl_label = vl_label.to(device)
with torch.no_grad():
vl_predictions = model(vl_feature, vl_sent_len, vl_doc_len)
vl_loss = criterion(vl_predictions, vl_label)
vl_loss_ls.append(vl_loss * num_sample)
vl_label_ls.extend(vl_label.clone().cpu())
vl_pred_ls.append(vl_predictions.clone().cpu())
vl_loss = sum(vl_loss_ls) / valid_set.__len__()
vl_pred = torch.cat(vl_pred_ls, 0)
vl_label = np.array(vl_label_ls)
vl_metrics = get_evaluation(
vl_label,
vl_pred.numpy(),
list_metrics=["accuracy", "confusion_matrix"])
output_file.write(
"Epoch: {}/{} \nValid loss: {} Valid accuracy: {} \nValid confusion matrix: \n{}\nTest loss: {} Test accuracy: {} \nTest confusion matrix: \n{}\n\n"
.format(epoch + 1, opt.num_epoches, vl_loss,
vl_metrics["accuracy"], vl_metrics["confusion_matrix"],
te_loss, test_metrics["accuracy"],
test_metrics["confusion_matrix"]))
print(
"Epoch: {}/{}, Lr: {},Valid Loss: {}, Valid Accuracy: {}, Test Loss: {}, Test Accuracy: {}"
.format(epoch + 1, opt.num_epoches,
optimizer.param_groups[0]['lr'], vl_loss,
vl_metrics["accuracy"], te_loss,
test_metrics["accuracy"]))
scheduler.step(vl_metrics["accuracy"])
model.train()
if vl_metrics["accuracy"] > best_acc:
best_acc = vl_metrics["accuracy"]
best_epoch = epoch
torch.save(model, opt.saved_path + os.sep + "whole_model_han")
# Early stopping
if epoch - best_epoch > opt.es_patience > 0:
print(
"Stop training at epoch {}. The lowest loss achieved is {}"
.format(epoch, te_loss))
break
if torch.cuda.is_available():
feature = feature.type('torch.FloatTensor').cuda()
label = label.cuda() # batch size * num of labels
available_label_ind = label!=-1
optimizer.zero_grad()
model._init_hidden_state()
predictions = model(feature)
loss = criterion(predictions[available_label_ind], label[available_label_ind])
# loss = criterion(predictions, label)
loss.backward()
optimizer.step()
with torch.no_grad(): #(Nest the remaining lines inside)
# Use tensor operations in get_evaluation to stay in the CUDA space
training_metrics = get_evaluation(label.cpu().numpy(), predictions.cpu().numpy(), list_metrics=["accuracy"])
print("Epoch: {}/{}, Iteration: {}/{}, Lr: {}, Loss: {}, Accuracy: {}".format(
epoch + 1,
opt.num_epoches,
iter + 1,
num_iter_per_epoch,
optimizer.param_groups[0]['lr'],
loss, training_metrics["accuracy"]))
if iter % 20 == 0:
model.eval()
loss_ls = []
te_label_ls = []