def evaluate_cnn(model, validation_generator, criterion, epoch, print_every=25): model.eval() losses = AverageMeter() accuracies = AverageMeter() aucs = AverageMeter() num_iter_per_epoch = len(validation_generator) y_true, y_pred, pos_scores = [], [], [] progress_bar = tqdm(enumerate(validation_generator), total=num_iter_per_epoch) for iter, batch in progress_bar: samples, labels = batch labels = labels.squeeze(1) if use_cuda: samples = samples.cuda() labels = labels.cuda() with torch.no_grad(): logits = model(samples) loss = criterion(logits, labels) ## validation metrics validation_metrics = get_evaluation(labels.cpu().numpy(), logits.cpu().detach().numpy(), metrics_list=['accuracy']) acc = validation_metrics['accuracy'] losses.update(loss.data, samples.size(0)) accuracies.update(acc, samples.size(0)) y_true += labels.cpu().numpy().tolist() y_pred += torch.max(logits, 1)[1].cpu().numpy().tolist() pos_scores += logits.cpu().detach().numpy()[:,1].tolist() if (iter % print_every == 0) and (iter > 0): print_str = "[Validation - Epoch: {}], Iteration: {}/{}, Loss: {}, Accuracy: {}" print(print_str.format( epoch + 1, iter, num_iter_per_epoch, losses.avg, accuracies.avg, )) try: vali_auc = metrics.roc_auc_score(y_true, pos_scores) except ValueError as e: vali_auc = 0 exit() print("Avg loss: {}, Acc: {}, AUC: {}".format(losses.avg.item(), accuracies.avg.item(), vali_auc)) return losses.avg.item(), accuracies.avg.item(), vali_auc
for n_iter, batch in tqdm(enumerate(test_generator), total=num_iter_per_epoch): features, labels = batch labels.sub_(1) if torch.cuda.is_available(): features = features.cuda() labels = labels.cuda() with torch.no_grad(): predictions = model(features) loss = criterion(predictions, labels) y_true += labels.cpu().numpy().tolist() y_pred += torch.max(predictions, 1)[1].cpu().numpy().tolist() validation_metrics = utils.get_evaluation(labels.cpu().numpy(), predictions.cpu().detach().numpy(), list_metrics=["accuracy", "f1_weighted", "f1_micro", "f1_macro"]) accuracy = validation_metrics['accuracy'] f1_weighted = validation_metrics['f1_weighted'] f1_micro = validation_metrics['f1_micro'] f1_macro = validation_metrics["f1_macro"] losses.update(loss.data, features.size(0)) accuracies.update(validation_metrics["accuracy"], features.size(0)) writer.add_scalar('Test/Loss', loss.item(), n_iter) writer.add_scalar('Test/Accuracy', accuracy,
def train_cnn(model, training_generator, optimizer, criterion, epoch, print_every=25): model.train() losses = AverageMeter() accuracies = AverageMeter() num_iter_per_epoch = len(training_generator) y_true, y_pred, pos_scores = [], [], [] progress_bar = tqdm(enumerate(training_generator), total=num_iter_per_epoch) for iter, batch in progress_bar: samples, labels = batch labels = labels.squeeze(1) if use_cuda: samples = samples.cuda() labels = labels.cuda() optimizer.zero_grad() logits = model(samples) loss = criterion(logits, labels) loss.backward() optimizer.step() ## training metrics training_metrics = get_evaluation( labels.cpu().numpy(), logits.cpu().detach().numpy(), metrics_list=["accuracy"], ) acc = training_metrics["accuracy"] losses.update(loss.data, samples.size(0)) accuracies.update(acc, samples.size(0)) y_true += labels.cpu().numpy().tolist() y_pred += torch.max(logits, 1)[1].cpu().numpy().tolist() pos_scores += logits.cpu().detach().numpy()[:, 1].tolist() lr = optimizer.state_dict()["param_groups"][0]["lr"] if (iter % print_every == 0) and (iter > 0): print_str = "[Training - Epoch: {}], LR: {}, Iteration: {}/{}, Loss: {}, Accuracy: {}" print( print_str.format( epoch + 1, lr, iter, num_iter_per_epoch, losses.avg, accuracies.avg, )) try: train_auc = metrics.roc_auc_score(y_true, pos_scores) except ValueError as e: print("y_prob = ", y_prob) print("y_true = ", y_true) train_auc = 0 exit() print("Avg loss: {}, Acc: {}, AUC: {}".format(losses.avg.item(), accuracies.avg.item(), train_auc)) return losses.avg.item(), accuracies.avg.item(), train_auc
def train(model, training_generator, optimizer, criterion, epoch, writer, log_file, scheduler, class_names, args, print_every=25): model.train() losses = utils.AverageMeter() accuracies = utils.AverageMeter() num_iter_per_epoch = len(training_generator) progress_bar = tqdm(enumerate(training_generator), total=num_iter_per_epoch) y_true = [] y_pred = [] for iter, batch in progress_bar: features, labels = batch if torch.cuda.is_available(): features = features.cuda() labels = labels.cuda() optimizer.zero_grad() predictions = model(features) y_true += labels.cpu().numpy().tolist() y_pred += torch.max(predictions, 1)[1].cpu().numpy().tolist() loss = criterion(predictions, labels) loss.backward() if args.scheduler == 'clr': scheduler.step() optimizer.step() training_metrics = utils.get_evaluation( labels.cpu().numpy(), predictions.cpu().detach().numpy(), list_metrics=["accuracy", "f1"]) losses.update(loss.data, features.size(0)) accuracies.update(training_metrics["accuracy"], features.size(0)) f1 = training_metrics['f1'] writer.add_scalar('Train/Loss', loss.item(), epoch * num_iter_per_epoch + iter) writer.add_scalar('Train/Accuracy', training_metrics['accuracy'], epoch * num_iter_per_epoch + iter) writer.add_scalar('Train/f1', f1, epoch * num_iter_per_epoch + iter) lr = optimizer.state_dict()["param_groups"][0]["lr"] if (iter % print_every == 0) and (iter > 0): print( "[Training - Epoch: {}], LR: {} , Iteration: {}/{} , Loss: {}, Accuracy: {}" .format(epoch + 1, lr, iter, num_iter_per_epoch, losses.avg, accuracies.avg)) if bool(args.log_f1): intermediate_report = classification_report(y_true, y_pred, output_dict=True) f1_by_class = 'F1 Scores by class: ' for class_name in class_names: f1_by_class += f"{class_name} : {np.round(intermediate_report[class_name]['f1-score'], 4)} |" print(f1_by_class) f1_train = f1_score(y_true, y_pred, average='weighted') writer.add_scalar('Train/loss/epoch', losses.avg, epoch + iter) writer.add_scalar('Train/acc/epoch', accuracies.avg, epoch + iter) writer.add_scalar('Train/f1/epoch', f1_train, epoch + iter) report = classification_report(y_true, y_pred) print(report) with open(log_file, 'a') as f: f.write(f'Training on Epoch {epoch} \n') f.write(f'Average loss: {losses.avg.item()} \n') f.write(f'Average accuracy: {accuracies.avg.item()} \n') f.write(f'F1 score: {f1_train} \n\n') f.write(report) f.write('*' * 25) f.write('\n') return losses.avg.item(), accuracies.avg.item(), f1_train
def evaluate(model, validation_generator, criterion, epoch, writer, log_file, print_every=25): model.eval() losses = utils.AverageMeter() accuracies = utils.AverageMeter() num_iter_per_epoch = len(validation_generator) y_true = [] y_pred = [] for iter, batch in tqdm(enumerate(validation_generator), total=num_iter_per_epoch): features, labels = batch if torch.cuda.is_available(): features = features.cuda() labels = labels.cuda() with torch.no_grad(): predictions = model(features) loss = criterion(predictions, labels) y_true += labels.cpu().numpy().tolist() y_pred += torch.max(predictions, 1)[1].cpu().numpy().tolist() validation_metrics = utils.get_evaluation( labels.cpu().numpy(), predictions.cpu().detach().numpy(), list_metrics=["accuracy", "f1"]) accuracy = validation_metrics['accuracy'] f1 = validation_metrics['f1'] losses.update(loss.data, features.size(0)) accuracies.update(validation_metrics["accuracy"], features.size(0)) writer.add_scalar('Test/Loss', loss.item(), epoch * num_iter_per_epoch + iter) writer.add_scalar('Test/Accuracy', accuracy, epoch * num_iter_per_epoch + iter) writer.add_scalar('Test/f1', f1, epoch * num_iter_per_epoch + iter) if (iter % print_every == 0) and (iter > 0): print( "[Validation - Epoch: {}] , Iteration: {}/{} , Loss: {}, Accuracy: {}" .format(epoch + 1, iter, num_iter_per_epoch, losses.avg, accuracies.avg)) f1_test = f1_score(y_true, y_pred, average='weighted') writer.add_scalar('Test/loss/epoch', losses.avg, epoch + iter) writer.add_scalar('Test/acc/epoch', accuracies.avg, epoch + iter) writer.add_scalar('Test/f1/epoch', f1_test, epoch + iter) report = classification_report(y_true, y_pred) print(report) with open(log_file, 'a') as f: f.write(f'Validation on Epoch {epoch} \n') f.write(f'Average loss: {losses.avg.item()} \n') f.write(f'Average accuracy: {accuracies.avg.item()} \n') f.write(f'F1 score {f1_test} \n\n') f.write(report) f.write('=' * 50) f.write('\n') return losses.avg.item(), accuracies.avg.item(), f1_test
def train(opt): dataset_train, dataset_test = fetch_dataset(opt.dataset) dataset_train = CharacterDataset(dataset_train, opt.max_length) dataset_test = CharacterDataset(dataset_test, opt.max_length) train_generator = DataLoader(dataset_train, batch_size=opt.batch_size, shuffle=True) test_generator = DataLoader(dataset_test, batch_size=opt.batch_size, shuffle=False) if opt.feature == "small": model = CharacterLevelCNN(input_length=opt.max_length, n_classes=dataset_train.num_classes, input_dim=len(opt.alphabet), n_conv_filters=256, n_fc_neurons=1024, init_std=0.05) elif opt.feature == "large": model = CharacterLevelCNN(input_length=opt.max_length, n_classes=dataset_test.num_classes, input_dim=len(opt.alphabet), n_conv_filters=1024, n_fc_neurons=2048, init_std=0.02) else: sys.exit("Invalid feature mode!") log_path = "{}_{}_{}".format(opt.log_path, opt.feature, opt.dataset) if os.path.isdir(log_path): shutil.rmtree(log_path) os.makedirs(log_path) writer = SummaryWriter(log_path) if torch.cuda.is_available(): model.cuda() criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) best_loss = 1e5 best_epoch = 0 model.train() num_iter_per_epoch = len(train_generator) for epoch in range(opt.num_epochs): for iter, batch in enumerate(train_generator): feature, label = batch label = label.long() if torch.cuda.is_available(): feature = feature.cuda() label = label.cuda() optimizer.zero_grad() pred = model(feature) loss = criterion(pred, label) loss.backward() optimizer.step() training_metrics = get_evaluation(label.cpu().numpy(), pred.cpu().detach().numpy(), list_metrics=["accuracy"]) print( "Epoch: %d/%d, Iteration: %d/%d, lr: %.6f, loss: %.6f, acc: %.6f" % (epoch + 1, opt.num_epochs, iter + 1, num_iter_per_epoch, optimizer.param_groups[0]['lr'], loss.item(), training_metrics["accuracy"])) writer.add_scalar('Train/Loss', loss, epoch * num_iter_per_epoch + iter) writer.add_scalar('Train/Accuracy', training_metrics["accuracy"], epoch * num_iter_per_epoch + iter) model.eval() loss_ls = [] te_label_ls = [] te_pred_ls = [] for batch in test_generator: with torch.no_grad(): te_feature, te_label = batch te_label = te_label.long() num_sample = len(te_label) if torch.cuda.is_available(): te_feature = te_feature.cuda() te_label = te_label.cuda() te_predictions = model(te_feature) te_loss = criterion(te_predictions, te_label) loss_ls.append(te_loss.item() * num_sample) te_label_ls.extend(te_label.clone().cpu()) te_pred_ls.append(te_predictions.clone().cpu()) te_loss = sum(loss_ls) / len(dataset_test) te_pred = torch.cat(te_pred_ls, 0) te_label = np.array(te_label_ls) test_metrics = get_evaluation( te_label, te_pred.numpy(), list_metrics=["accuracy", "confusion_matrix"]) print("Epoch: %d/%d, loss: %.6f, acc: %.6f,confusion matrix: \n%s" % (epoch + 1, opt.num_epochs, te_loss, test_metrics["accuracy"], test_metrics["confusion_matrix"])) writer.add_scalar('Test/Loss', te_loss, epoch) writer.add_scalar('Test/Accuracy', test_metrics["accuracy"], epoch) model.train() if te_loss + opt.es_min_delta < best_loss: best_loss = te_loss best_epoch = epoch torch.save( model.state_dict(), "./weights/char-cnn_%s_%s_%d.pth" % (opt.dataset, opt.feature, epoch)) # Early stopping if epoch - best_epoch > opt.es_patience > 0: print( "Stop training at epoch %d. The lowest loss achieved is %.6f at epoch %d" % (epoch, best_loss, best_epoch)) break
def train(model, training_generator, optimizer, criterion, epoch, start_iter, writer, log_file, scheduler, class_names, args, print_every=10): model.train() losses = utils.AverageMeter() accuracies = utils.AverageMeter() num_iter_per_epoch = len(training_generator) progress_bar = tqdm(enumerate(training_generator, start=start_iter), total=num_iter_per_epoch) y_true = [] y_pred = [] for iter, batch in progress_bar: features, labels = batch labels.sub_(1) if torch.cuda.is_available(): features = features.cuda() labels = labels.cuda() optimizer.zero_grad() predictions = model(features) y_true += labels.cpu().numpy().tolist() y_pred += torch.max(predictions, 1)[1].cpu().numpy().tolist() loss = criterion(predictions, labels) loss.backward() if args.get('Train', 'scheduler') == 'clr': scheduler.step() optimizer.step() training_metrics = utils.get_evaluation(labels.cpu().numpy(), predictions.cpu().detach().numpy(), list_metrics=["accuracy", "f1_weighted", "f1_micro", "f1_macro"]) losses.update(loss.data, features.size(0)) accuracies.update(training_metrics["accuracy"], features.size(0)) f1_weighted = training_metrics['f1_weighted'] f1_micro = training_metrics['f1_micro'] f1_macro = training_metrics["f1_macro"] writer.add_scalar('Train/Loss', loss.item(), epoch * num_iter_per_epoch + iter) writer.add_scalar('Train/Accuracy', training_metrics['accuracy'], epoch * num_iter_per_epoch + iter) writer.add_scalar('Train/f1-weighted', f1_weighted, epoch * num_iter_per_epoch + iter) writer.add_scalar('Train/f1-micro', f1_micro, epoch * num_iter_per_epoch + iter) writer.add_scalar('Train/f1-macro', f1_macro, epoch * num_iter_per_epoch + iter) #f1_scalars = {'f1_weighted': f1_weighted, 'f1_micro': f1_micro, 'f1_macro': f1_macro} #writer.add_scalars('Train/all-f1-scores', # f1_scalars, # epoch * num_iter_per_epoch + iter) for name, param in model.named_parameters(): writer.add_histogram(name, param, iter) lr = optimizer.state_dict()["param_groups"][0]["lr"] if (iter % print_every == 0) and (iter > 0): print("\n[Training - Epoch: {}], LR: {} , Iteration: {}/{} , Loss: {}, Accuracy: {}".format( epoch + 1, lr, iter, num_iter_per_epoch, losses.avg, accuracies.avg )) if bool(args.getboolean('Log', 'log_f1')): intermediate_report = classification_report( y_true, y_pred, output_dict=True) print(intermediate_report) # f1_by_class = 'F1 Scores by class: ' # for class_name in class_names: # f1_by_class += f"{class_name} : {np.round(intermediate_report[class_name]['f1-score'], 4)} |" # print(f1_by_class) f1_train_weighted = f1_score(y_true, y_pred, average='weighted') f1_train_micro = f1_score(y_true, y_pred, average='micro') f1_train_macro = f1_score(y_true, y_pred, average='macro') writer.add_scalar('Train/loss/epoch', losses.avg, epoch + iter) writer.add_scalar('Train/acc/epoch', accuracies.avg, epoch + iter) writer.add_scalar('Train/f1-weighted/epoch', f1_train_weighted, epoch + iter) writer.add_scalar('Train/f1-micro/epoch', f1_train_micro, epoch + iter) writer.add_scalar('Train/f1-macro/epoch', f1_train_macro, epoch + iter) #f1_scalars = {'f1_weighted': f1_train_weighted, 'f1_micro': f1_train_micro, 'f1_macro': f1_train_macro} #writer.add_scalars('Train/all-f1-scores/epoch', # f1_scalars, # epoch * num_iter_per_epoch + iter) report = classification_report(y_true, y_pred) print(report) with open(log_file, 'a') as f: f.write(f'Training on Epoch {epoch} \n') f.write(f'Average loss: {losses.avg.item()} \n') f.write(f'Average accuracy: {accuracies.avg.item()} \n') f.write(f'F1 Weighted score: {f1_train_weighted} \n\n') f.write(f'F1 Micro score: {f1_train_micro} \n\n') f.write(f'F1 Macro score: {f1_train_macro} \n\n') f.write(report) f.write('*' * 25) f.write('\n') return losses.avg.item(), accuracies.avg.item(), f1_train_weighted
def train( opt, task, data_path=None, fed_data=None, k_fold=0, cols_to_train=['tweets', 'judgments'], subsample=1, overwrite=True, config=''): """ Runs the HAN with the given parameters Args: opt (dict): Configuration for the model task (str): Name of the task data_path (str, optional): Path to the data. Defaults to None. fed_data (pd.DataFrame, optional): [description]. Defaults to None. k_fold (int, optional): [description]. Defaults to 0. cols_to_train (list, optional): [description]. Defaults to ['tweets', 'judgments']. subsample (int, optional): [description]. Defaults to 1. overwrite (bool, optional): [description]. Defaults to True. config (str, optional): [description]. Defaults to ''. Returns: [type]: [description] """ if torch.cuda.is_available(): torch.cuda.manual_seed(2019) else: torch.manual_seed(2019) output_file = open(opt['saved_path'] + "logs.txt", "w") output_file.write("Model's parameters: {}".format(opt.items())) training_params = {"batch_size": opt['batch_size'], "shuffle": False, "drop_last": True} test_params = {"batch_size": opt['batch_size'], "shuffle": False, "drop_last": False} max_word_length, max_sent_length = get_max_lengths( opt['train_set'], fed_data=fed_data) if fed_data is None: df_data = pd.read_csv(data_path, encoding='utf8', sep=',') else: df_train = pd.DataFrame(fed_data[0], columns=cols_to_train), df_test = pd.DataFrame(fed_data[1], columns=cols_to_train) kf = model_selection.KFold(n_splits=5) predicted_all_folds = [] true_all_folds = [] counter = 0 accuracies_all_folds = [] precision_all_folds = [] recall_all_folds = [] f1_all_folds = [] f1_macro_all_folds = [] f1_micro_all_folds = [] if fed_data is None: kf = model_selection.KFold(n_splits=5) iterator = kf.split(df_data) else: iterator = [(0, 0)] for train_index, test_index in iterator: counter += 1 print() print("CV fold: ", counter) print() if os.path.exists(opt['vocab_path']): os.remove(opt['vocab_path']) if fed_data is None: df_train, df_test = df_data.iloc[train_index], df_data.iloc[test_index] sep_idx = int(df_test.shape[0] / 2) df_valid = df_test[:sep_idx] df_test = df_test[sep_idx:] print( "Train size: ", df_train.shape, "Valid size: ", df_valid.shape, "Test size: ", df_test.shape) training_set = HANData( df_train, opt['vocab_path'], task, max_sent_length, max_word_length) training_generator = DataLoader(training_set, **training_params) test_set = HANData( df_test, opt['vocab_path'], task, max_sent_length, max_word_length) test_generator = DataLoader(test_set, **test_params) valid_set = HANData( df_valid, opt['vocab_path'], task, max_sent_length, max_word_length) valid_generator = DataLoader(valid_set, **test_params) model = HierAttNet( opt['word_hidden_size'], opt['sent_hidden_size'], opt['batch_size'], training_set.num_classes, opt['vocab_path'], max_sent_length, max_word_length) if torch.cuda.is_available(): model.cuda() criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=opt['lr']) best_loss = 1e5 best_epoch = 0 num_iter_per_epoch = len(training_generator) for epoch in range(opt['num_epochs']): model.train() for iter, (feature, label) in enumerate(training_generator): if torch.cuda.is_available(): feature = feature.cuda() label = label.cuda() optimizer.zero_grad() model._init_hidden_state() predictions = model(feature) print(predictions.shape) loss = criterion(predictions, label) loss.backward() optimizer.step() # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs. torch.nn.utils.clip_grad_norm_(model.parameters(), 0.25) training_metrics = get_evaluation( label.cpu().numpy(), predictions.cpu().detach().numpy(), list_metrics=["accuracy"]) print( "Epoch: {}/{}, Iteration: {}/{}, Lr: {}, Loss: {}, Accuracy: {}".format( epoch + 1, opt['num_epochs'], iter + 1, num_iter_per_epoch, optimizer.param_groups[0]['lr'], loss, training_metrics["accuracy"])) if epoch % opt['test_interval'] == 0: model.eval() loss_ls = [] te_label_ls = [] te_pred_ls = [] for te_feature, te_label in valid_generator: num_sample = len(te_label) if torch.cuda.is_available(): te_feature = te_feature.cuda() te_label = te_label.cuda() with torch.no_grad(): model._init_hidden_state(num_sample) te_predictions = model(te_feature) te_loss = criterion(te_predictions, te_label) loss_ls.append(te_loss * num_sample) te_label_ls.extend(te_label.clone().cpu()) te_pred_ls.append(te_predictions.clone().cpu()) te_loss = sum(loss_ls) / test_set.__len__() te_pred = torch.cat(te_pred_ls, 0) te_label = np.array(te_label_ls) test_metrics = get_evaluation( te_label, te_pred.numpy(), list_metrics=[ "accuracy", "confusion_matrix"]) print("Epoch: {}/{}, Lr: {}, Loss: {}, Accuracy: {}".format( epoch + 1, opt['num_epochs'], optimizer.param_groups[0]['lr'], te_loss, test_metrics["accuracy"])) if te_loss + opt['es_min_delta'] < best_loss: best_loss = te_loss best_epoch = epoch print('Saving model') torch.save( model, opt['saved_path'] + os.sep + f"{epoch}_{iter+1}_subsample_{subsample}_han.bin") # Early stopping if epoch - best_epoch > opt['es_patience'] > 0: print( "Stop training at epoch {}. The lowest loss achieved is {}".format( epoch, best_loss)) break print() print('Evaluation: ') print() torch.save( model, opt['saved_path'] + os.sep + "whole_model_han.pt") model.eval() model = torch.load(opt['saved_path'] + os.sep + "whole_model_han.pt") loss_ls = [] te_label_ls = [] te_pred_ls = [] for te_feature, te_label in test_generator: num_sample = len(te_label) if torch.cuda.is_available(): te_feature = te_feature.cuda() te_label = te_label.cuda() with torch.no_grad(): model._init_hidden_state(num_sample) te_predictions = model(te_feature) te_loss = criterion(te_predictions, te_label) loss_ls.append(te_loss * num_sample) te_label_ls.extend(te_label.clone().cpu()) te_pred_ls.append(te_predictions.clone().cpu()) te_pred = torch.cat(te_pred_ls, 0) te_label = np.array(te_label_ls) test_metrics = get_evaluation( te_label, te_pred.numpy(), list_metrics=[ "accuracy", "precision", "recall", "weighted f1", "macro f1", "f1", "confusion_matrix"]) true = te_label preds = np.argmax(te_pred.numpy(), -1) predicted_all_folds.extend(preds) true_all_folds.extend(true) f1 = test_metrics['f1'] macro_f1 = test_metrics['macro_f1'] micro_f1 = test_metrics['micro_f1'] rmse = test_metrics['rsme'] # print("Test set accuracy: {}".format(test_metrics["accuracy"])) # print("Test set precision: {}".format(test_metrics["precision"])) # print("Test set recall: {}".format(test_metrics["recall"])) # print("Test set weighted f1: {}".format(test_metrics["f1"])) # print("Test set macro f1: {}".format(test_metrics["macro_f1"])) # print("Test set cm: {}".format(test_metrics["confusion_matrix"])) for (metric, val) in test_metrics.items(): print("Test set {0}: {1}".format(metric, val)) accuracies_all_folds.append(test_metrics["accuracy"]) precision_all_folds.append(test_metrics["precision"]) recall_all_folds.append(test_metrics["recall"]) f1_all_folds.append(test_metrics["f1"]) f1_macro_all_folds.append(macro_f1) f1_micro_all_folds.append(micro_f1) print() print() print("Task: ", task) print("Accuracy: ", accuracy_score(true_all_folds, predicted_all_folds)) print( 'Confusion matrix: ', confusion_matrix( true_all_folds, predicted_all_folds)) print('All folds accuracy: ', accuracies_all_folds) print('All folds precision: ', precision_all_folds) print('All folds recall: ', recall_all_folds) print('All folds f1: ', f1_all_folds) print('All folds macro - f1: ', f1_macro_all_folds) print('All folds micro - f1: ', f1_micro_all_folds) if fed_data is not None: class FakeMagpie: def predict_from_text(self, text: str, return_float=False): df_single = pd.DataFrame( [[text, -1]], columns=[ cols_to_train[0], cols_to_train[1]]) el_set = HANData( df_single, opt['vocab_path'], task, max_sent_length, max_word_length) el_generator = DataLoader(el_set, **test_params) for te_feature, te_label in el_generator: num_sample = len(te_label) if torch.cuda.is_available(): te_feature = te_feature.cuda() te_label = te_label.cuda() with torch.no_grad(): model._init_hidden_state(num_sample) te_predictions = model(te_feature) if return_float: return float(te_predictions.clone().cpu()) else: return int(te_predictions.clone().cpu()) return FakeMagpie(), f1, macro_f1, micro_f1, rmse
def train(opt): device = torch.device("cuda" if (torch.cuda.is_available()) else "cpu") if torch.cuda.is_available(): torch.cuda.manual_seed(123) else: torch.manual_seed(123) output_file = open(opt.saved_path + os.sep + "logs.txt", "w") output_file.write("Model's parameters: {}".format(vars(opt))) training_params = { "batch_size": opt.batch_size, "shuffle": True, "drop_last": True } test_params = { "batch_size": opt.batch_size, "shuffle": False, "drop_last": False } # max_word_length, max_sent_length = get_max_lengths(opt.train_set) max_word_length, max_sent_length = 13, 24 vocab = read_vocab('data/yelp_review_full_csv/train.csv.txt') emb, word_to_ix = get_pretrained_word_embedding(opt.word2vec_path, vocab) df = pd.read_csv(opt.train_set, names=['label', 'text']) texts = np.array(df['text']) labels = np.array(df['label']) sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0) for train_index, test_index in sss.split(texts, labels): X_train, X_valid = texts[train_index], texts[test_index] y_train, y_valid = labels[train_index], labels[test_index] training_set = Custom_Dataset(X_train, y_train, word_to_ix, max_sent_length, max_word_length) valid_set = Custom_Dataset(X_valid, y_valid, word_to_ix, max_sent_length, max_word_length) training_generator = DataLoader(training_set, num_workers=32, **training_params) valid_generator = DataLoader(valid_set, num_workers=32, **training_params) df_test = pd.read_csv(opt.test_set, names=['label', 'text']) test_texts = np.array(df_test['text']) test_labels = np.array(df_test['label']) test_set = Custom_Dataset(test_texts, test_labels, word_to_ix, max_sent_length, max_word_length) test_generator = DataLoader(test_set, num_workers=32, **test_params) model = nn.DataParallel( HierarchicalAttention(opt.word_hidden_size, opt.sent_hidden_size, opt.batch_size, training_set.num_classes, emb, max_sent_length, max_word_length)) if os.path.isdir(opt.log_path): shutil.rmtree(opt.log_path) os.makedirs(opt.log_path) if torch.cuda.is_available(): model.to(device) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=opt.lr) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.1, patience=5, verbose=True, min_lr=1e-8) best_acc = 0. best_epoch = 0 model.train() num_iter_per_epoch = len(training_generator) for epoch in range(opt.num_epoches): print("Epoch " + str(epoch)) for feature, label, doc_len, sent_len in training_generator: if torch.cuda.is_available(): sent_len = torch.stack(sent_len, dim=1).to(device) doc_len = doc_len.to(device) feature = feature.to(device) label = label.to(device) optimizer.zero_grad() predictions = model(feature, sent_len, doc_len) loss = criterion(predictions, label) loss.backward() optimizer.step() training_metrics = get_evaluation(label.cpu().numpy(), predictions.cpu().detach().numpy(), list_metrics=["accuracy"]) print("Epoch: {}/{}, Lr: {}, Loss: {}, Accuracy: {}".format( epoch + 1, opt.num_epoches, optimizer.param_groups[0]['lr'], loss, training_metrics["accuracy"])) if epoch % opt.test_interval == 0: model.eval() loss_ls = [] te_label_ls = [] te_pred_ls = [] for te_feature, te_label, te_doc_len, te_sent_len in test_generator: num_sample = len(te_label) if torch.cuda.is_available(): te_sent_len = torch.stack(te_sent_len, dim=1).to(device) te_doc_len = te_doc_len.to(device) te_feature = te_feature.to(device) te_label = te_label.to(device) with torch.no_grad(): te_predictions = model(te_feature, te_sent_len, te_doc_len) te_loss = criterion(te_predictions, te_label) loss_ls.append(te_loss * num_sample) te_label_ls.extend(te_label.clone().cpu()) te_pred_ls.append(te_predictions.clone().cpu()) te_loss = sum(loss_ls) / test_set.__len__() te_pred = torch.cat(te_pred_ls, 0) te_label = np.array(te_label_ls) test_metrics = get_evaluation( te_label, te_pred.numpy(), list_metrics=["accuracy", "confusion_matrix"]) vl_loss_ls = [] vl_label_ls = [] vl_pred_ls = [] for vl_feature, vl_label, vl_doc_len, vl_sent_len in valid_generator: num_sample = len(vl_label) if torch.cuda.is_available(): vl_sent_len = torch.stack(vl_sent_len, dim=1).to(device) vl_doc_len = vl_doc_len.to(device) vl_feature = vl_feature.to(device) vl_label = vl_label.to(device) with torch.no_grad(): vl_predictions = model(vl_feature, vl_sent_len, vl_doc_len) vl_loss = criterion(vl_predictions, vl_label) vl_loss_ls.append(vl_loss * num_sample) vl_label_ls.extend(vl_label.clone().cpu()) vl_pred_ls.append(vl_predictions.clone().cpu()) vl_loss = sum(vl_loss_ls) / valid_set.__len__() vl_pred = torch.cat(vl_pred_ls, 0) vl_label = np.array(vl_label_ls) vl_metrics = get_evaluation( vl_label, vl_pred.numpy(), list_metrics=["accuracy", "confusion_matrix"]) output_file.write( "Epoch: {}/{} \nValid loss: {} Valid accuracy: {} \nValid confusion matrix: \n{}\nTest loss: {} Test accuracy: {} \nTest confusion matrix: \n{}\n\n" .format(epoch + 1, opt.num_epoches, vl_loss, vl_metrics["accuracy"], vl_metrics["confusion_matrix"], te_loss, test_metrics["accuracy"], test_metrics["confusion_matrix"])) print( "Epoch: {}/{}, Lr: {},Valid Loss: {}, Valid Accuracy: {}, Test Loss: {}, Test Accuracy: {}" .format(epoch + 1, opt.num_epoches, optimizer.param_groups[0]['lr'], vl_loss, vl_metrics["accuracy"], te_loss, test_metrics["accuracy"])) scheduler.step(vl_metrics["accuracy"]) model.train() if vl_metrics["accuracy"] > best_acc: best_acc = vl_metrics["accuracy"] best_epoch = epoch torch.save(model, opt.saved_path + os.sep + "whole_model_han") # Early stopping if epoch - best_epoch > opt.es_patience > 0: print( "Stop training at epoch {}. The lowest loss achieved is {}" .format(epoch, te_loss)) break
if torch.cuda.is_available(): feature = feature.type('torch.FloatTensor').cuda() label = label.cuda() # batch size * num of labels available_label_ind = label!=-1 optimizer.zero_grad() model._init_hidden_state() predictions = model(feature) loss = criterion(predictions[available_label_ind], label[available_label_ind]) # loss = criterion(predictions, label) loss.backward() optimizer.step() with torch.no_grad(): #(Nest the remaining lines inside) # Use tensor operations in get_evaluation to stay in the CUDA space training_metrics = get_evaluation(label.cpu().numpy(), predictions.cpu().numpy(), list_metrics=["accuracy"]) print("Epoch: {}/{}, Iteration: {}/{}, Lr: {}, Loss: {}, Accuracy: {}".format( epoch + 1, opt.num_epoches, iter + 1, num_iter_per_epoch, optimizer.param_groups[0]['lr'], loss, training_metrics["accuracy"])) if iter % 20 == 0: model.eval() loss_ls = [] te_label_ls = []