training_batch = ds.nextBatch(args.batch_size) if args.load_weights: classifier.load_weights(args.load_weights) bestAcc = 0 for epoch in range(args.epochs): trainingLoss = 0 tp, tn, fp, fn = 0, 0, 0, 0 for _ in range(args.updates_per_epoch): x, label = next(training_batch) loss = classifier.update(x, label) trainingLoss += loss tpb, tnb, fpb, fnb = classifier.evaluate(x, label) tp += tpb tn += tnb fp += fpb fn += fnb trainAcc, trainPrec, _, _, trainFOR = metric.metric(tp, tn, fp, fn) x_test, label_test = ds.batchTest() tp, tn, fp, fn = classifier.evaluate(x_test, label_test) testAcc, testPrec, _, _, testFOR = metric.metric(tp, tn, fp, fn) trainingLoss /= args.updates_per_epoch s = "Loss: {: .4f}".format(trainingLoss) print(epoch, s) print( "Training Accuracy = {:.4f}, Precision = {:.4f}, FOR = {:.4f}".format(
def main(): parser = argparse.ArgumentParser() parser.add_argument("--seed", default=42, type=int, help="Random seed") parser.add_argument("--bert_model", default='roberta-base', type=str, help="BERT model") parser.add_argument("--train_batch_size", default=15, type=int, help="Total batch size for training.") parser.add_argument("--eval_batch_size", default=8, type=int, help="Total batch size for eval.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=25.0, type=float, help="Total number of training epochs to perform.") parser.add_argument( "--warmup_proportion", default=0.1, type=float, help= "Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument( "--no_cuda", action='store_true', #Store_true: false help="Whether not to use CUDA when available") parser.add_argument( '--gradient_accumulation_steps', type=int, default=1, help= "Number of updates steps to accumulate before performing a backward/update pass." ) parser.add_argument('--max_grad_norm', help='gradient clipping for Max gradient norm.', required=False, default=1.0, type=float) parser.add_argument( '--label_smoothing', type=float, default=0.1, help= 'Coefficient for label smoothing (default: 0.1, if 0.0, no label smoothing)' ) parser.add_argument('--max_seq_length', type=int, default=128, help='Maximum number of paraphrases for each sentence') parser.add_argument("--do_lower_case", action='store_true', help="Whether to lowercase input string") # Special params parser.add_argument('--train_file_path', type=str, default=None, help='Training data path') parser.add_argument('--dev_file_path', type=str, default=None, help='Validation data path') parser.add_argument('--oos_dev_file_path', type=str, default=None, help='Out-of-Scope validation data path') parser.add_argument('--output_dir', type=str, default=None, help='Output file path') parser.add_argument('--save_model_path', type=str, default='', help='path to save the model checkpoints') parser.add_argument('--few_shot_num', type=int, default=5, help='Number of training examples for each class') parser.add_argument('--num_trials', type=int, default=10, help='Number of trials to see robustness') parser.add_argument("--do_predict", action='store_true', help="do_predict the model") parser.add_argument("--do_final_test", action='store_true', help="do_predict the model") args = parser.parse_args() random.seed(args.seed) N = args.few_shot_num T = args.num_trials train_file_path = args.train_file_path dev_file_path = args.dev_file_path train_examples, dev_examples = load_intent_datasets( train_file_path, dev_file_path, args.do_lower_case) sampled_tasks = [sample(N, train_examples) for i in range(T)] if args.oos_dev_file_path is not None: oos_dev_examples = load_intent_examples(args.oos_dev_file_path, args.do_lower_case) else: oos_dev_examples = [] label_lists = [] intent_train_examples = [] intent_dev_examples = [] intent_oos_dev_examples = [] for i in range(T): tasks = sampled_tasks[i] label_lists.append([]) intent_train_examples.append([]) intent_dev_examples.append( [InputExample(e.text, None, e.label) for e in dev_examples]) intent_oos_dev_examples.append( [InputExample(e.text, None, None) for e in oos_dev_examples]) for task in tasks: label = task['task'] examples = task['examples'] label_lists[-1].append(label) for j in range(len(examples)): intent_train_examples[-1].append( InputExample(examples[j], None, label)) if args.output_dir is not None: folder_name = '{}/{}-shot-{}/'.format(args.output_dir, N, args.bert_model) if not os.path.exists(folder_name): os.makedirs(folder_name) file_name = 'batch_{}---epoch_{}---lr_{}'.format( args.train_batch_size, args.num_train_epochs, args.learning_rate) file_name = '{}__oos-threshold'.format(file_name) if args.do_final_test: file_name = file_name + '_TEST.txt' else: file_name = file_name + '.txt' f = open(folder_name + file_name, 'w') else: f = None for j in range(T): save_model_path = '{}_{}'.format(folder_name + args.save_model_path, j + 1) if os.path.exists(save_model_path): assert args.do_predict else: assert not args.do_predict if args.save_model_path and os.path.exists(save_model_path): model = Classifier(path=save_model_path, label_list=label_lists[j], args=args) else: model = Classifier(path=None, label_list=label_lists[j], args=args) model.train(intent_train_examples[j]) if args.save_model_path: if not os.path.exists(save_model_path): os.mkdir(save_model_path) model.save(save_model_path) in_domain_preds = model.evaluate(intent_dev_examples[j]) oos_preds = model.evaluate(intent_oos_dev_examples[j]) in_acc = calc_in_acc(dev_examples, in_domain_preds, THRESHOLDS) oos_recall = calc_oos_recall(oos_preds, THRESHOLDS) oos_prec = calc_oos_precision(in_domain_preds, oos_preds, THRESHOLDS) oos_f1 = calc_oos_f1(oos_recall, oos_prec) print_results(THRESHOLDS, in_acc, oos_recall, oos_prec, oos_f1) if f is not None: for i in range(len(in_acc)): f.write('{},{},{},{} '.format(in_acc[i], oos_recall[i], oos_prec[i], oos_f1[i])) f.write('\n') if f is not None: f.close()