single_dev_datasets = [] for dataloader in dev_dataset_loader: single_dev_datasets.append([dl.dataset for dl in dataloader]) pickle.dump(single_dev_datasets, open( args.pickle + "/temp_single_dev.p", "wb" )) single_test_datasets = [] for dataloader in test_dataset_loader: single_test_datasets.append([dl.dataset for dl in dataloader]) print("DUMP temp_single_test") pickle.dump(single_test_datasets, open( args.pickle + "/temp_single_test.p", "wb" )) #agent.eval_batch_corpus(dev_dataset_loader, train_args['dev_file'], corpus2crf) agent.eval_batch_corpus(test_dataset_loader, train_args['test_file'], corpus2crf) # global prediction if args.if_pred: idx2tag = {idx: tag for tag, idx in tag2idx.items()} for file_name in args.pred_file: pred_lines = [] with codecs.open(file_name, 'r', 'utf-8') as f: pred_lines = f.readlines() pred_tokens, gold_labels = utils.read_corpus(pred_lines) pred_corpus_tagspace = set(["<start>", "<pad>"]) for sent_labels in gold_labels: pred_corpus_tagspace |= set(sent_labels) # local pred: use crfs binds to each training corpus
args.epoch = args.start_epoch + args.epoch epoch_list = range(args.start_epoch, args.epoch) else: args.epoch += 1 epoch_list = range(1, args.epoch) predictor = Predictor(tag2idx, packer, label_seq=True, batch_size=50) evaluator = Evaluator(predictor, packer, tag2idx, args.eva_matrix, args.pred_method) trainer = Trainer(ner_model, packer, crit_ner, crit_lm, optimizer, evaluator, crf2corpus, args.plateau) trainer.train(crf2train_dataloader, crf2dev_dataloader, dev_dataset_loader, epoch_list, args) trainer.eval_batch_corpus(dev_dataset_loader, args.dev_file, args.corpus2crf) try: print("Load from PICKLE") single_testset = pickle.load( open(args.pickle + "/temp_single_test.p", "rb")) test_dataset_loader = [] for datasets_tuple in single_testset: test_dataset_loader.append([ torch.utils.data.DataLoader(tup, 50, shuffle=False, drop_last=False) for tup in datasets_tuple ]) except: