end = start + l p = predLabels[start:end] c = correcLabels[start:end] pred_valid.append(p) corr_valid.append(c) start = end newSentencesValid = [] for i, s in enumerate(valid_words): sent = [] assert len(s) == len(valid_efs[i]) == len(pred_valid[i]) for j, item in enumerate(s): sent.append([item, valid_efs[i][j], pred_valid[i][j]]) newSentencesValid.append(sent) newSentencesValid_, newLabelsValid, newPredsValid = dp.wordLevelGeneration( newSentencesValid) p_valid, r_valid, f1_valid = dp.compute_precision_recall_f1( newLabelsValid, newPredsValid, args.flag, 1) print("Precision: {}, Recall: {}, F1: {}".format( p_valid, r_valid, f1_valid)) if f1_valid <= trainer.bestResult: time += 1 else: trainer.bestResult = f1_valid time = 0 trainer.save(( "saved_model/addUnlabeledData_{}_{}_lr_{}_prior_{}_beta_{}_gamma_{}_size_{}" ).format(args.dataset, args.flag, trainer.learningRate, trainer.m, trainer.beta, trainer.gamma, args.unlabel_size))
temp = [] temp2 = [] for word, ef, lf in s: temp.append(word) temp2.append(ef) test_words.append(temp) test_efs.append(temp2) newSentencesTest = [] for i, s in enumerate(test_words): sent = [] for j, item in enumerate(s): sent.append([item, test_efs[i][j], pred_test[i][j]]) newSentencesTest.append(sent) newSentencesTest_, newLabelsTest, newPredsTest = dp.wordLevelGeneration( newSentencesTest) p, r, f1 = dp.compute_precision_recall_f1(newLabelsTest, newPredsTest, args.flag, 1) print("Precision: {}, Recall: {}, F1: {}".format(p, r, f1)) if args.output: outputFile = "result/no_feature_pu_" + args.dataset + "_" + args.flag + "_" + str( args.set) + ".txt" with open(outputFile, "w") as fw: for i, sent in enumerate(test_words): preds = pred_test[i] corrs = test_efs[i] for j, w in enumerate(sent): pred = preds[j] corr = corrs[j] fw.write(("{} {} {}\n").format(w, corr, pred))
def main_feature(): parser = argparse.ArgumentParser(description="PU NER") # data parser.add_argument('--lr', type=float, default=1e-4, help='learning rate') parser.add_argument('--beta', type=float, default=0.0, help='beta of pu learning (default 0.0)') parser.add_argument('--gamma', type=float, default=1.0, help='gamma of pu learning (default 1.0)') parser.add_argument('--drop_out', type=float, default=0.5, help='dropout rate') parser.add_argument('--m', type=float, default=0.3, help='class balance rate') parser.add_argument('--flag', default="Disease", help='entity type (PER/LOC/ORG/MISC)') parser.add_argument('--dataset', default="disease", help='name of the dataset') parser.add_argument('--batch_size', type=int, default=100, help='batch size for training and testing') parser.add_argument('--print_time', type=int, default=1, help='epochs for printing result') parser.add_argument('--pert', type=float, default=1.0, help='percentage of data use for training') parser.add_argument('--type', type=str, default='bnpu', help='pu learning type (bnpu/bpu/upu)') # bpu upu parser.add_argument('--num', type=str, default='0', help='file number') args = parser.parse_args() dp = DataPrepare(args.dataset) mutils = FeaturedDetectionModelUtils(dp) trainSet, validSet, testSet, prior = mutils.load_dataset( args.flag, args.dataset, args.pert, args.num) f1_best_train = 0 trainSize = len(trainSet) validSize = len(validSet) testSize = len(testSet) print(("train set size: {}, valid set size: {}, test set size: {}").format( trainSize, validSize, testSize)) charcnn = CharCNN(dp.char2Idx) wordnet = WordNet(dp.wordEmbeddings, dp.word2Idx) casenet = CaseNet(dp.caseEmbeddings, dp.case2Idx) featurenet = FeatureNet() pulstmcnn = PULSTMCNN(dp, charcnn, wordnet, casenet, featurenet, 150, 200, 1, args.drop_out) if torch.cuda.is_available: charcnn.cuda() wordnet.cuda() casenet.cuda() featurenet.cuda() pulstmcnn.cuda() torch.cuda.manual_seed(1013) trainer = Trainer(pulstmcnn, prior, args.beta, args.gamma, args.lr, args.m) time = 0 bar = ProgressBar(maxval=int((len(trainSet) - 1) / args.batch_size)) train_sentences = dp.read_origin_file("data/" + args.dataset + "/train.txt") trainSize = int(len(train_sentences) * args.pert) train_sentences = train_sentences[:trainSize] train_words = [] train_efs = [] for s in train_sentences: temp = [] temp2 = [] for word, ef, lf in s: temp.append(word) temp2.append(ef) train_words.append(temp) train_efs.append(temp2) valid_sentences = dp.read_origin_file("data/" + args.dataset + "/valid.txt") valid_words = [] valid_efs = [] for s in valid_sentences: temp = [] temp2 = [] for word, ef, lf in s: temp.append(word) temp2.append(ef) valid_words.append(temp) valid_efs.append(temp2) test_sentences = dp.read_origin_file("data/" + args.dataset + "/test.txt") test_words = [] test_efs = [] for s in test_sentences: temp = [] temp2 = [] for word, ef, lf in s: temp.append(word) temp2.append(ef) test_words.append(temp) test_efs.append(temp2) for e in range(1, 1000): print("Epoch: {}".format(e)) bar.start() risks = [] prisks = [] nrisks = [] for step, (x_word_batch, x_case_batch, x_char_batch, x_feature_batch, y_batch, flag_batch) in enumerate( mutils.iterateSet(trainSet, batchSize=args.batch_size, mode="TRAIN")): bar.update(step) batch = [ x_word_batch, x_case_batch, x_char_batch, x_feature_batch, y_batch, flag_batch ] acc, risk, prisk, nrisk = trainer.train_mini_batch(batch, args) risks.append(risk) prisks.append(prisk) nrisks.append(nrisk) meanRisk = np.mean(np.array(risks)) meanRisk2 = np.mean(np.array(prisks)) meanRisk3 = np.mean(np.array(nrisks)) print("risk: {}, prisk: {}, nrisk: {}".format(meanRisk, meanRisk2, meanRisk3)) if e % 5 == 0: trainer.decay_learning_rate(e, args.lr) if e % args.print_time == 0: # train set pred_train = [] corr_train = [] for step, (x_word_train_batch, x_case_train_batch, x_char_train_batch, x_feature_train_batch, y_train_batch) in enumerate( mutils.iterateSet(trainSet, batchSize=100, mode="TEST", shuffle=False)): trainBatch = [ x_word_train_batch, x_case_train_batch, x_char_train_batch, x_feature_train_batch ] correcLabels = [] for x in y_train_batch: for xi in x: correcLabels.append(xi) lengths = [len(x) for x in x_word_train_batch] predLabels, _ = trainer.test(trainBatch, lengths) correcLabels = np.array(correcLabels) # print(predLabels) # print(correcLabels) assert len(predLabels) == len(correcLabels) start = 0 for i, l in enumerate(lengths): end = start + l p = predLabels[start:end] c = correcLabels[start:end] pred_train.append(p) corr_train.append(c) start = end # prec, rec, f1 = dp.evaluate_ner_tagging("data/" + args.dataset + "/train.txt",pred_train) newSentences = [] for i, s in enumerate(train_words): sent = [] assert len(s) == len(train_efs[i]) == len(pred_train[i]) for j, item in enumerate(s): sent.append([item, train_efs[i][j], pred_train[i][j]]) newSentences.append(sent) newSentences_, newLabels, newPreds = dp.wordLevelGeneration( newSentences) print("newlabels : ", len(newLabels)) print("newpreds : ", len(newPreds)) print(newLabels[0]) p_train, r_train, f1_train = dp.compute_precision_recall_f1( newLabels, newPreds, args.flag, 1) print("Precision: {}, Recall: {}, F1: {}".format( p_train, r_train, f1_train)) print(trainer.bestResult) if f1_train > f1_best_train: f1_best_train = f1_train p_best_train = p_train r_best_train = f1_train p_train_partial, r_train_partial, f1_train_partial, partial_entity_train, partial_ratio_train = dp.compute_precision_recall_f1_partial( newLabels, newPreds, args.flag, 1, True) # valid set pred_valid = [] corr_valid = [] for step, (x_word_test_batch, x_case_test_batch, x_char_test_batch, x_feature_test_batch, y_test_batch) in enumerate( mutils.iterateSet(validSet, batchSize=100, mode="TEST", shuffle=False)): validBatch = [ x_word_test_batch, x_case_test_batch, x_char_test_batch, x_feature_test_batch ] correcLabels = [] for x in y_test_batch: for xi in x: correcLabels.append(xi) lengths = [len(x) for x in x_word_test_batch] predLabels, _ = trainer.test(validBatch, lengths) correcLabels = np.array(correcLabels) assert len(predLabels) == len(correcLabels) start = 0 for i, l in enumerate(lengths): end = start + l p = predLabels[start:end] c = correcLabels[start:end] pred_valid.append(p) corr_valid.append(c) start = end newSentencesValid = [] for i, s in enumerate(valid_words): sent = [] assert len(s) == len(valid_efs[i]) == len(pred_valid[i]) for j, item in enumerate(s): sent.append([item, valid_efs[i][j], pred_valid[i][j]]) newSentencesValid.append(sent) newSentencesValid_, newLabelsValid, newPredsValid = dp.wordLevelGeneration( newSentencesValid) p_valid, r_valid, f1_valid = dp.compute_precision_recall_f1( newLabelsValid, newPredsValid, args.flag, 1) print("Precision: {}, Recall: {}, F1: {}".format( p_valid, r_valid, f1_valid)) if f1_valid <= trainer.bestResult: time += 1 else: trainer.bestResult = f1_valid time = 0 trainer.bestResult = f1_valid best_precision_valid = p_valid best_rappel_valid = r_valid p_valid_partial, r_valid_partial, f1_valid_partial, partial_entity_valid, partial_ratio_valid = dp.compute_precision_recall_f1_partial( newLabelsValid, newPredsValid, args.flag, 1, True) trainer.save(( "saved_model/{}_{}_{}_{}_lr_{}_prior_{}_beta_{}_gamma_{}_percent_{}" ).format(args.num, args.type, args.dataset, args.flag, trainer.learningRate, trainer.m, trainer.beta, trainer.gamma, args.pert)) if time > 5: f_result_valid = open('result_val.txt', 'a') print(("BEST RESULT ON VALIDATE DATA:{}").format( trainer.bestResult), file=f_result_valid) break pulstmcnn.load_state_dict( torch.load( "saved_model/{}_{}_{}_{}_lr_{}_prior_{}_beta_{}_gamma_{}_percent_{}" .format(args.num, args.type, args.dataset, args.flag, trainer.learningRate, trainer.m, trainer.beta, trainer.gamma, args.pert))) pred_test = [] corr_test = [] for step, (x_word_test_batch, x_case_test_batch, x_char_test_batch, x_feature_test_batch, y_test_batch) in enumerate( mutils.iterateSet(testSet, batchSize=100, mode="TEST", shuffle=False)): testBatch = [ x_word_test_batch, x_case_test_batch, x_char_test_batch, x_feature_test_batch ] correcLabels = [] for x in y_test_batch: for xi in x: correcLabels.append(xi) lengths = [len(x) for x in x_word_test_batch] predLabels, _ = trainer.test(testBatch, lengths) correcLabels = np.array(correcLabels) assert len(predLabels) == len(correcLabels) start = 0 for i, l in enumerate(lengths): end = start + l p = predLabels[start:end] c = correcLabels[start:end] pred_test.append(p) corr_test.append(c) start = end newSentencesTest = [] for i, s in enumerate(test_words): sent = [] assert len(s) == len(test_efs[i]) == len(pred_test[i]) for j, item in enumerate(s): sent.append([item, test_efs[i][j], pred_test[i][j]]) newSentencesTest.append(sent) newSentencesValid_, newLabelsValid, newPredsValid = dp.wordLevelGeneration( newSentencesTest) p_valid, r_valid, f1_valid = dp.compute_precision_recall_f1( newLabelsValid, newPredsValid, args.flag, 1) p_test_partial, r_test_partial, f1_test_partial, partial_entity_test, partial_ratio_test = dp.compute_precision_recall_f1_partial( newLabelsValid, newPredsValid, args.flag, 1, True) dutils = utils.dict_utils.DictUtils() inDico = get_dico(dp, dutils, "data/disease/test.txt", "fulldico.txt", "Disease", "Train", "disease") p_test_gene, r_test_gene, f1_test_gene, partial_test_gene, rpartial_test_gene, f1partial_test_gene, nb_notindico = dp.compute_metrics_generalize( newLabelsValid, newPredsValid, args.flag, 1, inDico) result_file = "results/" + args.num + "_final_results.txt" with open(result_file, "a+") as fw: print(args.lr, args.beta, args.gamma, args.drop_out, args.m, file=fw) print(p_best_train, r_best_train, f1_best_train, file=fw) print(p_train_partial, r_train_partial, f1_train_partial, partial_entity_train, partial_ratio_train, file=fw) print(best_precision_valid, best_rappel_valid, trainer.bestResult, file=fw) print(p_valid_partial, r_valid_partial, f1_valid_partial, partial_entity_valid, partial_ratio_valid, file=fw) print(p_valid, r_valid, f1_valid, file=fw) print(p_test_partial, r_test_partial, f1_test_partial, partial_entity_test, partial_ratio_test, file=fw) print(p_test_gene, r_test_gene, f1_test_gene, file=fw) print(partial_test_gene, rpartial_test_gene, f1partial_test_gene, file=fw) print(nb_notindico, file=fw)
for word, ef, lf in s: temp.append(word) temp2.append(ef) test_words.append(temp) test_efs.append(temp2) lens.append(len(s)) per_result = get_output(filenames[0]) loc_result = get_output(filenames[1]) org_result = get_output(filenames[2]) # misc_result = get_output(filenames[3]) # get result final_res = get_final_result(per_result, loc_result, org_result) newSentencesTest = [] for i, s in enumerate(test_words): sent = [] for j, item in enumerate(s): sent.append([item, test_efs[i][j], final_res[i][j]]) newSentencesTest.append(sent) newSentences, newLabels, newPreds = dp.wordLevelGeneration( newSentencesTest) p, r, f1 = prf1(newLabels, newPreds) print(p, r, f1) # get conflict c, w, p = get_conflict(per_result, loc_result, org_result) print(c, w, p)