def parse_words(file_obj, tag_scheme=None): word_list = parsed_documents_to_words(parse_conll_train(file_obj.read())) if tag_scheme is not None: tag_scheme.encode(word_list, 'gold_label') return word_list
def main(): parser = argparse.ArgumentParser() parser.add_argument('-i', '--test_file', default=L1_TAGGED_TEST_FILE_PATH, help='path to test file with l1 tags') parser.add_argument('-m', '--model_file', default=L2_MODEL_FILE_PATH, help='path to model parameters') parser.add_argument('-l', '--lexicon_file', default=L2_CLASS_LEXICON_FILE_PATH, help='path to label lexicon') parser.add_argument('-o', '--output_file', default=L2_TAGGED_TEST_FILE_PATH, help='path to output file') args = parser.parse_args() logger.info('Loading trained model.') trained_model, class_lexicon = load_model(args.model_file, args.lexicon_file) logger.info('Reading test file') with open(args.test_file, 'rb') as f: test_data = f.read() logger.info('Parsing test data.') parsed_documents_list = parse_conll_tagged(test_data) logger.info('Parse completed (%d documents).' % len(parsed_documents_list)) logger.info('Building word list from test data.') test_words = parsed_documents_to_words(parsed_documents_list) logger.info('Annotate test words.') annotate_data(test_words) logger.info( 'Set tag as prev_level_tag for prev level aggregation features.') for word in test_words: word.prev_level_tag = word.tag logger.info('Annotate words with prev level entities.') annotate_prev_level_entities(test_words) logger.info('Start greedy decoding.') greedy_decoding(test_words, trained_model, class_lexicon, L2_FEATURES) logger.info('Convert tag scheme from BILOU to IOB.') BILOU.decode(test_words, tag_attr='tag') logger.info('Formatting output.') parsed_documents_list = words_to_parsed_documents(test_words) output_data = format_conll_tagged(parsed_documents_list) logger.info('Dumping output.') with open(args.output_file, 'wb') as f: f.write(output_data)
def main(): parser = argparse.ArgumentParser() parser.add_argument('-t', '--train_file', default=L1_TRAIN_FILE_PATH, help='path to training file') parser.add_argument('-b', '--brown_dir', default=BROWN_CLUSTERS_DIR_PATH, help='path to brown cluster directory') args = parser.parse_args() logger.info('Reading train data.') with open(args.train_file, 'rb') as f: train_data = f.read() logger.info('Parsing train data.') parsed_document_list = parse_conll_train(train_data) logger.info('Parse completed (%d documents).' % len(parsed_document_list)) logger.info('Building word list from train data.') training_words = parsed_documents_to_words(parsed_document_list) logger.info('Parsing Brown clusters directory.') all_clusters_dict = parse_brown_clusters_directory(args.brown_dir) logger.info('Calculating prefix len for each path.') cluster_to_path_histogram_by_offset = dict() cluster_to_prefix_histogram_by_offset = dict() cluster_to_prefix_to_len_by_offset = dict() for corpus_name, cluster_dict in all_clusters_dict.iteritems(): path_histogram_by_offset = dict() prefix_histogram_by_offset = dict() path_to_len_by_offset = dict() for offset in (-2, -1, 0, 1, 2): path_to_type_histogram = build_path_histogram( cluster_dict, training_words, offset) path_histogram_by_offset[offset] = path_to_type_histogram prefix_to_type_histogram = build_prefix_histogram( path_to_type_histogram) prefix_histogram_by_offset[offset] = prefix_to_type_histogram path_to_len = get_path_to_prefix_len(path_to_type_histogram, prefix_to_type_histogram) path_to_len_by_offset[offset] = path_to_len cluster_to_path_histogram_by_offset[ corpus_name] = path_histogram_by_offset cluster_to_prefix_histogram_by_offset[ corpus_name] = prefix_histogram_by_offset cluster_to_prefix_to_len_by_offset[corpus_name] = path_to_len_by_offset logger.info('Dumping result to file.') with open(BROWN_PREFIX_TO_LEN_FILE_PATH, 'wb') as f: json.dump(cluster_to_prefix_to_len_by_offset, f, indent=4)
def main(): tagged_file_path = L1_TAGGED_TEST_FILE_PATH if len(sys.argv) == 1 else sys.argv[1] # parse word list from tagged file raw_data = open(tagged_file_path, 'rb').read() word_list = parsed_documents_to_words(parse_conll_tagged(raw_data)) # extract tagged and gold entities (per entity type) tagged_entities_by_type = extract_entities_by_types(word_list, tag_attr='tag') gold_entities_by_type = extract_entities_by_types(word_list, tag_attr='gold_label') # print stats per entity type for group in sorted(gold_entities_by_type.keys()): precision, recall, f1 = get_stats(tagged_entities_by_type[group], gold_entities_by_type[group]) print "%15s" % group, print 'prec: %.2f' % precision, print 'recall: %.2f' % recall, print 'f1: %.2f' % f1, print len(tagged_entities_by_type[group])
def main(): parser = argparse.ArgumentParser() parser.add_argument('-t', '--train_file', default=L1_TRAIN_FILE_PATH, help='path to training file') parser.add_argument('-m', '--model_file', default=L1_MODEL_FILE_PATH, help='path to save trained model parameters') parser.add_argument('-l', '--lexicon_file', default=L1_CLASS_LEXICON_FILE_PATH, help='path to save label lexicon') args = parser.parse_args() logger.info('Reading train data.') with open(args.train_file, 'rb') as f: train_data = f.read() logger.info('Parsing train data.') parsed_document_list = parse_conll_train(train_data) logger.info('Parse completed (%d documents).' % len(parsed_document_list)) logger.info('Building word list from train data.') training_words = parsed_documents_to_words(parsed_document_list) logger.info('Annotate training data.') annotate_data(training_words) logger.info('Convert gold label tag scheme from IOB to BILOU.') BILOU.encode(training_words, tag_attr='gold_label') logger.info('Set gold label as tag (for model tag features).') for word in training_words: word.tag = word.gold_label logger.info('Train model.') trained_model_params, class_lexicon = train(training_words, L1_FEATURES, NUM_TRAIN_ITERATIONS) save_model(trained_model_params, args.model_file, class_lexicon, args.lexicon_file)
def _document_text_to_words(document_text): parsed_document_mock = list() for sentence_text in document_text.splitlines(): parsed_sentence_mock = map(lambda word_text: {'text': word_text}, sentence_text.split()) parsed_document_mock.append(parsed_sentence_mock) return parsed_documents_to_words([parsed_document_mock])