def per_sentence_bionlp_fscores_nbest(test_filenames, gold_dir): import tempfile from DataSet import DataSet from cStringIO import StringIO gold_dir = path(gold_dir) sentences = DataSet.from_filenames(*test_filenames) for docid, sentences_in_doc in sentences.group_by_metadata('DOC'): # if docid != '9361029': # continue print 'DOC:', docid our_total_proposed = 0 our_total_matched = 0 for i, sentence in enumerate(sentences_in_doc): for j, parse in enumerate(sentence): print "DOC:", docid, 'Sentence:', i, 'Parse:', j our_score_components = parse.bionlp_fscore_components(sentence) matched, gold, proposed = our_score_components our_total_proposed += proposed our_total_matched += matched conll_version = StringIO() parse.write_conll(conll_version, include_metadata=False, sentence=sentence) conll_version.seek(0) conll_version = conll_version.read() import BioNLPConversionDB converter = BioNLPConversionDB.get_converter() bionlp_events_string = converter.convert(conll_version) if 0: print 'Events ---' print bionlp_events_string print 'Events ---' temp_test_dir = path(tempfile.mkdtemp(prefix=docid + '-')) temp_test_filename = path(temp_test_dir / docid + '.a2.t1') temp_test_file = file(temp_test_filename, 'w') temp_test_file.write(bionlp_events_string) temp_test_file.close() real_score_components = real_evaluation_bionlp_components( temp_test_dir, gold_dir) if our_score_components != real_score_components: real_evaluation_bionlp_components(temp_test_dir, gold_dir, show_output=True) temp_test_dir.rmtree() if our_score_components != real_score_components: print "Ours:", our_score_components print 'Real:', real_score_components print 'Events ---' print bionlp_events_string print 'Events ---' raise 'mismatch'
def per_sentence_bionlp_fscores(test_filename, test_dir, gold_dir): import tempfile from DataSet import DataSet from cStringIO import StringIO test_dir = path(test_dir) gold_dir = path(gold_dir) sentences = DataSet.from_filenames(test_filename) for docid, sentences_in_doc in sentences.group_by_metadata('DOC'): # if docid != '9015187': # if docid != '9081693': # if docid != '9257843': # if docid != '8108127': # if docid != '9115366': # if docid != '9361029': # continue print 'DOC:', docid our_total_proposed = 0 our_total_matched = 0 for sentence in sentences_in_doc: parse = sentence.gold_parse sentence.parses = [parse] our_score_components = parse.bionlp_fscore_components(sentence) matched, gold, proposed = our_score_components our_total_proposed += proposed our_total_matched += matched conll_version = StringIO() parse.write_conll(conll_version, include_metadata=False, sentence=sentence) conll_version.seek(0) conll_version = conll_version.read() import BioNLPConversionDB converter = BioNLPConversionDB.get_converter() bionlp_events_string = converter.convert(conll_version) if 1: print 'Events ---' print bionlp_events_string print 'Events ---' print "Ours:", our_score_components temp_test_dir = path(tempfile.mkdtemp(prefix=docid + '-')) temp_test_filename = path(temp_test_dir/docid + '.a2.t1') temp_test_file = file(temp_test_filename, 'w') temp_test_file.write(bionlp_events_string) temp_test_file.close() real_score_components = real_evaluation_bionlp_components(temp_test_dir, gold_dir) print 'Real:', real_score_components if our_score_components != real_score_components: real_evaluation_bionlp_components(temp_test_dir, gold_dir, show_output=True) temp_test_dir.rmtree() if our_score_components != real_score_components: raise 'mismatch'
def per_sentence_bionlp_fscores_nbest(test_filenames, gold_dir): import tempfile from DataSet import DataSet from cStringIO import StringIO gold_dir = path(gold_dir) sentences = DataSet.from_filenames(*test_filenames) for docid, sentences_in_doc in sentences.group_by_metadata('DOC'): # if docid != '9361029': # continue print 'DOC:', docid our_total_proposed = 0 our_total_matched = 0 for i, sentence in enumerate(sentences_in_doc): for j, parse in enumerate(sentence): print "DOC:", docid, 'Sentence:', i, 'Parse:', j our_score_components = parse.bionlp_fscore_components(sentence) matched, gold, proposed = our_score_components our_total_proposed += proposed our_total_matched += matched conll_version = StringIO() parse.write_conll(conll_version, include_metadata=False, sentence=sentence) conll_version.seek(0) conll_version = conll_version.read() import BioNLPConversionDB converter = BioNLPConversionDB.get_converter() bionlp_events_string = converter.convert(conll_version) if 0: print 'Events ---' print bionlp_events_string print 'Events ---' temp_test_dir = path(tempfile.mkdtemp(prefix=docid + '-')) temp_test_filename = path(temp_test_dir/docid + '.a2.t1') temp_test_file = file(temp_test_filename, 'w') temp_test_file.write(bionlp_events_string) temp_test_file.close() real_score_components = real_evaluation_bionlp_components(temp_test_dir, gold_dir) if our_score_components != real_score_components: real_evaluation_bionlp_components(temp_test_dir, gold_dir, show_output=True) temp_test_dir.rmtree() if our_score_components != real_score_components: print "Ours:", our_score_components print 'Real:', real_score_components print 'Events ---' print bionlp_events_string print 'Events ---' raise 'mismatch'
def per_sentence_bionlp_fscores(test_filename, test_dir, gold_dir): import tempfile from DataSet import DataSet from cStringIO import StringIO test_dir = path(test_dir) gold_dir = path(gold_dir) sentences = DataSet.from_filenames(test_filename) for docid, sentences_in_doc in sentences.group_by_metadata('DOC'): # if docid != '9015187': # if docid != '9081693': # if docid != '9257843': # if docid != '8108127': # if docid != '9115366': # if docid != '9361029': # continue print 'DOC:', docid our_total_proposed = 0 our_total_matched = 0 for sentence in sentences_in_doc: parse = sentence.gold_parse sentence.parses = [parse] our_score_components = parse.bionlp_fscore_components(sentence) matched, gold, proposed = our_score_components our_total_proposed += proposed our_total_matched += matched conll_version = StringIO() parse.write_conll(conll_version, include_metadata=False, sentence=sentence) conll_version.seek(0) conll_version = conll_version.read() import BioNLPConversionDB converter = BioNLPConversionDB.get_converter() bionlp_events_string = converter.convert(conll_version) if 1: print 'Events ---' print bionlp_events_string print 'Events ---' print "Ours:", our_score_components temp_test_dir = path(tempfile.mkdtemp(prefix=docid + '-')) temp_test_filename = path(temp_test_dir / docid + '.a2.t1') temp_test_file = file(temp_test_filename, 'w') temp_test_file.write(bionlp_events_string) temp_test_file.close() real_score_components = real_evaluation_bionlp_components( temp_test_dir, gold_dir) print 'Real:', real_score_components if our_score_components != real_score_components: real_evaluation_bionlp_components(temp_test_dir, gold_dir, show_output=True) temp_test_dir.rmtree() if our_score_components != real_score_components: raise 'mismatch'