def predict_and_output(cls, parser, options, sentences, output_file):
total_result = ScorerResult.zero()
with open(output_file + ".txt", "w") as f, \
open(output_file + ".graph", "w") as f_graph, \
open(output_file + ".gold_graph", "w") as f_gold_graph:
for sent_id, graph in parser.predict(sentences):
with gzip.open(options.deepbank_dir + "/" + sent_id + ".gz",
"rb") as f_gz:
contents = f_gz.read().decode("utf-8")
fields = contents.strip().split("\n\n")
if options.graph_type == "eds":
eds_literal = fields[-2]
eds_literal = re.sub(r"\{.*\}", "", eds_literal)
e = eds.loads_one(eds_literal)
gold_graph = EDSScorer.from_eds(e, sent_id)
f_gold_graph.write(eds_for_smatch(sent_id, e))
else:
assert options.graph_type == "dmrs"
m = simplemrs.loads_one(fields[-3])
gold_graph = EDSScorer.from_mrs(m)
f_gold_graph.write(mrs_for_smatch(sent_id, m))
result = EDSScorer.from_hypergraph(graph).compare_with(
gold_graph,
True, log_func=lambda x: print(x, file=f))
f.write(str(result))
f_graph.write(output_hg(sent_id, graph))
total_result += result
print("Total:")
print(total_result)
f.write(str(total_result))
current_path = os.path.dirname(__file__)
os.system('{}/../utils/smatch_1 {} {} > {}.smatch'.format(
current_path, output_file + ".graph", output_file + ".gold_graph", output_file))
os.system('cat {}.smatch'.format(output_file))
def eds(self):
"""
Deserialize and return an Eds object for native- or
JSON-formatted EDS data; otherwise return the original string.
"""
_eds = self.get('eds')
if _eds is not None:
if isinstance(_eds, dict):
_eds = eds.Eds.from_dict(_eds)
elif isinstance(_eds, stringtypes):
_eds = eds.loads_one(_eds)
return _eds
def eds(self):
"""
Deserialize and return an Eds object for native- or
JSON-formatted EDS data; otherwise return the original string.
"""
_eds = self.get('eds')
if _eds is not None:
if isinstance(_eds, dict):
_eds = eds.Eds.from_dict(_eds)
elif isinstance(_eds, stringtypes):
_eds = eds.loads_one(_eds)
return _eds
def test_deserialize():
e = eds.loads_one('{}')
assert e.top is None
assert len(e.nodes()) == 0
e = eds.loads_one('{:}')
assert e.top is None
assert len(e.nodes()) == 0
e = eds.loads_one('{e2: e2:_rain_v_1<3:9>[]}')
assert e.top == 'e2'
assert len(e.nodes()) == 1
assert e.nodes()[0].pred == '_rain_v_1_rel'
e = eds.loads_one('{: e2:_rain_v_1<3:9>[]}')
assert e.top is None
assert len(e.nodes()) == 1
assert e.nodes()[0].pred == '_rain_v_1_rel'
e = eds.loads_one('{e2:\n'
' e2:_rain_v_1<3:9>{e SF prop, TENSE pres}[]\n'
'}')
assert e.top == 'e2'
assert len(e.nodes()) == 1
assert e.nodes()[0].properties == {'SF': 'prop', 'TENSE': 'pres'}
e = eds.loads_one('{e2: (fragmented)\n'
'|e5:_nearly_x_deg<0:6>[]\n'
' _1:_every_q<7:12>[BV x3]\n'
' x3:_dog_n_1<13:16>[]\n'
' e2:_bark_v_1<17:24>[ARG1 x3]\n'
'}')
assert e.top == 'e2'
assert len(e.nodes()) == 4
def test_deserialize():
e = eds.loads_one('{}')
assert e.top is None
assert len(e.nodes()) == 0
e = eds.loads_one('{:}')
assert e.top is None
assert len(e.nodes()) == 0
e = eds.loads_one('{e2: e2:_rain_v_1<3:9>[]}')
assert e.top == 'e2'
assert len(e.nodes()) == 1
assert e.nodes()[0].pred == '_rain_v_1_rel'
e = eds.loads_one('{: e2:_rain_v_1<3:9>[]}')
assert e.top is None
assert len(e.nodes()) == 1
assert e.nodes()[0].pred == '_rain_v_1_rel'
e = eds.loads_one(
'{e2:\n'
' e2:_rain_v_1<3:9>{e SF prop, TENSE pres}[]\n'
'}'
)
assert e.top == 'e2'
assert len(e.nodes()) == 1
assert e.nodes()[0].properties == {'SF': 'prop', 'TENSE': 'pres'}
e = eds.loads_one(
'{e2: (fragmented)\n'
'|e5:_nearly_x_deg<0:6>[]\n'
' _1:_every_q<7:12>[BV x3]\n'
' x3:_dog_n_1<13:16>[]\n'
' e2:_bark_v_1<17:24>[ARG1 x3]\n'
'}'
)
assert e.top == 'e2'
assert len(e.nodes()) == 4
def test_deserialize():
e = eds.loads_one('{}')
assert e.top is None
assert len(e.nodes()) == 0
e = eds.loads_one('{:}')
assert e.top is None
assert len(e.nodes()) == 0
e = eds.loads_one('{e2: e2:_rain_v_1<3:9>[]}')
assert e.top == 'e2'
assert len(e.nodes()) == 1
assert e.nodes()[0].pred == '_rain_v_1_rel'
e = eds.loads_one('{: e2:_rain_v_1<3:9>[]}')
assert e.top is None
assert len(e.nodes()) == 1
assert e.nodes()[0].pred == '_rain_v_1_rel'
e = eds.loads_one(
'{e2:\n'
' e2:_rain_v_1<3:9>{e SF prop, TENSE pres}[]\n'
'}'
)
assert e.top == 'e2'
assert len(e.nodes()) == 1
assert e.nodes()[0].properties == {'SF': 'prop', 'TENSE': 'pres'}
e = eds.loads_one(
'{e2: (fragmented)\n'
'|e5:_nearly_x_deg<0:6>[]\n'
' _1:_every_q<7:12>[BV x3]\n'
' x3:_dog_n_1<13:16>[]\n'
' e2:_bark_v_1<17:24>[ARG1 x3]\n'
'}'
)
assert e.top == 'e2'
assert len(e.nodes()) == 4
# GitHub issue #203
# _thing_n_of-about was tripping up the parser due to the hyphen,
# and the empty property list of _business_n_1 does not have a space
# before } (without the space is better, I think)
e = eds.loads_one(
'{e3:\n'
' _1:udef_q<0:35>[BV x6]\n'
' e9:_successful_a_1<0:10>{e SF prop, TENSE untensed, MOOD indicative, PROG -, PERF -}[ARG1 x6]\n'
' e10:_american_a_1<11:19>{e SF prop, TENSE untensed, MOOD indicative, PROG -, PERF -}[ARG1 x6]\n'
' e12:compound<20:35>{e SF prop, TENSE untensed, MOOD indicative, PROG -, PERF -}[ARG1 x6, ARG2 x11]\n'
' _2:udef_q<20:28>[BV x11]\n'
' x11:_business_n_1<20:28>{x}[]\n'
' x6:_owner_n_of<29:35>{x PERS 3, NUM pl, IND +}[]\n'
' e3:_do_v_1<36:38>{e SF prop, TENSE pres, MOOD indicative, PROG -, PERF -}[ARG1 x6, ARG2 x18]\n'
' _3:_the_q<39:42>[BV x18]\n'
' e23:_same_a_as<43:47>{e SF prop, TENSE untensed, MOOD indicative, PROG -, PERF -}[ARG1 x18]\n'
' e25:comp_equal<43:47>{e SF prop, TENSE untensed, MOOD indicative, PROG -, PERF -}[ARG1 e23]\n'
' x18:_thing_n_of-about<48:54>{x PERS 3, NUM sg, IND +}[]\n'
'}'
)
assert e.top == 'e3'
assert len(e.nodes()) == 12
assert e.nodes()[5].properties == {}
assert e.nodes()[11].pred == '_thing_n_of-about'
def mapper(options):
main_dir, bank, strip_tree, is_train, graph_type, detect_func_name = options
detect_func = {
"small": HRGDerivation.detect_small,
"large": HRGDerivation.detect_large,
"lexicalized": HRGDerivation.detect_lexicalized
}[detect_func_name]
result = []
with open(main_dir + bank, encoding="utf-8") as f:
if bank.startswith("."):
return
while True:
sent_id = f.readline().strip()
if not sent_id:
break
assert sent_id.startswith("#")
sent_id = sent_id[1:]
tree_literal = f.readline().strip()
try:
with gzip.open(
deepbank_export_path + bank + "/" + sent_id + ".gz",
"rb") as f_gz:
contents = f_gz.read().decode("utf-8")
cfg = ConstTree.from_java_code_deepbank_1_1(
tree_literal, contents)
# strip labels
if strip_tree == STRIP_ALL_LABELS or strip_tree == STRIP_INTERNAL_LABELS:
if strip_tree == STRIP_ALL_LABELS:
strip_label(cfg)
elif strip_tree == STRIP_INTERNAL_LABELS:
strip_label_internal(cfg)
strip_unary(cfg)
elif strip_tree == STRIP_TO_UNLABEL or strip_tree == FUZZY_TREE:
strip_to_unlabel(cfg)
cfg = cfg.condensed_unary_chain()
cfg.populate_spans_internal()
fix_punct_hyphen(cfg)
fields = contents.strip().split("\n\n")
if graph_type == "eds":
eds_literal = fields[-2]
eds_literal = re.sub("\{.*\}", "", eds_literal)
e = eds.loads_one(eds_literal)
hg = HyperGraph.from_eds(e)
elif graph_type == "dmrs":
mrs_literal = fields[-3]
mrs_obj = simplemrs.loads_one(mrs_literal)
hg = HyperGraph.from_mrs(mrs_obj)
else:
raise Exception("Invalid graph type!")
names, args = extract_features(hg, cfg)
if strip_tree == 3:
cfg = fuzzy_cfg(cfg, names)
derivations = CFGRule.extract(
hg,
cfg,
# draw=True,
sent_id=sent_id,
detect_func=detect_func)
sent_id_info = "# ID: " + sent_id + "\n"
span_info = "# DelphinSpans: " + repr(
[i.span for i in cfg.generate_words()]) + "\n"
args_info = "# Args: " + repr(list(args)) + "\n"
names_info = "# Names: " + repr(list(names)) + "\n"
header = cfg.get_words()
original_cfg = cfg.to_string(with_comma=False).replace(
"+++", "+!+")
rules = list(cfg.generate_rules())
assert len(derivations) == len(rules)
for syn_rule, cfg_rule in zip(derivations, rules):
assert cfg_rule.tag == syn_rule.lhs
new_name = "{}#{}".format(cfg_rule.tag,
len(syn_rule.hrg.lhs.nodes) \
if syn_rule.hrg is not None else 0)
cfg_rule.tag = new_name
additional_cfg = cfg.to_string(with_comma=False).replace(
"+++", "+!+")
if any(rule for rule in cfg.generate_rules()
if len(rule.child) > 2):
if is_train:
print("{} Not binary tree!".format(sent_id))
else:
raise Exception("Not binary tree!")
result.append((sent_id, derivations, header,
header + original_cfg, header + additional_cfg))
except Exception as e:
print(sent_id)
print(e.__class__.__name__)
traceback.print_exc()
return bank, result
def relate_eds_coref(coref_discourse, eds_discourse, db_discourse,
index_discourse, syntax_discourse):
def wash_name(name):
# 将"-"两端连起来作为一个词
name_list = name.split(' ')
new_list = []
i = 0
while i < len(name_list):
word = name_list[i]
word = word.replace("'", "’")
word = word.replace('``', '“')
word = word.replace('’’', '”')
word = word.replace('-LCB-', '{')
word = word.replace('-RCB-', '}')
if word == '-':
new_list[-1] = new_list[-1] + '-' + name_list[i + 1]
i += 1
elif word == '--':
new_list.append('–')
elif word == '-LRB-':
new_list.append('(')
elif word == '-RRB-':
new_list.append(')')
else:
new_list.append(word)
i += 1
return " ".join(new_list)
def match_name_sent(initial_name, sent, index_sent, span_index=None):
sent_str = " ".join(sent)
name = wash_name(initial_name)
if not name:
print("Initial name for void:", initial_name)
#print("Washed: ", name)
name_list = name.split(' ')
accumulated_pos = 0
result = []
for i in range(len(sent) - len(name_list) + 1):
flag = 1
for j in range(i, i + len(name_list)):
if sent[j] != name_list[j - i]:
flag = 0
if flag:
start = int(index_sent[i].split(':')[0])
end = int(index_sent[i + len(name_list) - 1].split(':')[-1])
#return start, end
result.append([start, end, i])
# 选择最近的span
if len(result):
span_start = int(span_index.split('-')[0])
distance = 10000
for k, span in enumerate(result):
if abs(span[2] - span_start) < distance:
distance = abs(span_start - span[2])
record = k
return result[record][:2]
# e.g. Packwood & Packwood-Roth
if len(name_list) == 1:
for i in range(len(sent)):
if name_list[0] in sent[i]:
start = int(index_sent[i].split(':')[0])
end = int(index_sent[i].split(':')[-1])
return start, end
if 'U.S .' in sent_str and 'U.S.' in name:
name = name.replace('U.S.', 'U.S .')
name_list = name.split(' ')
print(name_list)
print(name in sent_str)
for i in range(len(sent) - len(name_list) + 1):
flag = 1
for j in range(i, i + len(name_list)):
print(sent[j])
print(name_list[j - i])
print('')
if sent[j] != name_list[j - i]:
flag = 0
if flag:
start = int(index_sent[i].split(':')[0])
end = int(index_sent[i + len(name_list) -
1].split(':')[-1])
return start, end
print("Washed:", name)
print(sent_str)
print("")
return -1, -1
def get_related_node(start, end, nodes, sent=None):
res = []
for item in nodes.items():
item_start = item[1].lnk.data[0]
item_end = item[1].lnk.data[1]
if start - 2 <= item_start and end + 2 >= item_end and (
(item_start <= end and item_end >= start) or
(item_end >= start and item_start <= end)):
res.append(item[0])
#elif ((start <= item_start <= end) or (start <= item_end <= end)) and not item[0].startswith('e'):
elif ((start <= item_start <= end) or
(start <= item_end <= end)) and item[0].startswith('e'):
res.append(item[0])
return res
basic_information = []
for i in range(len(db_discourse)):
basic_information.append({
'sentence': " ".join(db_discourse[i]),
'eds': eds_discourse[i],
'syntax': syntax_discourse[i]
})
global_dict['discourse'] = basic_information
ambiguous = 0
count_all = 0
global not_matched_chunk
coref_information_discourse = []
for value in coref_discourse.values():
coref_information = []
for item in value:
name = item.split('|')[0]
sent_index = int(item.split('|')[1])
span_index = item.split('|')[2]
sent = db_discourse[sent_index]
if not sent:
continue
start, end = match_name_sent(name, sent,
index_discourse[sent_index],
span_index)
if start == -1 and end == -1:
not_matched_chunk += 1
eds_literal = " ".join(eds_discourse[sent_index])
eds_literal = re.sub("\{.*\}", "", eds_literal)
try:
eds = loads_one(eds_literal)
except:
print(eds_literal)
raise AssertionError('Error')
related_nodes = get_related_node(start, end, eds._nodes,
" ".join(sent))
subgraph = []
for node in related_nodes:
for line in eds_discourse[sent_index]:
if line.startswith(' ' + node):
subgraph.append(line)
break
try:
index_to_node = parse_eds(subgraph)
context = parse_eds(eds_discourse[sent_index][1:-1])
except:
for line in eds_discourse[sent_index][1:-1]:
print(line, end='')
traceback.print_exc()
exit(0)
try:
"""
if len([node for node in index_to_node.values() if node.index.startswith('i') and node.name == 'compound']):
print(name, '***', sent_index, ' ', " ".join(db_discourse[sent_index]), '\nSubgraph:')
for line in subgraph:
print(line, end='')
input('Check This.')
"""
cores, reference_count = match_pattern(index_to_node, context)
except:
print("Error occurred when extracting cores!")
print(name, '***', sent_index, ' ',
" ".join(db_discourse[sent_index]), '\nSubgraph:')
for line in subgraph:
print(line, end='')
traceback.print_exc()
exit(0)
count_all += 1
if len(cores) > 1 or len(cores) == 0:
check = [1 for cnt in reference_count if cnt > 0]
if len(check) == 1:
idx = [int(cnt > 0) * 1
for cnt in reference_count].index(1)
coref_information.append(item + '|' + cores[idx].index)
continue
if len(cores) > 1:
ambiguous += 1
else:
not_matched_chunk += 1
if debug:
print(start, end)
if db_discourse:
print(name, '***', sent_index, ' ',
" ".join(db_discourse[sent_index]),
'\nSubgraph:')
else:
print(name, '***', sent_index, '\nSubgraph:')
for line in subgraph:
print(line, end='')
print('*******')
for line in eds_discourse[sent_index][1:-1]:
print(line, end='')
for j, node in enumerate(cores):
print(node.index, reference_count[j])
#for line in eds_discourse[sent_index]:
# print(line, end='')
#print(eds_discourse[sent_index])
if str(input("Error occured. Check this. Continue? y/n\n")
) != 'y':
exit(0)
print('=============')
else:
coref_information.append(item + '|' + cores[0].index)
#if len(coref_information) > 1:
# for item in coref_information:
#file.write(item + '\n')
#file.write('\n')
coref_information_discourse.append(coref_information)
global_dict['coreference'] = coref_information_discourse
json.dump(global_dict, file)
return ambiguous, count_all