def compare(args):
"""
Compare two [incr tsdb()] profiles.
"""
from delphin.mrs import simplemrs, compare as mrs_compare
template = '{id}\t<{left},{shared},{right}>'
if args['--verbose'] >= 1:
template += '\t{string}'
test_profile = _prepare_input_profile(args['PROFILE'],
filters=args['--filter'],
applicators=args['--apply'])
gold_profile = _prepare_input_profile(args['GOLD'])
i_inputs = dict((row['parse:parse-id'], row['item:i-input'])
for row in test_profile.join('item', 'parse'))
matched_rows = itsdb.match_rows(test_profile.read_table('result'),
gold_profile.read_table('result'),
'parse-id')
for (key, testrows, goldrows) in matched_rows:
(test_unique, shared, gold_unique) = mrs_compare.compare_bags(
[simplemrs.loads_one(row['mrs']) for row in testrows],
[simplemrs.loads_one(row['mrs']) for row in goldrows])
print(
template.format(id=key,
string=i_inputs[key],
left=test_unique,
shared=shared,
right=gold_unique))
def test_match_rows():
assert list(
itsdb.match_rows([{
'i-id': '10',
'i-input': 'a'
}, {
'i-id': '20',
'i-input': 'b'
}], [{
'i-id': '20',
'i-input': 'c'
}, {
'i-id': '30',
'i-input': 'd'
}], 'i-id')) == [('10', [{
'i-id': '10',
'i-input': 'a'
}], []),
('20', [{
'i-id': '20',
'i-input': 'b'
}], [{
'i-id': '20',
'i-input': 'c'
}]), ('30', [], [{
'i-id': '30',
'i-input': 'd'
}])]
def compare(testsuite, gold, select='i-id i-input mrs'):
"""
Compare two [incr tsdb()] profiles.
Args:
testsuite (str, TestSuite): path to the test [incr tsdb()]
testsuite or a :class:`TestSuite` object
gold (str, TestSuite): path to the gold [incr tsdb()]
testsuite or a :class:`TestSuite` object
select: TSQL query to select (id, input, mrs) triples
(default: `i-id i-input mrs`)
Yields:
dict: Comparison results as::
{"id": "item identifier",
"input": "input sentence",
"test": number_of_unique_results_in_test,
"shared": number_of_shared_results,
"gold": number_of_unique_results_in_gold}
"""
from delphin.mrs import simplemrs, compare as mrs_compare
if not isinstance(testsuite, itsdb.TestSuite):
if isinstance(testsuite, itsdb.ItsdbProfile):
testsuite = testsuite.root
testsuite = itsdb.TestSuite(testsuite)
if not isinstance(gold, itsdb.TestSuite):
if isinstance(gold, itsdb.ItsdbProfile):
gold = gold.root
gold = itsdb.TestSuite(gold)
queryobj = tsql.inspect_query('select ' + select)
if len(queryobj['projection']) != 3:
raise ValueError('select does not return 3 fields: ' + select)
input_select = '{} {}'.format(queryobj['projection'][0],
queryobj['projection'][1])
i_inputs = dict(tsql.select(input_select, testsuite))
matched_rows = itsdb.match_rows(tsql.select(select, testsuite),
tsql.select(select, gold), 0)
for (key, testrows, goldrows) in matched_rows:
(test_unique, shared, gold_unique) = mrs_compare.compare_bags(
[simplemrs.loads_one(row[2]) for row in testrows],
[simplemrs.loads_one(row[2]) for row in goldrows])
yield {
'id': key,
'input': i_inputs[key],
'test': test_unique,
'shared': shared,
'gold': gold_unique
}
def compare_mrs(dest_dir, gold_dir, log=None):
debug('Comparing output ({}) to gold ({})'.format(dest_dir, gold_dir), log)
test_profile = itsdb.ItsdbProfile(dest_dir)
gold_profile = itsdb.ItsdbProfile(gold_dir)
matched_rows = itsdb.match_rows(
test_profile.read_table('result'),
gold_profile.read_table('result'),
'parse-id'
)
success = True
for (key, testrows, goldrows) in matched_rows:
(test_unique, shared, gold_unique) = compare_bags(
[simplemrs.loads_one(row['mrs']) for row in testrows],
[simplemrs.loads_one(row['mrs']) for row in goldrows]
)
if test_unique or gold_unique:
success = False
info('{}\t<{},{},{}>'.format(key, test_unique, shared, gold_unique),
log)
debug('Completed comparison. Test {}.'
.format('succeeded' if success else 'failed'),
log)
return success