def compare(args): """ Compare two [incr tsdb()] profiles. """ from delphin.mrs import simplemrs, compare as mrs_compare template = '{id}\t<{left},{shared},{right}>' if args['--verbose'] >= 1: template += '\t{string}' test_profile = _prepare_input_profile(args['PROFILE'], filters=args['--filter'], applicators=args['--apply']) gold_profile = _prepare_input_profile(args['GOLD']) i_inputs = dict((row['parse:parse-id'], row['item:i-input']) for row in test_profile.join('item', 'parse')) matched_rows = itsdb.match_rows(test_profile.read_table('result'), gold_profile.read_table('result'), 'parse-id') for (key, testrows, goldrows) in matched_rows: (test_unique, shared, gold_unique) = mrs_compare.compare_bags( [simplemrs.loads_one(row['mrs']) for row in testrows], [simplemrs.loads_one(row['mrs']) for row in goldrows]) print( template.format(id=key, string=i_inputs[key], left=test_unique, shared=shared, right=gold_unique))
def test_match_rows(): assert list( itsdb.match_rows([{ 'i-id': '10', 'i-input': 'a' }, { 'i-id': '20', 'i-input': 'b' }], [{ 'i-id': '20', 'i-input': 'c' }, { 'i-id': '30', 'i-input': 'd' }], 'i-id')) == [('10', [{ 'i-id': '10', 'i-input': 'a' }], []), ('20', [{ 'i-id': '20', 'i-input': 'b' }], [{ 'i-id': '20', 'i-input': 'c' }]), ('30', [], [{ 'i-id': '30', 'i-input': 'd' }])]
def compare(testsuite, gold, select='i-id i-input mrs'): """ Compare two [incr tsdb()] profiles. Args: testsuite (str, TestSuite): path to the test [incr tsdb()] testsuite or a :class:`TestSuite` object gold (str, TestSuite): path to the gold [incr tsdb()] testsuite or a :class:`TestSuite` object select: TSQL query to select (id, input, mrs) triples (default: `i-id i-input mrs`) Yields: dict: Comparison results as:: {"id": "item identifier", "input": "input sentence", "test": number_of_unique_results_in_test, "shared": number_of_shared_results, "gold": number_of_unique_results_in_gold} """ from delphin.mrs import simplemrs, compare as mrs_compare if not isinstance(testsuite, itsdb.TestSuite): if isinstance(testsuite, itsdb.ItsdbProfile): testsuite = testsuite.root testsuite = itsdb.TestSuite(testsuite) if not isinstance(gold, itsdb.TestSuite): if isinstance(gold, itsdb.ItsdbProfile): gold = gold.root gold = itsdb.TestSuite(gold) queryobj = tsql.inspect_query('select ' + select) if len(queryobj['projection']) != 3: raise ValueError('select does not return 3 fields: ' + select) input_select = '{} {}'.format(queryobj['projection'][0], queryobj['projection'][1]) i_inputs = dict(tsql.select(input_select, testsuite)) matched_rows = itsdb.match_rows(tsql.select(select, testsuite), tsql.select(select, gold), 0) for (key, testrows, goldrows) in matched_rows: (test_unique, shared, gold_unique) = mrs_compare.compare_bags( [simplemrs.loads_one(row[2]) for row in testrows], [simplemrs.loads_one(row[2]) for row in goldrows]) yield { 'id': key, 'input': i_inputs[key], 'test': test_unique, 'shared': shared, 'gold': gold_unique }
def compare_mrs(dest_dir, gold_dir, log=None): debug('Comparing output ({}) to gold ({})'.format(dest_dir, gold_dir), log) test_profile = itsdb.ItsdbProfile(dest_dir) gold_profile = itsdb.ItsdbProfile(gold_dir) matched_rows = itsdb.match_rows( test_profile.read_table('result'), gold_profile.read_table('result'), 'parse-id' ) success = True for (key, testrows, goldrows) in matched_rows: (test_unique, shared, gold_unique) = compare_bags( [simplemrs.loads_one(row['mrs']) for row in testrows], [simplemrs.loads_one(row['mrs']) for row in goldrows] ) if test_unique or gold_unique: success = False info('{}\t<{},{},{}>'.format(key, test_unique, shared, gold_unique), log) debug('Completed comparison. Test {}.' .format('succeeded' if success else 'failed'), log) return success