def assemble(self):
"""Run assembly on the prepared statements.
This function loads all the prepared statements associated with the
corpus and then runs assembly on them.
"""
all_stmts = []
for record in self.dart_records:
stmts = self.sc.db.get_statements_for_document(
document_id=record['document_id'],
reader=record['reader'],
reader_version=record['reader_version'])
all_stmts += stmts
ia = IncrementalAssembler(all_stmts)
self.assembled_stmts = ia.get_statements()
self.metadata['num_statements'] = len(self.assembled_stmts)
def test_post_processing_all_stmts():
stmts = copy.deepcopy([s1, s2])
ia = IncrementalAssembler(stmts)
stmts_out = ia.get_statements()
# Check that we normalized concept names
assert stmts_out[0].subj.concept.name == 'agriculture'
# Check that we added flattened groundings
flat_grounding = [{'grounding': 'wm/concept/agriculture',
'name': 'agriculture', 'score': 1.0}]
assert stmts_out[0].subj.concept.db_refs['WM_FLAT'] == \
flat_grounding, flat_grounding
# Check that we added annotations
assert 'agents' in stmts_out[0].evidence[0].annotations
assert stmts_out[0].evidence[0].annotations['agents'] == {
'raw_text': ['some_text1', 'some_text2']
}, stmts_out[0].evidence[0].annotations['agents']
def assemble(self):
"""Run assembly on the prepared statements.
This function loads all the prepared statements associated with the
corpus and then runs assembly on them.
"""
all_stmts = []
logger.info('Loading statements from DB for %d records' %
len(self.dart_records))
for record in tqdm.tqdm(self.dart_records):
stmts = self.sc.db.get_statements_for_record(record['storage_key'])
all_stmts += stmts
logger.info('Instantiating incremental assembler with %d statements' %
len(all_stmts))
ia = IncrementalAssembler(all_stmts)
logger.info('Getting assembled statements')
self.assembled_stmts = ia.get_statements()
logger.info('Got %d assembled statements' % len(self.assembled_stmts))
self.metadata['num_statements'] = len(self.assembled_stmts)