def test_sentence_performance(events_service, sentences_service, test_results):
input_dir = Path(os.environ['BIOMEDICUS_TEST_DATA']) / 'sentences'
confusion = metrics.FirstTokenConfusion()
with EventsClient(address=events_service) as client, Pipeline(
RemoteProcessor(processor_id='biomedicus-sentences',
address=sentences_service),
LocalProcessor(metrics.Metrics(confusion,
tested='sentences',
target='Sentence'),
component_id='metrics',
client=client)) as pipeline:
for test_file in input_dir.glob('**/*.json'):
with JsonSerializer.file_to_event(test_file,
client=client) as event:
document = event.documents['plaintext']
results = pipeline.run(document)
print('F1 for event - "{}": {:0.3f} - elapsed: {}'.format(
event.event_id,
results[1].results['first_token_confusion']['f1'],
results[0].timing_info['process_method']))
print('Overall Precision:', confusion.precision)
print('Overall Recall:', confusion.recall)
print('Overall F1:', confusion.f1)
pipeline.print_times()
timing_info = pipeline.processor_timer_stats()[0].timing_info
test_results['Sentences'] = {
'Precision': confusion.precision,
'Recall': confusion.recall,
'F1': confusion.f1,
'Remote Call Duration': str(timing_info['remote_call'].mean),
'Process Method Duration': str(timing_info['process_method'].mean)
}
assert confusion.f1 > 0.85
def test_modification_detector_performance(events_service, modification_detector_service,
test_results):
input_dir = Path(os.environ['BIOMEDICUS_TEST_DATA']) / 'negation' / 'i2b2_2010'
confusion = metrics.FirstTokenConfusion()
metrics_processor = metrics.Metrics(confusion, tested='negated', target='i2b2concepts',
target_filter=is_negated)
with EventsClient(address=events_service) as client, Pipeline(
RemoteProcessor('biomedicus-negation', address=modification_detector_service,
params={'terms_index': 'i2b2concepts'}),
LocalProcessor(metrics_processor, component_id='metrics', client=client)
) as pipeline:
for test_file in input_dir.glob('**/*.pickle'):
with PickleSerializer.file_to_event(test_file, client=client) as event:
document = event.documents['plaintext']
results = pipeline.run(document)
print('F1 for event - "{}": {:0.3f} - elapsed: {}'.format(
event.event_id,
results.component_result('metrics').result_dict['first_token_confusion']['f1'],
results.component_result('biomedicus-negation').timing_info['process_method']
))
print('Overall Precision:', confusion.precision)
print('Overall Recall:', confusion.recall)
print('Overall F1:', confusion.f1)
pipeline.print_times()
timing_info = pipeline.processor_timer_stats('biomedicus-negation').timing_info
test_results['biomedicus-modification'] = {
'Gold Standard': "2010 i2b2-VA",
'Precision': confusion.precision,
'Recall': confusion.recall,
'F1': confusion.f1,
'Per-Document Mean Remote Call Duration': str(timing_info['remote_call'].mean),
'Per-Document Mean Process Method Duration': str(timing_info['process_method'].mean)
}