def test_goals():
p = create_pipeline(cfg, goal_layers=['text'])
assert p.nb_components() == 2
p = create_pipeline(cfg, goal_layers=['deps'])
assert p.nb_components() == 3
p = create_pipeline(cfg, goal_layers=['terms'])
assert p.nb_components() == 5
p = create_pipeline(cfg, goal_layers=['srl'])
assert p.nb_components() == 9
def test_full_pipeline():
p = pipeline.create_pipeline(cfg)
assert p.nb_components() == NB_COMPS
with open(four_words, 'r') as f:
summary = p.execute(f)
completed = [v for v in summary.values() if v == 'completed']
assert len(completed) == NB_COMPS
def test_create_pipeline_without_component_with_children_edges():
p = create_pipeline(cfg,
excepted_components=[
'vua-nominal-event-detection',
'vua-srl-dutch-nominal-events'
])
assert p.nb_components() == NB_COMPS - 2
def test_create_pipeline_with_in_layers_and_goals():
goal_layers = ['opinions', 'entities']
in_layers = ['terms', 'deps', 'constituents']
p = create_pipeline(cfg)
assert p.nb_components() == NB_COMPS
p.keep_from(in_layers)
assert p.nb_components() == NB_COMPS - 2
p.keep_until(goal_layers)
assert p.nb_components() == 6
p = create_pipeline(cfg, in_layers=in_layers, goal_layers=goal_layers)
assert p.nb_components() == 6
assert 'opinion-miner' in p.graph.keys()
assert 'ixa-pipe-nerc' in p.graph.keys()
assert 'ixa-pipe-ned' in p.graph.keys()
assert 'vua-ontotagging' in p.graph.keys()
def test_pipeline_detects_empty_output():
goal_layers = ['deps']
p = pipeline.create_pipeline(fail_empty_cfg, goal_layers=goal_layers)
assert p.nb_components() == 3
with open(single_word, 'r') as f:
summary = p.execute(f)
not_run = [v for v in summary.values() if v == 'not_run']
assert len(not_run) == 1
def test_executable_components_after_failure():
p = pipeline.create_pipeline(fail_cfg)
with open(single_word, 'r') as f:
summary = p.execute(f)
completed = [v for v in summary.values() if v == 'completed']
failed = [v for v in summary.values() if v == 'failed']
assert len(completed) == p.nb_components() - 1
assert len(failed) == 1
def test_pipeline_with_modified_component_args():
goal_layers = ['deps']
subargs = {'vua-alpino': '-t 0.2'}
p = pipeline.create_pipeline(cfg, goal_layers=goal_layers, subargs=subargs)
assert p.nb_components() == 3
with open(single_word, 'r') as f:
summary = p.execute(f)
completed = [v for v in summary.values() if v == 'completed']
assert len(completed) == 3
def test_component_fails_in_partially_generated_layer():
goal_layers = ['terms']
p = pipeline.create_pipeline(cfg,
in_layers=['terms'],
goal_layers=goal_layers)
scheduled = p.topological_sort()
assert len(scheduled) == 3
with open(part1_out, 'r') as f:
summary = p.execute(f)
failed = [v for v in summary.values() if v == 'failed']
assert len(failed) == 1
def test_finish_incomplete_pipeline():
goal_layers = ['opinions', 'timex']
p = pipeline.create_pipeline(cfg,
in_layers=['opinions', 'timex'],
goal_layers=goal_layers)
scheduled = p.topological_sort()
assert len(scheduled) == 2
with open(part1_out, 'r') as f:
summary = p.execute(f)
completed = [v for v in summary.values() if v == 'completed']
assert len(completed) == len(goal_layers)
def test_rescheduling_after_alpino_failure():
p = pipeline.create_pipeline(fail_alpino_cfg)
scheduled = p.topological_sort()
assert len(scheduled) == 4
with open(single_word, 'r') as f:
summary = p.execute(f)
completed = [v for v in summary.values() if v == 'completed']
failed = [v for v in summary.values() if v == 'failed']
not_run = [v for v in summary.values() if v == 'not_run']
assert len(not_run) == 1
assert len(completed) == 2
assert len(failed) == 1
def test_pipeline_creation():
p = create_pipeline(cfg)
assert p.nb_components() == NB_COMPS
scheduled = p.topological_sort()
assert len(scheduled) == NB_COMPS
def test_filtering_with_goal_layers_before_in_layers_has_no_effect():
p = create_pipeline(cfg, in_layers=['srl'])
nb_comps_in_filtered = p.nb_components()
goal_layers = ['terms']
p = create_pipeline(cfg, in_layers=['srl'], goal_layers=goal_layers)
assert p.nb_components() == nb_comps_in_filtered
def test_create_pipeline_without_component_with_dependents():
with pytest.raises(ValueError):
create_pipeline(cfg, excepted_components=['vua-wsd'])
def test_create_pipeline_without_layer_modifying_component():
excepted_components = ['ixa-pipe-ned']
p = create_pipeline(cfg, excepted_components=excepted_components)
assert p.nb_components() == NB_COMPS - 1
input_layers = []
if args.in_layers_str is not None:
input_layers = args.in_layers_str.split(',')
output_layers = []
if args.out_layers_str is not None:
output_layers = args.out_layers_str.split(',')
exclude_components = []
if args.exclude_components_str is not None:
exclude_components = args.exclude_components_str.split(',')
subargs = {}
if args.component_args is not None:
for argv in args.component_args.split(';'):
mod_name, opt_name, opt_val = argv.split(':')
subargs[mod_name] = '{} {}'.format(opt_name, opt_val)
try:
p = pipeline.create_pipeline(cfg_file,
in_layers=input_layers,
goal_layers=output_layers,
excepted_components=exclude_components,
bindir=bin_dir,
subargs=subargs)
input_file = sys.stdin
p.execute(input_file)
except ValueError as e:
logger.error(e)
