def test_goals(): p = create_pipeline(cfg, goal_layers=['text']) assert p.nb_components() == 2 p = create_pipeline(cfg, goal_layers=['deps']) assert p.nb_components() == 3 p = create_pipeline(cfg, goal_layers=['terms']) assert p.nb_components() == 5 p = create_pipeline(cfg, goal_layers=['srl']) assert p.nb_components() == 9
def test_full_pipeline(): p = pipeline.create_pipeline(cfg) assert p.nb_components() == NB_COMPS with open(four_words, 'r') as f: summary = p.execute(f) completed = [v for v in summary.values() if v == 'completed'] assert len(completed) == NB_COMPS
def test_create_pipeline_without_component_with_children_edges(): p = create_pipeline(cfg, excepted_components=[ 'vua-nominal-event-detection', 'vua-srl-dutch-nominal-events' ]) assert p.nb_components() == NB_COMPS - 2
def test_create_pipeline_with_in_layers_and_goals(): goal_layers = ['opinions', 'entities'] in_layers = ['terms', 'deps', 'constituents'] p = create_pipeline(cfg) assert p.nb_components() == NB_COMPS p.keep_from(in_layers) assert p.nb_components() == NB_COMPS - 2 p.keep_until(goal_layers) assert p.nb_components() == 6 p = create_pipeline(cfg, in_layers=in_layers, goal_layers=goal_layers) assert p.nb_components() == 6 assert 'opinion-miner' in p.graph.keys() assert 'ixa-pipe-nerc' in p.graph.keys() assert 'ixa-pipe-ned' in p.graph.keys() assert 'vua-ontotagging' in p.graph.keys()
def test_pipeline_detects_empty_output(): goal_layers = ['deps'] p = pipeline.create_pipeline(fail_empty_cfg, goal_layers=goal_layers) assert p.nb_components() == 3 with open(single_word, 'r') as f: summary = p.execute(f) not_run = [v for v in summary.values() if v == 'not_run'] assert len(not_run) == 1
def test_executable_components_after_failure(): p = pipeline.create_pipeline(fail_cfg) with open(single_word, 'r') as f: summary = p.execute(f) completed = [v for v in summary.values() if v == 'completed'] failed = [v for v in summary.values() if v == 'failed'] assert len(completed) == p.nb_components() - 1 assert len(failed) == 1
def test_pipeline_with_modified_component_args(): goal_layers = ['deps'] subargs = {'vua-alpino': '-t 0.2'} p = pipeline.create_pipeline(cfg, goal_layers=goal_layers, subargs=subargs) assert p.nb_components() == 3 with open(single_word, 'r') as f: summary = p.execute(f) completed = [v for v in summary.values() if v == 'completed'] assert len(completed) == 3
def test_component_fails_in_partially_generated_layer(): goal_layers = ['terms'] p = pipeline.create_pipeline(cfg, in_layers=['terms'], goal_layers=goal_layers) scheduled = p.topological_sort() assert len(scheduled) == 3 with open(part1_out, 'r') as f: summary = p.execute(f) failed = [v for v in summary.values() if v == 'failed'] assert len(failed) == 1
def test_finish_incomplete_pipeline(): goal_layers = ['opinions', 'timex'] p = pipeline.create_pipeline(cfg, in_layers=['opinions', 'timex'], goal_layers=goal_layers) scheduled = p.topological_sort() assert len(scheduled) == 2 with open(part1_out, 'r') as f: summary = p.execute(f) completed = [v for v in summary.values() if v == 'completed'] assert len(completed) == len(goal_layers)
def test_rescheduling_after_alpino_failure(): p = pipeline.create_pipeline(fail_alpino_cfg) scheduled = p.topological_sort() assert len(scheduled) == 4 with open(single_word, 'r') as f: summary = p.execute(f) completed = [v for v in summary.values() if v == 'completed'] failed = [v for v in summary.values() if v == 'failed'] not_run = [v for v in summary.values() if v == 'not_run'] assert len(not_run) == 1 assert len(completed) == 2 assert len(failed) == 1
def test_pipeline_creation(): p = create_pipeline(cfg) assert p.nb_components() == NB_COMPS scheduled = p.topological_sort() assert len(scheduled) == NB_COMPS
def test_filtering_with_goal_layers_before_in_layers_has_no_effect(): p = create_pipeline(cfg, in_layers=['srl']) nb_comps_in_filtered = p.nb_components() goal_layers = ['terms'] p = create_pipeline(cfg, in_layers=['srl'], goal_layers=goal_layers) assert p.nb_components() == nb_comps_in_filtered
def test_create_pipeline_without_component_with_dependents(): with pytest.raises(ValueError): create_pipeline(cfg, excepted_components=['vua-wsd'])
def test_create_pipeline_without_layer_modifying_component(): excepted_components = ['ixa-pipe-ned'] p = create_pipeline(cfg, excepted_components=excepted_components) assert p.nb_components() == NB_COMPS - 1
input_layers = [] if args.in_layers_str is not None: input_layers = args.in_layers_str.split(',') output_layers = [] if args.out_layers_str is not None: output_layers = args.out_layers_str.split(',') exclude_components = [] if args.exclude_components_str is not None: exclude_components = args.exclude_components_str.split(',') subargs = {} if args.component_args is not None: for argv in args.component_args.split(';'): mod_name, opt_name, opt_val = argv.split(':') subargs[mod_name] = '{} {}'.format(opt_name, opt_val) try: p = pipeline.create_pipeline(cfg_file, in_layers=input_layers, goal_layers=output_layers, excepted_components=exclude_components, bindir=bin_dir, subargs=subargs) input_file = sys.stdin p.execute(input_file) except ValueError as e: logger.error(e)