Beispiel #1
0
 def _run_test(configuration, box_eval):
   from pypeline.helpers.helpers import run_pipeline
   box_config = configure(configuration)
   box = initialise(box_config)
   
   run_pipeline(box, box_config, box_eval)
   thelp.diff(box_eval['cleaned_src_file_expected'], box_eval['cleaned_src_filename'])
   thelp.diff(box_eval['cleaned_trg_file_expected'], box_eval['cleaned_trg_filename'])
Beispiel #2
0
 def __test():
     configuration = {'src_lang':'de',
                      'src_tokenisation_dir':'tmptok',
                      'moses_installation_dir':os.path.abspath('../../../../')}
     values = {'src_filename':'tmp.de'}
     from pypeline.helpers.helpers import run_pipeline
     box_config = configure(configuration)
     box = initialise(configuration)
     print run_pipeline(box, values, None)
 def __test():
     configuration = {'src_lang':'en',
                      'trg_lang':'lt',
                      'moses_installation_dir':os.environ['MOSES_HOME'],
                      'giza_installation_dir':os.environ['GIZA_HOME'],
                      'translation_model_directory':'model-dir'}
     values = {'training_data_filename':'/Users/ianjohnson/work/MTM-2012/corpus/training/cleantrain'}
     from pypeline.helpers.helpers import run_pipeline
     box_config = configure(configuration)
     box = initialise(box_config)
     print run_pipeline(box, values, None)
Beispiel #4
0
 def __test():
     configuration = {
         'src_lang': 'de',
         'src_tokenisation_dir': 'tmptok',
         'moses_installation_dir': os.path.abspath('../../../../')
     }
     values = {'src_filename': 'tmp.de'}
     from pypeline.helpers.helpers import run_pipeline
     box_config = configure(configuration)
     box = initialise(configuration)
     print run_pipeline(box, values, None)
    def __test():
        configuration = {
            "src_lang": "en",
            "trg_lang": "lt",
            "moses_installation_dir": os.environ["MOSES_HOME"],
            "giza_installation_dir": os.environ["GIZA_HOME"],
            "translation_model_directory": "model-dir",
        }
        values = {"training_data_filename": "/Users/ianjohnson/work/MTM-2012/corpus/training/cleantrain"}
        from pypeline.helpers.helpers import run_pipeline

        box_config = configure(configuration)
        box = initialise(box_config)
        print run_pipeline(box, values, None)
Beispiel #6
0
 def __test():
     configuration = {'src_lang':'en',
                      'trg_lang':'lt',
                      'moses_installation_dir':os.path.abspath('../../../../'),
                      'mert_working_dir':'../../../../../tuning'}
     values = {'development_data_filename':'../../../../../corpus/tune',
               'moses_ini_file':'../../../../../model/model/moses.ini',
               'trg_language_model_filename':'../../../../../corpus/train.lt.lm',
               'trg_language_model_type':9,
               'trg_language_model_order':4}
     from pypeline.helpers.helpers import run_pipeline
     box_config = configure(configuration)
     box = initialise(configuration)
     print run_pipeline(box, values, None)
Beispiel #7
0
 def __test():
     configuration = {
         'src_lang': 'en',
         'trg_lang': 'lt',
         'moses_installation_dir': os.environ['MOSES_HOME'],
         'giza_installation_dir': os.environ['GIZA_HOME'],
         'translation_model_directory': 'model-dir'
     }
     values = {
         'training_data_filename':
         '/Users/ianjohnson/work/MTM-2012/corpus/training/cleantrain'
     }
     from pypeline.helpers.helpers import run_pipeline
     box_config = configure(configuration)
     box = initialise(box_config)
     print run_pipeline(box, values, None)
Beispiel #8
0
 def __test():
     configuration = {
         'src_lang': 'en',
         'trg_lang': 'lt',
         'moses_installation_dir': os.path.abspath('../../../../'),
         'mert_working_dir': '../../../../../tuning'
     }
     values = {
         'development_data_filename': '../../../../../corpus/tune',
         'moses_ini_file': '../../../../../model/model/moses.ini',
         'trg_language_model_filename': '../../../../../corpus/train.lt.lm',
         'trg_language_model_type': 9,
         'trg_language_model_order': 4
     }
     from pypeline.helpers.helpers import run_pipeline
     box_config = configure(configuration)
     box = initialise(configuration)
     print run_pipeline(box, values, None)
     def test_pypeline_with_subprocess_and_function_components(self):
          if sys.platform.startswith('win'):
               self.fail("Currently only this unit test is only supported on non-Windows platforms")
               
          rev_msg_one = "reverse(1)"
          rev_msg_two = "reverse(2)"
          upper_msg = "upper"

          reverse_command = os.path.join("src", "pypeline", "helpers", "tests", "reverse.sh")

          comp_proc_one = PypelineHelperUnitTest.__cons_and_start_subprocess_component(
               reverse_command, tuple(),
               lambda a, s: str(a['input']),
               lambda a, s: {'output': str(a)},
               state_mutator = lambda s: s.append(rev_msg_one) or s)
          try:
               comp_proc_two = PypelineHelperUnitTest.__cons_and_start_subprocess_component(
                    reverse_command, tuple(),
                    lambda a, s: str(a['input']),
                    lambda a, s: {'output': str(a)},
                    state_mutator = lambda s: s.append(rev_msg_two) or s)
               try:
                    comp_one = comp_proc_one[0]
                    comp_two = comp_proc_two[0]

                    upper_func = lambda a_string, s: a_string.upper()
                    comp_three = cons_function_component(upper_func,
                                                         state_mutator = lambda s: s.append(upper_msg) or s)

                    input_wire_func = lambda a, s: {'input': a}
                    input_wire = cons_wire(input_wire_func)
    
                    wire = cons_dictionary_wire({'output': 'input'})

                    output_to_string_func = lambda a, s: str(a['output'])
                    to_upper_wire = cons_wire(output_to_string_func)

                    output_wire_func = lambda a, s: str(a['output'])
                    output_wire = cons_wire(output_wire_func)

                    pipeline = cons_pipeline(input_wire,
                                             cons_wired_components(comp_one, comp_two, wire),
                                             to_upper_wire)
                    pipeline = cons_composed_component(pipeline, comp_three)

                    value = "hello world"
                    target = (upper_func(value, None), [rev_msg_one, rev_msg_two, upper_msg])
                    result = run_pipeline(pipeline, "hello world", list())

                    self.assertEquals(target, result)
               finally:
                    comp_proc_two[1].terminate()
                    comp_proc_two[1].wait()
          finally:
               comp_proc_one[1].terminate()
               comp_proc_one[1].wait()
     def test_pypeline_with_split_and_unsplit_wires(self):
          if sys.platform.startswith('win'):
               self.fail("Currently only this unit test is only supported on non-Windows platforms")
               
          rev_msg_one = "reverse(subprocess)"
          rev_msg_two = "reverse(function)"

          reverse_command = os.path.join("src", "pypeline", "helpers", "tests", "reverse.sh")

          reverse_func = lambda a, s: a[::-1]
          input_func = lambda a, s: str(a['input'])
          output_func = lambda a, s: {'output': str(a)}

          comp_proc_one = PypelineHelperUnitTest.__cons_and_start_subprocess_component(
               reverse_command, tuple(),
               input_func,
               output_func,
               state_mutator = lambda s: s.append(rev_msg_one) or s)
          try:
               comp_one = comp_proc_one[0]
               comp_two = cons_function_component(
                    reverse_func,
                    input_func,
                    output_func,
                    state_mutator = lambda s: s.append(rev_msg_two) or s)

               parallel_reverse_comp = cons_parallel_component(comp_one, comp_two)
               split_wire = cons_split_wire()
               unsplit_func = lambda a, b: {'subprocess_output' : a['output'],
                                            'function_output': b['output']}
               unsplit_wire = cons_unsplit_wire(unsplit_func)
               input_wire = cons_wire(lambda a, s: {'input': a})
               pipeline = cons_pipeline(input_wire,
                                        cons_composed_component(split_wire, parallel_reverse_comp),
                                        unsplit_wire)

               value = "hello world"
               result = run_pipeline(pipeline, "hello world", list())
               target_dict = {'output': reverse_func(value, None)}
               target_value = unsplit_func(target_dict, target_dict)
               target = (target_value, [rev_msg_one, rev_msg_two])
               self.assertEquals(target, result)
          finally:
               comp_proc_one[1].terminate()
               comp_proc_one[1].wait()