def _run_test(configuration, box_eval): from pypeline.helpers.helpers import run_pipeline box_config = configure(configuration) box = initialise(box_config) run_pipeline(box, box_config, box_eval) thelp.diff(box_eval['cleaned_src_file_expected'], box_eval['cleaned_src_filename']) thelp.diff(box_eval['cleaned_trg_file_expected'], box_eval['cleaned_trg_filename'])
def __test():
configuration = {'src_lang':'de',
'src_tokenisation_dir':'tmptok',
'moses_installation_dir':os.path.abspath('../../../../')}
values = {'src_filename':'tmp.de'}
from pypeline.helpers.helpers import run_pipeline
box_config = configure(configuration)
box = initialise(configuration)
print run_pipeline(box, values, None)
def __test():
configuration = {'src_lang':'en',
'trg_lang':'lt',
'moses_installation_dir':os.environ['MOSES_HOME'],
'giza_installation_dir':os.environ['GIZA_HOME'],
'translation_model_directory':'model-dir'}
values = {'training_data_filename':'/Users/ianjohnson/work/MTM-2012/corpus/training/cleantrain'}
from pypeline.helpers.helpers import run_pipeline
box_config = configure(configuration)
box = initialise(box_config)
print run_pipeline(box, values, None)
def __test():
configuration = {
'src_lang': 'de',
'src_tokenisation_dir': 'tmptok',
'moses_installation_dir': os.path.abspath('../../../../')
}
values = {'src_filename': 'tmp.de'}
from pypeline.helpers.helpers import run_pipeline
box_config = configure(configuration)
box = initialise(configuration)
print run_pipeline(box, values, None)
def __test():
configuration = {
"src_lang": "en",
"trg_lang": "lt",
"moses_installation_dir": os.environ["MOSES_HOME"],
"giza_installation_dir": os.environ["GIZA_HOME"],
"translation_model_directory": "model-dir",
}
values = {"training_data_filename": "/Users/ianjohnson/work/MTM-2012/corpus/training/cleantrain"}
from pypeline.helpers.helpers import run_pipeline
box_config = configure(configuration)
box = initialise(box_config)
print run_pipeline(box, values, None)
def __test():
configuration = {'src_lang':'en',
'trg_lang':'lt',
'moses_installation_dir':os.path.abspath('../../../../'),
'mert_working_dir':'../../../../../tuning'}
values = {'development_data_filename':'../../../../../corpus/tune',
'moses_ini_file':'../../../../../model/model/moses.ini',
'trg_language_model_filename':'../../../../../corpus/train.lt.lm',
'trg_language_model_type':9,
'trg_language_model_order':4}
from pypeline.helpers.helpers import run_pipeline
box_config = configure(configuration)
box = initialise(configuration)
print run_pipeline(box, values, None)
def __test():
configuration = {
'src_lang': 'en',
'trg_lang': 'lt',
'moses_installation_dir': os.environ['MOSES_HOME'],
'giza_installation_dir': os.environ['GIZA_HOME'],
'translation_model_directory': 'model-dir'
}
values = {
'training_data_filename':
'/Users/ianjohnson/work/MTM-2012/corpus/training/cleantrain'
}
from pypeline.helpers.helpers import run_pipeline
box_config = configure(configuration)
box = initialise(box_config)
print run_pipeline(box, values, None)
def __test():
configuration = {
'src_lang': 'en',
'trg_lang': 'lt',
'moses_installation_dir': os.path.abspath('../../../../'),
'mert_working_dir': '../../../../../tuning'
}
values = {
'development_data_filename': '../../../../../corpus/tune',
'moses_ini_file': '../../../../../model/model/moses.ini',
'trg_language_model_filename': '../../../../../corpus/train.lt.lm',
'trg_language_model_type': 9,
'trg_language_model_order': 4
}
from pypeline.helpers.helpers import run_pipeline
box_config = configure(configuration)
box = initialise(configuration)
print run_pipeline(box, values, None)
def test_pypeline_with_subprocess_and_function_components(self):
if sys.platform.startswith('win'):
self.fail("Currently only this unit test is only supported on non-Windows platforms")
rev_msg_one = "reverse(1)"
rev_msg_two = "reverse(2)"
upper_msg = "upper"
reverse_command = os.path.join("src", "pypeline", "helpers", "tests", "reverse.sh")
comp_proc_one = PypelineHelperUnitTest.__cons_and_start_subprocess_component(
reverse_command, tuple(),
lambda a, s: str(a['input']),
lambda a, s: {'output': str(a)},
state_mutator = lambda s: s.append(rev_msg_one) or s)
try:
comp_proc_two = PypelineHelperUnitTest.__cons_and_start_subprocess_component(
reverse_command, tuple(),
lambda a, s: str(a['input']),
lambda a, s: {'output': str(a)},
state_mutator = lambda s: s.append(rev_msg_two) or s)
try:
comp_one = comp_proc_one[0]
comp_two = comp_proc_two[0]
upper_func = lambda a_string, s: a_string.upper()
comp_three = cons_function_component(upper_func,
state_mutator = lambda s: s.append(upper_msg) or s)
input_wire_func = lambda a, s: {'input': a}
input_wire = cons_wire(input_wire_func)
wire = cons_dictionary_wire({'output': 'input'})
output_to_string_func = lambda a, s: str(a['output'])
to_upper_wire = cons_wire(output_to_string_func)
output_wire_func = lambda a, s: str(a['output'])
output_wire = cons_wire(output_wire_func)
pipeline = cons_pipeline(input_wire,
cons_wired_components(comp_one, comp_two, wire),
to_upper_wire)
pipeline = cons_composed_component(pipeline, comp_three)
value = "hello world"
target = (upper_func(value, None), [rev_msg_one, rev_msg_two, upper_msg])
result = run_pipeline(pipeline, "hello world", list())
self.assertEquals(target, result)
finally:
comp_proc_two[1].terminate()
comp_proc_two[1].wait()
finally:
comp_proc_one[1].terminate()
comp_proc_one[1].wait()
def test_pypeline_with_split_and_unsplit_wires(self):
if sys.platform.startswith('win'):
self.fail("Currently only this unit test is only supported on non-Windows platforms")
rev_msg_one = "reverse(subprocess)"
rev_msg_two = "reverse(function)"
reverse_command = os.path.join("src", "pypeline", "helpers", "tests", "reverse.sh")
reverse_func = lambda a, s: a[::-1]
input_func = lambda a, s: str(a['input'])
output_func = lambda a, s: {'output': str(a)}
comp_proc_one = PypelineHelperUnitTest.__cons_and_start_subprocess_component(
reverse_command, tuple(),
input_func,
output_func,
state_mutator = lambda s: s.append(rev_msg_one) or s)
try:
comp_one = comp_proc_one[0]
comp_two = cons_function_component(
reverse_func,
input_func,
output_func,
state_mutator = lambda s: s.append(rev_msg_two) or s)
parallel_reverse_comp = cons_parallel_component(comp_one, comp_two)
split_wire = cons_split_wire()
unsplit_func = lambda a, b: {'subprocess_output' : a['output'],
'function_output': b['output']}
unsplit_wire = cons_unsplit_wire(unsplit_func)
input_wire = cons_wire(lambda a, s: {'input': a})
pipeline = cons_pipeline(input_wire,
cons_composed_component(split_wire, parallel_reverse_comp),
unsplit_wire)
value = "hello world"
result = run_pipeline(pipeline, "hello world", list())
target_dict = {'output': reverse_func(value, None)}
target_value = unsplit_func(target_dict, target_dict)
target = (target_value, [rev_msg_one, rev_msg_two])
self.assertEquals(target, result)
finally:
comp_proc_one[1].terminate()
comp_proc_one[1].wait()
