def test_parallel_pypeline_with_split_and_unsplit_wires(self):
rev_msg_one = "reverse(top)"
rev_msg_two = "reverse(bottom)"
top_msg = "top"
bottom_msg = "bottom"
reverse_func = lambda a, s: a[::-1]
top_func = lambda a, s: " ".join([a, top_msg])
bottom_func = lambda a, s: " ".join([a, bottom_msg])
comp_rev_top = cons_function_component(reverse_func,
state_mutator = lambda s: s.append(rev_msg_one) or s)
comp_rev_bottom = cons_function_component(reverse_func,
state_mutator = lambda s: s.append(rev_msg_two) or s)
comp_para_top = cons_function_component(top_func,
state_mutator = lambda s: s.append(top_msg) or s)
comp_para_bottom = cons_function_component(bottom_func,
state_mutator = lambda s: s.append(bottom_msg) or s)
unsplit_func = lambda t, b: {'top': t, 'bottom': b}
pipeline = (comp_rev_top & comp_rev_bottom) >> \
(comp_para_top ** comp_para_bottom) >> \
cons_unsplit_wire(unsplit_func)
value = "hello world"
target = (unsplit_func(top_func(reverse_func(value, None), None),
bottom_func(reverse_func(value, None), None)),
[rev_msg_one, rev_msg_two, top_msg, bottom_msg])
result = ParallelPypelineHelperUnitTest.test(2, pipeline, "hello world", list())
self.assertEquals(target, result)
def test_parallel_first_and_second(self):
pi = 3.141
e = 2.718
value = {'pi' : pi, 'e' : e}
pipeline = cons_split_wire() >> \
(cons_dictionary_wire({'pi' : 'PI'}) >> cons_function_component(lambda a, s: {'PI' : a['PI']})).first() >> \
(cons_dictionary_wire({'e' : 'E'}) >> cons_function_component(lambda a, s: {'E' : a['E']})).second()
result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, None, eval_pipeline)
self.assertEquals(({'PI' : pi}, {'E' : e}), result)
def test_parallel_split(self):
pi = 3.141
value = {'pi' : pi}
pipeline = cons_function_component(lambda a, s: a) >> \
cons_split_wire() >> \
cons_function_component(lambda a, s: {'PI' : a['pi']}).first() >> \
(cons_function_component(lambda a, s: {'PI' : a['PI']}) ** \
cons_function_component(lambda a, s: a)) >> \
cons_unsplit_wire(lambda t, b: {'PI' : t['PI'], 'pi' : b['pi']})
result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, None, eval_pipeline)
self.assertEquals({'PI' : pi, 'pi' : pi}, result)
def test_parallel_run_eval_and_exec(self):
value = "hello world"
state = 0
input_msg = "input"
output_msg = "output"
input_func = lambda a, s: " ".join([input_msg, a])
output_func = lambda a, s: " ".join([a, output_msg])
function = lambda a, s: a.upper()
composition = lambda a, s: output_func(function(input_func(a, s), s), s)
state_func = lambda s: s + 1
pipeline = cons_function_component(function,
input_func,
output_func,
state_mutator = state_func)
result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, state, run_pipeline)
target = (composition(value, state), state_func(state))
self.assertEquals(target, result)
result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, state, eval_pipeline)
target = composition(value, state)
self.assertEquals(target, result)
result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, state, exec_pipeline)
target = state_func(state)
self.assertEquals(target, result)
def build_components(components, configuration, executor):
pipeline_components = dict()
pipeline_configuration = dict()
for component_id, module_name in components.items():
logger.info("Loading [%s] component from [%s]..." % (component_id, module_name))
module = __import__(module_name, fromlist = ['configure', 'initialise'])
# Component builds its own configuration object
config_func = getattr(module, 'configure')
component_config = config_func(configuration)
pipeline_configuration.update(component_config)
# Now build the component
init_func = getattr(module, 'initialise')
component_function = init_func(component_config)
# A wrapper for the component's function that submits to the executor
def get_component_function_wrapper(inner_function, comp_id, mod_name):
def component_function_wrapper(a, s):
logger.info("Running component [%s], from module [%s], with value [%s] and state [%s]..." % \
(comp_id, mod_name, a, s))
return inner_function(a, s)
return component_function_wrapper
# Arrowize the component
component = cons_function_component(get_component_function_wrapper(component_function, component_id, module_name))
# And store
pipeline_components[component_id] = component
return pipeline_components, pipeline_configuration
def test_parallel_if(self):
then_comp = cons_function_component(lambda a, s: {'z' : 'THEN'})
else_comp = cons_dictionary_wire({'c' : 'z'})
pipeline = cons_if_component(lambda a, s: a['a'] == True, then_comp, else_comp)
value = {'a' : True, 'b' : 'then', 'c' : 'else'}
result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, None, eval_pipeline)
self.assertEquals({'z' : 'THEN'}, result)
value = {'a' : False, 'b' : 'then', 'c' : 'else'}
result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, None, eval_pipeline)
self.assertEquals({'z' : 'else'}, result)
def test_serial_pypeline_with_function_components(self):
rev_msg_one = "reverse(1)"
rev_msg_two = "reverse(2)"
upper_msg = "upper"
reverse_function = lambda a, s: a[::-1]
upper_function = lambda a, s: a.upper()
comp_rev_one = cons_function_component(reverse_function,
state_mutator = lambda s: s.append(rev_msg_one) or s)
comp_rev_two = cons_function_component(reverse_function,
state_mutator = lambda s: s.append(rev_msg_two) or s)
comp_upper = cons_function_component(upper_function,
state_mutator = lambda s: s.append(upper_msg) or s)
pipeline = comp_rev_one >> comp_rev_two >> comp_upper
value = "hello world"
target = (upper_function(value, None), [rev_msg_one, rev_msg_two, upper_msg])
result = ParallelPypelineHelperUnitTest.test(2, pipeline, "hello world", list())
self.assertEquals(target, result)
def execute_module(executor, pcl_import_path, pcl_module, get_configuration_fn, get_inputs_fn):
"""Executes a PCL component in a concurrent environment. Provide a the concurrent execution environment, a colon separated PCL import path, the fully qualified PCL module name, and getter two functions. The configuration function receives the expected configuration keys and should return a dictionary, whose keys are the expected configuration, with appropriate values. The input function received the expected inputs and should return a dictionary, whose keys are the expected inputs, with appropriate values."""
# Set up Python path to import compiled PCL modules
pcl_import_path_bits = pcl_import_path.split(":")
for pcl_import_path_bit in pcl_import_path_bits:
if pcl_import_path_bit not in sys.path:
sys.path.append(pcl_import_path_bit)
# Import PCL
try:
pcl = __import__(pcl_module, fromlist = ['get_inputs',
'get_outputs',
'get_configuration',
'configure',
'initialise'])
except Exception as ex:
raise PCLImportError(ex)
# Get the pipeline
get_expected_inputs_fn = getattr(pcl, "get_inputs")
get_expected_outputs_fn = getattr(pcl, "get_outputs")
get_expected_configuration_fn = getattr(pcl, "get_configuration")
configure_fn = getattr(pcl, "configure")
initialise_fn = getattr(pcl, "initialise")
# Inputs, configuration values from the provided functions
pipeline_inputs = get_inputs_fn(get_expected_inputs_fn())
pipeline_configuration = get_configuration_fn(get_expected_configuration_fn())
# The expected outputs from the component
expected_outputs = get_expected_outputs_fn()
# Configure the PCL...
filtered_config = configure_fn(pipeline_configuration)
# ...and initialise
pipeline = initialise_fn(filtered_config)
if not isinstance(pipeline, KleisliArrow):
pipeline = cons_function_component(pipeline)
return (expected_outputs,
eval_pipeline(executor, pipeline, pipeline_inputs, pipeline_configuration))
def build_components(components, configuration, executor):
pipeline_components = dict()
pipeline_configuration = dict()
for component_id, module_name in components.items():
logger.info("Loading [%s] component from [%s]..." %
(component_id, module_name))
module = __import__(module_name, fromlist=['configure', 'initialise'])
# Component builds its own configuration object
config_func = getattr(module, 'configure')
component_config = config_func(configuration)
pipeline_configuration.update(component_config)
# Now build the component
init_func = getattr(module, 'initialise')
component_function = init_func(component_config)
# A wrapper for the component's function that submits to the executor
def get_component_function_wrapper(inner_function, comp_id, mod_name):
def component_function_wrapper(a, s):
logger.info("Running component [%s], from module [%s], with value [%s] and state [%s]..." % \
(comp_id, mod_name, a, s))
return inner_function(a, s)
return component_function_wrapper
# Arrowize the component
component = cons_function_component(
get_component_function_wrapper(component_function, component_id,
module_name))
# And store
pipeline_components[component_id] = component
return pipeline_components, pipeline_configuration
