def test_parallel_pypeline_with_split_and_unsplit_wires(self): rev_msg_one = "reverse(top)" rev_msg_two = "reverse(bottom)" top_msg = "top" bottom_msg = "bottom" reverse_func = lambda a, s: a[::-1] top_func = lambda a, s: " ".join([a, top_msg]) bottom_func = lambda a, s: " ".join([a, bottom_msg]) comp_rev_top = cons_function_component(reverse_func, state_mutator = lambda s: s.append(rev_msg_one) or s) comp_rev_bottom = cons_function_component(reverse_func, state_mutator = lambda s: s.append(rev_msg_two) or s) comp_para_top = cons_function_component(top_func, state_mutator = lambda s: s.append(top_msg) or s) comp_para_bottom = cons_function_component(bottom_func, state_mutator = lambda s: s.append(bottom_msg) or s) unsplit_func = lambda t, b: {'top': t, 'bottom': b} pipeline = (comp_rev_top & comp_rev_bottom) >> \ (comp_para_top ** comp_para_bottom) >> \ cons_unsplit_wire(unsplit_func) value = "hello world" target = (unsplit_func(top_func(reverse_func(value, None), None), bottom_func(reverse_func(value, None), None)), [rev_msg_one, rev_msg_two, top_msg, bottom_msg]) result = ParallelPypelineHelperUnitTest.test(2, pipeline, "hello world", list()) self.assertEquals(target, result)
def test_parallel_first_and_second(self): pi = 3.141 e = 2.718 value = {'pi' : pi, 'e' : e} pipeline = cons_split_wire() >> \ (cons_dictionary_wire({'pi' : 'PI'}) >> cons_function_component(lambda a, s: {'PI' : a['PI']})).first() >> \ (cons_dictionary_wire({'e' : 'E'}) >> cons_function_component(lambda a, s: {'E' : a['E']})).second() result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, None, eval_pipeline) self.assertEquals(({'PI' : pi}, {'E' : e}), result)
def test_parallel_split(self): pi = 3.141 value = {'pi' : pi} pipeline = cons_function_component(lambda a, s: a) >> \ cons_split_wire() >> \ cons_function_component(lambda a, s: {'PI' : a['pi']}).first() >> \ (cons_function_component(lambda a, s: {'PI' : a['PI']}) ** \ cons_function_component(lambda a, s: a)) >> \ cons_unsplit_wire(lambda t, b: {'PI' : t['PI'], 'pi' : b['pi']}) result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, None, eval_pipeline) self.assertEquals({'PI' : pi, 'pi' : pi}, result)
def test_parallel_run_eval_and_exec(self): value = "hello world" state = 0 input_msg = "input" output_msg = "output" input_func = lambda a, s: " ".join([input_msg, a]) output_func = lambda a, s: " ".join([a, output_msg]) function = lambda a, s: a.upper() composition = lambda a, s: output_func(function(input_func(a, s), s), s) state_func = lambda s: s + 1 pipeline = cons_function_component(function, input_func, output_func, state_mutator = state_func) result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, state, run_pipeline) target = (composition(value, state), state_func(state)) self.assertEquals(target, result) result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, state, eval_pipeline) target = composition(value, state) self.assertEquals(target, result) result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, state, exec_pipeline) target = state_func(state) self.assertEquals(target, result)
def build_components(components, configuration, executor): pipeline_components = dict() pipeline_configuration = dict() for component_id, module_name in components.items(): logger.info("Loading [%s] component from [%s]..." % (component_id, module_name)) module = __import__(module_name, fromlist = ['configure', 'initialise']) # Component builds its own configuration object config_func = getattr(module, 'configure') component_config = config_func(configuration) pipeline_configuration.update(component_config) # Now build the component init_func = getattr(module, 'initialise') component_function = init_func(component_config) # A wrapper for the component's function that submits to the executor def get_component_function_wrapper(inner_function, comp_id, mod_name): def component_function_wrapper(a, s): logger.info("Running component [%s], from module [%s], with value [%s] and state [%s]..." % \ (comp_id, mod_name, a, s)) return inner_function(a, s) return component_function_wrapper # Arrowize the component component = cons_function_component(get_component_function_wrapper(component_function, component_id, module_name)) # And store pipeline_components[component_id] = component return pipeline_components, pipeline_configuration
def test_parallel_if(self): then_comp = cons_function_component(lambda a, s: {'z' : 'THEN'}) else_comp = cons_dictionary_wire({'c' : 'z'}) pipeline = cons_if_component(lambda a, s: a['a'] == True, then_comp, else_comp) value = {'a' : True, 'b' : 'then', 'c' : 'else'} result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, None, eval_pipeline) self.assertEquals({'z' : 'THEN'}, result) value = {'a' : False, 'b' : 'then', 'c' : 'else'} result = ParallelPypelineHelperUnitTest.test(1, pipeline, value, None, eval_pipeline) self.assertEquals({'z' : 'else'}, result)
def test_serial_pypeline_with_function_components(self): rev_msg_one = "reverse(1)" rev_msg_two = "reverse(2)" upper_msg = "upper" reverse_function = lambda a, s: a[::-1] upper_function = lambda a, s: a.upper() comp_rev_one = cons_function_component(reverse_function, state_mutator = lambda s: s.append(rev_msg_one) or s) comp_rev_two = cons_function_component(reverse_function, state_mutator = lambda s: s.append(rev_msg_two) or s) comp_upper = cons_function_component(upper_function, state_mutator = lambda s: s.append(upper_msg) or s) pipeline = comp_rev_one >> comp_rev_two >> comp_upper value = "hello world" target = (upper_function(value, None), [rev_msg_one, rev_msg_two, upper_msg]) result = ParallelPypelineHelperUnitTest.test(2, pipeline, "hello world", list()) self.assertEquals(target, result)
def execute_module(executor, pcl_import_path, pcl_module, get_configuration_fn, get_inputs_fn): """Executes a PCL component in a concurrent environment. Provide a the concurrent execution environment, a colon separated PCL import path, the fully qualified PCL module name, and getter two functions. The configuration function receives the expected configuration keys and should return a dictionary, whose keys are the expected configuration, with appropriate values. The input function received the expected inputs and should return a dictionary, whose keys are the expected inputs, with appropriate values.""" # Set up Python path to import compiled PCL modules pcl_import_path_bits = pcl_import_path.split(":") for pcl_import_path_bit in pcl_import_path_bits: if pcl_import_path_bit not in sys.path: sys.path.append(pcl_import_path_bit) # Import PCL try: pcl = __import__(pcl_module, fromlist = ['get_inputs', 'get_outputs', 'get_configuration', 'configure', 'initialise']) except Exception as ex: raise PCLImportError(ex) # Get the pipeline get_expected_inputs_fn = getattr(pcl, "get_inputs") get_expected_outputs_fn = getattr(pcl, "get_outputs") get_expected_configuration_fn = getattr(pcl, "get_configuration") configure_fn = getattr(pcl, "configure") initialise_fn = getattr(pcl, "initialise") # Inputs, configuration values from the provided functions pipeline_inputs = get_inputs_fn(get_expected_inputs_fn()) pipeline_configuration = get_configuration_fn(get_expected_configuration_fn()) # The expected outputs from the component expected_outputs = get_expected_outputs_fn() # Configure the PCL... filtered_config = configure_fn(pipeline_configuration) # ...and initialise pipeline = initialise_fn(filtered_config) if not isinstance(pipeline, KleisliArrow): pipeline = cons_function_component(pipeline) return (expected_outputs, eval_pipeline(executor, pipeline, pipeline_inputs, pipeline_configuration))
def build_components(components, configuration, executor): pipeline_components = dict() pipeline_configuration = dict() for component_id, module_name in components.items(): logger.info("Loading [%s] component from [%s]..." % (component_id, module_name)) module = __import__(module_name, fromlist=['configure', 'initialise']) # Component builds its own configuration object config_func = getattr(module, 'configure') component_config = config_func(configuration) pipeline_configuration.update(component_config) # Now build the component init_func = getattr(module, 'initialise') component_function = init_func(component_config) # A wrapper for the component's function that submits to the executor def get_component_function_wrapper(inner_function, comp_id, mod_name): def component_function_wrapper(a, s): logger.info("Running component [%s], from module [%s], with value [%s] and state [%s]..." % \ (comp_id, mod_name, a, s)) return inner_function(a, s) return component_function_wrapper # Arrowize the component component = cons_function_component( get_component_function_wrapper(component_function, component_id, module_name)) # And store pipeline_components[component_id] = component return pipeline_components, pipeline_configuration