def test_workflow(): from xicam.core.execution.workflow import Workflow from xicam.core.execution.daskexecutor import DaskExecutor from xicam.plugins.operationplugin import output_names executor = DaskExecutor() @operation @output_names("square") def square(a=3) -> int: return a**2 @operation @output_names("sum") def my_sum(a, b=3) -> int: return a + b wf = Workflow() square = square() my_sum = my_sum() wf.add_operation(square) wf.add_operation(my_sum) wf.add_link(square, my_sum, "square", "a") assert wf.execute_synchronous(executor=executor) == [{"sum": 12}]
def test_execute_synchronous(self, sum_op, square_op, negative_op): workflow = Workflow() workflow.add_operations(sum_op, square_op, negative_op) workflow.add_link(sum_op, square_op, "sum", "n") workflow.add_link(square_op, negative_op, "square", "num") results = workflow.execute_synchronous(n1=2, n2=5) assert results == ({"negative": -49}, )
def test_end_node(self, double_and_triple_op, sum_op, square_op): workflow = Workflow() workflow.add_operations(double_and_triple_op, sum_op, square_op) workflow.add_link(sum_op, square_op, "sum", "n") workflow.add_link(square_op, double_and_triple_op, "square", "n") result = workflow.execute_synchronous(n1=1, n2=2) assert result == ({"double": 18, "triple": 27}, )
def test_start_node(self, double_and_triple_op, sum_op, square_op): workflow = Workflow() workflow.add_operations(double_and_triple_op, sum_op, square_op) workflow.add_link(double_and_triple_op, sum_op, "double", "n1") workflow.add_link(double_and_triple_op, sum_op, "triple", "n2") workflow.add_link(sum_op, square_op, "sum", "n") result = workflow.execute_synchronous(n=1) assert result == ({"square": 25}, )
def test_mutliple_end_nodes(double_and_triple_op, sum_op, square_op): workflow = Workflow() workflow.add_operations(double_and_triple_op, sum_op, square_op) workflow.add_link(sum_op, double_and_triple_op, "sum", "n") workflow.add_link(sum_op, square_op, "sum", "n") result = workflow.execute_synchronous(n1=2, n2=3) assert len(result) == 2 assert {"double": 10, "triple": 15} in result assert {"square": 25} in result
def test_two_copy_ops_to_one_op(self, double_and_triple_op, sum_op, square_op): # 1**2 + 2**2 => 5 workflow = Workflow() square_op.filled_values.update(n=1) square_op_2 = square_op.__class__() square_op_2.filled_values.update(n=2) square_op_2.output_names = ["square"] workflow.add_operations(sum_op, square_op, square_op_2) workflow.add_link(square_op, sum_op, "square", "n1") workflow.add_link(square_op_2, sum_op, "square", "n2") workflow._pretty_print() result = workflow.execute_synchronous() assert result == ({"sum": 5},)
def simple_workflow_with_intents(plot_op, abs_plot_op, blur_image_op, image_op): wf = Workflow() wf.add_operation(image_op) wf.add_operation(blur_image_op) wf.add_link(image_op, blur_image_op, "output_array", "arr") wf.add_operation(plot_op) wf.add_operation(abs_plot_op) wf.add_link(plot_op, abs_plot_op, "output1", "x_arr") wf.add_link(plot_op, abs_plot_op, "output2", "y_arr") return wf
def test_execute_all(self, qtbot, sum_op, square_op, negative_op): results = [{"negative": (1 + 2) ** 2 * -1}, {"negative": (3 + 4) ** 2 * -1}, {"negative": (5 + 6) ** 2 * -1}] def cb(*result): next_result = results.pop(0) assert result == next_result workflow = Workflow() workflow.add_operations(sum_op, square_op, negative_op) workflow.add_link(sum_op, square_op, "sum", "n") workflow.add_link(square_op, negative_op, "square", "num") n1_values = [1, 3, 5] n2_values = [2, 4, 6] # TODO -- we are only getting one result, should get three (3 pairs of n1/n2). workflow.execute_all(callback_slot=cb, n1=n1_values, n2=n2_values).result()
def test_two_ops_to_one_op(self, negative_op, square_op, sum_op): workflow = Workflow() workflow.add_operations(negative_op, square_op, sum_op) workflow.add_link(negative_op, sum_op, "negative", "n1") workflow.add_link(square_op, sum_op, "square", "n2") print(workflow.get_inbound_links(sum_op)) #from dask import visualize #visualize(workflow.as_dask_graph()[0], filename="/home/ihumphrey/graph") graph = workflow.as_dask_graph()[0] print(graph) for k, op in graph.items(): print(k, op[0].node.name) from dask.threaded import get negative_op.filled_values.update(num=3) square_op.filled_values.update(n=4) print(get(graph, '0')) # WHY is this an issue? Complains that sum missing required 'n2'
def simple_workflow(square_op, sum_op): from xicam.core.execution.workflow import Workflow wf = Workflow() square = square_op square2 = square_op.__class__() square2.filled_values["n"] = 2 wf.add_operation(square) wf.add_operation(square2) wf.add_operation(sum_op) wf.add_link(square, sum_op, "square", "n1") wf.add_link(square2, sum_op, "square", "n2") return wf
def test_execute_all(self, sum_op, square_op, negative_op): workflow = Workflow() workflow.add_operations(sum_op, square_op, negative_op) workflow.add_link(sum_op, square_op, "sum", "n") workflow.add_link(square_op, negative_op, "square", "num") n1_values = [1, 3, 5] n2_values = [2, 4, 6] # TODO -- we are only getting one result, should get three (3 pairs of n1/n2). results = list( workflow.execute_all(n1=n1_values, n2=n2_values).result()) assert results == [{ "negative": (1 + 2)**2 * -1 }, { "negative": (3 + 4)**2 * -1 }, { "negative": (5 + 6)**2 * -1 }]
def test_multiple_instances(square_op, sum_op): from xicam.core.execution.workflow import Workflow from xicam.core.execution.daskexecutor import DaskExecutor executor = DaskExecutor() wf = Workflow() square = square_op() square2 = square_op() square2.filled_values["a"] = 2 my_sum = sum_op() wf.add_operation(square) wf.add_operation(square2) wf.add_operation(my_sum) wf.add_link(square, my_sum, "square", "a") # 3**3 wf.add_link(square2, my_sum, "square", "b") # 2**2 assert wf.execute_synchronous(executor=executor) == [{"sum": 13}]
def test_multiple_instances(square_op, sum_op): from xicam.core.execution.workflow import Workflow from xicam.core.execution.daskexecutor import DaskExecutor executor = DaskExecutor() wf = Workflow() square = square_op() square2 = square_op() square2.filled_values['a'] = 2 my_sum = sum_op() wf.add_operation(square) wf.add_operation(square2) wf.add_operation(my_sum) wf.add_link(square, my_sum, 'square', 'a') wf.add_link(square2, my_sum, 'square', 'b') assert wf.execute_synchronous(executor=executor) == [{'sum': 13}]
def test_operation_links_multiple(self, sum_op, square_op, negative_op): # # (y) --> (n1) # my_func sum (n) --> (num) square # (x) --> (n2) # def my_func(x: int, y: int) -> (int, int): return y, x my_op = operation(my_func, output_names=("y", "x"))() workflow = Workflow() # workflow.add_operations(sum_op, square_op, negative_op, my_op) workflow.add_operations(my_op, sum_op, square_op, negative_op) link1 = (my_op, sum_op, "y", "n1") link2 = (my_op, sum_op, "x", "n2") link3 = (sum_op, square_op, "sum", "n") link4 = (square_op, negative_op, "square", "num") workflow.add_link(my_op, sum_op, "y", "n1") workflow.add_link(my_op, sum_op, "x", "n2") workflow.add_link(sum_op, square_op, "sum", "n") workflow.add_link(square_op, negative_op, "square", "num") # assert workflow.operation_links(my_op) == [link1, link2] # assert workflow.operation_links(sum_op) == [link3] # assert workflow.operation_links(square_op) == [link4] # assert workflow.operation_links(negative_op) == [] workflow._pretty_print() dask_graph, end_ids = workflow.as_dask_graph() # test execution results = workflow.execute_synchronous(x=3, y=5)