def test_end_node(self, double_and_triple_op, sum_op, square_op):
workflow = Workflow()
workflow.add_operations(double_and_triple_op, sum_op, square_op)
workflow.add_link(sum_op, square_op, "sum", "n")
workflow.add_link(square_op, double_and_triple_op, "square", "n")
result = workflow.execute_synchronous(n1=1, n2=2)
assert result == ({"double": 18, "triple": 27}, )
def test_execute_synchronous(self, sum_op, square_op, negative_op):
workflow = Workflow()
workflow.add_operations(sum_op, square_op, negative_op)
workflow.add_link(sum_op, square_op, "sum", "n")
workflow.add_link(square_op, negative_op, "square", "num")
results = workflow.execute_synchronous(n1=2, n2=5)
assert results == ({"negative": -49}, )
def __init__(self):
# Set up an invert operation
@OperationPlugin
@output_names('inverted_data')
def invert(data: np.ndarray = None) -> np.ndarray:
if issubclass(data.dtype.type, np.integer):
max = np.iinfo(data.dtype).max
else:
max = np.finfo(data.dtype).max
return max - data
# Set up our workflow
workflow = Workflow()
workflow.add_operation(invert)
workflow_editor = WorkflowEditor(workflow)
# Set up a GUI layout
center_widget = QLabel("test")
sample_stage_layout = GUILayout(center_widget)
# Set up stages
stages = {
"Sample Stage": GUILayout(center_widget, right=workflow_editor)
}
self.stages = stages
super(SamplePlugin, self).__init__()
def test_execute_synchronous_no_links_not_enough_kwargs(
self, sum_op, square_op, negative_op):
# TODO -- expect exception
workflow = Workflow()
workflow.add_operations(sum_op, square_op, negative_op)
results = workflow.execute_synchronous(n1=2, n2=5)
assert results == ({"negative": -49}, )
def test_workflow():
from xicam.core.execution.workflow import Workflow
from xicam.core.execution.daskexecutor import DaskExecutor
from xicam.plugins.operationplugin import output_names
executor = DaskExecutor()
@operation
@output_names("square")
def square(a=3) -> int:
return a**2
@operation
@output_names("sum")
def my_sum(a, b=3) -> int:
return a + b
wf = Workflow()
square = square()
my_sum = my_sum()
wf.add_operation(square)
wf.add_operation(my_sum)
wf.add_link(square, my_sum, "square", "a")
assert wf.execute_synchronous(executor=executor) == [{"sum": 12}]
def test_operation_links_multiple(self, sum_op, square_op, negative_op):
#
# (y) --> (n1)
# my_func sum (n) --> (num) square
# (x) --> (n2)
#
def my_func(x: int, y: int) -> (int, int):
return y, x
my_op = operation(my_func, output_names=("y", "x"))()
workflow = Workflow()
# workflow.add_operations(sum_op, square_op, negative_op, my_op)
workflow.add_operations(my_op, sum_op, square_op, negative_op)
link1 = (my_op, sum_op, "y", "n1")
link2 = (my_op, sum_op, "x", "n2")
link3 = (sum_op, square_op, "sum", "n")
link4 = (square_op, negative_op, "square", "num")
workflow.add_link(my_op, sum_op, "y", "n1")
workflow.add_link(my_op, sum_op, "x", "n2")
workflow.add_link(sum_op, square_op, "sum", "n")
workflow.add_link(square_op, negative_op, "square", "num")
# assert workflow.operation_links(my_op) == [link1, link2]
# assert workflow.operation_links(sum_op) == [link3]
# assert workflow.operation_links(square_op) == [link4]
# assert workflow.operation_links(negative_op) == []
workflow._pretty_print()
dask_graph, end_ids = workflow.as_dask_graph()
# test execution
results = workflow.execute_synchronous(x=3, y=5)
def __init__(self):
self.workflow = Workflow()
self.headermodel = QStandardItemModel()
# self.alignmenttabview = TabView(self.headermodel)
self.rawtabview = TabView(self.headermodel,
widgetcls=RAWViewer,
field='primary')
self.recontabs = QTabWidget()
self.workfloweditor = WorkflowEditor(self.workflow)
self.workfloweditor.setHidden(True)
self.tomotoolbar = TomoToolbar()
self.tomotoolbar.sigSliceReconstruction.connect(self.sliceReconstruct)
self.tomotoolbar.sigFullReconstruction.connect(self.fullReconstruction)
self.stages = {
'Alignment':
GUILayout(QLabel('Alignment'),
right=self.workfloweditor,
top=self.tomotoolbar),
'Preprocess':
GUILayout(self.rawtabview,
right=self.workfloweditor,
top=self.tomotoolbar),
'Reconstruct':
GUILayout(self.recontabs,
top=self.tomotoolbar,
right=self.workfloweditor),
}
super(TomographyPlugin, self).__init__()
def test_start_node(self, double_and_triple_op, sum_op, square_op):
workflow = Workflow()
workflow.add_operations(double_and_triple_op, sum_op, square_op)
workflow.add_link(double_and_triple_op, sum_op, "double", "n1")
workflow.add_link(double_and_triple_op, sum_op, "triple", "n2")
workflow.add_link(sum_op, square_op, "sum", "n")
result = workflow.execute_synchronous(n=1)
assert result == ({"square": 25}, )
def test_execute_no_links_diff_input_names(self, sum_op, square_op, negative_op):
# do the input names have to match in this case (more than one entry op)
operations = [sum_op, square_op, negative_op]
workflow = Workflow(name="test", operations=operations)
results = workflow.execute(n1=3, n2=-3, n=10, num=33).result()
assert len(results) == 3
assert {"sum": 0} in results
assert {"square": 100} in results
assert {"negative": -33} in results
def test_mutliple_end_nodes(double_and_triple_op, sum_op, square_op):
workflow = Workflow()
workflow.add_operations(double_and_triple_op, sum_op, square_op)
workflow.add_link(sum_op, double_and_triple_op, "sum", "n")
workflow.add_link(sum_op, square_op, "sum", "n")
result = workflow.execute_synchronous(n1=2, n2=3)
assert len(result) == 2
assert {"double": 10, "triple": 15} in result
assert {"square": 25} in result
def custom_parameter_workflow(custom_parameter_op):
from xicam.core.execution.workflow import Workflow
wf = Workflow()
custom_parameter_op = custom_parameter_op()
wf.add_operation(custom_parameter_op)
return wf
def test_execute_operation_no_default_no_value(self, sum_op):
# not sure how to test this....
def handle_exception(exception):
with pytest.raises(TypeError):
raise exception
workflow = Workflow()
workflow.add_operation(sum_op)
results = workflow.execute(except_slot=handle_exception).result()
print(results)
def test_execute_operation_default_input_value(self):
@operation
@output_names("doubled")
def double_op(x=10):
return x * 2
workflow = Workflow()
workflow.add_operation(double_op)
results = workflow.execute().result()
assert results == ({"doubled": 20}, )
def test_notify(self):
self.flag = False
def observer():
self.flag = not self.flag
workflow = Workflow()
workflow.attach(observer)
workflow.notify()
assert self.flag is True
def test_no_output_names(self):
@operation
def my_func(a1, a2):
return a1, a2
op = my_func()
w = Workflow()
w.add_operations(op)
result = w.execute_synchronous(a1=1, a2=2)
assert result == ({"my_func": 1}, )
def test_SAXSWorkflow():
# create processes
thresholdmask = ThresholdMaskPlugin()
qintegrate = QIntegratePlugin()
# set values
AI = AzimuthalIntegrator(.283,
5.24e-3,
4.085e-3,
0,
0,
0,
1.72e-4,
1.72e-4,
detector=Pilatus2M(),
wavelength=1.23984e-10)
thresholdmask.data.value = fabio.open(
'/Users/hari/Downloads/AGB_5S_USE_2_2m.edf').data
def AI_func():
from pyFAI.detectors import Pilatus2M
from pyFAI import AzimuthalIntegrator, units
return AzimuthalIntegrator(.283,
5.24e-3,
4.085e-3,
0,
0,
0,
1.72e-4,
1.72e-4,
detector=Pilatus2M(),
wavelength=1.23984e-10)
qintegrate.integrator.value = AI_func
qintegrate.npt.value = 1000
thresholdmask.minimum.value = 30
thresholdmask.maximum.value = 1e12
qintegrate.data.value = fabio.open(
'/Users/hari/Downloads/AGB_5S_USE_2_2m.edf').data
thresholdmask.neighborhood.value = 1
qintegrate.normalization_factor.value = 0.5
qintegrate.method.value = "numpy"
# connect processes
thresholdmask.mask.connect(qintegrate.mask)
# add processes to workflow
wf = Workflow('QIntegrate')
wf.addProcess(thresholdmask)
wf.addProcess(qintegrate)
dsk = DaskExecutor()
result = dsk.execute(wf)
print(result)
def test_one_output_name(self):
@operation
@output_names("return_val")
def my_func(a1, a2):
return a1, a2
op = my_func()
workflow = Workflow()
workflow.add_operations(op)
result = workflow.execute_synchronous(a1=1, a2=2)
assert result == ({"return_val": 1}, )
def test_execute_synchronous_no_links(self, sum_op, square_op, negative_op):
workflow = Workflow()
workflow.add_operations(sum_op, square_op, negative_op)
# n1, n2 -- inputs to sum_op;
# n -- input to square_op
# num -- input to negative_op
results = workflow.execute_synchronous(n1=2, n2=5, n=10, num=50)
assert len(results) == 3
assert {"sum": 7} in results
assert {"square": 100} in results
assert {"negative": -50} in results
def test_all_output_names(self):
@operation
@output_names('x', 'y')
def my_func(a1, a2):
return a1, a2
op = my_func()
workflow = Workflow()
workflow.add_operations(op)
result = workflow.execute_synchronous(a1=1, a2=2)
print(result)
assert result == ({'x': 1, 'y': 2}, )
def test_two_copy_ops_to_one_op(self, double_and_triple_op, sum_op, square_op):
# 1**2 + 2**2 => 5
workflow = Workflow()
square_op.filled_values.update(n=1)
square_op_2 = square_op.__class__()
square_op_2.filled_values.update(n=2)
square_op_2.output_names = ["square"]
workflow.add_operations(sum_op, square_op, square_op_2)
workflow.add_link(square_op, sum_op, "square", "n1")
workflow.add_link(square_op_2, sum_op, "square", "n2")
workflow._pretty_print()
result = workflow.execute_synchronous()
assert result == ({"sum": 5},)
def test_default(self, op, old):
w = Workflow(operations=[op])
result = w.execute_synchronous(executor=executor)
corrected_images = result[0]['corrected_images']
gains = get_default(op, 'gains')
assert np.array_equal(corrected_images,
op.filled_values['images'] * gains[0])
if TEST_CSX_TOOLS:
w.clear_operations()
w.add_operation(old)
result = w.execute_synchronous(executor=executor)
assert np.array_equal(result[0]['corrected_images'],
corrected_images)
def simple_workflow_with_intents(plot_op, abs_plot_op, blur_image_op,
image_op):
wf = Workflow()
wf.add_operation(image_op)
wf.add_operation(blur_image_op)
wf.add_link(image_op, blur_image_op, "output_array", "arr")
wf.add_operation(plot_op)
wf.add_operation(abs_plot_op)
wf.add_link(plot_op, abs_plot_op, "output1", "x_arr")
wf.add_link(plot_op, abs_plot_op, "output2", "y_arr")
return wf
def test_execute_all(self, qtbot, sum_op, square_op, negative_op):
results = [{"negative": (1 + 2) ** 2 * -1},
{"negative": (3 + 4) ** 2 * -1},
{"negative": (5 + 6) ** 2 * -1}]
def cb(*result):
next_result = results.pop(0)
assert result == next_result
workflow = Workflow()
workflow.add_operations(sum_op, square_op, negative_op)
workflow.add_link(sum_op, square_op, "sum", "n")
workflow.add_link(square_op, negative_op, "square", "num")
n1_values = [1, 3, 5]
n2_values = [2, 4, 6]
# TODO -- we are only getting one result, should get three (3 pairs of n1/n2).
workflow.execute_all(callback_slot=cb, n1=n1_values, n2=n2_values).result()
def simple_workflow(square_op, sum_op):
from xicam.core.execution.workflow import Workflow
wf = Workflow()
square = square_op
square2 = square_op.__class__()
square2.filled_values["n"] = 2
wf.add_operation(square)
wf.add_operation(square2)
wf.add_operation(sum_op)
wf.add_link(square, sum_op, "square", "n1")
wf.add_link(square2, sum_op, "square", "n2")
return wf
def test_two_ops_to_one_op(self, negative_op, square_op, sum_op):
workflow = Workflow()
workflow.add_operations(negative_op, square_op, sum_op)
workflow.add_link(negative_op, sum_op, "negative", "n1")
workflow.add_link(square_op, sum_op, "square", "n2")
print(workflow.get_inbound_links(sum_op))
#from dask import visualize
#visualize(workflow.as_dask_graph()[0], filename="/home/ihumphrey/graph")
graph = workflow.as_dask_graph()[0]
print(graph)
for k, op in graph.items():
print(k, op[0].node.name)
from dask.threaded import get
negative_op.filled_values.update(num=3)
square_op.filled_values.update(n=4)
print(get(graph, '0')) # WHY is this an issue? Complains that sum missing required 'n2'
def test_execute_no_links(self):
@operation
@output_names("doubled")
def double_op(n):
return n * 2
@operation
@output_names("tripled")
def triple_op(n):
return n * 3
workflow = Workflow()
workflow.add_operations(double_op, triple_op)
results = workflow.execute(n=10).result()
assert len(results) == 2
assert {"doubled": 20} in results
assert {"tripled": 30} in results
def test_execute_all(self, sum_op, square_op, negative_op):
workflow = Workflow()
workflow.add_operations(sum_op, square_op, negative_op)
workflow.add_link(sum_op, square_op, "sum", "n")
workflow.add_link(square_op, negative_op, "square", "num")
n1_values = [1, 3, 5]
n2_values = [2, 4, 6]
# TODO -- we are only getting one result, should get three (3 pairs of n1/n2).
results = list(
workflow.execute_all(n1=n1_values, n2=n2_values).result())
assert results == [{
"negative": (1 + 2)**2 * -1
}, {
"negative": (3 + 4)**2 * -1
}, {
"negative": (5 + 6)**2 * -1
}]
def test_multiple_instances(square_op, sum_op):
from xicam.core.execution.workflow import Workflow
from xicam.core.execution.daskexecutor import DaskExecutor
executor = DaskExecutor()
wf = Workflow()
square = square_op()
square2 = square_op()
square2.filled_values["a"] = 2
my_sum = sum_op()
wf.add_operation(square)
wf.add_operation(square2)
wf.add_operation(my_sum)
wf.add_link(square, my_sum, "square", "a") # 3**3
wf.add_link(square2, my_sum, "square", "b") # 2**2
assert wf.execute_synchronous(executor=executor) == [{"sum": 13}]
def test_multiple_instances(square_op, sum_op):
from xicam.core.execution.workflow import Workflow
from xicam.core.execution.daskexecutor import DaskExecutor
executor = DaskExecutor()
wf = Workflow()
square = square_op()
square2 = square_op()
square2.filled_values['a'] = 2
my_sum = sum_op()
wf.add_operation(square)
wf.add_operation(square2)
wf.add_operation(my_sum)
wf.add_link(square, my_sum, 'square', 'a')
wf.add_link(square2, my_sum, 'square', 'b')
assert wf.execute_synchronous(executor=executor) == [{'sum': 13}]
def test_notify_no_observers(self):
workflow = Workflow()
workflow.notify()