def test_multi_process_transparent_api_create_on_local_remote_ith_address_workflow(
):
files = examples.download_distributed_files()
wf = core.Workflow()
op = ops.result.displacement()
average = core.operators.math.norm_fc(op)
wf.add_operators([op, average])
wf.set_output_name("distrib", average.outputs.fields_container)
wf.set_input_name("ds", op.inputs.data_sources)
local_wf = core.Workflow()
merge = ops.utility.merge_fields_containers()
min_max = ops.min_max.min_max_fc(merge)
local_wf.add_operator(merge)
local_wf.add_operator(min_max)
local_wf.set_output_name("tot_output", min_max.outputs.field_max)
for i in files:
data_sources1 = core.DataSources(files[i])
remote_wf = wf.create_on_other_server(ip=local_servers[i].ip,
port=local_servers[i].port)
remote_wf.connect("ds", data_sources1)
local_wf.set_input_name("distrib" + str(i), merge, i)
local_wf.connect_with(remote_wf, ("distrib", "distrib" + str(i)))
max = local_wf.get_output("tot_output", core.types.field)
assert np.allclose(max.data, [10.03242272])
def test_multi_process_transparent_api_connect_local_op_remote_workflow():
files = examples.download_distributed_files()
workflows = []
for i in files:
wf = core.Workflow(server=local_servers[i])
op = ops.result.displacement(server=local_servers[i])
average = core.operators.math.norm_fc(op, server=local_servers[i])
wf.add_operators([op, average])
wf.set_output_name("distrib" + str(i),
average.outputs.fields_container)
wf.set_input_name("ds", op.inputs.data_sources)
workflows.append(wf)
local_wf = core.Workflow()
merge = ops.utility.merge_fields_containers()
min_max = ops.min_max.min_max_fc(merge)
local_wf.add_operator(merge)
local_wf.add_operator(min_max)
local_wf.set_output_name("tot_output", min_max.outputs.field_max)
for i, wf in enumerate(workflows):
data_sources1 = core.DataSources(files[i])
forward = ops.utility.forward(data_sources1)
wf.connect("ds", forward, 0)
local_wf.set_input_name("distrib" + str(i), merge, i)
local_wf.connect_with(wf)
max = local_wf.get_output("tot_output", core.types.field)
assert np.allclose(max.data, [10.03242272])
def test_simple_remote_workflow(simple_bar, local_server):
data_sources1 = core.DataSources(simple_bar)
wf = core.Workflow()
op = ops.result.displacement(data_sources=data_sources1)
average = core.operators.math.norm_fc(op)
wf.add_operators([op, average])
wf.set_output_name("out", average.outputs.fields_container)
local_wf = core.Workflow()
min_max = ops.min_max.min_max_fc()
local_wf.add_operator(min_max)
local_wf.set_input_name("in", min_max.inputs.fields_container)
local_wf.set_output_name("tot_output", min_max.outputs.field_max)
grpc_stream_provider = ops.metadata.streams_provider()
grpc_data_sources = core.DataSources()
grpc_data_sources.set_result_file_path(
local_server.ip + ":" + str(local_server.port), "grpc")
grpc_stream_provider.inputs.data_sources(grpc_data_sources)
remote_workflow_prov = core.Operator("remote_workflow_instantiate")
remote_workflow_prov.connect(3, grpc_stream_provider, 0)
remote_workflow_prov.connect(0, wf)
remote_workflow = remote_workflow_prov.get_output(0, core.types.workflow)
local_wf.connect_with(remote_workflow, ("out", "in"))
max = local_wf.get_output("tot_output", core.types.field)
assert np.allclose(max.data, [2.52368345e-05])
def test_multi_process_chain_remote_workflow():
files = examples.download_distributed_files()
wf = core.Workflow()
op = ops.result.displacement()
average = core.operators.math.norm_fc(op)
wf.add_operators([op, average])
wf.set_input_name("data_sources", op.inputs.data_sources)
wf.set_output_name("distrib", average.outputs.fields_container)
workflows = []
for i in files:
data_sources1 = core.DataSources(files[i])
grpc_stream_provider = ops.metadata.streams_provider()
grpc_data_sources = core.DataSources()
grpc_data_sources.set_result_file_path(
local_servers[i].ip + ":" + str(local_servers[i].port), "grpc")
grpc_stream_provider.inputs.data_sources(grpc_data_sources)
remote_workflow_prov = core.Operator("remote_workflow_instantiate")
remote_workflow_prov.connect(3, grpc_stream_provider, 0)
remote_workflow_prov.connect(0, wf)
remote_workflow = remote_workflow_prov.get_output(
0, core.types.workflow)
remote_workflow.connect("data_sources", data_sources1)
workflows.append(remote_workflow)
local_wf = core.Workflow()
merge = ops.utility.merge_fields_containers()
min_max = ops.min_max.min_max_fc(merge)
local_wf.add_operator(merge)
local_wf.add_operator(min_max)
local_wf.set_output_name("tot_output", min_max.outputs.field_max)
for i, wf in enumerate(workflows):
local_wf.set_input_name("distrib" + str(i), merge, i)
grpc_stream_provider = ops.metadata.streams_provider()
grpc_data_sources = core.DataSources()
grpc_data_sources.set_result_file_path(
local_servers[2].ip + ":" + str(local_servers[2].port), "grpc")
grpc_stream_provider.inputs.data_sources(grpc_data_sources)
remote_workflow_prov = core.Operator("remote_workflow_instantiate")
remote_workflow_prov.connect(3, grpc_stream_provider, 0)
remote_workflow_prov.connect(0, local_wf)
remote_workflow = remote_workflow_prov.get_output(0, core.types.workflow)
for i, wf in enumerate(workflows):
remote_workflow.connect_with(wf, ("distrib", "distrib" + str(i)))
max = remote_workflow.get_output("tot_output", core.types.field)
assert np.allclose(max.data, [10.03242272])
def test_remote_workflow_info(local_server):
wf = core.Workflow()
op = ops.result.displacement()
average = core.operators.math.norm_fc(op)
wf.add_operators([op, average])
wf.set_input_name("data_sources", op.inputs.data_sources)
wf.set_output_name("distrib", average.outputs.fields_container)
grpc_stream_provider = ops.metadata.streams_provider()
grpc_data_sources = core.DataSources()
grpc_data_sources.set_result_file_path(
local_server.ip + ":" + str(local_server.port), "grpc")
grpc_stream_provider.inputs.data_sources(grpc_data_sources)
remote_workflow_prov = core.Operator("remote_workflow_instantiate")
remote_workflow_prov.connect(3, grpc_stream_provider, 0)
remote_workflow_prov.connect(0, wf)
remote_workflow = remote_workflow_prov.get_output(0, core.types.workflow)
assert "data_sources" in remote_workflow.input_names
assert "distrib" in remote_workflow.output_names
