def test_load_function():
# function_bytes, module_name, function_name/class_name,
# function_interface
descriptor_func_bytes = gateway_client.serialize(
[None, __name__, MapFunc.__name__, "MapFunction"])
func = function.load_function(descriptor_func_bytes)
assert type(func) is MapFunc
def load_operator(descriptor_operator_bytes: bytes):
"""
Deserialize `descriptor_operator_bytes` to get operator info, then
create streaming operator.
Note that this function must be kept in sync with
`io.ray.streaming.runtime.python.GraphPbBuilder.serializeOperator`
Args:
descriptor_operator_bytes: serialized operator info
Returns:
a streaming operator
"""
assert len(descriptor_operator_bytes) > 0
function_desc_bytes, module_name, class_name \
= gateway_client.deserialize(descriptor_operator_bytes)
if function_desc_bytes:
return create_operator_with_func(
function.load_function(function_desc_bytes))
else:
assert module_name
assert class_name
mod = importlib.import_module(module_name)
cls = getattr(mod, class_name)
assert issubclass(cls, Operator)
print("cls", cls)
return cls()
def __init__(self, node_pb):
self.node_id = node_pb.node_id
self.node_type = NodeType[streaming_pb.NodeType.Name(
node_pb.node_type)]
self.parallelism = node_pb.parallelism
if node_pb.language == Language.PYTHON:
func_bytes = node_pb.function # python function descriptor
func = function.load_function(func_bytes)
self.stream_operator = operator.create_operator(func)
self.execution_tasks = [
ExecutionTask(task) for task in node_pb.execution_tasks
]
self.input_edges = [
ExecutionEdge(edge, node_pb.language)
for edge in node_pb.input_edges
]
self.output_edges = [
ExecutionEdge(edge, node_pb.language)
for edge in node_pb.output_edges
]