def load_function(descriptor_func_bytes: bytes):
"""
Deserialize `descriptor_func_bytes` to get function info, then
get or load streaming function.
Note that this function must be kept in sync with
`io.ray.streaming.runtime.python.GraphPbBuilder.serializeFunction`
Args:
descriptor_func_bytes: serialized function info
Returns:
a streaming function
"""
assert len(descriptor_func_bytes) > 0
function_bytes, module_name, function_name, function_interface \
= gateway_client.deserialize(descriptor_func_bytes)
if function_bytes:
return deserialize(function_bytes)
else:
assert module_name
assert function_interface
function_interface = getattr(sys.modules[__name__], function_interface)
mod = importlib.import_module(module_name)
assert function_name
func = getattr(mod, function_name)
# If func is a python function, user function is a simple python
# function, which will be wrapped as a SimpleXXXFunction.
# If func is a python class, user function is a sub class
# of XXXFunction.
if inspect.isfunction(func):
simple_func_class = _get_simple_function_class(function_interface)
return simple_func_class(func)
else:
assert issubclass(func, function_interface)
return func()
def load_function(descriptor_func_bytes: bytes):
"""
Deserialize `descriptor_func_bytes` to get function info, then
get or load streaming function.
Note that this function must be kept in sync with
`io.ray.streaming.runtime.python.GraphPbBuilder.serializeFunction`
Args:
descriptor_func_bytes: serialized function info
Returns:
a streaming function
"""
function_bytes, module_name, class_name, function_name, function_interface\
= gateway_client.deserialize(descriptor_func_bytes)
if function_bytes:
return deserialize(function_bytes)
else:
assert module_name
assert function_interface
function_interface = getattr(sys.modules[__name__], function_interface)
mod = importlib.import_module(module_name)
if class_name:
assert function_name is None
cls = getattr(mod, class_name)
assert issubclass(cls, function_interface)
return cls()
else:
assert function_name
func = getattr(mod, function_name)
simple_func_class = _get_simple_function_class(function_interface)
return simple_func_class(func)
def load_operator(descriptor_operator_bytes: bytes):
"""
Deserialize `descriptor_operator_bytes` to get operator info, then
create streaming operator.
Note that this function must be kept in sync with
`io.ray.streaming.runtime.python.GraphPbBuilder.serializeOperator`
Args:
descriptor_operator_bytes: serialized operator info
Returns:
a streaming operator
"""
assert len(descriptor_operator_bytes) > 0
function_desc_bytes, module_name, class_name \
= gateway_client.deserialize(descriptor_operator_bytes)
if function_desc_bytes:
return create_operator_with_func(
function.load_function(function_desc_bytes))
else:
assert module_name
assert class_name
mod = importlib.import_module(module_name)
cls = getattr(mod, class_name)
assert issubclass(cls, Operator)
print("cls", cls)
return cls()
def load_partition(descriptor_partition_bytes: bytes):
"""
Deserialize `descriptor_partition_bytes` to get partition info, then
get or load partition function.
Note that this function must be kept in sync with
`io.ray.streaming.runtime.python.GraphPbBuilder.serializePartition`
Args:
descriptor_partition_bytes: serialized partition info
Returns:
partition function
"""
partition_bytes, module_name, class_name, function_name =\
gateway_client.deserialize(descriptor_partition_bytes)
if partition_bytes:
return deserialize(partition_bytes)
else:
assert module_name
mod = importlib.import_module(module_name)
# If class_name is not None, user partition is a sub class
# of Partition.
# If function_name is not None, user partition is a simple python
# function, which will be wrapped as a SimplePartition.
if class_name:
assert function_name is None
cls = getattr(mod, class_name)
return cls()
else:
assert function_name
func = getattr(mod, function_name)
return SimplePartition(func)
def load_chained_operator(chained_operator_bytes: bytes):
"""Load chained operator from serialized operators and configs"""
serialized_operators, configs = gateway_client.deserialize(
chained_operator_bytes)
operators = [
load_operator(desc_bytes) for desc_bytes in serialized_operators
]
return ChainedOperator.new_chained_operator(operators, configs)
