def load_function(descriptor_func_bytes: bytes): """ Deserialize `descriptor_func_bytes` to get function info, then get or load streaming function. Note that this function must be kept in sync with `io.ray.streaming.runtime.python.GraphPbBuilder.serializeFunction` Args: descriptor_func_bytes: serialized function info Returns: a streaming function """ assert len(descriptor_func_bytes) > 0 function_bytes, module_name, function_name, function_interface \ = gateway_client.deserialize(descriptor_func_bytes) if function_bytes: return deserialize(function_bytes) else: assert module_name assert function_interface function_interface = getattr(sys.modules[__name__], function_interface) mod = importlib.import_module(module_name) assert function_name func = getattr(mod, function_name) # If func is a python function, user function is a simple python # function, which will be wrapped as a SimpleXXXFunction. # If func is a python class, user function is a sub class # of XXXFunction. if inspect.isfunction(func): simple_func_class = _get_simple_function_class(function_interface) return simple_func_class(func) else: assert issubclass(func, function_interface) return func()
def load_function(descriptor_func_bytes: bytes): """ Deserialize `descriptor_func_bytes` to get function info, then get or load streaming function. Note that this function must be kept in sync with `io.ray.streaming.runtime.python.GraphPbBuilder.serializeFunction` Args: descriptor_func_bytes: serialized function info Returns: a streaming function """ function_bytes, module_name, class_name, function_name, function_interface\ = gateway_client.deserialize(descriptor_func_bytes) if function_bytes: return deserialize(function_bytes) else: assert module_name assert function_interface function_interface = getattr(sys.modules[__name__], function_interface) mod = importlib.import_module(module_name) if class_name: assert function_name is None cls = getattr(mod, class_name) assert issubclass(cls, function_interface) return cls() else: assert function_name func = getattr(mod, function_name) simple_func_class = _get_simple_function_class(function_interface) return simple_func_class(func)
def load_operator(descriptor_operator_bytes: bytes): """ Deserialize `descriptor_operator_bytes` to get operator info, then create streaming operator. Note that this function must be kept in sync with `io.ray.streaming.runtime.python.GraphPbBuilder.serializeOperator` Args: descriptor_operator_bytes: serialized operator info Returns: a streaming operator """ assert len(descriptor_operator_bytes) > 0 function_desc_bytes, module_name, class_name \ = gateway_client.deserialize(descriptor_operator_bytes) if function_desc_bytes: return create_operator_with_func( function.load_function(function_desc_bytes)) else: assert module_name assert class_name mod = importlib.import_module(module_name) cls = getattr(mod, class_name) assert issubclass(cls, Operator) print("cls", cls) return cls()
def load_partition(descriptor_partition_bytes: bytes): """ Deserialize `descriptor_partition_bytes` to get partition info, then get or load partition function. Note that this function must be kept in sync with `io.ray.streaming.runtime.python.GraphPbBuilder.serializePartition` Args: descriptor_partition_bytes: serialized partition info Returns: partition function """ partition_bytes, module_name, class_name, function_name =\ gateway_client.deserialize(descriptor_partition_bytes) if partition_bytes: return deserialize(partition_bytes) else: assert module_name mod = importlib.import_module(module_name) # If class_name is not None, user partition is a sub class # of Partition. # If function_name is not None, user partition is a simple python # function, which will be wrapped as a SimplePartition. if class_name: assert function_name is None cls = getattr(mod, class_name) return cls() else: assert function_name func = getattr(mod, function_name) return SimplePartition(func)
def load_chained_operator(chained_operator_bytes: bytes): """Load chained operator from serialized operators and configs""" serialized_operators, configs = gateway_client.deserialize( chained_operator_bytes) operators = [ load_operator(desc_bytes) for desc_bytes in serialized_operators ] return ChainedOperator.new_chained_operator(operators, configs)