def test_load_operator():
# function_bytes, module_name, class_name,
descriptor_func_bytes = gateway_client.serialize(
[None, __name__, MapFunc.__name__, "MapFunction"])
descriptor_op_bytes = gateway_client.serialize(
[descriptor_func_bytes, "", ""])
map_operator = operator.load_operator(descriptor_op_bytes)
assert type(map_operator) is operator.MapOperator
descriptor_op_bytes = gateway_client.serialize(
[None, __name__, EmptyOperator.__name__])
test_operator = operator.load_operator(descriptor_op_bytes)
assert isinstance(test_operator, EmptyOperator)
def __init__(self, execution_vertex_pb):
self.execution_vertex_id = execution_vertex_pb.execution_vertex_id
self.execution_job_vertex_id = execution_vertex_pb \
.execution_job_vertex_id
self.execution_job_vertex_name = execution_vertex_pb \
.execution_job_vertex_name
self.execution_vertex_index = execution_vertex_pb\
.execution_vertex_index
self.parallelism = execution_vertex_pb.parallelism
if execution_vertex_pb\
.language == Language.PYTHON:
# python operator descriptor
operator_bytes = execution_vertex_pb.operator
if execution_vertex_pb.chained:
logger.info("Load chained operator")
self.stream_operator = operator.load_chained_operator(
operator_bytes)
else:
logger.info("Load operator")
self.stream_operator = operator.load_operator(operator_bytes)
self.worker_actor = None
if execution_vertex_pb.worker_actor:
self.worker_actor = ray.actor.ActorHandle. \
_deserialization_helper(execution_vertex_pb.worker_actor)
self.container_id = execution_vertex_pb.container_id
self.build_time = execution_vertex_pb.build_time
self.language = execution_vertex_pb.language
self.config = execution_vertex_pb.config
self.resource = execution_vertex_pb.resource
def __init__(self, vertex_pb):
self.execution_vertex_id = vertex_pb.execution_vertex_id
self.execution_job_vertex_Id = vertex_pb.execution_job_vertex_Id
self.execution_job_vertex_name = vertex_pb.execution_job_vertex_name
self.execution_vertex_index = vertex_pb.execution_vertex_index
self.parallelism = vertex_pb.parallelism
if vertex_pb.language == Language.PYTHON:
operator_bytes = vertex_pb.operator # python operator descriptor
self.stream_operator = operator.load_operator(operator_bytes)
self.worker_actor = ray.actor.ActorHandle. \
_deserialization_helper(vertex_pb.worker_actor)
self.container_id = vertex_pb.container_id
self.build_time = vertex_pb.build_time
self.language = vertex_pb.language
self.config = vertex_pb.config
self.resource = vertex_pb.resource
def __init__(self, node_pb):
self.node_id = node_pb.node_id
self.node_type = NodeType[streaming_pb.NodeType.Name(
node_pb.node_type)]
self.parallelism = node_pb.parallelism
if node_pb.language == Language.PYTHON:
operator_bytes = node_pb.operator # python operator descriptor
self.stream_operator = operator.load_operator(operator_bytes)
self.execution_tasks = [
ExecutionTask(task) for task in node_pb.execution_tasks
]
self.input_edges = [
ExecutionEdge(edge, node_pb.language)
for edge in node_pb.input_edges
]
self.output_edges = [
ExecutionEdge(edge, node_pb.language)
for edge in node_pb.output_edges
]