예제 #1
0
    def init(self, worker_context_bytes):
        worker_context = remote_call_pb.PythonJobWorkerContext()
        worker_context.ParseFromString(worker_context_bytes)
        self.worker_context = worker_context

        # build vertex context from pb
        self.execution_vertex_context = ExecutionVertexContext(
            worker_context.execution_vertex_context)

        # use vertex id as task id
        self.task_id = self.execution_vertex_context.get_task_id()

        # build and get processor from operator
        operator = self.execution_vertex_context.stream_operator
        self.stream_processor = processor.build_processor(operator)
        logger.info(
            "Initializing job worker, task_id: {}, operator: {}.".format(
                self.task_id, self.stream_processor))

        # get config from vertex
        self.config = self.execution_vertex_context.config

        if self.config.get(Config.CHANNEL_TYPE, Config.NATIVE_CHANNEL):
            self.reader_client = _streaming.ReaderClient()
            self.writer_client = _streaming.WriterClient()

        self.task = self.create_stream_task()

        logger.info("Job worker init succeeded.")
        return True
예제 #2
0
    def init(self, env):
        """init streaming actor"""
        env = pickle.loads(env)
        self.env = env
        logger.info("init operator instance %s", self.processor_name)

        if env.config.channel_type == Config.NATIVE_CHANNEL:
            core_worker = ray.worker.global_worker.core_worker
            reader_async_func = FunctionDescriptor(
                __name__, self.on_reader_message.__name__,
                self.__class__.__name__)
            reader_sync_func = FunctionDescriptor(
                __name__, self.on_reader_message_sync.__name__,
                self.__class__.__name__)
            self.reader_client = _streaming.ReaderClient(
                core_worker, reader_async_func, reader_sync_func)
            writer_async_func = FunctionDescriptor(
                __name__, self.on_writer_message.__name__,
                self.__class__.__name__)
            writer_sync_func = FunctionDescriptor(
                __name__, self.on_writer_message_sync.__name__,
                self.__class__.__name__)
            self.writer_client = _streaming.WriterClient(
                core_worker, writer_async_func, writer_sync_func)
        if len(self.input_channels) > 0:
            self.input_gate = DataInput(env, self.input_channels)
            self.input_gate.init()
        if len(self.output_channels) > 0:
            self.output_gate = DataOutput(
                env, self.output_channels,
                self.operator.partitioning_strategies)
            self.output_gate.init()
        logger.info("init operator instance %s succeed", self.processor_name)
        return True
예제 #3
0
파일: worker.py 프로젝트: yuishihara/ray
    def init(self, worker_context_bytes):
        worker_context = remote_call_pb.WorkerContext()
        worker_context.ParseFromString(worker_context_bytes)
        self.worker_context = worker_context
        self.task_id = worker_context.task_id
        self.config = worker_context.conf
        execution_graph = ExecutionGraph(worker_context.graph)
        self.execution_graph = execution_graph
        self.execution_task = self.execution_graph. \
            get_execution_task_by_task_id(self.task_id)
        self.execution_node = self.execution_graph. \
            get_execution_node_by_task_id(self.task_id)
        operator = self.execution_node.stream_operator
        self.stream_processor = processor.build_processor(operator)
        logger.info(
            "Initializing JobWorker, task_id: {}, operator: {}.".format(
                self.task_id, self.stream_processor))

        if self.config.get(Config.CHANNEL_TYPE, Config.NATIVE_CHANNEL):
            self.reader_client = _streaming.ReaderClient()
            self.writer_client = _streaming.WriterClient()

        self.task = self.create_stream_task()
        self.task.start()
        logger.info("JobWorker init succeed")
        return True
예제 #4
0
 def __init__(self):
     writer_async_func = PythonFunctionDescriptor(
         __name__, self.on_writer_message.__name__, self.__class__.__name__)
     writer_sync_func = PythonFunctionDescriptor(
         __name__, self.on_writer_message_sync.__name__,
         self.__class__.__name__)
     self.writer_client = _streaming.WriterClient(writer_async_func,
                                                  writer_sync_func)
     reader_async_func = PythonFunctionDescriptor(
         __name__, self.on_reader_message.__name__, self.__class__.__name__)
     reader_sync_func = PythonFunctionDescriptor(
         __name__, self.on_reader_message_sync.__name__,
         self.__class__.__name__)
     self.reader_client = _streaming.ReaderClient(reader_async_func,
                                                  reader_sync_func)
     self.writer = None
     self.output_channel_id = None
     self.reader = None
예제 #5
0
    def init(self, worker_context_bytes):
        logger.info("Start to init job worker")
        try:
            # deserialize context
            worker_context = remote_call_pb2.PythonJobWorkerContext()
            worker_context.ParseFromString(worker_context_bytes)
            self.worker_context = worker_context
            self.master_actor = ActorHandle._deserialization_helper(
                worker_context.master_actor)

            # build vertex context from pb
            self.execution_vertex_context = ExecutionVertexContext(
                worker_context.execution_vertex_context)
            self.execution_vertex = self\
                .execution_vertex_context.execution_vertex

            # save context
            job_worker_context_key = self.__get_job_worker_context_key()
            self.context_backend.put(job_worker_context_key,
                                     worker_context_bytes)

            # use vertex id as task id
            self.task_id = self.execution_vertex_context.get_task_id()
            # build and get processor from operator
            operator = self.execution_vertex_context.stream_operator
            self.stream_processor = processor.build_processor(operator)
            logger.info("Initializing job worker, exe_vertex_name={},"
                        "task_id: {}, operator: {}, pid={}".format(
                            self.execution_vertex_context.exe_vertex_name,
                            self.task_id, self.stream_processor, os.getpid()))

            # get config from vertex
            self.config = self.execution_vertex_context.config

            if self.config.get(Config.CHANNEL_TYPE, Config.NATIVE_CHANNEL):
                self.reader_client = _streaming.ReaderClient()
                self.writer_client = _streaming.WriterClient()

            logger.info("Job worker init succeeded.")
        except Exception:
            logger.exception("Error when init job worker.")
            return False
        return True
예제 #6
0
    def init(self, worker_context_bytes):
        worker_context = remote_call_pb.WorkerContext()
        worker_context.ParseFromString(worker_context_bytes)
        self.worker_context = worker_context
        self.task_id = worker_context.task_id
        self.config = worker_context.conf
        execution_graph = ExecutionGraph(worker_context.graph)
        self.execution_graph = execution_graph
        self.execution_task = self.execution_graph. \
            get_execution_task_by_task_id(self.task_id)
        self.execution_node = self.execution_graph. \
            get_execution_node_by_task_id(self.task_id)
        operator = self.execution_node.stream_operator
        self.stream_processor = processor.build_processor(operator)
        logger.info(
            "Initializing JobWorker, task_id: {}, operator: {}.".format(
                self.task_id, self.stream_processor))

        if self.config.get(Config.CHANNEL_TYPE, Config.NATIVE_CHANNEL):
            core_worker = ray.worker.global_worker.core_worker
            reader_async_func = PythonFunctionDescriptor(
                __name__, self.on_reader_message.__name__,
                self.__class__.__name__)
            reader_sync_func = PythonFunctionDescriptor(
                __name__, self.on_reader_message_sync.__name__,
                self.__class__.__name__)
            self.reader_client = _streaming.ReaderClient(
                core_worker, reader_async_func, reader_sync_func)
            writer_async_func = PythonFunctionDescriptor(
                __name__, self.on_writer_message.__name__,
                self.__class__.__name__)
            writer_sync_func = PythonFunctionDescriptor(
                __name__, self.on_writer_message_sync.__name__,
                self.__class__.__name__)
            self.writer_client = _streaming.WriterClient(
                core_worker, writer_async_func, writer_sync_func)

        self.task = self.create_stream_task()
        self.task.start()
        logger.info("JobWorker init succeed")
        return True
예제 #7
0
 def __init__(self):
     core_worker = ray.worker.global_worker.core_worker
     writer_async_func = FunctionDescriptor(__name__,
                                            self.on_writer_message.__name__,
                                            self.__class__.__name__)
     writer_sync_func = FunctionDescriptor(
         __name__, self.on_writer_message_sync.__name__,
         self.__class__.__name__)
     self.writer_client = _streaming.WriterClient(core_worker,
                                                  writer_async_func,
                                                  writer_sync_func)
     reader_async_func = FunctionDescriptor(__name__,
                                            self.on_reader_message.__name__,
                                            self.__class__.__name__)
     reader_sync_func = FunctionDescriptor(
         __name__, self.on_reader_message_sync.__name__,
         self.__class__.__name__)
     self.reader_client = _streaming.ReaderClient(core_worker,
                                                  reader_async_func,
                                                  reader_sync_func)
     self.writer = None
     self.output_channel_id = None
     self.reader = None
 def __init__(self):
     self.writer_client = _streaming.WriterClient()
     self.reader_client = _streaming.ReaderClient()
     self.writer = None
     self.output_channel_id = None
     self.reader = None