Ejemplo n.º 1
0
    def __init__(self, pplan_helper, in_stream, out_stream, looper,
                 sys_config):
        super(SpoutInstance, self).__init__(pplan_helper, in_stream,
                                            out_stream, looper, sys_config)
        self.topology_state = topology_pb2.TopologyState.Value("PAUSED")

        if not self.pplan_helper.is_spout:
            raise RuntimeError("No spout in physicial plan")

        context = self.pplan_helper.context
        self.spout_metrics = SpoutMetrics(self.pplan_helper)
        self.serializer = SerializerHelper.get_serializer(context)

        # acking related
        self.acking_enabled = context.get_cluster_config().get(
            constants.TOPOLOGY_ENABLE_ACKING, False)
        self.enable_message_timeouts = \
          context.get_cluster_config().get(constants.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS)
        Log.info("Enable ACK: %s" % str(self.acking_enabled))
        Log.info("Enable Message Timeouts: %s" %
                 str(self.enable_message_timeouts))

        # map <tuple_info.key -> tuple_info>, ordered by insertion time
        self.in_flight_tuples = collections.OrderedDict()
        self.immediate_acks = collections.deque()
        self.total_tuples_emitted = 0

        # load user's spout class
        spout_impl_class = super(SpoutInstance,
                                 self).load_py_instance(is_spout=True)
        self.spout_impl = spout_impl_class(delegate=self)
Ejemplo n.º 2
0
  def __init__(self, pplan_helper, in_stream, out_stream, looper):
    super(SpoutInstance, self).__init__(pplan_helper, in_stream, out_stream, looper)
    self.topology_state = topology_pb2.TopologyState.Value("PAUSED")

    if not self.pplan_helper.is_spout:
      raise RuntimeError("No spout in physicial plan")

    context = self.pplan_helper.context
    self.spout_metrics = SpoutMetrics(self.pplan_helper)
    self.serializer = SerializerHelper.get_serializer(context)

    # acking related
    self.acking_enabled = context.get_cluster_config().get(api_constants.TOPOLOGY_ENABLE_ACKING,
                                                           False)
    self.enable_message_timeouts = \
      context.get_cluster_config().get(api_constants.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS)
    Log.info("Enable ACK: %s" % str(self.acking_enabled))
    Log.info("Enable Message Timeouts: %s" % str(self.enable_message_timeouts))

    # map <tuple_info.key -> tuple_info>, ordered by insertion time
    self.in_flight_tuples = collections.OrderedDict()
    self.immediate_acks = collections.deque()
    self.total_tuples_emitted = 0

    # load user's spout class
    spout_impl_class = super(SpoutInstance, self).load_py_instance(is_spout=True)
    self.spout_impl = spout_impl_class(delegate=self)
Ejemplo n.º 3
0
    def handle_assignment_msg(self, pplan_helper):
        """Called when new NewInstanceAssignmentMessage arrives

    Tells this instance to become either spout/bolt.

    :param pplan_helper: PhysicalPlanHelper class to become
    """

        self.my_pplan_helper = pplan_helper
        self.my_pplan_helper.set_topology_context(self.metrics_collector)
        self.serializer = SerializerHelper.get_serializer(pplan_helper.context)

        if pplan_helper.is_spout:
            # Starting a spout
            my_spout = pplan_helper.get_my_spout()
            Log.info("Incarnating ourselves as spout: %s with task id %s",
                     pplan_helper.my_component_name,
                     str(pplan_helper.my_task_id))

            self.in_stream. \
              register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_SPOUT_READ_QUEUE_CAPACITY])
            self.out_stream. \
              register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_SPOUT_WRITE_QUEUE_CAPACITY])

            py_spout_instance = SpoutInstance(self.my_pplan_helper,
                                              self.in_stream, self.out_stream,
                                              self.looper)
            self.my_instance = AssignedInstance(is_spout=True,
                                                protobuf=my_spout,
                                                py_class=py_spout_instance)
        else:
            # Starting a bolt
            my_bolt = pplan_helper.get_my_bolt()
            Log.info("Incarnating ourselves as bolt: %s with task id %s",
                     pplan_helper.my_component_name,
                     str(pplan_helper.my_task_id))

            self.in_stream. \
              register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_BOLT_READ_QUEUE_CAPACITY])
            self.out_stream. \
              register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_BOLT_WRITE_QUEUE_CAPACITY])

            py_bolt_instance = BoltInstance(self.my_pplan_helper,
                                            self.in_stream, self.out_stream,
                                            self.looper)
            self.my_instance = AssignedInstance(is_spout=False,
                                                protobuf=my_bolt,
                                                py_class=py_bolt_instance)

        if pplan_helper.is_topology_running():
            try:
                self.start_instance_if_possible()
            except Exception as e:
                Log.error("Error with starting bolt/spout instance: " +
                          e.message)
                Log.error(traceback.format_exc())
        else:
            Log.info("The instance is deployed in deactivated state")
Ejemplo n.º 4
0
  def __init__(self, pplan_helper, in_stream, out_stream, looper):
    self.pplan_helper = pplan_helper
    self.in_stream = in_stream
    self.output_helper = OutgoingTupleHelper(self.pplan_helper, out_stream)
    self.looper = looper
    self.sys_config = system_config.get_sys_config()

    # will set a root logger here
    self.logger = logging.getLogger()
    context = pplan_helper.context
    mode = context.get_cluster_config().get(api_constants.TOPOLOGY_RELIABILITY_MODE,
                                            api_constants.TopologyReliabilityMode.ATMOST_ONCE)
    self.is_stateful = bool(mode == api_constants.TopologyReliabilityMode.EXACTLY_ONCE)
    self._stateful_state = None
    self.serializer = SerializerHelper.get_serializer(pplan_helper.context)
    self._initialized_global_metrics = False
Ejemplo n.º 5
0
  def __init__(self, pplan_helper, in_stream, out_stream, looper, sys_config):
    super(BoltInstance, self).__init__(pplan_helper, in_stream, out_stream, looper, sys_config)

    if self.pplan_helper.is_spout:
      raise RuntimeError("No bolt in physical plan")

    # bolt_config is auto-typed, not <str -> str> only
    context = self.pplan_helper.context
    self.bolt_metrics = BoltMetrics(self.pplan_helper)
    self.serializer = SerializerHelper.get_serializer(context)

    # acking related
    self.acking_enabled = context.get_cluster_config().get(constants.TOPOLOGY_ENABLE_ACKING, False)
    Log.info("Enable ACK: %s" % str(self.acking_enabled))

    # load user's bolt class
    bolt_impl_class = super(BoltInstance, self).load_py_instance(is_spout=False)
    self.bolt_impl = bolt_impl_class(delegate=self)
Ejemplo n.º 6
0
    def __init__(self, pplan_helper, in_stream, out_stream, looper):
        super(BoltInstance, self).__init__(pplan_helper, in_stream, out_stream,
                                           looper)

        if self.pplan_helper.is_spout:
            raise RuntimeError("No bolt in physical plan")

        # bolt_config is auto-typed, not <str -> str> only
        context = self.pplan_helper.context
        self.bolt_metrics = BoltMetrics(self.pplan_helper)
        self.serializer = SerializerHelper.get_serializer(context)

        # acking related
        self.acking_enabled = context.get_cluster_config().get(
            constants.TOPOLOGY_ENABLE_ACKING, False)
        Log.info("Enable ACK: %s" % str(self.acking_enabled))

        # load user's bolt class
        bolt_impl_class = super(BoltInstance,
                                self).load_py_instance(is_spout=False)
        self.bolt_impl = bolt_impl_class(delegate=self)
Ejemplo n.º 7
0
    def __init__(self, pplan_helper, in_stream, out_stream, looper):
        super(BoltInstance, self).__init__(pplan_helper, in_stream, out_stream,
                                           looper)

        if self.pplan_helper.is_spout:
            raise RuntimeError("No bolt in physical plan")

        # bolt_config is auto-typed, not <str -> str> only
        context = self.pplan_helper.context
        self.bolt_metrics = BoltMetrics(self.pplan_helper)
        self.serializer = SerializerHelper.get_serializer(context)

        # acking related
        mode = context.get_cluster_config().get(
            api_constants.TOPOLOGY_RELIABILITY_MODE,
            api_constants.TopologyReliabilityMode.ATMOST_ONCE)
        self.acking_enabled = bool(
            mode == api_constants.TopologyReliabilityMode.ATLEAST_ONCE)
        Log.info("Enable ACK: %s" % str(self.acking_enabled))

        # load user's bolt class
        bolt_impl_class = super(BoltInstance,
                                self).load_py_instance(is_spout=False)
        self.bolt_impl = bolt_impl_class(delegate=self)