def __init__(self, pplan_helper, in_stream, out_stream, looper,
sys_config):
super(SpoutInstance, self).__init__(pplan_helper, in_stream,
out_stream, looper, sys_config)
self.topology_state = topology_pb2.TopologyState.Value("PAUSED")
if not self.pplan_helper.is_spout:
raise RuntimeError("No spout in physicial plan")
context = self.pplan_helper.context
self.spout_metrics = SpoutMetrics(self.pplan_helper)
self.serializer = SerializerHelper.get_serializer(context)
# acking related
self.acking_enabled = context.get_cluster_config().get(
constants.TOPOLOGY_ENABLE_ACKING, False)
self.enable_message_timeouts = \
context.get_cluster_config().get(constants.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS)
Log.info("Enable ACK: %s" % str(self.acking_enabled))
Log.info("Enable Message Timeouts: %s" %
str(self.enable_message_timeouts))
# map <tuple_info.key -> tuple_info>, ordered by insertion time
self.in_flight_tuples = collections.OrderedDict()
self.immediate_acks = collections.deque()
self.total_tuples_emitted = 0
# load user's spout class
spout_impl_class = super(SpoutInstance,
self).load_py_instance(is_spout=True)
self.spout_impl = spout_impl_class(delegate=self)
def __init__(self, pplan_helper, in_stream, out_stream, looper):
super(SpoutInstance, self).__init__(pplan_helper, in_stream, out_stream, looper)
self.topology_state = topology_pb2.TopologyState.Value("PAUSED")
if not self.pplan_helper.is_spout:
raise RuntimeError("No spout in physicial plan")
context = self.pplan_helper.context
self.spout_metrics = SpoutMetrics(self.pplan_helper)
self.serializer = SerializerHelper.get_serializer(context)
# acking related
self.acking_enabled = context.get_cluster_config().get(api_constants.TOPOLOGY_ENABLE_ACKING,
False)
self.enable_message_timeouts = \
context.get_cluster_config().get(api_constants.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS)
Log.info("Enable ACK: %s" % str(self.acking_enabled))
Log.info("Enable Message Timeouts: %s" % str(self.enable_message_timeouts))
# map <tuple_info.key -> tuple_info>, ordered by insertion time
self.in_flight_tuples = collections.OrderedDict()
self.immediate_acks = collections.deque()
self.total_tuples_emitted = 0
# load user's spout class
spout_impl_class = super(SpoutInstance, self).load_py_instance(is_spout=True)
self.spout_impl = spout_impl_class(delegate=self)
def handle_assignment_msg(self, pplan_helper):
"""Called when new NewInstanceAssignmentMessage arrives
Tells this instance to become either spout/bolt.
:param pplan_helper: PhysicalPlanHelper class to become
"""
self.my_pplan_helper = pplan_helper
self.my_pplan_helper.set_topology_context(self.metrics_collector)
self.serializer = SerializerHelper.get_serializer(pplan_helper.context)
if pplan_helper.is_spout:
# Starting a spout
my_spout = pplan_helper.get_my_spout()
Log.info("Incarnating ourselves as spout: %s with task id %s",
pplan_helper.my_component_name,
str(pplan_helper.my_task_id))
self.in_stream. \
register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_SPOUT_READ_QUEUE_CAPACITY])
self.out_stream. \
register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_SPOUT_WRITE_QUEUE_CAPACITY])
py_spout_instance = SpoutInstance(self.my_pplan_helper,
self.in_stream, self.out_stream,
self.looper)
self.my_instance = AssignedInstance(is_spout=True,
protobuf=my_spout,
py_class=py_spout_instance)
else:
# Starting a bolt
my_bolt = pplan_helper.get_my_bolt()
Log.info("Incarnating ourselves as bolt: %s with task id %s",
pplan_helper.my_component_name,
str(pplan_helper.my_task_id))
self.in_stream. \
register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_BOLT_READ_QUEUE_CAPACITY])
self.out_stream. \
register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_BOLT_WRITE_QUEUE_CAPACITY])
py_bolt_instance = BoltInstance(self.my_pplan_helper,
self.in_stream, self.out_stream,
self.looper)
self.my_instance = AssignedInstance(is_spout=False,
protobuf=my_bolt,
py_class=py_bolt_instance)
if pplan_helper.is_topology_running():
try:
self.start_instance_if_possible()
except Exception as e:
Log.error("Error with starting bolt/spout instance: " +
e.message)
Log.error(traceback.format_exc())
else:
Log.info("The instance is deployed in deactivated state")
def __init__(self, pplan_helper, in_stream, out_stream, looper):
self.pplan_helper = pplan_helper
self.in_stream = in_stream
self.output_helper = OutgoingTupleHelper(self.pplan_helper, out_stream)
self.looper = looper
self.sys_config = system_config.get_sys_config()
# will set a root logger here
self.logger = logging.getLogger()
context = pplan_helper.context
mode = context.get_cluster_config().get(api_constants.TOPOLOGY_RELIABILITY_MODE,
api_constants.TopologyReliabilityMode.ATMOST_ONCE)
self.is_stateful = bool(mode == api_constants.TopologyReliabilityMode.EXACTLY_ONCE)
self._stateful_state = None
self.serializer = SerializerHelper.get_serializer(pplan_helper.context)
self._initialized_global_metrics = False
def __init__(self, pplan_helper, in_stream, out_stream, looper, sys_config):
super(BoltInstance, self).__init__(pplan_helper, in_stream, out_stream, looper, sys_config)
if self.pplan_helper.is_spout:
raise RuntimeError("No bolt in physical plan")
# bolt_config is auto-typed, not <str -> str> only
context = self.pplan_helper.context
self.bolt_metrics = BoltMetrics(self.pplan_helper)
self.serializer = SerializerHelper.get_serializer(context)
# acking related
self.acking_enabled = context.get_cluster_config().get(constants.TOPOLOGY_ENABLE_ACKING, False)
Log.info("Enable ACK: %s" % str(self.acking_enabled))
# load user's bolt class
bolt_impl_class = super(BoltInstance, self).load_py_instance(is_spout=False)
self.bolt_impl = bolt_impl_class(delegate=self)
def __init__(self, pplan_helper, in_stream, out_stream, looper):
super(BoltInstance, self).__init__(pplan_helper, in_stream, out_stream,
looper)
if self.pplan_helper.is_spout:
raise RuntimeError("No bolt in physical plan")
# bolt_config is auto-typed, not <str -> str> only
context = self.pplan_helper.context
self.bolt_metrics = BoltMetrics(self.pplan_helper)
self.serializer = SerializerHelper.get_serializer(context)
# acking related
self.acking_enabled = context.get_cluster_config().get(
constants.TOPOLOGY_ENABLE_ACKING, False)
Log.info("Enable ACK: %s" % str(self.acking_enabled))
# load user's bolt class
bolt_impl_class = super(BoltInstance,
self).load_py_instance(is_spout=False)
self.bolt_impl = bolt_impl_class(delegate=self)
def __init__(self, pplan_helper, in_stream, out_stream, looper):
super(BoltInstance, self).__init__(pplan_helper, in_stream, out_stream,
looper)
if self.pplan_helper.is_spout:
raise RuntimeError("No bolt in physical plan")
# bolt_config is auto-typed, not <str -> str> only
context = self.pplan_helper.context
self.bolt_metrics = BoltMetrics(self.pplan_helper)
self.serializer = SerializerHelper.get_serializer(context)
# acking related
mode = context.get_cluster_config().get(
api_constants.TOPOLOGY_RELIABILITY_MODE,
api_constants.TopologyReliabilityMode.ATMOST_ONCE)
self.acking_enabled = bool(
mode == api_constants.TopologyReliabilityMode.ATLEAST_ONCE)
Log.info("Enable ACK: %s" % str(self.acking_enabled))
# load user's bolt class
bolt_impl_class = super(BoltInstance,
self).load_py_instance(is_spout=False)
self.bolt_impl = bolt_impl_class(delegate=self)