def __init__(self, metrics_labels):
self.metrics_labels = metrics_labels
self.process_start_time = None
# as optimization
self._stat_total_processed_successfully = self.stat_total_processed_successfully.labels(
*self.metrics_labels)
self._stat_total_sys_exceptions = self.stat_total_sys_exceptions.labels(
*self.metrics_labels)
self._stat_total_user_exceptions = self.stat_total_user_exceptions.labels(
*self.metrics_labels)
self._stat_process_latency_ms = self.stat_process_latency_ms.labels(
*self.metrics_labels)
self._stat_last_invocation = self.stat_last_invocation.labels(
*self.metrics_labels)
self._stat_total_received = self.stat_total_received.labels(
*self.metrics_labels)
self._stat_total_processed_successfully_1min = self.stat_total_processed_successfully_1min.labels(
*self.metrics_labels)
self._stat_total_sys_exceptions_1min = self.stat_total_sys_exceptions_1min.labels(
*self.metrics_labels)
self._stat_total_user_exceptions_1min = self.stat_total_user_exceptions_1min.labels(
*self.metrics_labels)
self._stat_process_latency_ms_1min = self.stat_process_latency_ms_1min.labels(
*self.metrics_labels)
self._stat_total_received_1min = self.stat_total_received_1min.labels(
*self.metrics_labels)
# start time for windowed metrics
util.FixedTimer(60, self.reset, name="windowed-metrics-timer").start()
def __init__(self, metrics_labels):
self.metrics_labels = metrics_labels
self.process_start_time = None
# start time for windowed metrics
util.FixedTimer(60, self.reset).start()
def run(self):
# Setup state
self.state_context = self.setup_state()
# Setup consumers and input deserializers
mode = pulsar._pulsar.ConsumerType.Shared
if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value(
"FAILOVER"):
mode = pulsar._pulsar.ConsumerType.Failover
subscription_name = str(self.instance_config.function_details.tenant) + "/" + \
str(self.instance_config.function_details.namespace) + "/" + \
str(self.instance_config.function_details.name)
properties = util.get_properties(
util.getFullyQualifiedFunctionName(
self.instance_config.function_details.tenant,
self.instance_config.function_details.namespace,
self.instance_config.function_details.name),
self.instance_config.instance_id)
for topic, serde in self.instance_config.function_details.source.topicsToSerDeClassName.items(
):
if not serde:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), serde)
self.input_serdes[topic] = serde_kclass()
Log.debug("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic),
subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener,
self.input_serdes[topic]),
unacked_messages_timeout_ms=int(self.timeout_ms)
if self.timeout_ms else None,
properties=properties)
for topic, consumer_conf in self.instance_config.function_details.source.inputSpecs.items(
):
if not consumer_conf.serdeClassName:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(
os.path.dirname(self.user_code),
consumer_conf.serdeClassName)
self.input_serdes[topic] = serde_kclass()
Log.debug("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
consumer_args = {
"consumer_type":
mode,
"message_listener":
partial(self.message_listener, self.input_serdes[topic]),
"unacked_messages_timeout_ms":
int(self.timeout_ms) if self.timeout_ms else None,
"properties":
properties
}
if consumer_conf.HasField("receiverQueueSize"):
consumer_args[
"receiver_queue_size"] = consumer_conf.receiverQueueSize.value
if consumer_conf.isRegexPattern:
self.consumers[topic] = self.pulsar_client.subscribe(
re.compile(str(topic)), subscription_name, **consumer_args)
else:
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic), subscription_name, **consumer_args)
function_kclass = util.import_class(
os.path.dirname(self.user_code),
self.instance_config.function_details.className)
if function_kclass is None:
Log.critical("Could not import User Function Module %s" %
self.instance_config.function_details.className)
raise NameError("Could not import User Function Module %s" %
self.instance_config.function_details.className)
try:
self.function_class = function_kclass()
except:
self.function_purefunction = function_kclass
self.contextimpl = contextimpl.ContextImpl(
self.instance_config, Log, self.pulsar_client, self.user_code,
self.consumers, self.secrets_provider, self.metrics_labels,
self.state_context, self.stats)
# Now launch a thread that does execution
self.execution_thread = threading.Thread(target=self.actual_execution)
self.execution_thread.start()
# start proccess spawner health check timer
self.last_health_check_ts = time.time()
if self.expected_healthcheck_interval > 0:
timer = util.FixedTimer(self.expected_healthcheck_interval,
self.process_spawner_health_check_timer,
name="health-check-timer")
timer.start()