def run(self):
# Setup consumers and input deserializers
mode = pulsar._pulsar.ConsumerType.Exclusive
if self.atmost_once:
mode = pulsar._pulsar.ConsumerType.Shared
subscription_name = str(self.instance_config.function_config.tenant) + "/" + \
str(self.instance_config.function_config.namespace) + "/" + \
str(self.instance_config.function_config.name)
for topic, serde in self.instance_config.function_config.customSerdeInputs.items(
):
serde_kclass = util.import_class(os.path.dirname(self.user_code),
serde)
self.input_serdes[topic] = serde_kclass()
Log.info("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic),
subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener, topic,
self.input_serdes[topic]))
for topic in self.instance_config.function_config.inputs:
global DEFAULT_SERIALIZER
serde_kclass = util.import_class(os.path.dirname(self.user_code),
DEFAULT_SERIALIZER)
self.input_serdes[topic] = serde_kclass()
Log.info("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic),
subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener, topic,
self.input_serdes[topic]))
function_kclass = util.import_class(
os.path.dirname(self.user_code),
self.instance_config.function_config.className)
if function_kclass is None:
Log.critical("Could not import User Function Module %s" %
self.instance_config.function_config.className)
raise NameError("Could not import User Function Module %s" %
self.instance_config.function_config.className)
try:
self.function_class = function_kclass()
except:
self.function_purefunction = function_kclass
self.contextimpl = contextimpl.ContextImpl(self.instance_config, Log,
self.pulsar_client,
self.user_code,
self.consumers)
# Now launch a thread that does execution
self.exeuction_thread = threading.Thread(target=self.actual_execution)
self.exeuction_thread.start()
def run(self):
# Setup consumers and input deserializers
mode = pulsar._pulsar.ConsumerType.Shared
if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value(
"FAILOVER"):
mode = pulsar._pulsar.ConsumerType.Failover
subscription_name = str(self.instance_config.function_details.tenant) + "/" + \
str(self.instance_config.function_details.namespace) + "/" + \
str(self.instance_config.function_details.name)
for topic, serde in self.instance_config.function_details.source.topicsToSerDeClassName.items(
):
if not serde:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), serde)
self.input_serdes[topic] = serde_kclass()
Log.info("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic),
subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener, topic,
self.input_serdes[topic]),
unacked_messages_timeout_ms=int(self.timeout_ms)
if self.timeout_ms else None)
function_kclass = util.import_class(
os.path.dirname(self.user_code),
self.instance_config.function_details.className)
if function_kclass is None:
Log.critical("Could not import User Function Module %s" %
self.instance_config.function_details.className)
raise NameError("Could not import User Function Module %s" %
self.instance_config.function_details.className)
try:
self.function_class = function_kclass()
except:
self.function_purefunction = function_kclass
self.contextimpl = contextimpl.ContextImpl(self.instance_config, Log,
self.pulsar_client,
self.user_code,
self.consumers)
# Now launch a thread that does execution
self.exeuction_thread = threading.Thread(target=self.actual_execution)
self.exeuction_thread.start()
# start proccess spawner health check timer
self.last_health_check_ts = time.time()
Timer(30, self.process_spawner_health_check_timer).start()
def run(self):
# Setup state
self.state_context = self.setup_state()
# Setup consumers and input deserializers
mode = pulsar._pulsar.ConsumerType.Shared
if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value(
"FAILOVER"):
mode = pulsar._pulsar.ConsumerType.Failover
subscription_name = str(self.instance_config.function_details.tenant) + "/" + \
str(self.instance_config.function_details.namespace) + "/" + \
str(self.instance_config.function_details.name)
properties = util.get_properties(
util.getFullyQualifiedFunctionName(
self.instance_config.function_details.tenant,
self.instance_config.function_details.namespace,
self.instance_config.function_details.name),
self.instance_config.instance_id)
for topic, serde in self.instance_config.function_details.source.topicsToSerDeClassName.items(
):
if not serde:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), serde)
self.input_serdes[topic] = serde_kclass()
Log.debug("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic),
subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener,
self.input_serdes[topic]),
unacked_messages_timeout_ms=int(self.timeout_ms)
if self.timeout_ms else None,
properties=properties)
for topic, consumer_conf in self.instance_config.function_details.source.inputSpecs.items(
):
if not consumer_conf.serdeClassName:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(
os.path.dirname(self.user_code),
consumer_conf.serdeClassName)
self.input_serdes[topic] = serde_kclass()
Log.debug("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
consumer_args = {
"consumer_type":
mode,
"message_listener":
partial(self.message_listener, self.input_serdes[topic]),
"unacked_messages_timeout_ms":
int(self.timeout_ms) if self.timeout_ms else None,
"properties":
properties
}
if consumer_conf.HasField("receiverQueueSize"):
consumer_args[
"receiver_queue_size"] = consumer_conf.receiverQueueSize.value
if consumer_conf.isRegexPattern:
self.consumers[topic] = self.pulsar_client.subscribe(
re.compile(str(topic)), subscription_name, **consumer_args)
else:
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic), subscription_name, **consumer_args)
function_kclass = util.import_class(
os.path.dirname(self.user_code),
self.instance_config.function_details.className)
if function_kclass is None:
Log.critical("Could not import User Function Module %s" %
self.instance_config.function_details.className)
raise NameError("Could not import User Function Module %s" %
self.instance_config.function_details.className)
try:
self.function_class = function_kclass()
except:
self.function_purefunction = function_kclass
self.contextimpl = contextimpl.ContextImpl(
self.instance_config, Log, self.pulsar_client, self.user_code,
self.consumers, self.secrets_provider, self.metrics_labels,
self.state_context, self.stats)
# Now launch a thread that does execution
self.execution_thread = threading.Thread(target=self.actual_execution)
self.execution_thread.start()
# start proccess spawner health check timer
self.last_health_check_ts = time.time()
if self.expected_healthcheck_interval > 0:
timer = util.FixedTimer(self.expected_healthcheck_interval,
self.process_spawner_health_check_timer,
name="health-check-timer")
timer.start()
