def setup_output_serde(self):
if self.instance_config.function_details.sink.serDeClassName != None and \
len(self.instance_config.function_details.sink.serDeClassName) > 0:
serde_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_details.sink.serDeClassName)
self.output_serde = serde_kclass()
else:
global DEFAULT_SERIALIZER
serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
self.output_serde = serde_kclass()
def setup_output_serde(self):
if self.instance_config.function_details.sink.serDeClassName != None and \
len(self.instance_config.function_details.sink.serDeClassName) > 0:
serde_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_details.sink.serDeClassName)
self.output_serde = serde_kclass()
else:
global DEFAULT_SERIALIZER
serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
self.output_serde = serde_kclass()
def run(self):
# Setup consumers and input deserializers
mode = pulsar._pulsar.ConsumerType.Exclusive
if self.atmost_once:
mode = pulsar._pulsar.ConsumerType.Shared
subscription_name = str(self.instance_config.function_config.tenant) + "/" + \
str(self.instance_config.function_config.namespace) + "/" + \
str(self.instance_config.function_config.name)
for topic, serde in self.instance_config.function_config.customSerdeInputs.items(
):
serde_kclass = util.import_class(os.path.dirname(self.user_code),
serde)
self.input_serdes[topic] = serde_kclass()
Log.info("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic),
subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener, topic,
self.input_serdes[topic]))
for topic in self.instance_config.function_config.inputs:
global DEFAULT_SERIALIZER
serde_kclass = util.import_class(os.path.dirname(self.user_code),
DEFAULT_SERIALIZER)
self.input_serdes[topic] = serde_kclass()
Log.info("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic),
subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener, topic,
self.input_serdes[topic]))
function_kclass = util.import_class(
os.path.dirname(self.user_code),
self.instance_config.function_config.className)
if function_kclass is None:
Log.critical("Could not import User Function Module %s" %
self.instance_config.function_config.className)
raise NameError("Could not import User Function Module %s" %
self.instance_config.function_config.className)
try:
self.function_class = function_kclass()
except:
self.function_purefunction = function_kclass
self.contextimpl = contextimpl.ContextImpl(self.instance_config, Log,
self.pulsar_client,
self.user_code,
self.consumers)
# Now launch a thread that does execution
self.exeuction_thread = threading.Thread(target=self.actual_execution)
self.exeuction_thread.start()
def run(self):
# Setup consumers and input deserializers
mode = pulsar._pulsar.ConsumerType.Shared
if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value(
"FAILOVER"):
mode = pulsar._pulsar.ConsumerType.Failover
subscription_name = str(self.instance_config.function_details.tenant) + "/" + \
str(self.instance_config.function_details.namespace) + "/" + \
str(self.instance_config.function_details.name)
for topic, serde in self.instance_config.function_details.source.topicsToSerDeClassName.items(
):
if not serde:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), serde)
self.input_serdes[topic] = serde_kclass()
Log.info("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic),
subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener, topic,
self.input_serdes[topic]),
unacked_messages_timeout_ms=int(self.timeout_ms)
if self.timeout_ms else None)
function_kclass = util.import_class(
os.path.dirname(self.user_code),
self.instance_config.function_details.className)
if function_kclass is None:
Log.critical("Could not import User Function Module %s" %
self.instance_config.function_details.className)
raise NameError("Could not import User Function Module %s" %
self.instance_config.function_details.className)
try:
self.function_class = function_kclass()
except:
self.function_purefunction = function_kclass
self.contextimpl = contextimpl.ContextImpl(self.instance_config, Log,
self.pulsar_client,
self.user_code,
self.consumers)
# Now launch a thread that does execution
self.exeuction_thread = threading.Thread(target=self.actual_execution)
self.exeuction_thread.start()
# start proccess spawner health check timer
self.last_health_check_ts = time.time()
Timer(30, self.process_spawner_health_check_timer).start()
def publish(self, topic_name, message, serde_class_name="serde.IdentitySerDe", properties=None, compression_type=None, callback=None):
# Just make sure that user supplied values are properly typed
topic_name = str(topic_name)
serde_class_name = str(serde_class_name)
pulsar_compression_type = pulsar._pulsar.CompressionType.NONE
if compression_type is not None:
pulsar_compression_type = compression_type
if topic_name not in self.publish_producers:
self.publish_producers[topic_name] = self.pulsar_client.create_producer(
topic_name,
block_if_queue_full=True,
batching_enabled=True,
batching_max_publish_delay_ms=10,
compression_type=pulsar_compression_type,
properties=util.get_properties(util.getFullyQualifiedFunctionName(
self.instance_config.function_details.tenant,
self.instance_config.function_details.namespace,
self.instance_config.function_details.name),
self.instance_config.instance_id)
)
if serde_class_name not in self.publish_serializers:
serde_klass = util.import_class(self.user_code_dir, serde_class_name)
self.publish_serializers[serde_class_name] = serde_klass()
output_bytes = bytes(self.publish_serializers[serde_class_name].serialize(message))
self.publish_producers[topic_name].send_async(output_bytes, partial(self.callback_wrapper, callback, topic_name, self.get_message_id()), properties=properties)
def publish(self,
topic_name,
message,
serde_class_name="serde.IdentitySerDe",
properties=None,
compression_type=None):
# Just make sure that user supplied values are properly typed
topic_name = str(topic_name)
serde_class_name = str(serde_class_name)
pulsar_compression_type = pulsar._pulsar.CompressionType.NONE
if compression_type is not None:
pulsar_compression_type = compression_type
if topic_name not in self.publish_producers:
self.publish_producers[
topic_name] = self.pulsar_client.create_producer(
topic_name,
block_if_queue_full=True,
batching_enabled=True,
batching_max_publish_delay_ms=1,
max_pending_messages=100000,
compression_type=compression_type)
if serde_class_name not in self.publish_serializers:
serde_klass = util.import_class(self.user_code_dir,
serde_class_name)
self.publish_serializers[serde_class_name] = serde_klass()
output_bytes = bytes(
self.publish_serializers[serde_class_name].serialize(message))
self.publish_producers[topic_name].send_async(output_bytes,
None,
properties=properties)
def from_id(cls, id):
pipeline = Pipeline.objects.get(id=id)
klass = util.import_class(pipeline.class_path)
obj = klass(*pipeline.args, **pipeline.kwargs)
obj.pk = id
obj.parent = pipeline.parent_pipeline
obj.output = pipeline.output
return obj
def from_id(cls, id):
pipeline = Pipeline.objects.get(id=id)
klass = util.import_class(pipeline.class_path)
obj = klass(*pipeline.args, **pipeline.kwargs)
obj.pk = id
obj.parent = pipeline.parent_pipeline
obj.output = pipeline.output
return obj
def publish(self, topic_name, message, serde_class_name):
if topic_name not in self.publish_producers:
self.publish_producers[
topic_name] = self.pulsar_client.create_producer(
topic_name,
block_if_queue_full=True,
batching_enabled=True,
batching_max_publish_delay_ms=1,
max_pending_messages=100000)
if serde_class_name not in self.publish_serializers:
serde_klass = util.import_class(self.user_code_dir,
serde_class_name)
self.publish_serializers[serde_class_name] = serde_klass()
output_bytes = self.publish_serializers[serde_class_name].serialize(
message)
self.publish_producers[topic_name].send_async(output_bytes, None)
def publish(self, topic_name, message, serde_class_name="serde.IdentitySerDe", properties=None, compression_type=None):
# Just make sure that user supplied values are properly typed
topic_name = str(topic_name)
serde_class_name = str(serde_class_name)
pulsar_compression_type = pulsar._pulsar.CompressionType.NONE
if compression_type is not None:
pulsar_compression_type = compression_type
if topic_name not in self.publish_producers:
self.publish_producers[topic_name] = self.pulsar_client.create_producer(
topic_name,
block_if_queue_full=True,
batching_enabled=True,
batching_max_publish_delay_ms=1,
max_pending_messages=100000,
compression_type=pulsar_compression_type
)
if serde_class_name not in self.publish_serializers:
serde_klass = util.import_class(self.user_code_dir, serde_class_name)
self.publish_serializers[serde_class_name] = serde_klass()
output_bytes = bytes(self.publish_serializers[serde_class_name].serialize(message))
self.publish_producers[topic_name].send_async(output_bytes, None, properties=properties)
def create_logger(generator, logger_class, logger_config_file=None):
l_class = util.import_class(logger_class)
return l_class(generator, logger_config_file=logger_config_file)
def run(self):
# Setup state
self.state_context = self.setup_state()
# Setup consumers and input deserializers
mode = pulsar._pulsar.ConsumerType.Shared
if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value(
"FAILOVER"):
mode = pulsar._pulsar.ConsumerType.Failover
subscription_name = str(self.instance_config.function_details.tenant) + "/" + \
str(self.instance_config.function_details.namespace) + "/" + \
str(self.instance_config.function_details.name)
properties = util.get_properties(
util.getFullyQualifiedFunctionName(
self.instance_config.function_details.tenant,
self.instance_config.function_details.namespace,
self.instance_config.function_details.name),
self.instance_config.instance_id)
for topic, serde in self.instance_config.function_details.source.topicsToSerDeClassName.items(
):
if not serde:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), serde)
self.input_serdes[topic] = serde_kclass()
Log.debug("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic),
subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener,
self.input_serdes[topic]),
unacked_messages_timeout_ms=int(self.timeout_ms)
if self.timeout_ms else None,
properties=properties)
for topic, consumer_conf in self.instance_config.function_details.source.inputSpecs.items(
):
if not consumer_conf.serdeClassName:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(
os.path.dirname(self.user_code),
consumer_conf.serdeClassName)
self.input_serdes[topic] = serde_kclass()
Log.debug("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
consumer_args = {
"consumer_type":
mode,
"message_listener":
partial(self.message_listener, self.input_serdes[topic]),
"unacked_messages_timeout_ms":
int(self.timeout_ms) if self.timeout_ms else None,
"properties":
properties
}
if consumer_conf.HasField("receiverQueueSize"):
consumer_args[
"receiver_queue_size"] = consumer_conf.receiverQueueSize.value
if consumer_conf.isRegexPattern:
self.consumers[topic] = self.pulsar_client.subscribe(
re.compile(str(topic)), subscription_name, **consumer_args)
else:
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic), subscription_name, **consumer_args)
function_kclass = util.import_class(
os.path.dirname(self.user_code),
self.instance_config.function_details.className)
if function_kclass is None:
Log.critical("Could not import User Function Module %s" %
self.instance_config.function_details.className)
raise NameError("Could not import User Function Module %s" %
self.instance_config.function_details.className)
try:
self.function_class = function_kclass()
except:
self.function_purefunction = function_kclass
self.contextimpl = contextimpl.ContextImpl(
self.instance_config, Log, self.pulsar_client, self.user_code,
self.consumers, self.secrets_provider, self.metrics_labels,
self.state_context, self.stats)
# Now launch a thread that does execution
self.execution_thread = threading.Thread(target=self.actual_execution)
self.execution_thread.start()
# start proccess spawner health check timer
self.last_health_check_ts = time.time()
if self.expected_healthcheck_interval > 0:
timer = util.FixedTimer(self.expected_healthcheck_interval,
self.process_spawner_health_check_timer,
name="health-check-timer")
timer.start()
def main():
# Setup signal handlers
signal.signal(signal.SIGTERM, atexit_function)
signal.signal(signal.SIGHUP, atexit_function)
signal.signal(signal.SIGINT, atexit_function)
parser = argparse.ArgumentParser(
description='Pulsar Functions Python Instance')
parser.add_argument('--function_details',
required=True,
help='Function Details Json String')
parser.add_argument('--py',
required=True,
help='Full Path of Function Code File')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--function_id', required=True, help='Function Id')
parser.add_argument('--function_version',
required=True,
help='Function Version')
parser.add_argument('--pulsar_serviceurl',
required=True,
help='Pulsar Service Url')
parser.add_argument('--client_auth_plugin',
required=False,
help='Client authentication plugin')
parser.add_argument('--client_auth_params',
required=False,
help='Client authentication params')
parser.add_argument('--use_tls', required=False, help='Use tls')
parser.add_argument('--tls_allow_insecure_connection',
required=False,
help='Tls allow insecure connection')
parser.add_argument('--hostname_verification_enabled',
required=False,
help='Enable hostname verification')
parser.add_argument('--tls_trust_cert_path',
required=False,
help='Tls trust cert file path')
parser.add_argument('--port',
required=True,
help='Instance Port',
type=int)
parser.add_argument('--metrics_port',
required=True,
help="Port metrics will be exposed on",
type=int)
parser.add_argument('--max_buffered_tuples',
required=True,
help='Maximum number of Buffered tuples')
parser.add_argument('--logging_directory',
required=True,
help='Logging Directory')
parser.add_argument('--logging_file', required=True, help='Log file name')
parser.add_argument('--logging_config_file',
required=True,
help='Config file for logging')
parser.add_argument('--expected_healthcheck_interval',
required=True,
help='Expected time in seconds between health checks',
type=int)
parser.add_argument('--secrets_provider',
required=False,
help='The classname of the secrets provider')
parser.add_argument(
'--secrets_provider_config',
required=False,
help='The config that needs to be passed to secrets provider')
parser.add_argument(
'--install_usercode_dependencies',
required=False,
help=
'For packaged python like wheel files, do we need to install all dependencies',
type=bool)
parser.add_argument(
'--dependency_repository',
required=False,
help=
'For packaged python like wheel files, which repository to pull the dependencies from'
)
parser.add_argument(
'--extra_dependency_repository',
required=False,
help=
'For packaged python like wheel files, any extra repository to pull the dependencies from'
)
parser.add_argument('--state_storage_serviceurl',
required=False,
help='Managed State Storage Service Url')
parser.add_argument(
'--cluster_name',
required=True,
help='The name of the cluster this instance is running on')
args = parser.parse_args()
function_details = Function_pb2.FunctionDetails()
args.function_details = str(args.function_details)
if args.function_details[0] == '\'':
args.function_details = args.function_details[1:]
if args.function_details[-1] == '\'':
args.function_details = args.function_details[:-1]
json_format.Parse(args.function_details, function_details)
if os.path.splitext(str(args.py))[1] == '.whl':
if args.install_usercode_dependencies:
cmd = "pip install -t %s" % os.path.dirname(str(args.py))
if args.dependency_repository:
cmd = cmd + " -i %s" % str(args.dependency_repository)
if args.extra_dependency_repository:
cmd = cmd + " --extra-index-url %s" % str(
args.extra_dependency_repository)
cmd = cmd + " %s" % str(args.py)
retval = os.system(cmd)
if retval != 0:
print("Could not install user depedencies")
sys.exit(1)
else:
zpfile = zipfile.ZipFile(str(args.py), 'r')
zpfile.extractall(os.path.dirname(str(args.py)))
sys.path.insert(0, os.path.dirname(str(args.py)))
elif os.path.splitext(str(args.py))[1] == '.zip':
# Assumig zip file with format func.zip
# extract to folder function
# internal dir format
# "func/src"
# "func/requirements.txt"
# "func/deps"
# run pip install to target folder deps folder
zpfile = zipfile.ZipFile(str(args.py), 'r')
zpfile.extractall(os.path.dirname(str(args.py)))
basename = os.path.splitext(str(args.py))[0]
deps_dir = os.path.join(os.path.dirname(str(args.py)), basename,
"deps")
if os.path.isdir(deps_dir) and os.listdir(deps_dir):
# get all wheel files from deps directory
wheel_file_list = [
os.path.join(deps_dir, f) for f in os.listdir(deps_dir)
if os.path.isfile(os.path.join(deps_dir, f))
and os.path.splitext(f)[1] == '.whl'
]
cmd = "pip install -t %s --no-index --find-links %s %s" % (
os.path.dirname(str(
args.py)), deps_dir, " ".join(wheel_file_list))
Log.debug("Install python dependencies via cmd: %s" % cmd)
retval = os.system(cmd)
if retval != 0:
print(
"Could not install user depedencies specified by the zip file"
)
sys.exit(1)
# add python user src directory to path
sys.path.insert(
0, os.path.join(os.path.dirname(str(args.py)), basename, "src"))
log_file = os.path.join(
args.logging_directory,
util.getFullyQualifiedFunctionName(function_details.tenant,
function_details.namespace,
function_details.name),
"%s-%s.log" % (args.logging_file, args.instance_id))
log.init_logger(logging.INFO, log_file, args.logging_config_file)
Log.info("Starting Python instance with %s" % str(args))
authentication = None
use_tls = False
tls_allow_insecure_connection = False
tls_trust_cert_path = None
if args.client_auth_plugin and args.client_auth_params:
authentication = pulsar.Authentication(args.client_auth_plugin,
args.client_auth_params)
if args.use_tls == "true":
use_tls = True
if args.tls_allow_insecure_connection == "true":
tls_allow_insecure_connection = True
if args.tls_trust_cert_path:
tls_trust_cert_path = args.tls_trust_cert_path
pulsar_client = pulsar.Client(
args.pulsar_serviceurl,
authentication=authentication,
operation_timeout_seconds=30,
io_threads=1,
message_listener_threads=1,
concurrent_lookup_requests=50000,
log_conf_file_path=None,
use_tls=use_tls,
tls_trust_certs_file_path=tls_trust_cert_path,
tls_allow_insecure_connection=tls_allow_insecure_connection)
state_storage_serviceurl = None
if args.state_storage_serviceurl is not None:
state_storage_serviceurl = str(args.state_storage_serviceurl)
secrets_provider = None
if args.secrets_provider is not None:
secrets_provider = util.import_class(
os.path.dirname(inspect.getfile(inspect.currentframe())),
str(args.secrets_provider))
else:
secrets_provider = util.import_class(
os.path.dirname(inspect.getfile(inspect.currentframe())),
"secretsprovider.ClearTextSecretsProvider")
secrets_provider = secrets_provider()
secrets_provider_config = None
if args.secrets_provider_config is not None:
args.secrets_provider_config = str(args.secrets_provider_config)
if args.secrets_provider_config[0] == '\'':
args.secrets_provider_config = args.secrets_provider_config[1:]
if args.secrets_provider_config[-1] == '\'':
args.secrets_provider_config = args.secrets_provider_config[:-1]
secrets_provider_config = json.loads(str(args.secrets_provider_config))
secrets_provider.init(secrets_provider_config)
pyinstance = python_instance.PythonInstance(
str(args.instance_id), str(args.function_id),
str(args.function_version), function_details,
int(args.max_buffered_tuples), int(args.expected_healthcheck_interval),
str(args.py), pulsar_client, secrets_provider, args.cluster_name,
state_storage_serviceurl)
pyinstance.run()
server_instance = server.serve(args.port, pyinstance)
# Cannot use latest version of prometheus client because of thread leak
# prometheus_client.start_http_server(args.metrics_port)
# Use patched version of prometheus
# Contains fix from https://github.com/prometheus/client_python/pull/356
# This can be removed one the fix in is a official prometheus client release
prometheus_client_fix.start_http_server(args.metrics_port)
global to_run
while to_run:
time.sleep(1)
pyinstance.join()
# make sure to close all non-daemon threads before this!
sys.exit(0)
def main():
# Setup signal handlers
signal.signal(signal.SIGTERM, atexit_function)
signal.signal(signal.SIGHUP, atexit_function)
signal.signal(signal.SIGINT, atexit_function)
parser = argparse.ArgumentParser(description='Pulsar Functions Python Instance')
parser.add_argument('--function_details', required=True, help='Function Details Json String')
parser.add_argument('--py', required=True, help='Full Path of Function Code File')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--function_id', required=True, help='Function Id')
parser.add_argument('--function_version', required=True, help='Function Version')
parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url')
parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin')
parser.add_argument('--client_auth_params', required=False, help='Client authentication params')
parser.add_argument('--use_tls', required=False, help='Use tls')
parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection')
parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification')
parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path')
parser.add_argument('--port', required=True, help='Instance Port', type=int)
parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples')
parser.add_argument('--logging_directory', required=True, help='Logging Directory')
parser.add_argument('--logging_file', required=True, help='Log file name')
parser.add_argument('--logging_config_file', required=True, help='Config file for logging')
parser.add_argument('--expected_healthcheck_interval', required=True, help='Expected time in seconds between health checks', type=int)
parser.add_argument('--secrets_provider', required=False, help='The classname of the secrets provider')
parser.add_argument('--secrets_provider_config', required=False, help='The config that needs to be passed to secrets provider')
parser.add_argument('--install_usercode_dependencies', required=False, help='For packaged python like wheel files, do we need to install all dependencies', type=bool)
parser.add_argument('--dependency_repository', required=False, help='For packaged python like wheel files, which repository to pull the dependencies from')
parser.add_argument('--extra_dependency_repository', required=False, help='For packaged python like wheel files, any extra repository to pull the dependencies from')
args = parser.parse_args()
function_details = Function_pb2.FunctionDetails()
args.function_details = str(args.function_details)
if args.function_details[0] == '\'':
args.function_details = args.function_details[1:]
if args.function_details[-1] == '\'':
args.function_details = args.function_details[:-1]
json_format.Parse(args.function_details, function_details)
if os.path.splitext(str(args.py))[1] == '.whl':
if args.install_usercode_dependencies:
cmd = "pip install -t %s" % os.path.dirname(str(args.py))
if args.dependency_repository:
cmd = cmd + " -i %s" % str(args.dependency_repository)
if args.extra_dependency_repository:
cmd = cmd + " --extra-index-url %s" % str(args.extra_dependency_repository)
cmd = cmd + " %s" % str(args.py)
retval = os.system(cmd)
if retval != 0:
print "Could not install user depedencies"
sys.exit(1)
else:
zpfile = zipfile.ZipFile(str(args.py), 'r')
zpfile.extractall(os.path.dirname(str(args.py)))
sys.path.insert(0, os.path.dirname(str(args.py)))
elif os.path.splitext(str(args.py))[1] == '.zip':
# Assumig zip file with format func.zip
# extract to folder function
# internal dir format
# "func/src"
# "func/requirements.txt"
# "func/deps"
# run pip install to target folder deps folder
zpfile = zipfile.ZipFile(str(args.py), 'r')
zpfile.extractall(os.path.dirname(str(args.py)))
basename = os.path.splitext(str(args.py))[0]
requirements_txt_file = os.path.join(os.path.dirname(str(args.py)), basename, "requirements.txt")
deps_file = os.path.join(os.path.dirname(str(args.py)), basename, "deps")
cmd = "pip install -t %s -r %s --no-index --find-links %s" % (os.path.dirname(str(args.py)), requirements_txt_file, deps_file)
retval = os.system(cmd)
if retval != 0:
print "Could not install user depedencies specified by the zip file"
sys.exit(1)
sys.path.insert(0, os.path.join(os.path.dirname(str(args.py)), basename, "src"))
log_file = os.path.join(args.logging_directory,
util.getFullyQualifiedFunctionName(function_details.tenant, function_details.namespace, function_details.name),
"%s-%s.log" % (args.logging_file, args.instance_id))
log.init_logger(logging.INFO, log_file, args.logging_config_file)
Log.info("Starting Python instance with %s" % str(args))
authentication = None
use_tls = False
tls_allow_insecure_connection = False
tls_trust_cert_path = None
if args.client_auth_plugin and args.client_auth_params:
authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params)
if args.use_tls == "true":
use_tls = True
if args.tls_allow_insecure_connection == "true":
tls_allow_insecure_connection = True
if args.tls_trust_cert_path:
tls_trust_cert_path = args.tls_trust_cert_path
pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30, 1, 1, 50000, None, use_tls, tls_trust_cert_path, tls_allow_insecure_connection)
secrets_provider = None
if args.secrets_provider is not None:
secrets_provider = util.import_class(os.path.dirname(inspect.getfile(inspect.currentframe())), str(args.secrets_provider))
else:
secrets_provider = util.import_class(os.path.dirname(inspect.getfile(inspect.currentframe())), "secretsprovider.ClearTextSecretsProvider")
secrets_provider = secrets_provider()
secrets_provider_config = None
if args.secrets_provider_config is not None:
secrets_provider_config = json.loads(str(args.secrets_provider_config))
secrets_provider.init(secrets_provider_config)
pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id),
str(args.function_version), function_details,
int(args.max_buffered_tuples),
int(args.expected_healthcheck_interval),
str(args.py), pulsar_client, secrets_provider)
pyinstance.run()
server_instance = server.serve(args.port, pyinstance)
global to_run
while to_run:
time.sleep(1)
pyinstance.join()
sys.exit(1)
def create_logger(generator, logger_class, logger_config_file=None):
l_class = util.import_class(logger_class)
return l_class(generator, logger_config_file=logger_config_file)
)
try:
resp = command(node, *args)
except CommandError as e:
start_response(
'400 Bad Request',
[('Content-type', 'text/plain')],
)
return [str(e).encode('utf-8')]
start_response(
'200 OK',
[('Content-type', 'application/octet-stream')],
)
return [serialize(resp)]
if __name__ == '__main__':
node_cls = import_class(os.environ['DFS_NODE_CLASS'])
node = node_cls(*node_cls.get_args(os.environ))
def wsgi_app(env, start_response):
env['DFS_NODE_CLASS'] = node
return route_request(env, start_response)
with make_server(
os.environ.get('DFS_HOST', '0.0.0.0'),
int(os.environ.get('DFS_PORT', '8180')),
wsgi_app,
) as server:
server.serve_forever()
def run(self):
# Setup consumers and input deserializers
mode = pulsar._pulsar.ConsumerType.Shared
if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value("FAILOVER"):
mode = pulsar._pulsar.ConsumerType.Failover
subscription_name = str(self.instance_config.function_details.tenant) + "/" + \
str(self.instance_config.function_details.namespace) + "/" + \
str(self.instance_config.function_details.name)
for topic, serde in self.instance_config.function_details.source.topicsToSerDeClassName.items():
if not serde:
serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(os.path.dirname(self.user_code), serde)
self.input_serdes[topic] = serde_kclass()
Log.debug("Setting up consumer for topic %s with subname %s" % (topic, subscription_name))
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic), subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener, self.input_serdes[topic]),
unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms else None
)
for topic, consumer_conf in self.instance_config.function_details.source.inputSpecs.items():
if not consumer_conf.serdeClassName:
serde_kclass = util.import_class(os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(os.path.dirname(self.user_code), consumer_conf.serdeClassName)
self.input_serdes[topic] = serde_kclass()
Log.debug("Setting up consumer for topic %s with subname %s" % (topic, subscription_name))
if consumer_conf.isRegexPattern:
self.consumers[topic] = self.pulsar_client.subscribe(
re.compile(str(topic)), subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener, self.input_serdes[topic]),
unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms else None
)
else:
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic), subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener, self.input_serdes[topic]),
unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms else None
)
function_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_details.className)
if function_kclass is None:
Log.critical("Could not import User Function Module %s" % self.instance_config.function_details.className)
raise NameError("Could not import User Function Module %s" % self.instance_config.function_details.className)
try:
self.function_class = function_kclass()
except:
self.function_purefunction = function_kclass
self.contextimpl = contextimpl.ContextImpl(self.instance_config, Log, self.pulsar_client, self.user_code, self.consumers, self.secrets_provider)
# Now launch a thread that does execution
self.exeuction_thread = threading.Thread(target=self.actual_execution)
self.exeuction_thread.start()
# start proccess spawner health check timer
self.last_health_check_ts = time.time()
if self.expected_healthcheck_interval > 0:
Timer(self.expected_healthcheck_interval, self.process_spawner_health_check_timer).start()
