def setup_producer(self):
if self.instance_config.function_details.sink.topic != None and \
len(self.instance_config.function_details.sink.topic) > 0:
Log.debug("Setting up producer for topic %s" %
self.instance_config.function_details.sink.topic)
batch_type = pulsar.BatchingType.Default
if self.instance_config.function_details.sink.producerSpec.batchBuilder != None and \
len(self.instance_config.function_details.sink.producerSpec.batchBuilder) > 0:
batch_builder = self.instance_config.function_details.sink.producerSpec.batchBuilder
if batch_builder == "KEY_BASED":
batch_type = pulsar.BatchingType.KeyBased
self.producer = self.pulsar_client.create_producer(
str(self.instance_config.function_details.sink.topic),
block_if_queue_full=True,
batching_enabled=True,
batching_type=batch_type,
batching_max_publish_delay_ms=10,
compression_type=pulsar.CompressionType.LZ4,
# set send timeout to be infinity to prevent potential deadlock with consumer
# that might happen when consumer is blocked due to unacked messages
send_timeout_millis=0,
properties=util.get_properties(
util.getFullyQualifiedFunctionName(
self.instance_config.function_details.tenant,
self.instance_config.function_details.namespace,
self.instance_config.function_details.name),
self.instance_config.instance_id))
def publish(self, topic_name, message, serde_class_name="serde.IdentitySerDe", properties=None, compression_type=None, callback=None):
# Just make sure that user supplied values are properly typed
topic_name = str(topic_name)
serde_class_name = str(serde_class_name)
pulsar_compression_type = pulsar._pulsar.CompressionType.NONE
if compression_type is not None:
pulsar_compression_type = compression_type
if topic_name not in self.publish_producers:
self.publish_producers[topic_name] = self.pulsar_client.create_producer(
topic_name,
block_if_queue_full=True,
batching_enabled=True,
batching_max_publish_delay_ms=10,
compression_type=pulsar_compression_type,
properties=util.get_properties(util.getFullyQualifiedFunctionName(
self.instance_config.function_details.tenant,
self.instance_config.function_details.namespace,
self.instance_config.function_details.name),
self.instance_config.instance_id)
)
if serde_class_name not in self.publish_serializers:
serde_klass = util.import_class(self.user_code_dir, serde_class_name)
self.publish_serializers[serde_class_name] = serde_klass()
output_bytes = bytes(self.publish_serializers[serde_class_name].serialize(message))
self.publish_producers[topic_name].send_async(output_bytes, partial(self.callback_wrapper, callback, topic_name, self.get_message_id()), properties=properties)
def setup_producer(self):
if self.instance_config.function_details.sink.topic != None and \
len(self.instance_config.function_details.sink.topic) > 0:
Log.debug("Setting up producer for topic %s" % self.instance_config.function_details.sink.topic)
self.producer = self.pulsar_client.create_producer(
str(self.instance_config.function_details.sink.topic),
block_if_queue_full=True,
batching_enabled=True,
batching_max_publish_delay_ms=1,
# set send timeout to be infinity to prevent potential deadlock with consumer
# that might happen when consumer is blocked due to unacked messages
send_timeout_millis=0,
max_pending_messages=100000,
properties=util.get_properties(util.getFullyQualifiedFunctionName(
self.instance_config.function_details.tenant,
self.instance_config.function_details.namespace,
self.instance_config.function_details.name),
self.instance_config.instance_id)
)
def main():
# Setup signal handlers
signal.signal(signal.SIGTERM, atexit_function)
signal.signal(signal.SIGHUP, atexit_function)
signal.signal(signal.SIGINT, atexit_function)
parser = argparse.ArgumentParser(
description='Pulsar Functions Python Instance')
parser.add_argument('--function_details',
required=True,
help='Function Details Json String')
parser.add_argument('--py',
required=True,
help='Full Path of Function Code File')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--function_id', required=True, help='Function Id')
parser.add_argument('--function_version',
required=True,
help='Function Version')
parser.add_argument('--pulsar_serviceurl',
required=True,
help='Pulsar Service Url')
parser.add_argument('--client_auth_plugin',
required=False,
help='Client authentication plugin')
parser.add_argument('--client_auth_params',
required=False,
help='Client authentication params')
parser.add_argument('--use_tls', required=False, help='Use tls')
parser.add_argument('--tls_allow_insecure_connection',
required=False,
help='Tls allow insecure connection')
parser.add_argument('--hostname_verification_enabled',
required=False,
help='Enable hostname verification')
parser.add_argument('--tls_trust_cert_path',
required=False,
help='Tls trust cert file path')
parser.add_argument('--port',
required=True,
help='Instance Port',
type=int)
parser.add_argument('--max_buffered_tuples',
required=True,
help='Maximum number of Buffered tuples')
parser.add_argument('--logging_directory',
required=True,
help='Logging Directory')
parser.add_argument('--logging_file', required=True, help='Log file name')
args = parser.parse_args()
function_details = Function_pb2.FunctionDetails()
json_format.Parse(args.function_details, function_details)
log_file = os.path.join(
args.logging_directory,
util.getFullyQualifiedFunctionName(function_details.tenant,
function_details.namespace,
function_details.name),
"%s-%s.log" % (args.logging_file, args.instance_id))
log.init_rotating_logger(level=logging.INFO,
logfile=log_file,
max_files=5,
max_bytes=10 * 1024 * 1024)
Log.info("Starting Python instance with %s" % str(args))
authentication = None
use_tls = False
tls_allow_insecure_connection = False
tls_trust_cert_path = None
if args.client_auth_plugin and args.client_auth_params:
authentication = pulsar.Authentication(args.client_auth_plugin,
args.client_auth_params)
if args.use_tls == "true":
use_tls = True
if args.tls_allow_insecure_connection == "true":
tls_allow_insecure_connection = True
if args.tls_trust_cert_path:
tls_trust_cert_path = args.tls_trust_cert_path
pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30,
1, 1, 50000, None, use_tls,
tls_trust_cert_path,
tls_allow_insecure_connection)
pyinstance = python_instance.PythonInstance(str(args.instance_id),
str(args.function_id),
str(args.function_version),
function_details,
int(args.max_buffered_tuples),
str(args.py), pulsar_client)
pyinstance.run()
server_instance = server.serve(args.port, pyinstance)
global to_run
while to_run:
time.sleep(1)
pyinstance.join()
sys.exit(1)
def run(self):
# Setup state
self.state_context = self.setup_state()
# Setup consumers and input deserializers
mode = pulsar._pulsar.ConsumerType.Shared
if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value(
"FAILOVER"):
mode = pulsar._pulsar.ConsumerType.Failover
subscription_name = str(self.instance_config.function_details.tenant) + "/" + \
str(self.instance_config.function_details.namespace) + "/" + \
str(self.instance_config.function_details.name)
properties = util.get_properties(
util.getFullyQualifiedFunctionName(
self.instance_config.function_details.tenant,
self.instance_config.function_details.namespace,
self.instance_config.function_details.name),
self.instance_config.instance_id)
for topic, serde in self.instance_config.function_details.source.topicsToSerDeClassName.items(
):
if not serde:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), serde)
self.input_serdes[topic] = serde_kclass()
Log.debug("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic),
subscription_name,
consumer_type=mode,
message_listener=partial(self.message_listener,
self.input_serdes[topic]),
unacked_messages_timeout_ms=int(self.timeout_ms)
if self.timeout_ms else None,
properties=properties)
for topic, consumer_conf in self.instance_config.function_details.source.inputSpecs.items(
):
if not consumer_conf.serdeClassName:
serde_kclass = util.import_class(
os.path.dirname(self.user_code), DEFAULT_SERIALIZER)
else:
serde_kclass = util.import_class(
os.path.dirname(self.user_code),
consumer_conf.serdeClassName)
self.input_serdes[topic] = serde_kclass()
Log.debug("Setting up consumer for topic %s with subname %s" %
(topic, subscription_name))
consumer_args = {
"consumer_type":
mode,
"message_listener":
partial(self.message_listener, self.input_serdes[topic]),
"unacked_messages_timeout_ms":
int(self.timeout_ms) if self.timeout_ms else None,
"properties":
properties
}
if consumer_conf.HasField("receiverQueueSize"):
consumer_args[
"receiver_queue_size"] = consumer_conf.receiverQueueSize.value
if consumer_conf.isRegexPattern:
self.consumers[topic] = self.pulsar_client.subscribe(
re.compile(str(topic)), subscription_name, **consumer_args)
else:
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic), subscription_name, **consumer_args)
function_kclass = util.import_class(
os.path.dirname(self.user_code),
self.instance_config.function_details.className)
if function_kclass is None:
Log.critical("Could not import User Function Module %s" %
self.instance_config.function_details.className)
raise NameError("Could not import User Function Module %s" %
self.instance_config.function_details.className)
try:
self.function_class = function_kclass()
except:
self.function_purefunction = function_kclass
self.contextimpl = contextimpl.ContextImpl(
self.instance_config, Log, self.pulsar_client, self.user_code,
self.consumers, self.secrets_provider, self.metrics_labels,
self.state_context, self.stats)
# Now launch a thread that does execution
self.execution_thread = threading.Thread(target=self.actual_execution)
self.execution_thread.start()
# start proccess spawner health check timer
self.last_health_check_ts = time.time()
if self.expected_healthcheck_interval > 0:
timer = util.FixedTimer(self.expected_healthcheck_interval,
self.process_spawner_health_check_timer,
name="health-check-timer")
timer.start()
def main():
# Setup signal handlers
signal.signal(signal.SIGTERM, atexit_function)
signal.signal(signal.SIGHUP, atexit_function)
signal.signal(signal.SIGINT, atexit_function)
parser = argparse.ArgumentParser(
description='Pulsar Functions Python Instance')
parser.add_argument('--function_details',
required=True,
help='Function Details Json String')
parser.add_argument('--py',
required=True,
help='Full Path of Function Code File')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--function_id', required=True, help='Function Id')
parser.add_argument('--function_version',
required=True,
help='Function Version')
parser.add_argument('--pulsar_serviceurl',
required=True,
help='Pulsar Service Url')
parser.add_argument('--client_auth_plugin',
required=False,
help='Client authentication plugin')
parser.add_argument('--client_auth_params',
required=False,
help='Client authentication params')
parser.add_argument('--use_tls', required=False, help='Use tls')
parser.add_argument('--tls_allow_insecure_connection',
required=False,
help='Tls allow insecure connection')
parser.add_argument('--hostname_verification_enabled',
required=False,
help='Enable hostname verification')
parser.add_argument('--tls_trust_cert_path',
required=False,
help='Tls trust cert file path')
parser.add_argument('--port',
required=True,
help='Instance Port',
type=int)
parser.add_argument('--metrics_port',
required=True,
help="Port metrics will be exposed on",
type=int)
parser.add_argument('--max_buffered_tuples',
required=True,
help='Maximum number of Buffered tuples')
parser.add_argument('--logging_directory',
required=True,
help='Logging Directory')
parser.add_argument('--logging_file', required=True, help='Log file name')
parser.add_argument('--logging_config_file',
required=True,
help='Config file for logging')
parser.add_argument('--expected_healthcheck_interval',
required=True,
help='Expected time in seconds between health checks',
type=int)
parser.add_argument('--secrets_provider',
required=False,
help='The classname of the secrets provider')
parser.add_argument(
'--secrets_provider_config',
required=False,
help='The config that needs to be passed to secrets provider')
parser.add_argument(
'--install_usercode_dependencies',
required=False,
help=
'For packaged python like wheel files, do we need to install all dependencies',
type=bool)
parser.add_argument(
'--dependency_repository',
required=False,
help=
'For packaged python like wheel files, which repository to pull the dependencies from'
)
parser.add_argument(
'--extra_dependency_repository',
required=False,
help=
'For packaged python like wheel files, any extra repository to pull the dependencies from'
)
parser.add_argument('--state_storage_serviceurl',
required=False,
help='Managed State Storage Service Url')
parser.add_argument(
'--cluster_name',
required=True,
help='The name of the cluster this instance is running on')
args = parser.parse_args()
function_details = Function_pb2.FunctionDetails()
args.function_details = str(args.function_details)
if args.function_details[0] == '\'':
args.function_details = args.function_details[1:]
if args.function_details[-1] == '\'':
args.function_details = args.function_details[:-1]
json_format.Parse(args.function_details, function_details)
if os.path.splitext(str(args.py))[1] == '.whl':
if args.install_usercode_dependencies:
cmd = "pip install -t %s" % os.path.dirname(str(args.py))
if args.dependency_repository:
cmd = cmd + " -i %s" % str(args.dependency_repository)
if args.extra_dependency_repository:
cmd = cmd + " --extra-index-url %s" % str(
args.extra_dependency_repository)
cmd = cmd + " %s" % str(args.py)
retval = os.system(cmd)
if retval != 0:
print("Could not install user depedencies")
sys.exit(1)
else:
zpfile = zipfile.ZipFile(str(args.py), 'r')
zpfile.extractall(os.path.dirname(str(args.py)))
sys.path.insert(0, os.path.dirname(str(args.py)))
elif os.path.splitext(str(args.py))[1] == '.zip':
# Assumig zip file with format func.zip
# extract to folder function
# internal dir format
# "func/src"
# "func/requirements.txt"
# "func/deps"
# run pip install to target folder deps folder
zpfile = zipfile.ZipFile(str(args.py), 'r')
zpfile.extractall(os.path.dirname(str(args.py)))
basename = os.path.splitext(str(args.py))[0]
deps_dir = os.path.join(os.path.dirname(str(args.py)), basename,
"deps")
if os.path.isdir(deps_dir) and os.listdir(deps_dir):
# get all wheel files from deps directory
wheel_file_list = [
os.path.join(deps_dir, f) for f in os.listdir(deps_dir)
if os.path.isfile(os.path.join(deps_dir, f))
and os.path.splitext(f)[1] == '.whl'
]
cmd = "pip install -t %s --no-index --find-links %s %s" % (
os.path.dirname(str(
args.py)), deps_dir, " ".join(wheel_file_list))
Log.debug("Install python dependencies via cmd: %s" % cmd)
retval = os.system(cmd)
if retval != 0:
print(
"Could not install user depedencies specified by the zip file"
)
sys.exit(1)
# add python user src directory to path
sys.path.insert(
0, os.path.join(os.path.dirname(str(args.py)), basename, "src"))
log_file = os.path.join(
args.logging_directory,
util.getFullyQualifiedFunctionName(function_details.tenant,
function_details.namespace,
function_details.name),
"%s-%s.log" % (args.logging_file, args.instance_id))
log.init_logger(logging.INFO, log_file, args.logging_config_file)
Log.info("Starting Python instance with %s" % str(args))
authentication = None
use_tls = False
tls_allow_insecure_connection = False
tls_trust_cert_path = None
if args.client_auth_plugin and args.client_auth_params:
authentication = pulsar.Authentication(args.client_auth_plugin,
args.client_auth_params)
if args.use_tls == "true":
use_tls = True
if args.tls_allow_insecure_connection == "true":
tls_allow_insecure_connection = True
if args.tls_trust_cert_path:
tls_trust_cert_path = args.tls_trust_cert_path
pulsar_client = pulsar.Client(
args.pulsar_serviceurl,
authentication=authentication,
operation_timeout_seconds=30,
io_threads=1,
message_listener_threads=1,
concurrent_lookup_requests=50000,
log_conf_file_path=None,
use_tls=use_tls,
tls_trust_certs_file_path=tls_trust_cert_path,
tls_allow_insecure_connection=tls_allow_insecure_connection)
state_storage_serviceurl = None
if args.state_storage_serviceurl is not None:
state_storage_serviceurl = str(args.state_storage_serviceurl)
secrets_provider = None
if args.secrets_provider is not None:
secrets_provider = util.import_class(
os.path.dirname(inspect.getfile(inspect.currentframe())),
str(args.secrets_provider))
else:
secrets_provider = util.import_class(
os.path.dirname(inspect.getfile(inspect.currentframe())),
"secretsprovider.ClearTextSecretsProvider")
secrets_provider = secrets_provider()
secrets_provider_config = None
if args.secrets_provider_config is not None:
args.secrets_provider_config = str(args.secrets_provider_config)
if args.secrets_provider_config[0] == '\'':
args.secrets_provider_config = args.secrets_provider_config[1:]
if args.secrets_provider_config[-1] == '\'':
args.secrets_provider_config = args.secrets_provider_config[:-1]
secrets_provider_config = json.loads(str(args.secrets_provider_config))
secrets_provider.init(secrets_provider_config)
pyinstance = python_instance.PythonInstance(
str(args.instance_id), str(args.function_id),
str(args.function_version), function_details,
int(args.max_buffered_tuples), int(args.expected_healthcheck_interval),
str(args.py), pulsar_client, secrets_provider, args.cluster_name,
state_storage_serviceurl)
pyinstance.run()
server_instance = server.serve(args.port, pyinstance)
# Cannot use latest version of prometheus client because of thread leak
# prometheus_client.start_http_server(args.metrics_port)
# Use patched version of prometheus
# Contains fix from https://github.com/prometheus/client_python/pull/356
# This can be removed one the fix in is a official prometheus client release
prometheus_client_fix.start_http_server(args.metrics_port)
global to_run
while to_run:
time.sleep(1)
pyinstance.join()
# make sure to close all non-daemon threads before this!
sys.exit(0)
def main():
# Setup signal handlers
signal.signal(signal.SIGTERM, atexit_function)
signal.signal(signal.SIGHUP, atexit_function)
signal.signal(signal.SIGINT, atexit_function)
parser = argparse.ArgumentParser(description='Pulsar Functions Python Instance')
parser.add_argument('--function_details', required=True, help='Function Details Json String')
parser.add_argument('--py', required=True, help='Full Path of Function Code File')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--function_id', required=True, help='Function Id')
parser.add_argument('--function_version', required=True, help='Function Version')
parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url')
parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin')
parser.add_argument('--client_auth_params', required=False, help='Client authentication params')
parser.add_argument('--use_tls', required=False, help='Use tls')
parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection')
parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification')
parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path')
parser.add_argument('--port', required=True, help='Instance Port', type=int)
parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples')
parser.add_argument('--logging_directory', required=True, help='Logging Directory')
parser.add_argument('--logging_file', required=True, help='Log file name')
parser.add_argument('--logging_config_file', required=True, help='Config file for logging')
parser.add_argument('--expected_healthcheck_interval', required=True, help='Expected time in seconds between health checks', type=int)
parser.add_argument('--secrets_provider', required=False, help='The classname of the secrets provider')
parser.add_argument('--secrets_provider_config', required=False, help='The config that needs to be passed to secrets provider')
parser.add_argument('--install_usercode_dependencies', required=False, help='For packaged python like wheel files, do we need to install all dependencies', type=bool)
parser.add_argument('--dependency_repository', required=False, help='For packaged python like wheel files, which repository to pull the dependencies from')
parser.add_argument('--extra_dependency_repository', required=False, help='For packaged python like wheel files, any extra repository to pull the dependencies from')
args = parser.parse_args()
function_details = Function_pb2.FunctionDetails()
args.function_details = str(args.function_details)
if args.function_details[0] == '\'':
args.function_details = args.function_details[1:]
if args.function_details[-1] == '\'':
args.function_details = args.function_details[:-1]
json_format.Parse(args.function_details, function_details)
if os.path.splitext(str(args.py))[1] == '.whl':
if args.install_usercode_dependencies:
cmd = "pip install -t %s" % os.path.dirname(str(args.py))
if args.dependency_repository:
cmd = cmd + " -i %s" % str(args.dependency_repository)
if args.extra_dependency_repository:
cmd = cmd + " --extra-index-url %s" % str(args.extra_dependency_repository)
cmd = cmd + " %s" % str(args.py)
retval = os.system(cmd)
if retval != 0:
print "Could not install user depedencies"
sys.exit(1)
else:
zpfile = zipfile.ZipFile(str(args.py), 'r')
zpfile.extractall(os.path.dirname(str(args.py)))
sys.path.insert(0, os.path.dirname(str(args.py)))
elif os.path.splitext(str(args.py))[1] == '.zip':
# Assumig zip file with format func.zip
# extract to folder function
# internal dir format
# "func/src"
# "func/requirements.txt"
# "func/deps"
# run pip install to target folder deps folder
zpfile = zipfile.ZipFile(str(args.py), 'r')
zpfile.extractall(os.path.dirname(str(args.py)))
basename = os.path.splitext(str(args.py))[0]
requirements_txt_file = os.path.join(os.path.dirname(str(args.py)), basename, "requirements.txt")
deps_file = os.path.join(os.path.dirname(str(args.py)), basename, "deps")
cmd = "pip install -t %s -r %s --no-index --find-links %s" % (os.path.dirname(str(args.py)), requirements_txt_file, deps_file)
retval = os.system(cmd)
if retval != 0:
print "Could not install user depedencies specified by the zip file"
sys.exit(1)
sys.path.insert(0, os.path.join(os.path.dirname(str(args.py)), basename, "src"))
log_file = os.path.join(args.logging_directory,
util.getFullyQualifiedFunctionName(function_details.tenant, function_details.namespace, function_details.name),
"%s-%s.log" % (args.logging_file, args.instance_id))
log.init_logger(logging.INFO, log_file, args.logging_config_file)
Log.info("Starting Python instance with %s" % str(args))
authentication = None
use_tls = False
tls_allow_insecure_connection = False
tls_trust_cert_path = None
if args.client_auth_plugin and args.client_auth_params:
authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params)
if args.use_tls == "true":
use_tls = True
if args.tls_allow_insecure_connection == "true":
tls_allow_insecure_connection = True
if args.tls_trust_cert_path:
tls_trust_cert_path = args.tls_trust_cert_path
pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30, 1, 1, 50000, None, use_tls, tls_trust_cert_path, tls_allow_insecure_connection)
secrets_provider = None
if args.secrets_provider is not None:
secrets_provider = util.import_class(os.path.dirname(inspect.getfile(inspect.currentframe())), str(args.secrets_provider))
else:
secrets_provider = util.import_class(os.path.dirname(inspect.getfile(inspect.currentframe())), "secretsprovider.ClearTextSecretsProvider")
secrets_provider = secrets_provider()
secrets_provider_config = None
if args.secrets_provider_config is not None:
secrets_provider_config = json.loads(str(args.secrets_provider_config))
secrets_provider.init(secrets_provider_config)
pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id),
str(args.function_version), function_details,
int(args.max_buffered_tuples),
int(args.expected_healthcheck_interval),
str(args.py), pulsar_client, secrets_provider)
pyinstance.run()
server_instance = server.serve(args.port, pyinstance)
global to_run
while to_run:
time.sleep(1)
pyinstance.join()
sys.exit(1)
def main():
# Setup signal handlers
signal.signal(signal.SIGTERM, atexit_function)
signal.signal(signal.SIGHUP, atexit_function)
signal.signal(signal.SIGINT, atexit_function)
parser = argparse.ArgumentParser(
description='Pulsar Functions Python Instance')
parser.add_argument('--function_details',
required=True,
help='Function Details Json String')
parser.add_argument('--py',
required=True,
help='Full Path of Function Code File')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--function_id', required=True, help='Function Id')
parser.add_argument('--function_version',
required=True,
help='Function Version')
parser.add_argument('--pulsar_serviceurl',
required=True,
help='Pulsar Service Url')
parser.add_argument('--client_auth_plugin',
required=False,
help='Client authentication plugin')
parser.add_argument('--client_auth_params',
required=False,
help='Client authentication params')
parser.add_argument('--use_tls', required=False, help='Use tls')
parser.add_argument('--tls_allow_insecure_connection',
required=False,
help='Tls allow insecure connection')
parser.add_argument('--hostname_verification_enabled',
required=False,
help='Enable hostname verification')
parser.add_argument('--tls_trust_cert_path',
required=False,
help='Tls trust cert file path')
parser.add_argument('--port',
required=True,
help='Instance Port',
type=int)
parser.add_argument('--max_buffered_tuples',
required=True,
help='Maximum number of Buffered tuples')
parser.add_argument('--logging_directory',
required=True,
help='Logging Directory')
parser.add_argument('--logging_file', required=True, help='Log file name')
parser.add_argument('--expected_healthcheck_interval',
required=True,
help='Expected time in seconds between health checks',
type=int)
parser.add_argument(
'--install_usercode_dependencies',
required=False,
help=
'For packaged python like wheel files, do we need to install all dependencies',
type=bool)
args = parser.parse_args()
function_details = Function_pb2.FunctionDetails()
args.function_details = str(args.function_details)
if args.function_details[0] == '\'':
args.function_details = args.function_details[1:]
if args.function_details[-1] == '\'':
args.function_details = args.function_details[:-1]
json_format.Parse(args.function_details, function_details)
if os.path.splitext(str(args.py))[1] == '.whl':
if args.install_usercode_dependencies:
os.system("pip install -t %s %s" %
(os.path.dirname(str(args.py)), str(args.py)))
else:
zpfile = zipfile.ZipFile(str(args.py), 'r')
zpfile.extractall(os.path.dirname(str(args.py)))
sys.path.insert(0, os.path.dirname(str(args.py)))
log_file = os.path.join(
args.logging_directory,
util.getFullyQualifiedFunctionName(function_details.tenant,
function_details.namespace,
function_details.name),
"%s-%s.log" % (args.logging_file, args.instance_id))
log.init_rotating_logger(level=logging.INFO,
logfile=log_file,
max_files=5,
max_bytes=10 * 1024 * 1024)
Log.info("Starting Python instance with %s" % str(args))
authentication = None
use_tls = False
tls_allow_insecure_connection = False
tls_trust_cert_path = None
if args.client_auth_plugin and args.client_auth_params:
authentication = pulsar.Authentication(args.client_auth_plugin,
args.client_auth_params)
if args.use_tls == "true":
use_tls = True
if args.tls_allow_insecure_connection == "true":
tls_allow_insecure_connection = True
if args.tls_trust_cert_path:
tls_trust_cert_path = args.tls_trust_cert_path
pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30,
1, 1, 50000, None, use_tls,
tls_trust_cert_path,
tls_allow_insecure_connection)
pyinstance = python_instance.PythonInstance(
str(args.instance_id), str(args.function_id),
str(args.function_version), function_details,
int(args.max_buffered_tuples), int(args.expected_healthcheck_interval),
str(args.py), pulsar_client)
pyinstance.run()
server_instance = server.serve(args.port, pyinstance)
global to_run
while to_run:
time.sleep(1)
pyinstance.join()
sys.exit(1)
def main():
# Setup signal handlers
signal.signal(signal.SIGTERM, atexit_function)
signal.signal(signal.SIGHUP, atexit_function)
signal.signal(signal.SIGINT, atexit_function)
parser = argparse.ArgumentParser(description='Pulsar Functions Python Instance')
parser.add_argument('--function_classname', required=True, help='Function Class Name')
parser.add_argument('--py', required=True, help='Full Path of Function Code File')
parser.add_argument('--name', required=True, help='Function Name')
parser.add_argument('--tenant', required=True, help='Tenant Name')
parser.add_argument('--namespace', required=True, help='Namespace name')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--function_id', required=True, help='Function Id')
parser.add_argument('--function_version', required=True, help='Function Version')
parser.add_argument('--processing_guarantees', required=True, help='Processing Guarantees')
parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url')
parser.add_argument('--port', required=True, help='Instance Port', type=int)
parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples')
parser.add_argument('--user_config', required=False, help='User Config')
parser.add_argument('--logging_directory', required=True, help='Logging Directory')
parser.add_argument('--logging_file', required=True, help='Log file name')
parser.add_argument('--auto_ack', required=True, help='Enable Autoacking?')
parser.add_argument('--log_topic', required=False, help='Topic to send Log Messages')
parser.add_argument('--source_subscription_type', required=True, help='Subscription Type')
parser.add_argument('--source_topics_serde_classname', required=True, help='A mapping of Input topics to SerDe')
parser.add_argument('--sink_topic', required=False, help='Sink Topic')
parser.add_argument('--sink_serde_classname', required=False, help='Sink SerDe classname')
args = parser.parse_args()
log_file = os.path.join(args.logging_directory,
util.getFullyQualifiedFunctionName(args.tenant, args.namespace, args.name),
"%s-%s.log" % (args.logging_file, args.instance_id))
log.init_rotating_logger(level=logging.INFO, logfile=log_file,
max_files=5, max_bytes=10 * 1024 * 1024)
Log.info("Starting Python instance with %s" % str(args))
function_details = Function_pb2.FunctionDetails()
function_details.tenant = args.tenant
function_details.namespace = args.namespace
function_details.name = args.name
function_details.className = args.function_classname
sourceSpec = Function_pb2.SourceSpec()
sourceSpec.subscriptionType = Function_pb2.SubscriptionType.Value(args.source_subscription_type)
try:
source_topics_serde_classname_dict = json.loads(args.source_topics_serde_classname)
except ValueError:
log.critical("Cannot decode source_topics_serde_classname. This argument must be specifed as a JSON")
sys.exit(1)
if not source_topics_serde_classname_dict:
log.critical("source_topics_serde_classname cannot be empty")
for topics, serde_classname in source_topics_serde_classname_dict.items():
sourceSpec.topicsToSerDeClassName[topics] = serde_classname
function_details.source.MergeFrom(sourceSpec)
sinkSpec = Function_pb2.SinkSpec()
if args.sink_topic != None and len(args.sink_topic) != 0:
sinkSpec.topic = args.sink_topic
if args.sink_serde_classname != None and len(args.sink_serde_classname) != 0:
sinkSpec.serDeClassName = args.sink_serde_classname
function_details.sink.MergeFrom(sinkSpec)
function_details.processingGuarantees = Function_pb2.ProcessingGuarantees.Value(args.processing_guarantees)
if args.auto_ack == "true":
function_details.autoAck = True
else:
function_details.autoAck = False
if args.user_config != None and len(args.user_config) != 0:
function_details.userConfig = args.user_config
pulsar_client = pulsar.Client(args.pulsar_serviceurl)
pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id),
str(args.function_version), function_details,
int(args.max_buffered_tuples), str(args.py),
args.log_topic, pulsar_client)
pyinstance.run()
server_instance = server.serve(args.port, pyinstance)
global to_run
while to_run:
time.sleep(1)
pyinstance.join()
sys.exit(1)
def main():
# Setup signal handlers
signal.signal(signal.SIGTERM, atexit_function)
signal.signal(signal.SIGHUP, atexit_function)
signal.signal(signal.SIGINT, atexit_function)
parser = argparse.ArgumentParser(
description='Pulsar Functions Python Instance')
parser.add_argument('--function_classname',
required=True,
help='Function Class Name')
parser.add_argument('--py',
required=True,
help='Full Path of Function Code File')
parser.add_argument('--name', required=True, help='Function Name')
parser.add_argument('--tenant', required=True, help='Tenant Name')
parser.add_argument('--namespace', required=True, help='Namespace name')
parser.add_argument('--custom_serde_input_topics',
required=False,
help='Input Topics Requiring Custom Deserialization')
parser.add_argument('--custom_serde_classnames',
required=False,
help='Input Serde Classnames')
parser.add_argument('--input_topics',
required=False,
help='Input topics with default serde')
parser.add_argument('--output_topic', required=False, help='Output Topic')
parser.add_argument('--output_serde_classname',
required=False,
help='Output Serde Classnames')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--function_id', required=True, help='Function Id')
parser.add_argument('--function_version',
required=True,
help='Function Version')
parser.add_argument('--processing_guarantees',
required=True,
help='Processing Guarantees')
parser.add_argument('--subscription_type',
required=True,
help='Subscription Type')
parser.add_argument('--pulsar_serviceurl',
required=True,
help='Pulsar Service Url')
parser.add_argument('--port',
required=True,
help='Instance Port',
type=int)
parser.add_argument('--max_buffered_tuples',
required=True,
help='Maximum number of Buffered tuples')
parser.add_argument('--user_config', required=False, help='User Config')
parser.add_argument('--logging_directory',
required=True,
help='Logging Directory')
parser.add_argument('--logging_file', required=True, help='Log file name')
parser.add_argument('--auto_ack', required=True, help='Enable Autoacking?')
parser.add_argument('--log_topic',
required=False,
help='Topic to send Log Messages')
args = parser.parse_args()
log_file = os.path.join(
args.logging_directory,
util.getFullyQualifiedFunctionName(args.tenant, args.namespace,
args.name),
"%s-%s.log" % (args.logging_file, args.instance_id))
log.init_rotating_logger(level=logging.INFO,
logfile=log_file,
max_files=5,
max_bytes=10 * 1024 * 1024)
Log.info("Starting Python instance with %s" % str(args))
function_details = Function_pb2.FunctionDetails()
function_details.tenant = args.tenant
function_details.namespace = args.namespace
function_details.name = args.name
function_details.className = args.function_classname
if args.custom_serde_input_topics is None and args.input_topics is None:
Log.critical("Atleast one input topic must be present")
sys.exit(1)
if args.custom_serde_input_topics is not None and args.custom_serde_classnames is not None:
input_topics = args.custom_serde_input_topics.split(",")
input_serde = args.custom_serde_classnames.split(",")
if len(input_topics) != len(input_serde):
Log.critical(
"CustomSerde InputTopcis and Serde classnames should match")
sys.exit(1)
for i in xrange(len(input_topics)):
function_details.customSerdeInputs[
input_topics[i]] = input_serde[i]
if args.input_topics is not None:
for topic in args.input_topics.split(","):
function_details.inputs.append(topic)
if args.output_topic != None and len(args.output_topic) != 0:
function_details.output = args.output_topic
if args.output_serde_classname != None and len(
args.output_serde_classname) != 0:
function_details.outputSerdeClassName = args.output_serde_classname
function_details.processingGuarantees = Function_pb2.FunctionDetails.ProcessingGuarantees.Value(
args.processing_guarantees)
function_details.subscriptionType = Function_pb2.FunctionDetails.SubscriptionType.Values(
args.subscription_type)
if args.auto_ack == "true":
function_details.autoAck = True
else:
function_details.autoAck = False
if args.user_config != None and len(args.user_config) != 0:
user_config = json.loads(args.user_config)
for (key, value) in user_config.items():
function_details.userConfig[str(key)] = str(value)
pulsar_client = pulsar.Client(args.pulsar_serviceurl)
pyinstance = python_instance.PythonInstance(str(args.instance_id),
str(args.function_id),
str(args.function_version),
function_details,
int(args.max_buffered_tuples),
str(args.py), args.log_topic,
pulsar_client)
pyinstance.run()
server_instance = server.serve(args.port, pyinstance)
global to_run
while to_run:
time.sleep(1)
pyinstance.join()
sys.exit(1)
def main():
# Setup signal handlers
signal.signal(signal.SIGTERM, atexit_function)
signal.signal(signal.SIGHUP, atexit_function)
signal.signal(signal.SIGINT, atexit_function)
parser = argparse.ArgumentParser(
description='Pulsar Functions Python Instance')
parser.add_argument('--function_classname',
required=True,
help='Function Class Name')
parser.add_argument('--py',
required=True,
help='Full Path of Function Code File')
parser.add_argument('--name', required=True, help='Function Name')
parser.add_argument('--tenant', required=True, help='Tenant Name')
parser.add_argument('--namespace', required=True, help='Namespace name')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--function_id', required=True, help='Function Id')
parser.add_argument('--function_version',
required=True,
help='Function Version')
parser.add_argument('--processing_guarantees',
required=True,
help='Processing Guarantees')
parser.add_argument('--pulsar_serviceurl',
required=True,
help='Pulsar Service Url')
parser.add_argument('--client_auth_plugin',
required=False,
help='Client authentication plugin')
parser.add_argument('--client_auth_params',
required=False,
help='Client authentication params')
parser.add_argument('--use_tls', required=False, help='Use tls')
parser.add_argument('--tls_allow_insecure_connection',
required=False,
help='Tls allow insecure connection')
parser.add_argument('--hostname_verification_enabled',
required=False,
help='Enable hostname verification')
parser.add_argument('--tls_trust_cert_path',
required=False,
help='Tls trust cert file path')
parser.add_argument('--port',
required=True,
help='Instance Port',
type=int)
parser.add_argument('--max_buffered_tuples',
required=True,
help='Maximum number of Buffered tuples')
parser.add_argument('--user_config', required=False, help='User Config')
parser.add_argument('--logging_directory',
required=True,
help='Logging Directory')
parser.add_argument('--logging_file', required=True, help='Log file name')
parser.add_argument('--auto_ack', required=True, help='Enable Autoacking?')
parser.add_argument('--log_topic',
required=False,
help='Topic to send Log Messages')
parser.add_argument('--source_subscription_type',
required=True,
help='Subscription Type')
parser.add_argument('--source_topics_serde_classname',
required=True,
help='A mapping of Input topics to SerDe')
parser.add_argument(
'--topics_pattern',
required=False,
help=
'TopicsPattern to consume from list of topics under a namespace that match the pattern (not supported)'
)
parser.add_argument('--source_timeout_ms',
required=False,
help='Source message timeout in milliseconds')
parser.add_argument('--sink_topic', required=False, help='Sink Topic')
parser.add_argument('--sink_serde_classname',
required=False,
help='Sink SerDe classname')
args = parser.parse_args()
log_file = os.path.join(
args.logging_directory,
util.getFullyQualifiedFunctionName(args.tenant, args.namespace,
args.name),
"%s-%s.log" % (args.logging_file, args.instance_id))
log.init_rotating_logger(level=logging.INFO,
logfile=log_file,
max_files=5,
max_bytes=10 * 1024 * 1024)
Log.info("Starting Python instance with %s" % str(args))
function_details = Function_pb2.FunctionDetails()
function_details.tenant = args.tenant
function_details.namespace = args.namespace
function_details.name = args.name
function_details.className = args.function_classname
if args.topics_pattern:
raise ValueError('topics_pattern is not supported by python client')
sourceSpec = Function_pb2.SourceSpec()
sourceSpec.subscriptionType = Function_pb2.SubscriptionType.Value(
args.source_subscription_type)
try:
source_topics_serde_classname_dict = json.loads(
args.source_topics_serde_classname)
except ValueError:
Log.critical(
"Cannot decode source_topics_serde_classname. This argument must be specifed as a JSON"
)
sys.exit(1)
if not source_topics_serde_classname_dict:
Log.critical("source_topics_serde_classname cannot be empty")
for topics, serde_classname in source_topics_serde_classname_dict.items():
sourceSpec.topicsToSerDeClassName[topics] = serde_classname
if args.source_timeout_ms:
sourceSpec.timeoutMs = long(args.source_timeout_ms)
function_details.source.MergeFrom(sourceSpec)
sinkSpec = Function_pb2.SinkSpec()
if args.sink_topic != None and len(args.sink_topic) != 0:
sinkSpec.topic = args.sink_topic
if args.sink_serde_classname != None and len(
args.sink_serde_classname) != 0:
sinkSpec.serDeClassName = args.sink_serde_classname
function_details.sink.MergeFrom(sinkSpec)
function_details.processingGuarantees = Function_pb2.ProcessingGuarantees.Value(
args.processing_guarantees)
if args.auto_ack == "true":
function_details.autoAck = True
else:
function_details.autoAck = False
if args.user_config != None and len(args.user_config) != 0:
function_details.userConfig = args.user_config
authentication = None
use_tls = False
tls_allow_insecure_connection = False
tls_trust_cert_path = None
if args.client_auth_plugin and args.client_auth_params:
authentication = pulsar.Authentication(args.client_auth_plugin,
args.client_auth_params)
if args.use_tls == "true":
use_tls = True
if args.tls_allow_insecure_connection == "true":
tls_allow_insecure_connection = True
if args.tls_trust_cert_path:
tls_trust_cert_path = args.tls_trust_cert_path
pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30,
1, 1, 50000, None, use_tls,
tls_trust_cert_path,
tls_allow_insecure_connection)
pyinstance = python_instance.PythonInstance(str(args.instance_id),
str(args.function_id),
str(args.function_version),
function_details,
int(args.max_buffered_tuples),
str(args.py), args.log_topic,
pulsar_client)
pyinstance.run()
server_instance = server.serve(args.port, pyinstance)
global to_run
while to_run:
time.sleep(1)
pyinstance.join()
sys.exit(1)
