def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser( description='Pulsar Functions Python Instance') parser.add_argument('--function_classname', required=True, help='Function Class Name') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--name', required=True, help='Function Name') parser.add_argument('--tenant', required=True, help='Tenant Name') parser.add_argument('--namespace', required=True, help='Namespace name') parser.add_argument('--custom_serde_input_topics', required=False, help='Input Topics Requiring Custom Deserialization') parser.add_argument('--custom_serde_classnames', required=False, help='Input Serde Classnames') parser.add_argument('--input_topics', required=False, help='Input topics with default serde') parser.add_argument('--output_topic', required=False, help='Output Topic') parser.add_argument('--output_serde_classname', required=False, help='Output Serde Classnames') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--processing_guarantees', required=True, help='Processing Guarantees') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--user_config', required=False, help='User Config') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--auto_ack', required=True, help='Enable Autoacking?') parser.add_argument('--log_topic', required=False, help='Topic to send Log Messages') args = parser.parse_args() log_file = os.path.join(args.logging_directory, args.logging_file + ".log.0") log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=5, max_bytes=10 * 1024 * 1024) Log.info("Starting Python instance with %s" % str(args)) function_config = Function_pb2.FunctionConfig() function_config.tenant = args.tenant function_config.namespace = args.namespace function_config.name = args.name function_config.className = args.function_classname if args.custom_serde_input_topics is None and args.input_topics is None: Log.critical("Atleast one input topic must be present") sys.exit(1) if args.custom_serde_input_topics is not None and args.custom_serde_classnames is not None: input_topics = args.custom_serde_input_topics.split(",") input_serde = args.custom_serde_classnames.split(",") if len(input_topics) != len(input_serde): Log.critical( "CustomSerde InputTopcis and Serde classnames should match") sys.exit(1) for i in xrange(len(input_topics)): function_config.customSerdeInputs[input_topics[i]] = input_serde[i] if args.input_topics is not None: for topic in args.input_topics.split(","): function_config.inputs.append(topic) if args.output_topic != None and len(args.output_topic) != 0: function_config.output = args.output_topic if args.output_serde_classname != None and len( args.output_serde_classname) != 0: function_config.outputSerdeClassName = args.output_serde_classname function_config.processingGuarantees = Function_pb2.FunctionConfig.ProcessingGuarantees.Value( args.processing_guarantees) if args.auto_ack == "true": function_config.autoAck = True else: function_config.autoAck = False if args.user_config != None and len(args.user_config) != 0: user_config = json.loads(args.user_config) for (key, value) in user_config.items(): function_config.userConfig[str(key)] = str(value) pulsar_client = pulsar.Client(args.pulsar_serviceurl) pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), str(args.function_version), function_config, int(args.max_buffered_tuples), str(args.py), args.log_topic, pulsar_client) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser( description='Pulsar Functions Python Instance') parser.add_argument('--function_details', required=True, help='Function Details Json String') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin') parser.add_argument('--client_auth_params', required=False, help='Client authentication params') parser.add_argument('--use_tls', required=False, help='Use tls') parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection') parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification') parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') args = parser.parse_args() function_details = Function_pb2.FunctionDetails() json_format.Parse(args.function_details, function_details) log_file = os.path.join( args.logging_directory, util.getFullyQualifiedFunctionName(function_details.tenant, function_details.namespace, function_details.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=5, max_bytes=10 * 1024 * 1024) Log.info("Starting Python instance with %s" % str(args)) authentication = None use_tls = False tls_allow_insecure_connection = False tls_trust_cert_path = None if args.client_auth_plugin and args.client_auth_params: authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params) if args.use_tls == "true": use_tls = True if args.tls_allow_insecure_connection == "true": tls_allow_insecure_connection = True if args.tls_trust_cert_path: tls_trust_cert_path = args.tls_trust_cert_path pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30, 1, 1, 50000, None, use_tls, tls_trust_cert_path, tls_allow_insecure_connection) pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), str(args.py), pulsar_client) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser( description='Pulsar Functions Python Instance') parser.add_argument('--function_details', required=True, help='Function Details Json String') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin') parser.add_argument('--client_auth_params', required=False, help='Client authentication params') parser.add_argument('--use_tls', required=False, help='Use tls') parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection') parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification') parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--metrics_port', required=True, help="Port metrics will be exposed on", type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--logging_config_file', required=True, help='Config file for logging') parser.add_argument('--expected_healthcheck_interval', required=True, help='Expected time in seconds between health checks', type=int) parser.add_argument('--secrets_provider', required=False, help='The classname of the secrets provider') parser.add_argument( '--secrets_provider_config', required=False, help='The config that needs to be passed to secrets provider') parser.add_argument( '--install_usercode_dependencies', required=False, help= 'For packaged python like wheel files, do we need to install all dependencies', type=bool) parser.add_argument( '--dependency_repository', required=False, help= 'For packaged python like wheel files, which repository to pull the dependencies from' ) parser.add_argument( '--extra_dependency_repository', required=False, help= 'For packaged python like wheel files, any extra repository to pull the dependencies from' ) parser.add_argument('--state_storage_serviceurl', required=False, help='Managed State Storage Service Url') parser.add_argument( '--cluster_name', required=True, help='The name of the cluster this instance is running on') args = parser.parse_args() function_details = Function_pb2.FunctionDetails() args.function_details = str(args.function_details) if args.function_details[0] == '\'': args.function_details = args.function_details[1:] if args.function_details[-1] == '\'': args.function_details = args.function_details[:-1] json_format.Parse(args.function_details, function_details) if os.path.splitext(str(args.py))[1] == '.whl': if args.install_usercode_dependencies: cmd = "pip install -t %s" % os.path.dirname(str(args.py)) if args.dependency_repository: cmd = cmd + " -i %s" % str(args.dependency_repository) if args.extra_dependency_repository: cmd = cmd + " --extra-index-url %s" % str( args.extra_dependency_repository) cmd = cmd + " %s" % str(args.py) retval = os.system(cmd) if retval != 0: print("Could not install user depedencies") sys.exit(1) else: zpfile = zipfile.ZipFile(str(args.py), 'r') zpfile.extractall(os.path.dirname(str(args.py))) sys.path.insert(0, os.path.dirname(str(args.py))) elif os.path.splitext(str(args.py))[1] == '.zip': # Assumig zip file with format func.zip # extract to folder function # internal dir format # "func/src" # "func/requirements.txt" # "func/deps" # run pip install to target folder deps folder zpfile = zipfile.ZipFile(str(args.py), 'r') zpfile.extractall(os.path.dirname(str(args.py))) basename = os.path.splitext(str(args.py))[0] deps_dir = os.path.join(os.path.dirname(str(args.py)), basename, "deps") if os.path.isdir(deps_dir) and os.listdir(deps_dir): # get all wheel files from deps directory wheel_file_list = [ os.path.join(deps_dir, f) for f in os.listdir(deps_dir) if os.path.isfile(os.path.join(deps_dir, f)) and os.path.splitext(f)[1] == '.whl' ] cmd = "pip install -t %s --no-index --find-links %s %s" % ( os.path.dirname(str( args.py)), deps_dir, " ".join(wheel_file_list)) Log.debug("Install python dependencies via cmd: %s" % cmd) retval = os.system(cmd) if retval != 0: print( "Could not install user depedencies specified by the zip file" ) sys.exit(1) # add python user src directory to path sys.path.insert( 0, os.path.join(os.path.dirname(str(args.py)), basename, "src")) log_file = os.path.join( args.logging_directory, util.getFullyQualifiedFunctionName(function_details.tenant, function_details.namespace, function_details.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_logger(logging.INFO, log_file, args.logging_config_file) Log.info("Starting Python instance with %s" % str(args)) authentication = None use_tls = False tls_allow_insecure_connection = False tls_trust_cert_path = None if args.client_auth_plugin and args.client_auth_params: authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params) if args.use_tls == "true": use_tls = True if args.tls_allow_insecure_connection == "true": tls_allow_insecure_connection = True if args.tls_trust_cert_path: tls_trust_cert_path = args.tls_trust_cert_path pulsar_client = pulsar.Client( args.pulsar_serviceurl, authentication=authentication, operation_timeout_seconds=30, io_threads=1, message_listener_threads=1, concurrent_lookup_requests=50000, log_conf_file_path=None, use_tls=use_tls, tls_trust_certs_file_path=tls_trust_cert_path, tls_allow_insecure_connection=tls_allow_insecure_connection) state_storage_serviceurl = None if args.state_storage_serviceurl is not None: state_storage_serviceurl = str(args.state_storage_serviceurl) secrets_provider = None if args.secrets_provider is not None: secrets_provider = util.import_class( os.path.dirname(inspect.getfile(inspect.currentframe())), str(args.secrets_provider)) else: secrets_provider = util.import_class( os.path.dirname(inspect.getfile(inspect.currentframe())), "secretsprovider.ClearTextSecretsProvider") secrets_provider = secrets_provider() secrets_provider_config = None if args.secrets_provider_config is not None: args.secrets_provider_config = str(args.secrets_provider_config) if args.secrets_provider_config[0] == '\'': args.secrets_provider_config = args.secrets_provider_config[1:] if args.secrets_provider_config[-1] == '\'': args.secrets_provider_config = args.secrets_provider_config[:-1] secrets_provider_config = json.loads(str(args.secrets_provider_config)) secrets_provider.init(secrets_provider_config) pyinstance = python_instance.PythonInstance( str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), int(args.expected_healthcheck_interval), str(args.py), pulsar_client, secrets_provider, args.cluster_name, state_storage_serviceurl) pyinstance.run() server_instance = server.serve(args.port, pyinstance) # Cannot use latest version of prometheus client because of thread leak # prometheus_client.start_http_server(args.metrics_port) # Use patched version of prometheus # Contains fix from https://github.com/prometheus/client_python/pull/356 # This can be removed one the fix in is a official prometheus client release prometheus_client_fix.start_http_server(args.metrics_port) global to_run while to_run: time.sleep(1) pyinstance.join() # make sure to close all non-daemon threads before this! sys.exit(0)
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser(description='Pulsar Functions Python Instance') parser.add_argument('--function_details', required=True, help='Function Details Json String') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin') parser.add_argument('--client_auth_params', required=False, help='Client authentication params') parser.add_argument('--use_tls', required=False, help='Use tls') parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection') parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification') parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--logging_config_file', required=True, help='Config file for logging') parser.add_argument('--expected_healthcheck_interval', required=True, help='Expected time in seconds between health checks', type=int) parser.add_argument('--secrets_provider', required=False, help='The classname of the secrets provider') parser.add_argument('--secrets_provider_config', required=False, help='The config that needs to be passed to secrets provider') parser.add_argument('--install_usercode_dependencies', required=False, help='For packaged python like wheel files, do we need to install all dependencies', type=bool) parser.add_argument('--dependency_repository', required=False, help='For packaged python like wheel files, which repository to pull the dependencies from') parser.add_argument('--extra_dependency_repository', required=False, help='For packaged python like wheel files, any extra repository to pull the dependencies from') args = parser.parse_args() function_details = Function_pb2.FunctionDetails() args.function_details = str(args.function_details) if args.function_details[0] == '\'': args.function_details = args.function_details[1:] if args.function_details[-1] == '\'': args.function_details = args.function_details[:-1] json_format.Parse(args.function_details, function_details) if os.path.splitext(str(args.py))[1] == '.whl': if args.install_usercode_dependencies: cmd = "pip install -t %s" % os.path.dirname(str(args.py)) if args.dependency_repository: cmd = cmd + " -i %s" % str(args.dependency_repository) if args.extra_dependency_repository: cmd = cmd + " --extra-index-url %s" % str(args.extra_dependency_repository) cmd = cmd + " %s" % str(args.py) retval = os.system(cmd) if retval != 0: print "Could not install user depedencies" sys.exit(1) else: zpfile = zipfile.ZipFile(str(args.py), 'r') zpfile.extractall(os.path.dirname(str(args.py))) sys.path.insert(0, os.path.dirname(str(args.py))) elif os.path.splitext(str(args.py))[1] == '.zip': # Assumig zip file with format func.zip # extract to folder function # internal dir format # "func/src" # "func/requirements.txt" # "func/deps" # run pip install to target folder deps folder zpfile = zipfile.ZipFile(str(args.py), 'r') zpfile.extractall(os.path.dirname(str(args.py))) basename = os.path.splitext(str(args.py))[0] requirements_txt_file = os.path.join(os.path.dirname(str(args.py)), basename, "requirements.txt") deps_file = os.path.join(os.path.dirname(str(args.py)), basename, "deps") cmd = "pip install -t %s -r %s --no-index --find-links %s" % (os.path.dirname(str(args.py)), requirements_txt_file, deps_file) retval = os.system(cmd) if retval != 0: print "Could not install user depedencies specified by the zip file" sys.exit(1) sys.path.insert(0, os.path.join(os.path.dirname(str(args.py)), basename, "src")) log_file = os.path.join(args.logging_directory, util.getFullyQualifiedFunctionName(function_details.tenant, function_details.namespace, function_details.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_logger(logging.INFO, log_file, args.logging_config_file) Log.info("Starting Python instance with %s" % str(args)) authentication = None use_tls = False tls_allow_insecure_connection = False tls_trust_cert_path = None if args.client_auth_plugin and args.client_auth_params: authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params) if args.use_tls == "true": use_tls = True if args.tls_allow_insecure_connection == "true": tls_allow_insecure_connection = True if args.tls_trust_cert_path: tls_trust_cert_path = args.tls_trust_cert_path pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30, 1, 1, 50000, None, use_tls, tls_trust_cert_path, tls_allow_insecure_connection) secrets_provider = None if args.secrets_provider is not None: secrets_provider = util.import_class(os.path.dirname(inspect.getfile(inspect.currentframe())), str(args.secrets_provider)) else: secrets_provider = util.import_class(os.path.dirname(inspect.getfile(inspect.currentframe())), "secretsprovider.ClearTextSecretsProvider") secrets_provider = secrets_provider() secrets_provider_config = None if args.secrets_provider_config is not None: secrets_provider_config = json.loads(str(args.secrets_provider_config)) secrets_provider.init(secrets_provider_config) pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), int(args.expected_healthcheck_interval), str(args.py), pulsar_client, secrets_provider) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser( description='Pulsar Functions Python Instance') parser.add_argument('--function_details', required=True, help='Function Details Json String') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin') parser.add_argument('--client_auth_params', required=False, help='Client authentication params') parser.add_argument('--use_tls', required=False, help='Use tls') parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection') parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification') parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--expected_healthcheck_interval', required=True, help='Expected time in seconds between health checks', type=int) parser.add_argument( '--install_usercode_dependencies', required=False, help= 'For packaged python like wheel files, do we need to install all dependencies', type=bool) args = parser.parse_args() function_details = Function_pb2.FunctionDetails() args.function_details = str(args.function_details) if args.function_details[0] == '\'': args.function_details = args.function_details[1:] if args.function_details[-1] == '\'': args.function_details = args.function_details[:-1] json_format.Parse(args.function_details, function_details) if os.path.splitext(str(args.py))[1] == '.whl': if args.install_usercode_dependencies: os.system("pip install -t %s %s" % (os.path.dirname(str(args.py)), str(args.py))) else: zpfile = zipfile.ZipFile(str(args.py), 'r') zpfile.extractall(os.path.dirname(str(args.py))) sys.path.insert(0, os.path.dirname(str(args.py))) log_file = os.path.join( args.logging_directory, util.getFullyQualifiedFunctionName(function_details.tenant, function_details.namespace, function_details.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=5, max_bytes=10 * 1024 * 1024) Log.info("Starting Python instance with %s" % str(args)) authentication = None use_tls = False tls_allow_insecure_connection = False tls_trust_cert_path = None if args.client_auth_plugin and args.client_auth_params: authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params) if args.use_tls == "true": use_tls = True if args.tls_allow_insecure_connection == "true": tls_allow_insecure_connection = True if args.tls_trust_cert_path: tls_trust_cert_path = args.tls_trust_cert_path pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30, 1, 1, 50000, None, use_tls, tls_trust_cert_path, tls_allow_insecure_connection) pyinstance = python_instance.PythonInstance( str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), int(args.expected_healthcheck_interval), str(args.py), pulsar_client) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser(description='Pulsar Functions Python Instance') parser.add_argument('--function_classname', required=True, help='Function Class Name') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--name', required=True, help='Function Name') parser.add_argument('--tenant', required=True, help='Tenant Name') parser.add_argument('--namespace', required=True, help='Namespace name') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--processing_guarantees', required=True, help='Processing Guarantees') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--user_config', required=False, help='User Config') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--auto_ack', required=True, help='Enable Autoacking?') parser.add_argument('--log_topic', required=False, help='Topic to send Log Messages') parser.add_argument('--source_subscription_type', required=True, help='Subscription Type') parser.add_argument('--source_topics_serde_classname', required=True, help='A mapping of Input topics to SerDe') parser.add_argument('--sink_topic', required=False, help='Sink Topic') parser.add_argument('--sink_serde_classname', required=False, help='Sink SerDe classname') args = parser.parse_args() log_file = os.path.join(args.logging_directory, util.getFullyQualifiedFunctionName(args.tenant, args.namespace, args.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=5, max_bytes=10 * 1024 * 1024) Log.info("Starting Python instance with %s" % str(args)) function_details = Function_pb2.FunctionDetails() function_details.tenant = args.tenant function_details.namespace = args.namespace function_details.name = args.name function_details.className = args.function_classname sourceSpec = Function_pb2.SourceSpec() sourceSpec.subscriptionType = Function_pb2.SubscriptionType.Value(args.source_subscription_type) try: source_topics_serde_classname_dict = json.loads(args.source_topics_serde_classname) except ValueError: log.critical("Cannot decode source_topics_serde_classname. This argument must be specifed as a JSON") sys.exit(1) if not source_topics_serde_classname_dict: log.critical("source_topics_serde_classname cannot be empty") for topics, serde_classname in source_topics_serde_classname_dict.items(): sourceSpec.topicsToSerDeClassName[topics] = serde_classname function_details.source.MergeFrom(sourceSpec) sinkSpec = Function_pb2.SinkSpec() if args.sink_topic != None and len(args.sink_topic) != 0: sinkSpec.topic = args.sink_topic if args.sink_serde_classname != None and len(args.sink_serde_classname) != 0: sinkSpec.serDeClassName = args.sink_serde_classname function_details.sink.MergeFrom(sinkSpec) function_details.processingGuarantees = Function_pb2.ProcessingGuarantees.Value(args.processing_guarantees) if args.auto_ack == "true": function_details.autoAck = True else: function_details.autoAck = False if args.user_config != None and len(args.user_config) != 0: function_details.userConfig = args.user_config pulsar_client = pulsar.Client(args.pulsar_serviceurl) pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), str(args.py), args.log_topic, pulsar_client) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser( description='Pulsar Functions Python Instance') parser.add_argument('--function_classname', required=True, help='Function Class Name') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--name', required=True, help='Function Name') parser.add_argument('--tenant', required=True, help='Tenant Name') parser.add_argument('--namespace', required=True, help='Namespace name') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--processing_guarantees', required=True, help='Processing Guarantees') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin') parser.add_argument('--client_auth_params', required=False, help='Client authentication params') parser.add_argument('--use_tls', required=False, help='Use tls') parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection') parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification') parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--user_config', required=False, help='User Config') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--auto_ack', required=True, help='Enable Autoacking?') parser.add_argument('--log_topic', required=False, help='Topic to send Log Messages') parser.add_argument('--source_subscription_type', required=True, help='Subscription Type') parser.add_argument('--source_topics_serde_classname', required=True, help='A mapping of Input topics to SerDe') parser.add_argument( '--topics_pattern', required=False, help= 'TopicsPattern to consume from list of topics under a namespace that match the pattern (not supported)' ) parser.add_argument('--source_timeout_ms', required=False, help='Source message timeout in milliseconds') parser.add_argument('--sink_topic', required=False, help='Sink Topic') parser.add_argument('--sink_serde_classname', required=False, help='Sink SerDe classname') args = parser.parse_args() log_file = os.path.join( args.logging_directory, util.getFullyQualifiedFunctionName(args.tenant, args.namespace, args.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=5, max_bytes=10 * 1024 * 1024) Log.info("Starting Python instance with %s" % str(args)) function_details = Function_pb2.FunctionDetails() function_details.tenant = args.tenant function_details.namespace = args.namespace function_details.name = args.name function_details.className = args.function_classname if args.topics_pattern: raise ValueError('topics_pattern is not supported by python client') sourceSpec = Function_pb2.SourceSpec() sourceSpec.subscriptionType = Function_pb2.SubscriptionType.Value( args.source_subscription_type) try: source_topics_serde_classname_dict = json.loads( args.source_topics_serde_classname) except ValueError: Log.critical( "Cannot decode source_topics_serde_classname. This argument must be specifed as a JSON" ) sys.exit(1) if not source_topics_serde_classname_dict: Log.critical("source_topics_serde_classname cannot be empty") for topics, serde_classname in source_topics_serde_classname_dict.items(): sourceSpec.topicsToSerDeClassName[topics] = serde_classname if args.source_timeout_ms: sourceSpec.timeoutMs = long(args.source_timeout_ms) function_details.source.MergeFrom(sourceSpec) sinkSpec = Function_pb2.SinkSpec() if args.sink_topic != None and len(args.sink_topic) != 0: sinkSpec.topic = args.sink_topic if args.sink_serde_classname != None and len( args.sink_serde_classname) != 0: sinkSpec.serDeClassName = args.sink_serde_classname function_details.sink.MergeFrom(sinkSpec) function_details.processingGuarantees = Function_pb2.ProcessingGuarantees.Value( args.processing_guarantees) if args.auto_ack == "true": function_details.autoAck = True else: function_details.autoAck = False if args.user_config != None and len(args.user_config) != 0: function_details.userConfig = args.user_config authentication = None use_tls = False tls_allow_insecure_connection = False tls_trust_cert_path = None if args.client_auth_plugin and args.client_auth_params: authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params) if args.use_tls == "true": use_tls = True if args.tls_allow_insecure_connection == "true": tls_allow_insecure_connection = True if args.tls_trust_cert_path: tls_trust_cert_path = args.tls_trust_cert_path pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30, 1, 1, 50000, None, use_tls, tls_trust_cert_path, tls_allow_insecure_connection) pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), str(args.py), args.log_topic, pulsar_client) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)