def setup_producer(self): if self.instance_config.function_details.sink.topic != None and \ len(self.instance_config.function_details.sink.topic) > 0: Log.debug("Setting up producer for topic %s" % self.instance_config.function_details.sink.topic) batch_type = pulsar.BatchingType.Default if self.instance_config.function_details.sink.producerSpec.batchBuilder != None and \ len(self.instance_config.function_details.sink.producerSpec.batchBuilder) > 0: batch_builder = self.instance_config.function_details.sink.producerSpec.batchBuilder if batch_builder == "KEY_BASED": batch_type = pulsar.BatchingType.KeyBased self.producer = self.pulsar_client.create_producer( str(self.instance_config.function_details.sink.topic), block_if_queue_full=True, batching_enabled=True, batching_type=batch_type, batching_max_publish_delay_ms=10, compression_type=pulsar.CompressionType.LZ4, # set send timeout to be infinity to prevent potential deadlock with consumer # that might happen when consumer is blocked due to unacked messages send_timeout_millis=0, properties=util.get_properties( util.getFullyQualifiedFunctionName( self.instance_config.function_details.tenant, self.instance_config.function_details.namespace, self.instance_config.function_details.name), self.instance_config.instance_id))
def publish(self, topic_name, message, serde_class_name="serde.IdentitySerDe", properties=None, compression_type=None, callback=None): # Just make sure that user supplied values are properly typed topic_name = str(topic_name) serde_class_name = str(serde_class_name) pulsar_compression_type = pulsar._pulsar.CompressionType.NONE if compression_type is not None: pulsar_compression_type = compression_type if topic_name not in self.publish_producers: self.publish_producers[topic_name] = self.pulsar_client.create_producer( topic_name, block_if_queue_full=True, batching_enabled=True, batching_max_publish_delay_ms=10, compression_type=pulsar_compression_type, properties=util.get_properties(util.getFullyQualifiedFunctionName( self.instance_config.function_details.tenant, self.instance_config.function_details.namespace, self.instance_config.function_details.name), self.instance_config.instance_id) ) if serde_class_name not in self.publish_serializers: serde_klass = util.import_class(self.user_code_dir, serde_class_name) self.publish_serializers[serde_class_name] = serde_klass() output_bytes = bytes(self.publish_serializers[serde_class_name].serialize(message)) self.publish_producers[topic_name].send_async(output_bytes, partial(self.callback_wrapper, callback, topic_name, self.get_message_id()), properties=properties)
def setup_producer(self): if self.instance_config.function_details.sink.topic != None and \ len(self.instance_config.function_details.sink.topic) > 0: Log.debug("Setting up producer for topic %s" % self.instance_config.function_details.sink.topic) self.producer = self.pulsar_client.create_producer( str(self.instance_config.function_details.sink.topic), block_if_queue_full=True, batching_enabled=True, batching_max_publish_delay_ms=1, # set send timeout to be infinity to prevent potential deadlock with consumer # that might happen when consumer is blocked due to unacked messages send_timeout_millis=0, max_pending_messages=100000, properties=util.get_properties(util.getFullyQualifiedFunctionName( self.instance_config.function_details.tenant, self.instance_config.function_details.namespace, self.instance_config.function_details.name), self.instance_config.instance_id) )
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser( description='Pulsar Functions Python Instance') parser.add_argument('--function_details', required=True, help='Function Details Json String') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin') parser.add_argument('--client_auth_params', required=False, help='Client authentication params') parser.add_argument('--use_tls', required=False, help='Use tls') parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection') parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification') parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') args = parser.parse_args() function_details = Function_pb2.FunctionDetails() json_format.Parse(args.function_details, function_details) log_file = os.path.join( args.logging_directory, util.getFullyQualifiedFunctionName(function_details.tenant, function_details.namespace, function_details.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=5, max_bytes=10 * 1024 * 1024) Log.info("Starting Python instance with %s" % str(args)) authentication = None use_tls = False tls_allow_insecure_connection = False tls_trust_cert_path = None if args.client_auth_plugin and args.client_auth_params: authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params) if args.use_tls == "true": use_tls = True if args.tls_allow_insecure_connection == "true": tls_allow_insecure_connection = True if args.tls_trust_cert_path: tls_trust_cert_path = args.tls_trust_cert_path pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30, 1, 1, 50000, None, use_tls, tls_trust_cert_path, tls_allow_insecure_connection) pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), str(args.py), pulsar_client) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)
def run(self): # Setup state self.state_context = self.setup_state() # Setup consumers and input deserializers mode = pulsar._pulsar.ConsumerType.Shared if self.instance_config.function_details.source.subscriptionType == Function_pb2.SubscriptionType.Value( "FAILOVER"): mode = pulsar._pulsar.ConsumerType.Failover subscription_name = str(self.instance_config.function_details.tenant) + "/" + \ str(self.instance_config.function_details.namespace) + "/" + \ str(self.instance_config.function_details.name) properties = util.get_properties( util.getFullyQualifiedFunctionName( self.instance_config.function_details.tenant, self.instance_config.function_details.namespace, self.instance_config.function_details.name), self.instance_config.instance_id) for topic, serde in self.instance_config.function_details.source.topicsToSerDeClassName.items( ): if not serde: serde_kclass = util.import_class( os.path.dirname(self.user_code), DEFAULT_SERIALIZER) else: serde_kclass = util.import_class( os.path.dirname(self.user_code), serde) self.input_serdes[topic] = serde_kclass() Log.debug("Setting up consumer for topic %s with subname %s" % (topic, subscription_name)) self.consumers[topic] = self.pulsar_client.subscribe( str(topic), subscription_name, consumer_type=mode, message_listener=partial(self.message_listener, self.input_serdes[topic]), unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms else None, properties=properties) for topic, consumer_conf in self.instance_config.function_details.source.inputSpecs.items( ): if not consumer_conf.serdeClassName: serde_kclass = util.import_class( os.path.dirname(self.user_code), DEFAULT_SERIALIZER) else: serde_kclass = util.import_class( os.path.dirname(self.user_code), consumer_conf.serdeClassName) self.input_serdes[topic] = serde_kclass() Log.debug("Setting up consumer for topic %s with subname %s" % (topic, subscription_name)) consumer_args = { "consumer_type": mode, "message_listener": partial(self.message_listener, self.input_serdes[topic]), "unacked_messages_timeout_ms": int(self.timeout_ms) if self.timeout_ms else None, "properties": properties } if consumer_conf.HasField("receiverQueueSize"): consumer_args[ "receiver_queue_size"] = consumer_conf.receiverQueueSize.value if consumer_conf.isRegexPattern: self.consumers[topic] = self.pulsar_client.subscribe( re.compile(str(topic)), subscription_name, **consumer_args) else: self.consumers[topic] = self.pulsar_client.subscribe( str(topic), subscription_name, **consumer_args) function_kclass = util.import_class( os.path.dirname(self.user_code), self.instance_config.function_details.className) if function_kclass is None: Log.critical("Could not import User Function Module %s" % self.instance_config.function_details.className) raise NameError("Could not import User Function Module %s" % self.instance_config.function_details.className) try: self.function_class = function_kclass() except: self.function_purefunction = function_kclass self.contextimpl = contextimpl.ContextImpl( self.instance_config, Log, self.pulsar_client, self.user_code, self.consumers, self.secrets_provider, self.metrics_labels, self.state_context, self.stats) # Now launch a thread that does execution self.execution_thread = threading.Thread(target=self.actual_execution) self.execution_thread.start() # start proccess spawner health check timer self.last_health_check_ts = time.time() if self.expected_healthcheck_interval > 0: timer = util.FixedTimer(self.expected_healthcheck_interval, self.process_spawner_health_check_timer, name="health-check-timer") timer.start()
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser( description='Pulsar Functions Python Instance') parser.add_argument('--function_details', required=True, help='Function Details Json String') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin') parser.add_argument('--client_auth_params', required=False, help='Client authentication params') parser.add_argument('--use_tls', required=False, help='Use tls') parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection') parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification') parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--metrics_port', required=True, help="Port metrics will be exposed on", type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--logging_config_file', required=True, help='Config file for logging') parser.add_argument('--expected_healthcheck_interval', required=True, help='Expected time in seconds between health checks', type=int) parser.add_argument('--secrets_provider', required=False, help='The classname of the secrets provider') parser.add_argument( '--secrets_provider_config', required=False, help='The config that needs to be passed to secrets provider') parser.add_argument( '--install_usercode_dependencies', required=False, help= 'For packaged python like wheel files, do we need to install all dependencies', type=bool) parser.add_argument( '--dependency_repository', required=False, help= 'For packaged python like wheel files, which repository to pull the dependencies from' ) parser.add_argument( '--extra_dependency_repository', required=False, help= 'For packaged python like wheel files, any extra repository to pull the dependencies from' ) parser.add_argument('--state_storage_serviceurl', required=False, help='Managed State Storage Service Url') parser.add_argument( '--cluster_name', required=True, help='The name of the cluster this instance is running on') args = parser.parse_args() function_details = Function_pb2.FunctionDetails() args.function_details = str(args.function_details) if args.function_details[0] == '\'': args.function_details = args.function_details[1:] if args.function_details[-1] == '\'': args.function_details = args.function_details[:-1] json_format.Parse(args.function_details, function_details) if os.path.splitext(str(args.py))[1] == '.whl': if args.install_usercode_dependencies: cmd = "pip install -t %s" % os.path.dirname(str(args.py)) if args.dependency_repository: cmd = cmd + " -i %s" % str(args.dependency_repository) if args.extra_dependency_repository: cmd = cmd + " --extra-index-url %s" % str( args.extra_dependency_repository) cmd = cmd + " %s" % str(args.py) retval = os.system(cmd) if retval != 0: print("Could not install user depedencies") sys.exit(1) else: zpfile = zipfile.ZipFile(str(args.py), 'r') zpfile.extractall(os.path.dirname(str(args.py))) sys.path.insert(0, os.path.dirname(str(args.py))) elif os.path.splitext(str(args.py))[1] == '.zip': # Assumig zip file with format func.zip # extract to folder function # internal dir format # "func/src" # "func/requirements.txt" # "func/deps" # run pip install to target folder deps folder zpfile = zipfile.ZipFile(str(args.py), 'r') zpfile.extractall(os.path.dirname(str(args.py))) basename = os.path.splitext(str(args.py))[0] deps_dir = os.path.join(os.path.dirname(str(args.py)), basename, "deps") if os.path.isdir(deps_dir) and os.listdir(deps_dir): # get all wheel files from deps directory wheel_file_list = [ os.path.join(deps_dir, f) for f in os.listdir(deps_dir) if os.path.isfile(os.path.join(deps_dir, f)) and os.path.splitext(f)[1] == '.whl' ] cmd = "pip install -t %s --no-index --find-links %s %s" % ( os.path.dirname(str( args.py)), deps_dir, " ".join(wheel_file_list)) Log.debug("Install python dependencies via cmd: %s" % cmd) retval = os.system(cmd) if retval != 0: print( "Could not install user depedencies specified by the zip file" ) sys.exit(1) # add python user src directory to path sys.path.insert( 0, os.path.join(os.path.dirname(str(args.py)), basename, "src")) log_file = os.path.join( args.logging_directory, util.getFullyQualifiedFunctionName(function_details.tenant, function_details.namespace, function_details.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_logger(logging.INFO, log_file, args.logging_config_file) Log.info("Starting Python instance with %s" % str(args)) authentication = None use_tls = False tls_allow_insecure_connection = False tls_trust_cert_path = None if args.client_auth_plugin and args.client_auth_params: authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params) if args.use_tls == "true": use_tls = True if args.tls_allow_insecure_connection == "true": tls_allow_insecure_connection = True if args.tls_trust_cert_path: tls_trust_cert_path = args.tls_trust_cert_path pulsar_client = pulsar.Client( args.pulsar_serviceurl, authentication=authentication, operation_timeout_seconds=30, io_threads=1, message_listener_threads=1, concurrent_lookup_requests=50000, log_conf_file_path=None, use_tls=use_tls, tls_trust_certs_file_path=tls_trust_cert_path, tls_allow_insecure_connection=tls_allow_insecure_connection) state_storage_serviceurl = None if args.state_storage_serviceurl is not None: state_storage_serviceurl = str(args.state_storage_serviceurl) secrets_provider = None if args.secrets_provider is not None: secrets_provider = util.import_class( os.path.dirname(inspect.getfile(inspect.currentframe())), str(args.secrets_provider)) else: secrets_provider = util.import_class( os.path.dirname(inspect.getfile(inspect.currentframe())), "secretsprovider.ClearTextSecretsProvider") secrets_provider = secrets_provider() secrets_provider_config = None if args.secrets_provider_config is not None: args.secrets_provider_config = str(args.secrets_provider_config) if args.secrets_provider_config[0] == '\'': args.secrets_provider_config = args.secrets_provider_config[1:] if args.secrets_provider_config[-1] == '\'': args.secrets_provider_config = args.secrets_provider_config[:-1] secrets_provider_config = json.loads(str(args.secrets_provider_config)) secrets_provider.init(secrets_provider_config) pyinstance = python_instance.PythonInstance( str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), int(args.expected_healthcheck_interval), str(args.py), pulsar_client, secrets_provider, args.cluster_name, state_storage_serviceurl) pyinstance.run() server_instance = server.serve(args.port, pyinstance) # Cannot use latest version of prometheus client because of thread leak # prometheus_client.start_http_server(args.metrics_port) # Use patched version of prometheus # Contains fix from https://github.com/prometheus/client_python/pull/356 # This can be removed one the fix in is a official prometheus client release prometheus_client_fix.start_http_server(args.metrics_port) global to_run while to_run: time.sleep(1) pyinstance.join() # make sure to close all non-daemon threads before this! sys.exit(0)
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser(description='Pulsar Functions Python Instance') parser.add_argument('--function_details', required=True, help='Function Details Json String') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin') parser.add_argument('--client_auth_params', required=False, help='Client authentication params') parser.add_argument('--use_tls', required=False, help='Use tls') parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection') parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification') parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--logging_config_file', required=True, help='Config file for logging') parser.add_argument('--expected_healthcheck_interval', required=True, help='Expected time in seconds between health checks', type=int) parser.add_argument('--secrets_provider', required=False, help='The classname of the secrets provider') parser.add_argument('--secrets_provider_config', required=False, help='The config that needs to be passed to secrets provider') parser.add_argument('--install_usercode_dependencies', required=False, help='For packaged python like wheel files, do we need to install all dependencies', type=bool) parser.add_argument('--dependency_repository', required=False, help='For packaged python like wheel files, which repository to pull the dependencies from') parser.add_argument('--extra_dependency_repository', required=False, help='For packaged python like wheel files, any extra repository to pull the dependencies from') args = parser.parse_args() function_details = Function_pb2.FunctionDetails() args.function_details = str(args.function_details) if args.function_details[0] == '\'': args.function_details = args.function_details[1:] if args.function_details[-1] == '\'': args.function_details = args.function_details[:-1] json_format.Parse(args.function_details, function_details) if os.path.splitext(str(args.py))[1] == '.whl': if args.install_usercode_dependencies: cmd = "pip install -t %s" % os.path.dirname(str(args.py)) if args.dependency_repository: cmd = cmd + " -i %s" % str(args.dependency_repository) if args.extra_dependency_repository: cmd = cmd + " --extra-index-url %s" % str(args.extra_dependency_repository) cmd = cmd + " %s" % str(args.py) retval = os.system(cmd) if retval != 0: print "Could not install user depedencies" sys.exit(1) else: zpfile = zipfile.ZipFile(str(args.py), 'r') zpfile.extractall(os.path.dirname(str(args.py))) sys.path.insert(0, os.path.dirname(str(args.py))) elif os.path.splitext(str(args.py))[1] == '.zip': # Assumig zip file with format func.zip # extract to folder function # internal dir format # "func/src" # "func/requirements.txt" # "func/deps" # run pip install to target folder deps folder zpfile = zipfile.ZipFile(str(args.py), 'r') zpfile.extractall(os.path.dirname(str(args.py))) basename = os.path.splitext(str(args.py))[0] requirements_txt_file = os.path.join(os.path.dirname(str(args.py)), basename, "requirements.txt") deps_file = os.path.join(os.path.dirname(str(args.py)), basename, "deps") cmd = "pip install -t %s -r %s --no-index --find-links %s" % (os.path.dirname(str(args.py)), requirements_txt_file, deps_file) retval = os.system(cmd) if retval != 0: print "Could not install user depedencies specified by the zip file" sys.exit(1) sys.path.insert(0, os.path.join(os.path.dirname(str(args.py)), basename, "src")) log_file = os.path.join(args.logging_directory, util.getFullyQualifiedFunctionName(function_details.tenant, function_details.namespace, function_details.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_logger(logging.INFO, log_file, args.logging_config_file) Log.info("Starting Python instance with %s" % str(args)) authentication = None use_tls = False tls_allow_insecure_connection = False tls_trust_cert_path = None if args.client_auth_plugin and args.client_auth_params: authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params) if args.use_tls == "true": use_tls = True if args.tls_allow_insecure_connection == "true": tls_allow_insecure_connection = True if args.tls_trust_cert_path: tls_trust_cert_path = args.tls_trust_cert_path pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30, 1, 1, 50000, None, use_tls, tls_trust_cert_path, tls_allow_insecure_connection) secrets_provider = None if args.secrets_provider is not None: secrets_provider = util.import_class(os.path.dirname(inspect.getfile(inspect.currentframe())), str(args.secrets_provider)) else: secrets_provider = util.import_class(os.path.dirname(inspect.getfile(inspect.currentframe())), "secretsprovider.ClearTextSecretsProvider") secrets_provider = secrets_provider() secrets_provider_config = None if args.secrets_provider_config is not None: secrets_provider_config = json.loads(str(args.secrets_provider_config)) secrets_provider.init(secrets_provider_config) pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), int(args.expected_healthcheck_interval), str(args.py), pulsar_client, secrets_provider) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser( description='Pulsar Functions Python Instance') parser.add_argument('--function_details', required=True, help='Function Details Json String') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin') parser.add_argument('--client_auth_params', required=False, help='Client authentication params') parser.add_argument('--use_tls', required=False, help='Use tls') parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection') parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification') parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--expected_healthcheck_interval', required=True, help='Expected time in seconds between health checks', type=int) parser.add_argument( '--install_usercode_dependencies', required=False, help= 'For packaged python like wheel files, do we need to install all dependencies', type=bool) args = parser.parse_args() function_details = Function_pb2.FunctionDetails() args.function_details = str(args.function_details) if args.function_details[0] == '\'': args.function_details = args.function_details[1:] if args.function_details[-1] == '\'': args.function_details = args.function_details[:-1] json_format.Parse(args.function_details, function_details) if os.path.splitext(str(args.py))[1] == '.whl': if args.install_usercode_dependencies: os.system("pip install -t %s %s" % (os.path.dirname(str(args.py)), str(args.py))) else: zpfile = zipfile.ZipFile(str(args.py), 'r') zpfile.extractall(os.path.dirname(str(args.py))) sys.path.insert(0, os.path.dirname(str(args.py))) log_file = os.path.join( args.logging_directory, util.getFullyQualifiedFunctionName(function_details.tenant, function_details.namespace, function_details.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=5, max_bytes=10 * 1024 * 1024) Log.info("Starting Python instance with %s" % str(args)) authentication = None use_tls = False tls_allow_insecure_connection = False tls_trust_cert_path = None if args.client_auth_plugin and args.client_auth_params: authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params) if args.use_tls == "true": use_tls = True if args.tls_allow_insecure_connection == "true": tls_allow_insecure_connection = True if args.tls_trust_cert_path: tls_trust_cert_path = args.tls_trust_cert_path pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30, 1, 1, 50000, None, use_tls, tls_trust_cert_path, tls_allow_insecure_connection) pyinstance = python_instance.PythonInstance( str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), int(args.expected_healthcheck_interval), str(args.py), pulsar_client) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser(description='Pulsar Functions Python Instance') parser.add_argument('--function_classname', required=True, help='Function Class Name') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--name', required=True, help='Function Name') parser.add_argument('--tenant', required=True, help='Tenant Name') parser.add_argument('--namespace', required=True, help='Namespace name') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--processing_guarantees', required=True, help='Processing Guarantees') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--user_config', required=False, help='User Config') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--auto_ack', required=True, help='Enable Autoacking?') parser.add_argument('--log_topic', required=False, help='Topic to send Log Messages') parser.add_argument('--source_subscription_type', required=True, help='Subscription Type') parser.add_argument('--source_topics_serde_classname', required=True, help='A mapping of Input topics to SerDe') parser.add_argument('--sink_topic', required=False, help='Sink Topic') parser.add_argument('--sink_serde_classname', required=False, help='Sink SerDe classname') args = parser.parse_args() log_file = os.path.join(args.logging_directory, util.getFullyQualifiedFunctionName(args.tenant, args.namespace, args.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=5, max_bytes=10 * 1024 * 1024) Log.info("Starting Python instance with %s" % str(args)) function_details = Function_pb2.FunctionDetails() function_details.tenant = args.tenant function_details.namespace = args.namespace function_details.name = args.name function_details.className = args.function_classname sourceSpec = Function_pb2.SourceSpec() sourceSpec.subscriptionType = Function_pb2.SubscriptionType.Value(args.source_subscription_type) try: source_topics_serde_classname_dict = json.loads(args.source_topics_serde_classname) except ValueError: log.critical("Cannot decode source_topics_serde_classname. This argument must be specifed as a JSON") sys.exit(1) if not source_topics_serde_classname_dict: log.critical("source_topics_serde_classname cannot be empty") for topics, serde_classname in source_topics_serde_classname_dict.items(): sourceSpec.topicsToSerDeClassName[topics] = serde_classname function_details.source.MergeFrom(sourceSpec) sinkSpec = Function_pb2.SinkSpec() if args.sink_topic != None and len(args.sink_topic) != 0: sinkSpec.topic = args.sink_topic if args.sink_serde_classname != None and len(args.sink_serde_classname) != 0: sinkSpec.serDeClassName = args.sink_serde_classname function_details.sink.MergeFrom(sinkSpec) function_details.processingGuarantees = Function_pb2.ProcessingGuarantees.Value(args.processing_guarantees) if args.auto_ack == "true": function_details.autoAck = True else: function_details.autoAck = False if args.user_config != None and len(args.user_config) != 0: function_details.userConfig = args.user_config pulsar_client = pulsar.Client(args.pulsar_serviceurl) pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), str(args.py), args.log_topic, pulsar_client) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser( description='Pulsar Functions Python Instance') parser.add_argument('--function_classname', required=True, help='Function Class Name') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--name', required=True, help='Function Name') parser.add_argument('--tenant', required=True, help='Tenant Name') parser.add_argument('--namespace', required=True, help='Namespace name') parser.add_argument('--custom_serde_input_topics', required=False, help='Input Topics Requiring Custom Deserialization') parser.add_argument('--custom_serde_classnames', required=False, help='Input Serde Classnames') parser.add_argument('--input_topics', required=False, help='Input topics with default serde') parser.add_argument('--output_topic', required=False, help='Output Topic') parser.add_argument('--output_serde_classname', required=False, help='Output Serde Classnames') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--processing_guarantees', required=True, help='Processing Guarantees') parser.add_argument('--subscription_type', required=True, help='Subscription Type') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--user_config', required=False, help='User Config') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--auto_ack', required=True, help='Enable Autoacking?') parser.add_argument('--log_topic', required=False, help='Topic to send Log Messages') args = parser.parse_args() log_file = os.path.join( args.logging_directory, util.getFullyQualifiedFunctionName(args.tenant, args.namespace, args.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=5, max_bytes=10 * 1024 * 1024) Log.info("Starting Python instance with %s" % str(args)) function_details = Function_pb2.FunctionDetails() function_details.tenant = args.tenant function_details.namespace = args.namespace function_details.name = args.name function_details.className = args.function_classname if args.custom_serde_input_topics is None and args.input_topics is None: Log.critical("Atleast one input topic must be present") sys.exit(1) if args.custom_serde_input_topics is not None and args.custom_serde_classnames is not None: input_topics = args.custom_serde_input_topics.split(",") input_serde = args.custom_serde_classnames.split(",") if len(input_topics) != len(input_serde): Log.critical( "CustomSerde InputTopcis and Serde classnames should match") sys.exit(1) for i in xrange(len(input_topics)): function_details.customSerdeInputs[ input_topics[i]] = input_serde[i] if args.input_topics is not None: for topic in args.input_topics.split(","): function_details.inputs.append(topic) if args.output_topic != None and len(args.output_topic) != 0: function_details.output = args.output_topic if args.output_serde_classname != None and len( args.output_serde_classname) != 0: function_details.outputSerdeClassName = args.output_serde_classname function_details.processingGuarantees = Function_pb2.FunctionDetails.ProcessingGuarantees.Value( args.processing_guarantees) function_details.subscriptionType = Function_pb2.FunctionDetails.SubscriptionType.Values( args.subscription_type) if args.auto_ack == "true": function_details.autoAck = True else: function_details.autoAck = False if args.user_config != None and len(args.user_config) != 0: user_config = json.loads(args.user_config) for (key, value) in user_config.items(): function_details.userConfig[str(key)] = str(value) pulsar_client = pulsar.Client(args.pulsar_serviceurl) pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), str(args.py), args.log_topic, pulsar_client) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)
def main(): # Setup signal handlers signal.signal(signal.SIGTERM, atexit_function) signal.signal(signal.SIGHUP, atexit_function) signal.signal(signal.SIGINT, atexit_function) parser = argparse.ArgumentParser( description='Pulsar Functions Python Instance') parser.add_argument('--function_classname', required=True, help='Function Class Name') parser.add_argument('--py', required=True, help='Full Path of Function Code File') parser.add_argument('--name', required=True, help='Function Name') parser.add_argument('--tenant', required=True, help='Tenant Name') parser.add_argument('--namespace', required=True, help='Namespace name') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--function_id', required=True, help='Function Id') parser.add_argument('--function_version', required=True, help='Function Version') parser.add_argument('--processing_guarantees', required=True, help='Processing Guarantees') parser.add_argument('--pulsar_serviceurl', required=True, help='Pulsar Service Url') parser.add_argument('--client_auth_plugin', required=False, help='Client authentication plugin') parser.add_argument('--client_auth_params', required=False, help='Client authentication params') parser.add_argument('--use_tls', required=False, help='Use tls') parser.add_argument('--tls_allow_insecure_connection', required=False, help='Tls allow insecure connection') parser.add_argument('--hostname_verification_enabled', required=False, help='Enable hostname verification') parser.add_argument('--tls_trust_cert_path', required=False, help='Tls trust cert file path') parser.add_argument('--port', required=True, help='Instance Port', type=int) parser.add_argument('--max_buffered_tuples', required=True, help='Maximum number of Buffered tuples') parser.add_argument('--user_config', required=False, help='User Config') parser.add_argument('--logging_directory', required=True, help='Logging Directory') parser.add_argument('--logging_file', required=True, help='Log file name') parser.add_argument('--auto_ack', required=True, help='Enable Autoacking?') parser.add_argument('--log_topic', required=False, help='Topic to send Log Messages') parser.add_argument('--source_subscription_type', required=True, help='Subscription Type') parser.add_argument('--source_topics_serde_classname', required=True, help='A mapping of Input topics to SerDe') parser.add_argument( '--topics_pattern', required=False, help= 'TopicsPattern to consume from list of topics under a namespace that match the pattern (not supported)' ) parser.add_argument('--source_timeout_ms', required=False, help='Source message timeout in milliseconds') parser.add_argument('--sink_topic', required=False, help='Sink Topic') parser.add_argument('--sink_serde_classname', required=False, help='Sink SerDe classname') args = parser.parse_args() log_file = os.path.join( args.logging_directory, util.getFullyQualifiedFunctionName(args.tenant, args.namespace, args.name), "%s-%s.log" % (args.logging_file, args.instance_id)) log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=5, max_bytes=10 * 1024 * 1024) Log.info("Starting Python instance with %s" % str(args)) function_details = Function_pb2.FunctionDetails() function_details.tenant = args.tenant function_details.namespace = args.namespace function_details.name = args.name function_details.className = args.function_classname if args.topics_pattern: raise ValueError('topics_pattern is not supported by python client') sourceSpec = Function_pb2.SourceSpec() sourceSpec.subscriptionType = Function_pb2.SubscriptionType.Value( args.source_subscription_type) try: source_topics_serde_classname_dict = json.loads( args.source_topics_serde_classname) except ValueError: Log.critical( "Cannot decode source_topics_serde_classname. This argument must be specifed as a JSON" ) sys.exit(1) if not source_topics_serde_classname_dict: Log.critical("source_topics_serde_classname cannot be empty") for topics, serde_classname in source_topics_serde_classname_dict.items(): sourceSpec.topicsToSerDeClassName[topics] = serde_classname if args.source_timeout_ms: sourceSpec.timeoutMs = long(args.source_timeout_ms) function_details.source.MergeFrom(sourceSpec) sinkSpec = Function_pb2.SinkSpec() if args.sink_topic != None and len(args.sink_topic) != 0: sinkSpec.topic = args.sink_topic if args.sink_serde_classname != None and len( args.sink_serde_classname) != 0: sinkSpec.serDeClassName = args.sink_serde_classname function_details.sink.MergeFrom(sinkSpec) function_details.processingGuarantees = Function_pb2.ProcessingGuarantees.Value( args.processing_guarantees) if args.auto_ack == "true": function_details.autoAck = True else: function_details.autoAck = False if args.user_config != None and len(args.user_config) != 0: function_details.userConfig = args.user_config authentication = None use_tls = False tls_allow_insecure_connection = False tls_trust_cert_path = None if args.client_auth_plugin and args.client_auth_params: authentication = pulsar.Authentication(args.client_auth_plugin, args.client_auth_params) if args.use_tls == "true": use_tls = True if args.tls_allow_insecure_connection == "true": tls_allow_insecure_connection = True if args.tls_trust_cert_path: tls_trust_cert_path = args.tls_trust_cert_path pulsar_client = pulsar.Client(args.pulsar_serviceurl, authentication, 30, 1, 1, 50000, None, use_tls, tls_trust_cert_path, tls_allow_insecure_connection) pyinstance = python_instance.PythonInstance(str(args.instance_id), str(args.function_id), str(args.function_version), function_details, int(args.max_buffered_tuples), str(args.py), args.log_topic, pulsar_client) pyinstance.run() server_instance = server.serve(args.port, pyinstance) global to_run while to_run: time.sleep(1) pyinstance.join() sys.exit(1)