def main(): LOG_FORMAT = '%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(levelname)s: %(message)s' logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) logger.info('Starting microservice.py:main') sys.path.append(os.getcwd()) parser = argparse.ArgumentParser() parser.add_argument("interface_name", type=str, help="Name of the user interface.") parser.add_argument("api_type", type=str, choices=["REST", "GRPC", "FBS"]) parser.add_argument("--service-type", type=str, choices=[ "MODEL", "ROUTER", "TRANSFORMER", "COMBINER", "OUTLIER_DETECTOR" ], default="MODEL") parser.add_argument("--persistence", nargs='?', default=0, const=1, type=int) parser.add_argument("--parameters", type=str, default=os.environ.get(PARAMETERS_ENV_NAME, "[]")) parser.add_argument("--log-level", type=str, default="INFO") parser.add_argument("--tracing", nargs='?', default=int(os.environ.get("TRACING", "0")), const=1, type=int) args = parser.parse_args() parameters = parse_parameters(json.loads(args.parameters)) # set up log level log_level_num = getattr(logging, args.log_level.upper(), None) if not isinstance(log_level_num, int): raise ValueError('Invalid log level: %s', args.log_level) logger.setLevel(log_level_num) logger.debug("Log level set to %s:%s", args.log_level, log_level_num) annotations = load_annotations() logger.info("Annotations: %s", annotations) interface_file = importlib.import_module(args.interface_name) user_class = getattr(interface_file, args.interface_name) if args.persistence: logger.info('Restoring persisted component') user_object = persistence.restore(user_class, parameters) persistence.persist(user_object, parameters.get("push_frequency")) else: user_object = user_class(**parameters) # set log level for the imported microservice type seldon_microservice.logger.setLevel(log_level_num) port = int(os.environ.get(SERVICE_PORT_ENV_NAME, DEFAULT_PORT)) if args.tracing: tracer = setup_tracing(args.interface_name) if args.api_type == "REST": def rest_prediction_server(): app = seldon_microservice.get_rest_microservice(user_object) if args.tracing: from flask_opentracing import FlaskTracer tracing = FlaskTracer(tracer, True, app) app.run(host='0.0.0.0', port=port) logger.info("REST microservice running on port %i", port) server1_func = rest_prediction_server elif args.api_type == "GRPC": def grpc_prediction_server(): if args.tracing: from grpc_opentracing import open_tracing_server_interceptor logger.info("Adding tracer") interceptor = open_tracing_server_interceptor(tracer) else: interceptor = None server = seldon_microservice.get_grpc_server( user_object, annotations=annotations, trace_interceptor=interceptor) server.add_insecure_port(f"0.0.0.0:{port}") server.start() logger.info("GRPC microservice Running on port %i", port) while True: time.sleep(1000) server1_func = grpc_prediction_server else: server1_func = None if hasattr(user_object, 'custom_service') and callable( getattr(user_object, 'custom_service')): server2_func = user_object.custom_service else: server2_func = None logger.info('Starting servers') start_servers(server1_func, server2_func)
def main(): LOG_FORMAT = ( "%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(levelname)s: %(message)s" ) logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) logger.info("Starting microservice.py:main") logger.info(f"Seldon Core version: {__version__}") sys.path.append(os.getcwd()) parser = argparse.ArgumentParser() parser.add_argument("interface_name", type=str, help="Name of the user interface.") parser.add_argument( "--service-type", type=str, choices=[ "MODEL", "ROUTER", "TRANSFORMER", "COMBINER", "OUTLIER_DETECTOR" ], default="MODEL", ) parser.add_argument("--persistence", nargs="?", default=0, const=1, type=int) parser.add_argument("--parameters", type=str, default=os.environ.get(PARAMETERS_ENV_NAME, "[]")) parser.add_argument( "--log-level", type=str, choices=["DEBUG", "INFO", "WARNING", "ERROR"], default=DEFAULT_LOG_LEVEL, help="Log level of the inference server.", ) parser.add_argument( "--debug", nargs="?", type=bool, default=getenv_as_bool(DEBUG_ENV, default=False), const=True, help="Enable debug mode.", ) parser.add_argument( "--tracing", nargs="?", default=int(os.environ.get("TRACING", "0")), const=1, type=int, ) # gunicorn settings, defaults are from # http://docs.gunicorn.org/en/stable/settings.html parser.add_argument( "--workers", type=int, default=int(os.environ.get("GUNICORN_WORKERS", "1")), help="Number of Gunicorn workers for handling requests.", ) parser.add_argument( "--threads", type=int, default=int(os.environ.get("GUNICORN_THREADS", "10")), help="Number of threads to run per Gunicorn worker.", ) parser.add_argument( "--max-requests", type=int, default=int(os.environ.get("GUNICORN_MAX_REQUESTS", "0")), help= "Maximum number of requests gunicorn worker will process before restarting.", ) parser.add_argument( "--max-requests-jitter", type=int, default=int(os.environ.get("GUNICORN_MAX_REQUESTS_JITTER", "0")), help="Maximum random jitter to add to max-requests.", ) parser.add_argument( "--single-threaded", type=int, default=int(os.environ.get("FLASK_SINGLE_THREADED", "0")), help= "Force the Flask app to run single-threaded. Also applies to Gunicorn.", ) parser.add_argument( "--http-port", type=int, default=int( os.environ.get(HTTP_SERVICE_PORT_ENV_NAME, DEFAULT_HTTP_PORT)), help="Set http port of seldon service", ) parser.add_argument( "--grpc-port", type=int, default=int( os.environ.get(GRPC_SERVICE_PORT_ENV_NAME, DEFAULT_GRPC_PORT)), help="Set grpc port of seldon service", ) parser.add_argument( "--metrics-port", type=int, default=int( os.environ.get(METRICS_SERVICE_PORT_ENV_NAME, DEFAULT_METRICS_PORT)), help="Set metrics port of seldon service", ) parser.add_argument("--pidfile", type=str, default=None, help="A file path to use for the PID file") parser.add_argument( "--access-log", nargs="?", type=bool, default=getenv_as_bool(GUNICORN_ACCESS_LOG_ENV, default=False), const=True, help="Enable gunicorn access log.", ) args, remaining = parser.parse_known_args() if len(remaining) > 0: logger.error( f"Unknown args {remaining}. Note since 1.5.0 this CLI does not take API type (REST, GRPC)" ) sys.exit(-1) parameters = parse_parameters(json.loads(args.parameters)) setup_logger(args.log_level, args.debug) # set flask trace jaeger extra tags jaeger_extra_tags = list( filter( lambda x: (x != ""), [ tag.strip() for tag in os.environ.get("JAEGER_EXTRA_TAGS", "").split(",") ], )) logger.info("Parse JAEGER_EXTRA_TAGS %s", jaeger_extra_tags) annotations = load_annotations() logger.info("Annotations: %s", annotations) parts = args.interface_name.rsplit(".", 1) if len(parts) == 1: logger.info("Importing %s", args.interface_name) interface_file = importlib.import_module(args.interface_name) user_class = getattr(interface_file, args.interface_name) else: logger.info("Importing submodule %s", parts) interface_file = importlib.import_module(parts[0]) user_class = getattr(interface_file, parts[1]) if args.persistence: logger.info("Restoring persisted component") user_object = persistence.restore(user_class, parameters) persistence.persist(user_object, parameters.get("push_frequency")) else: user_object = user_class(**parameters) http_port = args.http_port grpc_port = args.grpc_port metrics_port = args.metrics_port # if args.tracing: # tracer = setup_tracing(args.interface_name) seldon_metrics = SeldonMetrics(worker_id_func=os.getpid) # TODO why 2 ways to create metrics server # seldon_metrics = SeldonMetrics( # worker_id_func=lambda: threading.current_thread().name # ) if args.debug: # Start Flask debug server def rest_prediction_server(): app = seldon_microservice.get_rest_microservice( user_object, seldon_metrics) try: user_object.load() except (NotImplementedError, AttributeError): pass if args.tracing: logger.info("Tracing branch is active") from flask_opentracing import FlaskTracing tracer = setup_tracing(args.interface_name) logger.info("Set JAEGER_EXTRA_TAGS %s", jaeger_extra_tags) FlaskTracing(tracer, True, app, jaeger_extra_tags) app.run( host="0.0.0.0", port=http_port, threaded=False if args.single_threaded else True, ) logger.info( "REST microservice running on port %i single-threaded=%s", http_port, args.single_threaded, ) server1_func = rest_prediction_server else: # Start production server def rest_prediction_server(): options = { "bind": "%s:%s" % ("0.0.0.0", http_port), "accesslog": accesslog(args.access_log), "loglevel": args.log_level.lower(), "timeout": 5000, "threads": threads(args.threads, args.single_threaded), "workers": args.workers, "max_requests": args.max_requests, "max_requests_jitter": args.max_requests_jitter, "post_worker_init": post_worker_init, "worker_exit": partial(worker_exit, seldon_metrics=seldon_metrics), } if args.pidfile is not None: options["pidfile"] = args.pidfile app = seldon_microservice.get_rest_microservice( user_object, seldon_metrics) UserModelApplication( app, user_object, jaeger_extra_tags, args.interface_name, options=options, ).run() logger.info("REST gunicorn microservice running on port %i", http_port) server1_func = rest_prediction_server def grpc_prediction_server(): if args.tracing: from grpc_opentracing import open_tracing_server_interceptor logger.info("Adding tracer") tracer = setup_tracing(args.interface_name) interceptor = open_tracing_server_interceptor(tracer) else: interceptor = None server = seldon_microservice.get_grpc_server( user_object, seldon_metrics, annotations=annotations, trace_interceptor=interceptor, ) try: user_object.load() except (NotImplementedError, AttributeError): pass server.add_insecure_port(f"0.0.0.0:{grpc_port}") server.start() logger.info("GRPC microservice Running on port %i", grpc_port) while True: time.sleep(1000) server2_func = grpc_prediction_server def rest_metrics_server(): app = seldon_microservice.get_metrics_microservice(seldon_metrics) if args.debug: app.run(host="0.0.0.0", port=metrics_port) else: options = { "bind": "%s:%s" % ("0.0.0.0", metrics_port), "accesslog": accesslog(args.access_log), "loglevel": args.log_level.lower(), "timeout": 5000, "max_requests": args.max_requests, "max_requests_jitter": args.max_requests_jitter, "post_worker_init": post_worker_init, } if args.pidfile is not None: options["pidfile"] = args.pidfile StandaloneApplication(app, options=options).run() logger.info("REST metrics microservice running on port %i", metrics_port) metrics_server_func = rest_metrics_server if hasattr(user_object, "custom_service") and callable( getattr(user_object, "custom_service")): server3_func = user_object.custom_service else: server3_func = None logger.info("Starting servers") start_servers(server1_func, server2_func, server3_func, metrics_server_func)
def main(): LOG_FORMAT = '%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(levelname)s: %(message)s' logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) logger.info('Starting microservice.py:main') sys.path.append(os.getcwd()) parser = argparse.ArgumentParser() parser.add_argument("interface_name", type=str, help="Name of the user interface.") parser.add_argument("api_type", type=str, choices=["REST", "GRPC", "FBS"]) parser.add_argument("--service-type", type=str, choices=[ "MODEL", "ROUTER", "TRANSFORMER", "COMBINER", "OUTLIER_DETECTOR"], default="MODEL") parser.add_argument("--persistence", nargs='?', default=0, const=1, type=int) parser.add_argument("--parameters", type=str, default=os.environ.get(PARAMETERS_ENV_NAME, "[]")) parser.add_argument("--log-level", type=str, default="INFO") parser.add_argument("--tracing", nargs='?', default=int(os.environ.get("TRACING", "0")), const=1, type=int) args = parser.parse_args() parameters = parse_parameters(json.loads(args.parameters)) # set up log level log_level_num = getattr(logging, args.log_level.upper(), None) if not isinstance(log_level_num, int): raise ValueError('Invalid log level: %s', args.log_level) logger.setLevel(log_level_num) logger.debug("Log level set to %s:%s", args.log_level, log_level_num) DEBUG = False if parameters.get(DEBUG_PARAMETER): parameters.pop(DEBUG_PARAMETER) DEBUG = True annotations = load_annotations() logger.info("Annotations: %s", annotations) interface_file = importlib.import_module(args.interface_name) user_class = getattr(interface_file, args.interface_name) if args.persistence: logger.info('Restoring persisted component') user_object = persistence.restore(user_class, parameters, debug=DEBUG) persistence.persist(user_object, parameters.get("push_frequency")) else: user_object = user_class(**parameters) if args.service_type == "MODEL": import seldon_core.model_microservice as seldon_microservice elif args.service_type == "ROUTER": import seldon_core.router_microservice as seldon_microservice elif args.service_type == "TRANSFORMER": import seldon_core.transformer_microservice as seldon_microservice elif args.service_type == "COMBINER": import seldon_core.combiner_microservice as seldon_microservice elif args.service_type == "OUTLIER_DETECTOR": import seldon_core.outlier_detector_microservice as seldon_microservice # set log level for the imported microservice type seldon_microservice.logger.setLevel(log_level_num) port = int(os.environ.get(SERVICE_PORT_ENV_NAME, DEFAULT_PORT)) if args.tracing: logger.info("Initializing tracing") from jaeger_client import Config jaeger_serv = os.environ.get("JAEGER_AGENT_HOST","0.0.0.0") jaeger_port = os.environ.get("JAEGER_AGENT_PORT", 5775) jaeger_config = os.environ.get("JAEGER_CONFIG_PATH",None) if jaeger_config is None: logger.info("Using default tracing config") config = Config( config={ # usually read from some yaml config 'sampler': { 'type': 'const', 'param': 1, }, 'local_agent': { 'reporting_host': jaeger_serv, 'reporting_port': jaeger_port, }, 'logging': True, }, service_name=args.interface_name, validate=True, ) else: logger.info("Loading tracing config from %s",jaeger_config) import yaml with open(jaeger_config, 'r') as stream: config_dict = yaml.load(stream) config = Config( config=config_dict, service_name=args.interface_name, validate=True, ) # this call also sets opentracing.tracer tracer = config.initialize_tracer() if args.api_type == "REST": def rest_prediction_server(): app = seldon_microservice.get_rest_microservice( user_object, debug=DEBUG) if args.tracing: from flask_opentracing import FlaskTracer tracing = FlaskTracer(tracer,True, app) app.run(host='0.0.0.0', port=port) logger.info("REST microservice running on port %i",port) server1_func = rest_prediction_server elif args.api_type == "GRPC": def grpc_prediction_server(): if args.tracing: from grpc_opentracing import open_tracing_server_interceptor logger.info("Adding tracer") interceptor = open_tracing_server_interceptor(tracer) else: interceptor = None server = seldon_microservice.get_grpc_server( user_object, debug=DEBUG, annotations=annotations, trace_interceptor=interceptor) server.add_insecure_port("0.0.0.0:{}".format(port)) server.start() logger.info("GRPC microservice Running on port %i",port) while True: time.sleep(1000) server1_func = grpc_prediction_server elif args.api_type == "FBS": def fbs_prediction_server(): seldon_microservice.run_flatbuffers_server(user_object, port) logger.info("FBS microservice Running on port %i",port) server1_func = fbs_prediction_server else: server1_func = None if hasattr(user_object, 'custom_service') and callable(getattr(user_object, 'custom_service')): server2_func = user_object.custom_service else: server2_func = None logger.info('Starting servers') startServers(server1_func, server2_func)
def main(): LOG_FORMAT = '%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(levelname)s: %(message)s' logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) logger.info('Starting microservice.py:main') sys.path.append(os.getcwd()) parser = argparse.ArgumentParser() parser.add_argument("interface_name", type=str, help="Name of the user interface.") parser.add_argument("api_type", type=str, choices=["REST", "GRPC", "FBS"]) parser.add_argument("--service-type", type=str, choices=[ "MODEL", "ROUTER", "TRANSFORMER", "COMBINER", "OUTLIER_DETECTOR" ], default="MODEL") parser.add_argument("--persistence", nargs='?', default=0, const=1, type=int) parser.add_argument("--parameters", type=str, default=os.environ.get(PARAMETERS_ENV_NAME, "[]")) parser.add_argument("--log-level", type=str, default='INFO') args = parser.parse_args() parameters = parse_parameters(json.loads(args.parameters)) # set up log level log_level_num = getattr(logging, args.log_level.upper(), None) if not isinstance(log_level_num, int): raise ValueError('Invalid log level: %s', args.log_level) logger.setLevel(log_level_num) DEBUG = False if parameters.get(DEBUG_PARAMETER): parameters.pop(DEBUG_PARAMETER) DEBUG = True annotations = load_annotations() logger.info("Annotations: %s", annotations) interface_file = importlib.import_module(args.interface_name) user_class = getattr(interface_file, args.interface_name) if args.persistence: logger.info('Restoring persisted component') user_object = persistence.restore(user_class, parameters, debug=DEBUG) persistence.persist(user_object, parameters.get("push_frequency")) else: user_object = user_class(**parameters) if args.service_type == "MODEL": import seldon_core.model_microservice as seldon_microservice elif args.service_type == "ROUTER": import seldon_core.router_microservice as seldon_microservice elif args.service_type == "TRANSFORMER": import seldon_core.transformer_microservice as seldon_microservice elif args.service_type == "COMBINER": import seldon_core.combiner_microservice as seldon_microservice elif args.service_type == "OUTLIER_DETECTOR": import seldon_core.outlier_detector_microservice as seldon_microservice port = int(os.environ.get(SERVICE_PORT_ENV_NAME, DEFAULT_PORT)) if args.api_type == "REST": def rest_prediction_server(): app = seldon_microservice.get_rest_microservice(user_object, debug=DEBUG) app.run(host='0.0.0.0', port=port) logger.info("REST microservice running on port %i", port) server1_func = rest_prediction_server elif args.api_type == "GRPC": def grpc_prediction_server(): server = seldon_microservice.get_grpc_server( user_object, debug=DEBUG, annotations=annotations) server.add_insecure_port("0.0.0.0:{}".format(port)) server.start() logger.info("GRPC microservice Running on port %i", port) while True: time.sleep(1000) server1_func = grpc_prediction_server elif args.api_type == "FBS": def fbs_prediction_server(): seldon_microservice.run_flatbuffers_server(user_object, port) logger.info("FBS microservice Running on port %i", port) server1_func = fbs_prediction_server else: server1_func = None if hasattr(user_object, 'custom_service') and callable( getattr(user_object, 'custom_service')): server2_func = user_object.custom_service else: server2_func = None logger.info('Starting servers') startServers(server1_func, server2_func)
def main(): LOG_FORMAT = ( "%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(levelname)s: %(message)s" ) logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) logger.info("Starting microservice.py:main") sys.path.append(os.getcwd()) parser = argparse.ArgumentParser() parser.add_argument("interface_name", type=str, help="Name of the user interface.") parser.add_argument("api_type", type=str, choices=["REST", "GRPC", "FBS"]) parser.add_argument( "--service-type", type=str, choices=["MODEL", "ROUTER", "TRANSFORMER", "COMBINER", "OUTLIER_DETECTOR"], default="MODEL", ) parser.add_argument("--persistence", nargs="?", default=0, const=1, type=int) parser.add_argument( "--parameters", type=str, default=os.environ.get(PARAMETERS_ENV_NAME, "[]") ) parser.add_argument("--log-level", type=str, default="INFO") parser.add_argument( "--tracing", nargs="?", default=int(os.environ.get("TRACING", "0")), const=1, type=int, ) # gunicorn settings, defaults are from http://docs.gunicorn.org/en/stable/settings.html parser.add_argument( "--workers", type=int, default=int(os.environ.get("GUNICORN_WORKERS", "1")), help="Number of gunicorn workers for handling requests.", ) parser.add_argument( "--max-requests", type=int, default=int(os.environ.get("GUNICORN_MAX_REQUESTS", "0")), help="Maximum number of requests gunicorn worker will process before restarting.", ) parser.add_argument( "--max-requests-jitter", type=int, default=int(os.environ.get("GUNICORN_MAX_REQUESTS_JITTER", "0")), help="Maximum random jitter to add to max-requests.", ) args = parser.parse_args() parameters = parse_parameters(json.loads(args.parameters)) # set flask trace jaeger extra tags jaeger_extra_tags = list( filter( lambda x: (x != ""), [tag.strip() for tag in os.environ.get("JAEGER_EXTRA_TAGS", "").split(",")], ) ) logger.info("Parse JAEGER_EXTRA_TAGS %s", jaeger_extra_tags) # set up log level log_level_raw = os.environ.get(LOG_LEVEL_ENV, args.log_level.upper()) log_level_num = getattr(logging, log_level_raw, None) if not isinstance(log_level_num, int): raise ValueError("Invalid log level: %s", args.log_level) logger.setLevel(log_level_num) logger.debug("Log level set to %s:%s", args.log_level, log_level_num) annotations = load_annotations() logger.info("Annotations: %s", annotations) parts = args.interface_name.rsplit(".", 1) if len(parts) == 1: logger.info("Importing %s", args.interface_name) interface_file = importlib.import_module(args.interface_name) user_class = getattr(interface_file, args.interface_name) else: logger.info("Importing submodule %s", parts) interface_file = importlib.import_module(parts[0]) user_class = getattr(interface_file, parts[1]) if args.persistence: logger.info("Restoring persisted component") user_object = persistence.restore(user_class, parameters) persistence.persist(user_object, parameters.get("push_frequency")) else: user_object = user_class(**parameters) # set log level for the imported microservice type seldon_microservice.logger.setLevel(log_level_num) logging.getLogger().setLevel(log_level_num) for handler in logger.handlers: handler.setLevel(log_level_num) port = int(os.environ.get(SERVICE_PORT_ENV_NAME, DEFAULT_PORT)) if args.tracing: tracer = setup_tracing(args.interface_name) if args.api_type == "REST": if args.workers > 1: def rest_prediction_server(): options = { "bind": "%s:%s" % ("0.0.0.0", port), "access_logfile": "-", "loglevel": "info", "timeout": 5000, "reload": "true", "workers": args.workers, "max_requests": args.max_requests, "max_requests_jitter": args.max_requests_jitter, } app = seldon_microservice.get_rest_microservice(user_object) StandaloneApplication(app, user_object, options=options).run() logger.info("REST gunicorn microservice running on port %i", port) server1_func = rest_prediction_server else: def rest_prediction_server(): app = seldon_microservice.get_rest_microservice(user_object) try: user_object.load() except (NotImplementedError, AttributeError): pass if args.tracing: logger.info("Tracing branch is active") from flask_opentracing import FlaskTracing logger.info("Set JAEGER_EXTRA_TAGS %s", jaeger_extra_tags) tracing = FlaskTracing(tracer, True, app, jaeger_extra_tags) app.run(host="0.0.0.0", port=port) logger.info("REST microservice running on port %i", port) server1_func = rest_prediction_server elif args.api_type == "GRPC": def grpc_prediction_server(): if args.tracing: from grpc_opentracing import open_tracing_server_interceptor logger.info("Adding tracer") interceptor = open_tracing_server_interceptor(tracer) else: interceptor = None server = seldon_microservice.get_grpc_server( user_object, annotations=annotations, trace_interceptor=interceptor ) try: user_object.load() except (NotImplementedError, AttributeError): pass server.add_insecure_port(f"0.0.0.0:{port}") server.start() logger.info("GRPC microservice Running on port %i", port) while True: time.sleep(1000) server1_func = grpc_prediction_server else: server1_func = None if hasattr(user_object, "custom_service") and callable( getattr(user_object, "custom_service") ): server2_func = user_object.custom_service else: server2_func = None logger.info("Starting servers") start_servers(server1_func, server2_func)