def main():
LOG_FORMAT = '%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(levelname)s: %(message)s'
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger.info('Starting microservice.py:main')
sys.path.append(os.getcwd())
parser = argparse.ArgumentParser()
parser.add_argument("interface_name",
type=str,
help="Name of the user interface.")
parser.add_argument("api_type", type=str, choices=["REST", "GRPC", "FBS"])
parser.add_argument("--service-type",
type=str,
choices=[
"MODEL", "ROUTER", "TRANSFORMER", "COMBINER",
"OUTLIER_DETECTOR"
],
default="MODEL")
parser.add_argument("--persistence",
nargs='?',
default=0,
const=1,
type=int)
parser.add_argument("--parameters",
type=str,
default=os.environ.get(PARAMETERS_ENV_NAME, "[]"))
parser.add_argument("--log-level", type=str, default="INFO")
parser.add_argument("--tracing",
nargs='?',
default=int(os.environ.get("TRACING", "0")),
const=1,
type=int)
args = parser.parse_args()
parameters = parse_parameters(json.loads(args.parameters))
# set up log level
log_level_num = getattr(logging, args.log_level.upper(), None)
if not isinstance(log_level_num, int):
raise ValueError('Invalid log level: %s', args.log_level)
logger.setLevel(log_level_num)
logger.debug("Log level set to %s:%s", args.log_level, log_level_num)
annotations = load_annotations()
logger.info("Annotations: %s", annotations)
interface_file = importlib.import_module(args.interface_name)
user_class = getattr(interface_file, args.interface_name)
if args.persistence:
logger.info('Restoring persisted component')
user_object = persistence.restore(user_class, parameters)
persistence.persist(user_object, parameters.get("push_frequency"))
else:
user_object = user_class(**parameters)
# set log level for the imported microservice type
seldon_microservice.logger.setLevel(log_level_num)
port = int(os.environ.get(SERVICE_PORT_ENV_NAME, DEFAULT_PORT))
if args.tracing:
tracer = setup_tracing(args.interface_name)
if args.api_type == "REST":
def rest_prediction_server():
app = seldon_microservice.get_rest_microservice(user_object)
if args.tracing:
from flask_opentracing import FlaskTracer
tracing = FlaskTracer(tracer, True, app)
app.run(host='0.0.0.0', port=port)
logger.info("REST microservice running on port %i", port)
server1_func = rest_prediction_server
elif args.api_type == "GRPC":
def grpc_prediction_server():
if args.tracing:
from grpc_opentracing import open_tracing_server_interceptor
logger.info("Adding tracer")
interceptor = open_tracing_server_interceptor(tracer)
else:
interceptor = None
server = seldon_microservice.get_grpc_server(
user_object,
annotations=annotations,
trace_interceptor=interceptor)
server.add_insecure_port(f"0.0.0.0:{port}")
server.start()
logger.info("GRPC microservice Running on port %i", port)
while True:
time.sleep(1000)
server1_func = grpc_prediction_server
else:
server1_func = None
if hasattr(user_object, 'custom_service') and callable(
getattr(user_object, 'custom_service')):
server2_func = user_object.custom_service
else:
server2_func = None
logger.info('Starting servers')
start_servers(server1_func, server2_func)
def main():
LOG_FORMAT = (
"%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(levelname)s: %(message)s"
)
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger.info("Starting microservice.py:main")
logger.info(f"Seldon Core version: {__version__}")
sys.path.append(os.getcwd())
parser = argparse.ArgumentParser()
parser.add_argument("interface_name",
type=str,
help="Name of the user interface.")
parser.add_argument(
"--service-type",
type=str,
choices=[
"MODEL", "ROUTER", "TRANSFORMER", "COMBINER", "OUTLIER_DETECTOR"
],
default="MODEL",
)
parser.add_argument("--persistence",
nargs="?",
default=0,
const=1,
type=int)
parser.add_argument("--parameters",
type=str,
default=os.environ.get(PARAMETERS_ENV_NAME, "[]"))
parser.add_argument(
"--log-level",
type=str,
choices=["DEBUG", "INFO", "WARNING", "ERROR"],
default=DEFAULT_LOG_LEVEL,
help="Log level of the inference server.",
)
parser.add_argument(
"--debug",
nargs="?",
type=bool,
default=getenv_as_bool(DEBUG_ENV, default=False),
const=True,
help="Enable debug mode.",
)
parser.add_argument(
"--tracing",
nargs="?",
default=int(os.environ.get("TRACING", "0")),
const=1,
type=int,
)
# gunicorn settings, defaults are from
# http://docs.gunicorn.org/en/stable/settings.html
parser.add_argument(
"--workers",
type=int,
default=int(os.environ.get("GUNICORN_WORKERS", "1")),
help="Number of Gunicorn workers for handling requests.",
)
parser.add_argument(
"--threads",
type=int,
default=int(os.environ.get("GUNICORN_THREADS", "10")),
help="Number of threads to run per Gunicorn worker.",
)
parser.add_argument(
"--max-requests",
type=int,
default=int(os.environ.get("GUNICORN_MAX_REQUESTS", "0")),
help=
"Maximum number of requests gunicorn worker will process before restarting.",
)
parser.add_argument(
"--max-requests-jitter",
type=int,
default=int(os.environ.get("GUNICORN_MAX_REQUESTS_JITTER", "0")),
help="Maximum random jitter to add to max-requests.",
)
parser.add_argument(
"--single-threaded",
type=int,
default=int(os.environ.get("FLASK_SINGLE_THREADED", "0")),
help=
"Force the Flask app to run single-threaded. Also applies to Gunicorn.",
)
parser.add_argument(
"--http-port",
type=int,
default=int(
os.environ.get(HTTP_SERVICE_PORT_ENV_NAME, DEFAULT_HTTP_PORT)),
help="Set http port of seldon service",
)
parser.add_argument(
"--grpc-port",
type=int,
default=int(
os.environ.get(GRPC_SERVICE_PORT_ENV_NAME, DEFAULT_GRPC_PORT)),
help="Set grpc port of seldon service",
)
parser.add_argument(
"--metrics-port",
type=int,
default=int(
os.environ.get(METRICS_SERVICE_PORT_ENV_NAME,
DEFAULT_METRICS_PORT)),
help="Set metrics port of seldon service",
)
parser.add_argument("--pidfile",
type=str,
default=None,
help="A file path to use for the PID file")
parser.add_argument(
"--access-log",
nargs="?",
type=bool,
default=getenv_as_bool(GUNICORN_ACCESS_LOG_ENV, default=False),
const=True,
help="Enable gunicorn access log.",
)
args, remaining = parser.parse_known_args()
if len(remaining) > 0:
logger.error(
f"Unknown args {remaining}. Note since 1.5.0 this CLI does not take API type (REST, GRPC)"
)
sys.exit(-1)
parameters = parse_parameters(json.loads(args.parameters))
setup_logger(args.log_level, args.debug)
# set flask trace jaeger extra tags
jaeger_extra_tags = list(
filter(
lambda x: (x != ""),
[
tag.strip()
for tag in os.environ.get("JAEGER_EXTRA_TAGS", "").split(",")
],
))
logger.info("Parse JAEGER_EXTRA_TAGS %s", jaeger_extra_tags)
annotations = load_annotations()
logger.info("Annotations: %s", annotations)
parts = args.interface_name.rsplit(".", 1)
if len(parts) == 1:
logger.info("Importing %s", args.interface_name)
interface_file = importlib.import_module(args.interface_name)
user_class = getattr(interface_file, args.interface_name)
else:
logger.info("Importing submodule %s", parts)
interface_file = importlib.import_module(parts[0])
user_class = getattr(interface_file, parts[1])
if args.persistence:
logger.info("Restoring persisted component")
user_object = persistence.restore(user_class, parameters)
persistence.persist(user_object, parameters.get("push_frequency"))
else:
user_object = user_class(**parameters)
http_port = args.http_port
grpc_port = args.grpc_port
metrics_port = args.metrics_port
# if args.tracing:
# tracer = setup_tracing(args.interface_name)
seldon_metrics = SeldonMetrics(worker_id_func=os.getpid)
# TODO why 2 ways to create metrics server
# seldon_metrics = SeldonMetrics(
# worker_id_func=lambda: threading.current_thread().name
# )
if args.debug:
# Start Flask debug server
def rest_prediction_server():
app = seldon_microservice.get_rest_microservice(
user_object, seldon_metrics)
try:
user_object.load()
except (NotImplementedError, AttributeError):
pass
if args.tracing:
logger.info("Tracing branch is active")
from flask_opentracing import FlaskTracing
tracer = setup_tracing(args.interface_name)
logger.info("Set JAEGER_EXTRA_TAGS %s", jaeger_extra_tags)
FlaskTracing(tracer, True, app, jaeger_extra_tags)
app.run(
host="0.0.0.0",
port=http_port,
threaded=False if args.single_threaded else True,
)
logger.info(
"REST microservice running on port %i single-threaded=%s",
http_port,
args.single_threaded,
)
server1_func = rest_prediction_server
else:
# Start production server
def rest_prediction_server():
options = {
"bind": "%s:%s" % ("0.0.0.0", http_port),
"accesslog": accesslog(args.access_log),
"loglevel": args.log_level.lower(),
"timeout": 5000,
"threads": threads(args.threads, args.single_threaded),
"workers": args.workers,
"max_requests": args.max_requests,
"max_requests_jitter": args.max_requests_jitter,
"post_worker_init": post_worker_init,
"worker_exit": partial(worker_exit,
seldon_metrics=seldon_metrics),
}
if args.pidfile is not None:
options["pidfile"] = args.pidfile
app = seldon_microservice.get_rest_microservice(
user_object, seldon_metrics)
UserModelApplication(
app,
user_object,
jaeger_extra_tags,
args.interface_name,
options=options,
).run()
logger.info("REST gunicorn microservice running on port %i", http_port)
server1_func = rest_prediction_server
def grpc_prediction_server():
if args.tracing:
from grpc_opentracing import open_tracing_server_interceptor
logger.info("Adding tracer")
tracer = setup_tracing(args.interface_name)
interceptor = open_tracing_server_interceptor(tracer)
else:
interceptor = None
server = seldon_microservice.get_grpc_server(
user_object,
seldon_metrics,
annotations=annotations,
trace_interceptor=interceptor,
)
try:
user_object.load()
except (NotImplementedError, AttributeError):
pass
server.add_insecure_port(f"0.0.0.0:{grpc_port}")
server.start()
logger.info("GRPC microservice Running on port %i", grpc_port)
while True:
time.sleep(1000)
server2_func = grpc_prediction_server
def rest_metrics_server():
app = seldon_microservice.get_metrics_microservice(seldon_metrics)
if args.debug:
app.run(host="0.0.0.0", port=metrics_port)
else:
options = {
"bind": "%s:%s" % ("0.0.0.0", metrics_port),
"accesslog": accesslog(args.access_log),
"loglevel": args.log_level.lower(),
"timeout": 5000,
"max_requests": args.max_requests,
"max_requests_jitter": args.max_requests_jitter,
"post_worker_init": post_worker_init,
}
if args.pidfile is not None:
options["pidfile"] = args.pidfile
StandaloneApplication(app, options=options).run()
logger.info("REST metrics microservice running on port %i", metrics_port)
metrics_server_func = rest_metrics_server
if hasattr(user_object, "custom_service") and callable(
getattr(user_object, "custom_service")):
server3_func = user_object.custom_service
else:
server3_func = None
logger.info("Starting servers")
start_servers(server1_func, server2_func, server3_func,
metrics_server_func)
def main():
LOG_FORMAT = '%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(levelname)s: %(message)s'
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger.info('Starting microservice.py:main')
sys.path.append(os.getcwd())
parser = argparse.ArgumentParser()
parser.add_argument("interface_name", type=str,
help="Name of the user interface.")
parser.add_argument("api_type", type=str, choices=["REST", "GRPC", "FBS"])
parser.add_argument("--service-type", type=str, choices=[
"MODEL", "ROUTER", "TRANSFORMER", "COMBINER", "OUTLIER_DETECTOR"], default="MODEL")
parser.add_argument("--persistence", nargs='?',
default=0, const=1, type=int)
parser.add_argument("--parameters", type=str,
default=os.environ.get(PARAMETERS_ENV_NAME, "[]"))
parser.add_argument("--log-level", type=str, default="INFO")
parser.add_argument("--tracing", nargs='?',
default=int(os.environ.get("TRACING", "0")), const=1, type=int)
args = parser.parse_args()
parameters = parse_parameters(json.loads(args.parameters))
# set up log level
log_level_num = getattr(logging, args.log_level.upper(), None)
if not isinstance(log_level_num, int):
raise ValueError('Invalid log level: %s', args.log_level)
logger.setLevel(log_level_num)
logger.debug("Log level set to %s:%s", args.log_level, log_level_num)
DEBUG = False
if parameters.get(DEBUG_PARAMETER):
parameters.pop(DEBUG_PARAMETER)
DEBUG = True
annotations = load_annotations()
logger.info("Annotations: %s", annotations)
interface_file = importlib.import_module(args.interface_name)
user_class = getattr(interface_file, args.interface_name)
if args.persistence:
logger.info('Restoring persisted component')
user_object = persistence.restore(user_class, parameters, debug=DEBUG)
persistence.persist(user_object, parameters.get("push_frequency"))
else:
user_object = user_class(**parameters)
if args.service_type == "MODEL":
import seldon_core.model_microservice as seldon_microservice
elif args.service_type == "ROUTER":
import seldon_core.router_microservice as seldon_microservice
elif args.service_type == "TRANSFORMER":
import seldon_core.transformer_microservice as seldon_microservice
elif args.service_type == "COMBINER":
import seldon_core.combiner_microservice as seldon_microservice
elif args.service_type == "OUTLIER_DETECTOR":
import seldon_core.outlier_detector_microservice as seldon_microservice
# set log level for the imported microservice type
seldon_microservice.logger.setLevel(log_level_num)
port = int(os.environ.get(SERVICE_PORT_ENV_NAME, DEFAULT_PORT))
if args.tracing:
logger.info("Initializing tracing")
from jaeger_client import Config
jaeger_serv = os.environ.get("JAEGER_AGENT_HOST","0.0.0.0")
jaeger_port = os.environ.get("JAEGER_AGENT_PORT", 5775)
jaeger_config = os.environ.get("JAEGER_CONFIG_PATH",None)
if jaeger_config is None:
logger.info("Using default tracing config")
config = Config(
config={ # usually read from some yaml config
'sampler': {
'type': 'const',
'param': 1,
},
'local_agent': {
'reporting_host': jaeger_serv,
'reporting_port': jaeger_port,
},
'logging': True,
},
service_name=args.interface_name,
validate=True,
)
else:
logger.info("Loading tracing config from %s",jaeger_config)
import yaml
with open(jaeger_config, 'r') as stream:
config_dict = yaml.load(stream)
config = Config(
config=config_dict,
service_name=args.interface_name,
validate=True,
)
# this call also sets opentracing.tracer
tracer = config.initialize_tracer()
if args.api_type == "REST":
def rest_prediction_server():
app = seldon_microservice.get_rest_microservice(
user_object, debug=DEBUG)
if args.tracing:
from flask_opentracing import FlaskTracer
tracing = FlaskTracer(tracer,True, app)
app.run(host='0.0.0.0', port=port)
logger.info("REST microservice running on port %i",port)
server1_func = rest_prediction_server
elif args.api_type == "GRPC":
def grpc_prediction_server():
if args.tracing:
from grpc_opentracing import open_tracing_server_interceptor
logger.info("Adding tracer")
interceptor = open_tracing_server_interceptor(tracer)
else:
interceptor = None
server = seldon_microservice.get_grpc_server(
user_object, debug=DEBUG, annotations=annotations, trace_interceptor=interceptor)
server.add_insecure_port("0.0.0.0:{}".format(port))
server.start()
logger.info("GRPC microservice Running on port %i",port)
while True:
time.sleep(1000)
server1_func = grpc_prediction_server
elif args.api_type == "FBS":
def fbs_prediction_server():
seldon_microservice.run_flatbuffers_server(user_object, port)
logger.info("FBS microservice Running on port %i",port)
server1_func = fbs_prediction_server
else:
server1_func = None
if hasattr(user_object, 'custom_service') and callable(getattr(user_object, 'custom_service')):
server2_func = user_object.custom_service
else:
server2_func = None
logger.info('Starting servers')
startServers(server1_func, server2_func)
def main():
LOG_FORMAT = '%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(levelname)s: %(message)s'
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger.info('Starting microservice.py:main')
sys.path.append(os.getcwd())
parser = argparse.ArgumentParser()
parser.add_argument("interface_name",
type=str,
help="Name of the user interface.")
parser.add_argument("api_type", type=str, choices=["REST", "GRPC", "FBS"])
parser.add_argument("--service-type",
type=str,
choices=[
"MODEL", "ROUTER", "TRANSFORMER", "COMBINER",
"OUTLIER_DETECTOR"
],
default="MODEL")
parser.add_argument("--persistence",
nargs='?',
default=0,
const=1,
type=int)
parser.add_argument("--parameters",
type=str,
default=os.environ.get(PARAMETERS_ENV_NAME, "[]"))
parser.add_argument("--log-level", type=str, default='INFO')
args = parser.parse_args()
parameters = parse_parameters(json.loads(args.parameters))
# set up log level
log_level_num = getattr(logging, args.log_level.upper(), None)
if not isinstance(log_level_num, int):
raise ValueError('Invalid log level: %s', args.log_level)
logger.setLevel(log_level_num)
DEBUG = False
if parameters.get(DEBUG_PARAMETER):
parameters.pop(DEBUG_PARAMETER)
DEBUG = True
annotations = load_annotations()
logger.info("Annotations: %s", annotations)
interface_file = importlib.import_module(args.interface_name)
user_class = getattr(interface_file, args.interface_name)
if args.persistence:
logger.info('Restoring persisted component')
user_object = persistence.restore(user_class, parameters, debug=DEBUG)
persistence.persist(user_object, parameters.get("push_frequency"))
else:
user_object = user_class(**parameters)
if args.service_type == "MODEL":
import seldon_core.model_microservice as seldon_microservice
elif args.service_type == "ROUTER":
import seldon_core.router_microservice as seldon_microservice
elif args.service_type == "TRANSFORMER":
import seldon_core.transformer_microservice as seldon_microservice
elif args.service_type == "COMBINER":
import seldon_core.combiner_microservice as seldon_microservice
elif args.service_type == "OUTLIER_DETECTOR":
import seldon_core.outlier_detector_microservice as seldon_microservice
port = int(os.environ.get(SERVICE_PORT_ENV_NAME, DEFAULT_PORT))
if args.api_type == "REST":
def rest_prediction_server():
app = seldon_microservice.get_rest_microservice(user_object,
debug=DEBUG)
app.run(host='0.0.0.0', port=port)
logger.info("REST microservice running on port %i", port)
server1_func = rest_prediction_server
elif args.api_type == "GRPC":
def grpc_prediction_server():
server = seldon_microservice.get_grpc_server(
user_object, debug=DEBUG, annotations=annotations)
server.add_insecure_port("0.0.0.0:{}".format(port))
server.start()
logger.info("GRPC microservice Running on port %i", port)
while True:
time.sleep(1000)
server1_func = grpc_prediction_server
elif args.api_type == "FBS":
def fbs_prediction_server():
seldon_microservice.run_flatbuffers_server(user_object, port)
logger.info("FBS microservice Running on port %i", port)
server1_func = fbs_prediction_server
else:
server1_func = None
if hasattr(user_object, 'custom_service') and callable(
getattr(user_object, 'custom_service')):
server2_func = user_object.custom_service
else:
server2_func = None
logger.info('Starting servers')
startServers(server1_func, server2_func)
def main():
LOG_FORMAT = (
"%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(levelname)s: %(message)s"
)
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger.info("Starting microservice.py:main")
sys.path.append(os.getcwd())
parser = argparse.ArgumentParser()
parser.add_argument("interface_name", type=str, help="Name of the user interface.")
parser.add_argument("api_type", type=str, choices=["REST", "GRPC", "FBS"])
parser.add_argument(
"--service-type",
type=str,
choices=["MODEL", "ROUTER", "TRANSFORMER", "COMBINER", "OUTLIER_DETECTOR"],
default="MODEL",
)
parser.add_argument("--persistence", nargs="?", default=0, const=1, type=int)
parser.add_argument(
"--parameters", type=str, default=os.environ.get(PARAMETERS_ENV_NAME, "[]")
)
parser.add_argument("--log-level", type=str, default="INFO")
parser.add_argument(
"--tracing",
nargs="?",
default=int(os.environ.get("TRACING", "0")),
const=1,
type=int,
)
# gunicorn settings, defaults are from http://docs.gunicorn.org/en/stable/settings.html
parser.add_argument(
"--workers",
type=int,
default=int(os.environ.get("GUNICORN_WORKERS", "1")),
help="Number of gunicorn workers for handling requests.",
)
parser.add_argument(
"--max-requests",
type=int,
default=int(os.environ.get("GUNICORN_MAX_REQUESTS", "0")),
help="Maximum number of requests gunicorn worker will process before restarting.",
)
parser.add_argument(
"--max-requests-jitter",
type=int,
default=int(os.environ.get("GUNICORN_MAX_REQUESTS_JITTER", "0")),
help="Maximum random jitter to add to max-requests.",
)
args = parser.parse_args()
parameters = parse_parameters(json.loads(args.parameters))
# set flask trace jaeger extra tags
jaeger_extra_tags = list(
filter(
lambda x: (x != ""),
[tag.strip() for tag in os.environ.get("JAEGER_EXTRA_TAGS", "").split(",")],
)
)
logger.info("Parse JAEGER_EXTRA_TAGS %s", jaeger_extra_tags)
# set up log level
log_level_raw = os.environ.get(LOG_LEVEL_ENV, args.log_level.upper())
log_level_num = getattr(logging, log_level_raw, None)
if not isinstance(log_level_num, int):
raise ValueError("Invalid log level: %s", args.log_level)
logger.setLevel(log_level_num)
logger.debug("Log level set to %s:%s", args.log_level, log_level_num)
annotations = load_annotations()
logger.info("Annotations: %s", annotations)
parts = args.interface_name.rsplit(".", 1)
if len(parts) == 1:
logger.info("Importing %s", args.interface_name)
interface_file = importlib.import_module(args.interface_name)
user_class = getattr(interface_file, args.interface_name)
else:
logger.info("Importing submodule %s", parts)
interface_file = importlib.import_module(parts[0])
user_class = getattr(interface_file, parts[1])
if args.persistence:
logger.info("Restoring persisted component")
user_object = persistence.restore(user_class, parameters)
persistence.persist(user_object, parameters.get("push_frequency"))
else:
user_object = user_class(**parameters)
# set log level for the imported microservice type
seldon_microservice.logger.setLevel(log_level_num)
logging.getLogger().setLevel(log_level_num)
for handler in logger.handlers:
handler.setLevel(log_level_num)
port = int(os.environ.get(SERVICE_PORT_ENV_NAME, DEFAULT_PORT))
if args.tracing:
tracer = setup_tracing(args.interface_name)
if args.api_type == "REST":
if args.workers > 1:
def rest_prediction_server():
options = {
"bind": "%s:%s" % ("0.0.0.0", port),
"access_logfile": "-",
"loglevel": "info",
"timeout": 5000,
"reload": "true",
"workers": args.workers,
"max_requests": args.max_requests,
"max_requests_jitter": args.max_requests_jitter,
}
app = seldon_microservice.get_rest_microservice(user_object)
StandaloneApplication(app, user_object, options=options).run()
logger.info("REST gunicorn microservice running on port %i", port)
server1_func = rest_prediction_server
else:
def rest_prediction_server():
app = seldon_microservice.get_rest_microservice(user_object)
try:
user_object.load()
except (NotImplementedError, AttributeError):
pass
if args.tracing:
logger.info("Tracing branch is active")
from flask_opentracing import FlaskTracing
logger.info("Set JAEGER_EXTRA_TAGS %s", jaeger_extra_tags)
tracing = FlaskTracing(tracer, True, app, jaeger_extra_tags)
app.run(host="0.0.0.0", port=port)
logger.info("REST microservice running on port %i", port)
server1_func = rest_prediction_server
elif args.api_type == "GRPC":
def grpc_prediction_server():
if args.tracing:
from grpc_opentracing import open_tracing_server_interceptor
logger.info("Adding tracer")
interceptor = open_tracing_server_interceptor(tracer)
else:
interceptor = None
server = seldon_microservice.get_grpc_server(
user_object, annotations=annotations, trace_interceptor=interceptor
)
try:
user_object.load()
except (NotImplementedError, AttributeError):
pass
server.add_insecure_port(f"0.0.0.0:{port}")
server.start()
logger.info("GRPC microservice Running on port %i", port)
while True:
time.sleep(1000)
server1_func = grpc_prediction_server
else:
server1_func = None
if hasattr(user_object, "custom_service") and callable(
getattr(user_object, "custom_service")
):
server2_func = user_object.custom_service
else:
server2_func = None
logger.info("Starting servers")
start_servers(server1_func, server2_func)
