def load_dependencies():
env = cs.HostingEnvironment()
saved_model_path = os.path.join(env.model_dir, 'export/Servo')
subprocess.Popen([
'tensorflow_model_server', '--port={}'.format(TF_SERVING_PORT),
'--model_name={}'.format(GENERIC_MODEL_NAME),
'--model_base_path={}'.format(saved_model_path)
])
def transformer(user_module):
env = cs.HostingEnvironment()
port = int(cs.Server.next_safe_port(env.port_range)) if env.port_range else DEFAULT_TF_SERVING_PORT
grpc_proxy_client = proxy_client.GRPCProxyClient(port)
_wait_model_to_load(grpc_proxy_client, TF_SERVING_MAXIMUM_LOAD_MODEL_TIME_IN_SECONDS)
return Transformer.from_module(user_module, grpc_proxy_client)
def load_dependencies():
env = cs.HostingEnvironment()
port = cs.Server.next_safe_port(env.port_range) if env.port_range else DEFAULT_TF_SERVING_PORT
saved_model_path = os.path.join(env.model_dir, 'export/Servo')
subprocess.Popen(['tensorflow_model_server',
'--port={}'.format(port),
'--model_name={}'.format(GENERIC_MODEL_NAME),
'--model_base_path={}'.format(saved_model_path)])
def start(cls):
"""Prepare the container for model serving, configure and launch the model server stack.
"""
logger.info("reading config")
env = cs.HostingEnvironment()
env.start_metrics_if_enabled()
if env.user_script_name:
Server._download_user_module(env)
logger.info('loading framework-specific dependencies')
framework = cs.ContainerEnvironment.load_framework()
framework.load_dependencies()
nginx_pid = 0
gunicorn_bind_address = '0.0.0.0:8080'
if env.use_nginx:
logger.info("starting nginx")
nginx_conf = pkg_resources.resource_filename(
'container_support', 'etc/nginx.conf')
subprocess.check_call(
['ln', '-sf', '/dev/stdout', '/var/log/nginx/access.log'])
subprocess.check_call(
['ln', '-sf', '/dev/stderr', '/var/log/nginx/error.log'])
gunicorn_bind_address = 'unix:/tmp/gunicorn.sock'
nginx_pid = subprocess.Popen(['nginx', '-c', nginx_conf]).pid
logger.info("starting gunicorn")
gunicorn_pid = subprocess.Popen([
"gunicorn", "--timeout",
str(env.model_server_timeout), "-k", "gevent", "-b",
gunicorn_bind_address, "--worker-connections",
str(1000 * env.model_server_workers), "-w",
str(env.model_server_workers), "container_support.wsgi:app"
]).pid
signal.signal(
signal.SIGTERM,
lambda a, b: Server._sigterm_handler(nginx_pid, gunicorn_pid))
children = set([nginx_pid, gunicorn_pid
]) if nginx_pid else gunicorn_pid
logger.info("inference server started. waiting on processes: %s" %
children)
while True:
pid, _ = os.wait()
if pid in children:
break
Server._sigterm_handler(nginx_pid, gunicorn_pid)
def from_env(cls):
cs.configure_logging()
logger.info("creating Server instance")
env = cs.HostingEnvironment()
env.pip_install_requirements()
logger.info("importing user module")
user_module = env.import_user_module(
) if env.user_script_name else None
framework = cs.ContainerEnvironment.load_framework()
transformer = framework.transformer(user_module)
server = Server("model server", transformer)
logger.info("returning initialized server")
return server