def test_start_model_server_default_service_handler(adapt, create_config,
sigterm, subprocess_popen,
subprocess_call):
model_server.start_model_server()
adapt.assert_called_once_with(model_server.DEFAULT_HANDLER_SERVICE)
create_config.assert_called_once_with()
mxnet_model_server_cmd = [
'mxnet-model-server',
'--start',
'--model-store',
model_server.DEFAULT_MMS_MODEL_DIRECTORY,
'--mms-config',
model_server.MMS_CONFIG_FILE,
'--log-config',
model_server.DEFAULT_MMS_LOG_FILE,
]
subprocess_popen.assert_called_once_with(mxnet_model_server_cmd)
sigterm.assert_called_once_with(MMS_PROCESS)
tail_cmd = ['tail', '-f', '/dev/null']
subprocess_call.assert_called_once_with(tail_cmd)
def _start_mms():
# by default the number of workers per model is 1, but we can configure it through the
# environment variable below if desired.
os.environ['MMS_DEFAULT_WORKERS_PER_MODEL'] = '2'
os.environ['OMP_NUM_THREADS'] = '8'
model_server.start_model_server(
handler_service='/home/model-server/model_handler.py:handle')
def test_start_model_server_default_service_handler(
adapt,
create_config,
exists,
install_requirements,
sigterm,
retrieve,
subprocess_popen,
subprocess_call,
):
model_server.start_model_server()
adapt.assert_called_once_with(model_server.DEFAULT_HANDLER_SERVICE)
create_config.assert_called_once_with()
exists.assert_called_once_with(REQUIREMENTS_PATH)
install_requirements.assert_called_once_with()
multi_model_server_cmd = [
"multi-model-server",
"--start",
"--model-store",
model_server.DEFAULT_MMS_MODEL_DIRECTORY,
"--mms-config",
model_server.MMS_CONFIG_FILE,
"--log-config",
model_server.DEFAULT_MMS_LOG_FILE,
]
subprocess_popen.assert_called_once_with(multi_model_server_cmd)
sigterm.assert_called_once_with(retrieve.return_value)
def test_start_model_server_custom_handler_service(adapt, create_config, sigterm, retrieve, subprocess_popen,
subprocess_call):
handler_service = Mock()
model_server.start_model_server(handler_service)
adapt.assert_called_once_with(handler_service)
def _start_mms():
# by default the number of workers per model is 1, but we can configure it through the
# environment variable below if desired.
# os.environ['SAGEMAKER_MODEL_SERVER_WORKERS'] = '2'
os.environ['SAGEMAKER_MODEL_SERVER_WORKERS'] = '2'
model_server.start_model_server(
handler_service='/opt/ml/code/app/model_handler_sm.py:handle')
def main():
_update_mxnet_env_vars()
# there's a race condition that causes the model server command to
# sometimes fail with 'bad address'. more investigation needed.
time.sleep(5)
model_server.start_model_server(handler_service=HANDLER_SERVICE)
def _start_mms():
# by default the number of workers per model is 1, but we can configure it through the
# environment variable below if desired.
os.environ['MMS_MAX_REQUEST_SIZE'] = '536870912'
os.environ['MMS_MAX_RESPONSE_SIZE'] = '536870912'
os.environ['MMS_DEFAULT_WORKERS_PER_MODEL'] = '4'
os.environ['MMS_JOB_QUEUE_SIZE'] = '500'
# os.environ['OMP_NUM_THREADS']='1'
os.environ['TVM_BIND_THREADS'] = '0'
# os.environ['MXNET_USE_OPERATOR_TUNING']='1'
model_server.start_model_server(
handler_service='/home/model-server/model_handler.py:handle')
def _start_model_server():
# there's a race condition that causes the model server command to
# sometimes fail with 'bad address'. more investigation needed
# retry starting mms until it's ready
model_server.start_model_server(handler_service=HANDLER_SERVICE)
from sagemaker_inference import model_server from . import handler_service HANDLER_SERVICE = handler_service.__file__ model_server.start_model_server(handler_service=HANDLER_SERVICE)
import sys
import os
import argparse
import logging
from sagemaker_inference import model_server
logging.getLogger().setLevel(logging.INFO)
if __name__ == "__main__":
model_server.start_model_server(handler_service="serving.handler")
def _start_model_server():
sleep(5)
model_server.start_model_server(handler_service=HANDLER_SERVICE)
def main():
_update_mxnet_env_vars()
model_server.start_model_server(handler_service=HANDLER_SERVICE)
def _start_model_server():
sleep(5)
model_server.start_model_server(
handler_service="/usr/local/bin/custom_handler.py:handle")
