def test_start_model_server_default_service_handler(adapt, create_config, sigterm, subprocess_popen, subprocess_call): model_server.start_model_server() adapt.assert_called_once_with(model_server.DEFAULT_HANDLER_SERVICE) create_config.assert_called_once_with() mxnet_model_server_cmd = [ 'mxnet-model-server', '--start', '--model-store', model_server.DEFAULT_MMS_MODEL_DIRECTORY, '--mms-config', model_server.MMS_CONFIG_FILE, '--log-config', model_server.DEFAULT_MMS_LOG_FILE, ] subprocess_popen.assert_called_once_with(mxnet_model_server_cmd) sigterm.assert_called_once_with(MMS_PROCESS) tail_cmd = ['tail', '-f', '/dev/null'] subprocess_call.assert_called_once_with(tail_cmd)
def _start_mms(): # by default the number of workers per model is 1, but we can configure it through the # environment variable below if desired. os.environ['MMS_DEFAULT_WORKERS_PER_MODEL'] = '2' os.environ['OMP_NUM_THREADS'] = '8' model_server.start_model_server( handler_service='/home/model-server/model_handler.py:handle')
def test_start_model_server_default_service_handler( adapt, create_config, exists, install_requirements, sigterm, retrieve, subprocess_popen, subprocess_call, ): model_server.start_model_server() adapt.assert_called_once_with(model_server.DEFAULT_HANDLER_SERVICE) create_config.assert_called_once_with() exists.assert_called_once_with(REQUIREMENTS_PATH) install_requirements.assert_called_once_with() multi_model_server_cmd = [ "multi-model-server", "--start", "--model-store", model_server.DEFAULT_MMS_MODEL_DIRECTORY, "--mms-config", model_server.MMS_CONFIG_FILE, "--log-config", model_server.DEFAULT_MMS_LOG_FILE, ] subprocess_popen.assert_called_once_with(multi_model_server_cmd) sigterm.assert_called_once_with(retrieve.return_value)
def test_start_model_server_custom_handler_service(adapt, create_config, sigterm, retrieve, subprocess_popen, subprocess_call): handler_service = Mock() model_server.start_model_server(handler_service) adapt.assert_called_once_with(handler_service)
def _start_mms(): # by default the number of workers per model is 1, but we can configure it through the # environment variable below if desired. # os.environ['SAGEMAKER_MODEL_SERVER_WORKERS'] = '2' os.environ['SAGEMAKER_MODEL_SERVER_WORKERS'] = '2' model_server.start_model_server( handler_service='/opt/ml/code/app/model_handler_sm.py:handle')
def main(): _update_mxnet_env_vars() # there's a race condition that causes the model server command to # sometimes fail with 'bad address'. more investigation needed. time.sleep(5) model_server.start_model_server(handler_service=HANDLER_SERVICE)
def _start_mms(): # by default the number of workers per model is 1, but we can configure it through the # environment variable below if desired. os.environ['MMS_MAX_REQUEST_SIZE'] = '536870912' os.environ['MMS_MAX_RESPONSE_SIZE'] = '536870912' os.environ['MMS_DEFAULT_WORKERS_PER_MODEL'] = '4' os.environ['MMS_JOB_QUEUE_SIZE'] = '500' # os.environ['OMP_NUM_THREADS']='1' os.environ['TVM_BIND_THREADS'] = '0' # os.environ['MXNET_USE_OPERATOR_TUNING']='1' model_server.start_model_server( handler_service='/home/model-server/model_handler.py:handle')
def _start_model_server(): # there's a race condition that causes the model server command to # sometimes fail with 'bad address'. more investigation needed # retry starting mms until it's ready model_server.start_model_server(handler_service=HANDLER_SERVICE)
from sagemaker_inference import model_server from . import handler_service HANDLER_SERVICE = handler_service.__file__ model_server.start_model_server(handler_service=HANDLER_SERVICE)
import sys import os import argparse import logging from sagemaker_inference import model_server logging.getLogger().setLevel(logging.INFO) if __name__ == "__main__": model_server.start_model_server(handler_service="serving.handler")
def _start_model_server(): sleep(5) model_server.start_model_server(handler_service=HANDLER_SERVICE)
def main(): _update_mxnet_env_vars() model_server.start_model_server(handler_service=HANDLER_SERVICE)
def _start_model_server(): sleep(5) model_server.start_model_server( handler_service="/usr/local/bin/custom_handler.py:handle")