def cleanup():
ts.stop_torchserve()
rm_dir('model_store')
rm_dir('logs')
# clean up residual from model-archiver IT suite.
rm_dir('model-archiver/model_archiver/htmlcov_ut model_archiver/model-archiver/htmlcov_it')
rm_file('ts_scripts/*_pb2*.py', True)
def cleanup():
ts.stop_torchserve()
rm_dir('model_store')
rm_dir('logs')
# clean up residual from model-archiver IT suite.
rm_dir(
'model-archiver/model_archiver/htmlcov_ut model_archiver/model-archiver/htmlcov_it'
)
def trigger_inference_tests():
""" Return exit code of newman execution of inference collection """
ts.start_torchserve(ncs=True,
model_store=MODEL_STORE_DIR,
log_file=TS_CONSOLE_LOG_FILE)
EXIT_CODE = os.system(
f"newman run -e {POSTMAN_ENV_FILE} {POSTMAN_COLLECTION_INFERENCE} -d {POSTMAN_INFERENCE_DATA_FILE} -r cli,htmlextra --reporter-htmlextra-export {ARTIFACTS_INFERENCE_DIR}/{REPORT_FILE} --verbose"
)
ts.stop_torchserve()
move_logs(TS_CONSOLE_LOG_FILE, ARTIFACTS_INFERENCE_DIR)
cleanup_model_store()
return EXIT_CODE
def trigger_management_tests():
""" Return exit code of newman execution of management collection """
ts.start_torchserve(ncs=True,
model_store=MODEL_STORE_DIR,
log_file=TS_CONSOLE_LOG_FILE)
EXIT_CODE = os.system(
f"newman run -e {POSTMAN_ENV_FILE} {POSTMAN_COLLECTION_MANAGEMENT} -r cli,html --reporter-html-export {ARTIFACTS_MANAGEMENT_DIR}/{REPORT_FILE} --verbose"
)
ts.stop_torchserve()
move_logs(TS_CONSOLE_LOG_FILE, ARTIFACTS_MANAGEMENT_DIR)
cleanup_model_store()
return EXIT_CODE
def trigger_https_tests_kfv2():
""" Return exit code of newman execution of https collection """
ts.start_torchserve(ncs=True,
model_store=MODEL_STORE_DIR,
config_file=TS_CONFIG_FILE_HTTPS_KFV2,
log_file=TS_CONSOLE_LOG_FILE)
EXIT_CODE = os.system(
f"newman run --insecure -e {POSTMAN_ENV_FILE} {POSTMAN_COLLECTION_HTTPS_KFV2} -r cli,htmlextra --reporter-htmlextra-export {ARTIFACTS_HTTPS_DIR_KFV2}/{REPORT_FILE} --verbose"
)
ts.stop_torchserve()
move_logs(TS_CONSOLE_LOG_FILE, ARTIFACTS_HTTPS_DIR_KFV2)
cleanup_model_store()
return EXIT_CODE
def trigger_inference_tests_kfv2():
""" Return exit code of newman execution of inference collection """
config_file = open("config.properties", "w")
config_file.write("service_envelope=kfservingv2")
config_file.close()
ts.start_torchserve(ncs=True,
model_store=MODEL_STORE_DIR,
config_file="config.properties",
log_file=TS_CONSOLE_LOG_FILE)
EXIT_CODE = os.system(
f"newman run -e {POSTMAN_ENV_FILE} {POSTMAN_COLLECTION_INFERENCE_KFV2} -d {POSTMAN_INFERENCE_DATA_FILE_KFV2} -r cli,htmlextra --reporter-htmlextra-export {ARTIFACTS_INFERENCE_DIR_KFV2}/{REPORT_FILE} --verbose"
)
ts.stop_torchserve()
move_logs(TS_CONSOLE_LOG_FILE, ARTIFACTS_INFERENCE_DIR_KFV2)
cleanup_model_store()
os.remove("config.properties")
return EXIT_CODE
def trigger_incr_timeout_inference_tests():
""" Return exit code of newman execution of increased timeout inference collection """
# Configuration with increased timeout
config_file = open("config.properties", "w")
config_file.write("default_response_timeout=300")
config_file.close()
ts.start_torchserve(ncs=True,
model_store=MODEL_STORE_DIR,
config_file="config.properties",
log_file=TS_CONSOLE_LOG_FILE)
EXIT_CODE = os.system(
f"newman run -e {POSTMAN_ENV_FILE} {POSTMAN_COLLECTION_INFERENCE} -d {POSTMAN_INCRSD_TIMEOUT_INFERENCE_DATA_FILE} -r cli,htmlextra --reporter-htmlextra-export {ARTIFACTS_INCRSD_TIMEOUT_INFERENCE_DIR}/{REPORT_FILE} --verbose"
)
ts.stop_torchserve()
move_logs(TS_CONSOLE_LOG_FILE, ARTIFACTS_INCRSD_TIMEOUT_INFERENCE_DIR)
cleanup_model_store()
os.remove("config.properties")
return EXIT_CODE
def test_workflow_sanity():
current_path = os.getcwd()
ts_log_file = os.path.join("logs", "ts_console.log")
os.makedirs("model_store", exist_ok=True)
os.makedirs("logs", exist_ok=True)
started = ts.start_torchserve(ncs=True,
log_file=ts_log_file,
model_store="model_store",
workflow_store="model_store")
if not started:
sys.exit(1)
# Register workflow
response = ts.register_workflow("densenet_wf")
if response and response.status_code == 200:
print(response.text)
else:
print(f"## Failed to register workflow")
sys.exit(1)
# Run prediction on workflow
response = ts.workflow_prediction("densenet",
"examples/image_classifier/kitten.jpg")
if response and response.status_code == 200:
print(response.text)
else:
print(f"## Failed to run inference on workflow - {response.text}")
sys.exit(1)
response = ts.unregister_workflow("densenet")
if response and response.status_code == 200:
print(response.text)
else:
print(f"## Failed to unregister workflow")
sys.exit(1)
stopped = ts.stop_torchserve()
if not stopped:
sys.exit(1)
def test_sanity():
generate_grpc_client_stubs()
import pathlib
pathlib.Path(__file__).parent.absolute()
print("## Started sanity tests")
resnet18_model = {
"name": "resnet-18",
"inputs": ["examples/image_classifier/kitten.jpg"],
"handler": "image_classifier"
}
models_to_validate = [{
"name": "fastrcnn",
"inputs": ["examples/object_detector/persons.jpg"],
"handler": "object_detector"
}, {
"name":
"fcn_resnet_101",
"inputs": [
"docs/images/blank_image.jpg",
"examples/image_segmenter/fcn/persons.jpg"
],
"handler":
"image_segmenter"
}, {
"name":
"my_text_classifier_v2",
"inputs": ["examples/text_classification/sample_text.txt"],
"handler":
"text_classification"
}, resnet18_model, {
"name":
"my_text_classifier_scripted_v2",
"inputs": ["examples/text_classification/sample_text.txt"],
"handler":
"text_classification"
}, {
"name": "alexnet_scripted",
"inputs": ["examples/image_classifier/kitten.jpg"],
"handler": "image_classifier"
}, {
"name":
"fcn_resnet_101_scripted",
"inputs": ["examples/image_segmenter/fcn/persons.jpg"],
"handler":
"image_segmenter"
}, {
"name":
"roberta_qa_no_torchscript",
"inputs":
["examples/Huggingface_Transformers/QA_artifacts/sample_text.txt"],
"handler":
"custom"
}, {
"name":
"bert_token_classification_no_torchscript",
"inputs": [
"examples/Huggingface_Transformers/Token_classification_artifacts/sample_text.txt"
],
"handler":
"custom"
}, {
"name":
"bert_seqc_without_torchscript",
"inputs": [
"examples/Huggingface_Transformers/Seq_classification_artifacts/sample_text.txt"
],
"handler":
"custom"
}]
ts_log_file = os.path.join("logs", "ts_console.log")
is_gpu_instance = utils.is_gpu_instance()
os.makedirs("model_store", exist_ok=True)
os.makedirs("logs", exist_ok=True)
if is_gpu_instance:
import torch
if not torch.cuda.is_available():
sys.exit("## Ohh its NOT running on GPU !")
started = ts.start_torchserve(log_file=ts_log_file)
if not started:
sys.exit(1)
for model in models_to_validate:
model_name = model["name"]
model_inputs = model["inputs"]
model_handler = model["handler"]
# Run gRPC sanity
register_model_grpc_cmd = f"python ts_scripts/torchserve_grpc_client.py register {model_name}"
status = os.system(register_model_grpc_cmd)
if status != 0:
print("## Failed to register model with torchserve")
sys.exit(1)
else:
print(
f"## Successfully registered {model_name} model with torchserve"
)
for input in model_inputs:
infer_model_grpc_cmd = f"python ts_scripts/torchserve_grpc_client.py infer {model_name} {input}"
status = os.system(infer_model_grpc_cmd)
if status != 0:
print(f"## Failed to run inference on {model_name} model")
sys.exit(1)
else:
print(f"## Successfully ran inference on {model_name} model.")
unregister_model_grpc_cmd = f"python ts_scripts/torchserve_grpc_client.py unregister {model_name}"
status = os.system(unregister_model_grpc_cmd)
if status != 0:
print(f"## Failed to unregister {model_name}")
sys.exit(1)
else:
print(f"## Successfully unregistered {model_name}")
# Run REST sanity
response = ts.register_model(model_name)
if response and response.status_code == 200:
print(
f"## Successfully registered {model_name} model with torchserve"
)
else:
print("## Failed to register model with torchserve")
sys.exit(1)
# For each input execute inference n=4 times
for input in model_inputs:
for i in range(4):
response = ts.run_inference(model_name, input)
if response and response.status_code == 200:
print(
f"## Successfully ran inference on {model_name} model."
)
else:
print(f"## Failed to run inference on {model_name} model")
sys.exit(1)
if is_gpu_instance:
if validate_model_on_gpu():
print(f"## Model {model_name} successfully loaded on GPU")
else:
sys.exit(
f"## Something went wrong, model {model_name} did not load on GPU!!"
)
# skip unregistering resnet-18 model to test snapshot feature with restart
if model != resnet18_model:
response = ts.unregister_model(model_name)
if response and response.status_code == 200:
print(f"## Successfully unregistered {model_name}")
else:
print(f"## Failed to unregister {model_name}")
sys.exit(1)
print(f"## {model_handler} handler is stable.")
stopped = ts.stop_torchserve()
if not stopped:
sys.exit(1)
# Restarting torchserve
# This should restart with the generated snapshot and resnet-18 model should be automatically registered
started = ts.start_torchserve(log_file=ts_log_file)
if not started:
sys.exit(1)
response = ts.run_inference(resnet18_model["name"],
resnet18_model["inputs"][0])
if response and response.status_code == 200:
print(
f"## Successfully ran inference on {resnet18_model['name']} model."
)
else:
print(f"## Failed to run inference on {resnet18_model['name']} model")
sys.exit(1)
stopped = ts.stop_torchserve()
if not stopped:
sys.exit(1)
links_ok = run_markdown_link_checker()
if not links_ok:
print("##WARNING : Broken links in docs.")
