def cleanup(): ts.stop_torchserve() rm_dir('model_store') rm_dir('logs') # clean up residual from model-archiver IT suite. rm_dir('model-archiver/model_archiver/htmlcov_ut model_archiver/model-archiver/htmlcov_it') rm_file('ts_scripts/*_pb2*.py', True)
def cleanup(): ts.stop_torchserve() rm_dir('model_store') rm_dir('logs') # clean up residual from model-archiver IT suite. rm_dir( 'model-archiver/model_archiver/htmlcov_ut model_archiver/model-archiver/htmlcov_it' )
def trigger_inference_tests(): """ Return exit code of newman execution of inference collection """ ts.start_torchserve(ncs=True, model_store=MODEL_STORE_DIR, log_file=TS_CONSOLE_LOG_FILE) EXIT_CODE = os.system( f"newman run -e {POSTMAN_ENV_FILE} {POSTMAN_COLLECTION_INFERENCE} -d {POSTMAN_INFERENCE_DATA_FILE} -r cli,htmlextra --reporter-htmlextra-export {ARTIFACTS_INFERENCE_DIR}/{REPORT_FILE} --verbose" ) ts.stop_torchserve() move_logs(TS_CONSOLE_LOG_FILE, ARTIFACTS_INFERENCE_DIR) cleanup_model_store() return EXIT_CODE
def trigger_management_tests(): """ Return exit code of newman execution of management collection """ ts.start_torchserve(ncs=True, model_store=MODEL_STORE_DIR, log_file=TS_CONSOLE_LOG_FILE) EXIT_CODE = os.system( f"newman run -e {POSTMAN_ENV_FILE} {POSTMAN_COLLECTION_MANAGEMENT} -r cli,html --reporter-html-export {ARTIFACTS_MANAGEMENT_DIR}/{REPORT_FILE} --verbose" ) ts.stop_torchserve() move_logs(TS_CONSOLE_LOG_FILE, ARTIFACTS_MANAGEMENT_DIR) cleanup_model_store() return EXIT_CODE
def trigger_https_tests_kfv2(): """ Return exit code of newman execution of https collection """ ts.start_torchserve(ncs=True, model_store=MODEL_STORE_DIR, config_file=TS_CONFIG_FILE_HTTPS_KFV2, log_file=TS_CONSOLE_LOG_FILE) EXIT_CODE = os.system( f"newman run --insecure -e {POSTMAN_ENV_FILE} {POSTMAN_COLLECTION_HTTPS_KFV2} -r cli,htmlextra --reporter-htmlextra-export {ARTIFACTS_HTTPS_DIR_KFV2}/{REPORT_FILE} --verbose" ) ts.stop_torchserve() move_logs(TS_CONSOLE_LOG_FILE, ARTIFACTS_HTTPS_DIR_KFV2) cleanup_model_store() return EXIT_CODE
def trigger_inference_tests_kfv2(): """ Return exit code of newman execution of inference collection """ config_file = open("config.properties", "w") config_file.write("service_envelope=kfservingv2") config_file.close() ts.start_torchserve(ncs=True, model_store=MODEL_STORE_DIR, config_file="config.properties", log_file=TS_CONSOLE_LOG_FILE) EXIT_CODE = os.system( f"newman run -e {POSTMAN_ENV_FILE} {POSTMAN_COLLECTION_INFERENCE_KFV2} -d {POSTMAN_INFERENCE_DATA_FILE_KFV2} -r cli,htmlextra --reporter-htmlextra-export {ARTIFACTS_INFERENCE_DIR_KFV2}/{REPORT_FILE} --verbose" ) ts.stop_torchserve() move_logs(TS_CONSOLE_LOG_FILE, ARTIFACTS_INFERENCE_DIR_KFV2) cleanup_model_store() os.remove("config.properties") return EXIT_CODE
def trigger_incr_timeout_inference_tests(): """ Return exit code of newman execution of increased timeout inference collection """ # Configuration with increased timeout config_file = open("config.properties", "w") config_file.write("default_response_timeout=300") config_file.close() ts.start_torchserve(ncs=True, model_store=MODEL_STORE_DIR, config_file="config.properties", log_file=TS_CONSOLE_LOG_FILE) EXIT_CODE = os.system( f"newman run -e {POSTMAN_ENV_FILE} {POSTMAN_COLLECTION_INFERENCE} -d {POSTMAN_INCRSD_TIMEOUT_INFERENCE_DATA_FILE} -r cli,htmlextra --reporter-htmlextra-export {ARTIFACTS_INCRSD_TIMEOUT_INFERENCE_DIR}/{REPORT_FILE} --verbose" ) ts.stop_torchserve() move_logs(TS_CONSOLE_LOG_FILE, ARTIFACTS_INCRSD_TIMEOUT_INFERENCE_DIR) cleanup_model_store() os.remove("config.properties") return EXIT_CODE
def test_workflow_sanity(): current_path = os.getcwd() ts_log_file = os.path.join("logs", "ts_console.log") os.makedirs("model_store", exist_ok=True) os.makedirs("logs", exist_ok=True) started = ts.start_torchserve(ncs=True, log_file=ts_log_file, model_store="model_store", workflow_store="model_store") if not started: sys.exit(1) # Register workflow response = ts.register_workflow("densenet_wf") if response and response.status_code == 200: print(response.text) else: print(f"## Failed to register workflow") sys.exit(1) # Run prediction on workflow response = ts.workflow_prediction("densenet", "examples/image_classifier/kitten.jpg") if response and response.status_code == 200: print(response.text) else: print(f"## Failed to run inference on workflow - {response.text}") sys.exit(1) response = ts.unregister_workflow("densenet") if response and response.status_code == 200: print(response.text) else: print(f"## Failed to unregister workflow") sys.exit(1) stopped = ts.stop_torchserve() if not stopped: sys.exit(1)
def test_sanity(): generate_grpc_client_stubs() import pathlib pathlib.Path(__file__).parent.absolute() print("## Started sanity tests") resnet18_model = { "name": "resnet-18", "inputs": ["examples/image_classifier/kitten.jpg"], "handler": "image_classifier" } models_to_validate = [{ "name": "fastrcnn", "inputs": ["examples/object_detector/persons.jpg"], "handler": "object_detector" }, { "name": "fcn_resnet_101", "inputs": [ "docs/images/blank_image.jpg", "examples/image_segmenter/fcn/persons.jpg" ], "handler": "image_segmenter" }, { "name": "my_text_classifier_v2", "inputs": ["examples/text_classification/sample_text.txt"], "handler": "text_classification" }, resnet18_model, { "name": "my_text_classifier_scripted_v2", "inputs": ["examples/text_classification/sample_text.txt"], "handler": "text_classification" }, { "name": "alexnet_scripted", "inputs": ["examples/image_classifier/kitten.jpg"], "handler": "image_classifier" }, { "name": "fcn_resnet_101_scripted", "inputs": ["examples/image_segmenter/fcn/persons.jpg"], "handler": "image_segmenter" }, { "name": "roberta_qa_no_torchscript", "inputs": ["examples/Huggingface_Transformers/QA_artifacts/sample_text.txt"], "handler": "custom" }, { "name": "bert_token_classification_no_torchscript", "inputs": [ "examples/Huggingface_Transformers/Token_classification_artifacts/sample_text.txt" ], "handler": "custom" }, { "name": "bert_seqc_without_torchscript", "inputs": [ "examples/Huggingface_Transformers/Seq_classification_artifacts/sample_text.txt" ], "handler": "custom" }] ts_log_file = os.path.join("logs", "ts_console.log") is_gpu_instance = utils.is_gpu_instance() os.makedirs("model_store", exist_ok=True) os.makedirs("logs", exist_ok=True) if is_gpu_instance: import torch if not torch.cuda.is_available(): sys.exit("## Ohh its NOT running on GPU !") started = ts.start_torchserve(log_file=ts_log_file) if not started: sys.exit(1) for model in models_to_validate: model_name = model["name"] model_inputs = model["inputs"] model_handler = model["handler"] # Run gRPC sanity register_model_grpc_cmd = f"python ts_scripts/torchserve_grpc_client.py register {model_name}" status = os.system(register_model_grpc_cmd) if status != 0: print("## Failed to register model with torchserve") sys.exit(1) else: print( f"## Successfully registered {model_name} model with torchserve" ) for input in model_inputs: infer_model_grpc_cmd = f"python ts_scripts/torchserve_grpc_client.py infer {model_name} {input}" status = os.system(infer_model_grpc_cmd) if status != 0: print(f"## Failed to run inference on {model_name} model") sys.exit(1) else: print(f"## Successfully ran inference on {model_name} model.") unregister_model_grpc_cmd = f"python ts_scripts/torchserve_grpc_client.py unregister {model_name}" status = os.system(unregister_model_grpc_cmd) if status != 0: print(f"## Failed to unregister {model_name}") sys.exit(1) else: print(f"## Successfully unregistered {model_name}") # Run REST sanity response = ts.register_model(model_name) if response and response.status_code == 200: print( f"## Successfully registered {model_name} model with torchserve" ) else: print("## Failed to register model with torchserve") sys.exit(1) # For each input execute inference n=4 times for input in model_inputs: for i in range(4): response = ts.run_inference(model_name, input) if response and response.status_code == 200: print( f"## Successfully ran inference on {model_name} model." ) else: print(f"## Failed to run inference on {model_name} model") sys.exit(1) if is_gpu_instance: if validate_model_on_gpu(): print(f"## Model {model_name} successfully loaded on GPU") else: sys.exit( f"## Something went wrong, model {model_name} did not load on GPU!!" ) # skip unregistering resnet-18 model to test snapshot feature with restart if model != resnet18_model: response = ts.unregister_model(model_name) if response and response.status_code == 200: print(f"## Successfully unregistered {model_name}") else: print(f"## Failed to unregister {model_name}") sys.exit(1) print(f"## {model_handler} handler is stable.") stopped = ts.stop_torchserve() if not stopped: sys.exit(1) # Restarting torchserve # This should restart with the generated snapshot and resnet-18 model should be automatically registered started = ts.start_torchserve(log_file=ts_log_file) if not started: sys.exit(1) response = ts.run_inference(resnet18_model["name"], resnet18_model["inputs"][0]) if response and response.status_code == 200: print( f"## Successfully ran inference on {resnet18_model['name']} model." ) else: print(f"## Failed to run inference on {resnet18_model['name']} model") sys.exit(1) stopped = ts.stop_torchserve() if not stopped: sys.exit(1) links_ok = run_markdown_link_checker() if not links_ok: print("##WARNING : Broken links in docs.")