def test_log_location_var_snapshot_enabled(): """ Validates that non metrics logs get saved in directory configured via LOG_LOCATION environment variable. """ # We stop torchserve here so that we can set LOG_LOCATION in environment variable and rerun torchserve test_utils.stop_torchserve() os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR run_log_location_var(no_config_snapshots=False) requests.post('http://127.0.0.1:8081/models?url=mnist.mar') # We stop torchserve again here so that we can remove the LOG_LOCATION setting from environment variable test_utils.stop_torchserve() print('Waiting to stop') time.sleep(15) del os.environ['LOG_LOCATION'] # In case of snapshot enabled, we get these three config files additionally in the custom directory assert len(glob.glob(path.join(test_utils.ROOT_DIR, 'config/*startup.cfg'))) >= 1 if platform.system() != "Windows": assert len( glob.glob(path.join(test_utils.ROOT_DIR, 'config/*shutdown.cfg'))) >= 1 assert len(glob.glob(path.join(test_utils.ROOT_DIR, 'config/*snap*.cfg'))) >= 1 for f in glob.glob(path.join(test_utils.ROOT_DIR, "*.log")): print("-------------Deleting " + f) os.remove(f) shutil.rmtree(path.join(test_utils.ROOT_DIR, 'config')) # Remove any old snapshots test_utils.delete_all_snapshots()
def test_replace_mar_file_with_dummy(): """Validates that torchserve will fail to start in the following scenario: 1) We use a snapshot file to start torchserve. The snapshot contains reference to "A" model file 2) "A" model file gets corrupted or is replaced by some dummy mar file with same name""" snapshot_created_on_management_api_invoke() # Start Torchserve using last snapshot state snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[0] test_utils.start_torchserve(snapshot_file=snapshot_cfg) response = requests.get('http://127.0.0.1:8081/models/') assert json.loads( response.content)['models'][0]['modelName'] == "densenet161" test_utils.stop_torchserve() # Now replace the registered model mar with dummy file replace_mar_file_with_dummy_mar_in_model_store( model_store="/workspace/model_store", model_mar="densenet161.mar") snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[0] test_utils.start_torchserve(snapshot_file=snapshot_cfg) try: response = requests.get('http://127.0.0.1:8081/models/') assert json.loads( response.content)['models'][0]['modelName'] == "densenet161" except: assert True, "Correct Model mar file not found" else: assert False, "Something is not right!! Successfully started Torchserve with a dummy mar file" finally: test_utils.delete_all_snapshots() test_utils.delete_model_store()
def test_log_location_and_metric_location_vars_snapshot_enabled(): """ Validates that metrics & non metrics related logs get saved in directory configured as per METRICS_LOCATION & LOG_LOCATION environment variables with snaphsot enabled. """ test_utils.stop_torchserve() test_utils.delete_all_snapshots() os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR os.environ['METRICS_LOCATION'] = test_utils.ROOT_DIR run_log_location_var(no_config_snapshots=False) run_metrics_location_var(no_config_snapshots=False) requests.post('http://127.0.0.1:8081/models?url=mnist.mar') # We stop torchserve again here so that we can remove the LOG_LOCATION & METRICS_LOCATION # setting from environment variable test_utils.stop_torchserve() del os.environ['LOG_LOCATION'] del os.environ['METRICS_LOCATION'] assert len(glob.glob(test_utils.ROOT_DIR + 'config/*startup.cfg')) >= 1 if platform.system() != "Windows": assert len( glob.glob(test_utils.ROOT_DIR + 'config/*shutdown.cfg')) >= 1 assert len(glob.glob(test_utils.ROOT_DIR + 'config/*snap*.cfg')) >= 1 for f in glob.glob(test_utils.ROOT_DIR + "*.log"): os.remove(f) shutil.rmtree(path.join(test_utils.ROOT_DIR, 'config'))
def test_metrics_location_var_snapshot_enabled_rdonly_dir(): """ Validates that we should not be able to create metrics related logs if the directory configured via 'METRICS_LOCATION' is a read only directory. """ # Torchserve cleanup is required here as we are going to set 'LOG_LOCATION' environment variable test_utils.stop_torchserve() test_utils.delete_all_snapshots() # First remove existing logs otherwise it may be a false positive case for f in glob.glob('logs/*.log'): os.remove(f) RDONLY_DIR = path.join(test_utils.ROOT_DIR, 'rdonly_dir') os.environ['METRICS_LOCATION'] = RDONLY_DIR try: run_metrics_location_var(custom_path=RDONLY_DIR, no_config_snapshots=False) requests.post('http://127.0.0.1:8081/models?url=mnist.mar') assert len(glob.glob('logs/access_log.log')) == 1 assert len(glob.glob('logs/model_log.log')) == 1 assert len(glob.glob('logs/ts_log.log')) == 1 assert len(glob.glob('logs/config/*snap*.cfg')) == 1 assert len(glob.glob(RDONLY_DIR + '/logs/model_metrics.log')) == 0 assert len(glob.glob(RDONLY_DIR + '/logs/ts_metrics.log')) == 0 finally: del os.environ['METRICS_LOCATION']
def snapshot_created_on_management_api_invoke(model_mar="densenet161.mar"): test_utils.delete_all_snapshots() test_utils.start_torchserve() requests.post( 'http://127.0.0.1:8081/models?url=https://torchserve.pytorch.org/mar_files/' + model_mar) time.sleep(10) test_utils.stop_torchserve()
def test_start_from_latest(): """ Validates if latest snapshot file is picked if we dont pass snapshot arg explicitly. """ test_utils.start_torchserve() response = requests.get('http://127.0.0.1:8081/models/') assert json.loads( response.content)['models'][0]['modelName'] == "densenet161" test_utils.stop_torchserve()
def test_snapshot_created_on_start_and_stop(): """ Validates that startup .cfg & shutdown.cfg are created upon start & stop. """ test_utils.delete_all_snapshots() test_utils.start_torchserve() test_utils.stop_torchserve() assert len(glob.glob('logs/config/*startup.cfg')) == 1 assert len(glob.glob('logs/config/*shutdown.cfg')) == 1
def test_no_config_snapshots_cli_option(): """ Validates that --no-config-snapshots works as expected. """ # Required to stop torchserve here so that all config files gets deleted test_utils.stop_torchserve() test_utils.delete_all_snapshots() test_utils.start_torchserve(no_config_snapshots=True) test_utils.stop_torchserve() assert len(glob.glob('logs/config/*.cfg')) == 0
def test_start_from_snapshot(): """ Validates if we can restore state from snapshot. """ snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[0] test_utils.start_torchserve(snapshot_file=snapshot_cfg) response = requests.get('http://127.0.0.1:8081/models/') assert json.loads( response.content)['models'][0]['modelName'] == "densenet161" test_utils.stop_torchserve()
def snapshot_created_on_management_api_invoke(model_mar="densenet161.mar"): test_utils.delete_all_snapshots() test_utils.start_torchserve() mar_path = "mar_path_{}".format(model_mar[0:-4]) if mar_path in test_utils.mar_file_table: requests.post('http://127.0.0.1:8081/models?url=' + model_mar) else: requests.post( 'http://127.0.0.1:8081/models?url=https://torchserve.pytorch.org/mar_files/' + model_mar) time.sleep(10) test_utils.stop_torchserve()
def test_start_from_non_existing_snapshot(): """ Validates that Torchserve should fail to start when we pass a non-existent snapshot as an input snapshot while starting Torchserve. """ test_utils.stop_torchserve() test_utils.start_torchserve(snapshot_file="logs/config/junk-snapshot.cfg") try: response = requests.get('http://127.0.0.1:8081/models/') except: assert True, "Failed to start Torchserve using a Non Existing Snapshot" else: assert False, "Something is not right!! Successfully started Torchserve " \ "using Non Existing Snapshot File!!"
def test_async_logging(): """Validates that we can use async_logging flag while starting Torchserve""" # Need to stop torchserve as we need to check if log files get generated with 'aysnc_logging' flag test_utils.stop_torchserve() for f in glob.glob("logs/*.log"): os.remove(f) # delete_all_snapshots() async_config_file = test_utils.ROOT_DIR + 'async-log-config.properties' with open(async_config_file, "w+") as f: f.write("async_logging=true") test_utils.start_torchserve(snapshot_file=async_config_file) assert len(glob.glob('logs/access_log.log')) == 1 assert len(glob.glob('logs/model_log.log')) == 1 assert len(glob.glob('logs/ts_log.log')) == 1
def test_async_logging_non_boolean(): '''Validates that Torchserve uses default value for async_logging flag if we assign a non boolean value to this flag''' test_utils.stop_torchserve() for f in glob.glob("logs/*.log"): os.remove(f) # delete_all_snapshots() async_config_file = test_utils.ROOT_DIR + 'async-log-config.properties' with open(async_config_file, "w+") as f: f.write("async_logging=2") test_utils.start_torchserve(snapshot_file=async_config_file) assert len(glob.glob('logs/access_log.log')) == 1 assert len(glob.glob('logs/model_log.log')) == 1 assert len(glob.glob('logs/ts_log.log')) == 1 test_utils.stop_torchserve()
def test_log_location_var_snapshot_disabled(): """ Validates that non metrics logs get saved in directory configured via LOG_LOCATION environment variable. """ # We stop torchserve here so that we can set LOG_LOCATION in environment variable and rerun torchserve test_utils.stop_torchserve() os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR run_log_location_var(no_config_snapshots=True) # We stop torchserve again here so that we can remove the LOG_LOCATION setting from environment variable test_utils.stop_torchserve() del os.environ['LOG_LOCATION'] for f in glob.glob(test_utils.ROOT_DIR + "*.log"): os.remove(f) # Remove any old snapshots test_utils.delete_all_snapshots()
def test_restart_torchserve_with_one_of_model_mar_removed(): """Validates that torchserve will fail to start in the following scenario: 1) We use a snapshot file to start torchserve. The snapshot contains reference to few model files 2) One of these model mar files are accidentally deleted from the model store""" # Register multiple models # 1st model test_utils.delete_model_store() test_utils.start_torchserve() requests.post( 'http://127.0.0.1:8081/models?url=https://torchserve.pytorch.org/mar_files/densenet161.mar' ) time.sleep(15) # 2nd model requests.post( 'http://127.0.0.1:8081/models?url=https://torchserve.pytorch.org/mar_files/mnist.mar' ) time.sleep(15) test_utils.stop_torchserve() # Start Torchserve test_utils.start_torchserve() response = requests.get('http://127.0.0.1:8081/models/') num_of_regd_models = len(json.loads(response.content)['models']) test_utils.stop_torchserve() # Now remove the registered model mar file (delete_mar_ fn) test_utils.delete_mar_file_from_model_store( model_store="/workspace/model_store", model_mar="densenet") # Start Torchserve with existing snapshot file containing reference to one of the model mar file # which is now missing from the model store snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[1] test_utils.start_torchserve(snapshot_file=snapshot_cfg) try: response = requests.get('http://127.0.0.1:8081/models/') except: assert True, "Failed to start Torchserve as one of reqd model mar file is missing" else: assert False, "Something is not right!! Started Torchserve successfully with a " \ "reqd model mar file missing from the model store!!" finally: test_utils.torchserve_cleanup()
def test_metrics_location_var_snapshot_enabled(): """ Validates that metrics related logs get saved in directory configured via METRICS_LOCATION environment variable. """ # We stop torchserve here so that we can set METRICS_LOCATION in environment variable and # restart torchserve test_utils.stop_torchserve() os.environ['METRICS_LOCATION'] = test_utils.ROOT_DIR run_metrics_location_var(no_config_snapshots=False) requests.post('http://127.0.0.1:8081/models?url=densenet161.mar') # We stop torchserve again here so that we can remove the METRICS_LOCATION setting # from environment variable test_utils.stop_torchserve() del os.environ['METRICS_LOCATION'] # In case of snapshot enabled, we get these three config files additionally in the custom directory assert len(glob.glob('logs/config/*startup.cfg')) >= 1 assert len(glob.glob('logs/config/*shutdown.cfg')) >= 1 assert len(glob.glob('logs/config/*snap*.cfg')) >= 1 for f in glob.glob(test_utils.ROOT_DIR + "*.log"): os.remove(f)
def test_log_location_var_snapshot_disabled_custom_path_read_only(): """ Validates that we should not be able to create non metrics related logs if the directory configured via 'LOG_LOCATION' is a read only directory. """ # Torchserve cleanup is required here as we are going to set 'LOG_LOCATION' environment variable test_utils.stop_torchserve() test_utils.delete_all_snapshots() # First remove existing logs otherwise it may be a false positive case for f in glob.glob('logs/*.log'): os.remove(f) RDONLY_DIR = path.join(test_utils.ROOT_DIR, 'rdonly_dir') os.environ['LOG_LOCATION'] = RDONLY_DIR try: run_log_location_var(custom_path=RDONLY_DIR, no_config_snapshots=True) assert len(glob.glob(RDONLY_DIR + '/logs/access_log.log')) == 0 assert len(glob.glob(RDONLY_DIR + '/logs/model_log.log')) == 0 assert len(glob.glob(RDONLY_DIR + '/logs/ts_log.log')) == 0 assert len(glob.glob('logs/model_metrics.log')) == 1 assert len(glob.glob('logs/ts_metrics.log')) == 1 finally: del os.environ['LOG_LOCATION']
def test_log_location_var_snapshot_enabled(): """ Validates that non metrics logs get saved in directory configured via LOG_LOCATION environment variable. """ # We stop torchserve here so that we can set LOG_LOCATION in environment variable and rerun torchserve test_utils.stop_torchserve() os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR run_log_location_var(no_config_snapshots=False) requests.post('http://127.0.0.1:8081/models?url=densenet161.mar') # We stop torchserve again here so that we can remove the LOG_LOCATION setting from environment variable test_utils.stop_torchserve() del os.environ['LOG_LOCATION'] # In case of snapshot enabled, we get these three config files additionally in the custom directory assert len(glob.glob(test_utils.ROOT_DIR + 'config/*startup.cfg')) >= 1 assert len(glob.glob(test_utils.ROOT_DIR + 'config/*shutdown.cfg')) >= 1 assert len(glob.glob(test_utils.ROOT_DIR + 'config/*snap*.cfg')) >= 1 for f in glob.glob(test_utils.ROOT_DIR + "*.log"): os.remove(f) cmd = ["rm", "-rf", test_utils.ROOT_DIR + 'config'] subprocess.run(cmd) # Remove any old snapshots test_utils.delete_all_snapshots()