def run_metrics_location_var(custom_path=test_utils.ROOT_DIR, no_config_snapshots=False):
test_utils.delete_all_snapshots()
test_utils.start_torchserve(no_config_snapshots=no_config_snapshots)
if os.access(custom_path, os.W_OK):
assert len(glob.glob(custom_path + '/ts_metrics.log')) == 1
assert len(glob.glob(custom_path + '/model_metrics.log')) == 1
def test_replace_mar_file_with_dummy():
"""Validates that torchserve will fail to start in the following scenario:
1) We use a snapshot file to start torchserve. The snapshot contains reference to "A" model file
2) "A" model file gets corrupted or is replaced by some dummy mar file with same name"""
snapshot_created_on_management_api_invoke()
# Start Torchserve using last snapshot state
snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[0]
test_utils.start_torchserve(snapshot_file=snapshot_cfg)
response = requests.get('http://127.0.0.1:8081/models/')
assert json.loads(
response.content)['models'][0]['modelName'] == "densenet161"
test_utils.stop_torchserve()
# Now replace the registered model mar with dummy file
replace_mar_file_with_dummy_mar_in_model_store(
model_store="/workspace/model_store", model_mar="densenet161.mar")
snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[0]
test_utils.start_torchserve(snapshot_file=snapshot_cfg)
try:
response = requests.get('http://127.0.0.1:8081/models/')
assert json.loads(
response.content)['models'][0]['modelName'] == "densenet161"
except:
assert True, "Correct Model mar file not found"
else:
assert False, "Something is not right!! Successfully started Torchserve with a dummy mar file"
finally:
test_utils.delete_all_snapshots()
test_utils.delete_model_store()
def test_malformed_ts_config():
"""Validates that Torchserve validates the config file parameters correctly and
ignores any unknown key-value parameters and starts successfully!!"""
cmd1 = ["cp", "" + test_utils.CODEBUILD_WD + "/benchmarks/config.properties", test_utils.ROOT_DIR]
cmd2 = ["cp", "" + test_utils.CODEBUILD_WD + "/benchmarks/config.properties",
test_utils.ROOT_DIR + "malformed-config.properties"]
subprocess.run(cmd1)
subprocess.run(cmd2)
config_file = "/workspace/config.properties"
malformed_config_file = "/workspace/malformed-config.properties"
with open(malformed_config_file, "r+") as f:
f.writelines(["non-keyvaluepair\n"])
# First validate well-formed config file
try:
conf_file = config_file
validate_config_file(conf_file)
# Next validate malformed config file
conf_file = malformed_config_file
validate_config_file(conf_file)
finally:
cmd1 = ["rm", "-rf", config_file]
cmd2 = ["rm", "-rf", malformed_config_file]
subprocess.run(cmd1)
subprocess.run(cmd2)
test_utils.delete_all_snapshots()
def test_log_location_var_snapshot_enabled():
"""
Validates that non metrics logs get saved in directory configured via LOG_LOCATION
environment variable.
"""
# We stop torchserve here so that we can set LOG_LOCATION in environment variable and rerun torchserve
test_utils.stop_torchserve()
os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR
run_log_location_var(no_config_snapshots=False)
requests.post('http://127.0.0.1:8081/models?url=mnist.mar')
# We stop torchserve again here so that we can remove the LOG_LOCATION setting from environment variable
test_utils.stop_torchserve()
print('Waiting to stop')
time.sleep(15)
del os.environ['LOG_LOCATION']
# In case of snapshot enabled, we get these three config files additionally in the custom directory
assert len(glob.glob(path.join(test_utils.ROOT_DIR,
'config/*startup.cfg'))) >= 1
if platform.system() != "Windows":
assert len(
glob.glob(path.join(test_utils.ROOT_DIR,
'config/*shutdown.cfg'))) >= 1
assert len(glob.glob(path.join(test_utils.ROOT_DIR,
'config/*snap*.cfg'))) >= 1
for f in glob.glob(path.join(test_utils.ROOT_DIR, "*.log")):
print("-------------Deleting " + f)
os.remove(f)
shutil.rmtree(path.join(test_utils.ROOT_DIR, 'config'))
# Remove any old snapshots
test_utils.delete_all_snapshots()
def test_log_location_and_metric_location_vars_snapshot_enabled():
"""
Validates that metrics & non metrics related logs get saved in directory configured as per
METRICS_LOCATION & LOG_LOCATION environment variables with snaphsot enabled.
"""
test_utils.stop_torchserve()
test_utils.delete_all_snapshots()
os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR
os.environ['METRICS_LOCATION'] = test_utils.ROOT_DIR
run_log_location_var(no_config_snapshots=False)
run_metrics_location_var(no_config_snapshots=False)
requests.post('http://127.0.0.1:8081/models?url=mnist.mar')
# We stop torchserve again here so that we can remove the LOG_LOCATION & METRICS_LOCATION
# setting from environment variable
test_utils.stop_torchserve()
del os.environ['LOG_LOCATION']
del os.environ['METRICS_LOCATION']
assert len(glob.glob(test_utils.ROOT_DIR + 'config/*startup.cfg')) >= 1
if platform.system() != "Windows":
assert len(
glob.glob(test_utils.ROOT_DIR + 'config/*shutdown.cfg')) >= 1
assert len(glob.glob(test_utils.ROOT_DIR + 'config/*snap*.cfg')) >= 1
for f in glob.glob(test_utils.ROOT_DIR + "*.log"):
os.remove(f)
shutil.rmtree(path.join(test_utils.ROOT_DIR, 'config'))
def test_metrics_location_var_snapshot_enabled_rdonly_dir():
"""
Validates that we should not be able to create metrics related logs if the directory configured
via 'METRICS_LOCATION' is a read only directory.
"""
# Torchserve cleanup is required here as we are going to set 'LOG_LOCATION' environment variable
test_utils.stop_torchserve()
test_utils.delete_all_snapshots()
# First remove existing logs otherwise it may be a false positive case
for f in glob.glob('logs/*.log'):
os.remove(f)
RDONLY_DIR = path.join(test_utils.ROOT_DIR, 'rdonly_dir')
os.environ['METRICS_LOCATION'] = RDONLY_DIR
try:
run_metrics_location_var(custom_path=RDONLY_DIR,
no_config_snapshots=False)
requests.post('http://127.0.0.1:8081/models?url=mnist.mar')
assert len(glob.glob('logs/access_log.log')) == 1
assert len(glob.glob('logs/model_log.log')) == 1
assert len(glob.glob('logs/ts_log.log')) == 1
assert len(glob.glob('logs/config/*snap*.cfg')) == 1
assert len(glob.glob(RDONLY_DIR + '/logs/model_metrics.log')) == 0
assert len(glob.glob(RDONLY_DIR + '/logs/ts_metrics.log')) == 0
finally:
del os.environ['METRICS_LOCATION']
def snapshot_created_on_management_api_invoke(model_mar="densenet161.mar"):
test_utils.delete_all_snapshots()
test_utils.start_torchserve()
requests.post(
'http://127.0.0.1:8081/models?url=https://torchserve.pytorch.org/mar_files/'
+ model_mar)
time.sleep(10)
test_utils.stop_torchserve()
def test_start_from_default():
"""
Validates that Default config is used if we dont use a config explicitly.
"""
test_utils.delete_all_snapshots()
test_utils.start_torchserve()
response = requests.get('http://127.0.0.1:8081/models/')
assert len(json.loads(response.content)['models']) == 0
def run_log_location_var(custom_path=test_utils.ROOT_DIR, no_config_snapshots=False):
test_utils.delete_all_snapshots()
test_utils.start_torchserve(no_config_snapshots=no_config_snapshots)
# This check warrants that we are not accidentally monitoring a readonly logs/snapshot directory
if os.access(custom_path, os.W_OK):
assert len(glob.glob(custom_path + '/access_log.log')) == 1
assert len(glob.glob(custom_path + '/model_log.log')) == 1
assert len(glob.glob(custom_path + '/ts_log.log')) == 1
def test_snapshot_created_on_start_and_stop():
"""
Validates that startup .cfg & shutdown.cfg are created upon start & stop.
"""
test_utils.delete_all_snapshots()
test_utils.start_torchserve()
test_utils.stop_torchserve()
assert len(glob.glob('logs/config/*startup.cfg')) == 1
assert len(glob.glob('logs/config/*shutdown.cfg')) == 1
def test_no_config_snapshots_cli_option():
"""
Validates that --no-config-snapshots works as expected.
"""
# Required to stop torchserve here so that all config files gets deleted
test_utils.stop_torchserve()
test_utils.delete_all_snapshots()
test_utils.start_torchserve(no_config_snapshots=True)
test_utils.stop_torchserve()
assert len(glob.glob('logs/config/*.cfg')) == 0
def validate_metrics_created(no_config_snapshots=False):
test_utils.delete_all_snapshots()
global NUM_STARTUP_CFG
# Reset NUM_STARTUP_CFG as we are deleting snapshots in the previous step
NUM_STARTUP_CFG = 0
test_utils.start_torchserve(no_config_snapshots=no_config_snapshots)
if not no_config_snapshots:
NUM_STARTUP_CFG += 1
assert len(glob.glob('logs/model_metrics.log')) == 1
assert len(glob.glob('logs/ts_metrics.log')) == 1
def snapshot_created_on_management_api_invoke(model_mar="densenet161.mar"):
test_utils.delete_all_snapshots()
test_utils.start_torchserve()
mar_path = "mar_path_{}".format(model_mar[0:-4])
if mar_path in test_utils.mar_file_table:
requests.post('http://127.0.0.1:8081/models?url=' + model_mar)
else:
requests.post(
'http://127.0.0.1:8081/models?url=https://torchserve.pytorch.org/mar_files/'
+ model_mar)
time.sleep(10)
test_utils.stop_torchserve()
def test_log_location_var_snapshot_disabled():
"""
Validates that non metrics logs get saved in directory configured via LOG_LOCATION
environment variable.
"""
# We stop torchserve here so that we can set LOG_LOCATION in environment variable and rerun torchserve
test_utils.stop_torchserve()
os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR
run_log_location_var(no_config_snapshots=True)
# We stop torchserve again here so that we can remove the LOG_LOCATION setting from environment variable
test_utils.stop_torchserve()
del os.environ['LOG_LOCATION']
for f in glob.glob(test_utils.ROOT_DIR + "*.log"):
os.remove(f)
# Remove any old snapshots
test_utils.delete_all_snapshots()
def test_torchserve_init_with_non_existent_model_store():
"""Validates that Torchserve fails to start if the model store directory is non existent """
test_utils.start_torchserve(model_store="/invalid_model_store",
snapshot_file=None,
no_config_snapshots=True)
try:
response = requests.get('http://127.0.0.1:8081/models/')
except:
assert True, "Failed to start Torchserve using non existent model-store directory"
else:
assert False, "Something is not right!! Successfully started Torchserve " \
"using non existent directory!!"
finally:
test_utils.delete_model_store()
test_utils.delete_all_snapshots()
def test_log_location_var_snapshot_disabled_custom_path_read_only():
"""
Validates that we should not be able to create non metrics related logs if the directory configured
via 'LOG_LOCATION' is a read only directory.
"""
# Torchserve cleanup is required here as we are going to set 'LOG_LOCATION' environment variable
test_utils.stop_torchserve()
test_utils.delete_all_snapshots()
# First remove existing logs otherwise it may be a false positive case
for f in glob.glob('logs/*.log'):
os.remove(f)
RDONLY_DIR = path.join(test_utils.ROOT_DIR, 'rdonly_dir')
os.environ['LOG_LOCATION'] = RDONLY_DIR
try:
run_log_location_var(custom_path=RDONLY_DIR, no_config_snapshots=True)
assert len(glob.glob(RDONLY_DIR + '/logs/access_log.log')) == 0
assert len(glob.glob(RDONLY_DIR + '/logs/model_log.log')) == 0
assert len(glob.glob(RDONLY_DIR + '/logs/ts_log.log')) == 0
assert len(glob.glob('logs/model_metrics.log')) == 1
assert len(glob.glob('logs/ts_metrics.log')) == 1
finally:
del os.environ['LOG_LOCATION']
def test_log_location_var_snapshot_enabled():
"""
Validates that non metrics logs get saved in directory configured via LOG_LOCATION
environment variable.
"""
# We stop torchserve here so that we can set LOG_LOCATION in environment variable and rerun torchserve
test_utils.stop_torchserve()
os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR
run_log_location_var(no_config_snapshots=False)
requests.post('http://127.0.0.1:8081/models?url=densenet161.mar')
# We stop torchserve again here so that we can remove the LOG_LOCATION setting from environment variable
test_utils.stop_torchserve()
del os.environ['LOG_LOCATION']
# In case of snapshot enabled, we get these three config files additionally in the custom directory
assert len(glob.glob(test_utils.ROOT_DIR + 'config/*startup.cfg')) >= 1
assert len(glob.glob(test_utils.ROOT_DIR + 'config/*shutdown.cfg')) >= 1
assert len(glob.glob(test_utils.ROOT_DIR + 'config/*snap*.cfg')) >= 1
for f in glob.glob(test_utils.ROOT_DIR + "*.log"):
os.remove(f)
cmd = ["rm", "-rf", test_utils.ROOT_DIR + 'config']
subprocess.run(cmd)
# Remove any old snapshots
test_utils.delete_all_snapshots()
def test_restart_torchserve_with_last_snapshot_with_model_mar_removed():
"""Validates that torchserve will fail to start in the following scenario:
1) We use a snapshot file to start torchserve. The snapshot contains reference to "A" model file
2) The "A" model mar file is accidentally deleted from the model store"""
# Register model using mgmt api
snapshot_created_on_management_api_invoke()
# Now remove the registered model mar file (delete_mar_ fn)
test_utils.delete_mar_file_from_model_store(
model_store="/workspace/model_store", model_mar="densenet")
# Start Torchserve with last generated snapshot file
snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[0]
test_utils.start_torchserve(snapshot_file=snapshot_cfg)
try:
response = requests.get('http://127.0.0.1:8081/models/')
except:
assert True, "Failed to start Torchserve properly as reqd model mar file is missing!!"
else:
assert False, "Something is not right!! Successfully started Torchserve without reqd mar file"
finally:
test_utils.delete_model_store()
test_utils.delete_all_snapshots()