def test_log_location_var_snapshot_enabled():
"""
Validates that non metrics logs get saved in directory configured via LOG_LOCATION
environment variable.
"""
# We stop torchserve here so that we can set LOG_LOCATION in environment variable and rerun torchserve
test_utils.stop_torchserve()
os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR
run_log_location_var(no_config_snapshots=False)
requests.post('http://127.0.0.1:8081/models?url=mnist.mar')
# We stop torchserve again here so that we can remove the LOG_LOCATION setting from environment variable
test_utils.stop_torchserve()
print('Waiting to stop')
time.sleep(15)
del os.environ['LOG_LOCATION']
# In case of snapshot enabled, we get these three config files additionally in the custom directory
assert len(glob.glob(path.join(test_utils.ROOT_DIR,
'config/*startup.cfg'))) >= 1
if platform.system() != "Windows":
assert len(
glob.glob(path.join(test_utils.ROOT_DIR,
'config/*shutdown.cfg'))) >= 1
assert len(glob.glob(path.join(test_utils.ROOT_DIR,
'config/*snap*.cfg'))) >= 1
for f in glob.glob(path.join(test_utils.ROOT_DIR, "*.log")):
print("-------------Deleting " + f)
os.remove(f)
shutil.rmtree(path.join(test_utils.ROOT_DIR, 'config'))
# Remove any old snapshots
test_utils.delete_all_snapshots()
def test_replace_mar_file_with_dummy():
"""Validates that torchserve will fail to start in the following scenario:
1) We use a snapshot file to start torchserve. The snapshot contains reference to "A" model file
2) "A" model file gets corrupted or is replaced by some dummy mar file with same name"""
snapshot_created_on_management_api_invoke()
# Start Torchserve using last snapshot state
snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[0]
test_utils.start_torchserve(snapshot_file=snapshot_cfg)
response = requests.get('http://127.0.0.1:8081/models/')
assert json.loads(
response.content)['models'][0]['modelName'] == "densenet161"
test_utils.stop_torchserve()
# Now replace the registered model mar with dummy file
replace_mar_file_with_dummy_mar_in_model_store(
model_store="/workspace/model_store", model_mar="densenet161.mar")
snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[0]
test_utils.start_torchserve(snapshot_file=snapshot_cfg)
try:
response = requests.get('http://127.0.0.1:8081/models/')
assert json.loads(
response.content)['models'][0]['modelName'] == "densenet161"
except:
assert True, "Correct Model mar file not found"
else:
assert False, "Something is not right!! Successfully started Torchserve with a dummy mar file"
finally:
test_utils.delete_all_snapshots()
test_utils.delete_model_store()
def test_log_location_and_metric_location_vars_snapshot_enabled():
"""
Validates that metrics & non metrics related logs get saved in directory configured as per
METRICS_LOCATION & LOG_LOCATION environment variables with snaphsot enabled.
"""
test_utils.stop_torchserve()
test_utils.delete_all_snapshots()
os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR
os.environ['METRICS_LOCATION'] = test_utils.ROOT_DIR
run_log_location_var(no_config_snapshots=False)
run_metrics_location_var(no_config_snapshots=False)
requests.post('http://127.0.0.1:8081/models?url=mnist.mar')
# We stop torchserve again here so that we can remove the LOG_LOCATION & METRICS_LOCATION
# setting from environment variable
test_utils.stop_torchserve()
del os.environ['LOG_LOCATION']
del os.environ['METRICS_LOCATION']
assert len(glob.glob(test_utils.ROOT_DIR + 'config/*startup.cfg')) >= 1
if platform.system() != "Windows":
assert len(
glob.glob(test_utils.ROOT_DIR + 'config/*shutdown.cfg')) >= 1
assert len(glob.glob(test_utils.ROOT_DIR + 'config/*snap*.cfg')) >= 1
for f in glob.glob(test_utils.ROOT_DIR + "*.log"):
os.remove(f)
shutil.rmtree(path.join(test_utils.ROOT_DIR, 'config'))
def test_metrics_location_var_snapshot_enabled_rdonly_dir():
"""
Validates that we should not be able to create metrics related logs if the directory configured
via 'METRICS_LOCATION' is a read only directory.
"""
# Torchserve cleanup is required here as we are going to set 'LOG_LOCATION' environment variable
test_utils.stop_torchserve()
test_utils.delete_all_snapshots()
# First remove existing logs otherwise it may be a false positive case
for f in glob.glob('logs/*.log'):
os.remove(f)
RDONLY_DIR = path.join(test_utils.ROOT_DIR, 'rdonly_dir')
os.environ['METRICS_LOCATION'] = RDONLY_DIR
try:
run_metrics_location_var(custom_path=RDONLY_DIR,
no_config_snapshots=False)
requests.post('http://127.0.0.1:8081/models?url=mnist.mar')
assert len(glob.glob('logs/access_log.log')) == 1
assert len(glob.glob('logs/model_log.log')) == 1
assert len(glob.glob('logs/ts_log.log')) == 1
assert len(glob.glob('logs/config/*snap*.cfg')) == 1
assert len(glob.glob(RDONLY_DIR + '/logs/model_metrics.log')) == 0
assert len(glob.glob(RDONLY_DIR + '/logs/ts_metrics.log')) == 0
finally:
del os.environ['METRICS_LOCATION']
def snapshot_created_on_management_api_invoke(model_mar="densenet161.mar"):
test_utils.delete_all_snapshots()
test_utils.start_torchserve()
requests.post(
'http://127.0.0.1:8081/models?url=https://torchserve.pytorch.org/mar_files/'
+ model_mar)
time.sleep(10)
test_utils.stop_torchserve()
def test_start_from_latest():
"""
Validates if latest snapshot file is picked if we dont pass snapshot arg explicitly.
"""
test_utils.start_torchserve()
response = requests.get('http://127.0.0.1:8081/models/')
assert json.loads(
response.content)['models'][0]['modelName'] == "densenet161"
test_utils.stop_torchserve()
def test_snapshot_created_on_start_and_stop():
"""
Validates that startup .cfg & shutdown.cfg are created upon start & stop.
"""
test_utils.delete_all_snapshots()
test_utils.start_torchserve()
test_utils.stop_torchserve()
assert len(glob.glob('logs/config/*startup.cfg')) == 1
assert len(glob.glob('logs/config/*shutdown.cfg')) == 1
def test_no_config_snapshots_cli_option():
"""
Validates that --no-config-snapshots works as expected.
"""
# Required to stop torchserve here so that all config files gets deleted
test_utils.stop_torchserve()
test_utils.delete_all_snapshots()
test_utils.start_torchserve(no_config_snapshots=True)
test_utils.stop_torchserve()
assert len(glob.glob('logs/config/*.cfg')) == 0
def test_start_from_snapshot():
"""
Validates if we can restore state from snapshot.
"""
snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[0]
test_utils.start_torchserve(snapshot_file=snapshot_cfg)
response = requests.get('http://127.0.0.1:8081/models/')
assert json.loads(
response.content)['models'][0]['modelName'] == "densenet161"
test_utils.stop_torchserve()
def snapshot_created_on_management_api_invoke(model_mar="densenet161.mar"):
test_utils.delete_all_snapshots()
test_utils.start_torchserve()
mar_path = "mar_path_{}".format(model_mar[0:-4])
if mar_path in test_utils.mar_file_table:
requests.post('http://127.0.0.1:8081/models?url=' + model_mar)
else:
requests.post(
'http://127.0.0.1:8081/models?url=https://torchserve.pytorch.org/mar_files/'
+ model_mar)
time.sleep(10)
test_utils.stop_torchserve()
def test_start_from_non_existing_snapshot():
"""
Validates that Torchserve should fail to start when we pass a non-existent snapshot
as an input snapshot while starting Torchserve.
"""
test_utils.stop_torchserve()
test_utils.start_torchserve(snapshot_file="logs/config/junk-snapshot.cfg")
try:
response = requests.get('http://127.0.0.1:8081/models/')
except:
assert True, "Failed to start Torchserve using a Non Existing Snapshot"
else:
assert False, "Something is not right!! Successfully started Torchserve " \
"using Non Existing Snapshot File!!"
def test_async_logging():
"""Validates that we can use async_logging flag while starting Torchserve"""
# Need to stop torchserve as we need to check if log files get generated with 'aysnc_logging' flag
test_utils.stop_torchserve()
for f in glob.glob("logs/*.log"):
os.remove(f)
# delete_all_snapshots()
async_config_file = test_utils.ROOT_DIR + 'async-log-config.properties'
with open(async_config_file, "w+") as f:
f.write("async_logging=true")
test_utils.start_torchserve(snapshot_file=async_config_file)
assert len(glob.glob('logs/access_log.log')) == 1
assert len(glob.glob('logs/model_log.log')) == 1
assert len(glob.glob('logs/ts_log.log')) == 1
def test_async_logging_non_boolean():
'''Validates that Torchserve uses default value for async_logging flag
if we assign a non boolean value to this flag'''
test_utils.stop_torchserve()
for f in glob.glob("logs/*.log"):
os.remove(f)
# delete_all_snapshots()
async_config_file = test_utils.ROOT_DIR + 'async-log-config.properties'
with open(async_config_file, "w+") as f:
f.write("async_logging=2")
test_utils.start_torchserve(snapshot_file=async_config_file)
assert len(glob.glob('logs/access_log.log')) == 1
assert len(glob.glob('logs/model_log.log')) == 1
assert len(glob.glob('logs/ts_log.log')) == 1
test_utils.stop_torchserve()
def test_log_location_var_snapshot_disabled():
"""
Validates that non metrics logs get saved in directory configured via LOG_LOCATION
environment variable.
"""
# We stop torchserve here so that we can set LOG_LOCATION in environment variable and rerun torchserve
test_utils.stop_torchserve()
os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR
run_log_location_var(no_config_snapshots=True)
# We stop torchserve again here so that we can remove the LOG_LOCATION setting from environment variable
test_utils.stop_torchserve()
del os.environ['LOG_LOCATION']
for f in glob.glob(test_utils.ROOT_DIR + "*.log"):
os.remove(f)
# Remove any old snapshots
test_utils.delete_all_snapshots()
def test_restart_torchserve_with_one_of_model_mar_removed():
"""Validates that torchserve will fail to start in the following scenario:
1) We use a snapshot file to start torchserve. The snapshot contains reference to few model files
2) One of these model mar files are accidentally deleted from the model store"""
# Register multiple models
# 1st model
test_utils.delete_model_store()
test_utils.start_torchserve()
requests.post(
'http://127.0.0.1:8081/models?url=https://torchserve.pytorch.org/mar_files/densenet161.mar'
)
time.sleep(15)
# 2nd model
requests.post(
'http://127.0.0.1:8081/models?url=https://torchserve.pytorch.org/mar_files/mnist.mar'
)
time.sleep(15)
test_utils.stop_torchserve()
# Start Torchserve
test_utils.start_torchserve()
response = requests.get('http://127.0.0.1:8081/models/')
num_of_regd_models = len(json.loads(response.content)['models'])
test_utils.stop_torchserve()
# Now remove the registered model mar file (delete_mar_ fn)
test_utils.delete_mar_file_from_model_store(
model_store="/workspace/model_store", model_mar="densenet")
# Start Torchserve with existing snapshot file containing reference to one of the model mar file
# which is now missing from the model store
snapshot_cfg = glob.glob('logs/config/*snap*.cfg')[1]
test_utils.start_torchserve(snapshot_file=snapshot_cfg)
try:
response = requests.get('http://127.0.0.1:8081/models/')
except:
assert True, "Failed to start Torchserve as one of reqd model mar file is missing"
else:
assert False, "Something is not right!! Started Torchserve successfully with a " \
"reqd model mar file missing from the model store!!"
finally:
test_utils.torchserve_cleanup()
def test_metrics_location_var_snapshot_enabled():
"""
Validates that metrics related logs get saved in directory configured via METRICS_LOCATION
environment variable.
"""
# We stop torchserve here so that we can set METRICS_LOCATION in environment variable and
# restart torchserve
test_utils.stop_torchserve()
os.environ['METRICS_LOCATION'] = test_utils.ROOT_DIR
run_metrics_location_var(no_config_snapshots=False)
requests.post('http://127.0.0.1:8081/models?url=densenet161.mar')
# We stop torchserve again here so that we can remove the METRICS_LOCATION setting
# from environment variable
test_utils.stop_torchserve()
del os.environ['METRICS_LOCATION']
# In case of snapshot enabled, we get these three config files additionally in the custom directory
assert len(glob.glob('logs/config/*startup.cfg')) >= 1
assert len(glob.glob('logs/config/*shutdown.cfg')) >= 1
assert len(glob.glob('logs/config/*snap*.cfg')) >= 1
for f in glob.glob(test_utils.ROOT_DIR + "*.log"):
os.remove(f)
def test_log_location_var_snapshot_disabled_custom_path_read_only():
"""
Validates that we should not be able to create non metrics related logs if the directory configured
via 'LOG_LOCATION' is a read only directory.
"""
# Torchserve cleanup is required here as we are going to set 'LOG_LOCATION' environment variable
test_utils.stop_torchserve()
test_utils.delete_all_snapshots()
# First remove existing logs otherwise it may be a false positive case
for f in glob.glob('logs/*.log'):
os.remove(f)
RDONLY_DIR = path.join(test_utils.ROOT_DIR, 'rdonly_dir')
os.environ['LOG_LOCATION'] = RDONLY_DIR
try:
run_log_location_var(custom_path=RDONLY_DIR, no_config_snapshots=True)
assert len(glob.glob(RDONLY_DIR + '/logs/access_log.log')) == 0
assert len(glob.glob(RDONLY_DIR + '/logs/model_log.log')) == 0
assert len(glob.glob(RDONLY_DIR + '/logs/ts_log.log')) == 0
assert len(glob.glob('logs/model_metrics.log')) == 1
assert len(glob.glob('logs/ts_metrics.log')) == 1
finally:
del os.environ['LOG_LOCATION']
def test_log_location_var_snapshot_enabled():
"""
Validates that non metrics logs get saved in directory configured via LOG_LOCATION
environment variable.
"""
# We stop torchserve here so that we can set LOG_LOCATION in environment variable and rerun torchserve
test_utils.stop_torchserve()
os.environ['LOG_LOCATION'] = test_utils.ROOT_DIR
run_log_location_var(no_config_snapshots=False)
requests.post('http://127.0.0.1:8081/models?url=densenet161.mar')
# We stop torchserve again here so that we can remove the LOG_LOCATION setting from environment variable
test_utils.stop_torchserve()
del os.environ['LOG_LOCATION']
# In case of snapshot enabled, we get these three config files additionally in the custom directory
assert len(glob.glob(test_utils.ROOT_DIR + 'config/*startup.cfg')) >= 1
assert len(glob.glob(test_utils.ROOT_DIR + 'config/*shutdown.cfg')) >= 1
assert len(glob.glob(test_utils.ROOT_DIR + 'config/*snap*.cfg')) >= 1
for f in glob.glob(test_utils.ROOT_DIR + "*.log"):
os.remove(f)
cmd = ["rm", "-rf", test_utils.ROOT_DIR + 'config']
subprocess.run(cmd)
# Remove any old snapshots
test_utils.delete_all_snapshots()