def run_test_suite(self, full_config):
"""Run benchmarks defined by full_config.
Args:
full_config: Config representing tests to run.
"""
# Left over from system that could have multiple instances for distributed
# tests. Currently uses first and only instance from list.
instance = cluster_local.UseLocalInstances(
virtual_env_path=full_config.get('virtual_env_path'))
# Folder to store suite results
full_config['test_suite_start_time'] = datetime.datetime.now(
).strftime('%Y%m%dT%H%M%S')
# Configs for the test suite
test_suite = command_builder.build_test_config_suite(
full_config, self.debug_level)
for _, test_configs in enumerate(test_suite):
last_config = None
for _, test_config in enumerate(test_configs):
last_config = test_config
# Executes oom test or the normal benchmark.
if test_config.get('oom_test'):
low = test_config['oom_low']
high = test_config['oom_high']
next_val = high
lowest_oom = high
while next_val != -1:
print(
'OOM testing--> low:{} high:{} next_val:{}'.format(
low, high, next_val))
test_config['batch_size'] = next_val
result_dir = self.run_benchmark(test_config, instance)
oom = reporting.check_oom(
os.path.join(result_dir, 'worker_0_stdout.log'))
if oom and next_val < lowest_oom:
lowest_oom = next_val
low, high, next_val = reporting.oom_batch_size_search(
low, high, next_val, oom)
print('Lowest OOM Value:{}'.format(lowest_oom))
else:
result_dir = self.run_benchmark(test_config, instance)
suite_dir_name = '{}_{}'.format(
last_config['test_suite_start_time'], last_config['test_id'])
reporting.process_folder(os.path.join(self.workspace, 'results',
suite_dir_name),
report_config=self.auto_test_config)
def run_test_suite(self, test_config):
"""Run benchmarks defined by full_config.
Args:
test_config: Config representing tests to run.
"""
# Folder to store suite results
test_config['test_suite_start_time'] = datetime.datetime.now(
).strftime('%Y%m%dT%H%M%S')
instance = cluster_local.UseLocalInstances()
for i in range(test_config['repeat']):
self.run_benchmark(test_config, instance, copy=i)
suite_dir_name = '{}_{}'.format(test_config['test_suite_start_time'],
test_config['test_id'])
reporting.process_folder(os.path.join(self.workspace, 'results',
suite_dir_name),
report_config=self.auto_test_config)