def run_test_suite(self, full_config): """Run benchmarks defined by full_config. Args: full_config: Config representing tests to run. """ # Left over from system that could have multiple instances for distributed # tests. Currently uses first and only instance from list. instance = cluster_local.UseLocalInstances( virtual_env_path=full_config.get('virtual_env_path')) # Folder to store suite results full_config['test_suite_start_time'] = datetime.datetime.now( ).strftime('%Y%m%dT%H%M%S') # Configs for the test suite test_suite = command_builder.build_test_config_suite( full_config, self.debug_level) for _, test_configs in enumerate(test_suite): last_config = None for _, test_config in enumerate(test_configs): last_config = test_config # Executes oom test or the normal benchmark. if test_config.get('oom_test'): low = test_config['oom_low'] high = test_config['oom_high'] next_val = high lowest_oom = high while next_val != -1: print( 'OOM testing--> low:{} high:{} next_val:{}'.format( low, high, next_val)) test_config['batch_size'] = next_val result_dir = self.run_benchmark(test_config, instance) oom = reporting.check_oom( os.path.join(result_dir, 'worker_0_stdout.log')) if oom and next_val < lowest_oom: lowest_oom = next_val low, high, next_val = reporting.oom_batch_size_search( low, high, next_val, oom) print('Lowest OOM Value:{}'.format(lowest_oom)) else: result_dir = self.run_benchmark(test_config, instance) suite_dir_name = '{}_{}'.format( last_config['test_suite_start_time'], last_config['test_id']) reporting.process_folder(os.path.join(self.workspace, 'results', suite_dir_name), report_config=self.auto_test_config)
def run_test_suite(self, test_config): """Run benchmarks defined by full_config. Args: test_config: Config representing tests to run. """ # Folder to store suite results test_config['test_suite_start_time'] = datetime.datetime.now( ).strftime('%Y%m%dT%H%M%S') instance = cluster_local.UseLocalInstances() for i in range(test_config['repeat']): self.run_benchmark(test_config, instance, copy=i) suite_dir_name = '{}_{}'.format(test_config['test_suite_start_time'], test_config['test_id']) reporting.process_folder(os.path.join(self.workspace, 'results', suite_dir_name), report_config=self.auto_test_config)