def test_job_suite_job_fail(self):
"""JobSuite with a failed job raises an error"""
self.mock_benchmark['path'] = 'abinaryhasnopath'
fail_job = Job(self.job_config, self.mock_benchmark)
suite = JobSuite({'name': 'suite', 'description': 'test'}, [fail_job])
with self.assertRaises(OSError):
suite.run()
def test_job_suite_job_fail(self):
"""JobSuite with a failed job raises an error"""
self.mock_benchmark['path'] = 'abinaryhasnopath'
fail_job = Job(self.job_config, self.mock_benchmark)
suite = JobSuite({'name': 'suite', 'description': 'test'}, [fail_job])
with self.assertRaises(OSError):
suite.run()
def test_job_suite(self):
"""JobSuite runs all jobs in the suite"""
jobs = [MagicMock() for i in range(10)]
for i, job in enumerate(jobs):
job.name = str(i)
job.safe_name = str(i)
job.metrics_config.names = ['a']
job.run.return_value = {'a': i}
suite = JobSuite({'name': 'suite', 'description': 'test'}, jobs)
suite.run()
metrics = suite.run()
expected = {str(i): {'a': i} for i in range(10)}
self.assertDictEqual(expected, metrics)
def test_job_suite(self):
"""JobSuite runs all jobs in the suite"""
jobs = [MagicMock() for i in range(10)]
for i, job in enumerate(jobs):
job.name = str(i)
job.safe_name = str(i)
job.metrics_config.names = ['a']
job.run.return_value = Metrics({'a': i})
suite = JobSuite({'name': 'suite', 'description': 'test'}, jobs)
suite.run()
metrics = suite.run()
expected = {str(i)+'.a': i for i in range(10)}
self.assertDictEqual(expected, metrics.metrics())
def main(args=sys.argv[1:]):
# register reporter plugins before setting up the parser
ReporterFactory.register('stdout', StdoutReporter)
parser = setup_parser()
args = parser.parse_args(args)
# warn is 30, should default to 30 when verbose=0
# each level below warning is 10 less than the previous
log_level = args.verbose*(-10) + 30
logging.basicConfig(format='%(levelname)s:%(name)s: %(message)s',
level=log_level)
logger = logging.getLogger(__name__)
logger.info('Loading benchmarks from "{}"'.format(args.benchmarks))
with open(args.benchmarks) as tests_file:
benchmarks = yaml.load(tests_file)
logger.info('Loading jobs from "{}"'.format(args.jobs_file))
with open(args.jobs_file) as jobs_file:
job_configs = yaml.load(jobs_file)
# on the first pass, construct all normal jobs, then make job suites
jobs = [Job(j, benchmarks[j['benchmark']]) for j in job_configs
if 'tests' not in j]
jobs = {j.name: j for j in jobs}
# after all the regulars jobs are created, create the job suites
for config in job_configs:
if 'tests' in config:
suite_jobs = []
for name in config['tests']:
try:
suite_jobs.append(jobs[name])
except KeyError:
logger.error('Couldn\'t find job "%s" in suite "%s"',
name, config['name'])
exit(1)
suite = JobSuite(config, suite_jobs)
jobs[suite.name] = suite
logger.info('Loaded {} benchmarks and {} jobs'
.format(len(benchmarks), len(jobs)))
args.command.run(args, jobs)