def test_save(self):
"""A json file is created in the right directory with the right name
when saving a job result."""
history = History('/history')
job = Job(
{
'args': ['somearg'],
'benchmark': 'bench',
'description': 'cool description',
'hooks': [],
'metrics': ['mysupercoolmetric'],
'name': 'job name',
}, {
'path': 'true',
'parser': 'parser',
})
now = datetime.now(timezone.utc)
expected_path = os.path.join(
'/history', 'job_name',
now.strftime('%Y-%m-%dT%H:%M:%SZ') + '.json')
# make sure file doesn't already exist
self.assertFalse(self.fs.Exists(expected_path))
history.save_job_result(job, Metrics({'mysupercoolmetric': 1}), now)
# make sure it exists now
self.assertTrue(self.fs.Exists(expected_path))
def run(self, args, jobs):
reporter = ReporterFactory.create('default')
if len(args.jobs) > 0:
for name in args.jobs:
if name not in jobs:
logger.error('No job "{}" found'.format(name))
exit(1)
jobs = {name: jobs[name] for name in args.jobs}
jobs = jobs.values()
print('Will run {} job(s)'.format(len(jobs)))
history = History(args.results)
now = datetime.now(timezone.utc)
for job in jobs:
print('Running "{}": {}'.format(job.name, job.description))
if not args.clowntown and not history.is_job_config_consistent(job):
logger.error('There was a previous run of "{}" that had a'
' different configuration, this is likely to make'
' your results confusing.'.format(job.name))
logger.error('You can proceed anyway using --clowntown')
exit(3)
metrics = job.run()
reporter.report(job, metrics)
history.save_job_result(job, metrics, now)
reporter.close()
def test_save(self):
"""A json file is created in the right directory with the right name
when saving a job result."""
history = History('/history')
job = Job({
'args': ['somearg'],
'benchmark': 'bench',
'description': 'cool description',
'hook': {
'hook': 'noop',
'options': []
},
'metrics': ['mysupercoolmetric'],
'name': 'job name',
}, {
'path': 'true',
'parser': 'parser',
})
now = datetime.now(timezone.utc)
expected_path = os.path.join(
'/history', 'job_name',
now.strftime('%Y-%m-%dT%H:%M:%SZ') + '.json')
# make sure file doesn't already exist
self.assertFalse(self.fs.Exists(expected_path))
history.save_job_result(job, Metrics({'mysupercoolmetric': 1}), now)
# make sure it exists now
self.assertTrue(self.fs.Exists(expected_path))
def run(self, args, jobs):
reporter = ReporterFactory.create('default')
if len(args.jobs) > 0:
for name in args.jobs:
if name not in jobs:
logger.error('No job "{}" found'.format(name))
exit(1)
jobs = {name: jobs[name] for name in args.jobs}
jobs = jobs.values()
print('Will run {} job(s)'.format(len(jobs)))
history = History(args.results)
now = datetime.now(timezone.utc)
for job in jobs:
print('Running "{}": {}'.format(job.name, job.description))
if not args.clowntown and not history.is_job_config_consistent(
job):
logger.error('There was a previous run of "{}" that had a'
' different configuration, this is likely to make'
' your results confusing.'.format(job.name))
logger.error('You can proceed anyway using --clowntown')
exit(3)
metrics = job.run()
anomalies = analyze(metrics, job, history)
reporter.report(job, metrics, anomalies)
history.save_job_result(job, metrics, now)
# if a correctness test failed, make it obvious
if False in metrics.correctness_tests().values():
# find which one(s) failed
for key, val in metrics.correctness_tests().items():
if not val:
logger.error('Correctness test "%s" failed', key)
reporter.close()
def run(self, args, jobs):
reporter = ReporterFactory.create('default')
if len(args.jobs) > 0:
for name in args.jobs:
if name not in jobs:
logger.error('No job "{}" found'.format(name))
exit(1)
jobs = {name: jobs[name] for name in args.jobs}
jobs = jobs.values()
print('Will run {} job(s)'.format(len(jobs)))
history = History(args.results)
now = datetime.now(timezone.utc)
for job in jobs:
print('Running "{}": {}'.format(job.name, job.description))
if not args.clowntown and not history.is_job_config_consistent(job):
logger.error('There was a previous run of "{}" that had a'
' different configuration, this is likely to make'
' your results confusing.'.format(job.name))
logger.error('You can proceed anyway using --clowntown')
exit(3)
metrics = job.run()
reporter.report(job, metrics)
history.save_job_result(job, metrics, now)
# if a correctness test failed, make it obvious
if False in metrics.correctness_tests().values():
# find which one(s) failed
for key, val in metrics.correctness_tests().items():
if not val:
logger.error('Correctness test "%s" failed', key)
reporter.close()