def test_save(self):
"""A json file is created in the right directory with the right name
when saving a job result."""
history = History('/history')
job = Job(
{
'args': ['somearg'],
'benchmark': 'bench',
'description': 'cool description',
'hooks': [],
'metrics': ['mysupercoolmetric'],
'name': 'job name',
}, {
'path': 'true',
'parser': 'parser',
})
now = datetime.now(timezone.utc)
expected_path = os.path.join(
'/history', 'job_name',
now.strftime('%Y-%m-%dT%H:%M:%SZ') + '.json')
# make sure file doesn't already exist
self.assertFalse(self.fs.Exists(expected_path))
history.save_job_result(job, Metrics({'mysupercoolmetric': 1}), now)
# make sure it exists now
self.assertTrue(self.fs.Exists(expected_path))
def run(self, args, jobs):
reporter = ReporterFactory.create(args.reporter)
if len(args.jobs) > 0:
for name in args.jobs:
if name not in jobs:
logger.error('No job "{}" found'.format(name))
exit(1)
jobs = {name: jobs[name] for name in args.jobs}
jobs = jobs.values()
history = History(args.results)
for job in jobs:
logger.info('Reporting result for "%s"', job.name)
results = history.load_historical_results(job)
if len(results) == 0:
logger.warn('No historical results for "%s", skipping',
job.name)
continue
latest = results[0]
metrics = latest.metrics
reporter.report(job, metrics)
reporter.close()
def test_save(self):
"""A json file is created in the right directory with the right name
when saving a job result."""
history = History('/history')
job = Job({
'args': ['somearg'],
'benchmark': 'bench',
'description': 'cool description',
'hook': {
'hook': 'noop',
'options': []
},
'metrics': ['mysupercoolmetric'],
'name': 'job name',
}, {
'path': 'true',
'parser': 'parser',
})
now = datetime.now(timezone.utc)
expected_path = os.path.join(
'/history', 'job_name',
now.strftime('%Y-%m-%dT%H:%M:%SZ') + '.json')
# make sure file doesn't already exist
self.assertFalse(self.fs.Exists(expected_path))
history.save_job_result(job, Metrics({'mysupercoolmetric': 1}), now)
# make sure it exists now
self.assertTrue(self.fs.Exists(expected_path))
def run(self, args, jobs):
reporter = ReporterFactory.create('default')
if len(args.jobs) > 0:
for name in args.jobs:
if name not in jobs:
logger.error('No job "{}" found'.format(name))
exit(1)
jobs = {name: jobs[name] for name in args.jobs}
jobs = jobs.values()
print('Will run {} job(s)'.format(len(jobs)))
history = History(args.results)
now = datetime.now(timezone.utc)
for job in jobs:
print('Running "{}": {}'.format(job.name, job.description))
if not args.clowntown and not history.is_job_config_consistent(job):
logger.error('There was a previous run of "{}" that had a'
' different configuration, this is likely to make'
' your results confusing.'.format(job.name))
logger.error('You can proceed anyway using --clowntown')
exit(3)
metrics = job.run()
reporter.report(job, metrics)
history.save_job_result(job, metrics, now)
reporter.close()
def run(self, args, jobs):
reporter = ReporterFactory.create(args.reporter)
if len(args.jobs) > 0:
for name in args.jobs:
if name not in jobs:
logger.error('No job "{}" found'.format(name))
exit(1)
jobs = {name: jobs[name] for name in args.jobs}
jobs = jobs.values()
history = History(args.results)
for job in jobs:
logger.info('Reporting result for "%s"', job.name)
results = history.load_historical_results(job)
if len(results) == 0:
logger.warn('No historical results for "%s", skipping',
job.name)
continue
latest = results[0]
metrics = latest.metrics
reporter.report(job, metrics)
reporter.close()
def test_consistency(self):
"""History is able to detect when a job configuration has changed."""
history = History('/history')
consistent_job = Job({
'args': ['somearg'],
'benchmark': 'bench',
'description': 'cool description',
'hook': {
'hook': 'noop',
'options': []
},
'metrics': ['mysupercoolmetric'],
'name': 'job name',
}, {
'path': 'true',
'parser': 'parser',
})
self.fs.CreateFile('/history/job_name/1.json',
contents='''
{
"config": {
"args": ["somearg"],
"benchmark": "bench",
"description": "cool description",
"hook": {
"hook": "noop",
"options": []
},
"metrics": ["mysupercoolmetric"],
"name": "job name",
"path": "true",
"parser": "parser"
},
"job": "job name",
"metrics": {
"mysupercoolmetric": 1
},
"timestamp": "2017-06-26T21:41:04"
}
''')
self.assertTrue(history.is_job_config_consistent(consistent_job))
inconsistent_job = consistent_job
inconsistent_job.config['args'] = ['some different arg']
self.assertFalse(history.is_job_config_consistent(inconsistent_job))
def test_consistency(self):
"""History is able to detect when a job configuration has changed."""
history = History('/history')
consistent_job = Job(
{
'args': ['somearg'],
'benchmark': 'bench',
'description': 'cool description',
'hooks': [],
'metrics': ['mysupercoolmetric'],
'name': 'job name',
}, {
'path': 'true',
'parser': 'parser',
})
self.fs.CreateFile('/history/job_name/1.json',
contents='''
{
"config": {
"args": ["somearg"],
"benchmark": "bench",
"description": "cool description",
"hooks": [],
"metrics": ["mysupercoolmetric"],
"name": "job name",
"path": "true",
"parser": "parser"
},
"job": "job name",
"metrics": {
"mysupercoolmetric": 1
},
"timestamp": "2017-06-26T21:41:04"
}
''')
self.assertTrue(history.is_job_config_consistent(consistent_job))
inconsistent_job = consistent_job
inconsistent_job.config['args'] = ['some different arg']
self.assertFalse(history.is_job_config_consistent(inconsistent_job))
def run(self, args, jobs):
reporter = ReporterFactory.create('default')
if len(args.jobs) > 0:
for name in args.jobs:
if name not in jobs:
logger.error('No job "{}" found'.format(name))
exit(1)
jobs = {name: jobs[name] for name in args.jobs}
jobs = jobs.values()
print('Will run {} job(s)'.format(len(jobs)))
history = History(args.results)
now = datetime.now(timezone.utc)
for job in jobs:
print('Running "{}": {}'.format(job.name, job.description))
if not args.clowntown and not history.is_job_config_consistent(
job):
logger.error('There was a previous run of "{}" that had a'
' different configuration, this is likely to make'
' your results confusing.'.format(job.name))
logger.error('You can proceed anyway using --clowntown')
exit(3)
metrics = job.run()
anomalies = analyze(metrics, job, history)
reporter.report(job, metrics, anomalies)
history.save_job_result(job, metrics, now)
# if a correctness test failed, make it obvious
if False in metrics.correctness_tests().values():
# find which one(s) failed
for key, val in metrics.correctness_tests().items():
if not val:
logger.error('Correctness test "%s" failed', key)
reporter.close()
def run(self, args, jobs):
reporter = ReporterFactory.create('default')
if len(args.jobs) > 0:
for name in args.jobs:
if name not in jobs:
logger.error('No job "{}" found'.format(name))
exit(1)
jobs = {name: jobs[name] for name in args.jobs}
jobs = jobs.values()
print('Will run {} job(s)'.format(len(jobs)))
history = History(args.results)
now = datetime.now(timezone.utc)
for job in jobs:
print('Running "{}": {}'.format(job.name, job.description))
if not args.clowntown and not history.is_job_config_consistent(job):
logger.error('There was a previous run of "{}" that had a'
' different configuration, this is likely to make'
' your results confusing.'.format(job.name))
logger.error('You can proceed anyway using --clowntown')
exit(3)
metrics = job.run()
reporter.report(job, metrics)
history.save_job_result(job, metrics, now)
# if a correctness test failed, make it obvious
if False in metrics.correctness_tests().values():
# find which one(s) failed
for key, val in metrics.correctness_tests().items():
if not val:
logger.error('Correctness test "%s" failed', key)
reporter.close()
def test_invalid_format(self):
"""History complains when a historical record is in an invalid format
(missing key(s))."""
history = History('/history')
job = Job({
'args': ['somearg'],
'benchmark': 'bench',
'description': 'cool description',
'metrics': ['mysupercoolmetric'],
'name': 'broken job',
}, {
'path': 'true',
'parser': 'parser',
})
self.fs.CreateFile('/history/broken_job/1.json',
contents='''
{
"config": {
"args": ["somearg"],
"benchmark": "bench",
"description": "cool description",
"hook": {
"hook": "noop",
"options": []
},
"metrics": ["mysupercoolmetric"],
"name": "job name",
"path": "true",
"parser": "parser"
},
"job": "broken_job",
"metrics": {
"mysupercoolmetric": 1
}
}''')
with self.assertRaises(KeyError):
history.load_historical_results(job)
def test_invalid_format(self):
"""History complains when a historical record is in an invalid format
(missing key(s))."""
history = History('/history')
job = Job(
{
'args': ['somearg'],
'benchmark': 'bench',
'description': 'cool description',
'metrics': ['mysupercoolmetric'],
'name': 'broken job',
}, {
'path': 'true',
'parser': 'parser',
})
self.fs.CreateFile('/history/broken_job/1.json',
contents='''
{
"config": {
"args": ["somearg"],
"benchmark": "bench",
"description": "cool description",
"hooks": [],
"metrics": ["mysupercoolmetric"],
"name": "job name",
"path": "true",
"parser": "parser"
},
"job": "broken_job",
"metrics": {
"mysupercoolmetric": 1
}
}''')
with self.assertRaises(KeyError):
history.load_historical_results(job)