def test_save(self): """A json file is created in the right directory with the right name when saving a job result.""" history = History('/history') job = Job( { 'args': ['somearg'], 'benchmark': 'bench', 'description': 'cool description', 'hooks': [], 'metrics': ['mysupercoolmetric'], 'name': 'job name', }, { 'path': 'true', 'parser': 'parser', }) now = datetime.now(timezone.utc) expected_path = os.path.join( '/history', 'job_name', now.strftime('%Y-%m-%dT%H:%M:%SZ') + '.json') # make sure file doesn't already exist self.assertFalse(self.fs.Exists(expected_path)) history.save_job_result(job, Metrics({'mysupercoolmetric': 1}), now) # make sure it exists now self.assertTrue(self.fs.Exists(expected_path))
def run(self, args, jobs): reporter = ReporterFactory.create(args.reporter) if len(args.jobs) > 0: for name in args.jobs: if name not in jobs: logger.error('No job "{}" found'.format(name)) exit(1) jobs = {name: jobs[name] for name in args.jobs} jobs = jobs.values() history = History(args.results) for job in jobs: logger.info('Reporting result for "%s"', job.name) results = history.load_historical_results(job) if len(results) == 0: logger.warn('No historical results for "%s", skipping', job.name) continue latest = results[0] metrics = latest.metrics reporter.report(job, metrics) reporter.close()
def test_save(self): """A json file is created in the right directory with the right name when saving a job result.""" history = History('/history') job = Job({ 'args': ['somearg'], 'benchmark': 'bench', 'description': 'cool description', 'hook': { 'hook': 'noop', 'options': [] }, 'metrics': ['mysupercoolmetric'], 'name': 'job name', }, { 'path': 'true', 'parser': 'parser', }) now = datetime.now(timezone.utc) expected_path = os.path.join( '/history', 'job_name', now.strftime('%Y-%m-%dT%H:%M:%SZ') + '.json') # make sure file doesn't already exist self.assertFalse(self.fs.Exists(expected_path)) history.save_job_result(job, Metrics({'mysupercoolmetric': 1}), now) # make sure it exists now self.assertTrue(self.fs.Exists(expected_path))
def run(self, args, jobs): reporter = ReporterFactory.create('default') if len(args.jobs) > 0: for name in args.jobs: if name not in jobs: logger.error('No job "{}" found'.format(name)) exit(1) jobs = {name: jobs[name] for name in args.jobs} jobs = jobs.values() print('Will run {} job(s)'.format(len(jobs))) history = History(args.results) now = datetime.now(timezone.utc) for job in jobs: print('Running "{}": {}'.format(job.name, job.description)) if not args.clowntown and not history.is_job_config_consistent(job): logger.error('There was a previous run of "{}" that had a' ' different configuration, this is likely to make' ' your results confusing.'.format(job.name)) logger.error('You can proceed anyway using --clowntown') exit(3) metrics = job.run() reporter.report(job, metrics) history.save_job_result(job, metrics, now) reporter.close()
def test_consistency(self): """History is able to detect when a job configuration has changed.""" history = History('/history') consistent_job = Job({ 'args': ['somearg'], 'benchmark': 'bench', 'description': 'cool description', 'hook': { 'hook': 'noop', 'options': [] }, 'metrics': ['mysupercoolmetric'], 'name': 'job name', }, { 'path': 'true', 'parser': 'parser', }) self.fs.CreateFile('/history/job_name/1.json', contents=''' { "config": { "args": ["somearg"], "benchmark": "bench", "description": "cool description", "hook": { "hook": "noop", "options": [] }, "metrics": ["mysupercoolmetric"], "name": "job name", "path": "true", "parser": "parser" }, "job": "job name", "metrics": { "mysupercoolmetric": 1 }, "timestamp": "2017-06-26T21:41:04" } ''') self.assertTrue(history.is_job_config_consistent(consistent_job)) inconsistent_job = consistent_job inconsistent_job.config['args'] = ['some different arg'] self.assertFalse(history.is_job_config_consistent(inconsistent_job))
def test_consistency(self): """History is able to detect when a job configuration has changed.""" history = History('/history') consistent_job = Job( { 'args': ['somearg'], 'benchmark': 'bench', 'description': 'cool description', 'hooks': [], 'metrics': ['mysupercoolmetric'], 'name': 'job name', }, { 'path': 'true', 'parser': 'parser', }) self.fs.CreateFile('/history/job_name/1.json', contents=''' { "config": { "args": ["somearg"], "benchmark": "bench", "description": "cool description", "hooks": [], "metrics": ["mysupercoolmetric"], "name": "job name", "path": "true", "parser": "parser" }, "job": "job name", "metrics": { "mysupercoolmetric": 1 }, "timestamp": "2017-06-26T21:41:04" } ''') self.assertTrue(history.is_job_config_consistent(consistent_job)) inconsistent_job = consistent_job inconsistent_job.config['args'] = ['some different arg'] self.assertFalse(history.is_job_config_consistent(inconsistent_job))
def run(self, args, jobs): reporter = ReporterFactory.create('default') if len(args.jobs) > 0: for name in args.jobs: if name not in jobs: logger.error('No job "{}" found'.format(name)) exit(1) jobs = {name: jobs[name] for name in args.jobs} jobs = jobs.values() print('Will run {} job(s)'.format(len(jobs))) history = History(args.results) now = datetime.now(timezone.utc) for job in jobs: print('Running "{}": {}'.format(job.name, job.description)) if not args.clowntown and not history.is_job_config_consistent( job): logger.error('There was a previous run of "{}" that had a' ' different configuration, this is likely to make' ' your results confusing.'.format(job.name)) logger.error('You can proceed anyway using --clowntown') exit(3) metrics = job.run() anomalies = analyze(metrics, job, history) reporter.report(job, metrics, anomalies) history.save_job_result(job, metrics, now) # if a correctness test failed, make it obvious if False in metrics.correctness_tests().values(): # find which one(s) failed for key, val in metrics.correctness_tests().items(): if not val: logger.error('Correctness test "%s" failed', key) reporter.close()
def run(self, args, jobs): reporter = ReporterFactory.create('default') if len(args.jobs) > 0: for name in args.jobs: if name not in jobs: logger.error('No job "{}" found'.format(name)) exit(1) jobs = {name: jobs[name] for name in args.jobs} jobs = jobs.values() print('Will run {} job(s)'.format(len(jobs))) history = History(args.results) now = datetime.now(timezone.utc) for job in jobs: print('Running "{}": {}'.format(job.name, job.description)) if not args.clowntown and not history.is_job_config_consistent(job): logger.error('There was a previous run of "{}" that had a' ' different configuration, this is likely to make' ' your results confusing.'.format(job.name)) logger.error('You can proceed anyway using --clowntown') exit(3) metrics = job.run() reporter.report(job, metrics) history.save_job_result(job, metrics, now) # if a correctness test failed, make it obvious if False in metrics.correctness_tests().values(): # find which one(s) failed for key, val in metrics.correctness_tests().items(): if not val: logger.error('Correctness test "%s" failed', key) reporter.close()
def test_invalid_format(self): """History complains when a historical record is in an invalid format (missing key(s)).""" history = History('/history') job = Job({ 'args': ['somearg'], 'benchmark': 'bench', 'description': 'cool description', 'metrics': ['mysupercoolmetric'], 'name': 'broken job', }, { 'path': 'true', 'parser': 'parser', }) self.fs.CreateFile('/history/broken_job/1.json', contents=''' { "config": { "args": ["somearg"], "benchmark": "bench", "description": "cool description", "hook": { "hook": "noop", "options": [] }, "metrics": ["mysupercoolmetric"], "name": "job name", "path": "true", "parser": "parser" }, "job": "broken_job", "metrics": { "mysupercoolmetric": 1 } }''') with self.assertRaises(KeyError): history.load_historical_results(job)
def test_invalid_format(self): """History complains when a historical record is in an invalid format (missing key(s)).""" history = History('/history') job = Job( { 'args': ['somearg'], 'benchmark': 'bench', 'description': 'cool description', 'metrics': ['mysupercoolmetric'], 'name': 'broken job', }, { 'path': 'true', 'parser': 'parser', }) self.fs.CreateFile('/history/broken_job/1.json', contents=''' { "config": { "args": ["somearg"], "benchmark": "bench", "description": "cool description", "hooks": [], "metrics": ["mysupercoolmetric"], "name": "job name", "path": "true", "parser": "parser" }, "job": "broken_job", "metrics": { "mysupercoolmetric": 1 } }''') with self.assertRaises(KeyError): history.load_historical_results(job)