def test_all(): cwd = Path(getcwd()) all_omts = [p.as_posix() for p in cwd.glob('**/*.omt')] th = TallyHolder() if environ.get('TRAVIS'): if not environ.get('OMV_ENGINE'): tallies = [parse_omt(t) for t in all_omts] else: engine = environ.get('OMV_ENGINE').lower() tallies = [parse_omt(t) for t in all_omts if load_yaml(t)['engine'].lower() == engine] else: tallies = [parse_omt(t) for t in all_omts] for t in tallies: th.add(t) results = [t.all_passed() for t in tallies] inform('') inform("%i test(s) run" % len(tallies), overline='-', underline='-', center=True) inform('') if all(results): inform("All tests passing!", underline='=', center=True) else: failed = [trim_path(t.omt) for t in tallies if not t.all_passed()] inform("Some test(s) failed: ", failed, underline='=') if is_verbose(): print('\n'+th.summary()+'\n') assert all(results)
def parse_omt(omt_path): inform('') inform("Running tests defined in ", trim_path(omt_path), underline='=', center=False) mepomt = OMVTestParser(omt_path) if not OMVEngines.has_key(mepomt.engine): inform("Error! Unrecognised engine: %s"%mepomt.engine) exit(1) engine = OMVEngines[mepomt.engine](mepomt.modelpath) experiments = [exp for exp in mepomt.generate_exps(engine)] tally = Tallyman(mepomt) try: engine.run() for exp in experiments: inform('Running checks for experiment: ', exp.name, indent=1) inform('') results = exp.check_all() inform('{:<30}{:^20}'.format('Observable', 'Test Passed'), underline='-', indent=3) for rn, rv in results.iteritems(): inform(u'{:<30}{:^20}'.format(rn, check(rv)), indent=3) tally.add_experiment(exp, results) except (EngineInstallationError, EngineExecutionError): # TODO: serialize exception info inform('ERROR running engine ', engine.name, indent=1, underline='-', overline='-') return tally
def add(self, tally): if not tally.engine in self.all_engines: self.all_engines.append(tally.engine) mp = trim_path(tally.modelpath) if not self.tallies.has_key(mp): self.tallies[mp] = {} mptallies = self.tallies[mp] if not mptallies.has_key(tally.engine): mptallies[tally.engine] = [] mptallies[tally.engine].append(tally)