def test_all():
cwd = Path(getcwd())
all_omts = [p.as_posix() for p in cwd.glob('**/*.omt')]
th = TallyHolder()
if environ.get('TRAVIS'):
if not environ.get('OMV_ENGINE'):
tallies = [parse_omt(t) for t in all_omts]
else:
engine = environ.get('OMV_ENGINE').lower()
tallies = [parse_omt(t)
for t in all_omts
if load_yaml(t)['engine'].lower() == engine]
else:
tallies = [parse_omt(t) for t in all_omts]
for t in tallies:
th.add(t)
results = [t.all_passed() for t in tallies]
inform('')
inform("%i test(s) run" % len(tallies),
overline='-', underline='-', center=True)
inform('')
if all(results):
inform("All tests passing!", underline='=', center=True)
else:
failed = [trim_path(t.omt) for t in tallies if not t.all_passed()]
inform("Some test(s) failed: ", failed, underline='=')
if is_verbose():
print('\n'+th.summary()+'\n')
assert all(results)
def parse_omt(omt_path):
inform('')
inform("Running tests defined in ", trim_path(omt_path),
underline='=', center=False)
mepomt = OMVTestParser(omt_path)
if not OMVEngines.has_key(mepomt.engine):
inform("Error! Unrecognised engine: %s"%mepomt.engine)
exit(1)
engine = OMVEngines[mepomt.engine](mepomt.modelpath)
experiments = [exp for exp in mepomt.generate_exps(engine)]
tally = Tallyman(mepomt)
try:
engine.run()
for exp in experiments:
inform('Running checks for experiment: ', exp.name, indent=1)
inform('')
results = exp.check_all()
inform('{:<30}{:^20}'.format('Observable', 'Test Passed'),
underline='-', indent=3)
for rn, rv in results.iteritems():
inform(u'{:<30}{:^20}'.format(rn, check(rv)), indent=3)
tally.add_experiment(exp, results)
except (EngineInstallationError, EngineExecutionError):
# TODO: serialize exception info
inform('ERROR running engine ', engine.name, indent=1,
underline='-', overline='-')
return tally
def add(self, tally):
if not tally.engine in self.all_engines:
self.all_engines.append(tally.engine)
mp = trim_path(tally.modelpath)
if not self.tallies.has_key(mp):
self.tallies[mp] = {}
mptallies = self.tallies[mp]
if not mptallies.has_key(tally.engine):
mptallies[tally.engine] = []
mptallies[tally.engine].append(tally)
def add(self, tally):
if not tally.engine in self.all_engines:
self.all_engines.append(tally.engine)
mp = trim_path(tally.modelpath)
if not self.tallies.has_key(mp):
self.tallies[mp] = {}
mptallies = self.tallies[mp]
if not mptallies.has_key(tally.engine):
mptallies[tally.engine] = []
mptallies[tally.engine].append(tally)
