def _run_test(self,
test: Test,
global_variables: dict,
output: str = 'full',
test_type='test') -> bool:
try:
self.var_holder.prepare_variables(test, global_variables)
logger.log_storage.test_start(test.file, test_type=test_type)
test.check_ignored()
with OptionalOutput(output == 'limited'):
test.run()
info(test_type.capitalize() + ' ' +
cut_path(self.tests_path, test.file) +
logger.green(' passed.'))
logger.log_storage.test_end(test.file, True, test_type=test_type)
return True
except SkipException:
info(test_type.capitalize() + ' ' +
cut_path(self.tests_path, test.file) +
logger.yellow(' skipped.'))
logger.log_storage.test_end(test.file,
True,
end_comment='Skipped',
test_type=test_type)
return True
except Exception as e:
warning(test_type.capitalize() + ' ' +
cut_path(self.tests_path, test.file) +
logger.red(' failed: ') + str(e))
debug(traceback.format_exc())
logger.log_storage.test_end(test.file,
False,
str(e),
test_type=test_type)
return False
def _run_finally(self, test, result: bool):
if test and test.final:
logger.log_storage.test_start(test.file,
test_type='{}_cleanup'.format(
test.file))
try:
test.run_finally(result)
info('Test ' + cut_path(self.tests_path, test.file) +
' [cleanup] ' + logger.green(' passed.'))
logger.log_storage.test_end(test.file,
True,
test_type='{} [cleanup]'.format(
test.file))
except Exception as e:
warning('Test ' + cut_path(self.tests_path, test.file) +
' [cleanup] ' + logger.red(' failed: ') + str(e))
debug(traceback.format_exc())
logger.log_storage.test_end(test.file,
False,
test_type='{} [cleanup]'.format(
test.file))
def print_summary(self, path):
from catcher.utils import logger
tests = [t for t in self._data if t.get('type') == 'test']
out_string = self.calculate_statistics(tests)
for test in tests:
out_string += '\nTest {}: '.format(
file_utils.cut_path(path, test['file']))
# select only step's ends, which belongs to the current test (excluding registered includes run)
step_finish_only = [
o for o in test['output']
if 'success' in o and o['nested'] == 0
]
if test['status'] == 'OK' and test['comment'] != 'Skipped':
out_string += logger.green('pass')
elif test['comment'] == 'Skipped':
out_string += logger.yellow('skipped')
else:
out_string += logger.red('fail') + ', on step {}'.format(
len(step_finish_only))
logger.info(out_string)
def run_tests(self, output: str = 'full') -> bool:
"""
Run the testcase
:param output: 'full' - all output possible. 'limited' - only test end report & summary. 'final' - only summary.
"""
try:
[mod.before() for mod in ModulesFactory().modules.values()]
results = []
for parse_result in self.parser.read_tests(self.tests_path):
if parse_result.should_run: # parse successful
variables = self.var_holder.variables # each test has it's own copy of global variables
variables[
'TEST_NAME'] = parse_result.test.file # variables are shared between test and includes
with OptionalOutput(output == 'final'):
for include in parse_result.run_on_include: # run all includes before the main test.
self._run_test(include,
variables,
output=output,
test_type='include')
result = self._run_test(parse_result.test,
variables,
output=output)
results.append(result)
self._run_finally(parse_result.test, result)
else: # parse failed (dependency/parsing problem)
warning('Test ' +
cut_path(self.tests_path, parse_result.test) +
logger.red(' failed: ') +
str(parse_result.parse_error))
logger.log_storage.test_parse_fail(
parse_result.test, parse_result.parse_error)
results.append(False)
return all(results)
finally:
logger.log_storage.write_report(join(self.path, 'reports'))
logger.log_storage.print_summary(self.tests_path)
[mod.after() for mod in ModulesFactory().modules.values()]