def _get_test_data(exch_test):
print('Test data set #%s' % exch_test)
start_exch = read_yaml_file(_get_test_file_name(exch_test, 'start_exch'))
finish_exch = read_yaml_file(_get_test_file_name(exch_test, 'finish_exch'))
merge_exch = read_yaml_file(_get_test_file_name(exch_test, 'merge_exch'))
return ExchangesCollection.create_from_log_data(start_exch, finish_exch, merge_exch)
def test_runner_skipped_configurations(with_dec_classpath, local_config, tmpdir, mock_pm):
"""
Test configurations correctly passed to TestRail report for skipped tests
:return:
"""
var_dir = _ensure_var_dir(tmpdir)
xunit_file = _ensure_xunit_file_empty(var_dir)
testrail_report_file = _ensure_tr_report_file_empty(var_dir)
suite_var_dir = str(var_dir.mkdir('suite-mock'))
config_path = str(var_dir.join('config.yaml'))
source = 'mock_test_module_with_test_configuration'
suite = 'mock'
module_name = 'suites.%s.%s.MockTestModuleWithTestConfiguration' % (suite, source)
test_prefix = module_name + '.'
config = deepcopy(local_config)
config.update({
'artifacts': {},
# 'attrib': 'test_runner',
# 'attr_match': 'any',
'suite_var_dir': suite_var_dir,
'suite_dir': join(dirname(__file__), 'res', 'decorators', 'suites'),
'remote': {
'suite_var_dir': suite_var_dir,
},
'config_path': config_path,
'zookeeper_enabled': False,
'pitr_enabled': False,
'compaction_enabled': True,
})
ssh_pool = LocalPool(local_config['ssh'])
test_module_source_file_name = '%s/%s/%s.py' % (config['suite_dir'], suite, source)
modules = {
'%s.%s' % (suite, source): {
'path': test_module_source_file_name,
'module_short_name': source,
}
}
test_configuration = '(pitr_enabled=false, compaction_enabled=true, zookeeper_enabled=false)'
expected_configuration_options = ['pitr_enabled', 'compaction_enabled', 'zookeeper_enabled']
expected_result = {
'test_main':
{'status': 'pass', 'type': None, 'message': None},
'test_zookeeper_only':
{'status': 'skipped', 'type': 'skipped cause of config.zookeeper_enabled is False', 'message': None},
}
expected_statuses_count = {'pass': 1,
'fail': 0,
'error': 0,
'skip': 1,
'total': len(expected_result)}
from tiden.tidenfabric import TidenFabric
TidenFabric().reset().setConfig(config)
tr = TidenRunner(config, modules=modules, ssh_pool=ssh_pool, plugin_manager=mock_pm, xunit_path=xunit_file)
tr.process_tests()
res = tr.get_tests_results()
res.create_testrail_report(config, report_file=basename(testrail_report_file))
_tests = res.get_tests()
print(_tests)
# validate raw test results
assert len(_tests) == len(expected_result)
for test_to_check in expected_result.keys():
status, error_type, message, test_name = res.get_test_details('{}{}{}'.format(test_prefix, test_to_check, test_configuration))
assert expected_result[test_to_check].get('status') == status
assert expected_result[test_to_check].get('type') == error_type
if expected_result[test_to_check].get('message') is None:
assert message is None
else:
assert expected_result[test_to_check].get('message') == message \
or expected_result[test_to_check].get('message') in message
for status, count in expected_statuses_count.items():
assert res.get_tests_num(status) == count
# validate generated TestRail .yaml report
tr_report = read_yaml_file(testrail_report_file)
assert type({}) == type(tr_report)
assert len(_tests) == len(tr_report)
for test_run, test in tr_report.items():
assert 'suite_run_id' in test
assert 'test_run_id' in test
assert test_run == test['test_run_id']
assert 'module' in test
assert test['module'] == module_name
assert 'test_configuration_options' in test
assert expected_configuration_options == test['test_configuration_options']
assert 'function' in test
assert test['function'] in expected_result.keys()
expected_test_result = expected_result[test['function']]
expected_status = res.util_status_to_testrail_status(expected_test_result['status'])
assert 'last_status' in test
assert expected_status == test['last_status']
# a test message will be either in 'message' or 'type' if 'message' is None
assert 'asserts' in test
assert type([]) == type(test['asserts'])
# currently Tiden generates only one assert per test
assert len(test['asserts']) == 1
assert type({}) == type(test['asserts'][0])
assert 'status' in test['asserts'][0]
assert expected_status == test['asserts'][0]['status']
expected_assert_message = expected_test_result['message'] if expected_test_result['message'] is not None else \
expected_test_result['type']
if expected_assert_message is not None:
assert res.util_filter_escape_seqs(expected_assert_message) in test['asserts'][0]['message']
# check all test run id's are unique
test_run_ids = [test['test_run_id'] for test in tr_report.values()]
assert len(test_run_ids) == len(set(test_run_ids))
# check all suite run id is the same
suite_run_ids = set([test['suite_run_id'] for test in tr_report.values()])
assert 1 == len(suite_run_ids)
def test_runner_basic(with_dec_classpath, local_config, tmpdir, mock_pm):
"""
Just test that after TidenRunner execution we've got correct test results ans correct exceptions in the
failed tests.
:return:
"""
var_dir = _ensure_var_dir(tmpdir)
xunit_file = _ensure_xunit_file_empty(var_dir)
testrail_report_file = _ensure_tr_report_file_empty(var_dir)
suite_var_dir = str(var_dir.mkdir('suite-mock'))
config_path = str(var_dir.join('config.yaml'))
suite = 'mock3'
module_short_name = 'mock_test_module_with_exceptions'
module_class_name = 'MockTestModuleWithExceptions'
module_name = 'suites.%s.%s.%s' % (suite, module_short_name, module_class_name)
test_prefix = module_name + '.'
config = deepcopy(local_config)
config.update({
'artifacts': {},
'attrib': 'test_runner',
'attr_match': 'any',
'suite_var_dir': suite_var_dir,
'suite_dir': join(dirname(__file__), 'res', 'decorators', 'suites'),
'remote': {
'suite_var_dir': suite_var_dir,
},
'config_path': config_path,
})
ssh_pool = LocalPool(local_config['ssh'])
test_module_source_file_name = '%s/%s/%s.py' % (config['suite_dir'], suite, module_short_name)
modules = {
'%s.%s' % (suite, module_short_name): {
'path': test_module_source_file_name,
'module_short_name': module_short_name,
}
}
expected_result = {
'test_should_pass':
{'status': 'pass', 'type': None, 'message': None},
'test_passed_with_result_message':
{'status': 'pass', 'type': None, 'message': 'WOO-HOO'},
'test_should_fail':
{'status': 'fail', 'type': 'TidenException', 'message': 'TidenException(\'Fake exception in test\')'},
'test_should_be_skipped':
{'status': 'skipped',
'type': 'skipped cause of expression evaluates to False at %s:45' % test_module_source_file_name,
'message': None},
'test_should_be_not_started':
{'status': 'skipped_no_start',
'type': 'skipped cause of attrib mismatch',
'message': None},
'test_with_exception_in_setup':
{'status': 'fail', 'type': 'TidenException', 'message': 'TidenException(\'Exception in test setup\')'},
'test_pass_with_exception_in_teardown':
{'status': 'pass', 'type': None, 'message': None},
'test_fail_with_exception_in_teardown':
{'status': 'fail', 'type': 'TidenException', 'message': 'TidenException(\'Fake exception in test\')'},
'test_should_fail_with_error':
{'status': 'error', 'type': 'OSError', 'message': 'IOError(\'Fake IO exception in test\')'},
}
expected_statuses_count = {'pass': 3,
'fail': 3,
'error': 1,
'skip': 2,
'total': len(expected_result)}
tr = TidenRunner(config, modules=modules, ssh_pool=ssh_pool, plugin_manager=mock_pm, xunit_path=xunit_file)
tr.process_tests()
res = tr.get_tests_results()
_tests = res.get_tests()
print(_tests)
# validate raw test results
assert len(_tests) == len(expected_result)
for test_to_check in expected_result.keys():
status, error_type, message, test_name = res.get_test_details('{}{}'.format(test_prefix, test_to_check))
assert expected_result[test_to_check].get('status') == status
assert expected_result[test_to_check].get('type') == error_type
if expected_result[test_to_check].get('message') is None:
assert message is None
else:
assert expected_result[test_to_check].get('message') == message \
or expected_result[test_to_check].get('message') in message
for status, count in expected_statuses_count.items():
assert res.get_tests_num(status) == count
# validate generated TestRail .yaml report
res.create_testrail_report(config, report_file=basename(testrail_report_file))
tr_report = read_yaml_file(testrail_report_file)
assert type({}) == type(tr_report)
assert len(_tests) == len(tr_report)
for test_run, test in tr_report.items():
assert 'suite_run_id' in test
assert 'test_run_id' in test
assert test_run == test['test_run_id']
assert 'module' in test
assert test['module'] == module_name
assert 'test_configuration_options' in test
assert [] == test['test_configuration_options']
assert 'function' in test
assert test['function'] in expected_result.keys()
expected_test_result = expected_result[test['function']]
expected_status = res.util_status_to_testrail_status(expected_test_result['status'])
assert 'last_status' in test
assert expected_status == test['last_status']
# a test message will be either in 'message' or 'type' if 'message' is None
assert 'asserts' in test
assert type([]) == type(test['asserts'])
# currently Tiden generates only one assert per test
assert len(test['asserts']) == 1
assert type({}) == type(test['asserts'][0])
assert 'status' in test['asserts'][0]
assert expected_status == test['asserts'][0]['status']
expected_assert_message = expected_test_result['message'] if expected_test_result['message'] is not None else \
expected_test_result['type']
if expected_assert_message is not None:
assert res.util_filter_escape_seqs(expected_assert_message) in test['asserts'][0]['message']
# check all test run id's are unique
test_run_ids = [test['test_run_id'] for test in tr_report.values()]
assert len(test_run_ids) == len(set(test_run_ids))
# check all suite run id is the same
suite_run_ids = set([test['suite_run_id'] for test in tr_report.values()])
assert 1 == len(suite_run_ids)