def stop_testcase(self, status, **kwargs):
from tiden.tidenfabric import TidenFabric
result_lines_collector = TidenFabric().getResultLinesCollector()
test_result_lines = result_lines_collector.get_lines()
# do not change fail status in iterations
if self.tests[self.current_test]['status'] == 'fail':
return
if self.tests[self.current_test]['status'] != status:
if self.tests[self.current_test]['status'] != 'running':
prev_status = self.tests[self.current_test]['status']
self.tests_num[prev_status] -= 1
self.tests_num[status] += 1
self.tests[self.current_test]['status'] = status
self.tests[self.current_test]['time'] = str(
exec_time(self.testcase_started, 1))
if status != 'pass':
self.tests[self.current_test]['xunit_info'] = {
'type':
kwargs.get('e').__class__.__name__,
'message':
self.util_filter_escape_seqs('\n'.join(test_result_lines) +
str(kwargs.get('tb')))
}
else:
self.tests[self.current_test]['xunit_info'] = {
'type':
None,
'message':
None if not test_result_lines else
self.util_filter_escape_seqs('\n'.join(test_result_lines))
}
if kwargs.get('known_issue'):
self.tests[self.current_test]['known_issue'] = kwargs.get(
'known_issue')
if self.tests[self.current_test]['status'] == "pass":
self.passed_with_issue[self.current_test] = kwargs.get(
'known_issue')
message = self.tests[self.current_test]['xunit_info']['message']
if message is None:
message = ''
if self.tests[self.current_test].get('xunit_info'):
self.tests[self.current_test]['xunit_info']['message'] = \
"*** Known issue: %s\n" % kwargs.get('known_issue') \
+ message
if kwargs.get('run_info'):
self.tests[self.current_test]['run_info'] = kwargs.get('run_info')
self.update_xunit()
def test_java_app(with_java_app_classpath, local_config, tmpdir, mock_pm):
from tiden.result import Result
from tiden.localpool import LocalPool
from tiden.tidenfabric import TidenFabric
from copy import deepcopy
from datetime import datetime
var_dir = str(tmpdir.mkdir('var'))
xunit_file = str(tmpdir.join('var').join('xunit.xml'))
tmpdir.join('var').join('xunit.xml').write('', ensure=True)
report_path = 'report.yaml'
config = deepcopy(local_config)
config.update({
'suite_name': 'mock',
'test_name': '*',
'suite_dir': join(dirname(__file__), 'res', 'java_app', 'suites'),
'dir_prefix': f'mock-{datetime.now().strftime("%y%m%d-%H%M%S")}',
})
config.update({
'suite_var_dir': str(tmpdir.join('var').mkdir(config['dir_prefix'])),
'remote': {
'artifacts': join(config['environment']['home'], 'artifacts'),
'suite_var_dir': join(config['environment']['home'], config['dir_prefix']),
},
'config_path': str(tmpdir.join('var').join('config.yaml')),
})
config.update({
'artifacts': {
'mockapp': {
'type': 'mockapp',
'path': join(var_dir, 'artifacts', 'mockapp'),
'remote_path': join(config['remote']['artifacts'], 'mockapp'),
}
},
})
ssh_pool = LocalPool(local_config['ssh'])
res = Result(xunit_path=xunit_file)
modules = {
'mock.mock_test_app': {
'path': '%s/mock/mock_test_app.py' % config['suite_dir'],
'module_short_name': 'mock_test_app',
},
}
from tiden.tidenrunner import TidenRunner
ssh_pool.connect()
TidenFabric().setSshPool(ssh_pool)
TidenFabric().setConfig(config)
tr = TidenRunner(config, modules=modules, ssh_pool=ssh_pool, plugin_manager=mock_pm, xunit_path=xunit_file)
tr.process_tests()
res = tr.get_tests_results()
res.flush_xunit()
res.create_testrail_report(config, report_file=str(report_path))
def test_fabric_not_assigned_option(example_dict_config):
config = TidenFabric().setConfig(example_dict_config)
assert 'config' == config.__doc__
# unknown option with '_enabled' in name should be treated as AttrObj
option = config.not_set_option_enabled
assert hasattr(option, '__parent__')
assert hasattr(option, '__name__')
assert 'config.not_set_option_enabled' == config.not_set_option_enabled.__name__(
)
assert hasattr(option, 'value')
assert option.value is None
# negating such AttrObj should return another AttrObj with inverted '__negated__' attribute
assert not option is None
assert not ~option is None
assert (~option).value
assert hasattr(option, '__negated__')
assert not option.__negated__
assert hasattr((~option), '__negated__')
assert (~option).__negated__
# other unknown options should be just None
option1 = config.not_set_option
assert not hasattr(option1, '__parent__')
assert not hasattr(option1, '__name__')
assert option1 is None
def start_testcase(self, tested_class, tested_attr):
from tiden.tidenfabric import TidenFabric
result_lines_collector = TidenFabric().getResultLinesCollector()
result_lines_collector.reset()
self.testcase_started = time()
self.tested_class = tested_class
self.tested_attr = tested_attr
tested_attr_name = tested_attr.split(
'(')[0] if '(' in tested_attr else tested_attr
self.current_test = "{}.{}.{}".format(tested_class.__module__,
tested_class.__class__.__name__,
tested_attr)
test_case_id = None
if hasattr(getattr(tested_class.__class__, tested_attr_name),
"__test_id__"):
test_ids = (getattr(
getattr(tested_class.__class__, tested_attr_name),
"__test_id__"))
test_case_id = int(test_ids[0])
if test_case_id == 0:
test_case_id = None
if self.tests.get(self.current_test):
self.tests[self.current_test].update({
"started": time(),
})
else:
self.tests_num['total'] += 1
self.tests[self.current_test] = {
'status':
'running',
'classname':
"%s.%s" %
(tested_class.__module__, tested_class.__class__.__name__),
'name':
str(tested_attr),
'time':
str(0),
'started':
time()
}
if test_case_id:
self.tests[self.current_test].update({
'test_case_id':
test_case_id,
})
def log_put(msg, level=3, **kwargs):
line_nums = __is_called_from_line()
prefix_str = ''
if len(line_nums) > 0:
line_num = "[%s]" % ' '.join(line_nums)
prefix_str = '[%s]%s ' % (
datetime.now().isoformat()[11:-7],
line_num
)
stdout.write('\r%s%s' % (prefix_str, msg))
stdout.flush()
logger = get_logger('tiden')
logger.info(msg)
if kwargs.get('report'):
from tiden.tidenfabric import TidenFabric
result_lines_collector = TidenFabric().getResultLinesCollector()
result_lines_collector.add_line(msg)
def init_ssh_pool(config):
log_print("*** Create SSH Pool ***", color='blue')
# Collect unique hosts
hosts = []
for name, data in config['environment'].items():
if name.endswith(
'_hosts'
) and not name == 'apps_use_global_hosts' and data is not None:
hosts.extend(data)
for name, data in config['environment'].items():
if isinstance(data, dict):
for inner_name, inner_data in data.items():
if inner_name.endswith('_hosts'):
hosts.extend(inner_data)
hosts = set(hosts)
config['ssh']['hosts'] = list(hosts)
# Calculate threads number
config['ssh']['threads_num'] = floor(sqrt(len(hosts)))
if config['ssh']['threads_num'] < cpu_count():
config['ssh']['threads_num'] = cpu_count()
if config['environment'].get('env_vars'):
config['ssh']['env_vars'] = config['environment']['env_vars']
write_yaml_file(config['config_path'], config)
# Make SSH connection pool
ssh_pool = None
if 'ansible' == config['connection_mode']:
try:
from tiden.ansiblepool import AnsiblePool
ssh_pool = AnsiblePool(config['ssh'])
except ImportError as e:
log_put('ERROR: unable to import AnsiblePool: %s' % e)
exit(1)
elif 'paramiko' == config['connection_mode']:
ssh_pool = SshPool(config['ssh'])
elif 'local' == config['connection_mode']:
config['ignite']['bind_to_host'] = True
config['ignite']['unique_node_ports'] = True
try:
from tiden.localpool import LocalPool
ssh_pool = LocalPool(config['ssh'])
except ImportError as e:
log_put('ERROR: unable to import LocalPool: %s' % e)
exit(1)
except NotImplementedError as e:
log_put('ERROR: %s' % e)
exit(1)
else:
log_put("ERROR: Unknown 'connection_mode' %s" %
config['connection_mode'])
exit(1)
if ssh_pool:
TidenFabric().setSshPool(ssh_pool)
ssh_pool.connect()
return ssh_pool
def log_print(msg=None, level=3, **kwargs):
if msg is None:
msg = ''
fmt_str = '{}{}'
colors = {'green': f'\033[32m{fmt_str}\033[0m',
'red': f'\033[91m{fmt_str}\033[0m',
'blue': f'\033[94m{fmt_str}\033[0m',
'yellow': f'\033[93m{fmt_str}\033[0m',
'pink': f'\033[95m{fmt_str}\033[0m',
'bold': f'\033[1m{fmt_str}\033[0m',
'debug': f'\033[35m{fmt_str}\033[0m'}
prefix_str = ''
if msg != '':
line_nums = __is_called_from_line()
if len(line_nums) > 0:
line_num = "[{}]".format(' '.join(line_nums))
prefix_str = '[{}]{} '.format(
datetime.now().isoformat()[11:-7],
line_num
)
if kwargs.get('color'):
fmt_str = colors.get(kwargs.get('color'), fmt_str)
if msg is not None:
print(fmt_str.format(prefix_str, msg))
logger = get_logger('tiden')
logger.info(msg)
else:
# stdout.write('\n')
# stdout.flush()
logger = get_logger('tiden')
logger.info('', skip_prefix=True)
if kwargs.get('report'):
from tiden.tidenfabric import TidenFabric
result_lines_collector = TidenFabric().getResultLinesCollector()
result_lines_collector.add_line(msg)
def test_fabric_simple(example_dict_config):
config = TidenFabric().setConfig(example_dict_config)
assert 'config' == config.__doc__
assert config.ignite
assert "ignite" == config.ignite.__doc__
assert config == config.ignite.__parent__
# assert config.ignite == config.ignite.pitr_enabled.__parent__ # this is not true anymore
assert config.ignite.pitr_enabled, "pitr_enabled should be True"
assert "pitr_enabled" == config.ignite.pitr_enabled.__doc__
assert 'config.ignite.pitr_enabled' == config.ignite.pitr_enabled.__name__(
)
assert not config.ignite.baseline_enabled, "baseline_enabled should be False"
assert not config.unknown_option_enabled, "unknown_option_enabled should be equal to False"
def test_tiden_hook():
hook_mgr = TidenFabric().get_hook_mgr()
plugins_path = hook_mgr.hook.tiden_get_plugins_path()
assert type(plugins_path) == list
assert len(plugins_path) == 1
assert type(plugins_path[0]) == list
assert len(plugins_path[0]) == 2
assert plugins_path[0][0] == join(dirname(dirname(abspath(__file__))), 'src', 'tiden', 'plugins')
assert plugins_path[0][1] == join(getcwd(), 'plugins')
pm = PluginManager({})
assert pm.plugins_paths == plugins_path[0]
assert pm.plugins == {}
pm = PluginManager({
'plugins': {
'DockerCleaner': {}
}
})
assert 'DockerCleaner' in pm.plugins
plugin = pm.plugins['DockerCleaner']
assert plugin['file'] == join(dirname(dirname(abspath(__file__))), 'src', 'tiden', 'plugins', 'dockercleaner.py')
assert plugin['class'] == 'DockerCleaner'
assert plugin['TIDEN_PLUGIN_VERSION'] == '1.0.0'
assert isinstance(plugin['instance'], TidenPlugin)
def test_fabric_update_dict_after_create(example_dict_config, clean_fabric):
c = example_dict_config.copy()
d = {'aa': 1}
b = d
d['bb'] = 2
assert 2 == b['bb']
class FFF:
def __init__(self, o):
self.q = o
f = FFF(d)
d['zz'] = 3
assert 3 == f.q['zz']
config = TidenFabric().getConfig(c)
c['simple'] = 42
c['artifacts'] = {
'test_artifact': {
'version': '1.0.0',
}
}
assert 42 == config.simple
assert '1.0.0' == config.artifacts.test_artifact.version
def _create_shared_file(self):
nas_manager = TidenFabric().getNasManager()
self.shared_file_path = nas_manager.touch_file(self.shared_file_name)
def main():
"""
Run Tiden tests
"""
log_print("*** Initialization ***", color='blue')
log_print('(c) 2017-{} GridGain Systems. All Rights Reserved'.format(
max(datetime.now().year, 2019)))
log_print(version)
exit_code = None
# parse arguments,
# load configuration,
# initialize working directories
config = TidenFabric().setConfig(setup_test_environment(
process_args())).obj
log_print('The configuration stored in %s' % config['config_path'])
logger = _get_default_logger(config)
sys.path.insert(0, abspath(getcwd()))
pm = PluginManager(config)
# prepare artifacts, artifact information is updated into config
# this must be done before tests collections,
# because some tests are applicable for specific artifacts only
log_print('*** Prepare artifacts ***', color='blue')
pm.do('before_prepare_artifacts', config)
remote_unzip_files, config = prepare(config)
if collect_only:
# we don't run any test, so no ssh pool nor plugin manager required
ssh_pool = None
pm = None
else:
# otherwise, create ssh pool,
# and prepare plugins to use it
ssh_pool = init_ssh_pool(config)
if pm.plugins:
log_print('*** Plugins ***', color='blue')
for name, plugin in pm.plugins.items():
log_print("%s, version %s" %
(name, plugin['TIDEN_PLUGIN_VERSION']))
pm.set(ssh=ssh_pool)
# initialize tests runner
log_print('*** Runner ***', color='blue')
tr = TidenRunner(config,
collect_only=collect_only,
ssh_pool=ssh_pool,
plugin_manager=pm)
if len(tr.modules.keys()) == 0:
log_print("Error: no test modules found")
exit(1)
log_print(
"%s module(s) matched %s.%s" %
(len(tr.modules.keys()), config['suite_name'], config['test_name']))
if collect_only:
tr.collect_tests()
else:
pm.do('before_hosts_setup')
init_remote_hosts(ssh_pool, config)
pm.do('after_hosts_setup')
upload_artifacts(ssh_pool, config, remote_unzip_files)
if pm.do_check('before_tests_run'):
tr.process_tests()
else:
exit_code = -1
pm.do('after_tests_run')
result = tr.get_tests_results()
result.flush_xunit()
result.print_summary()
result.create_testrail_report(config,
report_file=config.get('testrail_report'))
print_blue("Execution time %d:%02d:%02d " %
hms(int(time()) - result.get_started()))
if exit_code:
exit(exit_code)
def clean_fabric():
TidenFabric().reset()
def set_configuration_options(cfg_options, config, configuration):
from tiden.tidenfabric import TidenFabric
for i, cfg_option in enumerate(cfg_options):
config[cfg_option] = configuration[i]
TidenFabric().setConfig(config)
def test_runner_skipped_configurations(with_dec_classpath, local_config, tmpdir, mock_pm):
"""
Test configurations correctly passed to TestRail report for skipped tests
:return:
"""
var_dir = _ensure_var_dir(tmpdir)
xunit_file = _ensure_xunit_file_empty(var_dir)
testrail_report_file = _ensure_tr_report_file_empty(var_dir)
suite_var_dir = str(var_dir.mkdir('suite-mock'))
config_path = str(var_dir.join('config.yaml'))
source = 'mock_test_module_with_test_configuration'
suite = 'mock'
module_name = 'suites.%s.%s.MockTestModuleWithTestConfiguration' % (suite, source)
test_prefix = module_name + '.'
config = deepcopy(local_config)
config.update({
'artifacts': {},
# 'attrib': 'test_runner',
# 'attr_match': 'any',
'suite_var_dir': suite_var_dir,
'suite_dir': join(dirname(__file__), 'res', 'decorators', 'suites'),
'remote': {
'suite_var_dir': suite_var_dir,
},
'config_path': config_path,
'zookeeper_enabled': False,
'pitr_enabled': False,
'compaction_enabled': True,
})
ssh_pool = LocalPool(local_config['ssh'])
test_module_source_file_name = '%s/%s/%s.py' % (config['suite_dir'], suite, source)
modules = {
'%s.%s' % (suite, source): {
'path': test_module_source_file_name,
'module_short_name': source,
}
}
test_configuration = '(pitr_enabled=false, compaction_enabled=true, zookeeper_enabled=false)'
expected_configuration_options = ['pitr_enabled', 'compaction_enabled', 'zookeeper_enabled']
expected_result = {
'test_main':
{'status': 'pass', 'type': None, 'message': None},
'test_zookeeper_only':
{'status': 'skipped', 'type': 'skipped cause of config.zookeeper_enabled is False', 'message': None},
}
expected_statuses_count = {'pass': 1,
'fail': 0,
'error': 0,
'skip': 1,
'total': len(expected_result)}
from tiden.tidenfabric import TidenFabric
TidenFabric().reset().setConfig(config)
tr = TidenRunner(config, modules=modules, ssh_pool=ssh_pool, plugin_manager=mock_pm, xunit_path=xunit_file)
tr.process_tests()
res = tr.get_tests_results()
res.create_testrail_report(config, report_file=basename(testrail_report_file))
_tests = res.get_tests()
print(_tests)
# validate raw test results
assert len(_tests) == len(expected_result)
for test_to_check in expected_result.keys():
status, error_type, message, test_name = res.get_test_details('{}{}{}'.format(test_prefix, test_to_check, test_configuration))
assert expected_result[test_to_check].get('status') == status
assert expected_result[test_to_check].get('type') == error_type
if expected_result[test_to_check].get('message') is None:
assert message is None
else:
assert expected_result[test_to_check].get('message') == message \
or expected_result[test_to_check].get('message') in message
for status, count in expected_statuses_count.items():
assert res.get_tests_num(status) == count
# validate generated TestRail .yaml report
tr_report = read_yaml_file(testrail_report_file)
assert type({}) == type(tr_report)
assert len(_tests) == len(tr_report)
for test_run, test in tr_report.items():
assert 'suite_run_id' in test
assert 'test_run_id' in test
assert test_run == test['test_run_id']
assert 'module' in test
assert test['module'] == module_name
assert 'test_configuration_options' in test
assert expected_configuration_options == test['test_configuration_options']
assert 'function' in test
assert test['function'] in expected_result.keys()
expected_test_result = expected_result[test['function']]
expected_status = res.util_status_to_testrail_status(expected_test_result['status'])
assert 'last_status' in test
assert expected_status == test['last_status']
# a test message will be either in 'message' or 'type' if 'message' is None
assert 'asserts' in test
assert type([]) == type(test['asserts'])
# currently Tiden generates only one assert per test
assert len(test['asserts']) == 1
assert type({}) == type(test['asserts'][0])
assert 'status' in test['asserts'][0]
assert expected_status == test['asserts'][0]['status']
expected_assert_message = expected_test_result['message'] if expected_test_result['message'] is not None else \
expected_test_result['type']
if expected_assert_message is not None:
assert res.util_filter_escape_seqs(expected_assert_message) in test['asserts'][0]['message']
# check all test run id's are unique
test_run_ids = [test['test_run_id'] for test in tr_report.values()]
assert len(test_run_ids) == len(set(test_run_ids))
# check all suite run id is the same
suite_run_ids = set([test['suite_run_id'] for test in tr_report.values()])
assert 1 == len(suite_run_ids)
def test_runner_collect(with_dec_classpath, local_config, tmpdir, mock_pm):
var_dir = _ensure_var_dir(tmpdir)
xunit_file_collect = _ensure_xunit_file_empty(var_dir, '-collect')
xunit_file_process = _ensure_xunit_file_empty(var_dir, '-process')
testrail_report_file_collect = _ensure_tr_report_file_empty(var_dir, '-collect')
testrail_report_file_process = _ensure_tr_report_file_empty(var_dir, '-process')
suite_var_dir = str(var_dir.mkdir('suite-mock'))
config_path = str(var_dir.join('config.yaml'))
source = 'mock_test_module_with_test_configuration'
suite = 'mock'
module_name = 'suites.%s.%s.MockTestModuleWithTestConfiguration' % (suite, source)
test_prefix = module_name + '.'
config = deepcopy(local_config)
config.update({
'artifacts': {},
'suite_var_dir': suite_var_dir,
'suite_dir': join(dirname(__file__), 'res', 'decorators', 'suites'),
'remote': {
'suite_var_dir': suite_var_dir,
},
'config_path': config_path,
'zookeeper_enabled': False,
'pitr_enabled': False,
'compaction_enabled': True,
})
ssh_pool = LocalPool(local_config['ssh'])
test_module_source_file_name = '%s/%s/%s.py' % (config['suite_dir'], suite, source)
modules = {
'%s.%s' % (suite, source): {
'path': test_module_source_file_name,
'module_short_name': source,
}
}
test_configuration = '(pitr_enabled=false, compaction_enabled=true, zookeeper_enabled=false)'
expected_configuration_options = ['pitr_enabled', 'compaction_enabled', 'zookeeper_enabled']
expected_result = {
'test_main':
{'status': 'pass', 'type': None, 'message': None},
'test_zookeeper_only':
{'status': 'skipped', 'type': 'skipped cause of config.zookeeper_enabled is None', 'message': None},
}
from tiden.tidenfabric import TidenFabric
TidenFabric().reset().setConfig(config)
tr = TidenRunner(config, modules=modules, ssh_pool=ssh_pool, plugin_manager=mock_pm, xunit_path=xunit_file_collect)
tr.collect_tests()
res = tr.get_tests_results()
res.update_xunit()
res.create_testrail_report(config, report_file=basename(testrail_report_file_collect))
_tests = res.get_tests()
assert 12 == len(_tests)
print(_tests)
TidenFabric().reset().setConfig(config)
tr = TidenRunner(config, modules=modules, ssh_pool=ssh_pool, plugin_manager=mock_pm, xunit_path=xunit_file_process)
tr.process_tests()
res = tr.get_tests_results()
res.create_testrail_report(config, report_file=basename(testrail_report_file_process))
_tests = res.get_tests()
assert 2 == len(_tests)
print(_tests)
def _delete_shared_file(self):
nas_manager = TidenFabric().getNasManager()
nas_manager.delete_file(self.shared_file_name)
def _check_nas_manager_configured(self):
nas_manager = TidenFabric().getNasManager()
return nas_manager.is_configured()