def run_all(filename, case_filter=[]):
""" Helper function to run test cases defined in a file; to be called from __main__.
case_filter is an optional list of case names to run.
If not specified, all test cases are run.
"""
TinyFW.set_default_config(env_config_file=None, test_suite_name=TEST_SUITE)
test_methods = SearchCases.Search.search_test_cases(filename)
test_methods = filter(lambda m: not m.case_info['ignore'], test_methods)
test_cases = CaseConfig.Parser.apply_config(test_methods, None)
tests_failed = []
for case in test_cases:
test_name = case.test_method.__name__
if case_filter:
if case_filter[0].endswith('*'):
if not test_name.startswith(case_filter[0][:-1]):
continue
else:
if test_name not in case_filter:
continue
result = case.run()
if not result:
tests_failed.append(case)
if tests_failed:
print('The following tests have failed:')
for case in tests_failed:
print(' - ' + case.test_method.__name__)
raise SystemExit(1)
print('Tests pass')
def __init__(self, test_case, case_config, env_config_file=None):
super(Runner, self).__init__()
self.setDaemon(True)
if case_config:
test_suite_name = os.path.splitext(
os.path.basename(case_config))[0]
else:
test_suite_name = "TestRunner"
TinyFW.set_default_config(env_config_file=env_config_file,
test_suite_name=test_suite_name)
test_methods = SearchCases.Search.search_test_cases(test_case)
self.test_cases = CaseConfig.Parser.apply_config(
test_methods, case_config)
self.test_result = []
def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1,
level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs):
"""
decorator for idf custom tests (with default values for some keyword args).
:param app: test application class
:param dut: dut class
:param chip: chip supported, string or tuple
:param module: module, string
:param execution_time: execution time in minutes, int
:param level: test level, could be used to filter test cases, string
:param erase_nvs: if need to erase_nvs in DUT.start_app()
:param config_name: if specified, name of the app configuration
:param group: identifier to group custom tests (unused for now, defaults to "test-apps")
:param kwargs: other keyword args
:return: test method
"""
try:
# try to config the default behavior of erase nvs
dut.ERASE_NVS = erase_nvs
except AttributeError:
pass
original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
execution_time=execution_time, level=level, **kwargs)
def test(func):
test_func = original_method(func)
test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
return test_func
return test
def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", execution_time=1,
level="unit", erase_nvs=True, **kwargs):
"""
decorator for testing idf unit tests (with default values for some keyword args).
:param app: test application class
:param dut: dut class
:param chip: chip supported, string or tuple
:param module: module, string
:param execution_time: execution time in minutes, int
:param level: test level, could be used to filter test cases, string
:param erase_nvs: if need to erase_nvs in DUT.start_app()
:param kwargs: other keyword args
:return: test method
"""
try:
# try to config the default behavior of erase nvs
dut.ERASE_NVS = erase_nvs
except AttributeError:
pass
original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
execution_time=execution_time, level=level, **kwargs)
def test(func):
test_func = original_method(func)
test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
return test_func
return test
def run_all(filename, case_filter=[]):
""" Helper function to run test cases defined in a file; to be called from __main__.
case_filter is an optional list of case names to run.
If not specified, all test cases are run.
"""
TinyFW.set_default_config(env_config_file=None, test_suite_name=TEST_SUITE)
test_methods = SearchCases.Search.search_test_cases(filename)
test_methods = filter(lambda m: not m.case_info["ignore"], test_methods)
test_cases = CaseConfig.Parser.apply_config(test_methods, None)
tests_failed = []
for case in test_cases:
if case_filter and case.test_method.__name__ not in case_filter:
continue
result = case.run()
if not result:
tests_failed.append(case)
def test_func_generator(func,
app,
target,
ci_target,
module,
execution_time,
level,
erase_nvs,
drop_kwargs_dut=False,
**kwargs):
test_target = local_test_check(target)
dut = get_dut_class(test_target, erase_nvs)
if drop_kwargs_dut and 'dut' in kwargs: # panic_test() will inject dut, resolve conflicts here
dut = kwargs['dut']
del kwargs['dut']
original_method = TinyFW.test_method(
app=app,
dut=dut,
target=upper_list_or_str(target),
ci_target=upper_list_or_str(ci_target),
module=module,
execution_time=execution_time,
level=level,
erase_nvs=erase_nvs,
dut_dict=TARGET_DUT_CLS_DICT,
**kwargs)
test_func = original_method(func)
test_func.case_info["ID"] = format_case_id(target,
test_func.case_info["name"])
return test_func
def test_func_generator(func, app, target, ci_target, module, execution_time,
level, erase_nvs, **kwargs):
target = upper_list_or_str(target)
test_target = local_test_check(target)
if 'additional_duts' in kwargs:
dut_classes = deepcopy(TARGET_DUT_CLS_DICT)
dut_classes.update(kwargs['additional_duts'])
else:
dut_classes = TARGET_DUT_CLS_DICT
dut = get_dut_class(test_target, dut_classes, erase_nvs)
original_method = TinyFW.test_method(
app=app,
dut=dut,
target=target,
ci_target=upper_list_or_str(ci_target),
module=module,
execution_time=execution_time,
level=level,
erase_nvs=erase_nvs,
dut_dict=dut_classes,
**kwargs)
test_func = original_method(func)
test_func.case_info["ID"] = format_case_id(target,
test_func.case_info["name"])
return test_func
def run_all(filename):
""" Helper function to run all test cases defined in a file; to be called from __main__. """
TinyFW.set_default_config(env_config_file=None, test_suite_name=TEST_SUITE)
test_methods = SearchCases.Search.search_test_cases(filename)
test_methods = filter(lambda m: not m.case_info["ignore"], test_methods)
test_cases = CaseConfig.Parser.apply_config(test_methods, None)
tests_failed = []
for case in test_cases:
result = case.run()
if not result:
tests_failed.append(case)
if tests_failed:
print("The following tests have failed:")
for case in tests_failed:
print(" - " + case.test_method.__name__)
raise SystemExit(1)
print("Tests pass")
def test(func):
test_target = local_test_check(target)
dut = get_dut_class(test_target, erase_nvs)
original_method = TinyFW.test_method(
app=app, dut=dut, target=upper_list_or_str(target), ci_target=upper_list_or_str(ci_target),
module=module, execution_time=execution_time, level=level, erase_nvs=erase_nvs,
dut_dict=TARGET_DUT_CLS_DICT, **kwargs
)
test_func = original_method(func)
test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
return test_func
def idf_example_test(app=Example,
dut=IDFDUT,
chip="ESP32",
module="examples",
execution_time=1,
level="example",
erase_nvs=True,
config_name=None,
**kwargs):
"""
decorator for testing idf examples (with default values for some keyword args).
:param app: test application class
:param dut: dut class
:param chip: chip supported, string or tuple
:param module: module, string
:param execution_time: execution time in minutes, int
:param level: test level, could be used to filter test cases, string
:param erase_nvs: if need to erase_nvs in DUT.start_app()
:param config_name: if specified, name of the app configuration
:param kwargs: other keyword args
:return: test method
"""
try:
# try to config the default behavior of erase nvs
dut.ERASE_NVS = erase_nvs
except AttributeError:
pass
original_method = TinyFW.test_method(app=app,
dut=dut,
chip=chip,
module=module,
execution_time=execution_time,
level=level,
**kwargs)
def test(func):
test_func = original_method(func)
return test_func
return test
""" example of writing test with TinyTestFW """
import re
import ttfw_idf
from tiny_test_fw import TinyFW
@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
def test_examples_protocol_https_request(env, extra_data):
"""
steps: |
1. join AP
2. connect to www.howsmyssl.com:443
3. send http request
"""
dut1 = env.get_dut('https_request', 'examples/protocols/https_request', dut_class=ttfw_idf.ESP32DUT)
dut1.start_app()
dut1.expect(re.compile(r'Connecting to www.howsmyssl.com:443'), timeout=30)
dut1.expect('Performing the SSL/TLS handshake')
dut1.expect('Certificate verified.', timeout=15)
dut1.expect_all(re.compile(r'Cipher suite is TLS-ECDHE-RSA-WITH-AES-128-GCM-SHA256'),
'Reading HTTP response',
timeout=20)
dut1.expect(re.compile(r'Completed (\d) requests'))
if __name__ == '__main__':
TinyFW.set_default_config(env_config_file='EnvConfigTemplate.yml', dut=ttfw_idf.IDFDUT)
test_examples_protocol_https_request()
continue
pair = test_item.split(r':', 1)
if len(pair) == 1 or pair[0] is 'name':
test_dict['name'] = pair[0]
elif len(pair) == 2:
if pair[0] == 'timeout' or pair[0] == 'child case num':
test_dict[pair[0]] = int(pair[1])
else:
test_dict[pair[0]] = pair[1]
else:
raise ValueError('Error in argument item {} of {}'.format(
test_item, test))
test_dict['app_bin'] = args.app_bin
list_of_dicts.append(test_dict)
TinyFW.set_default_config(env_config_file=args.env_config_file)
env_config = TinyFW.get_default_config()
env_config['app'] = ttfw_idf.UT
env_config['dut'] = ttfw_idf.IDFDUT
env_config['test_suite_name'] = 'unit_test_parsing'
test_env = Env.Env(**env_config)
detect_update_unit_test_info(test_env,
extra_data=list_of_dicts,
app_bin=args.app_bin)
for index in range(1, args.repeat + 1):
if args.repeat > 1:
Utility.console_log("Repetition {}".format(index), color="green")
for dic in list_of_dicts:
t = dic.get('type', SIMPLE_TEST_ID)
