def detect_backtrace(self, comp_data):
start = 0
while True:
match = self.BACKTRACE_PATTERN.search(comp_data, pos=start)
if match:
start = match.end()
Utility.console_log("[Backtrace]:{}".format(match.group(1)),
color="red")
# translate backtrace
addresses = self.BACKTRACE_ADDRESS_PATTERN.findall(
match.group(1))
translated_backtrace = ""
for addr in addresses:
ret = self.dut.lookup_pc_address(addr)
if ret:
translated_backtrace += ret + "\n"
if translated_backtrace:
Utility.console_log("Translated backtrace\n:" +
translated_backtrace,
color="yellow")
else:
Utility.console_log("Failed to translate backtrace",
color="yellow")
else:
break
def detect_backtrace(self, comp_data):
start = 0
while True:
match = self.BACKTRACE_PATTERN.search(comp_data, pos=start)
if match:
start = match.end()
Utility.console_log('[Backtrace]:{}'.format(match.group(1)),
color='red')
# translate backtrace
addresses = self.BACKTRACE_ADDRESS_PATTERN.findall(
match.group(1))
translated_backtrace = ''
for addr in addresses:
ret = self.dut.lookup_pc_address(addr)
if ret:
translated_backtrace += ret + '\n'
if translated_backtrace:
Utility.console_log('Translated backtrace\n:' +
translated_backtrace,
color='yellow')
else:
Utility.console_log('Failed to translate backtrace',
color='yellow')
else:
break
def run_multiple_devices_cases(env, extra_data):
"""
extra_data can be two types of value
1. as dict:
e.g.
{"name": "gpio master/slave test example",
"child case num": 2,
"config": "release",
"env_tag": "UT_T2_1"}
2. as list dict:
e.g.
[{"name": "gpio master/slave test example1",
"child case num": 2,
"config": "release",
"env_tag": "UT_T2_1"},
{"name": "gpio master/slave test example2",
"child case num": 2,
"config": "release",
"env_tag": "UT_T2_1"}]
"""
failed_cases = []
case_config = format_test_case_config(extra_data,
env.default_dut_cls.TARGET)
duts = {}
for ut_config in case_config:
Utility.console_log("Running unit test for config: " + ut_config, "O")
for one_case in case_config[ut_config]:
log_test_case(
"multi-device test",
one_case,
ut_config,
)
result = False
junit_test_case = TinyFW.JunitReport.create_test_case(
format_case_name(one_case))
try:
result = run_one_multiple_devices_case(duts, ut_config, env,
one_case,
one_case.get('app_bin'),
junit_test_case)
except TestCaseFailed:
pass # result is False, this is handled by the finally block
except Exception as e:
handle_unexpected_exception(junit_test_case, e)
finally:
if result:
Utility.console_log("Success: " +
format_case_name(one_case),
color="green")
else:
failed_cases.append(format_case_name(one_case))
Utility.console_log("Failed: " +
format_case_name(one_case),
color="red")
TinyFW.JunitReport.test_case_finish(junit_test_case)
# close all DUTs when finish running all cases for one config
for dut in duts:
env.close_dut(dut)
duts = {}
def test_examples_base_mac_address(env, extra_data):
dut = env.get_dut('base_mac_address', 'examples/system/base_mac_address')
dut.start_app()
dut.expect('BASE_MAC: Base MAC Address read from EFUSE BLK0', timeout=30)
hex_r = r', '.join((r'0x([0-9a-f]{1,2})',) * 6)
mac_m = dut.expect(re.compile(r'BASE_MAC: Using "' + hex_r + r'" as base MAC address'), timeout=5)
Utility.console_log('BASE_MAC detected: {}'.format(':'.join(mac_m)))
def get_expected_mac_string(increment):
'''
Return the string representation of the MAC address mac_m with the last octet incremented.
mac_m is an array of strings in hexa-decimal format without the '0x' prefix.
'''
return ', '.join(['0x{}'.format(m) for m in mac_m[:-1]] + [hex(int(mac_m[-1], 16) + increment)])
dut.expect_all('WIFI_STA MAC: ' + get_expected_mac_string(0),
'SoftAP MAC: ' + get_expected_mac_string(1),
'BT MAC: ' + get_expected_mac_string(2),
'Ethernet MAC: ' + get_expected_mac_string(3),
timeout=10)
def _call_espcoredump(self, extra_args, coredump_file_name,
output_file_name):
# no "with" here, since we need the file to be open for later inspection by the test case
self.coredump_output = open(output_file_name, "w")
espcoredump_script = os.path.join(os.environ["IDF_PATH"], "components",
"espcoredump", "espcoredump.py")
espcoredump_args = [
sys.executable,
espcoredump_script,
"info_corefile",
"--core",
coredump_file_name,
]
espcoredump_args += extra_args
espcoredump_args.append(self.app.elf_file)
Utility.console_log("Running " + " ".join(espcoredump_args))
Utility.console_log("espcoredump output is written to " +
self.coredump_output.name)
subprocess.check_call(espcoredump_args, stdout=self.coredump_output)
self.coredump_output.flush()
self.coredump_output.seek(0)
def _call_espcoredump(self, extra_args, coredump_file_name,
output_file_name):
# no "with" here, since we need the file to be open for later inspection by the test case
self.coredump_output = open(output_file_name, 'w')
espcoredump_script = os.path.join(os.environ['IDF_PATH'], 'components',
'espcoredump', 'espcoredump.py')
espcoredump_args = [
sys.executable,
espcoredump_script,
'info_corefile',
'--core',
coredump_file_name,
]
espcoredump_args += extra_args
espcoredump_args.append(self.app.elf_file)
Utility.console_log('Running ' + ' '.join(espcoredump_args))
Utility.console_log('espcoredump output is written to ' +
self.coredump_output.name)
subprocess.check_call(espcoredump_args, stdout=self.coredump_output)
self.coredump_output.flush()
self.coredump_output.seek(0)
def test_examples_protocol_https_request_dynamic_buffers(env, extra_data):
# Check for connection using crt bundle with mbedtls dynamic resource enabled
dut1 = env.get_dut('https_request_ssldyn',
'examples/protocols/https_request',
dut_class=ttfw_idf.ESP32DUT,
app_config_name='ssldyn')
# check and log bin size
Utility.console_log('[app_config_name] - {}'.format(dut1.app.config_name))
binary_file = os.path.join(dut1.app.binary_path, 'https_request.bin')
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance('https_request_bin_size',
'{}KB'.format(bin_size // 1024))
# start test
dut1.start_app()
dut1.expect('Loaded app from partition at offset', timeout=30)
try:
ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'),
timeout=60)
print('Connected to AP with IP: {}'.format(ip_address))
except DUT.ExpectTimeout:
raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
# only check if one connection is established
Utility.console_log(
"Testing for \"https_request using crt bundle\" with mbedtls dynamic resource enabled"
)
try:
dut1.expect(re.compile('https_request using crt bundle'), timeout=30)
dut1.expect_all('Connection established...',
'Reading HTTP response...', 'HTTP/1.1 200 OK',
re.compile('connection closed'))
except Exception:
Utility.console_log(
"Failed the test for \"https_request using crt bundle\" when mbedtls dynamic resource was enabled"
)
raise
Utility.console_log(
"Passed the test for \"https_request using crt bundle\" when mbedtls dynamic resource was enabled"
)
# Read free heap size
res = dut1.expect(ttfw_idf.MINIMUM_FREE_HEAP_SIZE_RE, timeout=20)
if not res:
raise ValueError('Maximum heap size info not found')
ttfw_idf.print_heap_size('https_request', dut1.app.config_name,
dut1.TARGET, res[0])
env.close_dut('https_request_ssldyn')
def parallel_sessions_adder(dut, port, max_sessions):
# POSTs on /adder in parallel sessions
Utility.console_log("[test] POST {pipelined} on /adder in " + str(max_sessions) + " sessions =>", end=' ')
t = []
# Create all sessions
for i in range(max_sessions):
t.append(adder_thread(i, dut, port))
for i in range(len(t)):
t[i].start()
for i in range(len(t)):
t[i].join()
res = True
for i in range(len(t)):
if not test_val("Thread" + str(i) + " Failed", t[i].adder_result(), True):
res = False
t[i].close()
if (res):
Utility.console_log("Success")
return res
def test_examples_ulp_adc(env, extra_data):
dut = env.get_dut('ulp_adc', 'examples/system/ulp_adc')
dut.start_app()
dut.expect_all('Not ULP wakeup', 'Entering deep sleep', timeout=30)
for _ in range(5):
dut.expect('Deep sleep wakeup', timeout=60)
measurements_str = dut.expect(
re.compile(r'ULP did (\d+) measurements'), timeout=5)[0]
assert measurements_str is not None
measurements = int(measurements_str)
Utility.console_log('ULP did {} measurements'.format(measurements))
dut.expect('Thresholds: low=1500 high=2000', timeout=5)
value_str = dut.expect(
re.compile(r'Value=(\d+) was (above|below) threshold'),
timeout=5)[0]
assert value_str is not None
value = int(value_str)
Utility.console_log(
'Value {} was outside the boundaries'.format(value))
dut.expect('Entering deep sleep', timeout=60)
def run_test(self, proto, direction, atten_val):
"""
run test for one type, with specified atten_value and save the test result
:param proto: tcp or udp
:param direction: tx or rx
:param atten_val: attenuate value
"""
rssi = FAILED_TO_SCAN_RSSI
heap_size = INVALID_HEAP_SIZE
try:
server_raw_data, rssi, heap_size = self._test_once(
proto, direction)
throughput = self._save_test_result(
"{}_{}".format(proto, direction), server_raw_data, atten_val,
rssi, heap_size)
Utility.console_log("[{}][{}_{}][{}][{}]: {:.02f}".format(
self.config_name, proto, direction, rssi, self.ap_ssid,
throughput))
except Exception as e:
self._save_test_result("{}_{}".format(proto, direction), "",
atten_val, rssi, heap_size)
Utility.console_log("Failed during test: {}".format(e))
def start_gdb(self):
"""
Runs GDB and connects it to the "serial" port of the DUT.
After this, the DUT expect methods can no longer be used to capture output.
"""
self.stop_receive()
self._port_close()
Utility.console_log('Starting GDB...', 'orange')
self.gdb = GdbController(gdb_path=self.TOOLCHAIN_PREFIX + 'gdb')
# pygdbmi logs to console by default, make it log to a file instead
log_folder = self.app.get_log_folder(TEST_SUITE)
pygdbmi_log_file_name = os.path.join(
log_folder, 'pygdbmi_log_' + self.test_name + '.txt')
pygdbmi_logger = self.gdb.logger
pygdbmi_logger.setLevel(logging.DEBUG)
while pygdbmi_logger.hasHandlers():
pygdbmi_logger.removeHandler(pygdbmi_logger.handlers[0])
log_handler = logging.FileHandler(pygdbmi_log_file_name)
log_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
pygdbmi_logger.addHandler(log_handler)
# Set up logging for GDB remote protocol
gdb_remotelog_file_name = os.path.join(
log_folder, 'gdb_remote_log_' + self.test_name + '.txt')
self.gdb.write('-gdb-set remotelogfile ' + gdb_remotelog_file_name)
# Load the ELF file
self.gdb.write('-file-exec-and-symbols {}'.format(self.app.elf_file))
# Connect GDB to UART
Utility.console_log('Connecting to GDB Stub...', 'orange')
self.gdb.write('-gdb-set serial baud 115200')
responses = self.gdb.write('-target-select remote ' +
self.get_gdb_remote(),
timeout_sec=3)
# Make sure we get the 'stopped' notification
stop_response = self.find_gdb_response('stopped', 'notify', responses)
if not stop_response:
responses = self.gdb.write('-exec-interrupt', timeout_sec=3)
stop_response = self.find_gdb_response('stopped', 'notify',
responses)
assert stop_response
frame = stop_response['payload']['frame']
if 'file' not in frame:
frame['file'] = '?'
if 'line' not in frame:
frame['line'] = '?'
Utility.console_log(
'Stopped in {func} at {addr} ({file}:{line})'.format(**frame),
'orange')
# Drain remaining responses
self.gdb.get_gdb_response(raise_error_on_timeout=False)
def run_multiple_stage_cases(env, extra_data):
"""
extra_data can be 2 types of value
1. as dict: Mandantory keys: "name" and "child case num", optional keys: "reset" and others
3. as list of string or dict:
[case1, case2, case3, {"name": "restart from PRO CPU", "child case num": 2}, ...]
:param env: test env instance
:param extra_data: the case name or case list or case dictionary
:return: None
"""
case_config = format_test_case_config(extra_data)
# we don't want stop on failed case (unless some special scenarios we can't handle)
# this flag is used to log if any of the case failed during executing
# Before exit test function this flag is used to log if the case fails
failed_cases = []
for ut_config in case_config:
Utility.console_log("Running unit test for config: " + ut_config, "O")
dut = env.get_dut("unit-test-app",
app_path=UT_APP_PATH,
app_config_name=ut_config,
allow_dut_exception=True)
if len(case_config[ut_config]) > 0:
replace_app_bin(dut, "unit-test-app",
case_config[ut_config][0].get('app_bin'))
dut.start_app()
for one_case in case_config[ut_config]:
log_test_case("multi-stage test", one_case, ut_config)
performance_items = []
junit_test_case = TinyFW.JunitReport.create_test_case(
"[{}] {}".format(ut_config, one_case["name"]))
try:
run_one_multiple_stage_case(dut, one_case, junit_test_case)
performance_items = dut.get_performance_items()
except TestCaseFailed:
failed_cases.append(format_case_name(one_case))
except Exception as e:
junit_test_case.add_failure_info("Unexpected exception: " +
str(e))
failed_cases.append(format_case_name(one_case))
finally:
TinyFW.JunitReport.update_performance(performance_items)
TinyFW.JunitReport.test_case_finish(junit_test_case)
# close DUT when finish running all cases for one config
env.close_dut(dut.name)
# raise exception if any case fails
if failed_cases:
Utility.console_log("Failed Cases:", color="red")
for _case_name in failed_cases:
Utility.console_log("\t" + _case_name, color="red")
raise AssertionError("Unit Test Failed")
def test_examples_sd_card(
env, extra_data): # type: (ttfw_idf.Env.Env, None ) -> None
dut = env.get_dut('sd_card', 'examples/storage/sd_card/sdmmc')
dut.start_app()
dut.expect('example: Initializing SD card', timeout=20)
dut.expect('example: Using SDMMC peripheral', timeout=10)
# These lines are matched separately because of ASCII color codes in the output
name = dut.expect(re.compile(r'Name: (\w+)'), timeout=10)[0]
_type = dut.expect(re.compile(r'Type: (\S+)'), timeout=10)[0]
speed = dut.expect(re.compile(r'Speed: (\S+)'), timeout=10)[0]
size = dut.expect(re.compile(r'Size: (\S+)'), timeout=10)[0]
Utility.console_log('Card {} {} {}MHz {} found'.format(
name, _type, speed, size))
dut.expect_all('Opening file /sdcard/hello.txt',
'File written',
'Renaming file /sdcard/hello.txt to /sdcard/foo.txt',
'Reading file /sdcard/foo.txt',
"Read from file: 'Hello {}!'".format(name),
'Card unmounted',
timeout=10)
def handleMessage(self):
try:
j = json.loads(self.data)
except Exception as e:
Utility.console_log('Server ignores error: {}'.format(e), 'orange')
return
event = j.get('event')
if event and 'prog' in j and ((event == 'gdb_stub' and 'port' in j) or
(event == 'coredump' and 'file' in j)):
payload = {'event': 'debug_finished'}
self.sendMessage(json.dumps(payload))
Utility.console_log('Server sent: {}'.format(payload))
else:
Utility.console_log('Server received: {}'.format(j), 'orange')
def test_app_loadable_elf(env, extra_data):
rel_project_path = os.path.join('tools', 'test_apps', 'system',
'gdb_loadable_elf')
app_files = ['gdb_loadable_elf.elf']
example = ttfw_idf.LoadableElfTestApp(rel_project_path,
app_files,
target="esp32")
idf_path = example.get_sdk_path()
proj_path = os.path.join(idf_path, rel_project_path)
elf_path = os.path.join(example.binary_path, 'gdb_loadable_elf.elf')
esp_log_path = os.path.join(proj_path, 'esp.log')
with SerialThread(esp_log_path):
openocd_log = os.path.join(proj_path, 'openocd.log')
gdb_log = os.path.join(proj_path, 'gdb.log')
gdb_args = '-x {} --directory={}'.format(
os.path.join(proj_path, '.gdbinit.ci'),
os.path.join(proj_path, 'main'))
with ttfw_idf.OCDProcess(openocd_log), ttfw_idf.GDBProcess(
gdb_log, elf_path, gdb_args) as gdb:
gdb.pexpect_proc.sendline(
''
) # it is for "---Type <return> to continue, or q <return> to quit---"
i = gdb.pexpect_proc.expect_exact([
'Thread 1 hit Temporary breakpoint 2, app_main ()',
'Load failed'
])
if i == 0:
Utility.console_log('gdb is at breakpoint')
elif i == 1:
raise RuntimeError('Load has failed. Please examine the logs.')
else:
Utility.console_log('i = {}'.format(i))
Utility.console_log(str(gdb.pexpect_proc))
# This really should not happen. TIMEOUT and EOF failures are exceptions.
raise RuntimeError(
'An unknown error has occurred. Please examine the logs.')
gdb.pexpect_proc.expect_exact('(gdb)')
gdb.pexpect_proc.sendline('b esp_restart')
gdb.pexpect_proc.sendline('c')
gdb.pexpect_proc.expect_exact(
'Thread 1 hit Breakpoint 3, esp_restart ()')
if pexpect.run('grep "Restarting now." {}'.format(esp_log_path),
withexitstatus=True)[1]:
raise RuntimeError('Expected output from ESP was not received')
def check_time(prev_NY_time, prev_SH_time):
NY_str = dut.expect(re.compile(r'The current date/time in New York is: ({})'.format(TIME_FORMAT_REGEX)))[0]
SH_str = dut.expect(re.compile(r'The current date/time in Shanghai is: ({})'.format(TIME_FORMAT_REGEX)))[0]
Utility.console_log('New York: "{}"'.format(NY_str))
Utility.console_log('Shanghai: "{}"'.format(SH_str))
dut.expect('Entering deep sleep for 10 seconds')
Utility.console_log('Sleeping...')
new_NY_time = datetime.datetime.strptime(NY_str, TIME_FORMAT)
new_SH_time = datetime.datetime.strptime(SH_str, TIME_FORMAT)
# The initial time is not checked because datetime has problems with timezones
assert prev_NY_time is None or new_NY_time - prev_NY_time < TIME_DIFF
assert prev_SH_time is None or new_SH_time - prev_SH_time < TIME_DIFF
return (new_NY_time, new_SH_time)
def blehr_client_task(hr_obj, dut_addr):
interface = 'hci0'
ble_devname = 'blehr_sensor_1.0'
hr_srv_uuid = '180d'
hr_char_uuid = '2a37'
# Get BLE client module
ble_client_obj = lib_ble_client.BLE_Bluez_Client(interface,
devname=ble_devname,
devaddr=dut_addr)
if not ble_client_obj:
raise RuntimeError('Failed to get DBus-Bluez object')
# Discover Bluetooth Adapter and power on
is_adapter_set = ble_client_obj.set_adapter()
if not is_adapter_set:
raise RuntimeError('Adapter Power On failed !!')
# Connect BLE Device
is_connected = ble_client_obj.connect()
if not is_connected:
# Call disconnect to perform cleanup operations before exiting application
ble_client_obj.disconnect()
raise RuntimeError('Connection to device ' + str(ble_devname) +
' failed !!')
# Read Services
services_ret = ble_client_obj.get_services()
if services_ret:
Utility.console_log('\nServices\n')
Utility.console_log(str(services_ret))
else:
ble_client_obj.disconnect()
raise RuntimeError('Failure: Read Services failed')
'''
Blehr application run:
Start Notifications
Retrieve updated value
Stop Notifications
'''
blehr_ret = ble_client_obj.hr_update_simulation(hr_srv_uuid, hr_char_uuid)
if blehr_ret:
Utility.console_log('Success: blehr example test passed')
else:
raise RuntimeError('Failure: blehr example test failed')
# Call disconnect to perform cleanup operations before exiting application
ble_client_obj.disconnect()
def test_phy_multi_init_data_bin(env, _):
# type: (tiny_test_fw.Env.Env, None) -> None
config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*'))
config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files]
for name in config_names:
dut = env.get_dut('phy_multi_init_data_test', 'tools/test_apps/phy/phy_multi_init_data_test',app_config_name=name)
dut.start_app()
if 'CONFIG_ESP_PHY_MULTIPLE_INIT_DATA_BIN_EMBED' in dut.app.get_sdkconfig().keys():
Utility.console_log('multi init data bin embed test')
dut.expect('load embedded multi phy init data')
else:
Utility.console_log('multi init data bin test')
dut.expect('Support multiple PHY init data bins')
dut.expect('wifi_init finished')
Utility.console_log('Test Success')
def __init__(self, proj_path):
cmd = 'openocd -f board/esp32-wrover-kit-3.3v.cfg'
log_file = os.path.join(proj_path, 'openocd.log')
super(OCDProcess, self).__init__(cmd, log_file)
patterns = ['Info : Listening on port 3333 for gdb connections',
'Error: type \'esp32\' is missing virt2phys']
try:
while True:
i = self.p.expect_exact(patterns, timeout=30)
# TIMEOUT or EOF exceptions will be thrown upon other errors
if i == 0:
Utility.console_log('openocd is listening for gdb connections')
break # success
elif i == 1:
Utility.console_log('Ignoring error: "{}"'.format(patterns[i]))
# this error message is ignored because it is not a fatal error
except Exception:
Utility.console_log('openocd initialization has failed', 'R')
raise
def __init__(self, proj_path, elf_path):
cmd = 'xtensa-esp32-elf-gdb -x {} --directory={} {}'.format(os.path.join(proj_path, '.gdbinit.ci'),
os.path.join(proj_path, 'main'),
elf_path)
log_file = os.path.join(proj_path, 'gdb.log')
super(GDBProcess, self).__init__(cmd, log_file)
self.p.sendline('') # it is for "---Type <return> to continue, or q <return> to quit---"
i = self.p.expect_exact(['Thread 1 hit Temporary breakpoint 2, app_main ()',
'Load failed'])
if i == 0:
Utility.console_log('gdb is at breakpoint')
elif i == 1:
raise RuntimeError('Load has failed. Please examine the logs.')
else:
Utility.console_log('i = {}'.format(i))
Utility.console_log(str(self.p))
# This really should not happen. TIMEOUT and EOF failures are exceptions.
raise RuntimeError('An unknown error has occurred. Please examine the logs.')
self.p.expect_exact('(gdb)')
def run_test(self, proto, direction, atten_val): # type: (str, str, int) -> None
"""
run test for one type, with specified atten_value and save the test result
:param proto: tcp or udp
:param direction: tx or rx
:param atten_val: attenuate value
"""
rssi = FAILED_TO_SCAN_RSSI
heap_size = INVALID_HEAP_SIZE
try:
server_raw_data, rssi, heap_size = self._test_once(proto, direction)
throughput = self._save_test_result('{}_{}'.format(proto, direction),
server_raw_data, atten_val,
rssi, heap_size)
Utility.console_log('[{}][{}_{}][{}][{}]: {:.02f}'
.format(self.config_name, proto, direction, rssi, self.ap_ssid, throughput))
self.lowest_rssi_scanned = min(self.lowest_rssi_scanned, rssi)
except (ValueError, IndexError):
self._save_test_result('{}_{}'.format(proto, direction), '', atten_val, rssi, heap_size)
Utility.console_log('Fail to get throughput results.')
except AssertionError:
self.fail_to_scan += 1
Utility.console_log('Fail to scan AP.')
def test_examples_protocol_advanced_https_ota_example_anti_rollback(
env, extra_data):
"""
Working of OTA when anti_rollback is enabled and security version of new image is less than current one.
Application should return with error message in this case.
steps: |
1. join AP
2. Generate binary file with lower security version
3. Fetch OTA image over HTTPS
4. Check working of anti_rollback feature
"""
dut1 = env.get_dut('advanced_https_ota_example',
'examples/system/ota/advanced_https_ota',
dut_class=ttfw_idf.ESP32DUT,
app_config_name='anti_rollback')
Utility.console_log('Erasing the flash on the chip')
# erase the flash
dut1.erase_flash()
server_port = 8001
# Original binary file generated after compilation
bin_name = 'advanced_https_ota.bin'
# Modified firmware image to lower security version in its header. This is to enable negative test case
anti_rollback_bin_name = 'advanced_https_ota_lower_sec_version.bin'
# check and log bin size
binary_file = os.path.join(dut1.app.binary_path, bin_name)
file_size = os.path.getsize(binary_file)
with open(binary_file, 'rb+') as f:
with open(os.path.join(dut1.app.binary_path, anti_rollback_bin_name),
'wb+') as fo:
fo.write(f.read(file_size))
# Change security_version to 0 for negative test case
fo.seek(36)
fo.write(b'\x00')
binary_file = os.path.join(dut1.app.binary_path, anti_rollback_bin_name)
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance('advanced_https_ota_bin_size',
'{}KB'.format(bin_size // 1024))
# start test
host_ip = get_my_ip()
if (get_server_status(host_ip, server_port) is False):
thread1 = multiprocessing.Process(target=start_https_server,
args=(dut1.app.binary_path, host_ip,
server_port))
thread1.daemon = True
thread1.start()
dut1.start_app()
# Positive Case
dut1.expect('Loaded app from partition at offset', timeout=30)
try:
ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'),
timeout=30)
print('Connected to AP with IP: {}'.format(ip_address))
except DUT.ExpectTimeout:
thread1.terminate()
raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
dut1.expect('Starting Advanced OTA example', timeout=30)
# Use originally generated image with secure_version=1
print('writing to device: {}'.format('https://' + host_ip + ':' +
str(server_port) + '/' + bin_name))
dut1.write('https://' + host_ip + ':' + str(server_port) + '/' + bin_name)
dut1.expect('Loaded app from partition at offset', timeout=60)
dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30)
dut1.expect('App is valid, rollback cancelled successfully', 30)
# Negative Case
dut1.expect('Starting Advanced OTA example', timeout=30)
# Use modified image with secure_version=0
print('writing to device: {}'.format('https://' + host_ip + ':' +
str(server_port) + '/' +
anti_rollback_bin_name))
dut1.write('https://' + host_ip + ':' + str(server_port) + '/' +
anti_rollback_bin_name)
dut1.expect(
'New firmware security version is less than eFuse programmed, 0 < 1',
timeout=30)
try:
os.remove(binary_file)
except OSError:
pass
thread1.terminate()
def test_wifi_throughput_vs_rssi(env, extra_data):
"""
steps: |
1. build with best performance config
2. switch on one router
3. set attenuator value from 0-60 for each router
4. test TCP tx rx and UDP tx rx throughput
"""
att_port = env.get_variable('attenuator_port')
ap_list = env.get_variable('ap_list')
pc_nic_ip = env.get_pc_nic_info('pc_nic', 'ipv4')['addr']
apc_ip = env.get_variable('apc_ip')
pc_iperf_log_file = os.path.join(env.log_path, 'pc_iperf_log.md')
test_result = {
'tcp_tx': IperfUtility.TestResult('tcp', 'tx',
BEST_PERFORMANCE_CONFIG),
'tcp_rx': IperfUtility.TestResult('tcp', 'rx',
BEST_PERFORMANCE_CONFIG),
'udp_tx': IperfUtility.TestResult('udp', 'tx',
BEST_PERFORMANCE_CONFIG),
'udp_rx': IperfUtility.TestResult('udp', 'rx',
BEST_PERFORMANCE_CONFIG),
}
# 1. get DUT and download
dut = env.get_dut('iperf',
'examples/wifi/iperf',
app_config_name=BEST_PERFORMANCE_CONFIG)
dut.start_app()
dut.expect_any('iperf>', 'esp32>')
# 2. run test for each required att value
for ap_info in ap_list:
test_utility = IperfUtility.IperfTestUtility(
dut, BEST_PERFORMANCE_CONFIG, ap_info['ssid'], ap_info['password'],
pc_nic_ip, pc_iperf_log_file, test_result)
PowerControl.Control.control_rest(apc_ip, ap_info['outlet'], 'OFF')
PowerControl.Control.control(apc_ip, {ap_info['outlet']: 'ON'})
Attenuator.set_att(att_port, 0)
if not test_utility.wait_ap_power_on():
Utility.console_log(
'[{}] failed to power on, skip testing this AP'.format(
ap_info['ssid']),
color='red')
continue
for atten_val in ATTEN_VALUE_LIST:
assert Attenuator.set_att(att_port, atten_val) is True
try:
test_utility.run_all_cases(atten_val, NO_BANDWIDTH_LIMIT)
except AssertionError:
break
# 3. check test results
env.close_dut('iperf')
# 4. generate report
report = TestReport.ThroughputVsRssiReport(
os.path.join(env.log_path, 'Performance', 'STAThroughputVsRssiReport'),
test_result)
report.generate_report()
def test_wifi_throughput_with_different_configs(env, extra_data):
"""
steps: |
1. build iperf with specified configs
2. test throughput for all routers
"""
pc_nic_ip = env.get_pc_nic_info('pc_nic', 'ipv4')['addr']
pc_iperf_log_file = os.path.join(env.log_path, 'pc_iperf_log.md')
ap_info = {
'ssid': env.get_variable('ap_ssid'),
'password': env.get_variable('ap_password'),
}
config_names_raw = subprocess.check_output(
['ls', os.path.dirname(os.path.abspath(__file__))])
config_names = CONFIG_NAME_PATTERN.findall(config_names_raw)
if not config_names:
raise ValueError('no configs found in {}'.format(
os.path.dirname(__file__)))
test_result = dict()
sdkconfig_files = dict()
for config_name in config_names:
# 1. get the config
sdkconfig_files[config_name] = os.path.join(
os.path.dirname(__file__), 'sdkconfig.ci.{}'.format(config_name))
# 2. get DUT and download
dut = env.get_dut('iperf',
'examples/wifi/iperf',
app_config_name=config_name)
dut.start_app()
dut.expect_any('iperf>', 'esp32>')
# 3. run test for each required att value
test_result[config_name] = {
'tcp_tx': IperfUtility.TestResult('tcp', 'tx', config_name),
'tcp_rx': IperfUtility.TestResult('tcp', 'rx', config_name),
'udp_tx': IperfUtility.TestResult('udp', 'tx', config_name),
'udp_rx': IperfUtility.TestResult('udp', 'rx', config_name),
}
test_utility = IperfUtility.IperfTestUtility(
dut, config_name, ap_info['ssid'], ap_info['password'], pc_nic_ip,
pc_iperf_log_file, test_result[config_name])
for _ in range(RETRY_COUNT_FOR_BEST_PERFORMANCE):
test_utility.run_all_cases(0, NO_BANDWIDTH_LIMIT)
for result_type in test_result[config_name]:
summary = str(test_result[config_name][result_type])
if summary:
Utility.console_log(summary, color='orange')
# 4. check test results
env.close_dut('iperf')
# 5. generate report
report = TestReport.ThroughputForConfigsReport(
os.path.join(env.log_path, 'Performance',
'ThroughputForConfigsReport'), ap_info['ssid'],
test_result, sdkconfig_files)
report.generate_report()
def arbitrary_termination_test(dut, port):
Utility.console_log('[test] Arbitrary termination test =>', end=' ')
cases = [{
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut +
'\r\nCustom: SomeValue\r\n\r\n',
'code': '200',
'header': 'SomeValue'
}, {
'request':
'POST /echo HTTP/1.1\nHost: ' + dut + '\r\nCustom: SomeValue\r\n\r\n',
'code': '200',
'header': 'SomeValue'
}, {
'request':
'POST /echo HTTP/1.1\r\nHost: ' + dut + '\nCustom: SomeValue\r\n\r\n',
'code': '200',
'header': 'SomeValue'
}, {
'request':
'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom: SomeValue\n\r\n',
'code': '200',
'header': 'SomeValue'
}, {
'request':
'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom: SomeValue\r\n\n',
'code': '200',
'header': 'SomeValue'
}, {
'request':
'POST /echo HTTP/1.1\nHost: ' + dut + '\nCustom: SomeValue\n\n',
'code': '200',
'header': 'SomeValue'
}, {
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut +
'\r\nContent-Length: 5\n\r\nABCDE',
'code': '200',
'body': 'ABCDE'
}, {
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut +
'\r\nContent-Length: 5\r\n\nABCDE',
'code': '200',
'body': 'ABCDE'
}, {
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut +
'\r\nContent-Length: 5\n\nABCDE',
'code': '200',
'body': 'ABCDE'
}, {
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut +
'\r\nContent-Length: 5\n\n\rABCD',
'code': '200',
'body': '\rABCD'
}, {
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut +
'\r\r\nCustom: SomeValue\r\r\n\r\r\n',
'code': '400'
}, {
'request': 'POST /echo HTTP/1.1\r\r\nHost: ' + dut + '\r\n\r\n',
'code': '400'
}, {
'request': 'POST /echo HTTP/1.1\r\n\rHost: ' + dut + '\r\n\r\n',
'code': '400'
}, {
'request':
'POST /echo HTTP/1.1\r\nHost: ' + dut + '\rCustom: SomeValue\r\n',
'code': '400'
}, {
'request':
'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom: Some\rValue\r\n',
'code': '400'
}, {
'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut +
'\r\nCustom- SomeValue\r\n\r\n',
'code': '400'
}]
for case in cases:
s = Session(dut, port)
s.client.sendall((case['request']).encode())
resp_hdrs = s.read_resp_hdrs()
resp_body = s.read_resp_data()
s.close()
if not test_val('Response Code', case['code'], s.status):
return False
if 'header' in case.keys():
resp_hdr_val = None
if 'Custom' in resp_hdrs.keys():
resp_hdr_val = resp_hdrs['Custom']
if not test_val('Response Header', case['header'], resp_hdr_val):
return False
def packet_size_limit_test(dut, port, test_size):
Utility.console_log('[test] send size limit test =>', end=' ')
retry = 5
while (retry):
retry -= 1
Utility.console_log('data size = ', test_size)
s = http.client.HTTPConnection(dut + ':' + port, timeout=15)
random_data = ''.join(
string.printable[random.randint(0, len(string.printable)) - 1]
for _ in list(range(test_size)))
path = '/echo'
s.request('POST', url=path, body=random_data)
resp = s.getresponse()
if not test_val('Error', '200', str(resp.status)):
if test_val('Error', '500', str(resp.status)):
Utility.console_log('Data too large to be allocated')
test_size = test_size // 10
else:
Utility.console_log('Unexpected error')
s.close()
Utility.console_log('Retry...')
continue
resp = resp.read().decode()
result = (resp == random_data)
if not result:
test_val('Data size', str(len(random_data)), str(len(resp)))
s.close()
Utility.console_log('Retry...')
continue
s.close()
Utility.console_log('Success')
return True
Utility.console_log('Failed')
return False
s.client.sendall((case['request']).encode())
resp_hdrs = s.read_resp_hdrs()
resp_body = s.read_resp_data()
s.close()
if not test_val('Response Code', case['code'], s.status):
return False
if 'header' in case.keys():
resp_hdr_val = None
if 'Custom' in resp_hdrs.keys():
resp_hdr_val = resp_hdrs['Custom']
if not test_val('Response Header', case['header'], resp_hdr_val):
return False
if 'body' in case.keys():
if not test_val('Response Body', case['body'], resp_body):
return False
Utility.console_log('Success')
return True
def code_500_server_error_test(dut, port):
Utility.console_log('[test] 500 Server Error test =>', end=' ')
s = Session(dut, port)
# Sending a very large content length will cause malloc to fail
content_len = 2**30
s.client.sendall(
('POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: ' +
str(content_len) + '\r\n\r\nABCD').encode())
s.read_resp_hdrs()
s.read_resp_data()
if not test_val('Server Error', '500', s.status):
s.close()
def close(self):
super(IDFDUT, self).close()
if not self.allow_dut_exception and self.get_exceptions():
Utility.console_log("DUT exception detected on {}".format(self),
color="red")
raise IDFDUTException()
def _test_once(self, proto, direction): # type: (Any, str, str) -> Tuple[str, int, int]
""" do measure once for one type """
# connect and scan to get RSSI
dut_ip, rssi = self.setup()
assert direction in ['rx', 'tx']
assert proto in ['tcp', 'udp']
# run iperf test
if direction == 'tx':
with open(PC_IPERF_TEMP_LOG_FILE, 'w') as f:
if proto == 'tcp':
process = subprocess.Popen(['iperf', '-s', '-B', self.pc_nic_ip,
'-t', str(TEST_TIME), '-i', '1', '-f', 'm'],
stdout=f, stderr=f)
self.dut.write('iperf -c {} -i 1 -t {}'.format(self.pc_nic_ip, TEST_TIME))
else:
process = subprocess.Popen(['iperf', '-s', '-u', '-B', self.pc_nic_ip,
'-t', str(TEST_TIME), '-i', '1', '-f', 'm'],
stdout=f, stderr=f)
self.dut.write('iperf -c {} -u -i 1 -t {}'.format(self.pc_nic_ip, TEST_TIME))
for _ in range(TEST_TIMEOUT):
if process.poll() is not None:
break
time.sleep(1)
else:
process.terminate()
with open(PC_IPERF_TEMP_LOG_FILE, 'r') as f:
pc_raw_data = server_raw_data = f.read()
else:
with open(PC_IPERF_TEMP_LOG_FILE, 'w') as f:
if proto == 'tcp':
self.dut.write('iperf -s -i 1 -t {}'.format(TEST_TIME))
# wait until DUT TCP server created
try:
self.dut.expect('iperf tcp server create successfully', timeout=1)
except DUT.ExpectTimeout:
# compatible with old iperf example binary
Utility.console_log('create iperf tcp server fail')
process = subprocess.Popen(['iperf', '-c', dut_ip,
'-t', str(TEST_TIME), '-f', 'm'],
stdout=f, stderr=f)
else:
self.dut.write('iperf -s -u -i 1 -t {}'.format(TEST_TIME))
# wait until DUT TCP server created
try:
self.dut.expect('iperf udp server create successfully', timeout=1)
except DUT.ExpectTimeout:
# compatible with old iperf example binary
Utility.console_log('create iperf udp server fail')
process = subprocess.Popen(['iperf', '-c', dut_ip, '-u', '-b', '100M',
'-t', str(TEST_TIME), '-f', 'm'],
stdout=f, stderr=f)
for _ in range(TEST_TIMEOUT):
if process.poll() is not None:
break
time.sleep(1)
else:
process.terminate()
server_raw_data = self.dut.read()
with open(PC_IPERF_TEMP_LOG_FILE, 'r') as f:
pc_raw_data = f.read()
# save PC iperf logs to console
with open(self.pc_iperf_log_file, 'a+') as f:
f.write('## [{}] `{}`\r\n##### {}'
.format(self.config_name,
'{}_{}'.format(proto, direction),
time.strftime('%m-%d %H:%M:%S', time.localtime(time.time()))))
f.write('\r\n```\r\n\r\n' + pc_raw_data + '\r\n```\r\n')
self.dut.write('heap')
heap_size = self.dut.expect(re.compile(r'min heap size: (\d+)\D'))[0]
# return server raw data (for parsing test results) and RSSI
return server_raw_data, rssi, heap_size
def test_example_app_ble_central(env, extra_data):
"""
Steps:
1. Discover Bluetooth Adapter and Power On
"""
interface = 'hci0'
adv_host_name = "BleCentTestApp"
adv_iface_index = 0
adv_type = 'peripheral'
adv_uuid = '1811'
subprocess.check_output(['rm','-rf','/var/lib/bluetooth/*'])
subprocess.check_output(['hciconfig','hci0','reset'])
# Acquire DUT
dut = env.get_dut("blecent", "examples/bluetooth/nimble/blecent", dut_class=ttfw_idf.ESP32DUT)
# Get binary file
binary_file = os.path.join(dut.app.binary_path, "blecent.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("blecent_bin_size", "{}KB".format(bin_size // 1024))
# Upload binary and start testing
Utility.console_log("Starting blecent example test app")
dut.start_app()
dut.reset()
device_addr = ':'.join(re.findall('..', '%012x' % uuid.getnode()))
# Get BLE client module
ble_client_obj = lib_ble_client.BLE_Bluez_Client(interface)
if not ble_client_obj:
raise RuntimeError("Get DBus-Bluez object failed !!")
# Discover Bluetooth Adapter and power on
is_adapter_set = ble_client_obj.set_adapter()
if not is_adapter_set:
raise RuntimeError("Adapter Power On failed !!")
# Write device address to dut
dut.expect("BLE Host Task Started", timeout=60)
dut.write(device_addr + "\n")
'''
Blecent application run:
Create GATT data
Register GATT Application
Create Advertising data
Register advertisement
Start advertising
'''
ble_client_obj.start_advertising(adv_host_name, adv_iface_index, adv_type, adv_uuid)
# Call disconnect to perform cleanup operations before exiting application
ble_client_obj.disconnect()
# Check dut responses
dut.expect("Connection established", timeout=60)
dut.expect("Service discovery complete; status=0", timeout=60)
print("Service discovery passed\n\tService Discovery Status: 0")
dut.expect("GATT procedure initiated: read;", timeout=60)
dut.expect("Read complete; status=0", timeout=60)
print("Read passed\n\tSupportedNewAlertCategoryCharacteristic\n\tRead Status: 0")
dut.expect("GATT procedure initiated: write;", timeout=60)
dut.expect("Write complete; status=0", timeout=60)
print("Write passed\n\tAlertNotificationControlPointCharacteristic\n\tWrite Status: 0")
dut.expect("GATT procedure initiated: write;", timeout=60)
dut.expect("Subscribe complete; status=0", timeout=60)
print("Subscribe passed\n\tClientCharacteristicConfigurationDescriptor\n\tSubscribe Status: 0")
