def detect_backtrace(self, comp_data): start = 0 while True: match = self.BACKTRACE_PATTERN.search(comp_data, pos=start) if match: start = match.end() Utility.console_log("[Backtrace]:{}".format(match.group(1)), color="red") # translate backtrace addresses = self.BACKTRACE_ADDRESS_PATTERN.findall( match.group(1)) translated_backtrace = "" for addr in addresses: ret = self.dut.lookup_pc_address(addr) if ret: translated_backtrace += ret + "\n" if translated_backtrace: Utility.console_log("Translated backtrace\n:" + translated_backtrace, color="yellow") else: Utility.console_log("Failed to translate backtrace", color="yellow") else: break
def detect_backtrace(self, comp_data): start = 0 while True: match = self.BACKTRACE_PATTERN.search(comp_data, pos=start) if match: start = match.end() Utility.console_log('[Backtrace]:{}'.format(match.group(1)), color='red') # translate backtrace addresses = self.BACKTRACE_ADDRESS_PATTERN.findall( match.group(1)) translated_backtrace = '' for addr in addresses: ret = self.dut.lookup_pc_address(addr) if ret: translated_backtrace += ret + '\n' if translated_backtrace: Utility.console_log('Translated backtrace\n:' + translated_backtrace, color='yellow') else: Utility.console_log('Failed to translate backtrace', color='yellow') else: break
def run_multiple_devices_cases(env, extra_data): """ extra_data can be two types of value 1. as dict: e.g. {"name": "gpio master/slave test example", "child case num": 2, "config": "release", "env_tag": "UT_T2_1"} 2. as list dict: e.g. [{"name": "gpio master/slave test example1", "child case num": 2, "config": "release", "env_tag": "UT_T2_1"}, {"name": "gpio master/slave test example2", "child case num": 2, "config": "release", "env_tag": "UT_T2_1"}] """ failed_cases = [] case_config = format_test_case_config(extra_data, env.default_dut_cls.TARGET) duts = {} for ut_config in case_config: Utility.console_log("Running unit test for config: " + ut_config, "O") for one_case in case_config[ut_config]: log_test_case( "multi-device test", one_case, ut_config, ) result = False junit_test_case = TinyFW.JunitReport.create_test_case( format_case_name(one_case)) try: result = run_one_multiple_devices_case(duts, ut_config, env, one_case, one_case.get('app_bin'), junit_test_case) except TestCaseFailed: pass # result is False, this is handled by the finally block except Exception as e: handle_unexpected_exception(junit_test_case, e) finally: if result: Utility.console_log("Success: " + format_case_name(one_case), color="green") else: failed_cases.append(format_case_name(one_case)) Utility.console_log("Failed: " + format_case_name(one_case), color="red") TinyFW.JunitReport.test_case_finish(junit_test_case) # close all DUTs when finish running all cases for one config for dut in duts: env.close_dut(dut) duts = {}
def test_examples_base_mac_address(env, extra_data): dut = env.get_dut('base_mac_address', 'examples/system/base_mac_address') dut.start_app() dut.expect('BASE_MAC: Base MAC Address read from EFUSE BLK0', timeout=30) hex_r = r', '.join((r'0x([0-9a-f]{1,2})',) * 6) mac_m = dut.expect(re.compile(r'BASE_MAC: Using "' + hex_r + r'" as base MAC address'), timeout=5) Utility.console_log('BASE_MAC detected: {}'.format(':'.join(mac_m))) def get_expected_mac_string(increment): ''' Return the string representation of the MAC address mac_m with the last octet incremented. mac_m is an array of strings in hexa-decimal format without the '0x' prefix. ''' return ', '.join(['0x{}'.format(m) for m in mac_m[:-1]] + [hex(int(mac_m[-1], 16) + increment)]) dut.expect_all('WIFI_STA MAC: ' + get_expected_mac_string(0), 'SoftAP MAC: ' + get_expected_mac_string(1), 'BT MAC: ' + get_expected_mac_string(2), 'Ethernet MAC: ' + get_expected_mac_string(3), timeout=10)
def _call_espcoredump(self, extra_args, coredump_file_name, output_file_name): # no "with" here, since we need the file to be open for later inspection by the test case self.coredump_output = open(output_file_name, "w") espcoredump_script = os.path.join(os.environ["IDF_PATH"], "components", "espcoredump", "espcoredump.py") espcoredump_args = [ sys.executable, espcoredump_script, "info_corefile", "--core", coredump_file_name, ] espcoredump_args += extra_args espcoredump_args.append(self.app.elf_file) Utility.console_log("Running " + " ".join(espcoredump_args)) Utility.console_log("espcoredump output is written to " + self.coredump_output.name) subprocess.check_call(espcoredump_args, stdout=self.coredump_output) self.coredump_output.flush() self.coredump_output.seek(0)
def _call_espcoredump(self, extra_args, coredump_file_name, output_file_name): # no "with" here, since we need the file to be open for later inspection by the test case self.coredump_output = open(output_file_name, 'w') espcoredump_script = os.path.join(os.environ['IDF_PATH'], 'components', 'espcoredump', 'espcoredump.py') espcoredump_args = [ sys.executable, espcoredump_script, 'info_corefile', '--core', coredump_file_name, ] espcoredump_args += extra_args espcoredump_args.append(self.app.elf_file) Utility.console_log('Running ' + ' '.join(espcoredump_args)) Utility.console_log('espcoredump output is written to ' + self.coredump_output.name) subprocess.check_call(espcoredump_args, stdout=self.coredump_output) self.coredump_output.flush() self.coredump_output.seek(0)
def test_examples_protocol_https_request_dynamic_buffers(env, extra_data): # Check for connection using crt bundle with mbedtls dynamic resource enabled dut1 = env.get_dut('https_request_ssldyn', 'examples/protocols/https_request', dut_class=ttfw_idf.ESP32DUT, app_config_name='ssldyn') # check and log bin size Utility.console_log('[app_config_name] - {}'.format(dut1.app.config_name)) binary_file = os.path.join(dut1.app.binary_path, 'https_request.bin') bin_size = os.path.getsize(binary_file) ttfw_idf.log_performance('https_request_bin_size', '{}KB'.format(bin_size // 1024)) # start test dut1.start_app() dut1.expect('Loaded app from partition at offset', timeout=30) try: ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=60) print('Connected to AP with IP: {}'.format(ip_address)) except DUT.ExpectTimeout: raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP') # only check if one connection is established Utility.console_log( "Testing for \"https_request using crt bundle\" with mbedtls dynamic resource enabled" ) try: dut1.expect(re.compile('https_request using crt bundle'), timeout=30) dut1.expect_all('Connection established...', 'Reading HTTP response...', 'HTTP/1.1 200 OK', re.compile('connection closed')) except Exception: Utility.console_log( "Failed the test for \"https_request using crt bundle\" when mbedtls dynamic resource was enabled" ) raise Utility.console_log( "Passed the test for \"https_request using crt bundle\" when mbedtls dynamic resource was enabled" ) # Read free heap size res = dut1.expect(ttfw_idf.MINIMUM_FREE_HEAP_SIZE_RE, timeout=20) if not res: raise ValueError('Maximum heap size info not found') ttfw_idf.print_heap_size('https_request', dut1.app.config_name, dut1.TARGET, res[0]) env.close_dut('https_request_ssldyn')
def parallel_sessions_adder(dut, port, max_sessions): # POSTs on /adder in parallel sessions Utility.console_log("[test] POST {pipelined} on /adder in " + str(max_sessions) + " sessions =>", end=' ') t = [] # Create all sessions for i in range(max_sessions): t.append(adder_thread(i, dut, port)) for i in range(len(t)): t[i].start() for i in range(len(t)): t[i].join() res = True for i in range(len(t)): if not test_val("Thread" + str(i) + " Failed", t[i].adder_result(), True): res = False t[i].close() if (res): Utility.console_log("Success") return res
def test_examples_ulp_adc(env, extra_data): dut = env.get_dut('ulp_adc', 'examples/system/ulp_adc') dut.start_app() dut.expect_all('Not ULP wakeup', 'Entering deep sleep', timeout=30) for _ in range(5): dut.expect('Deep sleep wakeup', timeout=60) measurements_str = dut.expect( re.compile(r'ULP did (\d+) measurements'), timeout=5)[0] assert measurements_str is not None measurements = int(measurements_str) Utility.console_log('ULP did {} measurements'.format(measurements)) dut.expect('Thresholds: low=1500 high=2000', timeout=5) value_str = dut.expect( re.compile(r'Value=(\d+) was (above|below) threshold'), timeout=5)[0] assert value_str is not None value = int(value_str) Utility.console_log( 'Value {} was outside the boundaries'.format(value)) dut.expect('Entering deep sleep', timeout=60)
def run_test(self, proto, direction, atten_val): """ run test for one type, with specified atten_value and save the test result :param proto: tcp or udp :param direction: tx or rx :param atten_val: attenuate value """ rssi = FAILED_TO_SCAN_RSSI heap_size = INVALID_HEAP_SIZE try: server_raw_data, rssi, heap_size = self._test_once( proto, direction) throughput = self._save_test_result( "{}_{}".format(proto, direction), server_raw_data, atten_val, rssi, heap_size) Utility.console_log("[{}][{}_{}][{}][{}]: {:.02f}".format( self.config_name, proto, direction, rssi, self.ap_ssid, throughput)) except Exception as e: self._save_test_result("{}_{}".format(proto, direction), "", atten_val, rssi, heap_size) Utility.console_log("Failed during test: {}".format(e))
def start_gdb(self): """ Runs GDB and connects it to the "serial" port of the DUT. After this, the DUT expect methods can no longer be used to capture output. """ self.stop_receive() self._port_close() Utility.console_log('Starting GDB...', 'orange') self.gdb = GdbController(gdb_path=self.TOOLCHAIN_PREFIX + 'gdb') # pygdbmi logs to console by default, make it log to a file instead log_folder = self.app.get_log_folder(TEST_SUITE) pygdbmi_log_file_name = os.path.join( log_folder, 'pygdbmi_log_' + self.test_name + '.txt') pygdbmi_logger = self.gdb.logger pygdbmi_logger.setLevel(logging.DEBUG) while pygdbmi_logger.hasHandlers(): pygdbmi_logger.removeHandler(pygdbmi_logger.handlers[0]) log_handler = logging.FileHandler(pygdbmi_log_file_name) log_handler.setFormatter( logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) pygdbmi_logger.addHandler(log_handler) # Set up logging for GDB remote protocol gdb_remotelog_file_name = os.path.join( log_folder, 'gdb_remote_log_' + self.test_name + '.txt') self.gdb.write('-gdb-set remotelogfile ' + gdb_remotelog_file_name) # Load the ELF file self.gdb.write('-file-exec-and-symbols {}'.format(self.app.elf_file)) # Connect GDB to UART Utility.console_log('Connecting to GDB Stub...', 'orange') self.gdb.write('-gdb-set serial baud 115200') responses = self.gdb.write('-target-select remote ' + self.get_gdb_remote(), timeout_sec=3) # Make sure we get the 'stopped' notification stop_response = self.find_gdb_response('stopped', 'notify', responses) if not stop_response: responses = self.gdb.write('-exec-interrupt', timeout_sec=3) stop_response = self.find_gdb_response('stopped', 'notify', responses) assert stop_response frame = stop_response['payload']['frame'] if 'file' not in frame: frame['file'] = '?' if 'line' not in frame: frame['line'] = '?' Utility.console_log( 'Stopped in {func} at {addr} ({file}:{line})'.format(**frame), 'orange') # Drain remaining responses self.gdb.get_gdb_response(raise_error_on_timeout=False)
def run_multiple_stage_cases(env, extra_data): """ extra_data can be 2 types of value 1. as dict: Mandantory keys: "name" and "child case num", optional keys: "reset" and others 3. as list of string or dict: [case1, case2, case3, {"name": "restart from PRO CPU", "child case num": 2}, ...] :param env: test env instance :param extra_data: the case name or case list or case dictionary :return: None """ case_config = format_test_case_config(extra_data) # we don't want stop on failed case (unless some special scenarios we can't handle) # this flag is used to log if any of the case failed during executing # Before exit test function this flag is used to log if the case fails failed_cases = [] for ut_config in case_config: Utility.console_log("Running unit test for config: " + ut_config, "O") dut = env.get_dut("unit-test-app", app_path=UT_APP_PATH, app_config_name=ut_config, allow_dut_exception=True) if len(case_config[ut_config]) > 0: replace_app_bin(dut, "unit-test-app", case_config[ut_config][0].get('app_bin')) dut.start_app() for one_case in case_config[ut_config]: log_test_case("multi-stage test", one_case, ut_config) performance_items = [] junit_test_case = TinyFW.JunitReport.create_test_case( "[{}] {}".format(ut_config, one_case["name"])) try: run_one_multiple_stage_case(dut, one_case, junit_test_case) performance_items = dut.get_performance_items() except TestCaseFailed: failed_cases.append(format_case_name(one_case)) except Exception as e: junit_test_case.add_failure_info("Unexpected exception: " + str(e)) failed_cases.append(format_case_name(one_case)) finally: TinyFW.JunitReport.update_performance(performance_items) TinyFW.JunitReport.test_case_finish(junit_test_case) # close DUT when finish running all cases for one config env.close_dut(dut.name) # raise exception if any case fails if failed_cases: Utility.console_log("Failed Cases:", color="red") for _case_name in failed_cases: Utility.console_log("\t" + _case_name, color="red") raise AssertionError("Unit Test Failed")
def test_examples_sd_card( env, extra_data): # type: (ttfw_idf.Env.Env, None ) -> None dut = env.get_dut('sd_card', 'examples/storage/sd_card/sdmmc') dut.start_app() dut.expect('example: Initializing SD card', timeout=20) dut.expect('example: Using SDMMC peripheral', timeout=10) # These lines are matched separately because of ASCII color codes in the output name = dut.expect(re.compile(r'Name: (\w+)'), timeout=10)[0] _type = dut.expect(re.compile(r'Type: (\S+)'), timeout=10)[0] speed = dut.expect(re.compile(r'Speed: (\S+)'), timeout=10)[0] size = dut.expect(re.compile(r'Size: (\S+)'), timeout=10)[0] Utility.console_log('Card {} {} {}MHz {} found'.format( name, _type, speed, size)) dut.expect_all('Opening file /sdcard/hello.txt', 'File written', 'Renaming file /sdcard/hello.txt to /sdcard/foo.txt', 'Reading file /sdcard/foo.txt', "Read from file: 'Hello {}!'".format(name), 'Card unmounted', timeout=10)
def handleMessage(self): try: j = json.loads(self.data) except Exception as e: Utility.console_log('Server ignores error: {}'.format(e), 'orange') return event = j.get('event') if event and 'prog' in j and ((event == 'gdb_stub' and 'port' in j) or (event == 'coredump' and 'file' in j)): payload = {'event': 'debug_finished'} self.sendMessage(json.dumps(payload)) Utility.console_log('Server sent: {}'.format(payload)) else: Utility.console_log('Server received: {}'.format(j), 'orange')
def test_app_loadable_elf(env, extra_data): rel_project_path = os.path.join('tools', 'test_apps', 'system', 'gdb_loadable_elf') app_files = ['gdb_loadable_elf.elf'] example = ttfw_idf.LoadableElfTestApp(rel_project_path, app_files, target="esp32") idf_path = example.get_sdk_path() proj_path = os.path.join(idf_path, rel_project_path) elf_path = os.path.join(example.binary_path, 'gdb_loadable_elf.elf') esp_log_path = os.path.join(proj_path, 'esp.log') with SerialThread(esp_log_path): openocd_log = os.path.join(proj_path, 'openocd.log') gdb_log = os.path.join(proj_path, 'gdb.log') gdb_args = '-x {} --directory={}'.format( os.path.join(proj_path, '.gdbinit.ci'), os.path.join(proj_path, 'main')) with ttfw_idf.OCDProcess(openocd_log), ttfw_idf.GDBProcess( gdb_log, elf_path, gdb_args) as gdb: gdb.pexpect_proc.sendline( '' ) # it is for "---Type <return> to continue, or q <return> to quit---" i = gdb.pexpect_proc.expect_exact([ 'Thread 1 hit Temporary breakpoint 2, app_main ()', 'Load failed' ]) if i == 0: Utility.console_log('gdb is at breakpoint') elif i == 1: raise RuntimeError('Load has failed. Please examine the logs.') else: Utility.console_log('i = {}'.format(i)) Utility.console_log(str(gdb.pexpect_proc)) # This really should not happen. TIMEOUT and EOF failures are exceptions. raise RuntimeError( 'An unknown error has occurred. Please examine the logs.') gdb.pexpect_proc.expect_exact('(gdb)') gdb.pexpect_proc.sendline('b esp_restart') gdb.pexpect_proc.sendline('c') gdb.pexpect_proc.expect_exact( 'Thread 1 hit Breakpoint 3, esp_restart ()') if pexpect.run('grep "Restarting now." {}'.format(esp_log_path), withexitstatus=True)[1]: raise RuntimeError('Expected output from ESP was not received')
def check_time(prev_NY_time, prev_SH_time): NY_str = dut.expect(re.compile(r'The current date/time in New York is: ({})'.format(TIME_FORMAT_REGEX)))[0] SH_str = dut.expect(re.compile(r'The current date/time in Shanghai is: ({})'.format(TIME_FORMAT_REGEX)))[0] Utility.console_log('New York: "{}"'.format(NY_str)) Utility.console_log('Shanghai: "{}"'.format(SH_str)) dut.expect('Entering deep sleep for 10 seconds') Utility.console_log('Sleeping...') new_NY_time = datetime.datetime.strptime(NY_str, TIME_FORMAT) new_SH_time = datetime.datetime.strptime(SH_str, TIME_FORMAT) # The initial time is not checked because datetime has problems with timezones assert prev_NY_time is None or new_NY_time - prev_NY_time < TIME_DIFF assert prev_SH_time is None or new_SH_time - prev_SH_time < TIME_DIFF return (new_NY_time, new_SH_time)
def blehr_client_task(hr_obj, dut_addr): interface = 'hci0' ble_devname = 'blehr_sensor_1.0' hr_srv_uuid = '180d' hr_char_uuid = '2a37' # Get BLE client module ble_client_obj = lib_ble_client.BLE_Bluez_Client(interface, devname=ble_devname, devaddr=dut_addr) if not ble_client_obj: raise RuntimeError('Failed to get DBus-Bluez object') # Discover Bluetooth Adapter and power on is_adapter_set = ble_client_obj.set_adapter() if not is_adapter_set: raise RuntimeError('Adapter Power On failed !!') # Connect BLE Device is_connected = ble_client_obj.connect() if not is_connected: # Call disconnect to perform cleanup operations before exiting application ble_client_obj.disconnect() raise RuntimeError('Connection to device ' + str(ble_devname) + ' failed !!') # Read Services services_ret = ble_client_obj.get_services() if services_ret: Utility.console_log('\nServices\n') Utility.console_log(str(services_ret)) else: ble_client_obj.disconnect() raise RuntimeError('Failure: Read Services failed') ''' Blehr application run: Start Notifications Retrieve updated value Stop Notifications ''' blehr_ret = ble_client_obj.hr_update_simulation(hr_srv_uuid, hr_char_uuid) if blehr_ret: Utility.console_log('Success: blehr example test passed') else: raise RuntimeError('Failure: blehr example test failed') # Call disconnect to perform cleanup operations before exiting application ble_client_obj.disconnect()
def test_phy_multi_init_data_bin(env, _): # type: (tiny_test_fw.Env.Env, None) -> None config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.*')) config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files] for name in config_names: dut = env.get_dut('phy_multi_init_data_test', 'tools/test_apps/phy/phy_multi_init_data_test',app_config_name=name) dut.start_app() if 'CONFIG_ESP_PHY_MULTIPLE_INIT_DATA_BIN_EMBED' in dut.app.get_sdkconfig().keys(): Utility.console_log('multi init data bin embed test') dut.expect('load embedded multi phy init data') else: Utility.console_log('multi init data bin test') dut.expect('Support multiple PHY init data bins') dut.expect('wifi_init finished') Utility.console_log('Test Success')
def __init__(self, proj_path): cmd = 'openocd -f board/esp32-wrover-kit-3.3v.cfg' log_file = os.path.join(proj_path, 'openocd.log') super(OCDProcess, self).__init__(cmd, log_file) patterns = ['Info : Listening on port 3333 for gdb connections', 'Error: type \'esp32\' is missing virt2phys'] try: while True: i = self.p.expect_exact(patterns, timeout=30) # TIMEOUT or EOF exceptions will be thrown upon other errors if i == 0: Utility.console_log('openocd is listening for gdb connections') break # success elif i == 1: Utility.console_log('Ignoring error: "{}"'.format(patterns[i])) # this error message is ignored because it is not a fatal error except Exception: Utility.console_log('openocd initialization has failed', 'R') raise
def __init__(self, proj_path, elf_path): cmd = 'xtensa-esp32-elf-gdb -x {} --directory={} {}'.format(os.path.join(proj_path, '.gdbinit.ci'), os.path.join(proj_path, 'main'), elf_path) log_file = os.path.join(proj_path, 'gdb.log') super(GDBProcess, self).__init__(cmd, log_file) self.p.sendline('') # it is for "---Type <return> to continue, or q <return> to quit---" i = self.p.expect_exact(['Thread 1 hit Temporary breakpoint 2, app_main ()', 'Load failed']) if i == 0: Utility.console_log('gdb is at breakpoint') elif i == 1: raise RuntimeError('Load has failed. Please examine the logs.') else: Utility.console_log('i = {}'.format(i)) Utility.console_log(str(self.p)) # This really should not happen. TIMEOUT and EOF failures are exceptions. raise RuntimeError('An unknown error has occurred. Please examine the logs.') self.p.expect_exact('(gdb)')
def run_test(self, proto, direction, atten_val): # type: (str, str, int) -> None """ run test for one type, with specified atten_value and save the test result :param proto: tcp or udp :param direction: tx or rx :param atten_val: attenuate value """ rssi = FAILED_TO_SCAN_RSSI heap_size = INVALID_HEAP_SIZE try: server_raw_data, rssi, heap_size = self._test_once(proto, direction) throughput = self._save_test_result('{}_{}'.format(proto, direction), server_raw_data, atten_val, rssi, heap_size) Utility.console_log('[{}][{}_{}][{}][{}]: {:.02f}' .format(self.config_name, proto, direction, rssi, self.ap_ssid, throughput)) self.lowest_rssi_scanned = min(self.lowest_rssi_scanned, rssi) except (ValueError, IndexError): self._save_test_result('{}_{}'.format(proto, direction), '', atten_val, rssi, heap_size) Utility.console_log('Fail to get throughput results.') except AssertionError: self.fail_to_scan += 1 Utility.console_log('Fail to scan AP.')
def test_examples_protocol_advanced_https_ota_example_anti_rollback( env, extra_data): """ Working of OTA when anti_rollback is enabled and security version of new image is less than current one. Application should return with error message in this case. steps: | 1. join AP 2. Generate binary file with lower security version 3. Fetch OTA image over HTTPS 4. Check working of anti_rollback feature """ dut1 = env.get_dut('advanced_https_ota_example', 'examples/system/ota/advanced_https_ota', dut_class=ttfw_idf.ESP32DUT, app_config_name='anti_rollback') Utility.console_log('Erasing the flash on the chip') # erase the flash dut1.erase_flash() server_port = 8001 # Original binary file generated after compilation bin_name = 'advanced_https_ota.bin' # Modified firmware image to lower security version in its header. This is to enable negative test case anti_rollback_bin_name = 'advanced_https_ota_lower_sec_version.bin' # check and log bin size binary_file = os.path.join(dut1.app.binary_path, bin_name) file_size = os.path.getsize(binary_file) with open(binary_file, 'rb+') as f: with open(os.path.join(dut1.app.binary_path, anti_rollback_bin_name), 'wb+') as fo: fo.write(f.read(file_size)) # Change security_version to 0 for negative test case fo.seek(36) fo.write(b'\x00') binary_file = os.path.join(dut1.app.binary_path, anti_rollback_bin_name) bin_size = os.path.getsize(binary_file) ttfw_idf.log_performance('advanced_https_ota_bin_size', '{}KB'.format(bin_size // 1024)) # start test host_ip = get_my_ip() if (get_server_status(host_ip, server_port) is False): thread1 = multiprocessing.Process(target=start_https_server, args=(dut1.app.binary_path, host_ip, server_port)) thread1.daemon = True thread1.start() dut1.start_app() # Positive Case dut1.expect('Loaded app from partition at offset', timeout=30) try: ip_address = dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30) print('Connected to AP with IP: {}'.format(ip_address)) except DUT.ExpectTimeout: thread1.terminate() raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP') dut1.expect('Starting Advanced OTA example', timeout=30) # Use originally generated image with secure_version=1 print('writing to device: {}'.format('https://' + host_ip + ':' + str(server_port) + '/' + bin_name)) dut1.write('https://' + host_ip + ':' + str(server_port) + '/' + bin_name) dut1.expect('Loaded app from partition at offset', timeout=60) dut1.expect(re.compile(r' (sta|eth) ip: ([^,]+),'), timeout=30) dut1.expect('App is valid, rollback cancelled successfully', 30) # Negative Case dut1.expect('Starting Advanced OTA example', timeout=30) # Use modified image with secure_version=0 print('writing to device: {}'.format('https://' + host_ip + ':' + str(server_port) + '/' + anti_rollback_bin_name)) dut1.write('https://' + host_ip + ':' + str(server_port) + '/' + anti_rollback_bin_name) dut1.expect( 'New firmware security version is less than eFuse programmed, 0 < 1', timeout=30) try: os.remove(binary_file) except OSError: pass thread1.terminate()
def test_wifi_throughput_vs_rssi(env, extra_data): """ steps: | 1. build with best performance config 2. switch on one router 3. set attenuator value from 0-60 for each router 4. test TCP tx rx and UDP tx rx throughput """ att_port = env.get_variable('attenuator_port') ap_list = env.get_variable('ap_list') pc_nic_ip = env.get_pc_nic_info('pc_nic', 'ipv4')['addr'] apc_ip = env.get_variable('apc_ip') pc_iperf_log_file = os.path.join(env.log_path, 'pc_iperf_log.md') test_result = { 'tcp_tx': IperfUtility.TestResult('tcp', 'tx', BEST_PERFORMANCE_CONFIG), 'tcp_rx': IperfUtility.TestResult('tcp', 'rx', BEST_PERFORMANCE_CONFIG), 'udp_tx': IperfUtility.TestResult('udp', 'tx', BEST_PERFORMANCE_CONFIG), 'udp_rx': IperfUtility.TestResult('udp', 'rx', BEST_PERFORMANCE_CONFIG), } # 1. get DUT and download dut = env.get_dut('iperf', 'examples/wifi/iperf', app_config_name=BEST_PERFORMANCE_CONFIG) dut.start_app() dut.expect_any('iperf>', 'esp32>') # 2. run test for each required att value for ap_info in ap_list: test_utility = IperfUtility.IperfTestUtility( dut, BEST_PERFORMANCE_CONFIG, ap_info['ssid'], ap_info['password'], pc_nic_ip, pc_iperf_log_file, test_result) PowerControl.Control.control_rest(apc_ip, ap_info['outlet'], 'OFF') PowerControl.Control.control(apc_ip, {ap_info['outlet']: 'ON'}) Attenuator.set_att(att_port, 0) if not test_utility.wait_ap_power_on(): Utility.console_log( '[{}] failed to power on, skip testing this AP'.format( ap_info['ssid']), color='red') continue for atten_val in ATTEN_VALUE_LIST: assert Attenuator.set_att(att_port, atten_val) is True try: test_utility.run_all_cases(atten_val, NO_BANDWIDTH_LIMIT) except AssertionError: break # 3. check test results env.close_dut('iperf') # 4. generate report report = TestReport.ThroughputVsRssiReport( os.path.join(env.log_path, 'Performance', 'STAThroughputVsRssiReport'), test_result) report.generate_report()
def test_wifi_throughput_with_different_configs(env, extra_data): """ steps: | 1. build iperf with specified configs 2. test throughput for all routers """ pc_nic_ip = env.get_pc_nic_info('pc_nic', 'ipv4')['addr'] pc_iperf_log_file = os.path.join(env.log_path, 'pc_iperf_log.md') ap_info = { 'ssid': env.get_variable('ap_ssid'), 'password': env.get_variable('ap_password'), } config_names_raw = subprocess.check_output( ['ls', os.path.dirname(os.path.abspath(__file__))]) config_names = CONFIG_NAME_PATTERN.findall(config_names_raw) if not config_names: raise ValueError('no configs found in {}'.format( os.path.dirname(__file__))) test_result = dict() sdkconfig_files = dict() for config_name in config_names: # 1. get the config sdkconfig_files[config_name] = os.path.join( os.path.dirname(__file__), 'sdkconfig.ci.{}'.format(config_name)) # 2. get DUT and download dut = env.get_dut('iperf', 'examples/wifi/iperf', app_config_name=config_name) dut.start_app() dut.expect_any('iperf>', 'esp32>') # 3. run test for each required att value test_result[config_name] = { 'tcp_tx': IperfUtility.TestResult('tcp', 'tx', config_name), 'tcp_rx': IperfUtility.TestResult('tcp', 'rx', config_name), 'udp_tx': IperfUtility.TestResult('udp', 'tx', config_name), 'udp_rx': IperfUtility.TestResult('udp', 'rx', config_name), } test_utility = IperfUtility.IperfTestUtility( dut, config_name, ap_info['ssid'], ap_info['password'], pc_nic_ip, pc_iperf_log_file, test_result[config_name]) for _ in range(RETRY_COUNT_FOR_BEST_PERFORMANCE): test_utility.run_all_cases(0, NO_BANDWIDTH_LIMIT) for result_type in test_result[config_name]: summary = str(test_result[config_name][result_type]) if summary: Utility.console_log(summary, color='orange') # 4. check test results env.close_dut('iperf') # 5. generate report report = TestReport.ThroughputForConfigsReport( os.path.join(env.log_path, 'Performance', 'ThroughputForConfigsReport'), ap_info['ssid'], test_result, sdkconfig_files) report.generate_report()
def arbitrary_termination_test(dut, port): Utility.console_log('[test] Arbitrary termination test =>', end=' ') cases = [{ 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom: SomeValue\r\n\r\n', 'code': '200', 'header': 'SomeValue' }, { 'request': 'POST /echo HTTP/1.1\nHost: ' + dut + '\r\nCustom: SomeValue\r\n\r\n', 'code': '200', 'header': 'SomeValue' }, { 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\nCustom: SomeValue\r\n\r\n', 'code': '200', 'header': 'SomeValue' }, { 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom: SomeValue\n\r\n', 'code': '200', 'header': 'SomeValue' }, { 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom: SomeValue\r\n\n', 'code': '200', 'header': 'SomeValue' }, { 'request': 'POST /echo HTTP/1.1\nHost: ' + dut + '\nCustom: SomeValue\n\n', 'code': '200', 'header': 'SomeValue' }, { 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: 5\n\r\nABCDE', 'code': '200', 'body': 'ABCDE' }, { 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: 5\r\n\nABCDE', 'code': '200', 'body': 'ABCDE' }, { 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: 5\n\nABCDE', 'code': '200', 'body': 'ABCDE' }, { 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: 5\n\n\rABCD', 'code': '200', 'body': '\rABCD' }, { 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\r\nCustom: SomeValue\r\r\n\r\r\n', 'code': '400' }, { 'request': 'POST /echo HTTP/1.1\r\r\nHost: ' + dut + '\r\n\r\n', 'code': '400' }, { 'request': 'POST /echo HTTP/1.1\r\n\rHost: ' + dut + '\r\n\r\n', 'code': '400' }, { 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\rCustom: SomeValue\r\n', 'code': '400' }, { 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom: Some\rValue\r\n', 'code': '400' }, { 'request': 'POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nCustom- SomeValue\r\n\r\n', 'code': '400' }] for case in cases: s = Session(dut, port) s.client.sendall((case['request']).encode()) resp_hdrs = s.read_resp_hdrs() resp_body = s.read_resp_data() s.close() if not test_val('Response Code', case['code'], s.status): return False if 'header' in case.keys(): resp_hdr_val = None if 'Custom' in resp_hdrs.keys(): resp_hdr_val = resp_hdrs['Custom'] if not test_val('Response Header', case['header'], resp_hdr_val): return False
def packet_size_limit_test(dut, port, test_size): Utility.console_log('[test] send size limit test =>', end=' ') retry = 5 while (retry): retry -= 1 Utility.console_log('data size = ', test_size) s = http.client.HTTPConnection(dut + ':' + port, timeout=15) random_data = ''.join( string.printable[random.randint(0, len(string.printable)) - 1] for _ in list(range(test_size))) path = '/echo' s.request('POST', url=path, body=random_data) resp = s.getresponse() if not test_val('Error', '200', str(resp.status)): if test_val('Error', '500', str(resp.status)): Utility.console_log('Data too large to be allocated') test_size = test_size // 10 else: Utility.console_log('Unexpected error') s.close() Utility.console_log('Retry...') continue resp = resp.read().decode() result = (resp == random_data) if not result: test_val('Data size', str(len(random_data)), str(len(resp))) s.close() Utility.console_log('Retry...') continue s.close() Utility.console_log('Success') return True Utility.console_log('Failed') return False
s.client.sendall((case['request']).encode()) resp_hdrs = s.read_resp_hdrs() resp_body = s.read_resp_data() s.close() if not test_val('Response Code', case['code'], s.status): return False if 'header' in case.keys(): resp_hdr_val = None if 'Custom' in resp_hdrs.keys(): resp_hdr_val = resp_hdrs['Custom'] if not test_val('Response Header', case['header'], resp_hdr_val): return False if 'body' in case.keys(): if not test_val('Response Body', case['body'], resp_body): return False Utility.console_log('Success') return True def code_500_server_error_test(dut, port): Utility.console_log('[test] 500 Server Error test =>', end=' ') s = Session(dut, port) # Sending a very large content length will cause malloc to fail content_len = 2**30 s.client.sendall( ('POST /echo HTTP/1.1\r\nHost: ' + dut + '\r\nContent-Length: ' + str(content_len) + '\r\n\r\nABCD').encode()) s.read_resp_hdrs() s.read_resp_data() if not test_val('Server Error', '500', s.status): s.close()
def close(self): super(IDFDUT, self).close() if not self.allow_dut_exception and self.get_exceptions(): Utility.console_log("DUT exception detected on {}".format(self), color="red") raise IDFDUTException()
def _test_once(self, proto, direction): # type: (Any, str, str) -> Tuple[str, int, int] """ do measure once for one type """ # connect and scan to get RSSI dut_ip, rssi = self.setup() assert direction in ['rx', 'tx'] assert proto in ['tcp', 'udp'] # run iperf test if direction == 'tx': with open(PC_IPERF_TEMP_LOG_FILE, 'w') as f: if proto == 'tcp': process = subprocess.Popen(['iperf', '-s', '-B', self.pc_nic_ip, '-t', str(TEST_TIME), '-i', '1', '-f', 'm'], stdout=f, stderr=f) self.dut.write('iperf -c {} -i 1 -t {}'.format(self.pc_nic_ip, TEST_TIME)) else: process = subprocess.Popen(['iperf', '-s', '-u', '-B', self.pc_nic_ip, '-t', str(TEST_TIME), '-i', '1', '-f', 'm'], stdout=f, stderr=f) self.dut.write('iperf -c {} -u -i 1 -t {}'.format(self.pc_nic_ip, TEST_TIME)) for _ in range(TEST_TIMEOUT): if process.poll() is not None: break time.sleep(1) else: process.terminate() with open(PC_IPERF_TEMP_LOG_FILE, 'r') as f: pc_raw_data = server_raw_data = f.read() else: with open(PC_IPERF_TEMP_LOG_FILE, 'w') as f: if proto == 'tcp': self.dut.write('iperf -s -i 1 -t {}'.format(TEST_TIME)) # wait until DUT TCP server created try: self.dut.expect('iperf tcp server create successfully', timeout=1) except DUT.ExpectTimeout: # compatible with old iperf example binary Utility.console_log('create iperf tcp server fail') process = subprocess.Popen(['iperf', '-c', dut_ip, '-t', str(TEST_TIME), '-f', 'm'], stdout=f, stderr=f) else: self.dut.write('iperf -s -u -i 1 -t {}'.format(TEST_TIME)) # wait until DUT TCP server created try: self.dut.expect('iperf udp server create successfully', timeout=1) except DUT.ExpectTimeout: # compatible with old iperf example binary Utility.console_log('create iperf udp server fail') process = subprocess.Popen(['iperf', '-c', dut_ip, '-u', '-b', '100M', '-t', str(TEST_TIME), '-f', 'm'], stdout=f, stderr=f) for _ in range(TEST_TIMEOUT): if process.poll() is not None: break time.sleep(1) else: process.terminate() server_raw_data = self.dut.read() with open(PC_IPERF_TEMP_LOG_FILE, 'r') as f: pc_raw_data = f.read() # save PC iperf logs to console with open(self.pc_iperf_log_file, 'a+') as f: f.write('## [{}] `{}`\r\n##### {}' .format(self.config_name, '{}_{}'.format(proto, direction), time.strftime('%m-%d %H:%M:%S', time.localtime(time.time())))) f.write('\r\n```\r\n\r\n' + pc_raw_data + '\r\n```\r\n') self.dut.write('heap') heap_size = self.dut.expect(re.compile(r'min heap size: (\d+)\D'))[0] # return server raw data (for parsing test results) and RSSI return server_raw_data, rssi, heap_size
def test_example_app_ble_central(env, extra_data): """ Steps: 1. Discover Bluetooth Adapter and Power On """ interface = 'hci0' adv_host_name = "BleCentTestApp" adv_iface_index = 0 adv_type = 'peripheral' adv_uuid = '1811' subprocess.check_output(['rm','-rf','/var/lib/bluetooth/*']) subprocess.check_output(['hciconfig','hci0','reset']) # Acquire DUT dut = env.get_dut("blecent", "examples/bluetooth/nimble/blecent", dut_class=ttfw_idf.ESP32DUT) # Get binary file binary_file = os.path.join(dut.app.binary_path, "blecent.bin") bin_size = os.path.getsize(binary_file) ttfw_idf.log_performance("blecent_bin_size", "{}KB".format(bin_size // 1024)) # Upload binary and start testing Utility.console_log("Starting blecent example test app") dut.start_app() dut.reset() device_addr = ':'.join(re.findall('..', '%012x' % uuid.getnode())) # Get BLE client module ble_client_obj = lib_ble_client.BLE_Bluez_Client(interface) if not ble_client_obj: raise RuntimeError("Get DBus-Bluez object failed !!") # Discover Bluetooth Adapter and power on is_adapter_set = ble_client_obj.set_adapter() if not is_adapter_set: raise RuntimeError("Adapter Power On failed !!") # Write device address to dut dut.expect("BLE Host Task Started", timeout=60) dut.write(device_addr + "\n") ''' Blecent application run: Create GATT data Register GATT Application Create Advertising data Register advertisement Start advertising ''' ble_client_obj.start_advertising(adv_host_name, adv_iface_index, adv_type, adv_uuid) # Call disconnect to perform cleanup operations before exiting application ble_client_obj.disconnect() # Check dut responses dut.expect("Connection established", timeout=60) dut.expect("Service discovery complete; status=0", timeout=60) print("Service discovery passed\n\tService Discovery Status: 0") dut.expect("GATT procedure initiated: read;", timeout=60) dut.expect("Read complete; status=0", timeout=60) print("Read passed\n\tSupportedNewAlertCategoryCharacteristic\n\tRead Status: 0") dut.expect("GATT procedure initiated: write;", timeout=60) dut.expect("Write complete; status=0", timeout=60) print("Write passed\n\tAlertNotificationControlPointCharacteristic\n\tWrite Status: 0") dut.expect("GATT procedure initiated: write;", timeout=60) dut.expect("Subscribe complete; status=0", timeout=60) print("Subscribe passed\n\tClientCharacteristicConfigurationDescriptor\n\tSubscribe Status: 0")