def test_get_testcase_result(self): r = mbed_test_api.get_testcase_result(self.OUTOUT_CSTRING_TEST) self.assertEqual(len(r), 8) test_case_names = [ 'C strings: %e %E float formatting', 'C strings: %g %g float formatting', 'C strings: %i %d integer formatting', 'C strings: %u %d integer formatting', 'C strings: %x %E integer formatting', 'C strings: strpbrk', 'C strings: strtok' ] for test_case in test_case_names: tc = r[test_case] # If data structure is correct self.assertIn('utest_log', tc) self.assertIn('time_start', tc) self.assertIn('time_end', tc) self.assertIn('failed', tc) self.assertIn('result', tc) self.assertIn('passed', tc) self.assertIn('duration', tc) # values passed self.assertEqual(tc['passed'], 1) self.assertEqual(tc['failed'], 0) self.assertEqual(tc['result_text'], 'OK') # Failing test case tc = r['C strings: %f %f float formatting'] self.assertEqual(tc['passed'], 0) self.assertEqual(tc['failed'], 1) self.assertEqual(tc['result_text'], 'FAIL')
def test_get_testcase_result_tescase_name_and_count(self): r = mbed_test_api.get_testcase_result( self.OUTOUT_GENERIC_TESTS_TESCASE_NAME_AND_COUNT) self.assertEqual(len(r), 4) self.assertIn('Basic', r) self.assertIn('Blinky', r) self.assertIn('C++ heap', r) self.assertIn('C++ stack', r)
def test_get_testcase_result_tescase_name_and_count(self): r = mbed_test_api.get_testcase_result( self.OUTOUT_CSTRING_TEST_CASE_COUNT_AND_NAME) self.assertEqual(len(r), 2) self.assertIn('C strings: strpbrk', r) self.assertIn('C strings: strtok', r) self.assertEqual(r['C strings: strpbrk']['result_text'], 'SKIPPED') self.assertEqual(r['C strings: strtok']['result_text'], 'ERROR')
def test_get_testcase_result_start_tag_missing(self): result = mbed_test_api.get_testcase_result( self.OUTPUT_STARTTAG_MISSING) self.assertEqual(result['DNS query']['utest_log'], "__testcase_start tag not found.")
def run_host_test(image_path, disk, port, build_path, target_id, duration=10, micro=None, reset=None, verbose=False, copy_method=None, program_cycle_s=None, forced_reset_timeout=None, digest_source=None, json_test_cfg=None, max_failed_properties=5, enum_host_tests_path=None, global_resource_mgr=None, fast_model_connection=None, compare_log=None, num_sync_packtes=None, polling_timeout=None, retry_count=1, tags=None, run_app=None): """! This function runs host test supervisor (executes mbedhtrun) and checks output from host test process. @param image_path Path to binary file for flashing @param disk Currently mounted mbed-enabled devices disk (mount point) @param port Currently mounted mbed-enabled devices serial port (console) @param duration Test case timeout @param micro Mbed-enabled device name @param reset Reset type @param forced_reset_timeout Reset timeout (sec) @param verbose Verbose mode flag @param copy_method Copy method type (name) @param program_cycle_s Wait after flashing delay (sec) @param json_test_cfg Additional test configuration file path passed to host tests in JSON format @param max_failed_properties After how many unknown properties we will assume test is not ported @param enum_host_tests_path Directory where locally defined host tests may reside @param num_sync_packtes sync packets to send for host <---> device communication @param polling_timeout Timeout in sec for readiness of mount point and serial port of local or remote device @param tags Filter list of available devices under test to only run on devices with the provided list of tags [tag-filters tag1,tag] @param run_app Run application mode flag (we run application and grab serial port data) @param digest_source if None mbedhtrun will be executed. If 'stdin', stdin will be used via StdInObserver or file (if file name was given as switch option) @return Tuple with test results, test output, test duration times, test case results, and memory metrics. Return int > 0 if running mbedhtrun process failed. Retrun int < 0 if something went wrong during mbedhtrun execution. """ def get_binary_host_tests_dir(binary_path, level=2): """! Checks if in binary test group has host_tests directory @param binary_path Path to binary in test specification @param level How many directories above test host_tests dir exists @return Path to host_tests dir in group binary belongs too, None if not found """ try: binary_path_norm = os.path.normpath(binary_path) current_path_norm = os.path.normpath(os.getcwd()) host_tests_path = binary_path_norm.split( os.sep)[:-level] + ['host_tests'] build_dir_candidates = ['BUILD', '.build'] idx = None for build_dir_candidate in build_dir_candidates: if build_dir_candidate in host_tests_path: idx = host_tests_path.index(build_dir_candidate) break if idx is None: msg = 'The following directories were not in the path: %s' % ( ', '.join(build_dir_candidates)) raise Exception(msg) # Cut /<build dir>/tests/TOOLCHAIN/TARGET host_tests_path = host_tests_path[:idx] + host_tests_path[idx + 4:] host_tests_path = os.sep.join(host_tests_path) except Exception as e: gt_logger.gt_log_warn( "there was a problem while looking for host_tests directory") gt_logger.gt_log_tab("level %d, path: %s" % (level, binary_path)) gt_logger.gt_log_tab(str(e)) return None if os.path.isdir(host_tests_path): return host_tests_path return None if not enum_host_tests_path: # If there is -e specified we will try to find a host_tests path ourselves # # * Path to binary starts from "build" directory, and goes 4 levels # deep: ./build/tests/compiler/toolchain # * Binary is inside test group. # For example: <app>/tests/test_group_name/test_dir/*,cpp. # * We will search for directory called host_tests on the level of test group (level=2) # or on the level of tests directory (level=3). # # If host_tests directory is found above test code will will pass it to mbedhtrun using # switch -e <path_to_host_tests_dir> gt_logger.gt_log( "checking for 'host_tests' directory above image directory structure", print_text=verbose) test_group_ht_path = get_binary_host_tests_dir(image_path, level=2) TESTS_dir_ht_path = get_binary_host_tests_dir(image_path, level=3) if test_group_ht_path: enum_host_tests_path = test_group_ht_path elif TESTS_dir_ht_path: enum_host_tests_path = TESTS_dir_ht_path if enum_host_tests_path: gt_logger.gt_log_tab("found 'host_tests' directory in: '%s'" % enum_host_tests_path, print_text=verbose) else: gt_logger.gt_log_tab( "'host_tests' directory not found: two directory levels above image path checked", print_text=verbose) gt_logger.gt_log("selecting test case observer...", print_text=verbose) if digest_source: gt_logger.gt_log_tab("selected digest source: %s" % digest_source, print_text=verbose) # Select who will digest test case serial port data if digest_source == 'stdin': # When we want to scan stdin for test results raise NotImplementedError elif digest_source is not None: # When we want to open file to scan for test results raise NotImplementedError # Command executing CLI for host test supervisor (in detect-mode) cmd = [ "mbedhtrun", '-m', micro, '-p', port, '-f', '"%s"' % image_path, ] if enum_host_tests_path: cmd += ["-e", '"%s"' % enum_host_tests_path] if global_resource_mgr: # Use global resource manager to execute test # Example: # $ mbedhtrun -p :9600 -f "tests-mbed_drivers-generic_tests.bin" -m K64F --grm raas_client:10.2.203.31:8000 cmd += ['--grm', global_resource_mgr] else: # Use local resources to execute tests # Add extra parameters to host_test if disk: cmd += ["-d", disk] if copy_method: cmd += ["-c", copy_method] if target_id: cmd += ["-t", target_id] if reset: cmd += ["-r", reset] if run_app: cmd += ["--run"] # -f stores binary name! if fast_model_connection: # Use simulator resource manager to execute test # Example: # $ mbedhtrun -f "tests-mbed_drivers-generic_tests.elf" -m FVP_MPS2_M3 --fm DEFAULT cmd += ['--fm', fast_model_connection] if compare_log: cmd += ['--compare-log', compare_log] if program_cycle_s: cmd += ["-C", str(program_cycle_s)] if forced_reset_timeout: cmd += ["-R", str(forced_reset_timeout)] if json_test_cfg: cmd += ["--test-cfg", '"%s"' % str(json_test_cfg)] if num_sync_packtes: cmd += ["--sync", str(num_sync_packtes)] if tags: cmd += ["--tag-filters", tags] if polling_timeout: cmd += ["-P", str(polling_timeout)] gt_logger.gt_log_tab("calling mbedhtrun: %s" % " ".join(cmd), print_text=verbose) gt_logger.gt_log("mbed-host-test-runner: started") for retry in range(1, 1 + retry_count): start_time = time() returncode, htrun_output = run_htrun(cmd, verbose) end_time = time() if returncode < 0: return returncode elif returncode == 0: break gt_logger.gt_log("retry mbedhtrun {}/{}".format(retry, retry_count)) else: gt_logger.gt_log("{} failed after {} count".format(cmd, retry_count)) testcase_duration = end_time - start_time # Test case duration from reset to {end} htrun_output = get_printable_string(htrun_output) result = get_test_result(htrun_output) result_test_cases = get_testcase_result(htrun_output) test_cases_summary = get_testcase_summary(htrun_output) max_heap, reserved_heap, thread_stack_info = get_memory_metrics( htrun_output) thread_stack_summary = [] if thread_stack_info: thread_stack_summary = get_thread_stack_info_summary(thread_stack_info) memory_metrics = { "max_heap": max_heap, "reserved_heap": reserved_heap, "thread_stack_info": thread_stack_info, "thread_stack_summary": thread_stack_summary } get_coverage_data(build_path, htrun_output) gt_logger.gt_log("mbed-host-test-runner: stopped and returned '%s'" % result, print_text=verbose) return (result, htrun_output, testcase_duration, duration, result_test_cases, test_cases_summary, memory_metrics)