def print_aggregated(self, module_name, elem, failures_to_print): tests_executed = len(elem.findall('.//Test')) tests_passed = len(elem.findall('.//Test[@result="pass"]')) tests_failed = len(elem.findall('.//Test[@result="fail"]')) result = '%s_executed pass %s' % (module_name, str(tests_executed)) py_test_lib.add_result(self.result_output_file, result) result = '%s_passed pass %s' % (module_name, str(tests_passed)) py_test_lib.add_result(self.result_output_file, result) failed_result = 'pass' if tests_failed > 0: failed_result = 'fail' result = '%s_failed %s %s' % ( module_name, failed_result, str(tests_failed), ) py_test_lib.add_result(self.result_output_file, result) # output result to show if the module is done or not tests_done = elem.get('done', 'false') if tests_done == 'false': result = '%s_done fail' % module_name else: result = '%s_done pass' % module_name py_test_lib.add_result(self.result_output_file, result) Result = collections.namedtuple( 'Result', ['num_printed_failures', 'failures_skipped']) if failures_to_print == 0: return Result(0, False) # print failed test cases for debug num_printed_failures = 0 test_cases = elem.findall('.//TestCase') for test_case in test_cases: failed_tests = test_case.findall('.//Test[@result="fail"]') for failed_test in failed_tests: if num_printed_failures == failures_to_print: return Result(num_printed_failures, True) test_name = '%s/%s.%s' % ( module_name, test_case.get("name"), failed_test.get("name"), ) failures = failed_test.findall('.//Failure') failure_msg = '' for failure in failures: failure_msg = '%s \n %s' % ( failure_msg, failure.get('message'), ) self.logger.info('%s %s' % (test_name, failure_msg.strip())) num_printed_failures += 1 return Result(num_printed_failures, False)
def print_atomic(self, module_name, elem): test_cases = elem.findall('.//TestCase') for test_case in test_cases: tests = test_case.findall('.//Test') for atomic_test in tests: atomic_test_result = atomic_test.get("result") atomic_test_name = "%s/%s.%s" % ( module_name, test_case.get("name"), atomic_test.get("name"), ) py_test_lib.add_result( self.result_output_file, "%s %s" % (atomic_test_name, atomic_test_result), )
while loop < 20: r = requests.get(url, headers=headers) resp = yaml.load(r.text) currentsha_on_server = resp.get("deviceImage").get("image").get( "hash").get("sha256") if currentsha_on_server == sha: return 0 loop = loop + 1 time.sleep(30) if loop == 10: print "FAIL: Installed sha on device did not match" return -1 if match_sha_on_server(args.installed_sha) == 0: py_test_lib.add_result(RESULT_FILE, "installed-device-sha-match-server pass") r = requests.put(url, data=data, headers=headers) if match_sha_on_server(args.update_sha) == 0: py_test_lib.add_result(RESULT_FILE, "ota-update-to-%s pass" % args.update_sha) print "PASS: %s updated to %s successfully" % (args.devicename, args.update_sha) else: py_test_lib.add_result(RESULT_FILE, "ota-update-to-%s fail" % args.update_sha) print "FAIL: %s update to %s failed" % (args.devicename, args.update_sha) else: py_test_lib.add_result(RESULT_FILE, "installed-device-sha-match-server fail") print "FAIL: Insalled device sha to %s mismatched on the server" % args.devicename
'adb device %s is not available and reconnection attempts failed. Aborting.' % device.serial_or_address) if devices_to_detect: cleanup_and_exit( 1, 'TradeFed did not detect all available devices after %s retries. Aborting.' % tradefed_start_retry_count) logger.info('Starting TradeFed shell test.') try: child.expect(prompt, timeout=60) child.sendline(args.TEST_PARAMS) except pexpect.TIMEOUT: result = 'lunch-tf-shell fail' py_test_lib.add_result(RESULT_FILE, result) retry_check = RetryCheck(args.MAX_NUM_RUNS, args.RUNS_IF_UNCHANGED) # Loop while TradeFed is running. # This loop will rerun TradeFed if requested, until the number of failures stabilizes or a maximum # number of retries is reached. # Meanwhile, try to keep all devices accessible. For remote devices, use handshakes to inform remote # workers that their locally connected device needs to be reset. # The worker host side of the LAVA MultiNode messages is implemented in # wait-and-keep-local-device-accessible.yaml fail_to_complete = False # Assuming TradeFed is started from a clean environment, the first run will have the id 0 # Each retry gets a new session id. tradefed_session_id = 0 result_summary = None
def result_parser(xml_file, result_format): etree_file = open(xml_file, 'rb') etree_content = etree_file.read() rx = re.compile("&#([0-9]+);|&#x([0-9a-fA-F]+);") endpos = len(etree_content) pos = 0 while pos < endpos: # remove characters that don't conform to XML spec m = rx.search(etree_content, pos) if not m: break mstart, mend = m.span() target = m.group(1) if target: num = int(target) else: num = int(m.group(2), 16) # #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] if not (num in (0x9, 0xA, 0xD) or 0x20 <= num <= 0xD7FF or 0xE000 <= num <= 0xFFFD or 0x10000 <= num <= 0x10FFFF): etree_content = etree_content[:mstart] + etree_content[mend:] endpos = len(etree_content) # next time search again from the same position as this time # as the detected pattern was removed here pos = mstart else: # continue from the end of this match pos = mend try: root = ET.fromstring(etree_content) except ET.ParseError as e: logger.error('xml.etree.ElementTree.ParseError: %s' % e) logger.info('Please Check %s manually' % xml_file) sys.exit(1) logger.info('Test modules in %s: %s' % (xml_file, str(len(root.findall('Module'))))) failures_count = 0 for elem in root.findall('Module'): # Naming: Module Name + Test Case Name + Test Name if 'abi' in elem.attrib.keys(): module_name = '.'.join([elem.attrib['abi'], elem.attrib['name']]) else: module_name = elem.attrib['name'] if result_format == AGGREGATED: tests_executed = len(elem.findall('.//Test')) tests_passed = len(elem.findall('.//Test[@result="pass"]')) tests_failed = len(elem.findall('.//Test[@result="fail"]')) result = '%s_executed pass %s' % (module_name, str(tests_executed)) py_test_lib.add_result(RESULT_FILE, result) result = '%s_passed pass %s' % (module_name, str(tests_passed)) py_test_lib.add_result(RESULT_FILE, result) failed_result = 'pass' if tests_failed > 0: failed_result = 'fail' result = '%s_failed %s %s' % (module_name, failed_result, str(tests_failed)) py_test_lib.add_result(RESULT_FILE, result) # output result to show if the module is done or not tests_done = elem.get('done', 'false') if tests_done == 'false': result = '%s_done fail' % module_name else: result = '%s_done pass' % module_name py_test_lib.add_result(RESULT_FILE, result) if args.FAILURES_PRINTED > 0 and failures_count < args.FAILURES_PRINTED: # print failed test cases for debug test_cases = elem.findall('.//TestCase') for test_case in test_cases: failed_tests = test_case.findall('.//Test[@result="fail"]') for failed_test in failed_tests: test_name = '%s/%s.%s' % (module_name, test_case.get("name"), failed_test.get("name")) failures = failed_test.findall('.//Failure') failure_msg = '' for failure in failures: failure_msg = '%s \n %s' % (failure_msg, failure.get('message')) logger.info('%s %s' % (test_name, failure_msg.strip())) failures_count = failures_count + 1 if failures_count > args.FAILURES_PRINTED: logger.info('There are more than %d test cases ' 'failed, the output for the rest ' 'failed test cases will be ' 'skipped.' % (args.FAILURES_PRINTED)) # break the for loop of failed_tests break if failures_count > args.FAILURES_PRINTED: # break the for loop of test_cases break if result_format == ATOMIC: test_cases = elem.findall('.//TestCase') for test_case in test_cases: tests = test_case.findall('.//Test') for atomic_test in tests: atomic_test_result = atomic_test.get("result") atomic_test_name = "%s/%s.%s" % (module_name, test_case.get("name"), atomic_test.get("name")) py_test_lib.add_result( RESULT_FILE, "%s %s" % (atomic_test_name, atomic_test_result))
def result_parser(xml_file, result_format): etree_file = open(xml_file, 'rb') etree_content = etree_file.read() rx = re.compile("&#([0-9]+);|&#x([0-9a-fA-F]+);") endpos = len(etree_content) pos = 0 while pos < endpos: # remove characters that don't conform to XML spec m = rx.search(etree_content, pos) if not m: break mstart, mend = m.span() target = m.group(1) if target: num = int(target) else: num = int(m.group(2), 16) # #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] if not (num in (0x9, 0xA, 0xD) or 0x20 <= num <= 0xD7FF or 0xE000 <= num <= 0xFFFD or 0x10000 <= num <= 0x10FFFF): etree_content = etree_content[:mstart] + etree_content[mend:] endpos = len(etree_content) pos = mend try: root = ET.fromstring(etree_content) except ET.ParseError as e: logger.error('xml.etree.ElementTree.ParseError: %s' % e) logger.info('Please Check %s manually' % xml_file) sys.exit(1) logger.info('Test modules in %s: %s' % (xml_file, str(len(root.findall('Module'))))) for elem in root.findall('Module'): # Naming: Module Name + Test Case Name + Test Name if 'abi' in elem.attrib.keys(): module_name = '.'.join([elem.attrib['abi'], elem.attrib['name']]) else: module_name = elem.attrib['name'] if result_format == AGGREGATED: tests_executed = len(elem.findall('.//Test')) tests_passed = len(elem.findall('.//Test[@result="pass"]')) tests_failed = len(elem.findall('.//Test[@result="fail"]')) result = '%s_executed pass %s' % (module_name, str(tests_executed)) py_test_lib.add_result(RESULT_FILE, result) result = '%s_passed pass %s' % (module_name, str(tests_passed)) py_test_lib.add_result(RESULT_FILE, result) failed_result = 'pass' if tests_failed > 0: failed_result = 'fail' result = '%s_failed %s %s' % (module_name, failed_result, str(tests_failed)) py_test_lib.add_result(RESULT_FILE, result) if result_format == ATOMIC: test_cases = elem.findall('.//TestCase') for test_case in test_cases: tests = test_case.findall('.//Test') for atomic_test in tests: atomic_test_result = atomic_test.get("result") atomic_test_name = "%s/%s.%s" % (module_name, test_case.get("name"), atomic_test.get("name")) py_test_lib.add_result( RESULT_FILE, "%s %s" % (atomic_test_name, atomic_test_result))
command = "android-cts/tools/cts-tradefed run commandAndExit " + args.TEST_PARAMS if args.TEST_PATH == "android-vts": os.environ["VTS_ROOT"] = os.getcwd() command = "android-vts/tools/vts-tradefed run commandAndExit " + args.TEST_PARAMS if command is None: logger.error("Not supported path: %s" % args.TEST_PATH) sys.exit(1) child = subprocess.Popen(shlex.split(command), stderr=subprocess.STDOUT, stdout=tradefed_stdout) fail_to_complete = child.wait() if fail_to_complete: py_test_lib.add_result(RESULT_FILE, "tradefed-test-run fail") else: py_test_lib.add_result(RESULT_FILE, "tradefed-test-run pass") logger.info("Tradefed test finished") tradefed_stdout.close() tradefed_logcat.kill() tradefed_logcat_out.close() # Locate and parse test result. result_dir = "%s/results" % args.TEST_PATH test_result = "test_result.xml" if os.path.exists(result_dir) and os.path.isdir(result_dir): for root, dirs, files in os.walk(result_dir): for name in files: if name == test_result:
def print_aggregated(self, module_name, elem, failures_to_print): tests_executed = len(elem.findall('.//Test')) tests_passed = len(elem.findall('.//Test[@result="pass"]')) tests_failed = len(elem.findall('.//Test[@result="fail"]')) result = '%s_executed pass %s' % (module_name, str(tests_executed)) py_test_lib.add_result(self.result_output_file, result) result = '%s_passed pass %s' % (module_name, str(tests_passed)) py_test_lib.add_result(self.result_output_file, result) failed_result = 'pass' if tests_failed > 0: failed_result = 'fail' result = '%s_failed %s %s' % ( module_name, failed_result, str(tests_failed), ) py_test_lib.add_result(self.result_output_file, result) # output result to show if the module is done or not tests_done = elem.get('done', 'false') if tests_done == 'false': result = '%s_done fail' % module_name else: result = '%s_done pass' % module_name py_test_lib.add_result(self.result_output_file, result) Result = collections.namedtuple( 'Result', ['num_printed_failures', 'failures_skipped'] ) if failures_to_print == 0: return Result(0, False) # print failed test cases for debug num_printed_failures = 0 test_cases = elem.findall('.//TestCase') for test_case in test_cases: failed_tests = test_case.findall('.//Test[@result="fail"]') for failed_test in failed_tests: if num_printed_failures == failures_to_print: return Result(num_printed_failures, True) test_name = '%s/%s.%s' % ( module_name, test_case.get("name"), failed_test.get("name"), ) failures = failed_test.findall('.//Failure') failure_msg = '' for failure in failures: failure_msg = '%s \n %s' % ( failure_msg, failure.get('message'), ) self.logger.info('%s %s' % (test_name, failure_msg.strip())) num_printed_failures += 1 return Result(num_printed_failures, False)
def result_parser(xml_file, result_format): etree_file = open(xml_file, 'rb') etree_content = etree_file.read() rx = re.compile("&#([0-9]+);|&#x([0-9a-fA-F]+);") endpos = len(etree_content) pos = 0 while pos < endpos: # remove characters that don't conform to XML spec m = rx.search(etree_content, pos) if not m: break mstart, mend = m.span() target = m.group(1) if target: num = int(target) else: num = int(m.group(2), 16) # #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] if not(num in (0x9, 0xA, 0xD) or 0x20 <= num <= 0xD7FF or 0xE000 <= num <= 0xFFFD or 0x10000 <= num <= 0x10FFFF): etree_content = etree_content[:mstart] + etree_content[mend:] endpos = len(etree_content) pos = mend try: root = ET.fromstring(etree_content) except ET.ParseError as e: logger.error('xml.etree.ElementTree.ParseError: %s' % e) logger.info('Please Check %s manually' % xml_file) sys.exit(1) logger.info('Test modules in %s: %s' % (xml_file, str(len(root.findall('Module'))))) failures_count = 0 for elem in root.findall('Module'): # Naming: Module Name + Test Case Name + Test Name if 'abi' in elem.attrib.keys(): module_name = '.'.join([elem.attrib['abi'], elem.attrib['name']]) else: module_name = elem.attrib['name'] if result_format == AGGREGATED: tests_executed = len(elem.findall('.//Test')) tests_passed = len(elem.findall('.//Test[@result="pass"]')) tests_failed = len(elem.findall('.//Test[@result="fail"]')) result = '%s_executed pass %s' % (module_name, str(tests_executed)) py_test_lib.add_result(RESULT_FILE, result) result = '%s_passed pass %s' % (module_name, str(tests_passed)) py_test_lib.add_result(RESULT_FILE, result) failed_result = 'pass' if tests_failed > 0: failed_result = 'fail' result = '%s_failed %s %s' % (module_name, failed_result, str(tests_failed)) py_test_lib.add_result(RESULT_FILE, result) # output result to show if the module is done or not tests_done = elem.get('done', 'false') if tests_done == 'false': result = '%s_done fail' % module_name else: result = '%s_done pass' % module_name py_test_lib.add_result(RESULT_FILE, result) if args.FAILURES_PRINTED > 0 and failures_count < args.FAILURES_PRINTED: # print failed test cases for debug test_cases = elem.findall('.//TestCase') for test_case in test_cases: failed_tests = test_case.findall('.//Test[@result="fail"]') for failed_test in failed_tests: test_name = '%s/%s.%s' % (module_name, test_case.get("name"), failed_test.get("name")) failures = failed_test.findall('.//Failure') failure_msg = '' for failure in failures: failure_msg = '%s \n %s' % (failure_msg, failure.get('message')) logger.info('%s %s' % (test_name, failure_msg.strip())) failures_count = failures_count + 1 if failures_count > args.FAILURES_PRINTED: logger.info('There are more than %d test cases ' 'failed, the output for the rest ' 'failed test cases will be ' 'skipped.' % (args.FAILURES_PRINTED)) #break the for loop of failed_tests break if failures_count > args.FAILURES_PRINTED: #break the for loop of test_cases break if result_format == ATOMIC: test_cases = elem.findall('.//TestCase') for test_case in test_cases: tests = test_case.findall('.//Test') for atomic_test in tests: atomic_test_result = atomic_test.get("result") atomic_test_name = "%s/%s.%s" % (module_name, test_case.get("name"), atomic_test.get("name")) py_test_lib.add_result( RESULT_FILE, "%s %s" % (atomic_test_name, atomic_test_result))
} data = json.dumps({"image": {"hash": args.update_sha}}) def match_sha_on_server(sha): loop = 0 while loop < 20: r = requests.get(url, headers=headers) resp = yaml.load(r.text) currentsha_on_server = resp.get("deviceImage").get("image").get("hash").get("sha256") if currentsha_on_server == sha: return 0 loop = loop + 1 time.sleep(30) if loop == 10: print "FAIL: Installed sha on device did not match" return -1 if match_sha_on_server(args.installed_sha) == 0: py_test_lib.add_result(RESULT_FILE, "installed-device-sha-match-server pass") r = requests.put(url, data=data, headers=headers) if match_sha_on_server(args.update_sha) == 0: py_test_lib.add_result(RESULT_FILE, "ota-update-to-%s pass" % args.update_sha) print "PASS: %s updated to %s successfully" % (args.devicename, args.update_sha) else: py_test_lib.add_result(RESULT_FILE, "ota-update-to-%s fail" % args.update_sha) print "FAIL: %s update to %s failed" % (args.devicename, args.update_sha) else: py_test_lib.add_result(RESULT_FILE, "installed-device-sha-match-server fail") print "FAIL: Insalled device sha to %s mismatched on the server" % args.devicename