def get_scan_result_from_module(xml_path): """ Gathers ScanResult information from an already existing XML file. It does two checks: 1. Check the testsuites element for an ErrorCode property, if present then set ScanResult to that error code 2. If there is no ErrorCode, determine if tests passed or failed based on attributes in testsuites :param xml_path: path to the XML file to read :return: ScanResult with the appropriate data or None if file cannot be read """ try: with open(xml_path, 'r') as fx: tree = ElementTree.parse(fx) except IOError: return None testsuites = tree.getroot() if testsuites is None: return None # Default to success code return_code = 0 # Search for the ErrorCode property properties = testsuites.find("properties") if properties is not None: for property in properties.findall("property"): if property.get('name') == "ErrorCode": return_code = int(property.get('value')) # If no error code yet, check for failures if not return_code: failures = int(testsuites.get('failures') or 0) if failures: return_code = 1 path = get_module_name_from_xml_filename(xml_path) error_msg = RunnerReturnCodes.to_string(return_code) return ScanResult(path=path, xml_path=xml_path, return_code=return_code, error_msg=error_msg)
def scan_one(args, extra, type_, scanner, runner_path, bootstrap_config, file_name, output_dir): """ Scan one module or executable :param args: command line arguments :param extra: extra parameters :param int type_: module or executable :param scanner: platform-specific scanner instance :param runner_path: path to test runner executable :param BootstrapConfig bootstrap_config: configuration object for bootstrapping modules :param file_name: filename of module to scan :param output_dir: directory for output :return: ScannerResult """ logger.info("{}: {}".format(type_, file_name)) xml_out = create_xml_output_filename(file_name, output_dir) if os.path.exists(xml_out): return # module has already been tested # for a more exhaustive list of options: # https://github.com/google/googletest/blob/master/googletest/docs/AdvancedGuide.md#running-a-subset-of-the-tests # --help lists command options # --gtest_list_tests just list what tests are in the module # --gtest_shuffle shuffle test ordering cmd_args = ["--gtest_output=xml:" + xml_out, "--gtest_color=yes"] cmd_args += extra if args.wait_for_debugger: # user has requested to attach a debugger when running cmd_args += ["--wait-for-debugger"] ret = 0 if type_ == ModuleType.LIBRARY: if args.integ: # user wants to run integration tests export_symbol = __module_integ_export_symbol__ cmd_args += ["--integ"] else: # just run unit tests export_symbol = __module_unit_export_symbol__ # run with bootstrapper ran_with_bootstrapper = False if bootstrap_config: module_name = os.path.split(file_name)[1] bootstrapper = bootstrap_config.get_bootstrapper(module_name) if bootstrapper: ran_with_bootstrapper = True try: working_dir = args.dir app = os.path.join(args.dir, bootstrapper.command_line[0]) if not os.path.isfile(app): logger.error( "bootstrap executable not found {}".format(app)) full_command_line = bootstrapper.command_line + tuple( cmd_args) ret = scanner.bootstrap(working_dir, full_command_line) except: ret = RunnerReturnCodes.UNEXPECTED_EXCEPTION logger.exception("bootstrap failed") # run with "runner_<platform>" as the implicit bootstrapper (no need to specify this # in the bootstrapper config file) if not ran_with_bootstrapper: if scanner.exports_symbol(file_name, export_symbol): try: ret = scanner.call(file_name, export_symbol, runner_path, args=cmd_args) except KeyboardInterrupt: raise except: ret = RunnerReturnCodes.UNEXPECTED_EXCEPTION logger.exception("module call failed") else: ret = RunnerReturnCodes.SYMBOL_NOT_FOUND elif type_ == ModuleType.EXECUTABLE: if scanner.exports_symbol(file_name, __executable_export_symbol__): try: cmd_args = ["--unittest"] + cmd_args ret = scanner.run(file_name, args=cmd_args) except KeyboardInterrupt: raise except: ret = RunnerReturnCodes.UNEXPECTED_EXCEPTION logger.exception("executable run failed") else: logger.error("Executable does not export correct symbol.") ret = RunnerReturnCodes.SYMBOL_NOT_FOUND else: raise NotImplementedError("module type not supported: " + str(type_)) err = RunnerReturnCodes.to_string(ret) return ScanResult(path=file_name, return_code=ret, xml_path=xml_out, error_msg=err)
def scan(args, extra): scanner = Scanner() output_dir = create_output_directory(args.output_path, args.no_timestamp) # setup logging setup_logging(os.path.join(output_dir, "aztest.log"), args.verbosity) logger.info("AZ Test Scanner") if not args.runner_path: runner_path = os.path.abspath( os.path.join(args.dir, scanner.__runner_exe__)) else: runner_path = os.path.abspath(args.runner_path) if not os.path.exists(runner_path): logger.exception("Invalid test runner path: {}".format(runner_path)) return bootstrap_config = None if args.bootstrap_config: with open(args.bootstrap_config) as json_file: bootstrap_config = BootstrapConfig(flatten=True) bootstrap_config.load(json.load(json_file)) add_dirs_to_path(args.add_path) scan_results = [] # list of ScanResult() # Find default filter files if they exist and add to user-defined lists whitelist_files = (args.whitelist_files if args.whitelist_files else []) + [get_default_whitelist()] blacklist_files = (args.blacklist_files if args.blacklist_files else []) + [get_default_blacklist()] # Create a FileApprover to determine if scanned files can be tested file_approver = FileApprover(whitelist_files, blacklist_files) module_failures = 0 # Dynamic Libraries / Modules if not __no_dll__: logger.info("Scanning for dynamic libraries") for file_name in scanner.enumerate_modules(args.dir): try: if args.limit and len(scan_results) >= args.limit: continue # reached scanning limit if args.only and not FileApprover.is_in_list( file_name, args.only.split(',')): continue # filename does not match any expected pattern if not file_approver.is_approved(file_name): continue result = scan_one(args, extra, ModuleType.LIBRARY, scanner, runner_path, bootstrap_config, file_name, output_dir) if result: scan_results += [result] if result.return_code != RunnerReturnCodes.TESTS_SUCCEEDED: logger.error( "Module FAILED: {}, with exit code: {} ({})". format( file_name, result.return_code, RunnerReturnCodes.to_string( result.return_code))) module_failures += 1 if not os.path.exists(result.xml_path): XMLGenerator.create_xml_output_file( result.xml_path, result.return_code, result.error_msg) except KeyboardInterrupt: logger.exception("Process interrupted by user.") break except: logger.exception("Module scan failed.") # Executables if args.exe: logger.info("Scanning for executables") for file_name in scanner.enumerate_executables(args.dir): if args.limit and len(scan_results) >= args.limit: continue # reached scanning limit if args.only and not FileApprover.is_in_list( file_name, args.only.split(',')): continue # filename does not match any expected pattern if not file_approver.is_approved(file_name): continue result = scan_one(args, extra, ModuleType.EXECUTABLE, scanner, runner_path, bootstrap_config, file_name, output_dir) if result: scan_results += [result] if result.return_code != RunnerReturnCodes.TESTS_SUCCEEDED: logger.error( "Module FAILED: {}, with exit code: {} ({})".format( file_name, result.return_code, RunnerReturnCodes.to_string(result.return_code))) module_failures += 1 if not os.path.exists(result.xml_path): XMLGenerator.create_xml_output_file( result.xml_path, result.return_code, result.error_msg) # Always save ScanResult data in a JSON file so we have access to it later scan_results_json = {'scan_results': []} for scan_result in scan_results: scan_results_json['scan_results'].append(scan_result._asdict()) json_path = os.path.join(output_dir, 'scan_results.json') with open(json_path, 'w') as f: json.dump(scan_results_json, f) if not args.no_html_report: # Convert the set of XML files into an HTML report HTMLReporter.create_html_report(scan_results, output_dir) HTMLReporter.create_html_failure_report(scan_results, output_dir) return 1 if module_failures > 0 else 0
def scan(args, extra): scanner = Scanner() output_dir = create_output_directory(args.output_path, args.no_timestamp) # setup logging log_path = os.path.join(output_dir, "aztest.log") setup_logging(log_path, args.verbosity) logger.info("AZ Test Scanner") if not args.runner_path: runner_path = os.path.abspath( os.path.join(args.dir, scanner.__runner_exe__)) else: runner_path = os.path.abspath(args.runner_path) if not os.path.exists(runner_path): logger.exception("Invalid test runner path: {}".format(runner_path)) return bootstrap_config = None if args.bootstrap_config: with open(args.bootstrap_config) as json_file: bootstrap_config = BootstrapConfig(flatten=True) bootstrap_config.load(json.load(json_file)) add_dirs_to_path(args.add_path) scan_results = [] # list of ScanResult() # Find default filter files if they exist and add to user-defined lists whitelist_files = (args.whitelist_files if args.whitelist_files else []) + [get_default_whitelist()] blacklist_files = (args.blacklist_files if args.blacklist_files else []) + [get_default_blacklist()] # Create a FileApprover to determine if scanned files can be tested file_approver = FileApprover(whitelist_files, blacklist_files) module_failures = 0 library_module_name_list = [] executable_name_list = [] # Dynamic Libraries / Modules if not __no_dll__: logger.info("Scanning for dynamic libraries") library_module_name_list = list(scanner.enumerate_modules(args.dir)) for file_name in library_module_name_list: try: if args.limit and len(scan_results) >= args.limit: continue # reached scanning limit if args.only and not FileApprover.is_in_list( file_name, args.only.split(',')): continue # filename does not match any expected pattern if not file_approver.is_approved(file_name): continue result = scan_one(args, extra, ModuleType.LIBRARY, scanner, runner_path, bootstrap_config, file_name, output_dir) if result: scan_results += [result] if result.return_code != RunnerReturnCodes.TESTS_SUCCEEDED: if result.return_code == RunnerReturnCodes.MODULE_SKIPPED: logger.info("Module SKIPPED: {}".format(file_name)) else: logger.error( "Module FAILED: {}, with exit code: {} ({})". format( file_name, result.return_code, RunnerReturnCodes.to_string( result.return_code))) module_failures += 1 if not os.path.exists(result.xml_path): XMLGenerator.create_xml_output_file( result.xml_path, result.return_code, result.error_msg) except KeyboardInterrupt: logger.exception("Process interrupted by user.") break except: logger.exception("Module scan failed.") # Executables if args.exe: logger.info("Scanning for executables") executable_name_list = list(scanner.enumerate_executables(args.dir)) for file_name in executable_name_list: if args.limit and len(scan_results) >= args.limit: continue # reached scanning limit if args.only and not FileApprover.is_in_list( file_name, args.only.split(',')): continue # filename does not match any expected pattern if not file_approver.is_approved(file_name): continue result = scan_one(args, extra, ModuleType.EXECUTABLE, scanner, runner_path, bootstrap_config, file_name, output_dir) if result: scan_results += [result] if result.return_code != RunnerReturnCodes.TESTS_SUCCEEDED: logger.error( "Module FAILED: {}, with exit code: {} ({})".format( file_name, result.return_code, RunnerReturnCodes.to_string(result.return_code))) module_failures += 1 if not os.path.exists(result.xml_path): XMLGenerator.create_xml_output_file( result.xml_path, result.return_code, result.error_msg) # Always save ScanResult data in a JSON file so we have access to it later scan_results_json = {'scan_results': []} for scan_result in scan_results: scan_results_json['scan_results'].append(scan_result._asdict()) json_path = os.path.join(output_dir, 'scan_results.json') with open(json_path, 'w') as f: json.dump(scan_results_json, f) print "----------------AUTOTEST SUMMARY -----------------" print("Log: {}".format(log_path)) print("JSON results: {}".format(json_path)) if not args.no_html_report: # Convert the set of XML files into an HTML report html_report = HTMLReporter.create_html_report(scan_results, output_dir) html_failure_report = HTMLReporter.create_html_failure_report( scan_results, output_dir) print("HTML report: {}".format(html_report)) print("HTML failure-only report: {}".format(html_failure_report)) print("Total modules found: {}".format(len(library_module_name_list))) if module_failures: colorama.init() print( Fore.RED + "**** {} modules had failures or errors ****".format( module_failures)) print(Style.RESET_ALL) else: print("No modules had failures nor errors.") print( "If a module has 1/0 tests failed or errored, this means the library failed to load." ) test_summary_results = { TESTS_RUN_KEY: 0, TESTS_PASSED_KEY: 0, TESTS_FAILED_KEY: 0, TESTS_ERRORED_KEY: 0, TESTS_SKIPPED_KEY: 0, TOTAL_TIME_TAKEN_KEY: 0, } for file_name in library_module_name_list: _print_summary_for_file_name(test_summary_results, file_name, output_dir) for file_name in executable_name_list: _print_summary_for_file_name(test_summary_results, file_name, output_dir) print "\nTotal tests run: {0}".format( str(test_summary_results[TESTS_RUN_KEY])) print "Total tests passed: {0}".format( str(test_summary_results[TESTS_PASSED_KEY])) print "Total tests failed: {0}".format( str(test_summary_results[TESTS_FAILED_KEY])) print "Total tests errored: {0}".format( str(test_summary_results[TESTS_ERRORED_KEY])) print "Total tests skipped: {0}".format( str(test_summary_results[TESTS_SKIPPED_KEY])) print "Total test time taken: {0}\n".format( str(test_summary_results[TOTAL_TIME_TAKEN_KEY])) print "----------------AUTOTEST SUMMARY -----------------" return 1 if module_failures > 0 else 0
def test_ToString_UnknownCode_ReturnsDefaultMessage(self): return_code = -1 # Unknown error code error_string = RunnerReturnCodes.to_string(return_code) self.assertEqual(error_string, "Unknown return code")
def test_ToString_FailedCode_ReturnsMessage(self): return_code = 1 # TESTS_FAILED error_string = RunnerReturnCodes.to_string(return_code) self.assertEqual(error_string, "Tests Failed")
def test_ToString_PassCode_ReturnsNone(self): return_code = 0 # TESTS_SUCCEEDED error_string = RunnerReturnCodes.to_string(return_code) self.assertIsNone(error_string)