def scan(args, extra): scanner = Scanner() output_dir = create_output_directory(args.output_path, args.no_timestamp) # setup logging setup_logging(os.path.join(output_dir, "aztest.log"), args.verbosity) logger.info("AZ Test Scanner") if not args.runner_path: runner_path = os.path.abspath( os.path.join(args.dir, scanner.__runner_exe__)) else: runner_path = os.path.abspath(args.runner_path) if not os.path.exists(runner_path): logger.exception("Invalid test runner path: {}".format(runner_path)) return bootstrap_config = None if args.bootstrap_config: with open(args.bootstrap_config) as json_file: bootstrap_config = BootstrapConfig(flatten=True) bootstrap_config.load(json.load(json_file)) add_dirs_to_path(args.add_path) scan_results = [] # list of ScanResult() # Find default filter files if they exist and add to user-defined lists whitelist_files = (args.whitelist_files if args.whitelist_files else []) + [get_default_whitelist()] blacklist_files = (args.blacklist_files if args.blacklist_files else []) + [get_default_blacklist()] # Create a FileApprover to determine if scanned files can be tested file_approver = FileApprover(whitelist_files, blacklist_files) module_failures = 0 # Dynamic Libraries / Modules if not __no_dll__: logger.info("Scanning for dynamic libraries") for file_name in scanner.enumerate_modules(args.dir): try: if args.limit and len(scan_results) >= args.limit: continue # reached scanning limit if args.only and not FileApprover.is_in_list( file_name, args.only.split(',')): continue # filename does not match any expected pattern if not file_approver.is_approved(file_name): continue result = scan_one(args, extra, ModuleType.LIBRARY, scanner, runner_path, bootstrap_config, file_name, output_dir) if result: scan_results += [result] if result.return_code != RunnerReturnCodes.TESTS_SUCCEEDED: module_failures += 1 if not os.path.exists(result.xml_path): XMLGenerator.create_xml_output_file( result.xml_path, result.return_code, result.error_msg) except KeyboardInterrupt: logger.exception("Process interrupted by user.") break except: logger.exception("Module scan failed.") # Executables if args.exe: logger.info("Scanning for executables") for file_name in scanner.enumerate_executables(args.dir): if args.limit and len(scan_results) >= args.limit: continue # reached scanning limit if args.only and not FileApprover.is_in_list( file_name, args.only.split(',')): continue # filename does not match any expected pattern if not file_approver.is_approved(file_name): continue result = scan_one(args, extra, ModuleType.EXECUTABLE, scanner, runner_path, bootstrap_config, file_name, output_dir) if result: scan_results += [result] if result.return_code != RunnerReturnCodes.TESTS_SUCCEEDED: module_failures += 1 if not os.path.exists(result.xml_path): XMLGenerator.create_xml_output_file( result.xml_path, result.return_code, result.error_msg) if args.html_report: # Convert the set of XML files into an HTML report HTMLReporter.create_html_report(scan_results, output_dir) HTMLReporter.create_html_failure_report(scan_results, output_dir) return 1 if module_failures > 0 else 0
def scan(args, extra): scanner = Scanner() output_dir = create_output_directory(args.output_path, args.no_timestamp) # setup logging log_path = os.path.join(output_dir, "aztest.log") setup_logging(log_path, args.verbosity) logger.info("AZ Test Scanner") if not args.runner_path: runner_path = os.path.abspath( os.path.join(args.dir, scanner.__runner_exe__)) else: runner_path = os.path.abspath(args.runner_path) if not os.path.exists(runner_path): logger.exception("Invalid test runner path: {}".format(runner_path)) return bootstrap_config = None if args.bootstrap_config: with open(args.bootstrap_config) as json_file: bootstrap_config = BootstrapConfig(flatten=True) bootstrap_config.load(json.load(json_file)) add_dirs_to_path(args.add_path) scan_results = [] # list of ScanResult() # Find default filter files if they exist and add to user-defined lists whitelist_files = (args.whitelist_files if args.whitelist_files else []) + [get_default_whitelist()] blacklist_files = (args.blacklist_files if args.blacklist_files else []) + [get_default_blacklist()] # Create a FileApprover to determine if scanned files can be tested file_approver = FileApprover(whitelist_files, blacklist_files) module_failures = 0 library_module_name_list = [] executable_name_list = [] # Dynamic Libraries / Modules if not __no_dll__: logger.info("Scanning for dynamic libraries") library_module_name_list = list(scanner.enumerate_modules(args.dir)) for file_name in library_module_name_list: try: if args.limit and len(scan_results) >= args.limit: continue # reached scanning limit if args.only and not FileApprover.is_in_list( file_name, args.only.split(',')): continue # filename does not match any expected pattern if not file_approver.is_approved(file_name): continue result = scan_one(args, extra, ModuleType.LIBRARY, scanner, runner_path, bootstrap_config, file_name, output_dir) if result: scan_results += [result] if result.return_code != RunnerReturnCodes.TESTS_SUCCEEDED: if result.return_code == RunnerReturnCodes.MODULE_SKIPPED: logger.info("Module SKIPPED: {}".format(file_name)) else: logger.error( "Module FAILED: {}, with exit code: {} ({})". format( file_name, result.return_code, RunnerReturnCodes.to_string( result.return_code))) module_failures += 1 if not os.path.exists(result.xml_path): XMLGenerator.create_xml_output_file( result.xml_path, result.return_code, result.error_msg) except KeyboardInterrupt: logger.exception("Process interrupted by user.") break except: logger.exception("Module scan failed.") # Executables if args.exe: logger.info("Scanning for executables") executable_name_list = list(scanner.enumerate_executables(args.dir)) for file_name in executable_name_list: if args.limit and len(scan_results) >= args.limit: continue # reached scanning limit if args.only and not FileApprover.is_in_list( file_name, args.only.split(',')): continue # filename does not match any expected pattern if not file_approver.is_approved(file_name): continue result = scan_one(args, extra, ModuleType.EXECUTABLE, scanner, runner_path, bootstrap_config, file_name, output_dir) if result: scan_results += [result] if result.return_code != RunnerReturnCodes.TESTS_SUCCEEDED: logger.error( "Module FAILED: {}, with exit code: {} ({})".format( file_name, result.return_code, RunnerReturnCodes.to_string(result.return_code))) module_failures += 1 if not os.path.exists(result.xml_path): XMLGenerator.create_xml_output_file( result.xml_path, result.return_code, result.error_msg) # Always save ScanResult data in a JSON file so we have access to it later scan_results_json = {'scan_results': []} for scan_result in scan_results: scan_results_json['scan_results'].append(scan_result._asdict()) json_path = os.path.join(output_dir, 'scan_results.json') with open(json_path, 'w') as f: json.dump(scan_results_json, f) print "----------------AUTOTEST SUMMARY -----------------" print("Log: {}".format(log_path)) print("JSON results: {}".format(json_path)) if not args.no_html_report: # Convert the set of XML files into an HTML report html_report = HTMLReporter.create_html_report(scan_results, output_dir) html_failure_report = HTMLReporter.create_html_failure_report( scan_results, output_dir) print("HTML report: {}".format(html_report)) print("HTML failure-only report: {}".format(html_failure_report)) print("Total modules found: {}".format(len(library_module_name_list))) if module_failures: colorama.init() print( Fore.RED + "**** {} modules had failures or errors ****".format( module_failures)) print(Style.RESET_ALL) else: print("No modules had failures nor errors.") print( "If a module has 1/0 tests failed or errored, this means the library failed to load." ) test_summary_results = { TESTS_RUN_KEY: 0, TESTS_PASSED_KEY: 0, TESTS_FAILED_KEY: 0, TESTS_ERRORED_KEY: 0, TESTS_SKIPPED_KEY: 0, TOTAL_TIME_TAKEN_KEY: 0, } for file_name in library_module_name_list: _print_summary_for_file_name(test_summary_results, file_name, output_dir) for file_name in executable_name_list: _print_summary_for_file_name(test_summary_results, file_name, output_dir) print "\nTotal tests run: {0}".format( str(test_summary_results[TESTS_RUN_KEY])) print "Total tests passed: {0}".format( str(test_summary_results[TESTS_PASSED_KEY])) print "Total tests failed: {0}".format( str(test_summary_results[TESTS_FAILED_KEY])) print "Total tests errored: {0}".format( str(test_summary_results[TESTS_ERRORED_KEY])) print "Total tests skipped: {0}".format( str(test_summary_results[TESTS_SKIPPED_KEY])) print "Total test time taken: {0}\n".format( str(test_summary_results[TOTAL_TIME_TAKEN_KEY])) print "----------------AUTOTEST SUMMARY -----------------" return 1 if module_failures > 0 else 0