def run_retdec_build(config, db, cmd_runner, tested_commit): """Runs a build of RetDec. If the build fails, it prints and error message and sends an email to the author of the tested commit (if needed). """ build_start_date = datetime.now() build_id = db.insert_build_started_info(tested_commit, build_start_date) build_info = build_retdec(Directory(config['runner']['retdec_build_dir']), cmd_runner, int(config['runner']['build_procs'])) # Ensure that the same date appears after the build is finished # (otherwise, there may be a difference of a few seconds). build_info.start_date = build_start_date db.insert_build_ended_info(build_id, build_info) if not build_info.succeeded: LINES_COUNT = 30 error_msg = 'failed to build RetDec; ' +\ 'showing the last {} lines of the log:\n...\n'.format(LINES_COUNT) +\ '\n'.join(build_info.log.split('\n')[-LINES_COUNT:]).strip() logging.error(error_msg) print_error(error_msg) send_email_for_commit_if_needed(config, db, tested_commit) sys.exit(1)
def exit_if_on_windows(): """Exits the script if it is run on MS Windows.""" # This script needs support of signal.SIGCONT, socket.AF_UNIX, and # os.killpg(), which are not present on MS Windows. if on_windows(): print_error('this script cannot be run on MS Windows') sys.exit(1)
def ensure_is_run_from_script_dir(): """Ensures that the script is run from its directory.""" script_dir = os.path.abspath(os.path.dirname(__file__)) if os.getcwd() != script_dir: print_error( '{} has to be run from {}, not from {}'.format( os.path.basename(__file__), script_dir, os.getcwd() ) ) sys.exit(1)
def exit_if_already_running(): """Exits the script if it is already running.""" # The following approach is susceptible to race conditions, but since I was # not able to find a better way of doing this on Windows, I use it anyway. if os.path.isfile(PID_FILE_PATH): # Check if the process is already running. with open(PID_FILE_PATH, 'r') as f: pid = int(f.read()) if process_exists(pid): print_error('script is already running ({}) -> exiting'.format( PID_FILE_PATH)) sys.exit(1) # We may run. with open(PID_FILE_PATH, 'w') as f: f.write(str(os.getpid()))
def exit_if_already_running(): """Exits the script if it is already running for the current user.""" # Instead of storing the PID of the process in the filesystem and checking # whether it already exists, we utilize AF_UNIX sockets and the abstract # namespace. In this way, we don't have to manage any files. # Based on http://stackoverflow.com/a/7758075/2580955. # See also http://blog.eduardofleury.com/archives/2007/09/13 for more # information on AF_UNIX sockets. # # The socket has to be global. Otherwise, it would be closed once this # function finishes because of the garbage collector. global lock_socket lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) try: # To bind the socket to the abstract namespace, we have to start its # name with the null byte. lock_socket.bind('\0' + __file__ + get_script_user()) except socket.error: print_error('script is already running -> exiting') sys.exit(1)
def ensure_all_required_settings_are_set(config): """Ensures that all required settings in the given configuration are set.""" # [runner] -> clang_dir clang_dir = config['runner']['clang_dir'] if not clang_dir: print_error( "no 'clang_dir' in the [runner] section of config_local.ini " "(you have to add it)") sys.exit(1) elif not os.path.exists(clang_dir): print_error("'clang_dir' in the [runner] section of config_local.ini " "points to a non-existing directory") sys.exit(1) elif not os.path.exists(os.path.join(clang_dir, 'bin')): print_error("'clang_dir' in the [runner] section of config_local.ini " "does not seem to point to Clang") sys.exit(1) # [runner] -> retdec_build_dir retdec_build_dir = config['runner']['retdec_build_dir'] if not retdec_build_dir: print_error( "no 'retdec_build_dir' in the [runner] section of config_local.ini " "(you have to add it)") sys.exit(1) elif not os.path.exists(retdec_build_dir): print_error( "'retdec_build_dir' in the [runner] section of config_local.ini " "points to a non-existing directory") sys.exit(1) elif not os.path.exists(os.path.join(retdec_build_dir, 'CMakeCache.txt')): print_error( "'retdec_build_dir' in the [runner] section of config_local.ini " "does not seem to point to a RetDec build directory") sys.exit(1) # [runner] -> retdec_install_dir retdec_install_dir = config['runner']['retdec_install_dir'] if not retdec_install_dir: print_error( "no 'retdec_install_dir' in the [runner] section of config_local.ini " "(you have to add it)") sys.exit(1) elif not os.path.exists(retdec_install_dir): print_error( "'retdec_install_dir' in the [runner] section of config_local.ini " "points to a non-existing directory") sys.exit(1) elif not os.path.exists( os.path.join(retdec_install_dir, 'bin', 'retdec-decompiler.sh')): print_error( "'retdec_install_dir' in the [runner] section of config_local.ini " "does not seem to point to RetDec") sys.exit(1) # [runner] -> retdec_repo_dir retdec_repo_dir = config['runner']['retdec_repo_dir'] if not retdec_repo_dir: print_error( "no 'retdec_repo_dir' in the [runner] section of config_local.ini " "(you have to add it)") sys.exit(1) elif not os.path.exists(retdec_repo_dir): print_error( "'retdec_repo_dir' in the [runner] section of config_local.ini " "points to a non-existing directory") sys.exit(1) elif not os.path.exists(os.path.join(retdec_repo_dir, 'src', 'bin2llvmir')): print_error( "'retdec_repo_dir' in the [runner] section of config_local.ini " "does not seem to point to a RetDec repository") sys.exit(1) # [runner] -> tests_root_dir tests_root_dir = config['runner']['tests_root_dir'] if not tests_root_dir: print_error( "no 'tests_root_dir' in the [runner] section of config_local.ini " "(you have to add it)") sys.exit(1) elif not os.path.exists(tests_root_dir): print_error( "'tests_root_dir' in the [runner] section of config_local.ini " "points to a non-existing directory") sys.exit(1) elif not os.path.exists(os.path.join(tests_root_dir, 'bugs')): print_error( "'tests_root_dir' in the [runner] section of config_local.ini " "does not seem to point to regression tests") sys.exit(1) if config['runner'].getboolean('idaplugin_tests_enabled'): # [runner] -> idaplugin_ida_dir idaplugin_ida_dir = config['runner']['idaplugin_ida_dir'] if not idaplugin_ida_dir: print_error( "no 'idaplugin_ida_dir' in the [runner] section of config_local.ini " "(you have to add it to run tests for our IDA plugin)") sys.exit(1) elif not os.path.exists(idaplugin_ida_dir): print_error( "'idaplugin_ida_dir' in the [runner] section of config_local.ini " "points to a non-existing directory") sys.exit(1) elif not os.path.exists(os.path.join(idaplugin_ida_dir, 'plugins')): print_error( "'idaplugin_ida_dir' in the [runner] section of config_local.ini " "does not seem to point to IDA Pro") sys.exit(1) # [runner] -> idaplugin_script idaplugin_script = config['runner']['idaplugin_script'] if not idaplugin_script: print_error( "no 'idaplugin_script' in the [runner] section of config_local.ini " "(you have to add it to run tests for our IDA plugin)") sys.exit(1) elif not os.path.exists(idaplugin_script): print_error( "'idaplugin_script' in the [runner] section of config_local.ini " "points to a non-existing file") sys.exit(1)
with repo.checkout(tested_commit): # RetDec build. if args.build: run_retdec_build(config, db, cmd_runner, tested_commit) remove_results_from_previous_test_runs(tests_dir) # Find tests. test_cases = get_test_cases_to_run(tests_dir, tests_root_dir, excluded_dirs, config, args) if not test_cases: relative_excluded_dirs = ', '.join( os.path.relpath(dir.path, tests_root_dir.path) for dir in excluded_dirs) print_error( 'no {}tests found in {} (excluded directories: {})'.format( 'critical ' if args.only_critical else '', tests_dir.path, relative_excluded_dirs)) sys.exit(1) # Run them. print_prologue(tests_dir.path, test_cases, tested_commit, args.resume) tests_results = run_test_cases( test_cases, tested_commit, procs=get_num_of_procs_for_tests(config), resume=args.resume, lock=lock) print_summary(tests_results) # Send notifications (if needed).
remove_results_from_previous_test_runs(tests_dir) # Find tests. test_cases = get_test_cases_to_run( tests_dir, tests_root_dir, excluded_dirs, config, args ) if not test_cases: relative_excluded_dirs = ', '.join( os.path.relpath(dir.path, tests_root_dir.path) for dir in excluded_dirs ) print_error('no tests found in {} (excluded directories: {})'.format( tests_dir.path, relative_excluded_dirs, )) sys.exit(1) # Run them. print_prologue(tests_dir.path, test_cases, tested_commit, args.resume) tests_results = run_test_cases( test_cases, tested_commit, procs=get_num_of_procs_for_tests(config), resume=args.resume, lock=lock ) print_summary(tests_results) # Send notifications (if needed).