def run_retdec_build(config, db, cmd_runner, tested_commit):
"""Runs a build of RetDec.
If the build fails, it prints and error message and sends an email to the
author of the tested commit (if needed).
"""
build_start_date = datetime.now()
build_id = db.insert_build_started_info(tested_commit, build_start_date)
build_info = build_retdec(Directory(config['runner']['retdec_build_dir']),
cmd_runner, int(config['runner']['build_procs']))
# Ensure that the same date appears after the build is finished
# (otherwise, there may be a difference of a few seconds).
build_info.start_date = build_start_date
db.insert_build_ended_info(build_id, build_info)
if not build_info.succeeded:
LINES_COUNT = 30
error_msg = 'failed to build RetDec; ' +\
'showing the last {} lines of the log:\n...\n'.format(LINES_COUNT) +\
'\n'.join(build_info.log.split('\n')[-LINES_COUNT:]).strip()
logging.error(error_msg)
print_error(error_msg)
send_email_for_commit_if_needed(config, db, tested_commit)
sys.exit(1)
def exit_if_on_windows():
"""Exits the script if it is run on MS Windows."""
# This script needs support of signal.SIGCONT, socket.AF_UNIX, and
# os.killpg(), which are not present on MS Windows.
if on_windows():
print_error('this script cannot be run on MS Windows')
sys.exit(1)
def ensure_is_run_from_script_dir():
"""Ensures that the script is run from its directory."""
script_dir = os.path.abspath(os.path.dirname(__file__))
if os.getcwd() != script_dir:
print_error(
'{} has to be run from {}, not from {}'.format(
os.path.basename(__file__),
script_dir,
os.getcwd()
)
)
sys.exit(1)
def exit_if_already_running():
"""Exits the script if it is already running."""
# The following approach is susceptible to race conditions, but since I was
# not able to find a better way of doing this on Windows, I use it anyway.
if os.path.isfile(PID_FILE_PATH):
# Check if the process is already running.
with open(PID_FILE_PATH, 'r') as f:
pid = int(f.read())
if process_exists(pid):
print_error('script is already running ({}) -> exiting'.format(
PID_FILE_PATH))
sys.exit(1)
# We may run.
with open(PID_FILE_PATH, 'w') as f:
f.write(str(os.getpid()))
def exit_if_already_running():
"""Exits the script if it is already running for the current user."""
# Instead of storing the PID of the process in the filesystem and checking
# whether it already exists, we utilize AF_UNIX sockets and the abstract
# namespace. In this way, we don't have to manage any files.
# Based on http://stackoverflow.com/a/7758075/2580955.
# See also http://blog.eduardofleury.com/archives/2007/09/13 for more
# information on AF_UNIX sockets.
#
# The socket has to be global. Otherwise, it would be closed once this
# function finishes because of the garbage collector.
global lock_socket
lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
try:
# To bind the socket to the abstract namespace, we have to start its
# name with the null byte.
lock_socket.bind('\0' + __file__ + get_script_user())
except socket.error:
print_error('script is already running -> exiting')
sys.exit(1)
def ensure_all_required_settings_are_set(config):
"""Ensures that all required settings in the given configuration are set."""
# [runner] -> clang_dir
clang_dir = config['runner']['clang_dir']
if not clang_dir:
print_error(
"no 'clang_dir' in the [runner] section of config_local.ini "
"(you have to add it)")
sys.exit(1)
elif not os.path.exists(clang_dir):
print_error("'clang_dir' in the [runner] section of config_local.ini "
"points to a non-existing directory")
sys.exit(1)
elif not os.path.exists(os.path.join(clang_dir, 'bin')):
print_error("'clang_dir' in the [runner] section of config_local.ini "
"does not seem to point to Clang")
sys.exit(1)
# [runner] -> retdec_build_dir
retdec_build_dir = config['runner']['retdec_build_dir']
if not retdec_build_dir:
print_error(
"no 'retdec_build_dir' in the [runner] section of config_local.ini "
"(you have to add it)")
sys.exit(1)
elif not os.path.exists(retdec_build_dir):
print_error(
"'retdec_build_dir' in the [runner] section of config_local.ini "
"points to a non-existing directory")
sys.exit(1)
elif not os.path.exists(os.path.join(retdec_build_dir, 'CMakeCache.txt')):
print_error(
"'retdec_build_dir' in the [runner] section of config_local.ini "
"does not seem to point to a RetDec build directory")
sys.exit(1)
# [runner] -> retdec_install_dir
retdec_install_dir = config['runner']['retdec_install_dir']
if not retdec_install_dir:
print_error(
"no 'retdec_install_dir' in the [runner] section of config_local.ini "
"(you have to add it)")
sys.exit(1)
elif not os.path.exists(retdec_install_dir):
print_error(
"'retdec_install_dir' in the [runner] section of config_local.ini "
"points to a non-existing directory")
sys.exit(1)
elif not os.path.exists(
os.path.join(retdec_install_dir, 'bin', 'retdec-decompiler.sh')):
print_error(
"'retdec_install_dir' in the [runner] section of config_local.ini "
"does not seem to point to RetDec")
sys.exit(1)
# [runner] -> retdec_repo_dir
retdec_repo_dir = config['runner']['retdec_repo_dir']
if not retdec_repo_dir:
print_error(
"no 'retdec_repo_dir' in the [runner] section of config_local.ini "
"(you have to add it)")
sys.exit(1)
elif not os.path.exists(retdec_repo_dir):
print_error(
"'retdec_repo_dir' in the [runner] section of config_local.ini "
"points to a non-existing directory")
sys.exit(1)
elif not os.path.exists(os.path.join(retdec_repo_dir, 'src',
'bin2llvmir')):
print_error(
"'retdec_repo_dir' in the [runner] section of config_local.ini "
"does not seem to point to a RetDec repository")
sys.exit(1)
# [runner] -> tests_root_dir
tests_root_dir = config['runner']['tests_root_dir']
if not tests_root_dir:
print_error(
"no 'tests_root_dir' in the [runner] section of config_local.ini "
"(you have to add it)")
sys.exit(1)
elif not os.path.exists(tests_root_dir):
print_error(
"'tests_root_dir' in the [runner] section of config_local.ini "
"points to a non-existing directory")
sys.exit(1)
elif not os.path.exists(os.path.join(tests_root_dir, 'bugs')):
print_error(
"'tests_root_dir' in the [runner] section of config_local.ini "
"does not seem to point to regression tests")
sys.exit(1)
if config['runner'].getboolean('idaplugin_tests_enabled'):
# [runner] -> idaplugin_ida_dir
idaplugin_ida_dir = config['runner']['idaplugin_ida_dir']
if not idaplugin_ida_dir:
print_error(
"no 'idaplugin_ida_dir' in the [runner] section of config_local.ini "
"(you have to add it to run tests for our IDA plugin)")
sys.exit(1)
elif not os.path.exists(idaplugin_ida_dir):
print_error(
"'idaplugin_ida_dir' in the [runner] section of config_local.ini "
"points to a non-existing directory")
sys.exit(1)
elif not os.path.exists(os.path.join(idaplugin_ida_dir, 'plugins')):
print_error(
"'idaplugin_ida_dir' in the [runner] section of config_local.ini "
"does not seem to point to IDA Pro")
sys.exit(1)
# [runner] -> idaplugin_script
idaplugin_script = config['runner']['idaplugin_script']
if not idaplugin_script:
print_error(
"no 'idaplugin_script' in the [runner] section of config_local.ini "
"(you have to add it to run tests for our IDA plugin)")
sys.exit(1)
elif not os.path.exists(idaplugin_script):
print_error(
"'idaplugin_script' in the [runner] section of config_local.ini "
"points to a non-existing file")
sys.exit(1)
with repo.checkout(tested_commit):
# RetDec build.
if args.build:
run_retdec_build(config, db, cmd_runner, tested_commit)
remove_results_from_previous_test_runs(tests_dir)
# Find tests.
test_cases = get_test_cases_to_run(tests_dir, tests_root_dir,
excluded_dirs, config, args)
if not test_cases:
relative_excluded_dirs = ', '.join(
os.path.relpath(dir.path, tests_root_dir.path)
for dir in excluded_dirs)
print_error(
'no {}tests found in {} (excluded directories: {})'.format(
'critical ' if args.only_critical else '',
tests_dir.path, relative_excluded_dirs))
sys.exit(1)
# Run them.
print_prologue(tests_dir.path, test_cases, tested_commit,
args.resume)
tests_results = run_test_cases(
test_cases,
tested_commit,
procs=get_num_of_procs_for_tests(config),
resume=args.resume,
lock=lock)
print_summary(tests_results)
# Send notifications (if needed).
remove_results_from_previous_test_runs(tests_dir)
# Find tests.
test_cases = get_test_cases_to_run(
tests_dir,
tests_root_dir,
excluded_dirs,
config,
args
)
if not test_cases:
relative_excluded_dirs = ', '.join(
os.path.relpath(dir.path, tests_root_dir.path) for dir in excluded_dirs
)
print_error('no tests found in {} (excluded directories: {})'.format(
tests_dir.path,
relative_excluded_dirs,
))
sys.exit(1)
# Run them.
print_prologue(tests_dir.path, test_cases, tested_commit, args.resume)
tests_results = run_test_cases(
test_cases,
tested_commit,
procs=get_num_of_procs_for_tests(config),
resume=args.resume,
lock=lock
)
print_summary(tests_results)
# Send notifications (if needed).
