def _pre_run_cleanup(self):
"""Common cleanup before running a testcase."""
# Cleanup any existing application instances and user profile directories.
# Cleaning up temp user profile directories. Should be done before calling
# |get_command_line_for_application| call since that creates dependencies in
# the profile folder.
process_handler.terminate_stale_application_instances()
shell.clear_temp_directory()
def beat(previous_state, log_filename):
"""Run a cycle of heartbeat checks to ensure bot is running."""
# Handle case when run_bot.py script is stuck. If yes, kill its process.
task_end_time = tasks.get_task_end_time()
if psutil and task_end_time and dates.time_has_expired(
task_end_time, seconds=tasks.TASK_COMPLETION_BUFFER):
# Get absolute path to |run_bot| script. We use this to identify unique
# instances of bot running on a particular host.
startup_scripts_directory = environment.get_startup_scripts_directory()
bot_file_path = os.path.join(startup_scripts_directory, 'run_bot')
for process in psutil.process_iter():
try:
command_line = ' '.join(process.cmdline())
except (psutil.AccessDenied, psutil.NoSuchProcess, OSError):
sys.exc_clear()
continue
# Find the process running the main bot script.
if bot_file_path not in command_line:
continue
process_id = process.pid
logs.log('Killing stale bot (pid %d) which seems to have stuck.' %
process_id)
try:
process_handler.terminate_root_and_child_processes(process_id)
except Exception:
logs.log_error('Failed to terminate stale bot processes.')
# Minor cleanup to avoid disk space issues on bot restart.
process_handler.terminate_stale_application_instances()
shell.clear_temp_directory()
shell.clear_testcase_directories()
# Concerned stale processes should be killed. Now, delete the stale task.
tasks.track_task_end()
# Figure out when the log file was last modified.
try:
current_state = str(os.path.getmtime(log_filename))
except Exception:
current_state = None
logs.log('Old state %s, current state %s.' %
(previous_state, current_state))
# Only update the heartbeat if the log file was modified.
if current_state and current_state != previous_state:
# Try updating the heartbeat. If an error occurs, just
# wait and return None.
if not data_handler.update_heartbeat():
return None
# Heartbeat is successfully updated.
return current_state
def _pre_setup(self):
"""Common pre-setup."""
self._reset_cwd()
shell.clear_temp_directory()
# Clean up build directory if last one was partial.
partial_build_file_path = os.path.join(self.build_dir, PARTIAL_BUILD_FILE)
if os.path.exists(partial_build_file_path):
self.delete()
if self.base_build_dir:
_setup_build_directories(self.base_build_dir)
environment.set_value('APP_REVISION', self.revision)
environment.set_value('APP_PATH', '')
environment.set_value('APP_PATH_DEBUG', '')
def cleanup_task_state(): """Cleans state before and after a task is executed.""" # Cleanup stale processes. process_handler.cleanup_stale_processes() # Clear build urls, temp and testcase directories. shell.clear_build_urls_directory() shell.clear_crash_stacktraces_directory() shell.clear_testcase_directories() shell.clear_temp_directory() shell.clear_system_temp_directory() shell.clear_device_temp_directories() # Reset memory tool environment variables. environment.reset_current_memory_tool_options() # Call python's garbage collector. utils.python_gc()
def _run_libfuzzer_testcase(testcase, testcase_file_path):
"""Run libFuzzer testcase, and return the CrashResult."""
# Cleanup any existing application instances and temp directories.
process_handler.cleanup_stale_processes()
shell.clear_temp_directory()
if environment.is_trusted_host():
from bot.untrusted_runner import file_host
file_host.copy_file_to_worker(
testcase_file_path, file_host.rebase_to_worker_root(testcase_file_path))
test_timeout = environment.get_value('TEST_TIMEOUT',
process_handler.DEFAULT_TEST_TIMEOUT)
repro_command = tests.get_command_line_for_application(
file_to_run=testcase_file_path, needs_http=testcase.http_flag)
return_code, crash_time, output = process_handler.run_process(
repro_command, timeout=test_timeout)
return CrashResult(return_code, crash_time, output)
def store_testcase_dependencies_from_bundled_testcase_archive(
metadata, testcase, testcase_file_path):
"""Store testcase dependencies from a bundled testcase archive
(with multiple testcases)."""
# Nothing to do if this is not a bundled testcase archive with
# multiple testcase.
if not metadata.bundled:
return 1
# Cleanup, before re-running the app to get resource list.
process_handler.terminate_stale_application_instances()
shell.clear_temp_directory()
# Increase the test timeout to warmup timeout. This is needed
# since we use warmup timeout in test_for_crash_with_retries.
warmup_timeout = environment.get_value('WARMUP_TIMEOUT')
environment.set_value('TEST_TIMEOUT', warmup_timeout)
env_copy = environment.copy()
crash_queue = process_handler.get_queue()
thread_index = 0
thread = process_handler.get_process()(
target=testcase_manager.run_testcase_and_return_result_in_queue,
args=(crash_queue, thread_index, testcase_file_path, testcase.gestures,
env_copy))
thread.start()
thread.join(warmup_timeout)
if thread.is_alive():
try:
thread.terminate()
except:
logs.log_error('Process termination failed or was not needed.')
# We should be able to reproduce this reproducible crash. If not,
# something wrong happened and we would just try to redo task.
if not crash_queue.empty():
crash = crash_queue.get()
fuzzed_key, archived, absolute_path, archive_filename = (
setup.archive_testcase_and_dependencies_in_gcs(crash.resource_list,
testcase_file_path))
if archived:
testcase.archive_state = data_types.ArchiveStatus.FUZZED
else:
testcase.archive_state = 0
testcase.fuzzed_keys = fuzzed_key
metadata.blobstore_key = fuzzed_key
testcase.absolute_path = absolute_path
if archive_filename:
testcase.archive_filename = archive_filename
metadata.filename = os.path.basename(archive_filename)
else:
metadata.filename = os.path.basename(testcase_file_path)
metadata.bundled = False
metadata.put()
else:
logs.log_error('Could not get crash data from queue. Retrying task.')
tasks.add_task('analyze', testcase.key.id(), testcase.job_type)
return None
return 1
def update_source_code():
"""Updates source code files with latest version from appengine."""
process_handler.cleanup_stale_processes()
shell.clear_temp_directory()
root_directory = environment.get_value('ROOT_DIR')
temp_directory = environment.get_value('BOT_TMPDIR')
temp_archive = os.path.join(temp_directory, 'clusterfuzz-source.zip')
try:
storage.copy_file_from(get_source_url(), temp_archive)
except Exception:
logs.log_error('Could not retrieve source code archive from url.')
return
try:
file_list = archive.get_file_list(temp_archive)
zip_archive = zipfile.ZipFile(temp_archive, 'r')
except Exception:
logs.log_error('Bad zip file.')
return
src_directory = os.path.join(root_directory, 'src')
output_directory = os.path.dirname(root_directory)
error_occurred = False
normalized_file_set = set()
for filepath in file_list:
filename = os.path.basename(filepath)
# This file cannot be updated on the fly since it is running as server.
if filename == 'adb':
continue
absolute_filepath = os.path.join(output_directory, filepath)
if os.path.altsep:
absolute_filepath = absolute_filepath.replace(os.path.altsep, os.path.sep)
if os.path.realpath(absolute_filepath) != absolute_filepath:
continue
normalized_file_set.add(absolute_filepath)
try:
file_extension = os.path.splitext(filename)[1]
# Remove any .so files first before overwriting, as they can be loaded
# in the memory of existing processes. Overwriting them directly causes
# segfaults in existing processes (e.g. run.py).
if file_extension == '.so' and os.path.exists(absolute_filepath):
os.remove(absolute_filepath)
# On Windows, to update DLLs (and native .pyd extensions), we rename it
# first so that we can install the new version.
if (environment.platform() == 'WINDOWS' and
file_extension in ['.dll', '.pyd'] and
os.path.exists(absolute_filepath)):
_rename_dll_for_update(absolute_filepath)
except Exception:
logs.log_error('Failed to remove or move %s before extracting new '
'version.' % absolute_filepath)
try:
extracted_path = zip_archive.extract(filepath, output_directory)
external_attr = zip_archive.getinfo(filepath).external_attr
mode = (external_attr >> 16) & 0o777
mode |= 0o440
os.chmod(extracted_path, mode)
except:
error_occurred = True
logs.log_error(
'Failed to extract file %s from source archive.' % filepath)
zip_archive.close()
if error_occurred:
return
clear_pyc_files(src_directory)
clear_old_files(src_directory, normalized_file_set)
local_manifest_path = os.path.join(root_directory,
utils.LOCAL_SOURCE_MANIFEST)
source_version = utils.read_data_from_file(
local_manifest_path, eval_data=False)
logs.log('Source code updated to %s.' % source_version)
def test_for_reproducibility(testcase_path, expected_state,
expected_security_flag, test_timeout, http_flag,
gestures):
"""Test to see if a crash is fully reproducible or is a one-time crasher."""
# Cleanup any existing application instances and user profile directories.
# Cleaning up temp clears user profile directories and should be done before
# calling |get_command_line_for_application| call since that creates
# dependencies in the profile folder.
process_handler.terminate_stale_application_instances()
shell.clear_temp_directory()
app_directory = environment.get_value('APP_DIR')
command = get_command_line_for_application(testcase_path,
needs_http=http_flag)
crash_count = 0
crash_retries = environment.get_value('CRASH_RETRIES')
reproducible_crash_target_count = crash_retries * REPRODUCIBILITY_FACTOR
warmup_timeout = environment.get_value('WARMUP_TIMEOUT')
logs.log('Testing for crash (command="%s").' % command)
round_number = 0
for round_number in xrange(1, crash_retries + 1):
# Bail out early if there is no hope of finding a reproducible crash.
if (crash_retries - round_number + crash_count + 1 <
reproducible_crash_target_count):
break
run_timeout = warmup_timeout if round_number == 1 else test_timeout
return_code, crash_time, output = process_handler.run_process(
command,
timeout=run_timeout,
gestures=gestures,
current_working_directory=app_directory)
process_handler.terminate_stale_application_instances()
crash_result = CrashResult(return_code, crash_time, output)
if not crash_result.is_crash():
continue
state = crash_result.get_symbolized_data()
crash_state = state.crash_state
security_flag = crash_result.is_security_issue()
# If we don't have an expected crash state, set it to the one from initial
# crash.
if not expected_state:
expected_state = crash_state
if security_flag != expected_security_flag:
logs.log('Detected a crash without the correct security flag.')
continue
crash_comparer = CrashComparer(crash_state, expected_state)
if not crash_comparer.is_similar():
logs.log('Detected a crash with an unrelated state: '
'Expected(%s), Found(%s).' %
(expected_state, crash_state))
continue
crash_count += 1
if crash_count >= reproducible_crash_target_count:
logs.log('Crash is reproducible.')
return True
logs.log('Crash is not reproducible. Crash count: %d/%d.' %
(crash_count, round_number))
return False
def test_for_crash_with_retries(testcase,
testcase_path,
test_timeout,
http_flag=False,
compare_crash=True):
"""Test for a crash and return crash parameters like crash type, crash state,
crash stacktrace, etc."""
# Cleanup any existing application instances and user profile directories.
# Cleaning up temp clears user profile directories and should be done before
# calling |get_command_line_for_application| call since that creates
# dependencies in the profile folder.
process_handler.terminate_stale_application_instances()
shell.clear_temp_directory()
app_directory = environment.get_value('APP_DIR')
command = get_command_line_for_application(testcase_path,
needs_http=http_flag)
crash_retries = environment.get_value('CRASH_RETRIES')
flaky_stacktrace = testcase.flaky_stack
warmup_timeout = environment.get_value('WARMUP_TIMEOUT')
logs.log('Testing for crash (command="%s").' % command)
for round_number in xrange(1, crash_retries + 1):
run_timeout = warmup_timeout if round_number == 1 else test_timeout
return_code, crash_time, output = process_handler.run_process(
command,
timeout=run_timeout,
gestures=testcase.gestures,
current_working_directory=app_directory)
process_handler.terminate_stale_application_instances()
crash_result = CrashResult(return_code, crash_time, output)
if not crash_result.is_crash():
continue
state = crash_result.get_symbolized_data()
logs.log('Crash occurred in %d seconds (round %d). State:\n%s' %
(crash_time, round_number, state.crash_state))
if not compare_crash or not testcase.crash_state:
logs.log('Crash stacktrace comparison skipped.')
return crash_result
if flaky_stacktrace:
logs.log('Crash stacktrace is marked flaky, skipping comparison.')
return crash_result
if crash_result.should_ignore():
logs.log('Crash stacktrace matched ignore signatures, ignored.')
continue
if crash_result.is_security_issue() != testcase.security_flag:
logs.log('Crash security flag does not match, ignored.')
continue
crash_comparer = CrashComparer(state.crash_state, testcase.crash_state)
if crash_comparer.is_similar():
logs.log('Crash stacktrace is similar to original stacktrace.')
return crash_result
else:
logs.log('Crash stacktrace does not match original stacktrace.')
logs.log("Didn't crash at all.")
crash_result = CrashResult(return_code=0, crash_time=0, output=output)
return crash_result
