def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
os.environ['BOT_TMPDIR'] = os.path.join(self.tmp_dir, 'bot_tmpdir')
test_helpers.patch(self, [
'clusterfuzz._internal.datastore.data_handler.'
'get_data_bundle_bucket_name',
'clusterfuzz._internal.system.environment.'
'set_environment_parameters_from_file',
])
test_helpers.patch_environ(self)
# Our tests write data/logs into subdirs of ROOT_DIR. Pivot the ROOT_DIR to
# a temporary one.
new_root = _create_test_root()
os.environ['ROOT_DIR'] = new_root
self.saved_cwd = os.getcwd()
os.chdir(new_root)
environment.set_bot_environment()
fuzz_inputs = os.environ['FUZZ_INPUTS']
shell.remove_directory(fuzz_inputs, recreate=True)
worker_fuzz_inputs = file_host.rebase_to_worker_root(fuzz_inputs)
shell.remove_directory(worker_fuzz_inputs, recreate=True)
environment.set_value('GSUTIL_PATH', os.path.dirname(_which('gsutil')))
test_utils.setup_pubsub('test-clusterfuzz')
test_utils.create_pubsub_topic(pubsub.PubSubClient(),
'test-clusterfuzz',
'jobs-project-linux')
def test_syzkaller_kasan_android_with_env(self):
"""Test syzkaller kasan."""
environment.set_value('OS_OVERRIDE', 'ANDROID_KERNEL')
environment.set_bot_environment()
self._real_read_data_from_file = utils.read_data_from_file
test_helpers.patch(self, [
'clusterfuzz._internal.platforms.android.fetch_artifact.get',
'clusterfuzz._internal.platforms.android.kernel_utils.get_kernel_hash_and_build_id',
'clusterfuzz._internal.platforms.android.kernel_utils.get_kernel_name',
'clusterfuzz._internal.platforms.android.settings.get_product_brand',
'clusterfuzz._internal.google_cloud_utils.storage.get_file_from_cache_if_exists',
'clusterfuzz._internal.google_cloud_utils.storage.store_file_in_cache',
'clusterfuzz._internal.base.utils.write_data_to_file',
'clusterfuzz._internal.base.utils.read_data_from_file'
])
self.mock.get.side_effect = _mock_fetch_artifact_get
self.mock.get_kernel_hash_and_build_id.return_value = '40e9b2ff3a2', '12345'
self.mock.get_kernel_name.return_value = 'device_kernel'
self.mock.get_product_brand.return_value = 'google'
self.mock.get_file_from_cache_if_exists.return_value = False
self.mock.store_file_in_cache.return_value = None
self.mock.write_data_to_file = None
self.mock.read_data_from_file.side_effect = self._mock_read_data_from_file
data = self._read_test_data('kasan_syzkaller_android.txt')
expected_stack = self._read_test_data(
'kasan_syzkaller_android_linkified.txt')
actual_state = stack_analyzer.get_crash_data(data)
self.assertEqual(actual_state.crash_stacktrace, expected_stack)
def main():
"""Run a cycle of heartbeat checks to ensure Android device is running."""
logs.configure('android_heartbeat')
dates.initialize_timezone_from_environment()
environment.set_bot_environment()
monitor.initialize()
if environment.is_android_cuttlefish():
android.adb.set_cuttlefish_device_serial()
device_serial = environment.get_value('ANDROID_SERIAL')
while True:
state = android.adb.get_device_state()
if state == android.adb.DEVICE_NOT_FOUND_STRING.format(
serial=device_serial):
android.adb.connect_to_cuttlefish_device()
state = android.adb.get_device_state()
logs.log('Android device %s state: %s' % (device_serial, state))
monitoring_metrics.ANDROID_UPTIME.increment_by(
int(state == 'device'), {
'serial': device_serial or '',
'platform': environment.get_platform_group() or '',
})
time.sleep(data_types.ANDROID_HEARTBEAT_WAIT_INTERVAL)
if data_handler.bot_run_timed_out():
break
def main():
"""Update the heartbeat if there is bot activity."""
if len(sys.argv) < 2:
print('Usage: %s <log file>' % sys.argv[0])
return
environment.set_bot_environment()
logs.configure('run_heartbeat')
log_filename = sys.argv[1]
previous_state = None
# Get absolute path to heartbeat script and interpreter needed to execute it.
startup_scripts_directory = environment.get_startup_scripts_directory()
beat_script_path = os.path.join(startup_scripts_directory, BEAT_SCRIPT)
beat_interpreter = shell.get_interpreter(beat_script_path)
assert beat_interpreter
while True:
beat_command = [
beat_interpreter, beat_script_path,
str(previous_state), log_filename
]
try:
previous_state = subprocess.check_output(beat_command,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
logs.log_error('Failed to beat.', output=e.output)
except Exception:
logs.log_error('Failed to beat.')
# See if our run timed out, if yes bail out.
if data_handler.bot_run_timed_out():
break
def _prepare_initial_environment(build_directory, iterations, verbose):
"""Prepare common environment variables that don't depend on the job."""
# Create a temporary directory to use as ROOT_DIR with a copy of the default
# bot and configuration directories nested under it.
root_dir = environment.get_value('ROOT_DIR')
temp_root_dir = tempfile.mkdtemp()
environment.set_value('ROOT_DIR', temp_root_dir)
def _update_directory(directory_name, ignore_paths=None):
"""Copy a subdirectory from a checkout to a temp directory."""
if not ignore_paths:
ignore_paths = []
shutil.copytree(os.path.join(root_dir, directory_name),
os.path.join(temp_root_dir, directory_name),
ignore=lambda directory, contents: contents
if directory in ignore_paths else [])
_update_directory('bot')
_update_directory('configs')
_update_directory('resources')
_update_directory('src',
ignore_paths=[
os.path.join(root_dir, 'src', 'appengine'),
os.path.join(root_dir, 'src', 'bazel-bin'),
os.path.join(root_dir, 'src', 'bazel-genfiles'),
os.path.join(root_dir, 'src', 'bazel-out'),
os.path.join(root_dir, 'src', 'bazel-src'),
os.path.join(root_dir, 'src', 'clusterfuzz',
'_internal', 'tests'),
])
environment.set_value('CONFIG_DIR_OVERRIDE',
os.path.join(temp_root_dir, 'configs', 'test'))
environment.set_value(
'PYTHONPATH',
os.pathsep.join(
[os.path.join(temp_root_dir, 'src'),
appengine.find_sdk_path()]))
environment.set_bot_environment()
# Overrides that should not be set to the default values.
environment.set_value('APP_DIR', build_directory)
environment.set_value('BUILD_DIR', build_directory)
environment.set_value('BUILDS_DIR', build_directory)
# Some functionality must be disabled when running the tool.
environment.set_value('REPRODUCE_TOOL', True)
environment.set_value('TASK_NAME', 'reproduce')
# Force logging to console for this process and child processes.
if verbose:
environment.set_value('LOG_TO_CONSOLE', True)
if iterations:
environment.set_value('CRASH_RETRIES', iterations)
def test_set_bot_environment_default_variables(self):
"""Tests that set_bot_environment() sets default variables as
intended."""
self.assertNotIn('TEST_TIMEOUT', os.environ)
self.assertNotIn('VERSION_PATTERN', os.environ)
self.assertNotIn('WATCH_FOR_PROCESS_EXIT', os.environ)
# Now test that setting default variables works properly.
environment.set_bot_environment()
self.assertEqual(environment.get_value('TEST_TIMEOUT'), 10)
self.assertEqual(environment.get_value('VERSION_PATTERN'), '')
self.assertEqual(environment.get_value('WATCH_FOR_PROCESS_EXIT'),
False)
def main():
logs.configure('heartbeat')
dates.initialize_timezone_from_environment()
environment.set_bot_environment()
if sys.argv[1] == 'None':
previous_state = None
else:
previous_state = sys.argv[1]
log_filename = sys.argv[2]
try:
sys.stdout.write(str(beat(previous_state, log_filename)))
except Exception:
logs.log_error('Failed to beat.')
time.sleep(data_types.HEARTBEAT_WAIT_INTERVAL)
def test_lkl_linkification(self):
"""Test lkl linkification."""
environment.set_bot_environment()
self._real_read_data_from_file = utils.read_data_from_file
test_helpers.patch(self, [
'clusterfuzz._internal.platforms.android.fetch_artifact.get',
'clusterfuzz._internal.google_cloud_utils.storage.get_file_from_cache_if_exists',
'clusterfuzz._internal.google_cloud_utils.storage.store_file_in_cache',
'clusterfuzz._internal.base.utils.write_data_to_file',
'clusterfuzz._internal.base.utils.read_data_from_file'
])
self.mock.get.side_effect = _mock_fetch_artifact_get
self.mock.get_file_from_cache_if_exists.return_value = False
self.mock.store_file_in_cache.return_value = None
self.mock.write_data_to_file = None
self.mock.read_data_from_file.side_effect = self._mock_read_data_from_file
data = self._read_test_data('lkl_libfuzzer_symbolized.txt')
expected_stack = self._read_test_data(
'lkl_libfuzzer_symbolized_linkified.txt')
actual_state = stack_analyzer.get_crash_data(data)
self.assertEqual(actual_state.crash_stacktrace, expected_stack)
def main():
root_directory = environment.get_value('ROOT_DIR')
if not root_directory:
print(
'Please set ROOT_DIR environment variable to the root of the source '
'checkout before running. Exiting.')
print('For an example, check init.bash in the local directory.')
return
set_start_time()
environment.set_bot_environment()
persistent_cache.initialize()
logs.configure('run')
# Python buffering can otherwise cause exception logs in the child run_*.py
# processes to be lost.
environment.set_value('PYTHONUNBUFFERED', 1)
# Create command strings to launch bot and heartbeat.
base_directory = environment.get_startup_scripts_directory()
log_directory = environment.get_value('LOG_DIR')
bot_log = os.path.join(log_directory, 'bot.log')
bot_script_path = os.path.join(base_directory, BOT_SCRIPT)
bot_interpreter = shell.get_interpreter(bot_script_path)
assert bot_interpreter
bot_command = '%s %s' % (bot_interpreter, bot_script_path)
heartbeat_script_path = os.path.join(base_directory, HEARTBEAT_SCRIPT)
heartbeat_interpreter = shell.get_interpreter(heartbeat_script_path)
assert heartbeat_interpreter
heartbeat_command = '%s %s %s' % (heartbeat_interpreter,
heartbeat_script_path, bot_log)
run_loop(bot_command, heartbeat_command)
logs.log('Exit run.py')
def setUp(self):
helpers.patch_environ(self)
environment.set_value('OS_OVERRIDE', 'ANDROID')
environment.set_bot_environment()
adb.setup_adb()
adb.run_as_root()
# Check if the build succeeded based on the existence of the
# local archive file.
if os.path.exists(archive_path_local):
# Build success. Now, copy it to google cloud storage and make it
# public.
os.system('gsutil cp %s %s' % (archive_path_local, archive_path_remote))
os.system('gsutil acl set public-read %s' % archive_path_remote)
logs.log('Build succeeded, created %s.' % archive_filename)
else:
LAST_BUILD[tool_and_build_type] = ''
logs.log_error('Build failed, unable to create %s.' % archive_filename)
logs.log('Completed cycle, waiting for %d secs.' % wait_time)
time.sleep(wait_time)
if __name__ == '__main__':
# Make sure environment is correctly configured.
logs.configure('run_bot')
environment.set_bot_environment()
fail_wait = environment.get_value('FAIL_WAIT')
# Continue this forever.
while True:
try:
main()
except Exception:
logs.log_error('Failed to create build.')
time.sleep(fail_wait)
def main():
"""Prepare the configuration options and start requesting tasks."""
logs.configure('run_bot')
root_directory = environment.get_value('ROOT_DIR')
if not root_directory:
print('Please set ROOT_DIR environment variable to the root of the source '
'checkout before running. Exiting.')
print('For an example, check init.bash in the local directory.')
return
dates.initialize_timezone_from_environment()
environment.set_bot_environment()
monitor.initialize()
if not profiler.start_if_needed('python_profiler_bot'):
sys.exit(-1)
fuzzers_init.run()
if environment.is_trusted_host(ensure_connected=False):
from clusterfuzz._internal.bot.untrusted_runner import host
host.init()
if environment.is_untrusted_worker():
# Track revision since we won't go into the task_loop.
update_task.track_revision()
from clusterfuzz._internal.bot.untrusted_runner import \
untrusted as untrusted_worker
untrusted_worker.start_server()
assert False, 'Unreachable code'
while True:
# task_loop should be an infinite loop,
# unless we run into an exception.
error_stacktrace, clean_exit, task_payload = task_loop()
# Print the error trace to the console.
if not clean_exit:
print('Exception occurred while running "%s".' % task_payload)
print('-' * 80)
print(error_stacktrace)
print('-' * 80)
should_terminate = (
clean_exit or errors.error_in_list(error_stacktrace,
errors.BOT_ERROR_TERMINATION_LIST))
if should_terminate:
return
logs.log_error(
'Task exited with exception (payload="%s").' % task_payload,
error_stacktrace=error_stacktrace)
should_hang = errors.error_in_list(error_stacktrace,
errors.BOT_ERROR_HANG_LIST)
if should_hang:
logs.log('Start hanging forever.')
while True:
# Sleep to avoid consuming 100% of CPU.
time.sleep(60)
# See if our run timed out, if yes bail out.
if data_handler.bot_run_timed_out():
return