def create_variant_tasks_if_needed(testcase):
"""Creates a variant task if needed."""
testcase_id = testcase.key.id()
jobs = ndb_utils.get_all_from_model(data_types.Job)
for job in jobs:
# The variant needs to be tested in a different job type than us.
job_type = job.name
if testcase.job_type == job_type:
continue
# Don't try to reproduce engine fuzzer testcase with blackbox fuzzer
# testcases and vice versa.
if (environment.is_engine_fuzzer_job(testcase.job_type) !=
environment.is_engine_fuzzer_job(job_type)):
continue
# Skip experimental jobs.
job_environment = job.get_environment()
if utils.string_is_true(job_environment.get('EXPERIMENTAL')):
continue
# Don't look for variants in other projects.
project_name = data_handler.get_project_name(job_type)
if testcase.project_name != project_name:
continue
queue = tasks.queue_for_platform(job.platform)
tasks.add_task('variant', testcase_id, job_type, queue)
variant = data_handler.get_testcase_variant(testcase_id, job_type)
variant.status = data_types.TestcaseVariantStatus.PENDING
variant.put()
def create_variant_tasks_if_needed(testcase):
"""Creates a variant task if needed."""
if testcase.duplicate_of:
# If another testcase exists with same params, no need to spend cycles on
# calculating variants again.
return
testcase_id = testcase.key.id()
project = data_handler.get_project_name(testcase.job_type)
jobs = data_types.Job.query(data_types.Job.project == project)
for job in jobs:
# The variant needs to be tested in a different job type than us.
job_type = job.name
if testcase.job_type == job_type:
continue
# Don't try to reproduce engine fuzzer testcase with blackbox fuzzer
# testcases and vice versa.
if (environment.is_engine_fuzzer_job(testcase.job_type) !=
environment.is_engine_fuzzer_job(job_type)):
continue
# Skip experimental jobs.
job_environment = job.get_environment()
if utils.string_is_true(job_environment.get('EXPERIMENTAL')):
continue
queue = tasks.queue_for_platform(job.platform)
tasks.add_task('variant', testcase_id, job_type, queue)
variant = data_handler.get_testcase_variant(testcase_id, job_type)
variant.status = data_types.TestcaseVariantStatus.PENDING
variant.put()
def prepare_environment_for_testcase(testcase):
"""Set various environment variables based on the test case."""
_setup_memory_tools_environment(testcase)
# Setup environment variable for windows size and location properties.
# Explicit override to avoid using the default one from job definition since
# that contains unsubsituted vars like $WIDTH, etc.
environment.set_value('WINDOW_ARG', testcase.window_argument)
# Adjust timeout based on the stored multiplier (if available).
if testcase.timeout_multiplier:
test_timeout = environment.get_value('TEST_TIMEOUT')
environment.set_value('TEST_TIMEOUT',
int(test_timeout * testcase.timeout_multiplier))
# Add FUZZ_TARGET to environment if this is a fuzz target testcase.
fuzz_target = testcase.get_metadata('fuzzer_binary_name')
if fuzz_target:
environment.set_value('FUZZ_TARGET', fuzz_target)
# Override APP_ARGS with minimized arguments (if available). Don't do this
# for variant task since other job types can have its own set of required
# arguments, so use the full set of arguments of that job.
task_name = environment.get_value('TASK_NAME')
app_args = _get_application_arguments(testcase, task_name)
if app_args:
environment.set_value('APP_ARGS', app_args)
# TODO(ochang): Remove once migration is completed. Testcases generated by the
# new pipeline aren't compatible with the legacy launchers.
if (environment.is_engine_fuzzer_job(testcase.job_type)
and '%TESTCASE%' not in testcase.minimized_arguments):
environment.set_value('USE_NEW_ENGINE_IMPL', True)
def _setup_x():
"""Start Xvfb and blackbox before running the test application."""
if environment.platform() != "LINUX":
return []
if environment.is_engine_fuzzer_job():
# For engine fuzzer jobs like AFL, libFuzzer, Xvfb is not needed as the
# those fuzz targets do not needed a UI.
return []
environment.set_value("DISPLAY", DISPLAY)
print("Creating virtual display...")
xvfb_runner = new_process.ProcessRunner("/usr/bin/Xvfb")
xvfb_process = xvfb_runner.run(additional_args=[
DISPLAY,
"-screen",
"0",
"1280x1024x24",
"-ac",
"-nolisten",
"tcp",
])
time.sleep(PROCESS_START_WAIT_SECONDS)
blackbox_runner = new_process.ProcessRunner("/usr/bin/blackbox")
blackbox_process = blackbox_runner.run()
time.sleep(PROCESS_START_WAIT_SECONDS)
# Return all handles we create so they can be terminated properly at exit.
return [xvfb_process, blackbox_process]
def _get_variant_testcase_for_job(testcase, job_type):
"""Return a testcase entity for variant task use. This changes the fuzz
target params for a particular fuzzing engine."""
if testcase.job_type == job_type:
# Update stack operation on same testcase.
return testcase
if not environment.is_engine_fuzzer_job(testcase.job_type):
# For blackbox fuzzer testcases, there is no change of fuzzer required.
return testcase
engine_name = environment.get_engine_for_job(job_type)
project = data_handler.get_project_name(job_type)
binary_name = testcase.get_metadata("fuzzer_binary_name")
fully_qualified_fuzzer_name = data_types.fuzz_target_fully_qualified_name(
engine_name, project, binary_name)
variant_testcase = data_types.clone_entity(testcase)
variant_testcase.key = testcase.key
variant_testcase.fuzzer_name = engine_name
variant_testcase.overridden_fuzzer_name = fully_qualified_fuzzer_name
variant_testcase.job_type = job_type
# Remove put() method to avoid updates. DO NOT REMOVE THIS.
variant_testcase.put = lambda: None
return variant_testcase
def setup_additional_args_for_app():
"""Select additional args for the specified app at random."""
if environment.is_engine_fuzzer_job():
# Not applicable to engine fuzzers.
return
app_name = environment.get_value('APP_NAME')
if not app_name:
return
# Convert the app_name to lowercase. Case may vary by platform.
app_name = app_name.lower()
# Hack: strip file extensions that may be appended on various platforms.
extensions_to_strip = ['.exe', '.apk']
for extension in extensions_to_strip:
app_name = utils.strip_from_right(app_name, extension)
trials = data_types.Trial.query(data_types.Trial.app_name == app_name)
trials = [trial for trial in trials if random.random() < trial.probability]
if not trials:
return
app_args = environment.get_value('APP_ARGS', '') + ' ' + trials[0].app_args
trial_app_args = trials[0].app_args
for trial in trials[1:]:
app_args += ' ' + trial.app_args
trial_app_args += ' ' + trial.app_args
environment.set_value('APP_ARGS', app_args)
environment.set_value('TRIAL_APP_ARGS', trial_app_args)
def _setup_x():
"""Start Xvfb and blackbox before running the test application."""
if environment.platform() != 'LINUX':
return []
if environment.is_engine_fuzzer_job():
# For engine fuzzer jobs like AFL, libFuzzer, Xvfb is not needed as the
# those fuzz targets do not needed a UI.
return []
environment.set_value('DISPLAY', DISPLAY)
print('Starting Xvfb...')
xvfb_runner = new_process.ProcessRunner('/usr/bin/Xvfb')
xvfb_process = xvfb_runner.run(additional_args=[
DISPLAY, '-screen', '0', '1280x1024x24', '-ac', '-nolisten', 'tcp'
])
time.sleep(PROCESS_START_WAIT_SECONDS)
print('Starting blackbox...')
blackbox_runner = new_process.ProcessRunner('/usr/bin/blackbox')
blackbox_process = blackbox_runner.run()
time.sleep(PROCESS_START_WAIT_SECONDS)
# Return all handles we create so they can be terminated properly at exit.
return [xvfb_process, blackbox_process]
def initialize_device():
"""Prepares android device for app install."""
if environment.is_engine_fuzzer_job() or environment.is_kernel_fuzzer_job(
):
# These steps are not applicable to libFuzzer and syzkaller jobs and can
# brick a device on trying to configure device build settings.
return
adb.setup_adb()
# General device configuration settings.
configure_system_build_properties()
configure_device_settings()
add_test_accounts_if_needed()
# Setup AddressSanitizer if needed.
sanitizer.setup_asan_if_needed()
# Reboot device as above steps would need it and also it brings device in a
# good state.
reboot()
# Make sure we are running as root after restart.
adb.run_as_root()
# Other configuration tasks (only to done after reboot).
wifi.configure()
setup_host_and_device_forwarder_if_needed()
settings.change_se_linux_to_permissive_mode()
app.wait_until_optimization_complete()
ui.clear_notifications()
ui.unlock_screen()
def start_web_server_if_needed():
"""Start web server for blackbox fuzzer jobs (non-engine fuzzer jobs)."""
if environment.is_engine_fuzzer_job():
return
try:
http_server.start()
except Exception:
logs.log_error('Failed to start web server, skipping.')
def update_job_weights():
"""Update job weights."""
for job in data_types.Job.query():
multiplier = DEFAULT_MULTIPLIER
if environment.is_engine_fuzzer_job(job.name):
targets_count = ndb.Key(data_types.FuzzTargetsCount, job.name).get()
if targets_count:
multiplier = targets_count.count
update_job_weight(job.name, multiplier)
def should_attempt_phase(testcase, phase):
"""Return true if we should we attempt a minimization phase."""
if (phase == MinimizationPhase.ARGUMENTS and
environment.is_engine_fuzzer_job()):
# Should not minimize arguments list for engine based fuzzer jobs.
return False
current_phase = testcase.get_metadata(
'minimization_phase', default=MinimizationPhase.GESTURES)
return phase >= current_phase
def update_job_weights():
"""Update job weights."""
for job in data_types.Job.query():
multiplier = DEFAULT_MULTIPLIER
if environment.is_engine_fuzzer_job(job.name):
targets_count = ndb.Key(data_types.FuzzTargetsCount,
job.name).get()
# If the count is 0, it may be due to a bad build or some other issue. Use
# the default weight in that case to allow for recovery.
if targets_count and targets_count.count:
multiplier = targets_count.count
update_job_weight(job.name, multiplier)
def _patch_rpaths(self, instrumented_library_paths):
"""Patch rpaths of builds to point to instrumented libraries."""
if environment.is_engine_fuzzer_job():
# Import here as this path is not available in App Engine context.
from bot.fuzzers import utils as fuzzer_utils
for target_path in fuzzer_utils.get_fuzz_targets(self.build_dir):
self._patch_rpath(target_path, instrumented_library_paths)
else:
app_path = environment.get_value('APP_PATH')
if app_path:
self._patch_rpath(app_path, instrumented_library_paths)
app_path_debug = environment.get_value('APP_PATH_DEBUG')
if app_path_debug:
self._patch_rpath(app_path_debug, instrumented_library_paths)
def _set_random_fuzz_target_for_fuzzing_if_needed(fuzz_targets, target_weights):
"""Sets a random fuzz target for fuzzing."""
if not environment.is_engine_fuzzer_job():
return
# TODO(ochang): Untie this dependency on knowledge of the current task.
task_name = environment.get_value('TASK_NAME')
if task_name != 'fuzz':
return
fuzz_targets = list(fuzz_targets)
if not fuzz_targets:
logs.log_error('No fuzz targets found. Unable to pick random one.')
return
fuzz_target = fuzzer_selection.select_fuzz_target(fuzz_targets,
target_weights)
environment.set_value('FUZZ_TARGET', fuzz_target)
logs.log('Picked fuzz target %s for fuzzing.' % fuzz_target)
def execute_task(testcase_id, job_type):
"""Run analyze task."""
# Reset redzones.
environment.reset_current_memory_tool_options(redzone_size=128)
# Unset window location size and position properties so as to use default.
environment.set_value('WINDOW_ARG', '')
# Locate the testcase associated with the id.
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED)
metadata = data_types.TestcaseUploadMetadata.query(
data_types.TestcaseUploadMetadata.testcase_id == int(testcase_id)).get()
if not metadata:
logs.log_error(
'Testcase %s has no associated upload metadata.' % testcase_id)
testcase.key.delete()
return
is_lsan_enabled = environment.get_value('LSAN')
if is_lsan_enabled:
# Creates empty local blacklist so all leaks will be visible to uploader.
leak_blacklist.create_empty_local_blacklist()
# Store the bot name and timestamp in upload metadata.
bot_name = environment.get_value('BOT_NAME')
metadata.bot_name = bot_name
metadata.timestamp = datetime.datetime.utcnow()
metadata.put()
# Adjust the test timeout, if user has provided one.
if metadata.timeout:
environment.set_value('TEST_TIMEOUT', metadata.timeout)
# Adjust the number of retries, if user has provided one.
if metadata.retries is not None:
environment.set_value('CRASH_RETRIES', metadata.retries)
# Setup testcase and get absolute testcase path.
file_list, _, testcase_file_path = setup.setup_testcase(testcase)
if not file_list:
return
# Set up a custom or regular build based on revision.
build_manager.setup_build(testcase.crash_revision)
# Check if we have an application path. If not, our build failed
# to setup correctly.
app_path = environment.get_value('APP_PATH')
if not app_path:
data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR,
'Build setup failed')
if data_handler.is_first_retry_for_task(testcase):
build_fail_wait = environment.get_value('FAIL_WAIT')
tasks.add_task(
'analyze', testcase_id, job_type, wait_time=build_fail_wait)
else:
close_invalid_testcase_and_update_status(testcase, metadata,
'Build setup failed')
return
# Update initial testcase information.
testcase.absolute_path = testcase_file_path
testcase.job_type = job_type
testcase.binary_flag = utils.is_binary_file(testcase_file_path)
testcase.queue = tasks.default_queue()
testcase.crash_state = ''
# Set initial testcase metadata fields (e.g. build url, etc).
data_handler.set_initial_testcase_metadata(testcase)
# Update minimized arguments and use ones provided during user upload.
if not testcase.minimized_arguments:
minimized_arguments = environment.get_value('APP_ARGS') or ''
additional_command_line_flags = testcase.get_metadata(
'uploaded_additional_args')
if additional_command_line_flags:
minimized_arguments += ' %s' % additional_command_line_flags
environment.set_value('APP_ARGS', minimized_arguments)
testcase.minimized_arguments = minimized_arguments
# Update other fields not set at upload time.
testcase.crash_revision = environment.get_value('APP_REVISION')
data_handler.set_initial_testcase_metadata(testcase)
testcase.put()
# Initialize some variables.
gestures = testcase.gestures
http_flag = testcase.http_flag
test_timeout = environment.get_value('TEST_TIMEOUT')
# Get the crash output.
result = testcase_manager.test_for_crash_with_retries(
testcase,
testcase_file_path,
test_timeout,
http_flag=http_flag,
compare_crash=False)
# If we don't get a crash, try enabling http to see if we can get a crash.
# Skip engine fuzzer jobs (e.g. libFuzzer, AFL) for which http testcase paths
# are not applicable.
if (not result.is_crash() and not http_flag and
not environment.is_engine_fuzzer_job()):
result_with_http = testcase_manager.test_for_crash_with_retries(
testcase,
testcase_file_path,
test_timeout,
http_flag=True,
compare_crash=False)
if result_with_http.is_crash():
logs.log('Testcase needs http flag for crash.')
http_flag = True
result = result_with_http
# Refresh our object.
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
# Set application command line with the correct http flag.
application_command_line = (
testcase_manager.get_command_line_for_application(
testcase_file_path, needs_http=http_flag))
# Get the crash data.
crashed = result.is_crash()
crash_time = result.get_crash_time()
state = result.get_symbolized_data()
unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False)
# Get crash info object with minidump info. Also, re-generate unsymbolized
# stacktrace if needed.
crash_info, _ = (
crash_uploader.get_crash_info_and_stacktrace(
application_command_line, state.crash_stacktrace, gestures))
if crash_info:
testcase.minidump_keys = crash_info.store_minidump()
if not crashed:
# Could not reproduce the crash.
log_message = (
'Testcase didn\'t crash in %d seconds (with retries)' % test_timeout)
data_handler.update_testcase_comment(
testcase, data_types.TaskState.FINISHED, log_message)
# For an unreproducible testcase, retry once on another bot to confirm
# our results and in case this bot is in a bad state which we didn't catch
# through our usual means.
if data_handler.is_first_retry_for_task(testcase):
testcase.status = 'Unreproducible, retrying'
testcase.put()
tasks.add_task('analyze', testcase_id, job_type)
return
# In the general case, we will not attempt to symbolize if we do not detect
# a crash. For user uploads, we should symbolize anyway to provide more
# information about what might be happening.
crash_stacktrace_output = utils.get_crash_stacktrace_output(
application_command_line, state.crash_stacktrace,
unsymbolized_crash_stacktrace)
testcase.crash_stacktrace = data_handler.filter_stacktrace(
crash_stacktrace_output)
close_invalid_testcase_and_update_status(testcase, metadata,
'Unreproducible')
# A non-reproducing testcase might still impact production branches.
# Add the impact task to get that information.
task_creation.create_impact_task_if_needed(testcase)
return
# Update http flag and re-run testcase to store dependencies (for bundled
# archives only).
testcase.http_flag = http_flag
if not store_testcase_dependencies_from_bundled_testcase_archive(
metadata, testcase, testcase_file_path):
return
# Update testcase crash parameters.
testcase.crash_type = state.crash_type
testcase.crash_address = state.crash_address
testcase.crash_state = state.crash_state
# Try to guess if the bug is security or not.
security_flag = crash_analyzer.is_security_issue(
state.crash_stacktrace, state.crash_type, state.crash_address)
testcase.security_flag = security_flag
# If it is, guess the severity.
if security_flag:
testcase.security_severity = severity_analyzer.get_security_severity(
state.crash_type, state.crash_stacktrace, job_type, bool(gestures))
log_message = ('Testcase crashed in %d seconds (r%d)' %
(crash_time, testcase.crash_revision))
data_handler.update_testcase_comment(testcase, data_types.TaskState.FINISHED,
log_message)
# See if we have to ignore this crash.
if crash_analyzer.ignore_stacktrace(state.crash_stacktrace):
close_invalid_testcase_and_update_status(testcase, metadata, 'Irrelavant')
return
# Test for reproducibility.
one_time_crasher_flag = not testcase_manager.test_for_reproducibility(
testcase_file_path, state.crash_state, security_flag, test_timeout,
http_flag, gestures)
testcase.one_time_crasher_flag = one_time_crasher_flag
# Check to see if this is a duplicate.
project_name = data_handler.get_project_name(job_type)
existing_testcase = data_handler.find_testcase(
project_name, state.crash_type, state.crash_state, security_flag)
if existing_testcase:
# If the existing test case is unreproducible and we are, replace the
# existing test case with this one.
if existing_testcase.one_time_crasher_flag and not one_time_crasher_flag:
duplicate_testcase = existing_testcase
original_testcase = testcase
else:
duplicate_testcase = testcase
original_testcase = existing_testcase
metadata.status = 'Duplicate'
metadata.duplicate_of = existing_testcase.key.id()
duplicate_testcase.status = 'Duplicate'
duplicate_testcase.duplicate_of = original_testcase.key.id()
duplicate_testcase.put()
# Set testcase and metadata status if not set already.
if testcase.status != 'Duplicate':
testcase.status = 'Processed'
metadata.status = 'Confirmed'
# Add new leaks to global blacklist to avoid detecting duplicates.
# Only add if testcase has a direct leak crash and if it's reproducible.
if is_lsan_enabled:
leak_blacklist.add_crash_to_global_blacklist_if_needed(testcase)
# Add application specific information in the trace.
crash_stacktrace_output = utils.get_crash_stacktrace_output(
application_command_line, state.crash_stacktrace,
unsymbolized_crash_stacktrace)
testcase.crash_stacktrace = data_handler.filter_stacktrace(
crash_stacktrace_output)
# Update the testcase values.
testcase.put()
# Update the upload metadata.
metadata.security_flag = security_flag
metadata.put()
# Create tasks to
# 1. Minimize testcase (minimize).
# 2. Find regression range (regression).
# 3. Find testcase impact on production branches (impact).
# 4. Check whether testcase is fixed (progression).
# 5. Get second stacktrace from another job in case of
# one-time crashers (stack).
task_creation.create_tasks(testcase)
def execute_task(testcase_id, job_type):
"""Run a test case with a different job type to see if they reproduce."""
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
if environment.is_engine_fuzzer_job(
testcase.job_type) != environment.is_engine_fuzzer_job(job_type):
# We should never reach here. But in case we do, we should bail out as
# otherwise we will run into exceptions.
return
# Use a cloned testcase entity with different fuzz target paramaters for
# a different fuzzing engine.
original_job_type = testcase.job_type
testcase = _get_variant_testcase_for_job(testcase, job_type)
# Setup testcase and its dependencies.
file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type)
if not file_list:
return
# Set up a custom or regular build. We explicitly omit the crash revision
# since we want to test against the latest build here.
build_manager.setup_build()
# Check if we have an application path. If not, our build failed to setup
# correctly.
if not build_manager.check_app_path():
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(
testcase,
data_types.TaskState.ERROR,
"Build setup failed with job: " + job_type,
)
return
# Disable gestures if we're running on a different platform from that of
# the original test case.
use_gestures = testcase.platform == environment.platform().lower()
# Reproduce the crash.
app_path = environment.get_value("APP_PATH")
command = testcase_manager.get_command_line_for_application(
testcase_file_path, app_path=app_path, needs_http=testcase.http_flag)
test_timeout = environment.get_value("TEST_TIMEOUT", 10)
revision = environment.get_value("APP_REVISION")
result = testcase_manager.test_for_crash_with_retries(
testcase,
testcase_file_path,
test_timeout,
http_flag=testcase.http_flag,
use_gestures=use_gestures,
compare_crash=False,
)
if result.is_crash() and not result.should_ignore():
crash_state = result.get_state()
crash_type = result.get_type()
security_flag = result.is_security_issue()
gestures = testcase.gestures if use_gestures else None
one_time_crasher_flag = not testcase_manager.test_for_reproducibility(
testcase.fuzzer_name,
testcase.actual_fuzzer_name(),
testcase_file_path,
crash_state,
security_flag,
test_timeout,
testcase.http_flag,
gestures,
)
if one_time_crasher_flag:
status = data_types.TestcaseVariantStatus.FLAKY
else:
status = data_types.TestcaseVariantStatus.REPRODUCIBLE
crash_comparer = CrashComparer(crash_state, testcase.crash_state)
is_similar = (crash_comparer.is_similar()
and security_flag == testcase.security_flag)
unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False)
symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True)
crash_stacktrace_output = utils.get_crash_stacktrace_output(
command, symbolized_crash_stacktrace,
unsymbolized_crash_stacktrace)
else:
status = data_types.TestcaseVariantStatus.UNREPRODUCIBLE
is_similar = False
crash_type = None
crash_state = None
security_flag = False
crash_stacktrace_output = "No crash occurred."
if original_job_type == job_type:
# This case happens when someone clicks 'Update last tested stacktrace using
# trunk build' button.
testcase = data_handler.get_testcase_by_id(testcase_id)
testcase.last_tested_crash_stacktrace = data_handler.filter_stacktrace(
crash_stacktrace_output)
testcase.set_metadata("last_tested_crash_revision",
revision,
update_testcase=True)
else:
# Regular case of variant analysis.
variant = data_handler.get_testcase_variant(testcase_id, job_type)
variant.status = status
variant.revision = revision
variant.crash_type = crash_type
variant.crash_state = crash_state
variant.security_flag = security_flag
variant.is_similar = is_similar
# Explicitly skipping crash stacktrace for now as it make entities larger
# and we plan to use only crash paramaters in UI.
variant.put()
def execute_task(testcase_id, job_type):
"""Run a test case with a different job type to see if they reproduce."""
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
if (environment.is_engine_fuzzer_job(testcase.job_type) !=
environment.is_engine_fuzzer_job(job_type)):
# We should never reach here. But in case we do, we should bail out as
# otherwise we will run into exceptions.
return
# Setup testcase and its dependencies.
fuzzer_override = builtin_fuzzers.get_fuzzer_for_job(job_type)
file_list, _, testcase_file_path = setup.setup_testcase(
testcase, fuzzer_override=fuzzer_override)
if not file_list:
return
# Set up a custom or regular build. We explicitly omit the crash revision
# since we want to test against the latest build here.
build_manager.setup_build()
# Check if we have an application path. If not, our build failed to setup
# correctly.
app_path = environment.get_value('APP_PATH')
if not app_path:
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(
testcase, data_types.TaskState.ERROR,
'Build setup failed with job: ' + job_type)
return
# Reproduce the crash.
command = testcase_manager.get_command_line_for_application(
testcase_file_path, app_path=app_path, needs_http=testcase.http_flag)
test_timeout = environment.get_value('TEST_TIMEOUT', 10)
revision = environment.get_value('APP_REVISION')
result = testcase_manager.test_for_crash_with_retries(
testcase,
testcase_file_path,
test_timeout,
http_flag=testcase.http_flag,
compare_crash=False)
if result.is_crash() and not result.should_ignore():
crash_state = result.get_state()
crash_type = result.get_type()
security_flag = result.is_security_issue()
one_time_crasher_flag = not testcase_manager.test_for_reproducibility(
testcase_file_path, crash_state, security_flag, test_timeout,
testcase.http_flag, testcase.gestures)
if one_time_crasher_flag:
status = data_types.TestcaseVariantStatus.FLAKY
else:
status = data_types.TestcaseVariantStatus.REPRODUCIBLE
crash_comparer = CrashComparer(crash_state, testcase.crash_state)
is_similar = (crash_comparer.is_similar()
and security_flag == testcase.security_flag)
unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False)
symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True)
crash_stacktrace_output = utils.get_crash_stacktrace_output(
command, symbolized_crash_stacktrace,
unsymbolized_crash_stacktrace)
else:
status = data_types.TestcaseVariantStatus.UNREPRODUCIBLE
is_similar = False
crash_type = None
crash_state = None
security_flag = False
crash_stacktrace_output = 'No crash occurred.'
testcase = data_handler.get_testcase_by_id(testcase_id)
if testcase.job_type == job_type:
# This case happens when someone clicks 'Update last tested stacktrace using
# trunk build' button.
testcase.last_tested_crash_stacktrace = (
data_handler.filter_stacktrace(crash_stacktrace_output))
testcase.set_metadata('last_tested_crash_revision',
revision,
update_testcase=False)
else:
# Regular case of variant analysis.
variant = data_handler.get_testcase_variant(testcase_id, job_type)
variant.status = status
variant.revision = revision
variant.crash_type = crash_type
variant.crash_state = crash_state
variant.security_flag = security_flag
variant.is_similar = is_similar
# Explicitly skipping crash stacktrace for now as it make entities larger
# and we plan to use only crash paramaters in UI.
variant.put()
def do_post(self):
"""Upload a testcase."""
email = helpers.get_user_email()
testcase_id = request.get('testcaseId')
uploaded_file = self.get_upload()
if testcase_id and not uploaded_file:
testcase = helpers.get_testcase(testcase_id)
if not access.can_user_access_testcase(testcase):
raise helpers.AccessDeniedException()
# Use minimized testcase for upload (if available).
key = (testcase.minimized_keys if testcase.minimized_keys
and testcase.minimized_keys != 'NA' else
testcase.fuzzed_keys)
uploaded_file = blobs.get_blob_info(key)
# Extract filename part from blob.
uploaded_file.filename = os.path.basename(
uploaded_file.filename.replace('\\', os.sep))
job_type = request.get('job')
if not job_type:
raise helpers.EarlyExitException('Missing job name.', 400)
if (not data_types.Job.VALID_NAME_REGEX.match(job_type)
or not job_type in data_handler.get_all_job_type_names()):
raise helpers.EarlyExitException('Invalid job name.', 400)
fuzzer_name = request.get('fuzzer')
job_type_lowercase = job_type.lower()
if 'libfuzzer' in job_type_lowercase:
fuzzer_name = 'libFuzzer'
elif 'afl' in job_type_lowercase:
fuzzer_name = 'afl'
elif 'honggfuzz' in job_type_lowercase:
fuzzer_name = 'honggfuzz'
is_engine_job = fuzzer_name and environment.is_engine_fuzzer_job(
job_type)
target_name = request.get('target')
if not is_engine_job and target_name:
raise helpers.EarlyExitException(
'Target name is not applicable to non-engine jobs (AFL, libFuzzer).',
400)
if is_engine_job and not target_name:
raise helpers.EarlyExitException(
'Missing target name for engine job (AFL, libFuzzer).', 400)
if (target_name
and not data_types.Fuzzer.VALID_NAME_REGEX.match(target_name)):
raise helpers.EarlyExitException('Invalid target name.', 400)
fully_qualified_fuzzer_name = ''
if is_engine_job and target_name:
fully_qualified_fuzzer_name, target_name = find_fuzz_target(
fuzzer_name, target_name, job_type)
if (not access.has_access(need_privileged_access=False,
job_type=job_type,
fuzzer_name=(fully_qualified_fuzzer_name
or fuzzer_name))
and not _is_uploader_allowed(email)):
raise helpers.AccessDeniedException()
multiple_testcases = bool(request.get('multiple'))
http_flag = bool(request.get('http'))
high_end_job = bool(request.get('highEnd'))
bug_information = request.get('issue')
crash_revision = request.get('revision')
timeout = request.get('timeout')
retries = request.get('retries')
bug_summary_update_flag = bool(request.get('updateIssue'))
quiet_flag = bool(request.get('quiet'))
additional_arguments = request.get('args')
app_launch_command = request.get('cmd')
platform_id = request.get('platform')
issue_labels = request.get('issue_labels')
gestures = request.get('gestures') or '[]'
testcase_metadata = request.get('metadata', {})
if testcase_metadata:
try:
testcase_metadata = json.loads(testcase_metadata)
except Exception:
raise helpers.EarlyExitException('Invalid metadata JSON.', 400)
if not isinstance(testcase_metadata, dict):
raise helpers.EarlyExitException(
'Metadata is not a JSON object.', 400)
if issue_labels:
testcase_metadata['issue_labels'] = issue_labels
try:
gestures = ast.literal_eval(gestures)
except Exception:
raise helpers.EarlyExitException('Failed to parse gestures.', 400)
archive_state = 0
bundled = False
file_path_input = ''
# Certain modifications such as app launch command, issue updates are only
# allowed for privileged users.
privileged_user = access.has_access(need_privileged_access=True)
if not privileged_user:
if bug_information or bug_summary_update_flag:
raise helpers.EarlyExitException(
'You are not privileged to update existing issues.', 400)
need_privileged_access = utils.string_is_true(
data_handler.get_value_from_job_definition(
job_type, 'PRIVILEGED_ACCESS'))
if need_privileged_access:
raise helpers.EarlyExitException(
'You are not privileged to run this job type.', 400)
if app_launch_command:
raise helpers.EarlyExitException(
'You are not privileged to run arbitrary launch commands.',
400)
if (testcase_metadata
and not _allow_unprivileged_metadata(testcase_metadata)):
raise helpers.EarlyExitException(
'You are not privileged to set testcase metadata.', 400)
if additional_arguments:
raise helpers.EarlyExitException(
'You are not privileged to add command-line arguments.',
400)
if gestures:
raise helpers.EarlyExitException(
'You are not privileged to run arbitrary gestures.', 400)
# TODO(aarya): Remove once AFL is migrated to engine pipeline.
if target_name:
additional_arguments = '%TESTCASE%'
if crash_revision and crash_revision.isdigit():
crash_revision = int(crash_revision)
else:
crash_revision = 0
if bug_information == '0': # Auto-recover from this bad input.
bug_information = None
if bug_information and not bug_information.isdigit():
raise helpers.EarlyExitException('Bug is not a number.', 400)
if not timeout:
timeout = 0
elif not timeout.isdigit() or timeout == '0':
raise helpers.EarlyExitException(
'Testcase timeout must be a number greater than 0.', 400)
else:
timeout = int(timeout)
if timeout > 120:
raise helpers.EarlyExitException(
'Testcase timeout may not be greater than 120 seconds.',
400)
if retries:
if retries.isdigit():
retries = int(retries)
else:
retries = None
if retries is None or retries > MAX_RETRIES:
raise helpers.EarlyExitException(
'Testcase retries must be a number less than %d.' %
MAX_RETRIES, 400)
else:
retries = None
job_queue = tasks.queue_for_job(job_type, is_high_end=high_end_job)
if uploaded_file is not None:
filename = ''.join([
x for x in uploaded_file.filename
if x not in ' ;/?:@&=+$,{}|<>()\\'
])
key = str(uploaded_file.key())
if archive.is_archive(filename):
archive_state = data_types.ArchiveStatus.FUZZED
if archive_state:
if multiple_testcases:
# Create a job to unpack an archive.
metadata = data_types.BundledArchiveMetadata()
metadata.blobstore_key = key
metadata.timeout = timeout
metadata.job_queue = job_queue
metadata.job_type = job_type
metadata.http_flag = http_flag
metadata.archive_filename = filename
metadata.uploader_email = email
metadata.gestures = gestures
metadata.crash_revision = crash_revision
metadata.additional_arguments = additional_arguments
metadata.bug_information = bug_information
metadata.platform_id = platform_id
metadata.app_launch_command = app_launch_command
metadata.fuzzer_name = fuzzer_name
metadata.overridden_fuzzer_name = fully_qualified_fuzzer_name
metadata.fuzzer_binary_name = target_name
metadata.put()
tasks.add_task('unpack',
str(metadata.key.id()),
job_type,
queue=tasks.queue_for_job(job_type))
# Create a testcase metadata object to show the user their upload.
upload_metadata = data_types.TestcaseUploadMetadata()
upload_metadata.timestamp = datetime.datetime.utcnow()
upload_metadata.filename = filename
upload_metadata.blobstore_key = key
upload_metadata.original_blobstore_key = key
upload_metadata.status = 'Pending'
upload_metadata.bundled = True
upload_metadata.uploader_email = email
upload_metadata.retries = retries
upload_metadata.bug_summary_update_flag = bug_summary_update_flag
upload_metadata.quiet_flag = quiet_flag
upload_metadata.additional_metadata_string = json.dumps(
testcase_metadata)
upload_metadata.put()
helpers.log('Uploaded multiple testcases.',
helpers.VIEW_OPERATION)
return self.render_json({'multiple': True})
file_path_input = guess_input_file(uploaded_file, filename)
if not file_path_input:
raise helpers.EarlyExitException((
"Unable to detect which file to launch. The main file\'s name "
'must contain either of %s.' % str(RUN_FILE_PATTERNS)),
400)
else:
raise helpers.EarlyExitException('Please select a file to upload.',
400)
testcase_id = data_handler.create_user_uploaded_testcase(
key,
key,
archive_state,
filename,
file_path_input,
timeout,
job_type,
job_queue,
http_flag,
gestures,
additional_arguments,
bug_information,
crash_revision,
email,
platform_id,
app_launch_command,
fuzzer_name,
fully_qualified_fuzzer_name,
target_name,
bundled,
retries,
bug_summary_update_flag,
quiet_flag,
additional_metadata=testcase_metadata)
if not quiet_flag:
testcase = data_handler.get_testcase_by_id(testcase_id)
issue = issue_tracker_utils.get_issue_for_testcase(testcase)
if issue:
report_url = data_handler.TESTCASE_REPORT_URL.format(
domain=data_handler.get_domain(), testcase_id=testcase_id)
comment = ('ClusterFuzz is analyzing your testcase. '
'Developers can follow the progress at %s.' %
report_url)
issue.save(new_comment=comment)
helpers.log('Uploaded testcase %s' % testcase_id,
helpers.VIEW_OPERATION)
return self.render_json({'id': '%s' % testcase_id})
def process_command(task):
"""Figures out what to do with the given task and executes the command."""
logs.log("Executing command '%s'" % task.payload())
if not task.payload().strip():
logs.log_error('Empty task received.')
return
# Parse task payload.
task_name = task.command
task_argument = task.argument
job_name = task.job
environment.set_value('TASK_NAME', task_name)
environment.set_value('TASK_ARGUMENT', task_argument)
environment.set_value('JOB_NAME', job_name)
if job_name != 'none':
job = data_types.Job.query(data_types.Job.name == job_name).get()
# Job might be removed. In that case, we don't want an exception
# raised and causing this task to be retried by another bot.
if not job:
logs.log_error("Job '%s' not found." % job_name)
return
if not job.platform:
error_string = "No platform set for job '%s'" % job_name
logs.log_error(error_string)
raise errors.BadStateError(error_string)
# A misconfiguration led to this point. Clean up the job if necessary.
job_queue_suffix = tasks.queue_suffix_for_platform(job.platform)
bot_queue_suffix = tasks.default_queue_suffix()
if job_queue_suffix != bot_queue_suffix:
# This happens rarely, store this as a hard exception.
logs.log_error(
'Wrong platform for job %s: job queue [%s], bot queue [%s].' %
(job_name, job_queue_suffix, bot_queue_suffix))
# Try to recreate the job in the correct task queue.
new_queue = (
tasks.high_end_queue() if task.high_end else tasks.regular_queue())
new_queue += job_queue_suffix
# Command override is continuously run by a bot. If we keep failing
# and recreating the task, it will just DoS the entire task queue.
# So, we don't create any new tasks in that case since it needs
# manual intervention to fix the override anyway.
if not task.is_command_override:
try:
tasks.add_task(task_name, task_argument, job_name, new_queue)
except Exception:
# This can happen on trying to publish on a non-existent topic, e.g.
# a topic for a high-end bot on another platform. In this case, just
# give up.
logs.log_error('Failed to fix platform and re-add task.')
# Add a wait interval to avoid overflowing task creation.
failure_wait_interval = environment.get_value('FAIL_WAIT')
time.sleep(failure_wait_interval)
return
if task_name != 'fuzz':
# Make sure that our platform id matches that of the testcase (for
# non-fuzz tasks).
testcase = data_handler.get_entity_by_type_and_id(data_types.Testcase,
task_argument)
if testcase:
current_platform_id = environment.get_platform_id()
testcase_platform_id = testcase.platform_id
# This indicates we are trying to run this job on the wrong platform.
# This can happen when you have different type of devices (e.g
# android) on the same platform group. In this case, we just recreate
# the task.
if (task_name != 'variant' and testcase_platform_id and
not utils.fields_match(testcase_platform_id, current_platform_id)):
logs.log(
'Testcase %d platform (%s) does not match with ours (%s), exiting'
% (testcase.key.id(), testcase_platform_id, current_platform_id))
tasks.add_task(task_name, task_argument, job_name)
return
# Some fuzzers contain additional environment variables that should be
# set for them. Append these for tests generated by these fuzzers and for
# the fuzz command itself.
fuzzer_name = None
if task_name == 'fuzz':
fuzzer_name = task_argument
elif testcase:
fuzzer_name = testcase.fuzzer_name
# Get job's environment string.
environment_string = job.get_environment_string()
if task_name == 'minimize':
# Let jobs specify a different job and fuzzer to minimize with.
job_environment = job.get_environment()
minimize_job_override = job_environment.get('MINIMIZE_JOB_OVERRIDE')
if minimize_job_override:
minimize_job = data_types.Job.query(
data_types.Job.name == minimize_job_override).get()
if minimize_job:
environment.set_value('JOB_NAME', minimize_job_override)
environment_string = minimize_job.get_environment_string()
environment_string += '\nORIGINAL_JOB_NAME = %s\n' % job_name
job_name = minimize_job_override
else:
logs.log_error(
'Job for minimization not found: %s.' % minimize_job_override)
# Fallback to using own job for minimization.
minimize_fuzzer_override = job_environment.get('MINIMIZE_FUZZER_OVERRIDE')
fuzzer_name = minimize_fuzzer_override or fuzzer_name
if fuzzer_name and not environment.is_engine_fuzzer_job(job_name):
fuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == fuzzer_name).get()
additional_default_variables = ''
additional_variables_for_job = ''
if (fuzzer and hasattr(fuzzer, 'additional_environment_string') and
fuzzer.additional_environment_string):
for line in fuzzer.additional_environment_string.splitlines():
# Job specific values may be defined in fuzzer additional
# environment variable name strings in the form
# job_name:VAR_NAME = VALUE.
if '=' in line and ':' in line.split('=', 1)[0]:
fuzzer_job_name, environment_definition = line.split(':', 1)
if fuzzer_job_name == job_name:
additional_variables_for_job += '\n%s' % environment_definition
continue
additional_default_variables += '\n%s' % line
environment_string += additional_default_variables
environment_string += additional_variables_for_job
# Update environment for the job.
update_environment_for_job(environment_string)
# Match the cpu architecture with the ones required in the job definition.
# If they don't match, then bail out and recreate task.
if not is_supported_cpu_arch_for_job():
logs.log(
'Unsupported cpu architecture specified in job definition, exiting.')
tasks.add_task(task_name, task_argument, job_name)
return
# Initial cleanup.
cleanup_task_state()
start_web_server_if_needed()
try:
run_command(task_name, task_argument, job_name)
finally:
# Final clean up.
cleanup_task_state()
