def get_impact_on_build(build_type, current_version, testcase, testcase_file_path): """Return impact and additional trace on a prod build given build_type.""" build = build_manager.setup_production_build(build_type) if not build: raise BuildFailedException('Build setup failed for %s' % build_type.capitalize()) if not build_manager.check_app_path(): raise AppFailedException() version = build.revision if version == current_version: return Impact(current_version, likely=False) app_path = environment.get_value('APP_PATH') command = testcase_manager.get_command_line_for_application( testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) result = testcase_manager.test_for_crash_with_retries( testcase, testcase_file_path, environment.get_value('TEST_TIMEOUT'), http_flag=testcase.http_flag) if result.is_crash(): symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True) unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) stacktrace = utils.get_crash_stacktrace_output( command, symbolized_crash_stacktrace, unsymbolized_crash_stacktrace, build_type) return Impact(version, likely=False, extra_trace=stacktrace) return Impact()
def _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, revision, update_metadata=False): """Test to see if a test case reproduces in the specified revision.""" build_manager.setup_build(revision) if not build_manager.check_app_path(): raise errors.BuildSetupError(revision, job_type) if testcase_manager.check_for_bad_build(job_type, revision): log_message = 'Bad build at r%d. Skipping' % revision testcase = data_handler.get_testcase_by_id(testcase.key.id()) data_handler.update_testcase_comment(testcase, data_types.TaskState.WIP, log_message) raise errors.BadBuildError(revision, job_type) test_timeout = environment.get_value('TEST_TIMEOUT', 10) result = testcase_manager.test_for_crash_with_retries( testcase, testcase_file_path, test_timeout, http_flag=testcase.http_flag) _log_output(revision, result) if update_metadata: _update_issue_metadata(testcase) return result
def _verify_target_exists(build_directory): """Ensure that we can find the test target before running it. Separated into its own function to simplify test behavior.""" if not build_manager.check_app_path(): raise errors.ReproduceToolUnrecoverableError( 'Unable to locate app binary in {build_directory}.'.format( build_directory=build_directory))
def _testcase_reproduces_in_revision(testcase, testcase_file_path, job_type, revision, should_log=True, min_revision=None, max_revision=None): """Test to see if a test case reproduces in the specified revision.""" if should_log: log_message = 'Testing r%d' % revision if min_revision is not None and max_revision is not None: log_message += ' (current range %d:%d)' % (min_revision, max_revision) testcase = data_handler.get_testcase_by_id(testcase.key.id()) data_handler.update_testcase_comment(testcase, data_types.TaskState.WIP, log_message) build_manager.setup_build(revision) if not build_manager.check_app_path(): raise errors.BuildSetupError(revision, job_type) if testcase_manager.check_for_bad_build(job_type, revision): log_message = 'Bad build at r%d. Skipping' % revision testcase = data_handler.get_testcase_by_id(testcase.key.id()) data_handler.update_testcase_comment(testcase, data_types.TaskState.WIP, log_message) raise errors.BadBuildError(revision, job_type) test_timeout = environment.get_value('TEST_TIMEOUT', 10) result = testcase_manager.test_for_crash_with_retries( testcase, testcase_file_path, test_timeout, http_flag=testcase.http_flag) return result.is_crash()
def _check_fixed_for_custom_binary(testcase, job_type, testcase_file_path): """Simplified fixed check for test cases using custom binaries.""" revision = environment.get_value('APP_REVISION') # Update comments to reflect bot information and clean up old comments. testcase_id = testcase.key.id() testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) build_manager.setup_build() if not build_manager.check_app_path(): testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment( testcase, data_types.TaskState.ERROR, 'Build setup failed for custom binary') build_fail_wait = environment.get_value('FAIL_WAIT') tasks.add_task( 'progression', testcase_id, job_type, wait_time=build_fail_wait) return test_timeout = environment.get_value('TEST_TIMEOUT', 10) result = testcase_manager.test_for_crash_with_retries( testcase, testcase_file_path, test_timeout, http_flag=testcase.http_flag) _log_output(revision, result) # Re-fetch to finalize testcase updates in branches below. testcase = data_handler.get_testcase_by_id(testcase.key.id()) # If this still crashes on the most recent build, it's not fixed. The task # will be rescheduled by a cron job and re-attempted eventually. if result.is_crash(): app_path = environment.get_value('APP_PATH') command = testcase_manager.get_command_line_for_application( testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True) unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) stacktrace = utils.get_crash_stacktrace_output( command, symbolized_crash_stacktrace, unsymbolized_crash_stacktrace) testcase.last_tested_crash_stacktrace = data_handler.filter_stacktrace( stacktrace) data_handler.update_progression_completion_metadata( testcase, revision, is_crash=True, message='still crashes on latest custom build') return if result.unexpected_crash: testcase.set_metadata( 'crashes_on_unexpected_state', True, update_testcase=False) else: testcase.delete_metadata( 'crashes_on_unexpected_state', update_testcase=False) # Retry once on another bot to confirm our results and in case this bot is in # a bad state which we didn't catch through our usual means. if data_handler.is_first_retry_for_task(testcase, reset_after_retry=True): tasks.add_task('progression', testcase_id, job_type) data_handler.update_progression_completion_metadata(testcase, revision) return # The bug is fixed. testcase.fixed = 'Yes' testcase.open = False data_handler.update_progression_completion_metadata( testcase, revision, message='fixed on latest custom build')
def execute_task(testcase_id, job_type): """Run a test case with a different job type to see if they reproduce.""" testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return if (environment.is_engine_fuzzer_job(testcase.job_type) != environment.is_engine_fuzzer_job(job_type)): # We should never reach here. But in case we do, we should bail out as # otherwise we will run into exceptions. return # Use a cloned testcase entity with different fuzz target paramaters for # a different fuzzing engine. original_job_type = testcase.job_type testcase = _get_variant_testcase_for_job(testcase, job_type) # Setup testcase and its dependencies. file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type) if not file_list: return # Set up a custom or regular build. We explicitly omit the crash revision # since we want to test against the latest build here. try: build_manager.setup_build() except errors.BuildNotFoundError: logs.log_warn('Matching build not found.') return # Check if we have an application path. If not, our build failed to setup # correctly. if not build_manager.check_app_path(): testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment( testcase, data_types.TaskState.ERROR, 'Build setup failed with job: ' + job_type) return # Disable gestures if we're running on a different platform from that of # the original test case. use_gestures = testcase.platform == environment.platform().lower() # Reproduce the crash. app_path = environment.get_value('APP_PATH') command = testcase_manager.get_command_line_for_application( testcase_file_path, app_path=app_path, needs_http=testcase.http_flag) test_timeout = environment.get_value('TEST_TIMEOUT', 10) revision = environment.get_value('APP_REVISION') result = testcase_manager.test_for_crash_with_retries( testcase, testcase_file_path, test_timeout, http_flag=testcase.http_flag, use_gestures=use_gestures, compare_crash=False) if result.is_crash() and not result.should_ignore(): crash_state = result.get_state() crash_type = result.get_type() security_flag = result.is_security_issue() gestures = testcase.gestures if use_gestures else None one_time_crasher_flag = not testcase_manager.test_for_reproducibility( testcase.fuzzer_name, testcase.actual_fuzzer_name(), testcase_file_path, crash_state, security_flag, test_timeout, testcase.http_flag, gestures) if one_time_crasher_flag: status = data_types.TestcaseVariantStatus.FLAKY else: status = data_types.TestcaseVariantStatus.REPRODUCIBLE crash_comparer = CrashComparer(crash_state, testcase.crash_state) is_similar = (crash_comparer.is_similar() and security_flag == testcase.security_flag) unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) symbolized_crash_stacktrace = result.get_stacktrace(symbolized=True) crash_stacktrace_output = utils.get_crash_stacktrace_output( command, symbolized_crash_stacktrace, unsymbolized_crash_stacktrace) else: status = data_types.TestcaseVariantStatus.UNREPRODUCIBLE is_similar = False crash_type = None crash_state = None security_flag = False crash_stacktrace_output = 'No crash occurred.' if original_job_type == job_type: # This case happens when someone clicks 'Update last tested stacktrace using # trunk build' button. testcase = data_handler.get_testcase_by_id(testcase_id) testcase.last_tested_crash_stacktrace = ( data_handler.filter_stacktrace(crash_stacktrace_output)) testcase.set_metadata('last_tested_crash_revision', revision, update_testcase=True) else: # Regular case of variant analysis. variant = data_handler.get_testcase_variant(testcase_id, job_type) variant.status = status variant.revision = revision variant.crash_type = crash_type variant.crash_state = crash_state variant.security_flag = security_flag variant.is_similar = is_similar variant.platform = environment.platform().lower() # Explicitly skipping crash stacktrace for now as it make entities larger # and we plan to use only crash paramaters in UI. variant.put()
def execute_task(testcase_id, job_type): """Run analyze task.""" # Reset redzones. environment.reset_current_memory_tool_options(redzone_size=128) # Unset window location size and position properties so as to use default. environment.set_value('WINDOW_ARG', '') # Locate the testcase associated with the id. testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) metadata = data_types.TestcaseUploadMetadata.query( data_types.TestcaseUploadMetadata.testcase_id == int( testcase_id)).get() if not metadata: logs.log_error('Testcase %s has no associated upload metadata.' % testcase_id) testcase.key.delete() return is_lsan_enabled = environment.get_value('LSAN') if is_lsan_enabled: # Creates empty local blacklist so all leaks will be visible to uploader. leak_blacklist.create_empty_local_blacklist() # Store the bot name and timestamp in upload metadata. bot_name = environment.get_value('BOT_NAME') metadata.bot_name = bot_name metadata.timestamp = datetime.datetime.utcnow() metadata.put() # Adjust the test timeout, if user has provided one. if metadata.timeout: environment.set_value('TEST_TIMEOUT', metadata.timeout) # Adjust the number of retries, if user has provided one. if metadata.retries is not None: environment.set_value('CRASH_RETRIES', metadata.retries) # Set up testcase and get absolute testcase path. file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type) if not file_list: return # Set up build. setup_build(testcase) # Check if we have an application path. If not, our build failed # to setup correctly. if not build_manager.check_app_path(): data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, 'Build setup failed') if data_handler.is_first_retry_for_task(testcase): build_fail_wait = environment.get_value('FAIL_WAIT') tasks.add_task('analyze', testcase_id, job_type, wait_time=build_fail_wait) else: data_handler.close_invalid_uploaded_testcase( testcase, metadata, 'Build setup failed') return # Update initial testcase information. testcase.absolute_path = testcase_file_path testcase.job_type = job_type testcase.binary_flag = utils.is_binary_file(testcase_file_path) testcase.queue = tasks.default_queue() testcase.crash_state = '' # Set initial testcase metadata fields (e.g. build url, etc). data_handler.set_initial_testcase_metadata(testcase) # Update minimized arguments and use ones provided during user upload. if not testcase.minimized_arguments: minimized_arguments = environment.get_value('APP_ARGS') or '' additional_command_line_flags = testcase.get_metadata( 'uploaded_additional_args') if additional_command_line_flags: minimized_arguments += ' %s' % additional_command_line_flags environment.set_value('APP_ARGS', minimized_arguments) testcase.minimized_arguments = minimized_arguments # Update other fields not set at upload time. testcase.crash_revision = environment.get_value('APP_REVISION') data_handler.set_initial_testcase_metadata(testcase) testcase.put() # Initialize some variables. gestures = testcase.gestures http_flag = testcase.http_flag test_timeout = environment.get_value('TEST_TIMEOUT') # Get the crash output. result = testcase_manager.test_for_crash_with_retries(testcase, testcase_file_path, test_timeout, http_flag=http_flag, compare_crash=False) # If we don't get a crash, try enabling http to see if we can get a crash. # Skip engine fuzzer jobs (e.g. libFuzzer, AFL) for which http testcase paths # are not applicable. if (not result.is_crash() and not http_flag and not environment.is_engine_fuzzer_job()): result_with_http = testcase_manager.test_for_crash_with_retries( testcase, testcase_file_path, test_timeout, http_flag=True, compare_crash=False) if result_with_http.is_crash(): logs.log('Testcase needs http flag for crash.') http_flag = True result = result_with_http # Refresh our object. testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return # Set application command line with the correct http flag. application_command_line = ( testcase_manager.get_command_line_for_application( testcase_file_path, needs_http=http_flag)) # Get the crash data. crashed = result.is_crash() crash_time = result.get_crash_time() state = result.get_symbolized_data() unsymbolized_crash_stacktrace = result.get_stacktrace(symbolized=False) # Get crash info object with minidump info. Also, re-generate unsymbolized # stacktrace if needed. crash_info, _ = (crash_uploader.get_crash_info_and_stacktrace( application_command_line, state.crash_stacktrace, gestures)) if crash_info: testcase.minidump_keys = crash_info.store_minidump() if not crashed: # Could not reproduce the crash. log_message = ('Testcase didn\'t crash in %d seconds (with retries)' % test_timeout) data_handler.update_testcase_comment(testcase, data_types.TaskState.FINISHED, log_message) # In the general case, we will not attempt to symbolize if we do not detect # a crash. For user uploads, we should symbolize anyway to provide more # information about what might be happening. crash_stacktrace_output = utils.get_crash_stacktrace_output( application_command_line, state.crash_stacktrace, unsymbolized_crash_stacktrace) testcase.crash_stacktrace = data_handler.filter_stacktrace( crash_stacktrace_output) # For an unreproducible testcase, retry once on another bot to confirm # our results and in case this bot is in a bad state which we didn't catch # through our usual means. if data_handler.is_first_retry_for_task(testcase): testcase.status = 'Unreproducible, retrying' testcase.put() tasks.add_task('analyze', testcase_id, job_type) return data_handler.close_invalid_uploaded_testcase(testcase, metadata, 'Unreproducible') # A non-reproducing testcase might still impact production branches. # Add the impact task to get that information. task_creation.create_impact_task_if_needed(testcase) return # Update testcase crash parameters. testcase.http_flag = http_flag testcase.crash_type = state.crash_type testcase.crash_address = state.crash_address testcase.crash_state = state.crash_state crash_stacktrace_output = utils.get_crash_stacktrace_output( application_command_line, state.crash_stacktrace, unsymbolized_crash_stacktrace) testcase.crash_stacktrace = data_handler.filter_stacktrace( crash_stacktrace_output) # Try to guess if the bug is security or not. security_flag = crash_analyzer.is_security_issue(state.crash_stacktrace, state.crash_type, state.crash_address) testcase.security_flag = security_flag # If it is, guess the severity. if security_flag: testcase.security_severity = severity_analyzer.get_security_severity( state.crash_type, state.crash_stacktrace, job_type, bool(gestures)) log_message = ('Testcase crashed in %d seconds (r%d)' % (crash_time, testcase.crash_revision)) data_handler.update_testcase_comment(testcase, data_types.TaskState.FINISHED, log_message) # See if we have to ignore this crash. if crash_analyzer.ignore_stacktrace(state.crash_stacktrace): data_handler.close_invalid_uploaded_testcase(testcase, metadata, 'Irrelavant') return # Test for reproducibility. one_time_crasher_flag = not testcase_manager.test_for_reproducibility( testcase.fuzzer_name, testcase.actual_fuzzer_name(), testcase_file_path, state.crash_state, security_flag, test_timeout, http_flag, gestures) testcase.one_time_crasher_flag = one_time_crasher_flag # Check to see if this is a duplicate. data_handler.check_uploaded_testcase_duplicate(testcase, metadata) # Set testcase and metadata status if not set already. if testcase.status == 'Duplicate': # For testcase uploaded by bots (with quiet flag), don't create additional # tasks. if metadata.quiet_flag: data_handler.close_invalid_uploaded_testcase( testcase, metadata, 'Duplicate') return else: # New testcase. testcase.status = 'Processed' metadata.status = 'Confirmed' # Reset the timestamp as well, to respect # data_types.MIN_ELAPSED_TIME_SINCE_REPORT. Otherwise it may get filed by # triage task prematurely without the grouper having a chance to run on this # testcase. testcase.timestamp = utils.utcnow() # Add new leaks to global blacklist to avoid detecting duplicates. # Only add if testcase has a direct leak crash and if it's reproducible. if is_lsan_enabled: leak_blacklist.add_crash_to_global_blacklist_if_needed(testcase) # Update the testcase values. testcase.put() # Update the upload metadata. metadata.security_flag = security_flag metadata.put() _add_default_issue_metadata(testcase) # Create tasks to # 1. Minimize testcase (minimize). # 2. Find regression range (regression). # 3. Find testcase impact on production branches (impact). # 4. Check whether testcase is fixed (progression). # 5. Get second stacktrace from another job in case of # one-time crashes (stack). task_creation.create_tasks(testcase)
def execute_task(testcase_id, job_type): """Execute a symbolize command.""" # Locate the testcase associated with the id. testcase = data_handler.get_testcase_by_id(testcase_id) # We should atleast have a symbolized debug or release build. if not build_manager.has_symbolized_builds(): return data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) # Setup testcase and its dependencies. file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type) if not file_list: return # Initialize variables. build_fail_wait = environment.get_value('FAIL_WAIT') old_crash_stacktrace = data_handler.get_stacktrace(testcase) sym_crash_type = testcase.crash_type sym_crash_address = testcase.crash_address sym_crash_state = testcase.crash_state sym_redzone = DEFAULT_REDZONE warmup_timeout = environment.get_value('WARMUP_TIMEOUT') # Decide which build revision to use. if testcase.crash_stacktrace == 'Pending': # This usually happen when someone clicked the 'Update stacktrace from # trunk' button on the testcase details page. In this case, we are forced # to use trunk. No revision -> trunk build. build_revision = None else: build_revision = testcase.crash_revision # Set up a custom or regular build based on revision. build_manager.setup_build(build_revision) # Get crash revision used in setting up build. crash_revision = environment.get_value('APP_REVISION') if not build_manager.check_app_path(): testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, 'Build setup failed') tasks.add_task('symbolize', testcase_id, job_type, wait_time=build_fail_wait) return # ASAN tool settings (if the tool is used). # See if we can get better stacks with higher redzone sizes. # A UAF might actually turn out to be OOB read/write with a bigger redzone. if environment.tool_matches('ASAN', job_type) and testcase.security_flag: redzone = MAX_REDZONE while redzone >= MIN_REDZONE: environment.reset_current_memory_tool_options( redzone_size=testcase.redzone, disable_ubsan=testcase.disable_ubsan) process_handler.terminate_stale_application_instances() command = testcase_manager.get_command_line_for_application( testcase_file_path, needs_http=testcase.http_flag) return_code, crash_time, output = (process_handler.run_process( command, timeout=warmup_timeout, gestures=testcase.gestures)) crash_result = CrashResult(return_code, crash_time, output) if crash_result.is_crash() and 'AddressSanitizer' in output: state = crash_result.get_symbolized_data() security_flag = crash_result.is_security_issue() if (not crash_analyzer.ignore_stacktrace( state.crash_stacktrace) and security_flag == testcase.security_flag and state.crash_type == testcase.crash_type and (state.crash_type != sym_crash_type or state.crash_state != sym_crash_state)): logs.log( 'Changing crash parameters.\nOld : %s, %s, %s' % (sym_crash_type, sym_crash_address, sym_crash_state)) sym_crash_type = state.crash_type sym_crash_address = state.crash_address sym_crash_state = state.crash_state sym_redzone = redzone old_crash_stacktrace = state.crash_stacktrace logs.log( '\nNew : %s, %s, %s' % (sym_crash_type, sym_crash_address, sym_crash_state)) break redzone /= 2 # We should have atleast a symbolized debug or a release build. symbolized_builds = build_manager.setup_symbolized_builds(crash_revision) if (not symbolized_builds or (not build_manager.check_app_path() and not build_manager.check_app_path('APP_PATH_DEBUG'))): testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, 'Build setup failed') tasks.add_task('symbolize', testcase_id, job_type, wait_time=build_fail_wait) return # Increase malloc_context_size to get all stack frames. Default is 30. environment.reset_current_memory_tool_options( redzone_size=sym_redzone, malloc_context_size=STACK_FRAME_COUNT, symbolize_inline_frames=True, disable_ubsan=testcase.disable_ubsan) # TSAN tool settings (if the tool is used). if environment.tool_matches('TSAN', job_type): environment.set_tsan_max_history_size() # Do the symbolization if supported by this application. result, sym_crash_stacktrace = (get_symbolized_stacktraces( testcase_file_path, testcase, old_crash_stacktrace, sym_crash_state)) # Update crash parameters. testcase = data_handler.get_testcase_by_id(testcase_id) testcase.crash_type = sym_crash_type testcase.crash_address = sym_crash_address testcase.crash_state = sym_crash_state testcase.crash_stacktrace = ( data_handler.filter_stacktrace(sym_crash_stacktrace)) if not result: data_handler.update_testcase_comment( testcase, data_types.TaskState.ERROR, 'Unable to reproduce crash, skipping ' 'stacktrace update') else: # Switch build url to use the less-optimized symbolized build with better # stacktrace. build_url = environment.get_value('BUILD_URL') if build_url: testcase.set_metadata('build_url', build_url, update_testcase=False) data_handler.update_testcase_comment(testcase, data_types.TaskState.FINISHED) testcase.symbolized = True testcase.crash_revision = crash_revision testcase.put() # We might have updated the crash state. See if we need to marked as duplicate # based on other testcases. data_handler.handle_duplicate_entry(testcase) task_creation.create_blame_task_if_needed(testcase) # Switch current directory before builds cleanup. root_directory = environment.get_value('ROOT_DIR') os.chdir(root_directory) # Cleanup symbolized builds which are space-heavy. symbolized_builds.delete()