def test_syzkaller_kasan_android_with_env(self): """Test syzkaller kasan.""" environment.set_value('OS_OVERRIDE', 'ANDROID_KERNEL') environment.set_bot_environment() self._real_read_data_from_file = utils.read_data_from_file test_helpers.patch(self, [ 'clusterfuzz._internal.platforms.android.fetch_artifact.get', 'clusterfuzz._internal.platforms.android.kernel_utils.get_kernel_hash_and_build_id', 'clusterfuzz._internal.platforms.android.kernel_utils.get_kernel_name', 'clusterfuzz._internal.platforms.android.settings.get_product_brand', 'clusterfuzz._internal.google_cloud_utils.storage.get_file_from_cache_if_exists', 'clusterfuzz._internal.google_cloud_utils.storage.store_file_in_cache', 'clusterfuzz._internal.base.utils.write_data_to_file', 'clusterfuzz._internal.base.utils.read_data_from_file' ]) self.mock.get.side_effect = _mock_fetch_artifact_get self.mock.get_kernel_hash_and_build_id.return_value = '40e9b2ff3a2', '12345' self.mock.get_kernel_name.return_value = 'device_kernel' self.mock.get_product_brand.return_value = 'google' self.mock.get_file_from_cache_if_exists.return_value = False self.mock.store_file_in_cache.return_value = None self.mock.write_data_to_file = None self.mock.read_data_from_file.side_effect = self._mock_read_data_from_file data = self._read_test_data('kasan_syzkaller_android.txt') expected_stack = self._read_test_data( 'kasan_syzkaller_android_linkified.txt') actual_state = stack_analyzer.get_crash_data(data) self.assertEqual(actual_state.crash_stacktrace, expected_stack)
def is_still_crashing(st_index, stacktrace): """Check if the the given stackstrace indicates the testcase is still crashing""" state = stack_analyzer.get_crash_data(stacktrace, fuzz_target=fuzz_target_name, symbolize_flag=False, already_symbolized=True, detect_ooms_and_hangs=True) crash_comparer = CrashComparer(state.crash_state, testcase.crash_state) if not crash_comparer.is_similar(): return False logs.log(f'State for trial {st_index} of {testcase_id} ' f'remains similar' f'(old_state={testcase.crash_state}, ' f'new_state={state.crash_state}).') is_security = crash_analyzer.is_security_issue(state.crash_stacktrace, state.crash_type, state.crash_address) if is_security != testcase.security_flag: return False logs.log(f'Security flag for trial {st_index} of {testcase_id} ' f'still matches' f'({testcase.security_flag}).') return True
def get_unsymbolized_data(self): """Compute unsymbolized crash data if necessary or return cached result.""" if self._unsymbolized_crash_data: return self._unsymbolized_crash_data self._unsymbolized_crash_data = stack_analyzer.get_crash_data( self.output, symbolize_flag=False) return self._unsymbolized_crash_data
def get_symbolized_data(self) -> CrashInfo: """Compute symbolized crash data if necessary or return cached result.""" if self._symbolized_crash_data: return self._symbolized_crash_data self._symbolized_crash_data = stack_analyzer.get_crash_data( self.output, symbolize_flag=True) return self._symbolized_crash_data
def test_parse_output_to_processed_report(self): """Tests if given output parses to the expected symbolized stack bytes.""" self.needs_file_delete = False state = stack_analyzer.get_crash_data(SAMPLE_OUTPUT_TO_PARSE) actual_report_bytes = crash_uploader.get_symbolized_stack_bytes( state.crash_type, state.crash_address, state.frames) with open(EXPECTED_PROCESSED_REPORT_PATH, 'rb') as expected_report: expected_report_bytes = expected_report.read() self.assertEqual(actual_report_bytes, expected_report_bytes)
def handle_update(testcase, revision, stacktrace, error): """Handle update.""" logs.log('Got external update for testcase.', testcase_id=testcase.key.id()) if error: _mark_errored(testcase, revision, error) return last_tested_revision = (testcase.get_metadata('last_tested_revision') or testcase.crash_revision) if revision < last_tested_revision: logs.log_warn(f'Revision {revision} less than previously tested ' f'revision {last_tested_revision}.') return fuzz_target = testcase.get_fuzz_target() if fuzz_target: fuzz_target_name = fuzz_target.binary else: fuzz_target_name = None # Record use of fuzz target to avoid garbage collection (since fuzz_task does # not run). data_handler.record_fuzz_target(fuzz_target.engine, fuzz_target.binary, testcase.job_type) state = stack_analyzer.get_crash_data(stacktrace, fuzz_target=fuzz_target_name, symbolize_flag=False, already_symbolized=True, detect_ooms_and_hangs=True) crash_comparer = CrashComparer(state.crash_state, testcase.crash_state) if not crash_comparer.is_similar(): logs.log(f'State no longer similar (' f'testcase_id={testcase.key.id()}, ' f'old_state={testcase.crash_state}, ' f'new_state={state.crash_state})') _mark_as_fixed(testcase, revision) return is_security = crash_analyzer.is_security_issue(state.crash_stacktrace, state.crash_type, state.crash_address) if is_security != testcase.security_flag: logs.log(f'Security flag for {testcase.key.id()} no longer matches.') _mark_as_fixed(testcase, revision) return logs.log(f'{testcase.key.id()} still crashes.') testcase.last_tested_crash_stacktrace = stacktrace data_handler.update_progression_completion_metadata(testcase, revision, is_crash=True)
def post(self): """Handle a post request.""" if not auth.get_current_user(): raise helpers.AccessDeniedException() project = request.get('project') fuzz_target = request.get('fuzz_target') stacktrace = request.get('stacktrace') state = stack_analyzer.get_crash_data( stacktrace, symbolize_flag=False, fuzz_target=fuzz_target, already_symbolized=True, detect_ooms_and_hangs=True) security_flag = crash_analyzer.is_security_issue( state.crash_stacktrace, state.crash_type, state.crash_address) result = { 'state': state.crash_state, 'type': state.crash_type, 'security': security_flag, } duplicate_testcase = data_handler.find_testcase( project, state.crash_type, state.crash_state, security_flag) if duplicate_testcase: result['result'] = 'duplicate' result['duplicate_id'] = duplicate_testcase.key.id() bug_id = ( duplicate_testcase.bug_information or duplicate_testcase.group_bug_information) if bug_id: result['bug_id'] = str(bug_id) else: result['result'] = 'new' return self.render_json(result)
def test_lkl_linkification(self): """Test lkl linkification.""" environment.set_bot_environment() self._real_read_data_from_file = utils.read_data_from_file test_helpers.patch(self, [ 'clusterfuzz._internal.platforms.android.fetch_artifact.get', 'clusterfuzz._internal.google_cloud_utils.storage.get_file_from_cache_if_exists', 'clusterfuzz._internal.google_cloud_utils.storage.store_file_in_cache', 'clusterfuzz._internal.base.utils.write_data_to_file', 'clusterfuzz._internal.base.utils.read_data_from_file' ]) self.mock.get.side_effect = _mock_fetch_artifact_get self.mock.get_file_from_cache_if_exists.return_value = False self.mock.store_file_in_cache.return_value = None self.mock.write_data_to_file = None self.mock.read_data_from_file.side_effect = self._mock_read_data_from_file data = self._read_test_data('lkl_libfuzzer_symbolized.txt') expected_stack = self._read_test_data( 'lkl_libfuzzer_symbolized_linkified.txt') actual_state = stack_analyzer.get_crash_data(data) self.assertEqual(actual_state.crash_stacktrace, expected_stack)
def do_post(self): """Upload a testcase.""" email = helpers.get_user_email() testcase_id = request.get('testcaseId') uploaded_file = self.get_upload() if testcase_id and not uploaded_file: testcase = helpers.get_testcase(testcase_id) if not access.can_user_access_testcase(testcase): raise helpers.AccessDeniedException() # Use minimized testcase for upload (if available). key = (testcase.minimized_keys if testcase.minimized_keys and testcase.minimized_keys != 'NA' else testcase.fuzzed_keys) uploaded_file = blobs.get_blob_info(key) # Extract filename part from blob. uploaded_file.filename = os.path.basename( uploaded_file.filename.replace('\\', os.sep)) job_type = request.get('job') if not job_type: raise helpers.EarlyExitException('Missing job name.', 400) job = data_types.Job.query(data_types.Job.name == job_type).get() if not job: raise helpers.EarlyExitException('Invalid job name.', 400) fuzzer_name = request.get('fuzzer') job_type_lowercase = job_type.lower() for engine in fuzzing.ENGINES: if engine.lower() in job_type_lowercase: fuzzer_name = engine is_engine_job = fuzzer_name and environment.is_engine_fuzzer_job( job_type) target_name = request.get('target') if not is_engine_job and target_name: raise helpers.EarlyExitException( 'Target name is not applicable to non-engine jobs (AFL, libFuzzer).', 400) if is_engine_job and not target_name: raise helpers.EarlyExitException( 'Missing target name for engine job (AFL, libFuzzer).', 400) if (target_name and not data_types.Fuzzer.VALID_NAME_REGEX.match(target_name)): raise helpers.EarlyExitException('Invalid target name.', 400) fully_qualified_fuzzer_name = '' if is_engine_job and target_name: if job.is_external(): # External jobs don't run and set FuzzTarget entities as part of # fuzz_task. Set it here instead. fuzz_target = (data_handler.record_fuzz_target( fuzzer_name, target_name, job_type)) fully_qualified_fuzzer_name = fuzz_target.fully_qualified_name( ) target_name = fuzz_target.binary else: fully_qualified_fuzzer_name, target_name = find_fuzz_target( fuzzer_name, target_name, job_type) if (not access.has_access(need_privileged_access=False, job_type=job_type, fuzzer_name=(fully_qualified_fuzzer_name or fuzzer_name)) and not _is_uploader_allowed(email)): raise helpers.AccessDeniedException() multiple_testcases = bool(request.get('multiple')) http_flag = bool(request.get('http')) high_end_job = bool(request.get('highEnd')) bug_information = request.get('issue') crash_revision = request.get('revision') timeout = request.get('timeout') retries = request.get('retries') bug_summary_update_flag = bool(request.get('updateIssue')) quiet_flag = bool(request.get('quiet')) additional_arguments = request.get('args') app_launch_command = request.get('cmd') platform_id = request.get('platform') issue_labels = request.get('issue_labels') gestures = request.get('gestures') or '[]' stacktrace = request.get('stacktrace') crash_data = None if job.is_external(): if not stacktrace: raise helpers.EarlyExitException( 'Stacktrace required for external jobs.', 400) if not crash_revision: raise helpers.EarlyExitException( 'Revision required for external jobs.', 400) crash_data = stack_analyzer.get_crash_data( stacktrace, fuzz_target=target_name, symbolize_flag=False, already_symbolized=True, detect_ooms_and_hangs=True) elif stacktrace: raise helpers.EarlyExitException( 'Should not specify stacktrace for non-external jobs.', 400) testcase_metadata = request.get('metadata', {}) if testcase_metadata: try: testcase_metadata = json.loads(testcase_metadata) except Exception as e: raise helpers.EarlyExitException('Invalid metadata JSON.', 400) from e if not isinstance(testcase_metadata, dict): raise helpers.EarlyExitException( 'Metadata is not a JSON object.', 400) if issue_labels: testcase_metadata['issue_labels'] = issue_labels try: gestures = ast.literal_eval(gestures) except Exception as e: raise helpers.EarlyExitException('Failed to parse gestures.', 400) from e archive_state = 0 bundled = False file_path_input = '' # Certain modifications such as app launch command, issue updates are only # allowed for privileged users. privileged_user = access.has_access(need_privileged_access=True) if not privileged_user: if bug_information or bug_summary_update_flag: raise helpers.EarlyExitException( 'You are not privileged to update existing issues.', 400) need_privileged_access = utils.string_is_true( data_handler.get_value_from_job_definition( job_type, 'PRIVILEGED_ACCESS')) if need_privileged_access: raise helpers.EarlyExitException( 'You are not privileged to run this job type.', 400) if app_launch_command: raise helpers.EarlyExitException( 'You are not privileged to run arbitrary launch commands.', 400) if (testcase_metadata and not _allow_unprivileged_metadata(testcase_metadata)): raise helpers.EarlyExitException( 'You are not privileged to set testcase metadata.', 400) if additional_arguments: raise helpers.EarlyExitException( 'You are not privileged to add command-line arguments.', 400) if gestures: raise helpers.EarlyExitException( 'You are not privileged to run arbitrary gestures.', 400) if crash_revision and crash_revision.isdigit(): crash_revision = int(crash_revision) else: crash_revision = 0 if bug_information == '0': # Auto-recover from this bad input. bug_information = None if bug_information and not bug_information.isdigit(): raise helpers.EarlyExitException('Bug is not a number.', 400) if not timeout: timeout = 0 elif not timeout.isdigit() or timeout == '0': raise helpers.EarlyExitException( 'Testcase timeout must be a number greater than 0.', 400) else: timeout = int(timeout) if timeout > 120: raise helpers.EarlyExitException( 'Testcase timeout may not be greater than 120 seconds.', 400) if retries: if retries.isdigit(): retries = int(retries) else: retries = None if retries is None or retries > MAX_RETRIES: raise helpers.EarlyExitException( 'Testcase retries must be a number less than %d.' % MAX_RETRIES, 400) else: retries = None job_queue = tasks.queue_for_job(job_type, is_high_end=high_end_job) if uploaded_file is not None: filename = ''.join([ x for x in uploaded_file.filename if x not in ' ;/?:@&=+$,{}|<>()\\' ]) key = str(uploaded_file.key()) if archive.is_archive(filename): archive_state = data_types.ArchiveStatus.FUZZED if archive_state: if multiple_testcases: # Create a job to unpack an archive. metadata = data_types.BundledArchiveMetadata() metadata.blobstore_key = key metadata.timeout = timeout metadata.job_queue = job_queue metadata.job_type = job_type metadata.http_flag = http_flag metadata.archive_filename = filename metadata.uploader_email = email metadata.gestures = gestures metadata.crash_revision = crash_revision metadata.additional_arguments = additional_arguments metadata.bug_information = bug_information metadata.platform_id = platform_id metadata.app_launch_command = app_launch_command metadata.fuzzer_name = fuzzer_name metadata.overridden_fuzzer_name = fully_qualified_fuzzer_name metadata.fuzzer_binary_name = target_name metadata.put() tasks.add_task('unpack', str(metadata.key.id()), job_type, queue=tasks.queue_for_job(job_type)) # Create a testcase metadata object to show the user their upload. upload_metadata = data_types.TestcaseUploadMetadata() upload_metadata.timestamp = datetime.datetime.utcnow() upload_metadata.filename = filename upload_metadata.blobstore_key = key upload_metadata.original_blobstore_key = key upload_metadata.status = 'Pending' upload_metadata.bundled = True upload_metadata.uploader_email = email upload_metadata.retries = retries upload_metadata.bug_summary_update_flag = bug_summary_update_flag upload_metadata.quiet_flag = quiet_flag upload_metadata.additional_metadata_string = json.dumps( testcase_metadata) upload_metadata.bug_information = bug_information upload_metadata.put() helpers.log('Uploaded multiple testcases.', helpers.VIEW_OPERATION) return self.render_json({'multiple': True}) file_path_input = guess_input_file(uploaded_file, filename) if not file_path_input: raise helpers.EarlyExitException(( "Unable to detect which file to launch. The main file\'s name " 'must contain either of %s.' % str(RUN_FILE_PATTERNS)), 400) else: raise helpers.EarlyExitException('Please select a file to upload.', 400) testcase_id = data_handler.create_user_uploaded_testcase( key, key, archive_state, filename, file_path_input, timeout, job, job_queue, http_flag, gestures, additional_arguments, bug_information, crash_revision, email, platform_id, app_launch_command, fuzzer_name, fully_qualified_fuzzer_name, target_name, bundled, retries, bug_summary_update_flag, quiet_flag, additional_metadata=testcase_metadata, crash_data=crash_data) if not quiet_flag: testcase = data_handler.get_testcase_by_id(testcase_id) issue = issue_tracker_utils.get_issue_for_testcase(testcase) if issue: report_url = data_handler.TESTCASE_REPORT_URL.format( domain=data_handler.get_domain(), testcase_id=testcase_id) comment = ('ClusterFuzz is analyzing your testcase. ' 'Developers can follow the progress at %s.' % report_url) issue.save(new_comment=comment) helpers.log('Uploaded testcase %s' % testcase_id, helpers.VIEW_OPERATION) return self.render_json({'id': '%s' % testcase_id})
def process_bad_units(self, bad_units_path, quarantine_corpus_path, crashes): """Process bad units found during merge.""" # TODO(ochang): A lot of this function is similar to parts of fuzz_task. # Ideally fuzz_task can be refactored in a way that lets us share the common # code. environment.reset_current_memory_tool_options( redzone_size=DEFAULT_REDZONE) self.runner.process_sanitizer_options() logs.log('Processing bad units.') corpus_file_paths = _get_corpus_file_paths(bad_units_path) num_bad_units = 0 # Run each corpus item individually. for i, unit_path in enumerate(corpus_file_paths, 1): if i % 100 == 0: logs.log('Up to %d' % i) unit_name = os.path.basename(unit_path) if unit_name.startswith('timeout-') or unit_name.startswith( 'oom-'): # Don't waste time re-running timeout or oom testcases. self._quarantine_unit(unit_path, quarantine_corpus_path) num_bad_units += 1 continue try: result = self._run_single_unit(unit_path) except TimeoutError: # Slow unit. Quarantine it. self._quarantine_unit(unit_path, quarantine_corpus_path) num_bad_units += 1 continue if not crash_analyzer.is_memory_tool_crash(result.output): # Didn't crash. continue # Get memory tool crash information. state = stack_analyzer.get_crash_data(result.output, symbolize_flag=True) # Crashed or caused a leak. Quarantine it. unit_path = self._quarantine_unit(unit_path, quarantine_corpus_path) num_bad_units += 1 if crash_analyzer.ignore_stacktrace(state.crash_stacktrace): continue # Local de-duplication. if state.crash_state not in crashes: security_flag = crash_analyzer.is_security_issue( state.crash_stacktrace, state.crash_type, state.crash_address) crashes[state.crash_state] = CorpusCrash( state.crash_state, state.crash_type, state.crash_address, state.crash_stacktrace, unit_path, security_flag) logs.log('Found %d bad units, %d unique crashes.' % (num_bad_units, len(crashes)))