def _make_bisection_request(pubsub_topic, testcase, target, bisect_type): """Make a bisection request to the external bisection service. Returns whether or not a request was actually made.""" if bisect_type == 'fixed': old_commit, new_commit = _get_commits(testcase.fixed, testcase.job_type) elif bisect_type == 'regressed': old_commit, new_commit = _get_commits(testcase.regression, testcase.job_type) else: raise ValueError('Invalid bisection type: ' + bisect_type) if not new_commit: # old_commit can be empty (i.e. '0' case), but new_commit should never be. return False old_commit, new_commit = _check_commits(testcase, bisect_type, old_commit, new_commit) reproducer = blobs.read_key(testcase.minimized_keys or testcase.fuzzed_keys) pubsub_client = pubsub.PubSubClient() pubsub_client.publish(pubsub_topic, [ pubsub.Message( reproducer, { 'type': bisect_type, 'project_name': target.project, 'sanitizer': environment.SANITIZER_NAME_MAP[ environment.get_memory_tool_name(testcase.job_type)], 'fuzz_target': target.binary, 'old_commit': old_commit, 'new_commit': new_commit, 'testcase_id': str(testcase.key.id()), 'issue_id': testcase.bug_information, 'crash_type': testcase.crash_type, 'crash_state': testcase.crash_state, 'security': str(testcase.security_flag), 'severity': severity_analyzer.severity_to_string( testcase.security_severity), 'timestamp': testcase.timestamp.isoformat(), }) ]) return True
def do_fork(): """Return whether or not to do fork mode.""" # TODO(crbug.com/920355): Reenable this when fork mode works with ChromeOS's # MSAN. job_name = environment.get_value('JOB_NAME') memory_tool = environment.get_memory_tool_name(job_name) if memory_tool == 'MSAN' and environment.is_chromeos_system_job(): return False return engine_common.decide_with_probability( engine_common.get_strategy_probability(strategy.FORK_STRATEGY, default=FORK_PROBABILITY))
def update_job_weight(job_name, multiplier): """Update a job weight.""" tool_name = environment.get_memory_tool_name(job_name) multiplier *= SANITIZER_WEIGHTS.get(tool_name, DEFAULT_SANITIZER_WEIGHT) query = data_types.FuzzerJob.query(data_types.FuzzerJob.job == job_name) changed_weights = [] for fuzzer_job in query: if fuzzer_job.multiplier != multiplier: fuzzer_job.multiplier = multiplier changed_weights.append(fuzzer_job) if changed_weights: ndb.put_multi(changed_weights)
def request_bisection(testcase, bisect_type): """Request precise bisection.""" pubsub_topic = local_config.ProjectConfig().get( 'bisect_service.pubsub_topic') if not pubsub_topic: return target = testcase.get_fuzz_target() if not target: return if bisect_type == 'fixed': old_commit, new_commit = _get_commits(testcase.fixed, testcase.job_type) elif bisect_type == 'regressed': old_commit, new_commit = _get_commits(testcase.regression, testcase.job_type) else: raise ValueError('Invalid bisection type: ' + bisect_type) reproducer = blobs.read_key(testcase.minimized_keys or testcase.fuzzed_keys) pubsub_client = pubsub.PubSubClient() pubsub_client.publish( pubsub_topic, pubsub.Message( reproducer, { 'type': bisect_type, 'project_name': target.project, 'sanitizer': environment.SANITIZER_NAME_MAP[ environment.get_memory_tool_name(testcase.job_type)], 'fuzz_target': target.binary, 'old_commit': old_commit, 'new_commit': new_commit, 'testcase_id': testcase.key.id(), 'issue_id': testcase.bug_information, 'crash_type': testcase.crash_type, 'security': str(testcase.security_flag), }))
def do_fork(): """Return whether or not to do fork mode.""" # TODO(metzman): Find a workaround for Windows command line limit before # re-enabling this. if environment.platform() == 'WINDOWS': return False # TODO(crbug.com/920355): Reenable this when fork mode works with ChromeOS's # MSAN. job_name = environment.get_value('JOB_NAME') memory_tool = environment.get_memory_tool_name(job_name) if memory_tool == 'MSAN' and environment.is_chromeos_system_job(): return False return engine_common.decide_with_probability( engine_common.get_strategy_probability(strategy.FORK_STRATEGY, default=FORK_PROBABILITY))
def _make_bisection_request(pubsub_topic, testcase, target, bisect_type): """Make a bisection request to the external bisection service.""" if bisect_type == 'fixed': old_commit, new_commit = _get_commits(testcase.fixed, testcase.job_type) elif bisect_type == 'regressed': old_commit, new_commit = _get_commits(testcase.regression, testcase.job_type) else: raise ValueError('Invalid bisection type: ' + bisect_type) if not old_commit or not new_commit: return reproducer = blobs.read_key(testcase.minimized_keys or testcase.fuzzed_keys) pubsub_client = pubsub.PubSubClient() pubsub_client.publish( pubsub_topic, pubsub.Message( reproducer, { 'type': bisect_type, 'project_name': target.project, 'sanitizer': environment.SANITIZER_NAME_MAP[ environment.get_memory_tool_name(testcase.job_type)], 'fuzz_target': target.binary, 'old_commit': old_commit, 'new_commit': new_commit, 'testcase_id': testcase.key.id(), 'issue_id': testcase.bug_information, 'crash_type': testcase.crash_type, 'security': str(testcase.security_flag), }))
def add_external_task(command, testcase_id, job): """Add external task.""" if command != 'progression': # Only progression is supported. return pubsub_client = pubsub.PubSubClient() topic_name = job.external_reproduction_topic assert topic_name is not None testcase = data_handler.get_testcase_by_id(testcase_id) fuzz_target = testcase.get_fuzz_target() memory_tool_name = environment.get_memory_tool_name(job.name) sanitizer = environment.SANITIZER_NAME_MAP.get(memory_tool_name) job_environment = job.get_environment() if job_environment.get('CUSTOM_BINARY'): raise RuntimeError('External jobs should never have custom binaries.') build_path = (job_environment.get('RELEASE_BUILD_BUCKET_PATH') or job_environment.get('FUZZ_TARGET_BUILD_BUCKET_PATH')) if build_path is None: raise RuntimeError(f'{job.name} has no build path defined.') min_revision = (testcase.get_metadata('last_tested_revision') or testcase.crash_revision) logs.log(f'Publishing external reproduction task for {testcase_id}.') attributes = { 'project': job.project, 'target': fuzz_target.binary, 'fuzzer': testcase.fuzzer_name, 'sanitizer': sanitizer, 'job': job.name, 'testcaseId': str(testcase_id), 'buildPath': build_path, 'minRevisionAbove': str(min_revision), } reproducer = blobs.read_key(testcase.minimized_keys or testcase.fuzzed_keys) message = pubsub.Message(data=reproducer, attributes=attributes) pubsub_client.publish(topic_name, [message])
def _setup_device_and_fuzzer(self): """Build a Device and Fuzzer object based on QEMU's settings.""" # These environment variables are set when start_qemu is run. # We need them in order to ssh / otherwise communicate with the VM. fuchsia_pkey_path = environment.get_value('FUCHSIA_PKEY_PATH') fuchsia_portnum = environment.get_value('FUCHSIA_PORTNUM') fuchsia_resources_dir = environment.get_value('FUCHSIA_RESOURCES_DIR') if (not fuchsia_pkey_path or not fuchsia_portnum or not fuchsia_resources_dir): raise fuchsia.errors.FuchsiaConfigError(( 'FUCHSIA_PKEY_PATH, FUCHSIA_PORTNUM, or FUCHSIA_RESOURCES_DIR was ' 'not set')) # Fuzzer objects communicate with the VM via a Device object, # which we set up here. fuchsia_resources_dir_plus_build = os.path.join( fuchsia_resources_dir, self.FUCHSIA_BUILD_REL_PATH) self.host = Host.from_dir(fuchsia_resources_dir_plus_build) self.device = Device(self.host, 'localhost', fuchsia_portnum) self.device.set_ssh_option('StrictHostKeyChecking no') self.device.set_ssh_option('UserKnownHostsFile=/dev/null') self.device.set_ssh_identity(fuchsia_pkey_path) # Fuchsia fuzzer names have the format {package_name}/{binary_name}. package, target = self.executable_path.split('/') test_data_dir = os.path.join(fuchsia_resources_dir_plus_build, self.FUZZER_TEST_DATA_REL_PATH, package, target) # Finally, we set up the Fuzzer object itself, which will run our fuzzer! sanitizer = environment.get_memory_tool_name( environment.get_value('JOB_NAME')).lower() self.fuzzer = Fuzzer(self.device, package, target, output=test_data_dir, foreground=True, sanitizer=sanitizer)
def _prepare_predator_message(testcase): """Prepare the json sent to the Predator service for the given test case.""" result, error_message = _is_predator_testcase(testcase) if not result: _set_predator_result_with_error(testcase, error_message) return None crash_revisions_dict, crash_revision_hash = _prepare_component_revisions_dict( testcase.crash_revision, testcase.job_type) # Do a None check since we can return {} for revision = 0. if crash_revisions_dict is None: _set_predator_result_with_error( testcase, 'Failed to fetch component revisions for revision %s.' % testcase.crash_revision) return None dependency_rolls = [] start_revision_hash = end_revision_hash = None if ':' in testcase.regression: regression_parts = testcase.regression.split(':', 1) start_revision = int(regression_parts[0]) end_revision = int(regression_parts[1]) start_revisions_dict, start_revision_hash = ( _prepare_component_revisions_dict(start_revision, testcase.job_type)) # Do a None check since we can return {} for revision = 0. if start_revisions_dict is None: _set_predator_result_with_error( testcase, 'Failed to fetch component revisions for revision %s.' % start_revision) return None end_revisions_dict, end_revision_hash = ( _prepare_component_revisions_dict(end_revision, testcase.job_type)) # Do a None check since we can return {} for revision = 0. if end_revisions_dict is None: _set_predator_result_with_error( testcase, 'Failed to fetch component revisions for revision %s.' % end_revision) return None if start_revision != 0: dependency_rolls = _compute_rolls(start_revisions_dict, end_revisions_dict) # Put the current revisions dictionary in the format predator expects. crash_revision_component_revisions_list = ( _format_component_revisions_for_predator(crash_revisions_dict)) # In addition to the start and end revisions, Predator expects the regression # range to include the dependency path and repository URL in the same way that # they would be included in the dependency rolls. Note that we do not take # this from the rolls dict directly as it may not be available. src_entry = [ entry for entry in crash_revision_component_revisions_list if entry['dep_path'] == 'src' ][0] # TODO(mbarbella): This is a hack since ClusterFuzz relies on "src" as a # special-cased path, but this is only going to be the correct repository # root path some of the time. For certain cases, we must update it. repo_url = src_entry['repo_url'] real_dep_path = SRC_COMPONENT_OVERRIDES.get(repo_url, 'src') if real_dep_path != 'src': for dependency_list in [ dependency_rolls, crash_revision_component_revisions_list ]: for entry in dependency_list: if entry['dep_path'] == 'src': entry['dep_path'] = real_dep_path break regression_range = { 'dep_path': real_dep_path, 'repo_url': repo_url, 'old_revision': start_revision_hash, 'new_revision': end_revision_hash, } crash_stacktrace = _filter_stacktrace( data_handler.get_stacktrace(testcase)) return pubsub.Message(data=json.dumps({ 'stack_trace': crash_stacktrace, 'crash_revision': crash_revision_hash, 'customized_data': { 'regression_range': regression_range, 'dependency_rolls': dependency_rolls, 'dependencies': crash_revision_component_revisions_list, 'crash_type': testcase.crash_type, 'crash_address': testcase.crash_address, 'sanitizer': environment.get_memory_tool_name(testcase.job_type), 'security_flag': testcase.security_flag, 'job_type': testcase.job_type, 'testcase_id': testcase.key.id() }, 'platform': testcase.platform, 'client_id': 'clusterfuzz', 'signature': testcase.crash_state, }).encode('utf-8'))