def clear_system_temp_directory(): """Clear system specific temp directory.""" def _delete_object(path, delete_func): """Delete a object with its delete function, ignoring any error.""" try: delete_func(path) except: pass if environment.get_value('SKIP_SYSTEM_TEMP_CLEANUP'): # This provides a way to avoid clearing system temporary directory when it # can interfere with other processes on the system. return # Cache system temp directory to avoid iterating through the system dir list # on every gettempdir call. Also, it helps to avoid a case where temp dir # fills up the disk and gets ignored by gettempdir. global _system_temp_dir if not _system_temp_dir: _system_temp_dir = tempfile.gettempdir() # Use a custom cleanup rather than using |remove_directory| since it # recreates the directory and can mess up permissions and symlinks. for root, dirs, files in walk(_system_temp_dir, topdown=False): for name in files: _delete_object(os.path.join(root, name), os.remove) for name in dirs: _delete_object(os.path.join(root, name), os.rmdir) logs.log('Cleared system temp directory: %s' % _system_temp_dir)
def run_and_wait(request, _): """Implementation of RunAndWait.""" process_runner = new_process.ProcessRunner(request.executable_path, request.default_args) args = {} protobuf_utils.get_protobuf_field(args, request.popen_args, 'bufsize') protobuf_utils.get_protobuf_field(args, request.popen_args, 'executable') protobuf_utils.get_protobuf_field(args, request.popen_args, 'shell') protobuf_utils.get_protobuf_field(args, request.popen_args, 'cwd') if request.popen_args.env_is_set: args['env'] = request.popen_args.env else: args['env'] = None args['additional_args'] = request.additional_args protobuf_utils.get_protobuf_field(args, request, 'timeout') protobuf_utils.get_protobuf_field(args, request, 'terminate_before_kill') protobuf_utils.get_protobuf_field(args, request, 'terminate_wait_time') protobuf_utils.get_protobuf_field(args, request, 'input_data') protobuf_utils.get_protobuf_field(args, request, 'max_stdout_len') logs.log('Running command: %s' % process_runner.get_command()) return untrusted_runner_pb2.RunAndWaitResponse( result=process_result_to_proto(process_runner.run_and_wait(**args)))
def send(to_email, subject, html_content): """Send email.""" sendgrid_api_key = db_config.get_value('sendgrid_api_key') if not sendgrid_api_key: logs.log_warn( 'Skipping email as SendGrid API key is not set in config.') return from_email = db_config.get_value('sendgrid_sender') if not from_email: logs.log_warn( 'Skipping email as SendGrid sender is not set in config.') return message = Mail(from_email=From(str(from_email)), to_emails=To(str(to_email)), subject=Subject(subject), html_content=HtmlContent(str(html_content))) try: sg = SendGridAPIClient(sendgrid_api_key) response = sg.send(message) logs.log('Sent email to %s.' % to_email, status_code=response.status_code, body=response.body, headers=response.headers) except Exception: logs.log_error('Failed to send email to %s.' % to_email)
def post(self, message): """Handle a post request.""" testcase_id = message.attributes.get('testcaseId') if not testcase_id: raise helpers.EarlyExitException('Missing testcaseId.', 400) revision = message.attributes.get('revision') if not revision or not revision.isdigit(): raise helpers.EarlyExitException('Missing revision.', 400) revision = int(revision) testcase = data_handler.get_testcase_by_id(testcase_id) job = data_types.Job.query( data_types.Job.name == testcase.job_type).get() if not job or not job.is_external(): raise helpers.EarlyExitException('Invalid job.', 400) if message.data: stacktrace = message.data.decode() else: logs.log(f'No stacktrace provided (testcase_id={testcase_id}).') stacktrace = '' error = message.attributes.get('error') handle_update(testcase, revision, stacktrace, error) return 'OK'
def save_crash_info_if_needed(testcase_id, crash_revision, job_type, crash_type, crash_address, crash_frames): """Saves crash report for chromium project, skip otherwise.""" if data_handler.get_project_name(job_type) != 'chromium': return None serialized_crash_stack_frames = get_symbolized_stack_bytes( crash_type, crash_address, crash_frames) if not serialized_crash_stack_frames: return None crash_info = CrashReportInfo( serialized_crash_stack_frames=serialized_crash_stack_frames) # Get product and version (required). platform = environment.platform() crash_info.product = PRODUCT_MAP[platform] crash_info.version = revisions.get_real_revision(crash_revision, job_type, display=True) # Update crash_info object with bot information and testcase id. crash_info.bot_id = environment.get_value('BOT_NAME') crash_info.testcase_id = int(testcase_id) # Store CrashInfo metadata. crash_report_metadata = crash_info.to_report_metadata() crash_report_metadata.job_type = job_type crash_report_metadata.crash_revision = crash_revision crash_report_metadata.put() logs.log('Created crash report entry for testcase %s.' % testcase_id) return crash_info
def main(): """Run a cycle of heartbeat checks to ensure Android device is running.""" logs.configure('android_heartbeat') dates.initialize_timezone_from_environment() environment.set_bot_environment() monitor.initialize() if environment.is_android_cuttlefish(): android.adb.set_cuttlefish_device_serial() device_serial = environment.get_value('ANDROID_SERIAL') while True: state = android.adb.get_device_state() if state == android.adb.DEVICE_NOT_FOUND_STRING.format( serial=device_serial): android.adb.connect_to_cuttlefish_device() state = android.adb.get_device_state() logs.log('Android device %s state: %s' % (device_serial, state)) monitoring_metrics.ANDROID_UPTIME.increment_by( int(state == 'device'), { 'serial': device_serial or '', 'platform': environment.get_platform_group() or '', }) time.sleep(data_types.ANDROID_HEARTBEAT_WAIT_INTERVAL) if data_handler.bot_run_timed_out(): break
def _process_project(project, bucket): """Collects coverage information for all fuzz targets in the given project and the total stats for the project.""" project_name = _basename(project) logs.log('Processing coverage for %s project.' % project_name) report_path = storage.get_cloud_storage_file_path(bucket, project) report_info = _read_json(report_path) if not report_info: logs.log_warn('Skipping code coverage for %s project.' % project_name) return # Iterate through report_info['fuzzer_stats_dir'] and prepare # CoverageInformation entities for invididual fuzz targets. entities = [] for fuzzer in storage.list_blobs(report_info['fuzzer_stats_dir'], recursive=False): entities.append( _process_fuzzer_stats(fuzzer, report_info, project_name, bucket)) logs.log('Processed coverage for %d targets in %s project.' % (len(entities), project_name)) # Prepare CoverageInformation entity for the total project stats. entities.append(_process_project_stats(report_info, project_name)) ndb_utils.put_multi(entities)
def update_fuzzer_jobs(fuzzer_entities, job_names): """Update fuzzer job mappings.""" to_delete = {} for fuzzer_entity_key in fuzzer_entities: fuzzer_entity = fuzzer_entity_key.get() for job in data_types.Job.query(): if not job.environment_string: continue job_environment = job.get_environment() if not utils.string_is_true(job_environment.get( 'MANAGED', 'False')): continue if job.name in job_names: continue logs.log('Deleting job %s' % job.name) to_delete[job.name] = job.key try: fuzzer_entity.jobs.remove(job.name) except ValueError: pass fuzzer_entity.put() fuzzer_selection.update_mappings_for_fuzzer(fuzzer_entity) if to_delete: ndb_utils.delete_multi(to_delete.values())
def add_crash_to_global_blacklist_if_needed(testcase): """Adds relevant function from testcase crash state to global blacklist.""" testcase_id = testcase.key.id() if not should_be_blacklisted(testcase): logs.log( 'Testcase %s is not a reproducible leak, skipping leak blacklist.' % testcase_id) return False function_name = get_leak_function_for_blacklist(testcase) if not function_name: logs.log_error( 'Testcase %s has invalid crash state, skipping leak blacklist.' % testcase_id) return False existing_query = data_types.Blacklist.query( data_types.Blacklist.function_name == function_name) existing_query = existing_query.filter( data_types.Blacklist.testcase_id == testcase_id) existing_query = existing_query.filter( data_types.Blacklist.tool_name == LSAN_TOOL_NAME) if existing_query.get(): logs.log_error('Item already in leak blacklist.') return False blacklist_item = data_types.Blacklist(function_name=function_name, testcase_id=testcase_id, tool_name=LSAN_TOOL_NAME) blacklist_item.put() logs.log('Added %s to leak blacklist.' % function_name) return blacklist_item
def notify_issue_update(testcase, status): """Notify that an issue update occurred (i.e. issue was filed or closed).""" topic = local_config.ProjectConfig().get('issue_updates.pubsub_topic') if not topic: return pubsub_client = pubsub.PubSubClient() pubsub_client.publish( topic, [ pubsub.Message( attributes={ 'crash_address': testcase.crash_address, 'crash_state': testcase.crash_state, 'crash_type': testcase.crash_type, 'issue_id': testcase.bug_information or '', 'security': str(testcase.security_flag).lower(), 'status': status, 'testcase_id': str(testcase.key.id()), }) ]) if status in ('verified', 'wontfix'): logs.log(f'Closing issue {testcase.github_issue_num} ' f'in GitHub repo {testcase.github_repo_id}: ' f'Testcase {testcase.key.id()} is marked as {status}.') oss_fuzz_github.close_issue(testcase)
def rsync_to_disk(self, directory, timeout=CORPUS_FILES_SYNC_TIMEOUT, delete=True): """Run gsutil to download corpus files from GCS. Overridden to have additional logging. Args: directory: Path to directory to sync to. timeout: Timeout for gsutil. delete: Whether or not to delete files on disk that don't exist locally. Returns: A bool indicating whether or not the command succeeded. """ result = GcsCorpus.rsync_to_disk( self, directory, timeout=timeout, delete=delete) if not result: return False # Checkout additional regressions corpus if set and ignore the result. if self._regressions_corpus: regressions_dir = os.path.join(directory, 'regressions') self._regressions_corpus.rsync_to_disk( regressions_dir, timeout=timeout, delete=False) num_files = _count_corpus_files(directory) if self._log_results: logs.log('%d corpus files downloaded for %s.' % (num_files, self._project_qualified_target_name)) return result
def rsync_from_disk(self, directory, timeout=CORPUS_FILES_SYNC_TIMEOUT, delete=True): """Upload local files to GCS and remove files which do not exist locally. Overridden to have additional logging. Args: directory: Path to directory to sync to. timeout: Timeout for gsutil. delete: Whether or not to delete files on GCS that don't exist locally. Returns: A bool indicating whether or not the command succeeded. """ result = GcsCorpus.rsync_from_disk( self, directory, timeout=timeout, delete=delete) num_files = _count_corpus_files(directory) if self._log_results: logs.log('%d corpus files uploaded for %s.' % (num_files, self._project_qualified_target_name)) return result
def combine_testcases_into_group(testcase_1, testcase_2, testcase_map): """Combine two testcases into a group.""" logs.log('Grouping testcase 1 ' '(crash_type=%s, crash_state=%s, security_flag=%s, group=%s) ' 'and testcase 2 ' '(crash_type=%s, crash_state=%s, security_flag=%s, group=%s).' % (testcase_1.crash_type, testcase_1.crash_state, testcase_1.security_flag, testcase_1.group_id, testcase_2.crash_type, testcase_2.crash_state, testcase_2.security_flag, testcase_2.group_id)) # If none of the two testcases have a group id, just assign a new group id to # both. if not testcase_1.group_id and not testcase_2.group_id: new_group_id = _get_new_group_id() testcase_1.group_id = new_group_id testcase_2.group_id = new_group_id return # If one of the testcase has a group id, then assign the other to reuse that # group id. if testcase_1.group_id and not testcase_2.group_id: testcase_2.group_id = testcase_1.group_id return if testcase_2.group_id and not testcase_1.group_id: testcase_1.group_id = testcase_2.group_id return # If both the testcase have their own groups, then just merge the two groups # together and reuse one of their group ids. group_id_to_reuse = testcase_1.group_id group_id_to_move = testcase_2.group_id for testcase in six.itervalues(testcase_map): if testcase.group_id == group_id_to_move: testcase.group_id = group_id_to_reuse
def is_still_crashing(st_index, stacktrace): """Check if the the given stackstrace indicates the testcase is still crashing""" state = stack_analyzer.get_crash_data(stacktrace, fuzz_target=fuzz_target_name, symbolize_flag=False, already_symbolized=True, detect_ooms_and_hangs=True) crash_comparer = CrashComparer(state.crash_state, testcase.crash_state) if not crash_comparer.is_similar(): return False logs.log(f'State for trial {st_index} of {testcase_id} ' f'remains similar' f'(old_state={testcase.crash_state}, ' f'new_state={state.crash_state}).') is_security = crash_analyzer.is_security_issue(state.crash_stacktrace, state.crash_type, state.crash_address) if is_security != testcase.security_flag: return False logs.log(f'Security flag for trial {st_index} of {testcase_id} ' f'still matches' f'({testcase.security_flag}).') return True
def file_issue(testcase): """File an issue to the GitHub repo of the project""" if not _filing_enabled(testcase): return if testcase.github_repo_id and testcase.github_issue_num: logs.log('Issue already filed under' f'issue number {testcase.github_issue_num} in ' f'Repo {testcase.github_repo_id}.') return access_token = _get_access_token() repo = _get_repo(testcase, access_token) if not repo: logs.log('Unable to file issues to the main repo of the project') return if not repo.has_issues: logs.log_warn('Unable to file issues to the main repo: ' 'Repo has disabled issues.') return issue = _post_issue(repo, testcase) _update_testcase_properties(testcase, repo, issue)
def _make_corpus_backup_public(target, corpus_fuzzer_name_override, corpus_backup_bucket_name): """Identifies old corpus backups and makes them public.""" corpus_backup_date = utils.utcnow().date() - datetime.timedelta( days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS) corpus_backup_url = corpus_manager.gcs_url_for_backup_file( corpus_backup_bucket_name, corpus_fuzzer_name_override or target.engine, target.project_qualified_name(), corpus_backup_date) if not storage.get(corpus_backup_url): logs.log_warn('Failed to find corpus backup %s.' % corpus_backup_url) return if not _set_public_acl_if_needed(corpus_backup_url): return filename = ( corpus_manager.PUBLIC_BACKUP_TIMESTAMP + os.extsep + corpus_manager.BACKUP_ARCHIVE_FORMAT) public_url = os.path.join(os.path.dirname(corpus_backup_url), filename) if not storage.copy_blob(corpus_backup_url, public_url): logs.log_error( 'Failed to overwrite %s with the latest public corpus backup.' % public_url) return if not _set_public_acl_if_needed(public_url): return logs.log('Corpus backup %s is now marked public.' % corpus_backup_url)
def _add_default_issue_metadata(testcase): """Adds the default issue metadata (e.g. components, labels) to testcase.""" default_metadata = engine_common.get_all_issue_metadata_for_testcase( testcase) if not default_metadata: return testcase_metadata = testcase.get_metadata() for key, default_value in six.iteritems(default_metadata): # Add the default issue metadata first. This gives preference to uploader # specified issue metadata. new_value_list = utils.parse_delimited(default_value, delimiter=',', strip=True, remove_empty=True) # Append uploader specified testcase metadata value to end (for preference). uploader_value = testcase_metadata.get(key, '') uploader_value_list = utils.parse_delimited(uploader_value, delimiter=',', strip=True, remove_empty=True) for value in uploader_value_list: if value not in new_value_list: new_value_list.append(value) new_value = ','.join(new_value_list) if new_value == uploader_value: continue logs.log('Updating issue metadata for {} from {} to {}.'.format( key, uploader_value, new_value)) testcase.set_metadata(key, new_value)
def _post_put_hook(self, _): if not self.key: # Failed put. An exception will be thrown automatically afterwards. return logs.log('Updated testcase %d (bug %s).' % (self.key.id(), self.bug_information or '-'))
def update_target_weights_for_engine(client, engine, specifications): """Update all fuzz target weights for the specified engine.""" matches = {} run_set = set() # All fuzzers with non-default weights must be tracked with a special # specification. This ensures that they will be restored to normal weight # once conditions causing adjustments are no longer met. target_jobs = data_types.FuzzTargetJob.query( data_types.FuzzTarget.engine == engine).filter( data_types.FuzzTargetJob.weight != 1.0) for target_job in target_jobs: matches[(target_job.fuzz_target_name, target_job.job)] = RESTORE_DEFAULT_MATCH for match in specifications: update_matches_for_specification(match, client, engine, matches, run_set) for (fuzzer, job), match in six.iteritems(matches): if (fuzzer, job) not in run_set: # This ensures that we don't reset weights for fuzzers with problems if # they didn't run in the time covered by our queries. continue update_weight_for_target(fuzzer, job, match) logs.log('Weight adjustments complete for engine %s.' % engine)
def _store_testcase_for_regression_testing(testcase, testcase_file_path): """Stores reproduction testcase for future regression testing in corpus pruning task.""" if testcase.open: # Store testcase only after the crash is fixed. return if not testcase.bug_information: # Only store crashes with bugs associated with them. return fuzz_target = data_handler.get_fuzz_target(testcase.overridden_fuzzer_name) if not fuzz_target: # No work to do, only applicable for engine fuzzers. return corpus = corpus_manager.FuzzTargetCorpus(fuzz_target.engine, fuzz_target.project_qualified_name()) regression_testcase_url = os.path.join( corpus.get_regressions_corpus_gcs_url(), utils.file_hash(testcase_file_path)) if storage.copy_file_to(testcase_file_path, regression_testcase_url): logs.log('Successfully stored testcase for regression testing: ' + regression_testcase_url) else: logs.log_error('Failed to store testcase for regression testing: ' + regression_testcase_url)
def _process_failures(self, projects, build_type): """Process failures.""" issue_tracker = issue_tracker_utils.get_issue_tracker() if not issue_tracker: raise OssFuzzBuildStatusException('Failed to get issue tracker.') for project in projects: project_name = project['name'] builds = project['history'] if not builds: continue build = builds[0] if build['success']: continue project_name = project['name'] # Do not file an issue for non-main build types, if there is a main build # failure for the same project, as the root cause might be the same. if build_type != MAIN_BUILD_TYPE: build_failure = get_build_failure(project_name, MAIN_BUILD_TYPE) if build_failure: continue build_failure = get_build_failure(project_name, build_type) build_time = get_build_time(build) if build_failure: if build_time <= build_failure.last_checked_timestamp: # No updates. continue else: build_failure = create_build_failure(project_name, build, build_type) build_failure.last_checked_timestamp = build_time build_failure.consecutive_failures += 1 if build_failure.consecutive_failures >= MIN_CONSECUTIVE_BUILD_FAILURES: if build_failure.issue_id is None: oss_fuzz_project = _get_oss_fuzz_project(project_name) if not oss_fuzz_project: logs.log( 'Project %s is disabled, skipping bug filing.' % project_name) continue build_failure.issue_id = file_bug(issue_tracker, project_name, build['build_id'], oss_fuzz_project.ccs, build_type) elif (build_failure.consecutive_failures - MIN_CONSECUTIVE_BUILD_FAILURES) % REMINDER_INTERVAL == 0: send_reminder(issue_tracker, build_failure.issue_id, build['build_id']) build_failure.put()
def _load_data(self, bigquery, fuzzer): """Load yesterday's stats into BigQuery.""" project_id = utils.get_application_id() yesterday = (self._utc_now().date() - datetime.timedelta(days=1)) date_string = yesterday.strftime('%Y%m%d') timestamp = utils.utc_date_to_timestamp(yesterday) dataset_id = fuzzer_stats.dataset_name(fuzzer) if not self._create_dataset_if_needed(bigquery, dataset_id): return for kind in STATS_KINDS: kind_name = kind.__name__ table_id = kind_name if kind == fuzzer_stats.TestcaseRun: schema = fuzzer_stats_schema.get(fuzzer) else: schema = kind.SCHEMA if not self._create_table_if_needed(bigquery, dataset_id, table_id, schema): continue gcs_path = fuzzer_stats.get_gcs_stats_path(kind_name, fuzzer, timestamp) load = { 'destinationTable': { 'projectId': project_id, 'tableId': table_id + '$' + date_string, 'datasetId': dataset_id, }, 'schemaUpdateOptions': [ 'ALLOW_FIELD_ADDITION', ], 'sourceFormat': 'NEWLINE_DELIMITED_JSON', 'sourceUris': ['gs:/' + gcs_path + '*.json'], 'writeDisposition': 'WRITE_TRUNCATE', } if schema is not None: load['schema'] = schema job_body = { 'configuration': { 'load': load, }, } logs.log("Uploading job to BigQuery.", job_body=job_body) request = bigquery.jobs().insert(projectId=project_id, body=job_body) response = request.execute() # We cannot really check the response here, as the query might be still # running, but having a BigQuery jobId in the log would make our life # simpler if we ever have to manually check the status of the query. # See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query. logs.log('Response from BigQuery: %s' % response)
def wait_until_good_state(): """Check battery and make sure it is charged beyond minimum level and temperature thresholds.""" # Battery levels are not applicable on GCE. if environment.is_android_cuttlefish() or settings.is_automotive(): return # Make sure device is online. adb.wait_for_device() # Skip battery check if done recently. last_battery_check_time = persistent_cache.get_value( LAST_BATTERY_CHECK_TIME_KEY, constructor=datetime.datetime.utcfromtimestamp) if last_battery_check_time and not dates.time_has_expired( last_battery_check_time, seconds=BATTERY_CHECK_INTERVAL): return # Initialize variables. battery_level_threshold = environment.get_value('LOW_BATTERY_LEVEL_THRESHOLD', LOW_BATTERY_LEVEL_THRESHOLD) battery_temperature_threshold = environment.get_value( 'MAX_BATTERY_TEMPERATURE_THRESHOLD', MAX_BATTERY_TEMPERATURE_THRESHOLD) device_restarted = False while True: battery_information = get_battery_level_and_temperature() if battery_information is None: logs.log_error('Failed to get battery information, skipping check.') return battery_level = battery_information['level'] battery_temperature = battery_information['temperature'] logs.log('Battery information: level (%d%%), temperature (%.1f celsius).' % (battery_level, battery_temperature)) if (battery_level >= battery_level_threshold and battery_temperature <= battery_temperature_threshold): persistent_cache.set_value(LAST_BATTERY_CHECK_TIME_KEY, time.time()) return logs.log('Battery in bad battery state, putting device in sleep mode.') if not device_restarted: adb.reboot() device_restarted = True # Change thresholds to expected levels (only if they were below minimum # thresholds). if battery_level < battery_level_threshold: battery_level_threshold = EXPECTED_BATTERY_LEVEL if battery_temperature > battery_temperature_threshold: battery_temperature_threshold = EXPECTED_BATTERY_TEMPERATURE # Stopping shell should help with shutting off a lot of services that would # otherwise use up the battery. However, we need to turn it back on to get # battery status information. adb.stop_shell() time.sleep(BATTERY_CHARGE_INTERVAL) adb.start_shell()
def _find_existing_issue(repo, issue_title): """Checking if there is an existing open issue under the same name""" for issue in repo.get_issues(): if issue.title == issue_title: logs.log( f'Issue ({issue_title}) already exists in Repo ({repo.id}).') return issue return None
def _process_fuzzer_stats(fuzzer, project_info, project_name, bucket): """Processes coverage stats for a single fuzz target.""" fuzzer_name = data_types.fuzz_target_project_qualified_name( project_name, _basename(fuzzer)) fuzzer_info_path = storage.get_cloud_storage_file_path(bucket, fuzzer) logs.log('Processing fuzzer stats for %s (%s).' % (fuzzer_name, fuzzer_info_path)) return _coverage_information(fuzzer_info_path, fuzzer_name, project_info)
def _post_put_hook(self, _): if not self.key: # Failed put. An exception will be thrown automatically afterwards. return logs.log( f'Updated testcase {self.key.id()} (bug {self.bug_information or "-"}).' )
def get_introspector_index(): """Return introspector projects status""" if storage.exists(INTROSPECTOR_INDEX_JSON_URL): introspector_index = json.loads( storage.read_data(INTROSPECTOR_INDEX_JSON_URL)) else: introspector_index = {} logs.log('Loaded introspector status: %d' % len(introspector_index)) return introspector_index
def generate_weighted_strategy_pool(strategy_list, use_generator, engine_name): """Generate a strategy pool based on probability distribution from multi armed bandit experimentation.""" # If weighted strategy selection is enabled, there will be a distribution # stored in the environment. distribution = environment.get_value('STRATEGY_SELECTION_DISTRIBUTION') selection_method = environment.get_value('STRATEGY_SELECTION_METHOD', default_value='default') # Otherwise if weighted strategy selection is not enabled (strategy selection # method is default) or if we cannot query properly, generate strategy # pool according to default parameters. We pass the combined list of # multi-armed bandit strategies and manual strategies for consideration in # the default strategy selection process. if not distribution or selection_method == 'default': return generate_default_strategy_pool(strategy_list, use_generator) # Change the distribution to a list of named tuples rather than a list of # dictionaries so that we can use the random_weighted_choice function. Filter # out probability entries from other engines. distribution_tuples = [ StrategyCombination(strategy_name=elem['strategy_name'], probability=elem['probability']) for elem in distribution if elem['engine'] == engine_name ] if not distribution_tuples: logs.log_warn( 'Tried to generate a weighted strategy pool, but do not have ' 'strategy probabilities for %s fuzzing engine.' % engine_name) return generate_default_strategy_pool(strategy_list, use_generator) strategy_selection = utils.random_weighted_choice(distribution_tuples, 'probability') strategy_name = strategy_selection.strategy_name chosen_strategies = strategy_name.split(',') pool = StrategyPool() for strategy_tuple in strategy_list: if strategy_tuple.name in chosen_strategies: pool.add_strategy(strategy_tuple) # We consider certain strategies separately as those are only supported by a # small number of fuzz targets and should be used heavily when available. for value in [ strategy_entry for strategy_entry in strategy_list if strategy_entry.manually_enable ]: if do_strategy(value): pool.add_strategy(value) logs.log('Strategy pool was generated according to weighted distribution. ' 'Chosen strategies: ' + ', '.join(pool.strategy_names)) return pool
def start_instance(): """Start an instance via undercoat.""" handle = undercoat_api_command('start_instance').output.strip() logs.log('Started undercoat instance with handle %s' % handle) # Immediately save the handle in case we crash before stop_instance() # is called add_running_handle(handle) return handle
def close_bug(issue_tracker, issue_id, project_name): """Close a build failure bug.""" logs.log('Closing build failure bug (project=%s, issue_id=%s).' % (project_name, issue_id)) issue = issue_tracker.get_original_issue(issue_id) issue.status = 'Verified' issue.save( new_comment='The latest build has succeeded, closing this issue.', notify=True)