def auth_domain(): """Get the auth domain.""" domain = local_config.ProjectConfig().get('firebase.auth_domain') if domain: return domain return utils.get_application_id() + '.firebaseapp.com'
def get_current_user(): """Get the current logged in user, or None.""" if environment.is_local_development(): return User('user@localhost') current_request = request_cache.get_current_request() if local_config.AuthConfig().get('enable_loas'): loas_user = current_request.headers.get( 'X-AppEngine-LOAS-Peer-Username') if loas_user: return User(loas_user + '@google.com') iap_email = get_iap_email(current_request) if iap_email: return User(iap_email) cache_backing = request_cache.get_cache_backing() oauth_email = getattr(cache_backing, '_oauth_email', None) if oauth_email: return User(oauth_email) cached_email = getattr(cache_backing, '_cached_email', None) if cached_email: return User(cached_email) session_cookie = get_session_cookie() if not session_cookie: return None try: decoded_claims = decode_claims(get_session_cookie()) except AuthError: logs.log_warn('Invalid session cookie.') return None allowed_firebase_providers = local_config.ProjectConfig().get( 'firebase.auth_providers', ['google.com']) firebase_info = decoded_claims.get('firebase', {}) sign_in_provider = firebase_info.get('sign_in_provider') if sign_in_provider not in allowed_firebase_providers: logs.log_error(f'Firebase provider {sign_in_provider} is not enabled.') return None # Per https://docs.github.com/en/authentication/ # keeping-your-account-and-data-secure/authorizing-oauth-apps # GitHub requires emails to be verified before an OAuth app can be # authorized, so we make an exception. if (not decoded_claims.get('email_verified') and sign_in_provider != 'github.com'): return None email = decoded_claims.get('email') if not email: return None # We cache the email for this request if we've validated the user to make # subsequent get_current_user() calls fast. setattr(cache_backing, '_cached_email', email) return User(email)
def bootstrap_gcs(storage_path): """Bootstrap GCS.""" local_gcs_buckets_path = os.path.join(storage_path, 'local_gcs') if not os.path.exists(local_gcs_buckets_path): os.mkdir(local_gcs_buckets_path) config = local_config.ProjectConfig() test_blobs_bucket = os.environ.get('TEST_BLOBS_BUCKET') if test_blobs_bucket: create_local_bucket(local_gcs_buckets_path, test_blobs_bucket) else: create_local_bucket(local_gcs_buckets_path, config.get('blobs.bucket')) create_local_bucket(local_gcs_buckets_path, config.get('deployment.bucket')) create_local_bucket(local_gcs_buckets_path, config.get('bigquery.bucket')) create_local_bucket(local_gcs_buckets_path, config.get('backup.bucket')) create_local_bucket(local_gcs_buckets_path, config.get('logs.fuzzer.bucket')) create_local_bucket(local_gcs_buckets_path, config.get('env.CORPUS_BUCKET')) create_local_bucket(local_gcs_buckets_path, config.get('env.QUARANTINE_BUCKET')) create_local_bucket(local_gcs_buckets_path, config.get('env.SHARED_CORPUS_BUCKET')) create_local_bucket(local_gcs_buckets_path, config.get('env.FUZZ_LOGS_BUCKET')) create_local_bucket(local_gcs_buckets_path, config.get('env.MUTATOR_PLUGINS_BUCKET')) # Symlink local GCS bucket path to appengine src dir to bypass sandboxing # issues. common.symlink(src=local_gcs_buckets_path, target=os.path.join(appengine.SRC_DIR_PY, 'local_gcs'))
def notify_issue_update(testcase, status): """Notify that an issue update occurred (i.e. issue was filed or closed).""" topic = local_config.ProjectConfig().get('issue_updates.pubsub_topic') if not topic: return pubsub_client = pubsub.PubSubClient() pubsub_client.publish( topic, [ pubsub.Message( attributes={ 'crash_address': testcase.crash_address, 'crash_state': testcase.crash_state, 'crash_type': testcase.crash_type, 'issue_id': testcase.bug_information or '', 'security': str(testcase.security_flag).lower(), 'status': status, 'testcase_id': str(testcase.key.id()), }) ]) if status in ('verified', 'wontfix'): logs.log(f'Closing issue {testcase.github_issue_num} ' f'in GitHub repo {testcase.github_repo_id}: ' f'Testcase {testcase.key.id()} is marked as {status}.') oss_fuzz_github.close_issue(testcase)
def setUp(self): test_helpers.patch_environ(self) environment.remove_key('PROJECT_NAME') environment.remove_key('ISSUE_TRACKER') environment.remove_key('UPDATE_WEB_TESTS') self.config = local_config.ProjectConfig()
def _deployment_file_url(filename): """Helper to return deployment file url.""" deployment_bucket = local_config.ProjectConfig().get('deployment.bucket') if not deployment_bucket: return None return 'gs://{bucket}/{name}'.format(bucket=deployment_bucket, name=filename)
def blobs_bucket(): """Get the blobs bucket name.""" # Allow tests to override blobs bucket name safely. test_blobs_bucket = environment.get_value('TEST_BLOBS_BUCKET') if test_blobs_bucket: return test_blobs_bucket assert not environment.get_value('PY_UNITTESTS') return local_config.ProjectConfig().get('blobs.bucket')
def get(self): """Handle a get request.""" dest = request.get('dest', DEFAULT_REDIRECT) base_handler.check_redirect_url(dest) return self.render( 'login.html', { 'apiKey': local_config.ProjectConfig().get('firebase.api_key'), 'authDomain': auth.auth_domain(), 'dest': dest, })
def get(self): """Handle a GET request.""" # The task is supposed to be super reliable and never fail. If anything goes # wrong, we just fail with the exception going straight into StackDriver. logs.log('FuzzerCoverage task started.') bucket = local_config.ProjectConfig().get('coverage.reports.bucket') if not bucket: logs.log( 'Coverage bucket is not specified. Skipping FuzzerCoverage task.' ) return collect_fuzzer_coverage(bucket) logs.log('FuzzerCoverage task finished successfully.')
def _upload_kernel_coverage_data(kcov_path, kernel_bid): """Upload kcov data to a cloud storage bucket.""" bucket_name = local_config.ProjectConfig().get('coverage.reports.bucket') if not bucket_name: return formatted_date = str(utils.utcnow().date().isoformat()) identifier = environment.get_value('BOT_NAME') + str( utils.utcnow().isoformat()) gcs_url = (f'gs://{bucket_name}/syzkaller/{formatted_date}/{kernel_bid}/' f'{identifier}') if storage.copy_file_to(kcov_path, gcs_url): logs.log(f'Copied kcov data to {gcs_url}.')
def initialize(): """Initialize if monitoring is enabled for this bot.""" global _monitoring_v3_client global _flusher_thread if environment.get_value('LOCAL_DEVELOPMENT'): return if not local_config.ProjectConfig().get('monitoring.enabled'): return if check_module_loaded(monitoring_v3): _initialize_monitored_resource() _monitoring_v3_client = monitoring_v3.MetricServiceClient( credentials=credentials.get_default()[0]) _flusher_thread = _FlusherThread() _flusher_thread.start()
def ignore_stacktrace(crash_stacktrace): """Return whether the stacktrace needs to be ignored.""" # Filter crash based on search exclude pattern specified in job definition. search_excludes = environment.get_value('SEARCH_EXCLUDES') if search_excludes and re.search(search_excludes, crash_stacktrace): return True # Match stacktrace against custom defined blacklist regexes in project config. stack_blacklist_regexes = ( local_config.ProjectConfig().get('stacktrace.stack_blacklist_regexes')) if not stack_blacklist_regexes: return False stack_blacklist_regex = re.compile(r'(%s)' % '|'.join(stack_blacklist_regexes)) for line in crash_stacktrace.splitlines(): if stack_blacklist_regex.match(line): return True return False
def notify_issue_update(testcase, status): """Notify that an issue update occurred (i.e. issue was filed or closed).""" topic = local_config.ProjectConfig().get('issue_updates.pubsub_topic') if not topic: return pubsub_client = pubsub.PubSubClient() pubsub_client.publish(topic, [ pubsub.Message( attributes={ 'crash_address': testcase.crash_address, 'crash_state': testcase.crash_state, 'crash_type': testcase.crash_type, 'issue_id': testcase.bug_information or '', 'security': str(testcase.security_flag).lower(), 'status': status, 'testcase_id': str(testcase.key.id()), }) ])
def _get_revision_vars_url_format(job_type, platform_id=None): """Return REVISION_VARS_URL from job environment if available. Otherwise, default to one set in project.yaml. For custom binary jobs, this is not applicable.""" if job_type is None: # Force it to use env attribute in project.yaml. return local_config.ProjectConfig().get('env.REVISION_VARS_URL') custom_binary = data_handler.get_value_from_job_definition( job_type, 'CUSTOM_BINARY') if utils.string_is_true(custom_binary): return None rev_path = data_handler.get_value_from_job_definition_or_environment( job_type, 'REVISION_VARS_URL') rev_path = overrides.check_and_apply_overrides( rev_path, overrides.PLATFORM_ID_TO_REV_PATH_KEY, platform_id=platform_id) return rev_path
def get_crash_data(crash_data, symbolize_flag=True, fuzz_target=None, already_symbolized=False, detect_ooms_and_hangs=None) -> stacktraces.CrashInfo: """Get crash parameters from crash data. Crash parameters include crash type, address, state and stacktrace. If the stacktrace is not already symbolized, we will try to symbolize it unless |symbolize| flag is set to False. Symbolized stacktrace will contain inline frames, but we do exclude them for purposes of crash state generation (helps in testcase deduplication).""" # Decide whether to symbolize or not symbolize the input stacktrace. # Note that Fuchsia logs are always symbolized. if symbolize_flag: # Defer imports since stack_symbolizer pulls in a lot of things. from clusterfuzz._internal.crash_analysis.stack_parsing import \ stack_symbolizer crash_stacktrace_with_inlines = stack_symbolizer.symbolize_stacktrace( crash_data, enable_inline_frames=True) crash_stacktrace_without_inlines = stack_symbolizer.symbolize_stacktrace( crash_data, enable_inline_frames=False) else: # We are explicitly indicated to not symbolize using |symbolize_flag|. There # is no distinction between inline and non-inline frames for an unsymbolized # stacktrace. crash_stacktrace_with_inlines = crash_data crash_stacktrace_without_inlines = crash_data # Additional stack frame ignore regexes. custom_stack_frame_ignore_regexes = (local_config.ProjectConfig().get( 'stacktrace.stack_frame_ignore_regexes', [])) if environment.get_value('TASK_NAME') == 'analyze': detect_v8_runtime_errors = True else: detect_v8_runtime_errors = environment.get_value( 'DETECT_V8_RUNTIME_ERRORS', False) fuzz_target = fuzz_target or environment.get_value('FUZZ_TARGET') redzone_size = environment.get_value('REDZONE') if detect_ooms_and_hangs is None: detect_ooms_and_hangs = ( environment.get_value('REPORT_OOMS_AND_HANGS') and (not redzone_size or redzone_size <= MAX_REDZONE_SIZE_FOR_OOMS_AND_HANGS)) include_ubsan = 'halt_on_error=0' not in environment.get_value( 'UBSAN_OPTIONS', '') stack_parser = stacktraces.StackParser( symbolized=symbolize_flag or already_symbolized, detect_ooms_and_hangs=detect_ooms_and_hangs, detect_v8_runtime_errors=detect_v8_runtime_errors, custom_stack_frame_ignore_regexes=custom_stack_frame_ignore_regexes, fuzz_target=fuzz_target, include_ubsan=include_ubsan) result = stack_parser.parse(crash_stacktrace_without_inlines) # Use stacktrace with inlines for the result. if result.crash_stacktrace: result.crash_stacktrace = crash_stacktrace_with_inlines # Linkify Android Kernel or lkl stacktrace. linkify_kernel_or_lkl_stacktrace_if_needed(result) return result
def get_bucket(): """Return bucket for bigquery stats.""" return local_config.ProjectConfig().get('bigquery.bucket')
def upload_testcases_if_needed(fuzzer_name, testcase_list, testcase_directory, data_directory): """Upload test cases from the list to a cloud storage bucket.""" # Since builtin fuzzers have a coverage minimized corpus, no need to upload # test case samples for them. if fuzzer_name in fuzzing.ENGINES: return bucket_name = local_config.ProjectConfig().get( 'coverage.fuzzer-testcases.bucket') if not bucket_name: return files_list = [] has_testcases_in_testcase_directory = False has_testcases_in_data_directory = False for testcase_path in testcase_list: if testcase_path.startswith(testcase_directory): files_list.append( os.path.relpath(testcase_path, testcase_directory)) has_testcases_in_testcase_directory = True elif testcase_path.startswith(data_directory): files_list.append(os.path.relpath(testcase_path, data_directory)) has_testcases_in_data_directory = True if not files_list: return formatted_date = str(utils.utcnow().date()) gcs_base_url = 'gs://{bucket_name}/{date}/{fuzzer_name}/'.format( bucket_name=bucket_name, date=formatted_date, fuzzer_name=fuzzer_name) runner = gsutil.GSUtilRunner() batch_directory_blobs = storage.list_blobs(gcs_base_url) total_testcases = 0 for blob in batch_directory_blobs: if not blob.endswith(LIST_FILE_BASENAME): continue list_gcs_url = storage.get_cloud_storage_file_path(bucket_name, blob) data = storage.read_data(list_gcs_url) if not data: logs.log_error( 'Read no data from test case list at {gcs_url}'.format( gcs_url=list_gcs_url)) continue total_testcases += len(data.splitlines()) # If we've already uploaded enough test cases for this fuzzer today, return. if total_testcases >= TESTCASES_PER_DAY: return # Cap the number of files. testcases_limit = min(len(files_list), TESTCASES_PER_DAY - total_testcases) files_list = files_list[:testcases_limit] # Upload each batch of tests to its own unique sub-bucket. identifier = environment.get_value('BOT_NAME') + str(utils.utcnow()) gcs_base_url += utils.string_hash(identifier) list_gcs_url = gcs_base_url + '/' + LIST_FILE_BASENAME if not storage.write_data('\n'.join(files_list).encode('utf-8'), list_gcs_url): return if has_testcases_in_testcase_directory: # Sync everything in |testcase_directory| since it is fuzzer-generated. runner.rsync(testcase_directory, gcs_base_url) if has_testcases_in_data_directory: # Sync all fuzzer generated testcase in data bundle directory. runner.rsync(data_directory, gcs_base_url, exclusion_pattern=('(?!.*{fuzz_prefix})'.format( fuzz_prefix=testcase_manager.FUZZ_PREFIX))) # Sync all possible resource dependencies as a best effort. It matches # |resources-| prefix that a fuzzer can use to indicate resources. Also, it # matches resources directory that Chromium web_tests use for dependencies. runner.rsync(data_directory, gcs_base_url, exclusion_pattern='(?!.*resource)') logs.log('Synced {count} test cases to {gcs_url}.'.format( count=len(files_list), gcs_url=gcs_base_url))
def get_bucket(): """Return path to fuzzer logs bucket.""" return local_config.ProjectConfig().get('logs.fuzzer.bucket')
def _get_topic(): """Get the Pub/Sub topic for publishing tasks.""" return local_config.ProjectConfig().get('bisect_service.pubsub_topic')
def get(self): """Handles a GET request.""" libfuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == 'libFuzzer').get() if not libfuzzer: logs.log_error('Failed to get libFuzzer Fuzzer entity.') return afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get() if not afl: logs.log_error('Failed to get AFL Fuzzer entity.') return honggfuzz = data_types.Fuzzer.query( data_types.Fuzzer.name == 'honggfuzz').get() if not honggfuzz: logs.log_error('Failed to get honggfuzz Fuzzer entity.') return gft = data_types.Fuzzer.query( data_types.Fuzzer.name == 'googlefuzztest').get() if not gft: logs.log_error('Failed to get googlefuzztest Fuzzer entity.') return project_config = local_config.ProjectConfig() segregate_projects = project_config.get('segregate_projects') project_setup_configs = project_config.get('project_setup') project_names = set() job_names = set() fuzzer_entities = { 'afl': afl.key, 'honggfuzz': honggfuzz.key, 'googlefuzztest': gft.key, 'libfuzzer': libfuzzer.key, } for setup_config in project_setup_configs: bucket_config = setup_config.get('build_buckets') if not bucket_config: raise ProjectSetupError('Project setup buckets not specified.') config = ProjectSetup( BUILD_BUCKET_PATH_TEMPLATE, REVISION_URL, setup_config.get('build_type'), config_suffix=setup_config.get('job_suffix', ''), external_config=setup_config.get('external_config', ''), segregate_projects=segregate_projects, experimental_sanitizers=setup_config.get( 'experimental_sanitizers', []), engine_build_buckets={ 'libfuzzer': bucket_config.get('libfuzzer'), 'libfuzzer-i386': bucket_config.get('libfuzzer_i386'), 'afl': bucket_config.get('afl'), 'honggfuzz': bucket_config.get('honggfuzz'), 'googlefuzztest': bucket_config.get('googlefuzztest'), 'none': bucket_config.get('no_engine'), 'dataflow': bucket_config.get('dataflow'), }, fuzzer_entities=fuzzer_entities, add_info_labels=setup_config.get('add_info_labels', False), add_revision_mappings=setup_config.get('add_revision_mappings', False), additional_vars=setup_config.get('additional_vars')) projects_source = setup_config.get('source') if projects_source == 'oss-fuzz': projects = get_oss_fuzz_projects() elif projects_source.startswith(storage.GS_PREFIX): projects = get_projects_from_gcs(projects_source) else: raise ProjectSetupError('Invalid projects source: ' + projects_source) if not projects: raise ProjectSetupError('Missing projects list.') result = config.set_up(projects) project_names.update(result.project_names) job_names.update(result.job_names) cleanup_stale_projects(list(fuzzer_entities.values()), project_names, job_names, segregate_projects)
def set_bot_environment(): """Set environment for the bots.""" root_dir = get_value('ROOT_DIR') if not root_dir: # Error, bail out. return False # Reset our current working directory. Our's last job might # have left us in a non-existent temp directory. # Or ROOT_DIR might be deleted and recreated. os.chdir(root_dir) # Set some default directories. These can be overriden by config files below. bot_dir = os.path.join(root_dir, 'bot') if is_trusted_host(ensure_connected=False): worker_root_dir = os.environ['WORKER_ROOT_DIR'] os.environ['BUILDS_DIR'] = os.path.join(worker_root_dir, 'bot', 'builds') else: os.environ['BUILDS_DIR'] = os.path.join(bot_dir, 'builds') os.environ['BUILD_URLS_DIR'] = os.path.join(bot_dir, 'build-urls') os.environ['LOG_DIR'] = os.path.join(bot_dir, 'logs') os.environ['CACHE_DIR'] = os.path.join(bot_dir, 'cache') inputs_dir = os.path.join(bot_dir, 'inputs') os.environ['INPUT_DIR'] = inputs_dir os.environ['CRASH_STACKTRACES_DIR'] = os.path.join(inputs_dir, 'crash-stacks') os.environ['FUZZERS_DIR'] = os.path.join(inputs_dir, 'fuzzers') os.environ['DATA_BUNDLES_DIR'] = os.path.join(inputs_dir, 'data-bundles') os.environ['FUZZ_INPUTS'] = os.path.join(inputs_dir, 'fuzzer-testcases') os.environ['FUZZ_INPUTS_MEMORY'] = os.environ['FUZZ_INPUTS'] os.environ['FUZZ_INPUTS_DISK'] = os.path.join(inputs_dir, 'fuzzer-testcases-disk') os.environ['MUTATOR_PLUGINS_DIR'] = os.path.join(inputs_dir, 'mutator-plugins') os.environ['FUZZ_DATA'] = os.path.join(inputs_dir, 'fuzzer-common-data-bundles') os.environ['IMAGES_DIR'] = os.path.join(inputs_dir, 'images') os.environ['SYMBOLS_DIR'] = os.path.join(inputs_dir, 'symbols') os.environ['USER_PROFILE_ROOT_DIR'] = os.path.join(inputs_dir, 'user-profile-dirs') # Set bot name. if not get_value('BOT_NAME'): # If not defined, default to host name. os.environ['BOT_NAME'] = socket.gethostname().lower() # Local temp directory (non-tmpfs). local_tmp_dir = os.path.join(bot_dir, 'tmp') # Set BOT_TMPDIR if not already set. if not get_value('BOT_TMPDIR'): os.environ['BOT_TMPDIR'] = local_tmp_dir # Add common environment variables needed by Bazel test runner. # See https://docs.bazel.build/versions/master/test-encyclopedia.html. # NOTE: Do not use a tmpfs folder as some fuzz targets don't work. os.environ['TEST_TMPDIR'] = local_tmp_dir os.environ['TZ'] = 'UTC' # Sets the default configuration. Can be overridden by job environment. set_default_vars() # Set environment variable from local project configuration. from clusterfuzz._internal.config import local_config local_config.ProjectConfig().set_environment() # Success. return True