def auth_domain():
"""Get the auth domain."""
domain = local_config.ProjectConfig().get('firebase.auth_domain')
if domain:
return domain
return utils.get_application_id() + '.firebaseapp.com'
def get_current_user():
"""Get the current logged in user, or None."""
if environment.is_local_development():
return User('user@localhost')
current_request = request_cache.get_current_request()
if local_config.AuthConfig().get('enable_loas'):
loas_user = current_request.headers.get(
'X-AppEngine-LOAS-Peer-Username')
if loas_user:
return User(loas_user + '@google.com')
iap_email = get_iap_email(current_request)
if iap_email:
return User(iap_email)
cache_backing = request_cache.get_cache_backing()
oauth_email = getattr(cache_backing, '_oauth_email', None)
if oauth_email:
return User(oauth_email)
cached_email = getattr(cache_backing, '_cached_email', None)
if cached_email:
return User(cached_email)
session_cookie = get_session_cookie()
if not session_cookie:
return None
try:
decoded_claims = decode_claims(get_session_cookie())
except AuthError:
logs.log_warn('Invalid session cookie.')
return None
allowed_firebase_providers = local_config.ProjectConfig().get(
'firebase.auth_providers', ['google.com'])
firebase_info = decoded_claims.get('firebase', {})
sign_in_provider = firebase_info.get('sign_in_provider')
if sign_in_provider not in allowed_firebase_providers:
logs.log_error(f'Firebase provider {sign_in_provider} is not enabled.')
return None
# Per https://docs.github.com/en/authentication/
# keeping-your-account-and-data-secure/authorizing-oauth-apps
# GitHub requires emails to be verified before an OAuth app can be
# authorized, so we make an exception.
if (not decoded_claims.get('email_verified')
and sign_in_provider != 'github.com'):
return None
email = decoded_claims.get('email')
if not email:
return None
# We cache the email for this request if we've validated the user to make
# subsequent get_current_user() calls fast.
setattr(cache_backing, '_cached_email', email)
return User(email)
def bootstrap_gcs(storage_path):
"""Bootstrap GCS."""
local_gcs_buckets_path = os.path.join(storage_path, 'local_gcs')
if not os.path.exists(local_gcs_buckets_path):
os.mkdir(local_gcs_buckets_path)
config = local_config.ProjectConfig()
test_blobs_bucket = os.environ.get('TEST_BLOBS_BUCKET')
if test_blobs_bucket:
create_local_bucket(local_gcs_buckets_path, test_blobs_bucket)
else:
create_local_bucket(local_gcs_buckets_path, config.get('blobs.bucket'))
create_local_bucket(local_gcs_buckets_path,
config.get('deployment.bucket'))
create_local_bucket(local_gcs_buckets_path, config.get('bigquery.bucket'))
create_local_bucket(local_gcs_buckets_path, config.get('backup.bucket'))
create_local_bucket(local_gcs_buckets_path,
config.get('logs.fuzzer.bucket'))
create_local_bucket(local_gcs_buckets_path,
config.get('env.CORPUS_BUCKET'))
create_local_bucket(local_gcs_buckets_path,
config.get('env.QUARANTINE_BUCKET'))
create_local_bucket(local_gcs_buckets_path,
config.get('env.SHARED_CORPUS_BUCKET'))
create_local_bucket(local_gcs_buckets_path,
config.get('env.FUZZ_LOGS_BUCKET'))
create_local_bucket(local_gcs_buckets_path,
config.get('env.MUTATOR_PLUGINS_BUCKET'))
# Symlink local GCS bucket path to appengine src dir to bypass sandboxing
# issues.
common.symlink(src=local_gcs_buckets_path,
target=os.path.join(appengine.SRC_DIR_PY, 'local_gcs'))
def notify_issue_update(testcase, status):
"""Notify that an issue update occurred (i.e. issue was filed or closed)."""
topic = local_config.ProjectConfig().get('issue_updates.pubsub_topic')
if not topic:
return
pubsub_client = pubsub.PubSubClient()
pubsub_client.publish(
topic, [
pubsub.Message(
attributes={
'crash_address': testcase.crash_address,
'crash_state': testcase.crash_state,
'crash_type': testcase.crash_type,
'issue_id': testcase.bug_information or '',
'security': str(testcase.security_flag).lower(),
'status': status,
'testcase_id': str(testcase.key.id()),
})
])
if status in ('verified', 'wontfix'):
logs.log(f'Closing issue {testcase.github_issue_num} '
f'in GitHub repo {testcase.github_repo_id}: '
f'Testcase {testcase.key.id()} is marked as {status}.')
oss_fuzz_github.close_issue(testcase)
def setUp(self):
test_helpers.patch_environ(self)
environment.remove_key('PROJECT_NAME')
environment.remove_key('ISSUE_TRACKER')
environment.remove_key('UPDATE_WEB_TESTS')
self.config = local_config.ProjectConfig()
def _deployment_file_url(filename):
"""Helper to return deployment file url."""
deployment_bucket = local_config.ProjectConfig().get('deployment.bucket')
if not deployment_bucket:
return None
return 'gs://{bucket}/{name}'.format(bucket=deployment_bucket,
name=filename)
def blobs_bucket():
"""Get the blobs bucket name."""
# Allow tests to override blobs bucket name safely.
test_blobs_bucket = environment.get_value('TEST_BLOBS_BUCKET')
if test_blobs_bucket:
return test_blobs_bucket
assert not environment.get_value('PY_UNITTESTS')
return local_config.ProjectConfig().get('blobs.bucket')
def get(self):
"""Handle a get request."""
dest = request.get('dest', DEFAULT_REDIRECT)
base_handler.check_redirect_url(dest)
return self.render(
'login.html', {
'apiKey': local_config.ProjectConfig().get('firebase.api_key'),
'authDomain': auth.auth_domain(),
'dest': dest,
})
def get(self):
"""Handle a GET request."""
# The task is supposed to be super reliable and never fail. If anything goes
# wrong, we just fail with the exception going straight into StackDriver.
logs.log('FuzzerCoverage task started.')
bucket = local_config.ProjectConfig().get('coverage.reports.bucket')
if not bucket:
logs.log(
'Coverage bucket is not specified. Skipping FuzzerCoverage task.'
)
return
collect_fuzzer_coverage(bucket)
logs.log('FuzzerCoverage task finished successfully.')
def _upload_kernel_coverage_data(kcov_path, kernel_bid):
"""Upload kcov data to a cloud storage bucket."""
bucket_name = local_config.ProjectConfig().get('coverage.reports.bucket')
if not bucket_name:
return
formatted_date = str(utils.utcnow().date().isoformat())
identifier = environment.get_value('BOT_NAME') + str(
utils.utcnow().isoformat())
gcs_url = (f'gs://{bucket_name}/syzkaller/{formatted_date}/{kernel_bid}/'
f'{identifier}')
if storage.copy_file_to(kcov_path, gcs_url):
logs.log(f'Copied kcov data to {gcs_url}.')
def initialize():
"""Initialize if monitoring is enabled for this bot."""
global _monitoring_v3_client
global _flusher_thread
if environment.get_value('LOCAL_DEVELOPMENT'):
return
if not local_config.ProjectConfig().get('monitoring.enabled'):
return
if check_module_loaded(monitoring_v3):
_initialize_monitored_resource()
_monitoring_v3_client = monitoring_v3.MetricServiceClient(
credentials=credentials.get_default()[0])
_flusher_thread = _FlusherThread()
_flusher_thread.start()
def ignore_stacktrace(crash_stacktrace):
"""Return whether the stacktrace needs to be ignored."""
# Filter crash based on search exclude pattern specified in job definition.
search_excludes = environment.get_value('SEARCH_EXCLUDES')
if search_excludes and re.search(search_excludes, crash_stacktrace):
return True
# Match stacktrace against custom defined blacklist regexes in project config.
stack_blacklist_regexes = (
local_config.ProjectConfig().get('stacktrace.stack_blacklist_regexes'))
if not stack_blacklist_regexes:
return False
stack_blacklist_regex = re.compile(r'(%s)' %
'|'.join(stack_blacklist_regexes))
for line in crash_stacktrace.splitlines():
if stack_blacklist_regex.match(line):
return True
return False
def notify_issue_update(testcase, status):
"""Notify that an issue update occurred (i.e. issue was filed or closed)."""
topic = local_config.ProjectConfig().get('issue_updates.pubsub_topic')
if not topic:
return
pubsub_client = pubsub.PubSubClient()
pubsub_client.publish(topic, [
pubsub.Message(
attributes={
'crash_address': testcase.crash_address,
'crash_state': testcase.crash_state,
'crash_type': testcase.crash_type,
'issue_id': testcase.bug_information or '',
'security': str(testcase.security_flag).lower(),
'status': status,
'testcase_id': str(testcase.key.id()),
})
])
def _get_revision_vars_url_format(job_type, platform_id=None):
"""Return REVISION_VARS_URL from job environment if available. Otherwise,
default to one set in project.yaml. For custom binary jobs, this is not
applicable."""
if job_type is None:
# Force it to use env attribute in project.yaml.
return local_config.ProjectConfig().get('env.REVISION_VARS_URL')
custom_binary = data_handler.get_value_from_job_definition(
job_type, 'CUSTOM_BINARY')
if utils.string_is_true(custom_binary):
return None
rev_path = data_handler.get_value_from_job_definition_or_environment(
job_type, 'REVISION_VARS_URL')
rev_path = overrides.check_and_apply_overrides(
rev_path,
overrides.PLATFORM_ID_TO_REV_PATH_KEY,
platform_id=platform_id)
return rev_path
def get_crash_data(crash_data,
symbolize_flag=True,
fuzz_target=None,
already_symbolized=False,
detect_ooms_and_hangs=None) -> stacktraces.CrashInfo:
"""Get crash parameters from crash data.
Crash parameters include crash type, address, state and stacktrace.
If the stacktrace is not already symbolized, we will try to symbolize it
unless |symbolize| flag is set to False. Symbolized stacktrace will contain
inline frames, but we do exclude them for purposes of crash state generation
(helps in testcase deduplication)."""
# Decide whether to symbolize or not symbolize the input stacktrace.
# Note that Fuchsia logs are always symbolized.
if symbolize_flag:
# Defer imports since stack_symbolizer pulls in a lot of things.
from clusterfuzz._internal.crash_analysis.stack_parsing import \
stack_symbolizer
crash_stacktrace_with_inlines = stack_symbolizer.symbolize_stacktrace(
crash_data, enable_inline_frames=True)
crash_stacktrace_without_inlines = stack_symbolizer.symbolize_stacktrace(
crash_data, enable_inline_frames=False)
else:
# We are explicitly indicated to not symbolize using |symbolize_flag|. There
# is no distinction between inline and non-inline frames for an unsymbolized
# stacktrace.
crash_stacktrace_with_inlines = crash_data
crash_stacktrace_without_inlines = crash_data
# Additional stack frame ignore regexes.
custom_stack_frame_ignore_regexes = (local_config.ProjectConfig().get(
'stacktrace.stack_frame_ignore_regexes', []))
if environment.get_value('TASK_NAME') == 'analyze':
detect_v8_runtime_errors = True
else:
detect_v8_runtime_errors = environment.get_value(
'DETECT_V8_RUNTIME_ERRORS', False)
fuzz_target = fuzz_target or environment.get_value('FUZZ_TARGET')
redzone_size = environment.get_value('REDZONE')
if detect_ooms_and_hangs is None:
detect_ooms_and_hangs = (
environment.get_value('REPORT_OOMS_AND_HANGS')
and (not redzone_size
or redzone_size <= MAX_REDZONE_SIZE_FOR_OOMS_AND_HANGS))
include_ubsan = 'halt_on_error=0' not in environment.get_value(
'UBSAN_OPTIONS', '')
stack_parser = stacktraces.StackParser(
symbolized=symbolize_flag or already_symbolized,
detect_ooms_and_hangs=detect_ooms_and_hangs,
detect_v8_runtime_errors=detect_v8_runtime_errors,
custom_stack_frame_ignore_regexes=custom_stack_frame_ignore_regexes,
fuzz_target=fuzz_target,
include_ubsan=include_ubsan)
result = stack_parser.parse(crash_stacktrace_without_inlines)
# Use stacktrace with inlines for the result.
if result.crash_stacktrace:
result.crash_stacktrace = crash_stacktrace_with_inlines
# Linkify Android Kernel or lkl stacktrace.
linkify_kernel_or_lkl_stacktrace_if_needed(result)
return result
def get_bucket():
"""Return bucket for bigquery stats."""
return local_config.ProjectConfig().get('bigquery.bucket')
def upload_testcases_if_needed(fuzzer_name, testcase_list, testcase_directory,
data_directory):
"""Upload test cases from the list to a cloud storage bucket."""
# Since builtin fuzzers have a coverage minimized corpus, no need to upload
# test case samples for them.
if fuzzer_name in fuzzing.ENGINES:
return
bucket_name = local_config.ProjectConfig().get(
'coverage.fuzzer-testcases.bucket')
if not bucket_name:
return
files_list = []
has_testcases_in_testcase_directory = False
has_testcases_in_data_directory = False
for testcase_path in testcase_list:
if testcase_path.startswith(testcase_directory):
files_list.append(
os.path.relpath(testcase_path, testcase_directory))
has_testcases_in_testcase_directory = True
elif testcase_path.startswith(data_directory):
files_list.append(os.path.relpath(testcase_path, data_directory))
has_testcases_in_data_directory = True
if not files_list:
return
formatted_date = str(utils.utcnow().date())
gcs_base_url = 'gs://{bucket_name}/{date}/{fuzzer_name}/'.format(
bucket_name=bucket_name, date=formatted_date, fuzzer_name=fuzzer_name)
runner = gsutil.GSUtilRunner()
batch_directory_blobs = storage.list_blobs(gcs_base_url)
total_testcases = 0
for blob in batch_directory_blobs:
if not blob.endswith(LIST_FILE_BASENAME):
continue
list_gcs_url = storage.get_cloud_storage_file_path(bucket_name, blob)
data = storage.read_data(list_gcs_url)
if not data:
logs.log_error(
'Read no data from test case list at {gcs_url}'.format(
gcs_url=list_gcs_url))
continue
total_testcases += len(data.splitlines())
# If we've already uploaded enough test cases for this fuzzer today, return.
if total_testcases >= TESTCASES_PER_DAY:
return
# Cap the number of files.
testcases_limit = min(len(files_list), TESTCASES_PER_DAY - total_testcases)
files_list = files_list[:testcases_limit]
# Upload each batch of tests to its own unique sub-bucket.
identifier = environment.get_value('BOT_NAME') + str(utils.utcnow())
gcs_base_url += utils.string_hash(identifier)
list_gcs_url = gcs_base_url + '/' + LIST_FILE_BASENAME
if not storage.write_data('\n'.join(files_list).encode('utf-8'),
list_gcs_url):
return
if has_testcases_in_testcase_directory:
# Sync everything in |testcase_directory| since it is fuzzer-generated.
runner.rsync(testcase_directory, gcs_base_url)
if has_testcases_in_data_directory:
# Sync all fuzzer generated testcase in data bundle directory.
runner.rsync(data_directory,
gcs_base_url,
exclusion_pattern=('(?!.*{fuzz_prefix})'.format(
fuzz_prefix=testcase_manager.FUZZ_PREFIX)))
# Sync all possible resource dependencies as a best effort. It matches
# |resources-| prefix that a fuzzer can use to indicate resources. Also, it
# matches resources directory that Chromium web_tests use for dependencies.
runner.rsync(data_directory,
gcs_base_url,
exclusion_pattern='(?!.*resource)')
logs.log('Synced {count} test cases to {gcs_url}.'.format(
count=len(files_list), gcs_url=gcs_base_url))
def get_bucket():
"""Return path to fuzzer logs bucket."""
return local_config.ProjectConfig().get('logs.fuzzer.bucket')
def _get_topic():
"""Get the Pub/Sub topic for publishing tasks."""
return local_config.ProjectConfig().get('bisect_service.pubsub_topic')
def get(self):
"""Handles a GET request."""
libfuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == 'libFuzzer').get()
if not libfuzzer:
logs.log_error('Failed to get libFuzzer Fuzzer entity.')
return
afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get()
if not afl:
logs.log_error('Failed to get AFL Fuzzer entity.')
return
honggfuzz = data_types.Fuzzer.query(
data_types.Fuzzer.name == 'honggfuzz').get()
if not honggfuzz:
logs.log_error('Failed to get honggfuzz Fuzzer entity.')
return
gft = data_types.Fuzzer.query(
data_types.Fuzzer.name == 'googlefuzztest').get()
if not gft:
logs.log_error('Failed to get googlefuzztest Fuzzer entity.')
return
project_config = local_config.ProjectConfig()
segregate_projects = project_config.get('segregate_projects')
project_setup_configs = project_config.get('project_setup')
project_names = set()
job_names = set()
fuzzer_entities = {
'afl': afl.key,
'honggfuzz': honggfuzz.key,
'googlefuzztest': gft.key,
'libfuzzer': libfuzzer.key,
}
for setup_config in project_setup_configs:
bucket_config = setup_config.get('build_buckets')
if not bucket_config:
raise ProjectSetupError('Project setup buckets not specified.')
config = ProjectSetup(
BUILD_BUCKET_PATH_TEMPLATE,
REVISION_URL,
setup_config.get('build_type'),
config_suffix=setup_config.get('job_suffix', ''),
external_config=setup_config.get('external_config', ''),
segregate_projects=segregate_projects,
experimental_sanitizers=setup_config.get(
'experimental_sanitizers', []),
engine_build_buckets={
'libfuzzer': bucket_config.get('libfuzzer'),
'libfuzzer-i386': bucket_config.get('libfuzzer_i386'),
'afl': bucket_config.get('afl'),
'honggfuzz': bucket_config.get('honggfuzz'),
'googlefuzztest': bucket_config.get('googlefuzztest'),
'none': bucket_config.get('no_engine'),
'dataflow': bucket_config.get('dataflow'),
},
fuzzer_entities=fuzzer_entities,
add_info_labels=setup_config.get('add_info_labels', False),
add_revision_mappings=setup_config.get('add_revision_mappings',
False),
additional_vars=setup_config.get('additional_vars'))
projects_source = setup_config.get('source')
if projects_source == 'oss-fuzz':
projects = get_oss_fuzz_projects()
elif projects_source.startswith(storage.GS_PREFIX):
projects = get_projects_from_gcs(projects_source)
else:
raise ProjectSetupError('Invalid projects source: ' +
projects_source)
if not projects:
raise ProjectSetupError('Missing projects list.')
result = config.set_up(projects)
project_names.update(result.project_names)
job_names.update(result.job_names)
cleanup_stale_projects(list(fuzzer_entities.values()), project_names,
job_names, segregate_projects)
def set_bot_environment():
"""Set environment for the bots."""
root_dir = get_value('ROOT_DIR')
if not root_dir:
# Error, bail out.
return False
# Reset our current working directory. Our's last job might
# have left us in a non-existent temp directory.
# Or ROOT_DIR might be deleted and recreated.
os.chdir(root_dir)
# Set some default directories. These can be overriden by config files below.
bot_dir = os.path.join(root_dir, 'bot')
if is_trusted_host(ensure_connected=False):
worker_root_dir = os.environ['WORKER_ROOT_DIR']
os.environ['BUILDS_DIR'] = os.path.join(worker_root_dir, 'bot',
'builds')
else:
os.environ['BUILDS_DIR'] = os.path.join(bot_dir, 'builds')
os.environ['BUILD_URLS_DIR'] = os.path.join(bot_dir, 'build-urls')
os.environ['LOG_DIR'] = os.path.join(bot_dir, 'logs')
os.environ['CACHE_DIR'] = os.path.join(bot_dir, 'cache')
inputs_dir = os.path.join(bot_dir, 'inputs')
os.environ['INPUT_DIR'] = inputs_dir
os.environ['CRASH_STACKTRACES_DIR'] = os.path.join(inputs_dir,
'crash-stacks')
os.environ['FUZZERS_DIR'] = os.path.join(inputs_dir, 'fuzzers')
os.environ['DATA_BUNDLES_DIR'] = os.path.join(inputs_dir, 'data-bundles')
os.environ['FUZZ_INPUTS'] = os.path.join(inputs_dir, 'fuzzer-testcases')
os.environ['FUZZ_INPUTS_MEMORY'] = os.environ['FUZZ_INPUTS']
os.environ['FUZZ_INPUTS_DISK'] = os.path.join(inputs_dir,
'fuzzer-testcases-disk')
os.environ['MUTATOR_PLUGINS_DIR'] = os.path.join(inputs_dir,
'mutator-plugins')
os.environ['FUZZ_DATA'] = os.path.join(inputs_dir,
'fuzzer-common-data-bundles')
os.environ['IMAGES_DIR'] = os.path.join(inputs_dir, 'images')
os.environ['SYMBOLS_DIR'] = os.path.join(inputs_dir, 'symbols')
os.environ['USER_PROFILE_ROOT_DIR'] = os.path.join(inputs_dir,
'user-profile-dirs')
# Set bot name.
if not get_value('BOT_NAME'):
# If not defined, default to host name.
os.environ['BOT_NAME'] = socket.gethostname().lower()
# Local temp directory (non-tmpfs).
local_tmp_dir = os.path.join(bot_dir, 'tmp')
# Set BOT_TMPDIR if not already set.
if not get_value('BOT_TMPDIR'):
os.environ['BOT_TMPDIR'] = local_tmp_dir
# Add common environment variables needed by Bazel test runner.
# See https://docs.bazel.build/versions/master/test-encyclopedia.html.
# NOTE: Do not use a tmpfs folder as some fuzz targets don't work.
os.environ['TEST_TMPDIR'] = local_tmp_dir
os.environ['TZ'] = 'UTC'
# Sets the default configuration. Can be overridden by job environment.
set_default_vars()
# Set environment variable from local project configuration.
from clusterfuzz._internal.config import local_config
local_config.ProjectConfig().set_environment()
# Success.
return True
