def get_components_list(component_revisions_dict, job_type):
"""Return a prioritized order of components based on job type."""
components = sorted(component_revisions_dict.keys())
if utils.is_chromium():
# Components prioritization only applies to non-chromium projects.
return components
project_name = data_handler.get_project_name(job_type)
if not project_name:
# No project name found in job environment, return list as-is.
return components
main_repo = data_handler.get_main_repo(job_type)
project_src = '/src/' + project_name
for component in components.copy():
if component_revisions_dict[component]['url'] == main_repo:
# Matches recorded main repo.
components.remove(component)
components.insert(0, component)
break
if component == project_src:
components.remove(component)
components.insert(0, component)
break
if project_name.lower() in os.path.basename(component).lower():
components.remove(component)
components.insert(0, component)
# Keep trying in case an exact match is found later.
return components
def create_impact_task_if_needed(testcase):
"""Creates an impact task if needed."""
# Impact doesn't make sense for non-chromium projects.
if not utils.is_chromium():
return
# Impact is only applicable to chromium project, otherwise bail out.
if testcase.project_name != 'chromium':
return
# We cannot run impact job for custom binaries since we don't have any
# archived production builds for these.
if build_manager.is_custom_binary():
return
tasks.add_task('impact', testcase.key.id(), testcase.job_type)
def get(self):
"""Handles get request."""
email = helpers.get_user_email()
if not email:
raise helpers.AccessDeniedException()
is_privileged_or_domain_user = access.has_access(
need_privileged_access=False)
if is_privileged_or_domain_user or _is_uploader_allowed(email):
# Privileged, domain and upload users can see all job and fuzzer names.
allowed_jobs = data_handler.get_all_job_type_names()
allowed_fuzzers = data_handler.get_all_fuzzer_names_including_children(
include_parents=True)
else:
# Check if this is an external user with access to certain fuzzers/jobs.
allowed_jobs = external_users.allowed_jobs_for_user(email)
allowed_fuzzers = external_users.allowed_fuzzers_for_user(
email, include_from_jobs=True)
if not allowed_fuzzers and not allowed_jobs:
raise helpers.AccessDeniedException()
has_issue_tracker = bool(data_handler.get_issue_tracker_name())
result, params = get_result()
return self.render(
'upload.html', {
'fieldValues': {
'blackboxFuzzers':
filter_blackbox_fuzzers(allowed_fuzzers),
'jobs': allowed_jobs,
'targets': {
engine: filter_target_names(allowed_fuzzers, engine)
for engine in fuzzing.ENGINES
},
'isChromium': utils.is_chromium(),
'sandboxedJobs': data_types.INTERNAL_SANDBOXED_JOB_TYPES,
'csrfToken': form.generate_csrf_token(),
'isExternalUser': not is_privileged_or_domain_user,
'uploadInfo': gcs.prepare_blob_upload()._asdict(),
'hasIssueTracker': has_issue_tracker,
},
'params': params,
'result': result
})
def get(self):
"""Get and render the testcase list in HTML."""
result, params = get_result()
field_values = {
'projects':
data_handler.get_all_project_names(),
'fuzzers':
data_handler.get_all_fuzzer_names_including_children(
include_parents=True),
'jobs':
data_handler.get_all_job_type_names(),
'shouldShowImpact':
utils.is_chromium()
}
return self.render('testcase-list.html', {
'fieldValues': field_values,
'result': result,
'params': params
})
def create_blame_task_if_needed(testcase):
"""Creates a blame task if needed."""
# Blame doesn't work for non-chromium projects.
if not utils.is_chromium():
return
# Blame is only applicable to chromium project, otherwise bail out.
if testcase.project_name != 'chromium':
return
# We cannot run blame job for custom binaries since we don't have any context
# on the crash revision and regression range.
if build_manager.is_custom_binary():
return
# Don't send duplicate issues to Predator. This causes issues with metrics
# tracking and wastes cycles.
if testcase.status == 'Duplicate':
return
create_task = False
if testcase.one_time_crasher_flag:
# For unreproducible testcases, it is still beneficial to get component
# information from blame task.
create_task = True
else:
# Reproducible testcase.
# Step 1: Check if the regression task finished. If not, bail out.
if not testcase.regression:
return
# Step 2: Check if the symbolize task is applicable and finished. If not,
# bail out.
if build_manager.has_symbolized_builds() and not testcase.symbolized:
return
create_task = True
if create_task:
tasks.add_task('blame', testcase.key.id(), testcase.job_type)
return middleware
def register_routes(flask_app, routes):
"""Utility function to register all routes to the flask app."""
for route, handler in routes:
flask_app.add_url_rule(route, view_func=handler.as_view(route))
# Add item to the navigation menu. Order is important.
base_handler.add_menu('Testcases', '/testcases')
base_handler.add_menu('Fuzzer Statistics', '/fuzzer-stats')
base_handler.add_menu('Crash Statistics', '/crash-stats')
base_handler.add_menu('Upload Testcase', '/upload-testcase')
_is_chromium = utils.is_chromium()
_is_oss_fuzz = utils.is_oss_fuzz()
if _is_chromium:
base_handler.add_menu('Crashes by range', '/commit-range')
if not _is_oss_fuzz:
base_handler.add_menu('Fuzzers', '/fuzzers')
base_handler.add_menu('Corpora', '/corpora')
base_handler.add_menu('Bots', '/bots')
base_handler.add_menu('Jobs', '/jobs')
base_handler.add_menu('Configuration', '/configuration')
base_handler.add_menu('Report Bug', '/report-bug')
base_handler.add_menu('Documentation', '/docs')
def execute_task(testcase_id, job_type):
"""Attempt to find if the testcase affects release branches on Chromium."""
# This shouldn't ever get scheduled, but check just in case.
if not utils.is_chromium():
return
# Locate the testcase associated with the id.
testcase = data_handler.get_testcase_by_id(testcase_id)
# If this testcase is fixed, we should no longer be doing impact testing.
if testcase.fixed and testcase.is_impact_set_flag:
return
# For testcases with status unreproducible, we just do impact analysis just
# once.
if testcase.is_status_unreproducible() and testcase.is_impact_set_flag:
return
# Update comments only after checking the above bailout conditions.
data_handler.update_testcase_comment(testcase,
data_types.TaskState.STARTED)
# This task is not applicable to unreproducible testcases.
if testcase.one_time_crasher_flag:
data_handler.update_testcase_comment(
testcase, data_types.TaskState.ERROR,
'Not applicable for unreproducible testcases')
return
# This task is not applicable for custom binaries. We cannot remove the
# creation of such tasks specifically for custom binary testcase in cron,
# so exit gracefully.
if build_manager.is_custom_binary():
data_handler.update_testcase_comment(
testcase, data_types.TaskState.FINISHED,
'Not applicable for custom binaries')
return
# If we don't have a stable or beta build url pattern, we try to use build
# information url to make a guess.
if not build_manager.has_production_builds():
if not testcase.regression:
data_handler.update_testcase_comment(
testcase, data_types.TaskState.FINISHED,
'Cannot run without regression range, will re-run once regression '
'task finishes')
return
impacts = get_impacts_from_url(testcase.regression, testcase.job_type)
testcase = data_handler.get_testcase_by_id(testcase_id)
set_testcase_with_impacts(testcase, impacts)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.FINISHED)
return
# Setup testcase and its dependencies.
file_list, _, testcase_file_path = setup.setup_testcase(testcase, job_type)
if not file_list:
return
# Setup extended stable, stable, beta builds
# and get impact and crash stacktrace.
try:
impacts = get_impacts_on_prod_builds(testcase, testcase_file_path)
except BuildFailedException as error:
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(testcase,
data_types.TaskState.ERROR,
str(error))
tasks.add_task('impact',
testcase_id,
job_type,
wait_time=environment.get_value('FAIL_WAIT'))
return
testcase = data_handler.get_testcase_by_id(testcase_id)
set_testcase_with_impacts(testcase, impacts)
# Set stacktrace in case we have a unreproducible crash on trunk,
# but it crashes on one of the production builds.
if testcase.is_status_unreproducible() and impacts.get_extra_trace():
testcase.crash_stacktrace = data_handler.filter_stacktrace(
'%s\n\n%s' %
(data_handler.get_stacktrace(testcase), impacts.get_extra_trace()))
data_handler.update_testcase_comment(testcase,
data_types.TaskState.FINISHED)
def get_component_revisions_dict(revision, job_type, platform_id=None):
"""Retrieve revision vars dict."""
if revision == 0 or revision == '0' or revision is None:
# Return empty dict for zero start revision.
return {}
revision_vars_url_format = _get_revision_vars_url_format(
job_type, platform_id=platform_id)
if not revision_vars_url_format:
return None
project_name = data_handler.get_project_name(job_type)
revisions_dict = {}
if utils.is_chromium():
component = data_handler.get_component_name(job_type)
repository = data_handler.get_repository_for_component(component)
if repository and not _is_clank(revision_vars_url_format):
revision_hash = _git_commit_position_to_git_hash_for_chromium(
revision, repository)
if revision_hash is None:
return None
# FIXME: While we check for this explicitly appended component in all
# applicable cases that we know of within this codebase, if the dict
# is shared with an external service (e.g. Predator) we may need to clean
# this up beforehand.
revisions_dict['/src'] = {
'name': _get_component_display_name(component, project_name),
'url': _git_url_for_chromium_repository(repository),
'rev': revision_hash,
'commit_pos': revision
}
# Use revision hash for info url later.
revision = revision_hash
revision_vars_url = revision_vars_url_format % revision
url_content = _get_url_content(revision_vars_url)
if not url_content:
logs.log_error('Failed to get component revisions from %s.' %
revision_vars_url)
return None
# Parse as per DEPS format.
if _is_deps(revision_vars_url):
deps_revisions_dict = deps_to_revisions_dict(url_content)
if not deps_revisions_dict:
return None
revisions_dict.update(deps_revisions_dict)
return revisions_dict
# Parse as per Clank DEPS format.
if _is_clank(revision_vars_url):
return _clank_revision_file_to_revisions_dict(url_content)
# Default case: parse content as yaml.
revisions_dict = _to_dict(url_content)
if not revisions_dict:
logs.log_error('Failed to parse component revisions from %s.' %
revision_vars_url)
return None
# Parse as per source map format.
if revision_vars_url.endswith(SOURCE_MAP_EXTENSION):
revisions_dict = _src_map_to_revisions_dict(revisions_dict,
project_name)
return revisions_dict