def _add_default_issue_metadata(testcase):
"""Adds the default issue metadata (e.g. components, labels) to testcase."""
default_metadata = engine_common.get_all_issue_metadata_for_testcase(
testcase)
if not default_metadata:
return
testcase_metadata = testcase.get_metadata()
for key, default_value in six.iteritems(default_metadata):
# Add the default issue metadata first. This gives preference to uploader
# specified issue metadata.
new_value_list = utils.parse_delimited(default_value,
delimiter=',',
strip=True,
remove_empty=True)
# Append uploader specified testcase metadata value to end (for preference).
uploader_value = testcase_metadata.get(key, '')
uploader_value_list = utils.parse_delimited(uploader_value,
delimiter=',',
strip=True,
remove_empty=True)
for value in uploader_value_list:
if value not in new_value_list:
new_value_list.append(value)
new_value = ','.join(new_value_list)
if new_value == uploader_value:
continue
logs.log('Updating issue metadata for {} from {} to {}.'.format(
key, uploader_value, new_value))
testcase.set_metadata(key, new_value)
def _get_from_metadata(testcase, name):
"""Get values from testcase metadata."""
return utils.parse_delimited(
testcase.get_metadata(name, ''),
delimiter=',',
strip=True,
remove_empty=True)
def update_issue_ccs_from_owners_file(policy, testcase, issue):
"""Add cc to an issue based on owners list from owners file. This is
currently applicable to fuzz targets only."""
auto_cc_label = policy.label("auto_cc_from_owners")
if not auto_cc_label:
return
if not issue or not issue.is_open:
return
if testcase.get_metadata("has_issue_ccs_from_owners_file"):
return
ccs_list = utils.parse_delimited(
testcase.get_metadata("issue_owners", ""),
delimiter=",",
strip=True,
remove_empty=True,
)
if not ccs_list:
return
# If we've assigned the ccs before, it likely means we were incorrect.
# Don't try again for this particular issue.
if issue_tracker_utils.was_label_added(issue, auto_cc_label):
return
ccs_added = False
actions = list(issue.actions)
for cc in random.sample(ccs_list, min(AUTO_CC_LIMIT, len(ccs_list))):
if cc in issue.ccs:
continue
# If cc was previously manually removed from the cc list, we assume that
# they were incorrectly added. Don't try to add them again.
cc_was_removed = any(cc in action.ccs.removed for action in actions)
if cc_was_removed:
continue
issue.ccs.add(cc)
ccs_added = True
if not ccs_added:
# Everyone we'd expect to see has already been cced on the issue. No need
# to spam it with another comment. Also, set the metadata to avoid doing
# this again.
testcase.set_metadata("has_issue_ccs_from_owners_file", True)
return
issue_comment = (
"Automatically adding ccs based on OWNERS file / target commit history.")
if utils.is_oss_fuzz():
issue_comment += OSS_FUZZ_INCORRECT_COMMENT + "."
else:
issue_comment = _append_generic_incorrect_comment(issue_comment, policy,
issue, ".")
issue.labels.add(auto_cc_label)
issue.save(new_comment=issue_comment, notify=True)
def update_issue_ccs_from_owners_file(testcase, issue):
"""Add cc to an issue based on owners list from owners file. This is
currently applicable to fuzz targets only."""
if not issue or not issue.open:
return
# If we've assigned the ccs before, it likely means we were incorrect.
# Don't try again for this particular issue.
if issue.has_comment_with_label(
data_types.ISSUE_CLUSTERFUZZ_AUTO_CC_LABEL):
return
if testcase.get_metadata('has_issue_ccs_from_owners_file'):
return
ccs_list = utils.parse_delimited(testcase.get_metadata('issue_owners', ''),
delimiter=',',
strip=True,
remove_empty=True)
if not ccs_list:
return
ccs_added = False
comments = issue.get_comments()
for cc in random.sample(ccs_list, min(AUTO_CC_LIMIT, len(ccs_list))):
if issue.has_cc(cc):
continue
# If cc was previously manually removed from the cc list, we assume that
# they were incorrectly added. Don't try to add them again.
cc_was_removed = any(
('-%s' % cc) in comment.cc for comment in comments)
if cc_was_removed:
continue
issue.add_cc(cc)
ccs_added = True
if not ccs_added:
# Everyone we'd expect to see has already been cced on the issue. No need
# to spam it with another comment. Also, set the metadata to avoid doing
# this again.
testcase.set_metadata('has_issue_ccs_from_owners_file', True)
return
issue.comment = (
'Automatically adding ccs based on OWNERS file / target commit history.'
)
if utils.is_oss_fuzz():
issue.comment += OSS_FUZZ_INCORRECT_COMMENT
else:
issue.comment += INTERNAL_INCORRECT_COMMENT
issue.comment += '.'
issue.add_label(data_types.ISSUE_CLUSTERFUZZ_AUTO_CC_LABEL)
issue.save(send_email=True)
def get_issue_metadata(fuzz_target_path, extension):
"""Get issue metadata."""
metadata_file_path = fuzzer_utils.get_supporting_file(fuzz_target_path,
extension)
if environment.is_trusted_host():
metadata_file_path = fuzzer_utils.get_file_from_untrusted_worker(
metadata_file_path)
if not os.path.exists(metadata_file_path):
return []
with open(metadata_file_path) as handle:
return utils.parse_delimited(
handle, delimiter='\n', strip=True, remove_empty=True)
def get_issue_labels(fuzz_target_path):
"""Return list of issue labels given a fuzz target path."""
labels_file_path = fuzzer_utils.get_supporting_file(fuzz_target_path,
LABELS_FILE_EXTENSION)
if environment.is_trusted_host():
labels_file_path = fuzzer_utils.get_file_from_untrusted_worker(
labels_file_path)
if not os.path.exists(labels_file_path):
return []
with open(labels_file_path) as handle:
return utils.parse_delimited(
handle, delimiter='\n', strip=True, remove_empty=True)
def file_issue(testcase,
itm,
security_severity=None,
user_email=None,
additional_ccs=None):
"""File an issue for the given test case."""
issue = Issue()
issue.summary = data_handler.get_issue_summary(testcase)
issue.body = data_handler.get_issue_description(
testcase, reporter=user_email, show_reporter=True)
# Labels applied by default across all issue trackers.
issue.status = 'New'
issue.add_label('ClusterFuzz')
# Add label on memory tool used.
add_memory_tool_label_if_needed(issue, testcase)
# Add reproducibility flag label.
if testcase.one_time_crasher_flag:
issue.add_label('Unreproducible')
else:
issue.add_label('Reproducible')
# Add security severity flag label.
add_security_severity_label_if_needed(issue, testcase, security_severity)
# Get view restriction rules for the job.
issue_restrictions = data_handler.get_value_from_job_definition(
testcase.job_type, 'ISSUE_VIEW_RESTRICTIONS', 'security')
should_restrict_issue = (
issue_restrictions == 'all' or
(issue_restrictions == 'security' and testcase.security_flag))
# Chromium-specific labels.
if itm.project_name == 'chromium':
# A different status system is used on the chromium tracker. Since we
# have already reproduced the crash, we skip the Unconfirmed status.
issue.status = 'Untriaged'
# Add OS label.
if environment.is_chromeos_job(testcase.job_type):
# ChromeOS fuzzers run on Linux platform, so use correct OS-Chrome for
# tracking.
issue.add_label('OS-Chrome')
elif testcase.platform_id:
os_label = 'OS-%s' % ((testcase.platform_id.split(':')[0]).capitalize())
issue.add_label(os_label)
# Add view restrictions for internal job types.
add_view_restrictions_if_needed(issue, testcase)
if testcase.security_flag:
# Apply labels specific to security bugs.
issue.add_label('Restrict-View-SecurityTeam')
issue.add_label('Type-Bug-Security')
# Add reward labels if this is from an external fuzzer contribution.
fuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == testcase.fuzzer_name).get()
if fuzzer and fuzzer.external_contribution:
issue.add_label('reward-topanel')
issue.add_label('External-Fuzzer-Contribution')
data_handler.update_issue_impact_labels(testcase, issue)
else:
# Apply labels for functional (non-security) bugs.
if utils.sub_string_exists_in(NON_CRASH_TYPES, testcase.crash_type):
# Non-crashing test cases shouldn't be assigned Pri-1.
issue.add_label('Pri-2')
issue.add_label('Type-Bug')
else:
# Default functional bug labels.
issue.add_label('Pri-1')
issue.add_label('Stability-Crash')
issue.add_label('Type-Bug')
# AOSP-specific labels.
elif itm.project_name == 'android':
if testcase.security_flag:
# Security bug labels.
issue.add_cc('*****@*****.**')
issue.add_label('Type-Security')
issue.add_label('Restrict-View-Commit')
else:
# Functional bug labels.
issue.add_label('Type-Defect')
# OSS-Fuzz specific labels.
elif itm.project_name == 'oss-fuzz':
if testcase.security_flag:
# Security bug labels.
issue.add_label('Type-Bug-Security')
else:
# Functional bug labels.
issue.add_label('Type-Bug')
if should_restrict_issue:
issue.add_label('Restrict-View-Commit')
# Add additional labels from the job definition and fuzzer.
additional_labels = data_handler.get_additional_values_for_variable(
'AUTOMATIC_LABELS', testcase.job_type, testcase.fuzzer_name)
for label in additional_labels:
issue.add_label(label)
# Add additional components from the job definition and fuzzer.
automatic_components = data_handler.get_additional_values_for_variable(
'AUTOMATIC_COMPONENTS', testcase.job_type, testcase.fuzzer_name)
for component in automatic_components:
issue.add_component(component)
# Add additional ccs from the job definition and fuzzer.
ccs = data_handler.get_additional_values_for_variable(
'AUTOMATIC_CCS', testcase.job_type, testcase.fuzzer_name)
# For externally contributed fuzzers, potentially cc the author.
# Use fully qualified fuzzer name if one is available.
fully_qualified_fuzzer_name = (
testcase.overridden_fuzzer_name or testcase.fuzzer_name)
ccs += external_users.cc_users_for_fuzzer(fully_qualified_fuzzer_name,
testcase.security_flag)
ccs += external_users.cc_users_for_job(testcase.job_type,
testcase.security_flag)
# Add the user as a cc if requested, and any default ccs for this job.
# Check for additional ccs or labels from the job definition.
if additional_ccs:
ccs += [cc for cc in additional_ccs if cc not in ccs]
# For user uploads, we assume the uploader is interested in the issue.
if testcase.uploader_email and testcase.uploader_email not in ccs:
ccs.append(testcase.uploader_email)
if itm.project_name == 'oss-fuzz' and ccs:
# Add a reported label for deadline tracking.
issue.add_label(reported_label())
if issue.has_label_matching('Restrict-View-Commit'):
issue.body += '\n\n' + DEADLINE_NOTE
issue.body += '\n\n' + FIX_NOTE
issue.body += '\n\n' + QUESTIONS_NOTE
for cc in ccs:
issue.add_cc(cc)
# Add additional labels from testcase metadata.
metadata_labels = utils.parse_delimited(
testcase.get_metadata('issue_labels', ''),
delimiter=',',
strip=True,
remove_empty=True)
for label in metadata_labels:
issue.add_label(label)
issue.itm = itm
issue.reporter = user_email
issue.save()
# Update the testcase with this newly created issue.
testcase.bug_information = str(issue.id)
testcase.put()
data_handler.update_group_bug(testcase.group_id)
return issue.id
def file_issue(testcase,
issue_tracker,
security_severity=None,
user_email=None,
additional_ccs=None):
"""File an issue for the given test case."""
issue = issue_tracker.new_issue()
issue.title = data_handler.get_issue_summary(testcase)
issue.body = data_handler.get_issue_description(testcase,
reporter=user_email,
show_reporter=True)
policy = issue_tracker_policy.get(issue_tracker.project)
# Add reproducibility flag label.
if testcase.one_time_crasher_flag:
issue.labels.add(policy.label('unreproducible'))
else:
issue.labels.add(policy.label('reproducible'))
# Chromium-specific labels.
if issue_tracker.project and issue_tracker.project == 'chromium':
# Add reward labels if this is from an external fuzzer contribution.
fuzzer = data_types.Fuzzer.query(
data_types.Fuzzer.name == testcase.fuzzer_name).get()
if fuzzer and fuzzer.external_contribution:
issue.labels.add('reward-topanel')
issue.labels.add('External-Fuzzer-Contribution')
update_issue_impact_labels(testcase, issue)
# Add additional labels from the job definition and fuzzer.
additional_labels = data_handler.get_additional_values_for_variable(
'AUTOMATIC_LABELS', testcase.job_type, testcase.fuzzer_name)
for label in additional_labels:
issue.labels.add(label)
# Add additional components from the job definition and fuzzer.
automatic_components = data_handler.get_additional_values_for_variable(
'AUTOMATIC_COMPONENTS', testcase.job_type, testcase.fuzzer_name)
for component in automatic_components:
issue.components.add(component)
is_crash = not utils.sub_string_exists_in(NON_CRASH_TYPES,
testcase.crash_type)
properties = policy.get_new_issue_properties(
is_security=testcase.security_flag, is_crash=is_crash)
issue.status = properties.status
# Add additional ccs from the job definition and fuzzer.
ccs = data_handler.get_additional_values_for_variable(
'AUTOMATIC_CCS', testcase.job_type, testcase.fuzzer_name)
# For externally contributed fuzzers, potentially cc the author.
# Use fully qualified fuzzer name if one is available.
fully_qualified_fuzzer_name = (testcase.overridden_fuzzer_name
or testcase.fuzzer_name)
ccs += external_users.cc_users_for_fuzzer(fully_qualified_fuzzer_name,
testcase.security_flag)
ccs += external_users.cc_users_for_job(testcase.job_type,
testcase.security_flag)
# Add the user as a cc if requested, and any default ccs for this job.
# Check for additional ccs or labels from the job definition.
if additional_ccs:
ccs += [cc for cc in additional_ccs if cc not in ccs]
# For user uploads, we assume the uploader is interested in the issue.
if testcase.uploader_email and testcase.uploader_email not in ccs:
ccs.append(testcase.uploader_email)
ccs.extend(properties.ccs)
# Get view restriction rules for the job.
issue_restrictions = data_handler.get_value_from_job_definition(
testcase.job_type, 'ISSUE_VIEW_RESTRICTIONS', 'security')
should_restrict_issue = (issue_restrictions == 'all'
or (issue_restrictions == 'security'
and testcase.security_flag))
has_accountable_people = bool(ccs)
# Check for labels with special logic.
additional_labels = []
if should_restrict_issue:
additional_labels.append(policy.label('restrict_view'))
if has_accountable_people:
additional_labels.append(policy.label('reported'))
if testcase.security_flag:
additional_labels.append(policy.label('security_severity'))
additional_labels.append(policy.label('os'))
# Apply label substitutions.
for label in itertools.chain(properties.labels, additional_labels):
for result in apply_substitutions(label, testcase, security_severity):
issue.labels.add(result)
issue.body += properties.issue_body_footer
if (should_restrict_issue and has_accountable_people
and policy.deadline_policy_message):
issue.body += '\n\n' + policy.deadline_policy_message
for cc in ccs:
issue.ccs.add(cc)
# Add additional labels from testcase metadata.
metadata_labels = utils.parse_delimited(testcase.get_metadata(
'issue_labels', ''),
delimiter=',',
strip=True,
remove_empty=True)
for label in metadata_labels:
issue.labels.add(label)
issue.reporter = user_email
issue.save()
# Update the testcase with this newly created issue.
testcase.bug_information = str(issue.id)
testcase.put()
data_handler.update_group_bug(testcase.group_id)
return issue.id