def notify_closed_issue_if_testcase_is_open(testcase, issue):
"""Notify closed issue if associated testcase is still open after a certain
time period."""
# If the testcase is already closed, no more work to do.
if not testcase.open:
return
# Check testcase status, so as to skip unreproducible uploads.
if testcase.status not in ['Processed', 'Duplicate']:
return
# If there is no associated issue, then bail out.
if not issue or not testcase.bug_information:
return
# If the issue is still open, no work needs to be done. Bail out.
if issue.is_open:
return
# If we have already passed our deadline based on issue closed timestamp,
# no need to notify. We will close the testcase instead.
if (issue.closed_time and not dates.time_has_expired(
issue.closed_time,
days=data_types.NOTIFY_CLOSED_BUG_WITH_OPEN_TESTCASE_DEADLINE)):
return
# Check if there is ignore label on issue already. If yes, bail out.
if issue_tracker_utils.was_label_added(issue,
data_types.ISSUE_IGNORE_LABEL):
return
# Check if we did add the notification comment already. If yes, bail out.
if issue_tracker_utils.was_label_added(
issue, data_types.ISSUE_NEEDS_FEEDBACK_LABEL):
return
issue.labels.add(data_types.ISSUE_NEEDS_FEEDBACK_LABEL)
if issue.status in ['Fixed', 'Verified']:
issue_comment = (
'ClusterFuzz testcase %d is still reproducing on tip-of-tree build '
'(trunk).\n\nPlease re-test your fix against this testcase and if the '
'fix was incorrect or incomplete, please re-open the bug. Otherwise, '
'ignore this notification and add %s label.' %
(testcase.key.id(), data_types.ISSUE_MISTRIAGED_LABEL))
else:
# Covers WontFix, Archived cases.
issue_comment = (
'ClusterFuzz testcase %d is still reproducing on tip-of-tree build '
'(trunk).\n\nIf this testcase was not reproducible locally or '
'unworkable, ignore this notification and we will file another '
'bug soon with hopefully a better and workable testcase.\n\n'
'Otherwise, if this is not intended to be fixed (e.g. this is an '
'intentional crash), please add %s label to prevent future bug filing '
'with similar crash stacktrace.' %
(testcase.key.id(), data_types.ISSUE_IGNORE_LABEL))
issue.save(new_comment=issue_comment, notify=True)
logs.log('Notified closed issue for open testcase %d.' % testcase.key.id())
def notify_issue_if_testcase_is_invalid(testcase, issue):
"""Leave comments on associated issues when test cases are no longer valid."""
if not issue or not testcase.bug_information:
return
# If the issue is closed, there's no work to do.
if not issue.is_open:
return
# Currently, this only happens if a test case relies on a fuzzer that has
# been deleted. This can be modified if more cases are needed in the future.
if not testcase.get_metadata('fuzzer_was_deleted'):
return
# Check if we added this message once. If yes, bail out.
if issue_tracker_utils.was_label_added(
issue, data_types.ISSUE_INVALID_FUZZER_LABEL):
return
issue_comment = (
'ClusterFuzz testcase %d is associated with an obsolete fuzzer and can '
'no longer be processed. Please close the issue if it is no longer '
'actionable.') % testcase.key.id()
issue.labels.add(data_types.ISSUE_INVALID_FUZZER_LABEL)
issue.save(new_comment=issue_comment, notify=True)
logs.log('Closed issue %d for invalid testcase %d.' %
(issue.id, testcase.key.id()))
def mark_testcase_as_closed_if_issue_is_closed(testcase, issue):
"""Mark testcase as closed if the associated issue is closed."""
# If the testcase is already closed, no more work to do.
if not testcase.open:
return
# If there is no associated issue, then bail out.
if not issue or not testcase.bug_information:
return
# If the issue is still open, no work needs to be done. Bail out.
if issue.is_open:
return
# Make sure we passed our deadline based on issue closed timestamp.
if (issue.closed_time and not dates.time_has_expired(
issue.closed_time,
days=data_types.CLOSE_TESTCASE_WITH_CLOSED_BUG_DEADLINE)):
return
# If the issue has an ignore label, don't close the testcase and bail out.
# This helps to prevent new bugs from getting filed for legit WontFix cases.
if issue_tracker_utils.was_label_added(issue,
data_types.ISSUE_IGNORE_LABEL):
return
testcase.open = False
testcase.fixed = 'NA'
testcase.put()
logs.log('Closed testcase %d with issue closed.' % testcase.key.id())
def update_issue_ccs_from_owners_file(testcase, issue):
"""Add cc to an issue based on owners list from owners file. This is
currently applicable to fuzz targets only."""
if not issue or not issue.is_open:
return
# If we've assigned the ccs before, it likely means we were incorrect.
# Don't try again for this particular issue.
if issue_tracker_utils.was_label_added(
issue, data_types.ISSUE_CLUSTERFUZZ_AUTO_CC_LABEL):
return
if testcase.get_metadata('has_issue_ccs_from_owners_file'):
return
ccs_list = utils.parse_delimited(testcase.get_metadata('issue_owners', ''),
delimiter=',',
strip=True,
remove_empty=True)
if not ccs_list:
return
ccs_added = False
actions = list(issue.actions)
for cc in random.sample(ccs_list, min(AUTO_CC_LIMIT, len(ccs_list))):
if cc in issue.ccs:
continue
# If cc was previously manually removed from the cc list, we assume that
# they were incorrectly added. Don't try to add them again.
cc_was_removed = any(cc in action.ccs.removed for action in actions)
if cc_was_removed:
continue
issue.ccs.add(cc)
ccs_added = True
if not ccs_added:
# Everyone we'd expect to see has already been cced on the issue. No need
# to spam it with another comment. Also, set the metadata to avoid doing
# this again.
testcase.set_metadata('has_issue_ccs_from_owners_file', True)
return
issue_comment = (
'Automatically adding ccs based on OWNERS file / target commit history.'
)
if utils.is_oss_fuzz():
issue_comment += OSS_FUZZ_INCORRECT_COMMENT
else:
issue_comment += INTERNAL_INCORRECT_COMMENT
issue_comment += '.'
issue.labels.add(data_types.ISSUE_CLUSTERFUZZ_AUTO_CC_LABEL)
issue.save(new_comment=issue_comment, notify=True)
def update_fuzz_blocker_label(testcase, issue,
top_crashes_by_project_and_platform_map):
"""Add top crash label to issue."""
if not issue:
return
if not testcase.open:
return
top_crash_platforms = get_top_crash_platforms(
testcase, top_crashes_by_project_and_platform_map)
if not top_crash_platforms:
# Not a top crasher, bail out.
return
if issue_tracker_utils.was_label_added(
issue, data_types.ISSUE_FUZZ_BLOCKER_LABEL):
# Issue was already marked a top crasher, bail out.
return
if len(top_crash_platforms) == 1:
platform_message = '%s platform' % top_crash_platforms[0]
else:
platform_message = '%s and %s platforms' % (', '.join(
top_crash_platforms[:-1]), top_crash_platforms[-1])
fuzzer_name = (testcase.get_metadata('fuzzer_binary_name')
or testcase.fuzzer_name)
update_message = (
'This crash occurs very frequently on %s and is likely preventing the '
'fuzzer %s from making much progress. Fixing this will allow more bugs '
'to be found.' % (platform_message, fuzzer_name))
if utils.is_oss_fuzz():
update_message += OSS_FUZZ_INCORRECT_COMMENT
else:
update_message += '\n\nMarking this bug as a blocker for next Beta release.'
update_message += INTERNAL_INCORRECT_COMMENT
update_message += (' and remove the %s label.' %
data_types.ISSUE_RELEASEBLOCK_BETA_LABEL)
issue.labels.add(data_types.ISSUE_RELEASEBLOCK_BETA_LABEL)
# Update with the next beta for trunk, and remove existing milestone label.
beta_milestone_label = (
'M-%d' %
build_info.get_release_milestone('head', testcase.platform))
if beta_milestone_label not in issue.labels:
issue.labels.remove_by_prefix('M-')
issue.labels.add(beta_milestone_label)
logs.log(update_message)
issue.labels.add(data_types.ISSUE_FUZZ_BLOCKER_LABEL)
issue.save(new_comment=update_message, notify=True)
def update_os_labels(testcase, issue):
"""Add OS labels to issue."""
if not issue:
return
platforms = get_crash_occurrence_platforms(testcase)
logs.log('Found %d platforms for the testcase %d.' %
(len(platforms), testcase.key.id()),
platforms=platforms)
for platform in platforms:
os_label = 'OS-%s' % platform
if not issue_tracker_utils.was_label_added(issue, os_label):
issue.labels.add(os_label)
issue.save(notify=False)
logs.log('Updated labels of issue %d.' % issue.id, labels=issue.labels)
def update_component_labels(testcase, issue):
"""Add components to the issue if needed."""
if not issue:
return
components = _get_predator_result_item(testcase,
'suspected_components',
default=[])
# Remove components already in issue or whose more specific variants exist.
filtered_components = []
for component in components:
found_component_in_issue = any(
component == issue_component
or issue_component.startswith(component + '>')
for issue_component in issue.components)
if not found_component_in_issue:
filtered_components.append(component)
if not filtered_components:
# If there are no new components to add, then we shouldn't make any changes
# to issue.
return
# Don't run on issues we've already applied automatic components to in case
# labels are removed manually. This may cause issues in the event that we
# rerun a test case, but it seems like a reasonable tradeoff to avoid spam.
if issue_tracker_utils.was_label_added(
issue, data_types.ISSUE_PREDATOR_AUTO_COMPONENTS_LABEL):
return
for filtered_component in filtered_components:
issue.components.add(filtered_component)
issue.labels.add(data_types.ISSUE_PREDATOR_AUTO_COMPONENTS_LABEL)
issue_comment = (
'Automatically applying components based on crash stacktrace and '
'information from OWNERS files.\n\n'
'If this is incorrect, please apply the %s label.' %
data_types.ISSUE_PREDATOR_WRONG_COMPONENTS_LABEL)
issue.save(new_comment=issue_comment, notify=True)
def update_issue_owner_and_ccs_from_predator_results(testcase,
issue,
only_allow_ccs=False):
"""Assign the issue to an appropriate owner if possible."""
if not issue or not issue.is_open:
return
# If the issue already has an owner, we don't need to update the bug.
if issue.assignee:
return
# If we've assigned an owner or cc once before, it likely means we were
# incorrect. Don't try again for this particular issue.
if (issue_tracker_utils.was_label_added(
issue, data_types.ISSUE_PREDATOR_AUTO_OWNER_LABEL)
or issue_tracker_utils.was_label_added(
issue, data_types.ISSUE_PREDATOR_AUTO_CC_LABEL)):
return
# If there are more than 3 suspected CLs, we can't be confident in the
# results. Just skip any sort of notification to CL authors in this case.
suspected_cls = _get_predator_result_item(testcase, 'suspected_cls')
if not suspected_cls or len(suspected_cls) > 3:
return
# Validate that the suspected CLs have all of the information we need before
# continuing. This allows us to assume that they are well-formed later,
# avoiding any potential exceptions that would interrupt this task.
for suspected_cl in suspected_cls:
url = suspected_cl.get('url')
description = suspected_cl.get('description')
author = suspected_cl.get('author')
if not url or not description or not author:
logs.log_error(
'Suspected CL for testcase %d is missing required information.'
% testcase.key.id())
return
if len(suspected_cls) == 1 and not only_allow_ccs:
suspected_cl = suspected_cls[0]
# If this owner has already been assigned before but has since been removed,
# don't assign it to them again.
for action in issue.actions:
if action.assignee == suspected_cls[0]['author']:
return
# We have high confidence for the single-CL case, so we assign the owner.
issue.labels.add(data_types.ISSUE_PREDATOR_AUTO_OWNER_LABEL)
issue.assignee = suspected_cl['author']
issue.status = 'Assigned'
issue_comment = (
'Automatically assigning owner based on suspected regression '
'changelist %s (%s).\n\n'
'If this is incorrect, please let us know why and apply the %s '
'label. If you aren\'t the correct owner for this issue, please '
'unassign yourself as soon as possible so it can be re-triaged.' %
(suspected_cl['url'], suspected_cl['description'],
data_types.ISSUE_PREDATOR_WRONG_CL_LABEL))
else:
if testcase.get_metadata('has_issue_ccs_from_predator_results'):
return
issue_comment = (
'Automatically adding ccs based on suspected regression changelists:'
'\n\n')
ccs_added = False
for suspected_cl in suspected_cls:
# Update the comment with the suspected CL, regardless of whether or not
# we're ccing the author. This might, for example, catch the attention of
# someone who has already been cced.
author = suspected_cl['author']
issue_comment += '%s by %s - %s\n\n' % (
suspected_cl['description'], author, suspected_cl['url'])
if author in issue.ccs:
continue
# If an author has previously been manually removed from the cc list,
# we assume they were incorrectly added. Don't try to add them again.
author_was_removed = False
for action in issue.actions:
if author in action.ccs.removed:
author_was_removed = True
break
if author_was_removed:
continue
issue.ccs.add(author)
ccs_added = True
if not ccs_added:
# Everyone we'd expect to see has already been cced on the issue. No need
# to spam it with another comment. Also, set the metadata to avoid doing
# this again.
testcase.set_metadata('has_issue_ccs_from_owners_file', True)
return
issue.labels.add(data_types.ISSUE_PREDATOR_AUTO_CC_LABEL)
issue_comment += (
'If this is incorrect, please let us know why and apply the %s label.'
% data_types.ISSUE_PREDATOR_WRONG_CL_LABEL)
try:
issue.save(new_comment=issue_comment, notify=True)
except HttpError:
# If we see such an error when we aren't setting an owner, it's unexpected.
if only_allow_ccs or not issue.assignee:
logs.log_error('Unable to update issue for test case %d.' %
testcase.key.id())
return
# Retry without setting the owner. They may not be a chromium project
# member, in which case we can try falling back to cc.
issue = issue_tracker_utils.get_issue_for_testcase(testcase)
update_issue_owner_and_ccs_from_predator_results(testcase,
issue,
only_allow_ccs=True)
def mark_unreproducible_testcase_and_issue_as_closed_after_deadline(
testcase, issue):
"""Closes an unreproducible testcase and its associated issue after a certain
time period."""
# If the testcase is already closed, no more work to do.
if not testcase.open:
return
# Check testcase status, so as to skip unreproducible uploads.
if testcase.status not in ['Processed', 'Duplicate']:
return
# Make sure that this testcase is an unreproducible bug. If not, bail out.
if not testcase.one_time_crasher_flag:
return
# Make sure that this testcase has an associated bug. If not, bail out.
if not testcase.bug_information:
return
# If this testcase was manually uploaded, don't change issue state as our
# reproduction result might be incorrect.
if testcase.uploader_email:
return
# Make sure that there is an associated bug and it is in open state.
if not issue or not issue.is_open:
return
# Check if there are any reproducible open testcases are associated with
# this bug. If yes, return.
similar_testcase = data_types.Testcase.query(
data_types.Testcase.bug_information == testcase.bug_information,
ndb_utils.is_true(data_types.Testcase.open),
ndb_utils.is_false(data_types.Testcase.one_time_crasher_flag)).get()
if similar_testcase:
return
# Make sure that testcase is atleast older than
# |UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE|, otherwise it will be seen in
# crash stats anyway.
if (testcase.timestamp and not dates.time_has_expired(
testcase.timestamp,
days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE)):
return
# Handle testcase that turned from reproducible to unreproducible. Account
# for the recent progression task run time.
last_tested_crash_time = testcase.get_metadata('last_tested_crash_time')
if (last_tested_crash_time and not dates.time_has_expired(
last_tested_crash_time,
days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE)):
return
# Make that there is no crash seen in the deadline period.
if get_crash_occurrence_platforms(
testcase, data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE):
return
# As a last check, do the expensive call of actually checking all issue
# comments to make sure we we didn't get called out on issue mistriage.
if issue_tracker_utils.was_label_added(issue,
data_types.ISSUE_MISTRIAGED_LABEL):
return
# Close associated issue and testcase.
comment = ('ClusterFuzz testcase %d is flaky and no longer crashes, '
'so closing issue.' % testcase.key.id())
if utils.is_oss_fuzz():
comment += OSS_FUZZ_INCORRECT_COMMENT
else:
comment += INTERNAL_INCORRECT_COMMENT
comment += ' and re-open the issue.'
issue.status = 'WontFix'
issue.save(new_comment=comment, notify=True)
testcase.fixed = 'NA'
testcase.open = False
testcase.put()
logs.log('Closed unreproducible testcase %d and associated issue.' %
testcase.key.id())
def mark_issue_as_closed_if_testcase_is_fixed(testcase, issue):
"""Mark an issue as fixed if all of its associated reproducible testcase are
fixed."""
# If there is no associated issue, then bail out.
if not issue or not testcase.bug_information:
return
# If the issue is closed in a status other than Fixed, like Duplicate, WontFix
# or Archived, we shouldn't change it. Bail out.
if not issue.is_open and issue.status != 'Fixed':
return
# Check testcase status, so as to skip unreproducible uploads.
if testcase.status not in ['Processed', 'Duplicate']:
return
# If the testcase is still open, no work needs to be done. Bail out.
if testcase.open:
return
# FIXME: Find a better solution to skip over reproducible tests that are now
# showing up a flaky (esp when we are unable to reproduce crash in original
# crash revision).
if testcase.fixed == 'NA':
return
# We can only verify fixed issues for reproducible testcases. If the testcase
# is unreproducible, bail out. Exception is if we explicitly marked this as
# fixed.
if testcase.one_time_crasher_flag and testcase.fixed != 'Yes':
return
# Make sure that no other testcases associated with this issue are open.
similar_testcase = data_types.Testcase.query(
data_types.Testcase.bug_information == testcase.bug_information,
ndb_utils.is_true(data_types.Testcase.open),
ndb_utils.is_false(data_types.Testcase.one_time_crasher_flag)).get()
if similar_testcase:
return
# As a last check, do the expensive call of actually checking all issue
# comments to make sure we didn't do the verification already and we didn't
# get called out on issue mistriage.
if (issue_tracker_utils.was_label_added(issue,
data_types.ISSUE_VERIFIED_LABEL)
or issue_tracker_utils.was_label_added(
issue, data_types.ISSUE_MISTRIAGED_LABEL)):
return
issue.labels.add(data_types.ISSUE_VERIFIED_LABEL)
comment = ('ClusterFuzz testcase %d is verified as fixed, '
'so closing issue as verified.' % testcase.key.id())
if utils.is_oss_fuzz():
comment += OSS_FUZZ_INCORRECT_COMMENT
else:
comment += INTERNAL_INCORRECT_COMMENT
comment += ' and re-open the issue.'
issue.status = 'Verified'
issue.save(new_comment=comment, notify=True)
logs.log('Closed issue %d for fixed testcase %d.' %
(issue.id, testcase.key.id()))