def notify_closed_issue_if_testcase_is_open(testcase, issue): """Notify closed issue if associated testcase is still open after a certain time period.""" # If the testcase is already closed, no more work to do. if not testcase.open: return # Check testcase status, so as to skip unreproducible uploads. if testcase.status not in ['Processed', 'Duplicate']: return # If there is no associated issue, then bail out. if not issue or not testcase.bug_information: return # If the issue is still open, no work needs to be done. Bail out. if issue.is_open: return # If we have already passed our deadline based on issue closed timestamp, # no need to notify. We will close the testcase instead. if (issue.closed_time and not dates.time_has_expired( issue.closed_time, days=data_types.NOTIFY_CLOSED_BUG_WITH_OPEN_TESTCASE_DEADLINE)): return # Check if there is ignore label on issue already. If yes, bail out. if issue_tracker_utils.was_label_added(issue, data_types.ISSUE_IGNORE_LABEL): return # Check if we did add the notification comment already. If yes, bail out. if issue_tracker_utils.was_label_added( issue, data_types.ISSUE_NEEDS_FEEDBACK_LABEL): return issue.labels.add(data_types.ISSUE_NEEDS_FEEDBACK_LABEL) if issue.status in ['Fixed', 'Verified']: issue_comment = ( 'ClusterFuzz testcase %d is still reproducing on tip-of-tree build ' '(trunk).\n\nPlease re-test your fix against this testcase and if the ' 'fix was incorrect or incomplete, please re-open the bug. Otherwise, ' 'ignore this notification and add %s label.' % (testcase.key.id(), data_types.ISSUE_MISTRIAGED_LABEL)) else: # Covers WontFix, Archived cases. issue_comment = ( 'ClusterFuzz testcase %d is still reproducing on tip-of-tree build ' '(trunk).\n\nIf this testcase was not reproducible locally or ' 'unworkable, ignore this notification and we will file another ' 'bug soon with hopefully a better and workable testcase.\n\n' 'Otherwise, if this is not intended to be fixed (e.g. this is an ' 'intentional crash), please add %s label to prevent future bug filing ' 'with similar crash stacktrace.' % (testcase.key.id(), data_types.ISSUE_IGNORE_LABEL)) issue.save(new_comment=issue_comment, notify=True) logs.log('Notified closed issue for open testcase %d.' % testcase.key.id())
def notify_issue_if_testcase_is_invalid(testcase, issue): """Leave comments on associated issues when test cases are no longer valid.""" if not issue or not testcase.bug_information: return # If the issue is closed, there's no work to do. if not issue.is_open: return # Currently, this only happens if a test case relies on a fuzzer that has # been deleted. This can be modified if more cases are needed in the future. if not testcase.get_metadata('fuzzer_was_deleted'): return # Check if we added this message once. If yes, bail out. if issue_tracker_utils.was_label_added( issue, data_types.ISSUE_INVALID_FUZZER_LABEL): return issue_comment = ( 'ClusterFuzz testcase %d is associated with an obsolete fuzzer and can ' 'no longer be processed. Please close the issue if it is no longer ' 'actionable.') % testcase.key.id() issue.labels.add(data_types.ISSUE_INVALID_FUZZER_LABEL) issue.save(new_comment=issue_comment, notify=True) logs.log('Closed issue %d for invalid testcase %d.' % (issue.id, testcase.key.id()))
def mark_testcase_as_closed_if_issue_is_closed(testcase, issue): """Mark testcase as closed if the associated issue is closed.""" # If the testcase is already closed, no more work to do. if not testcase.open: return # If there is no associated issue, then bail out. if not issue or not testcase.bug_information: return # If the issue is still open, no work needs to be done. Bail out. if issue.is_open: return # Make sure we passed our deadline based on issue closed timestamp. if (issue.closed_time and not dates.time_has_expired( issue.closed_time, days=data_types.CLOSE_TESTCASE_WITH_CLOSED_BUG_DEADLINE)): return # If the issue has an ignore label, don't close the testcase and bail out. # This helps to prevent new bugs from getting filed for legit WontFix cases. if issue_tracker_utils.was_label_added(issue, data_types.ISSUE_IGNORE_LABEL): return testcase.open = False testcase.fixed = 'NA' testcase.put() logs.log('Closed testcase %d with issue closed.' % testcase.key.id())
def update_issue_ccs_from_owners_file(testcase, issue): """Add cc to an issue based on owners list from owners file. This is currently applicable to fuzz targets only.""" if not issue or not issue.is_open: return # If we've assigned the ccs before, it likely means we were incorrect. # Don't try again for this particular issue. if issue_tracker_utils.was_label_added( issue, data_types.ISSUE_CLUSTERFUZZ_AUTO_CC_LABEL): return if testcase.get_metadata('has_issue_ccs_from_owners_file'): return ccs_list = utils.parse_delimited(testcase.get_metadata('issue_owners', ''), delimiter=',', strip=True, remove_empty=True) if not ccs_list: return ccs_added = False actions = list(issue.actions) for cc in random.sample(ccs_list, min(AUTO_CC_LIMIT, len(ccs_list))): if cc in issue.ccs: continue # If cc was previously manually removed from the cc list, we assume that # they were incorrectly added. Don't try to add them again. cc_was_removed = any(cc in action.ccs.removed for action in actions) if cc_was_removed: continue issue.ccs.add(cc) ccs_added = True if not ccs_added: # Everyone we'd expect to see has already been cced on the issue. No need # to spam it with another comment. Also, set the metadata to avoid doing # this again. testcase.set_metadata('has_issue_ccs_from_owners_file', True) return issue_comment = ( 'Automatically adding ccs based on OWNERS file / target commit history.' ) if utils.is_oss_fuzz(): issue_comment += OSS_FUZZ_INCORRECT_COMMENT else: issue_comment += INTERNAL_INCORRECT_COMMENT issue_comment += '.' issue.labels.add(data_types.ISSUE_CLUSTERFUZZ_AUTO_CC_LABEL) issue.save(new_comment=issue_comment, notify=True)
def update_fuzz_blocker_label(testcase, issue, top_crashes_by_project_and_platform_map): """Add top crash label to issue.""" if not issue: return if not testcase.open: return top_crash_platforms = get_top_crash_platforms( testcase, top_crashes_by_project_and_platform_map) if not top_crash_platforms: # Not a top crasher, bail out. return if issue_tracker_utils.was_label_added( issue, data_types.ISSUE_FUZZ_BLOCKER_LABEL): # Issue was already marked a top crasher, bail out. return if len(top_crash_platforms) == 1: platform_message = '%s platform' % top_crash_platforms[0] else: platform_message = '%s and %s platforms' % (', '.join( top_crash_platforms[:-1]), top_crash_platforms[-1]) fuzzer_name = (testcase.get_metadata('fuzzer_binary_name') or testcase.fuzzer_name) update_message = ( 'This crash occurs very frequently on %s and is likely preventing the ' 'fuzzer %s from making much progress. Fixing this will allow more bugs ' 'to be found.' % (platform_message, fuzzer_name)) if utils.is_oss_fuzz(): update_message += OSS_FUZZ_INCORRECT_COMMENT else: update_message += '\n\nMarking this bug as a blocker for next Beta release.' update_message += INTERNAL_INCORRECT_COMMENT update_message += (' and remove the %s label.' % data_types.ISSUE_RELEASEBLOCK_BETA_LABEL) issue.labels.add(data_types.ISSUE_RELEASEBLOCK_BETA_LABEL) # Update with the next beta for trunk, and remove existing milestone label. beta_milestone_label = ( 'M-%d' % build_info.get_release_milestone('head', testcase.platform)) if beta_milestone_label not in issue.labels: issue.labels.remove_by_prefix('M-') issue.labels.add(beta_milestone_label) logs.log(update_message) issue.labels.add(data_types.ISSUE_FUZZ_BLOCKER_LABEL) issue.save(new_comment=update_message, notify=True)
def update_os_labels(testcase, issue): """Add OS labels to issue.""" if not issue: return platforms = get_crash_occurrence_platforms(testcase) logs.log('Found %d platforms for the testcase %d.' % (len(platforms), testcase.key.id()), platforms=platforms) for platform in platforms: os_label = 'OS-%s' % platform if not issue_tracker_utils.was_label_added(issue, os_label): issue.labels.add(os_label) issue.save(notify=False) logs.log('Updated labels of issue %d.' % issue.id, labels=issue.labels)
def update_component_labels(testcase, issue): """Add components to the issue if needed.""" if not issue: return components = _get_predator_result_item(testcase, 'suspected_components', default=[]) # Remove components already in issue or whose more specific variants exist. filtered_components = [] for component in components: found_component_in_issue = any( component == issue_component or issue_component.startswith(component + '>') for issue_component in issue.components) if not found_component_in_issue: filtered_components.append(component) if not filtered_components: # If there are no new components to add, then we shouldn't make any changes # to issue. return # Don't run on issues we've already applied automatic components to in case # labels are removed manually. This may cause issues in the event that we # rerun a test case, but it seems like a reasonable tradeoff to avoid spam. if issue_tracker_utils.was_label_added( issue, data_types.ISSUE_PREDATOR_AUTO_COMPONENTS_LABEL): return for filtered_component in filtered_components: issue.components.add(filtered_component) issue.labels.add(data_types.ISSUE_PREDATOR_AUTO_COMPONENTS_LABEL) issue_comment = ( 'Automatically applying components based on crash stacktrace and ' 'information from OWNERS files.\n\n' 'If this is incorrect, please apply the %s label.' % data_types.ISSUE_PREDATOR_WRONG_COMPONENTS_LABEL) issue.save(new_comment=issue_comment, notify=True)
def update_issue_owner_and_ccs_from_predator_results(testcase, issue, only_allow_ccs=False): """Assign the issue to an appropriate owner if possible.""" if not issue or not issue.is_open: return # If the issue already has an owner, we don't need to update the bug. if issue.assignee: return # If we've assigned an owner or cc once before, it likely means we were # incorrect. Don't try again for this particular issue. if (issue_tracker_utils.was_label_added( issue, data_types.ISSUE_PREDATOR_AUTO_OWNER_LABEL) or issue_tracker_utils.was_label_added( issue, data_types.ISSUE_PREDATOR_AUTO_CC_LABEL)): return # If there are more than 3 suspected CLs, we can't be confident in the # results. Just skip any sort of notification to CL authors in this case. suspected_cls = _get_predator_result_item(testcase, 'suspected_cls') if not suspected_cls or len(suspected_cls) > 3: return # Validate that the suspected CLs have all of the information we need before # continuing. This allows us to assume that they are well-formed later, # avoiding any potential exceptions that would interrupt this task. for suspected_cl in suspected_cls: url = suspected_cl.get('url') description = suspected_cl.get('description') author = suspected_cl.get('author') if not url or not description or not author: logs.log_error( 'Suspected CL for testcase %d is missing required information.' % testcase.key.id()) return if len(suspected_cls) == 1 and not only_allow_ccs: suspected_cl = suspected_cls[0] # If this owner has already been assigned before but has since been removed, # don't assign it to them again. for action in issue.actions: if action.assignee == suspected_cls[0]['author']: return # We have high confidence for the single-CL case, so we assign the owner. issue.labels.add(data_types.ISSUE_PREDATOR_AUTO_OWNER_LABEL) issue.assignee = suspected_cl['author'] issue.status = 'Assigned' issue_comment = ( 'Automatically assigning owner based on suspected regression ' 'changelist %s (%s).\n\n' 'If this is incorrect, please let us know why and apply the %s ' 'label. If you aren\'t the correct owner for this issue, please ' 'unassign yourself as soon as possible so it can be re-triaged.' % (suspected_cl['url'], suspected_cl['description'], data_types.ISSUE_PREDATOR_WRONG_CL_LABEL)) else: if testcase.get_metadata('has_issue_ccs_from_predator_results'): return issue_comment = ( 'Automatically adding ccs based on suspected regression changelists:' '\n\n') ccs_added = False for suspected_cl in suspected_cls: # Update the comment with the suspected CL, regardless of whether or not # we're ccing the author. This might, for example, catch the attention of # someone who has already been cced. author = suspected_cl['author'] issue_comment += '%s by %s - %s\n\n' % ( suspected_cl['description'], author, suspected_cl['url']) if author in issue.ccs: continue # If an author has previously been manually removed from the cc list, # we assume they were incorrectly added. Don't try to add them again. author_was_removed = False for action in issue.actions: if author in action.ccs.removed: author_was_removed = True break if author_was_removed: continue issue.ccs.add(author) ccs_added = True if not ccs_added: # Everyone we'd expect to see has already been cced on the issue. No need # to spam it with another comment. Also, set the metadata to avoid doing # this again. testcase.set_metadata('has_issue_ccs_from_owners_file', True) return issue.labels.add(data_types.ISSUE_PREDATOR_AUTO_CC_LABEL) issue_comment += ( 'If this is incorrect, please let us know why and apply the %s label.' % data_types.ISSUE_PREDATOR_WRONG_CL_LABEL) try: issue.save(new_comment=issue_comment, notify=True) except HttpError: # If we see such an error when we aren't setting an owner, it's unexpected. if only_allow_ccs or not issue.assignee: logs.log_error('Unable to update issue for test case %d.' % testcase.key.id()) return # Retry without setting the owner. They may not be a chromium project # member, in which case we can try falling back to cc. issue = issue_tracker_utils.get_issue_for_testcase(testcase) update_issue_owner_and_ccs_from_predator_results(testcase, issue, only_allow_ccs=True)
def mark_unreproducible_testcase_and_issue_as_closed_after_deadline( testcase, issue): """Closes an unreproducible testcase and its associated issue after a certain time period.""" # If the testcase is already closed, no more work to do. if not testcase.open: return # Check testcase status, so as to skip unreproducible uploads. if testcase.status not in ['Processed', 'Duplicate']: return # Make sure that this testcase is an unreproducible bug. If not, bail out. if not testcase.one_time_crasher_flag: return # Make sure that this testcase has an associated bug. If not, bail out. if not testcase.bug_information: return # If this testcase was manually uploaded, don't change issue state as our # reproduction result might be incorrect. if testcase.uploader_email: return # Make sure that there is an associated bug and it is in open state. if not issue or not issue.is_open: return # Check if there are any reproducible open testcases are associated with # this bug. If yes, return. similar_testcase = data_types.Testcase.query( data_types.Testcase.bug_information == testcase.bug_information, ndb_utils.is_true(data_types.Testcase.open), ndb_utils.is_false(data_types.Testcase.one_time_crasher_flag)).get() if similar_testcase: return # Make sure that testcase is atleast older than # |UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE|, otherwise it will be seen in # crash stats anyway. if (testcase.timestamp and not dates.time_has_expired( testcase.timestamp, days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE)): return # Handle testcase that turned from reproducible to unreproducible. Account # for the recent progression task run time. last_tested_crash_time = testcase.get_metadata('last_tested_crash_time') if (last_tested_crash_time and not dates.time_has_expired( last_tested_crash_time, days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE)): return # Make that there is no crash seen in the deadline period. if get_crash_occurrence_platforms( testcase, data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE): return # As a last check, do the expensive call of actually checking all issue # comments to make sure we we didn't get called out on issue mistriage. if issue_tracker_utils.was_label_added(issue, data_types.ISSUE_MISTRIAGED_LABEL): return # Close associated issue and testcase. comment = ('ClusterFuzz testcase %d is flaky and no longer crashes, ' 'so closing issue.' % testcase.key.id()) if utils.is_oss_fuzz(): comment += OSS_FUZZ_INCORRECT_COMMENT else: comment += INTERNAL_INCORRECT_COMMENT comment += ' and re-open the issue.' issue.status = 'WontFix' issue.save(new_comment=comment, notify=True) testcase.fixed = 'NA' testcase.open = False testcase.put() logs.log('Closed unreproducible testcase %d and associated issue.' % testcase.key.id())
def mark_issue_as_closed_if_testcase_is_fixed(testcase, issue): """Mark an issue as fixed if all of its associated reproducible testcase are fixed.""" # If there is no associated issue, then bail out. if not issue or not testcase.bug_information: return # If the issue is closed in a status other than Fixed, like Duplicate, WontFix # or Archived, we shouldn't change it. Bail out. if not issue.is_open and issue.status != 'Fixed': return # Check testcase status, so as to skip unreproducible uploads. if testcase.status not in ['Processed', 'Duplicate']: return # If the testcase is still open, no work needs to be done. Bail out. if testcase.open: return # FIXME: Find a better solution to skip over reproducible tests that are now # showing up a flaky (esp when we are unable to reproduce crash in original # crash revision). if testcase.fixed == 'NA': return # We can only verify fixed issues for reproducible testcases. If the testcase # is unreproducible, bail out. Exception is if we explicitly marked this as # fixed. if testcase.one_time_crasher_flag and testcase.fixed != 'Yes': return # Make sure that no other testcases associated with this issue are open. similar_testcase = data_types.Testcase.query( data_types.Testcase.bug_information == testcase.bug_information, ndb_utils.is_true(data_types.Testcase.open), ndb_utils.is_false(data_types.Testcase.one_time_crasher_flag)).get() if similar_testcase: return # As a last check, do the expensive call of actually checking all issue # comments to make sure we didn't do the verification already and we didn't # get called out on issue mistriage. if (issue_tracker_utils.was_label_added(issue, data_types.ISSUE_VERIFIED_LABEL) or issue_tracker_utils.was_label_added( issue, data_types.ISSUE_MISTRIAGED_LABEL)): return issue.labels.add(data_types.ISSUE_VERIFIED_LABEL) comment = ('ClusterFuzz testcase %d is verified as fixed, ' 'so closing issue as verified.' % testcase.key.id()) if utils.is_oss_fuzz(): comment += OSS_FUZZ_INCORRECT_COMMENT else: comment += INTERNAL_INCORRECT_COMMENT comment += ' and re-open the issue.' issue.status = 'Verified' issue.save(new_comment=comment, notify=True) logs.log('Closed issue %d for fixed testcase %d.' % (issue.id, testcase.key.id()))