def get_issue_tracker_for_testcase(testcase):
"""Get an IssueTracker or raise EarlyExitException."""
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(testcase)
if not issue_tracker:
raise EarlyExitException(
"The testcase doesn't have a corresponding issue tracker", 404)
return issue_tracker
def check_public_testcase(self, blob_info, testcase):
"""Check public testcase."""
if blob_info.key() != testcase.minimized_keys:
return False
if not testcase.bug_information:
return False
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(
testcase)
issue = issue_tracker.get_issue(testcase.bug_information)
if not issue:
return False
# If the issue is explicitly marked as view restricted to committers only
# (OSS-Fuzz only), then don't allow public download.
if 'restrict-view-commit' in issue.labels:
return False
# For OSS-Fuzz, delay the disclosure of the reproducer by 30 days.
# If the deadline had previously exceeded, the reproducer was made public
# already so exclude that case.
if (utils.is_oss_fuzz() and 'deadline-exceeded' not in issue.labels
and issue.closed_time and not dates.time_has_expired(
issue.closed_time, days=_OSS_FUZZ_REPRODUCER_DELAY)):
return False
return True
def get(self):
"""Handle a cron job."""
@memoize.wrap(memoize.FifoInMemory(256))
def cc_users_for_job(job_type, security_flag):
"""Return users to CC for a job."""
# Memoized per cron run.
return external_users.cc_users_for_job(job_type, security_flag)
for testcase in get_open_testcases_with_bugs():
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(
testcase)
if not issue_tracker:
logging.error('Failed to get issue tracker manager for %s',
testcase.key.id())
continue
policy = issue_tracker_policy.get(issue_tracker.project)
reported_label = policy.label('reported')
if not reported_label:
return
reported_pattern = issue_filer.get_label_pattern(reported_label)
try:
issue = issue_tracker.get_original_issue(
testcase.bug_information)
except:
logging.error('Error occurred when fetching issue %s.',
testcase.bug_information)
continue
if not issue or not issue.is_open:
continue
ccs = cc_users_for_job(testcase.job_type, testcase.security_flag)
new_ccs = [cc for cc in ccs if cc not in issue.ccs]
if not new_ccs:
# Nothing to do.
continue
for cc in new_ccs:
logging.info('CCing %s on %s', cc, issue.id)
issue.ccs.add(cc)
comment = None
if (not issue.labels.has_with_pattern(reported_pattern)
and not data_handler.get_value_from_job_definition(
testcase.job_type, 'DISABLE_DISCLOSURE', False)):
# Add reported label and deadline comment if necessary.
for result in issue_filer.apply_substitutions(
policy, reported_label, testcase):
issue.labels.add(result)
if policy.label('restrict_view') in issue.labels:
logging.info('Adding deadline comment on %s', issue.id)
comment = policy.deadline_policy_message
issue.save(new_comment=comment, notify=True)
def can_user_access_testcase(testcase):
"""Checks if the current user can access the testcase."""
config = db_config.get()
need_privileged_access = (testcase.security_flag
and not config.relax_security_bug_restrictions)
if has_access(fuzzer_name=testcase.actual_fuzzer_name(),
job_type=testcase.job_type,
need_privileged_access=need_privileged_access):
return True
user_email = helpers.get_user_email()
if testcase.uploader_email and testcase.uploader_email == user_email:
return True
# Allow owners of bugs to see associated test cases and test case groups.
issue_id = testcase.bug_information or testcase.group_bug_information
if not issue_id:
return False
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(
testcase)
associated_issue = issue_tracker.get_issue(issue_id)
if not associated_issue:
return False
# Look at both associated issue and original issue (if the associated one
# is a duplicate of the original issue).
issues_to_check = [associated_issue]
if associated_issue.merged_into:
original_issue = issue_tracker.get_original_issue(issue_id)
if original_issue:
issues_to_check.append(original_issue)
relaxed_restrictions = (config.relax_testcase_restrictions
or _is_domain_allowed(user_email))
for issue in issues_to_check:
if relaxed_restrictions:
if (any(utils.emails_equal(user_email, cc) for cc in issue.ccs)
or utils.emails_equal(user_email, issue.assignee)
or utils.emails_equal(user_email, issue.reporter)):
return True
elif utils.emails_equal(user_email, issue.assignee):
return True
return False
def check_public_testcase(self, blob_info, testcase):
"""Check public testcase."""
if blob_info.key() != testcase.minimized_keys:
return False
if not testcase.bug_information:
return False
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(testcase)
issue = issue_tracker.get_issue(testcase.bug_information)
if not issue:
return False
# If the issue is explicitly marked as view restricted to committers only
# (OSS-Fuzz only), then don't allow public download.
if 'restrict-view-commit' in issue.labels:
return False
return True
def handle_public_testcase(self, blob_info, testcase, fuzzer_binary_name):
"""Handle public testcase."""
if blob_info.key() != testcase.minimized_keys:
return False
if not testcase.bug_information:
return False
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(
testcase)
issue = issue_tracker.get_issue(testcase.bug_information)
if not issue:
return False
# If the issue is explicitly marked as view restricted to committers only
# (OSS-Fuzz only), then don't allow public download.
if 'restrict-view-commit' in issue.labels:
return False
self._send_blob(blob_info,
testcase.key.id(),
is_minimized=True,
fuzzer_binary_name=fuzzer_binary_name)
return True
def get(self):
"""Handle a get request."""
try:
grouper.group_testcases()
except:
logs.log_error('Error occurred while grouping test cases.')
return
# Free up memory after group task run.
utils.python_gc()
# Get a list of jobs excluded from bug filing.
excluded_jobs = _get_excluded_jobs()
# Get a list of all jobs. This is used to filter testcases whose jobs have
# been removed.
all_jobs = data_handler.get_all_job_type_names()
for testcase_id in data_handler.get_open_testcase_id_iterator():
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
# Already deleted.
continue
# Skip if testcase's job is removed.
if testcase.job_type not in all_jobs:
continue
# Skip if testcase's job is in exclusions list.
if testcase.job_type in excluded_jobs:
continue
# Skip if we are running progression task at this time.
if testcase.get_metadata('progression_pending'):
continue
# If the testcase has a bug filed already, no triage is needed.
if _is_bug_filed(testcase):
continue
# Check if the crash is important, i.e. it is either a reproducible crash
# or an unreproducible crash happening frequently.
if not _is_crash_important(testcase):
continue
# Require that all tasks like minimizaton, regression testing, etc have
# finished.
if not data_handler.critical_tasks_completed(testcase):
continue
# For testcases that are not part of a group, wait an additional time till
# group task completes.
# FIXME: In future, grouping might be dependent on regression range, so we
# would have to add an additional wait time.
if not testcase.group_id and not dates.time_has_expired(
testcase.timestamp,
hours=data_types.MIN_ELAPSED_TIME_SINCE_REPORT):
continue
# If this project does not have an associated issue tracker, we cannot
# file this crash anywhere.
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(
testcase)
if not issue_tracker:
continue
# If there are similar issues to this test case already filed or recently
# closed, skip filing a duplicate bug.
if _check_and_update_similar_bug(testcase, issue_tracker):
continue
# Clean up old triage messages that would be not applicable now.
testcase.delete_metadata(TRIAGE_MESSAGE_KEY, update_testcase=False)
# File the bug first and then create filed bug metadata.
try:
issue_filer.file_issue(testcase, issue_tracker)
except Exception:
logs.log_error('Failed to file issue for testcase %d.' %
testcase_id)
continue
_create_filed_bug_metadata(testcase)
logs.log('Filed new issue %s for testcase %d.' %
(testcase.bug_information, testcase_id))
def group_testcases():
"""Group testcases based on rules like same bug numbers, similar crash
states, etc."""
testcase_map = {}
cached_issue_map = {}
for testcase_id in data_handler.get_open_testcase_id_iterator():
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
# Already deleted.
continue
# Remove duplicates early on to avoid large groups.
if (not testcase.bug_information and not testcase.uploader_email and
has_testcase_with_same_params(testcase, testcase_map)):
logs.log('Deleting duplicate testcase %d.' % testcase_id)
testcase.key.delete()
continue
# Wait for minimization to finish as this might change crash params such
# as type and may mark it as duplicate / closed.
if not testcase.minimized_keys:
continue
# Store needed testcase attributes into |testcase_map|.
testcase_map[testcase_id] = TestcaseAttributes()
testcase_attributes = testcase_map[testcase_id]
for attribute_name in FORWARDED_ATTRIBUTES:
setattr(testcase_attributes, attribute_name,
getattr(testcase, attribute_name))
# Store original issue mappings in the testcase attributes.
if testcase.bug_information:
issue_id = int(testcase.bug_information)
project_name = testcase.project_name
if (project_name in cached_issue_map and
issue_id in cached_issue_map[project_name]):
testcase_attributes.issue_id = (
cached_issue_map[project_name][issue_id])
else:
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(
testcase)
if not issue_tracker:
continue
# Determine the original issue id traversing the list of duplicates.
try:
issue = issue_tracker.get_original_issue(issue_id)
original_issue_id = issue.id
except:
# If we are unable to access the issue, then we can't determine
# the original issue id. Assume that it is the same as issue id.
logs.log_error(
'Unable to determine original issue for %d.' % issue_id)
original_issue_id = issue_id
if project_name not in cached_issue_map:
cached_issue_map[project_name] = {}
cached_issue_map[project_name][issue_id] = original_issue_id
cached_issue_map[project_name][original_issue_id] = original_issue_id
testcase_attributes.issue_id = original_issue_id
# No longer needed. Free up some memory.
cached_issue_map.clear()
group_testcases_with_similar_states(testcase_map)
group_testcases_with_same_issues(testcase_map)
group_leader.choose(testcase_map)
# TODO(aarya): Replace with an optimized implementation using dirty flag.
# Update the group mapping in testcase object.
for testcase_id in data_handler.get_open_testcase_id_iterator():
if testcase_id not in testcase_map:
# A new testcase that was just created. Skip for now, will be grouped in
# next iteration of group task.
continue
# If we are part of a group, then calculate the number of testcases in that
# group and lowest issue id of issues associated with testcases in that
# group.
updated_group_id = testcase_map[testcase_id].group_id
updated_is_leader = testcase_map[testcase_id].is_leader
updated_group_id_count = 0
updated_group_bug_information = 0
if updated_group_id:
for other_testcase in six.itervalues(testcase_map):
if other_testcase.group_id != updated_group_id:
continue
updated_group_id_count += 1
# Update group issue id to be lowest issue id in the entire group.
if other_testcase.issue_id is None:
continue
if (not updated_group_bug_information or
updated_group_bug_information > other_testcase.issue_id):
updated_group_bug_information = other_testcase.issue_id
# If this group id is used by only one testcase, then remove it.
if updated_group_id_count == 1:
data_handler.delete_group(updated_group_id, update_testcases=False)
updated_group_id = 0
updated_group_bug_information = 0
updated_is_leader = True
# If this group has more than the maximum allowed testcases, log an error
# so that the sheriff can later debug what caused this. Usually, this is a
# bug in grouping logic OR a ever changing crash signature (e.g. slightly
# different crash types or crash states). We cannot bail out as otherwise,
# we will not group the testcase leading to a spam of new filed bugs.
if updated_group_id_count > GROUP_MAX_TESTCASE_LIMIT:
logs.log_error(
'Group %d exceeds maximum allowed testcases.' % updated_group_id)
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
# Already deleted.
continue
is_changed = (
(testcase.group_id != updated_group_id) or
(testcase.group_bug_information != updated_group_bug_information) or
(testcase.is_leader != updated_is_leader))
if not is_changed:
# If nothing is changed, no more work to do. It's faster this way.
continue
testcase.group_bug_information = updated_group_bug_information
testcase.group_id = updated_group_id
testcase.is_leader = updated_is_leader
testcase.put()
logs.log(
'Updated testcase %d group to %d.' % (testcase_id, updated_group_id))
def group_testcases():
"""Group testcases based on rules like same bug numbers, similar crash
states, etc."""
testcase_map = {}
cached_issue_map = {}
for testcase_id in data_handler.get_open_testcase_id_iterator():
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
# Already deleted.
continue
# Remove duplicates early on to avoid large groups.
if (not testcase.bug_information and not testcase.uploader_email
and _has_testcase_with_same_params(testcase, testcase_map)):
logs.log('Deleting duplicate testcase %d.' % testcase_id)
testcase.key.delete()
continue
# Wait for minimization to finish as this might change crash params such
# as type and may mark it as duplicate / closed.
if not testcase.minimized_keys:
continue
# Store needed testcase attributes into |testcase_map|.
testcase_map[testcase_id] = TestcaseAttributes(testcase_id)
testcase_attributes = testcase_map[testcase_id]
for attribute_name in FORWARDED_ATTRIBUTES:
setattr(testcase_attributes, attribute_name,
getattr(testcase, attribute_name))
# Store original issue mappings in the testcase attributes.
if testcase.bug_information:
issue_id = int(testcase.bug_information)
project_name = testcase.project_name
if (project_name in cached_issue_map
and issue_id in cached_issue_map[project_name]):
testcase_attributes.issue_id = (
cached_issue_map[project_name][issue_id])
else:
issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase(
testcase)
if not issue_tracker:
logs.log_error(
'Unable to access issue tracker for issue %d.' %
issue_id)
testcase_attributes.issue_id = issue_id
continue
# Determine the original issue id traversing the list of duplicates.
try:
issue = issue_tracker.get_original_issue(issue_id)
original_issue_id = int(issue.id)
except:
# If we are unable to access the issue, then we can't determine
# the original issue id. Assume that it is the same as issue id.
logs.log_error(
'Unable to determine original issue for issue %d.' %
issue_id)
testcase_attributes.issue_id = issue_id
continue
if project_name not in cached_issue_map:
cached_issue_map[project_name] = {}
cached_issue_map[project_name][issue_id] = original_issue_id
cached_issue_map[project_name][
original_issue_id] = original_issue_id
testcase_attributes.issue_id = original_issue_id
# No longer needed. Free up some memory.
cached_issue_map.clear()
_group_testcases_with_similar_states(testcase_map)
_group_testcases_with_same_issues(testcase_map)
_shrink_large_groups_if_needed(testcase_map)
group_leader.choose(testcase_map)
# TODO(aarya): Replace with an optimized implementation using dirty flag.
# Update the group mapping in testcase object.
for testcase_id in data_handler.get_open_testcase_id_iterator():
if testcase_id not in testcase_map:
# A new testcase that was just created. Skip for now, will be grouped in
# next iteration of group task.
continue
# If we are part of a group, then calculate the number of testcases in that
# group and lowest issue id of issues associated with testcases in that
# group.
updated_group_id = testcase_map[testcase_id].group_id
updated_is_leader = testcase_map[testcase_id].is_leader
updated_group_id_count = 0
updated_group_bug_information = 0
if updated_group_id:
for other_testcase in six.itervalues(testcase_map):
if other_testcase.group_id != updated_group_id:
continue
updated_group_id_count += 1
# Update group issue id to be lowest issue id in the entire group.
if other_testcase.issue_id is None:
continue
if (not updated_group_bug_information
or updated_group_bug_information >
other_testcase.issue_id):
updated_group_bug_information = other_testcase.issue_id
# If this group id is used by only one testcase, then remove it.
if updated_group_id_count == 1:
data_handler.delete_group(updated_group_id, update_testcases=False)
updated_group_id = 0
updated_group_bug_information = 0
updated_is_leader = True
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
# Already deleted.
continue
is_changed = (
(testcase.group_id != updated_group_id) or
(testcase.group_bug_information != updated_group_bug_information)
or (testcase.is_leader != updated_is_leader))
if not testcase.get_metadata('ran_grouper'):
testcase.set_metadata('ran_grouper',
True,
update_testcase=not is_changed)
if not is_changed:
continue
testcase.group_bug_information = updated_group_bug_information
testcase.group_id = updated_group_id
testcase.is_leader = updated_is_leader
testcase.put()
logs.log('Updated testcase %d group to %d.' %
(testcase_id, updated_group_id))
