def _get_commits(repo, regress_commit_or_range, fix_commit_or_range):
"""Get commits for analysis."""
regress_commits = _get_commit_range(repo, regress_commit_or_range)
if len(regress_commits) > COMMIT_RANGE_LIMIT:
raise osv.ImpactError('Too many commits in regression range.')
fix_commits = _get_commit_range(repo, fix_commit_or_range)
if len(fix_commits) > COMMIT_RANGE_LIMIT:
logging.warning('Too many commits in fix range.')
# Rather than bail out here and potentially leaving a Bug as "unfixed"
# indefinitely, we continue.
return regress_commits, fix_commits
def process_impact_task(source_id, message):
"""Process an impact task."""
logging.info('Processing impact task for %s', source_id)
regress_result = ndb.Key(osv.RegressResult, source_id).get()
if not regress_result:
logging.error('Missing RegressResult for %s', source_id)
return
fix_result = ndb.Key(osv.FixResult, source_id).get()
if not fix_result:
logging.warning('Missing FixResult for %s', source_id)
fix_result = osv.FixResult()
# Check if there is an existing Bug for the same source, but with a different
# allocated ID. This shouldn't happen.
allocated_bug_id = message.attributes['allocated_id']
existing_bug = osv.Bug.query(osv.Bug.source_id == source_id).get()
if existing_bug and existing_bug.key.id() != allocated_bug_id:
logging.error('Bug entry already exists for %s with a different ID %s',
source_id, existing_bug.key.id())
return
if existing_bug and existing_bug.status == osv.BugStatus.INVALID:
logging.warning('Bug %s already marked as invalid.', existing_bug.key.id())
return
if existing_bug:
public = existing_bug.public
else:
raise osv.ImpactError('Task requested without Bug allocated.')
# TODO(ochang): Handle changing repo types? e.g. SVN -> Git.
repo_url = regress_result.repo_url or fix_result.repo_url
if not repo_url:
raise osv.ImpactError('No repo_url set')
issue_id = fix_result.issue_id or regress_result.issue_id
fix_commit = fix_result.commit
with tempfile.TemporaryDirectory() as tmp_dir:
repo = osv.clone_with_retries(repo_url, tmp_dir)
# If not a precise fix commit, try to find the exact one by going through
# commit messages (oss-fuzz only).
if source_id.startswith(SOURCE_PREFIX) and ':' in fix_commit:
start_commit, end_commit = fix_commit.split(':')
commit = find_oss_fuzz_fix_via_commit(repo, start_commit, end_commit,
source_id, issue_id)
if commit:
logging.info('Found exact fix commit %s via commit message (oss-fuzz)',
commit)
fix_commit = commit
# Actually compute the affected commits/tags.
result = osv.get_affected(repo, regress_result.commit, fix_commit)
logging.info('Found affected %s', ', '.join(result.tags))
# If the range resolved to a single commit, simplify it.
if len(result.fix_commits) == 1:
fix_commit = result.fix_commits[0]
elif not result.fix_commits:
# Not fixed.
fix_commit = ''
if len(result.regress_commits) == 1:
regress_commit = result.regress_commits[0]
else:
regress_commit = regress_result.commit
project = fix_result.project or regress_result.project
ecosystem = fix_result.ecosystem or regress_result.ecosystem
summary = fix_result.summary or regress_result.summary
details = fix_result.details or regress_result.details
severity = fix_result.severity or regress_result.severity
reference_urls = fix_result.reference_urls or regress_result.reference_urls
update_affected_commits(allocated_bug_id, result, project, ecosystem, public)
existing_bug.repo_url = repo_url
existing_bug.fixed = fix_commit
existing_bug.regressed = regress_commit
existing_bug.affected = result.tags
existing_bug.affected_fuzzy = osv.normalize_tags(result.tags)
existing_bug.confidence = result.confidence
existing_bug.issue_id = issue_id
existing_bug.project = project
existing_bug.ecosystem = ecosystem
existing_bug.summary = summary
existing_bug.details = details
existing_bug.status = osv.BugStatus.PROCESSED
existing_bug.severity = severity
existing_bug.reference_urls = reference_urls
existing_bug.additional_commit_ranges = []
# Don't display additional ranges for imprecise commits, as they can be
# confusing.
if ':' in existing_bug.fixed or ':' in existing_bug.regressed:
existing_bug.put()
return
def _sort_key(value):
# Allow sorting of None values.
return (value[0] or '', value[1] or '')
for introduced_in, fixed_in in sorted(result.affected_ranges, key=_sort_key):
if (introduced_in == existing_bug.regressed and
(fixed_in or '') == existing_bug.fixed):
# Don't include the main range.
continue
existing_bug.additional_commit_ranges.append(
osv.CommitRange(introduced_in=introduced_in, fixed_in=fixed_in))
existing_bug.put()
def process_impact_task(source_id, message):
"""Process an impact task."""
logging.info('Processing impact task for %s', source_id)
regress_result = ndb.Key(osv.RegressResult, source_id).get()
if not regress_result:
logging.error('Missing RegressResult for %s', source_id)
return
fix_result = ndb.Key(osv.FixResult, source_id).get()
if not fix_result:
logging.warning('Missing FixResult for %s', source_id)
fix_result = osv.FixResult()
# Check if there is an existing Bug for the same source, but with a different
# allocated ID. This shouldn't happen.
allocated_bug_id = message.attributes['allocated_id']
existing_bug = osv.Bug.query(osv.Bug.source_id == source_id).get()
if existing_bug and existing_bug.key.id() != allocated_bug_id:
logging.error('Bug entry already exists for %s with a different ID %s',
source_id, existing_bug.key.id())
return
if existing_bug and existing_bug.status == osv.BugStatus.INVALID:
logging.warning('Bug %s already marked as invalid.',
existing_bug.key.id())
return
if existing_bug:
public = existing_bug.public
else:
raise osv.ImpactError('Task requested without Bug allocated.')
repo_url = regress_result.repo_url or fix_result.repo_url
if not repo_url:
raise osv.ImpactError('No repo_url set')
# Always populate Bug attributes, even if the remainder of the analysis fails.
# This does not mark the Bug as being valid.
set_bug_attributes(existing_bug, regress_result, fix_result)
existing_bug.put()
issue_id = fix_result.issue_id or regress_result.issue_id
fix_commit = fix_result.commit
with tempfile.TemporaryDirectory() as tmp_dir:
repo = osv.clone_with_retries(repo_url, tmp_dir)
# If not a precise fix commit, try to find the exact one by going through
# commit messages (oss-fuzz only).
if source_id.startswith(SOURCE_PREFIX) and ':' in fix_commit:
start_commit, end_commit = fix_commit.split(':')
commit = find_oss_fuzz_fix_via_commit(repo, start_commit,
end_commit, source_id,
issue_id)
if commit:
logging.info(
'Found exact fix commit %s via commit message (oss-fuzz)',
commit)
fix_commit = commit
# Actually compute the affected commits/tags.
repo_analyzer = osv.RepoAnalyzer()
result = repo_analyzer.get_affected(repo, regress_result.commit,
fix_commit)
affected_tags = sorted(list(result.tags))
logging.info('Found affected %s', ', '.join(affected_tags))
# If the range resolved to a single commit, simplify it.
if len(result.fix_commits) == 1:
fix_commit = result.fix_commits[0]
elif not result.fix_commits:
# Not fixed.
fix_commit = ''
if (len(result.regress_commits) == 1
and osv.UNKNOWN_COMMIT not in regress_result.commit):
regress_commit = result.regress_commits[0]
else:
regress_commit = regress_result.commit
project = fix_result.project or regress_result.project
ecosystem = fix_result.ecosystem or regress_result.ecosystem
osv.update_affected_commits(allocated_bug_id, result.commits, project,
ecosystem, public)
affected_tags = sorted(list(result.tags))
existing_bug.fixed = fix_commit
existing_bug.regressed = regress_commit
existing_bug.affected = affected_tags
existing_bug.affected_fuzzy = osv.normalize_tags(affected_tags)
existing_bug.status = osv.BugStatus.PROCESSED
# For the AffectedRange, use the first commit in the regress commit range, and
# the last commit in the fix commit range.
introduced = result.regress_commits[0] if result.regress_commits else ''
fixed = result.fix_commits[-1] if result.fix_commits else ''
existing_bug.affected_ranges = [
osv.AffectedRange(type='GIT',
repo_url=repo_url,
introduced=introduced,
fixed=fixed),
]
# Expose range data in `database_specific`.
database_specific = {}
if ':' in existing_bug.regressed:
database_specific['introduced_range'] = existing_bug.regressed
if ':' in existing_bug.fixed:
database_specific['fixed_range'] = existing_bug.fixed
if database_specific:
existing_bug.database_specific = database_specific
# Don't display additional ranges for imprecise commits, as they can be
# confusing.
if ':' in existing_bug.fixed or ':' in existing_bug.regressed:
existing_bug.put()
return
def _sort_key(value):
# Allow sorting of None values.
return (value[0] or '', value[1] or '')
for introduced_in, fixed_in in sorted(result.affected_ranges,
key=_sort_key):
if not fixed_in:
fixed_in = ''
if (introduced_in == existing_bug.regressed
and fixed_in == existing_bug.fixed):
# Don't repeat the main range.
continue
existing_bug.affected_ranges.append(
osv.AffectedRange(type='GIT',
repo_url=repo_url,
introduced=introduced_in,
fixed=fixed_in))
existing_bug.put()