def _analyze_vulnerability(self, source_repo, repo, vulnerability, path,
original_sha256):
"""Analyze vulnerability and push new changes."""
# Add OSS-Fuzz
bug = osv.Bug.get_by_id(vulnerability.id)
if bug:
fix_result = osv.FixResult.get_by_id(bug.source_id)
if fix_result:
add_fix_information(vulnerability, bug, fix_result)
result = osv.analyze(vulnerability,
analyze_git=not source_repo.ignore_git,
detect_cherrypicks=source_repo.detect_cherrypicks,
versions_from_repo=source_repo.versions_from_repo)
if not result.has_changes:
return result
if not source_repo.editable:
return result
output_path = os.path.join(osv.repo_path(repo), path)
if self._push_new_ranges_and_versions(source_repo, repo, vulnerability,
output_path, original_sha256):
logging.info('Updated range/versions for vulnerability %s.',
vulnerability.id)
return result
logging.warning('Discarding changes for %s due to conflicts.',
vulnerability.id)
raise UpdateConflictError
def _source_update(self, message):
"""Source update."""
source = message.attributes['source']
path = message.attributes['path']
original_sha256 = message.attributes['original_sha256']
source_repo = osv.get_source_repository(source)
repo = osv.clone_with_retries(
source_repo.repo_url,
os.path.join(self._sources_dir, source),
callbacks=self._git_callbacks(source_repo))
yaml_path = os.path.join(osv.repo_path(repo), path)
current_sha256 = osv.sha256(yaml_path)
if current_sha256 != original_sha256:
logging.warning(
'sha256sum of %s no longer matches (expected=%s vs current=%s).',
path, original_sha256, current_sha256)
return
try:
vulnerability = osv.parse_vulnerability(yaml_path)
except Exception as e:
logging.error('Failed to parse vulnerability %s: %s', yaml_path, e)
return
self._do_update(source_repo, repo, vulnerability, yaml_path,
original_sha256)
def _request_analysis_external(self, source_repo, repo, path):
"""Request analysis."""
original_sha256 = osv.sha256(os.path.join(osv.repo_path(repo), path))
self._publisher.publish(_TASKS_TOPIC,
data=b'',
type='update',
source=source_repo.name,
path=path,
original_sha256=original_sha256)
def _source_update(self, message):
"""Source update."""
source = message.attributes['source']
path = message.attributes['path']
source_repo = osv.get_source_repository(source)
repo = osv.clone_with_retries(
source_repo.repo_url,
os.path.join(self._sources_dir, source),
callbacks=self._git_callbacks(source_repo))
yaml_path = os.path.join(osv.repo_path(repo), path)
vulnerability = osv.parse_vulnerability(yaml_path)
self._do_update(source_repo, repo, vulnerability, yaml_path)
def _request_analysis_external(self, source_repo, repo, path, deleted=False):
"""Request analysis."""
if deleted:
original_sha256 = ''
else:
original_sha256 = osv.sha256(os.path.join(osv.repo_path(repo), path))
self._publisher.publish(
_TASKS_TOPIC,
data=b'',
type='update',
source=source_repo.name,
path=path,
original_sha256=original_sha256,
deleted=str(deleted).lower())
def _request_analysis(self, bug, source_repo, repo):
"""Request analysis."""
if bug.source_of_truth == osv.SourceOfTruth.SOURCE_REPO:
path = osv.source_path(source_repo, bug)
file_path = os.path.join(osv.repo_path(repo), path)
if not os.path.exists(file_path):
logging.info(
'Skipping analysis for %s as the source file no longer exists.',
path)
return
original_sha256 = osv.sha256(file_path)
self._request_analysis_external(source_repo, original_sha256, path)
else:
self._request_internal_analysis(bug)
def import_new_oss_fuzz_entries(self, repo, oss_fuzz_source):
"""Import new entries."""
exported = []
vulnerabilities_path = os.path.join(
osv.repo_path(repo), oss_fuzz_source.directory_path or '')
for bug in osv.Bug.query(
osv.Bug.source_of_truth == osv.SourceOfTruth.INTERNAL):
if bug.status != osv.BugStatus.PROCESSED:
continue
if not bug.public:
continue
source_name, _ = osv.parse_source_id(bug.source_id)
if source_name != oss_fuzz_source.name:
continue
vulnerability_path = os.path.join(vulnerabilities_path,
osv.source_path(bug))
os.makedirs(os.path.dirname(vulnerability_path), exist_ok=True)
if os.path.exists(vulnerability_path):
continue
logging.info('Writing %s', bug.key.id())
osv.vulnerability_to_yaml(bug.to_vulnerability(),
vulnerability_path)
# The source of truth is now this yaml file.
bug.source_of_truth = osv.SourceOfTruth.SOURCE_REPO
exported.append(bug)
# Commit Vulnerability changes back to the oss-fuzz source repository.
repo.index.add_all()
diff = repo.index.diff_to_tree(repo.head.peel().tree)
if not diff:
logging.info('No new entries, skipping committing.')
return
logging.info('Commiting and pushing new entries')
if osv.push_source_changes(repo, 'Import from OSS-Fuzz',
self._git_callbacks(oss_fuzz_source)):
ndb.put_multi(exported)
def _source_update(self, message):
"""Source update."""
source = message.attributes['source']
path = message.attributes['path']
original_sha256 = message.attributes['original_sha256']
deleted = message.attributes['deleted'] == 'true'
source_repo = osv.get_source_repository(source)
repo = osv.ensure_updated_checkout(
source_repo.repo_url,
os.path.join(self._sources_dir, source),
git_callbacks=self._git_callbacks(source_repo))
yaml_path = os.path.join(osv.repo_path(repo), path)
if not os.path.exists(yaml_path):
logging.info('%s was deleted.', yaml_path)
if deleted:
self._handle_deleted(yaml_path)
return
if deleted:
logging.info('Deletion request but source still exists, aborting.')
return
current_sha256 = osv.sha256(yaml_path)
if current_sha256 != original_sha256:
logging.warning(
'sha256sum of %s no longer matches (expected=%s vs current=%s).',
path, original_sha256, current_sha256)
return
try:
vulnerability = osv.parse_vulnerability(yaml_path)
except Exception as e:
logging.error('Failed to parse vulnerability %s: %s', yaml_path, e)
return
self._do_update(source_repo, repo, vulnerability, yaml_path, path,
original_sha256)
def import_new_oss_fuzz_entries(self, repo, oss_fuzz_source):
"""Import new entries."""
# TODO(ochang): Make this more efficient by recording whether or not we
# imported already in Datastore.
vulnerabilities_path = os.path.join(
osv.repo_path(repo), oss_fuzz_source.directory_path or '')
for bug in osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED):
if not bug.public:
continue
source_name, source_id = osv.parse_source_id(bug.source_id)
if source_name != oss_fuzz_source.name:
continue
project_dir = os.path.join(vulnerabilities_path, bug.project)
os.makedirs(project_dir, exist_ok=True)
vulnerability_path = os.path.join(
project_dir, source_id + VULNERABILITY_EXTENSION)
if os.path.exists(vulnerability_path):
continue
logging.info('Writing %s', bug.key.id())
osv.vulnerability_to_yaml(bug.to_vulnerability_new(),
vulnerability_path)
# Commit Vulnerability changes back to the oss-fuzz source repository.
repo.index.add_all()
diff = repo.index.diff_to_tree(repo.head.peel().tree)
if not diff:
logging.info('No new entries, skipping committing.')
return
logging.info('Commiting and pushing new entries')
osv.push_source_changes(repo, 'Import from OSS-Fuzz',
self._git_callbacks(oss_fuzz_source))
def _source_update(self, message):
"""Source update."""
source = message.attributes['source']
path = message.attributes['path']
original_sha256 = message.attributes['original_sha256']
deleted = message.attributes['deleted'] == 'true'
source_repo = osv.get_source_repository(source)
if source_repo.type == osv.SourceRepositoryType.GIT:
repo = osv.ensure_updated_checkout(
source_repo.repo_url,
os.path.join(self._sources_dir, source),
git_callbacks=self._git_callbacks(source_repo),
branch=source_repo.repo_branch)
vuln_path = os.path.join(osv.repo_path(repo), path)
if not os.path.exists(vuln_path):
logging.info('%s was deleted.', vuln_path)
if deleted:
self._handle_deleted(source_repo, path)
return
if deleted:
logging.info(
'Deletion request but source still exists, aborting.')
return
try:
vulnerabilities = osv.parse_vulnerabilities(
vuln_path, key_path=source_repo.key_path)
except Exception as e:
logging.error('Failed to parse vulnerability %s: %s',
vuln_path, e)
return
current_sha256 = osv.sha256(vuln_path)
elif source_repo.type == osv.SourceRepositoryType.BUCKET:
storage_client = storage.Client()
bucket = storage_client.bucket(source_repo.bucket)
try:
blob = bucket.blob(path).download_as_bytes()
except google.cloud.exceptions.NotFound:
logging.error('Bucket path %s does not exist.', path)
return
current_sha256 = osv.sha256_bytes(blob)
try:
vulnerabilities = osv.parse_vulnerabilities_from_data(
blob,
extension=os.path.splitext(path)[1],
key_path=source_repo.key_path)
except Exception as e:
logging.error('Failed to parse vulnerability %s: %s', path, e)
return
repo = None
else:
raise RuntimeError('Unsupported SourceRepository type.')
if current_sha256 != original_sha256:
logging.warning(
'sha256sum of %s no longer matches (expected=%s vs current=%s).',
path, original_sha256, current_sha256)
return
for vulnerability in vulnerabilities:
self._do_update(source_repo, repo, vulnerability, path,
original_sha256)
def _process_updates_git(self, source_repo):
"""Process updates for a git source_repo."""
repo = self.checkout(source_repo)
walker = repo.walk(repo.head.target, pygit2.GIT_SORT_TOPOLOGICAL)
if source_repo.last_synced_hash:
walker.hide(source_repo.last_synced_hash)
# Get list of changed files since last sync.
changed_entries = set()
deleted_entries = set()
for commit in walker:
if commit.author.email == osv.AUTHOR_EMAIL:
continue
if _NO_UPDATE_MARKER in commit.message:
logging.info('Skipping commit %s as no update marker found.', commit.id)
continue
logging.info('Processing commit %s from %s', commit.id,
commit.author.email)
for parent in commit.parents:
diff = repo.diff(parent, commit)
for delta in diff.deltas:
if delta.old_file and _is_vulnerability_file(source_repo,
delta.old_file.path):
if delta.status == pygit2.GIT_DELTA_DELETED:
deleted_entries.add(delta.old_file.path)
continue
changed_entries.add(delta.old_file.path)
if delta.new_file and _is_vulnerability_file(source_repo,
delta.new_file.path):
changed_entries.add(delta.new_file.path)
# Create tasks for changed files.
for changed_entry in changed_entries:
if source_repo.ignore_file(changed_entry):
continue
path = os.path.join(osv.repo_path(repo), changed_entry)
if not os.path.exists(path):
# Path no longer exists. It must have been deleted in another commit.
continue
logging.info('Re-analysis triggered for %s', changed_entry)
original_sha256 = osv.sha256(path)
self._request_analysis_external(source_repo, original_sha256,
changed_entry)
# Mark deleted entries as invalid.
for deleted_entry in deleted_entries:
if source_repo.ignore_file(deleted_entry):
continue
path = os.path.join(osv.repo_path(repo), deleted_entry)
if os.path.exists(path):
# Path still exists. It must have been added back in another commit.
continue
logging.info('Marking %s as invalid', deleted_entry)
original_sha256 = ''
self._request_analysis_external(
source_repo, original_sha256, deleted_entry, deleted=True)
source_repo.last_synced_hash = str(repo.head.target)
source_repo.put()