def _source_update(self, message):
"""Source update."""
source = message.attributes['source']
path = message.attributes['path']
original_sha256 = message.attributes['original_sha256']
source_repo = osv.get_source_repository(source)
repo = osv.clone_with_retries(
source_repo.repo_url,
os.path.join(self._sources_dir, source),
callbacks=self._git_callbacks(source_repo))
yaml_path = os.path.join(osv.repo_path(repo), path)
current_sha256 = osv.sha256(yaml_path)
if current_sha256 != original_sha256:
logging.warning(
'sha256sum of %s no longer matches (expected=%s vs current=%s).',
path, original_sha256, current_sha256)
return
try:
vulnerability = osv.parse_vulnerability(yaml_path)
except Exception as e:
logging.error('Failed to parse vulnerability %s: %s', yaml_path, e)
return
self._do_update(source_repo, repo, vulnerability, yaml_path,
original_sha256)
def _request_analysis_external(self, source_repo, repo, path):
"""Request analysis."""
original_sha256 = osv.sha256(os.path.join(osv.repo_path(repo), path))
self._publisher.publish(_TASKS_TOPIC,
data=b'',
type='update',
source=source_repo.name,
path=path,
original_sha256=original_sha256)
def _request_analysis_external(self, source_repo, repo, path, deleted=False):
"""Request analysis."""
if deleted:
original_sha256 = ''
else:
original_sha256 = osv.sha256(os.path.join(osv.repo_path(repo), path))
self._publisher.publish(
_TASKS_TOPIC,
data=b'',
type='update',
source=source_repo.name,
path=path,
original_sha256=original_sha256,
deleted=str(deleted).lower())
def _request_analysis(self, bug, source_repo, repo):
"""Request analysis."""
if bug.source_of_truth == osv.SourceOfTruth.SOURCE_REPO:
path = osv.source_path(source_repo, bug)
file_path = os.path.join(osv.repo_path(repo), path)
if not os.path.exists(file_path):
logging.info(
'Skipping analysis for %s as the source file no longer exists.',
path)
return
original_sha256 = osv.sha256(file_path)
self._request_analysis_external(source_repo, original_sha256, path)
else:
self._request_internal_analysis(bug)
def _source_update(self, message):
"""Source update."""
source = message.attributes['source']
path = message.attributes['path']
original_sha256 = message.attributes['original_sha256']
deleted = message.attributes['deleted'] == 'true'
source_repo = osv.get_source_repository(source)
repo = osv.ensure_updated_checkout(
source_repo.repo_url,
os.path.join(self._sources_dir, source),
git_callbacks=self._git_callbacks(source_repo))
yaml_path = os.path.join(osv.repo_path(repo), path)
if not os.path.exists(yaml_path):
logging.info('%s was deleted.', yaml_path)
if deleted:
self._handle_deleted(yaml_path)
return
if deleted:
logging.info('Deletion request but source still exists, aborting.')
return
current_sha256 = osv.sha256(yaml_path)
if current_sha256 != original_sha256:
logging.warning(
'sha256sum of %s no longer matches (expected=%s vs current=%s).',
path, original_sha256, current_sha256)
return
try:
vulnerability = osv.parse_vulnerability(yaml_path)
except Exception as e:
logging.error('Failed to parse vulnerability %s: %s', yaml_path, e)
return
self._do_update(source_repo, repo, vulnerability, yaml_path, path,
original_sha256)
def _source_update(self, message):
"""Source update."""
source = message.attributes['source']
path = message.attributes['path']
original_sha256 = message.attributes['original_sha256']
deleted = message.attributes['deleted'] == 'true'
source_repo = osv.get_source_repository(source)
if source_repo.type == osv.SourceRepositoryType.GIT:
repo = osv.ensure_updated_checkout(
source_repo.repo_url,
os.path.join(self._sources_dir, source),
git_callbacks=self._git_callbacks(source_repo),
branch=source_repo.repo_branch)
vuln_path = os.path.join(osv.repo_path(repo), path)
if not os.path.exists(vuln_path):
logging.info('%s was deleted.', vuln_path)
if deleted:
self._handle_deleted(source_repo, path)
return
if deleted:
logging.info(
'Deletion request but source still exists, aborting.')
return
try:
vulnerabilities = osv.parse_vulnerabilities(
vuln_path, key_path=source_repo.key_path)
except Exception as e:
logging.error('Failed to parse vulnerability %s: %s',
vuln_path, e)
return
current_sha256 = osv.sha256(vuln_path)
elif source_repo.type == osv.SourceRepositoryType.BUCKET:
storage_client = storage.Client()
bucket = storage_client.bucket(source_repo.bucket)
try:
blob = bucket.blob(path).download_as_bytes()
except google.cloud.exceptions.NotFound:
logging.error('Bucket path %s does not exist.', path)
return
current_sha256 = osv.sha256_bytes(blob)
try:
vulnerabilities = osv.parse_vulnerabilities_from_data(
blob,
extension=os.path.splitext(path)[1],
key_path=source_repo.key_path)
except Exception as e:
logging.error('Failed to parse vulnerability %s: %s', path, e)
return
repo = None
else:
raise RuntimeError('Unsupported SourceRepository type.')
if current_sha256 != original_sha256:
logging.warning(
'sha256sum of %s no longer matches (expected=%s vs current=%s).',
path, original_sha256, current_sha256)
return
for vulnerability in vulnerabilities:
self._do_update(source_repo, repo, vulnerability, path,
original_sha256)
def _process_updates_git(self, source_repo):
"""Process updates for a git source_repo."""
repo = self.checkout(source_repo)
walker = repo.walk(repo.head.target, pygit2.GIT_SORT_TOPOLOGICAL)
if source_repo.last_synced_hash:
walker.hide(source_repo.last_synced_hash)
# Get list of changed files since last sync.
changed_entries = set()
deleted_entries = set()
for commit in walker:
if commit.author.email == osv.AUTHOR_EMAIL:
continue
if _NO_UPDATE_MARKER in commit.message:
logging.info('Skipping commit %s as no update marker found.', commit.id)
continue
logging.info('Processing commit %s from %s', commit.id,
commit.author.email)
for parent in commit.parents:
diff = repo.diff(parent, commit)
for delta in diff.deltas:
if delta.old_file and _is_vulnerability_file(source_repo,
delta.old_file.path):
if delta.status == pygit2.GIT_DELTA_DELETED:
deleted_entries.add(delta.old_file.path)
continue
changed_entries.add(delta.old_file.path)
if delta.new_file and _is_vulnerability_file(source_repo,
delta.new_file.path):
changed_entries.add(delta.new_file.path)
# Create tasks for changed files.
for changed_entry in changed_entries:
if source_repo.ignore_file(changed_entry):
continue
path = os.path.join(osv.repo_path(repo), changed_entry)
if not os.path.exists(path):
# Path no longer exists. It must have been deleted in another commit.
continue
logging.info('Re-analysis triggered for %s', changed_entry)
original_sha256 = osv.sha256(path)
self._request_analysis_external(source_repo, original_sha256,
changed_entry)
# Mark deleted entries as invalid.
for deleted_entry in deleted_entries:
if source_repo.ignore_file(deleted_entry):
continue
path = os.path.join(osv.repo_path(repo), deleted_entry)
if os.path.exists(path):
# Path still exists. It must have been added back in another commit.
continue
logging.info('Marking %s as invalid', deleted_entry)
original_sha256 = ''
self._request_analysis_external(
source_repo, original_sha256, deleted_entry, deleted=True)
source_repo.last_synced_hash = str(repo.head.target)
source_repo.put()