def _export_ecosystem_to_bucket(self, ecosystem, tmp_dir):
"""Export ecosystem vulns to bucket."""
logging.info('Exporting vulnerabilities for ecosystem %s', ecosystem)
storage_client = storage.Client()
bucket = storage_client.get_bucket(self._export_bucket)
zip_path = os.path.join(tmp_dir, 'all.zip')
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zip_file:
for bug in osv.Bug.query(osv.Bug.ecosystem == ecosystem):
if not bug.public or not bug.status == osv.BugStatus.PROCESSED:
continue
file_path = os.path.join(tmp_dir, bug.id() + '.json')
osv.write_vulnerability(
bug.to_vulnerability(include_source=True), file_path)
zip_file.write(file_path, os.path.basename(file_path))
def upload_single(source_path, target_path):
"""Upload a single vulnerability."""
logging.info('Uploading %s', target_path)
try:
blob = bucket.blob(target_path)
blob.upload_from_filename(source_path)
except Exception as e:
logging.error('Failed to export: %s', e)
with concurrent.futures.ThreadPoolExecutor(
max_workers=_EXPORT_WORKERS) as executor:
for filename in os.listdir(tmp_dir):
executor.submit(upload_single, os.path.join(tmp_dir, filename),
f'{ecosystem}/{filename}')
def _push_new_ranges_and_versions(self, source_repo, repo, vulnerability,
output_path, original_sha256):
"""Pushes new ranges and versions."""
osv.write_vulnerability(vulnerability,
output_path,
key_path=source_repo.key_path)
repo.index.add_all()
return osv.push_source_changes(repo,
f'Update {vulnerability.id}',
self._git_callbacks(source_repo),
expected_hashes={
output_path: original_sha256,
})
def import_new_oss_fuzz_entries(self, repo, oss_fuzz_source):
"""Import new entries."""
exported = []
for bug in osv.Bug.query(
osv.Bug.source_of_truth == osv.SourceOfTruth.INTERNAL):
if bug.status != osv.BugStatus.PROCESSED:
continue
if not bug.public:
continue
# We don't index this as INTERNAL generally implies OSS-Fuzz anyway (at
# time of writing).
source_name, _ = osv.parse_source_id(bug.source_id)
if source_name != oss_fuzz_source.name:
continue
vulnerability_path = os.path.join(
osv.repo_path(repo), osv.source_path(oss_fuzz_source, bug))
os.makedirs(os.path.dirname(vulnerability_path), exist_ok=True)
if os.path.exists(vulnerability_path):
continue
logging.info('Writing %s', bug.key.id())
osv.write_vulnerability(
bug.to_vulnerability(v0_7=False, v0_8=True), vulnerability_path)
# The source of truth is now this yaml file.
bug.source_of_truth = osv.SourceOfTruth.SOURCE_REPO
exported.append(bug)
# Commit Vulnerability changes back to the oss-fuzz source repository.
repo.index.add_all()
diff = repo.index.diff_to_tree(repo.head.peel().tree)
if not diff:
logging.info('No new entries, skipping committing.')
return
logging.info('Commiting and pushing new entries')
if osv.push_source_changes(repo, 'Import from OSS-Fuzz',
self._git_callbacks(oss_fuzz_source)):
ndb.put_multi(exported)