def export_oss_fuzz_to_bucket(self):
"""Export OSS-Fuzz vulns to bucket."""
storage_client = storage.Client()
bucket = storage_client.get_bucket(self._oss_fuzz_export_bucket)
def export_oss_fuzz(vulnerability, testcase_id, issue_id):
"""Export a single vulnerability."""
try:
blob = bucket.blob(f'testcase/{testcase_id}.json')
data = json.dumps(json_format.MessageToDict(vulnerability))
blob.upload_from_string(data)
if not issue_id:
return
blob = bucket.blob(f'issue/{issue_id}.json')
blob.upload_from_string(data)
except Exception as e:
logging.error('Failed to export: %s', e)
with concurrent.futures.ThreadPoolExecutor(
max_workers=_EXPORT_WORKERS) as executor:
for bug in osv.Bug.query(osv.Bug.ecosystem == 'OSS-Fuzz'):
if not bug.public:
continue
_, source_id = osv.parse_source_id(bug.source_id)
executor.submit(export_oss_fuzz, bug.to_vulnerability(), source_id,
bug.issue_id)
def process_oss_fuzz(self, oss_fuzz_source):
"""Process OSS-Fuzz source data."""
# Export OSS-Fuzz Vulnerability data into source repository.
# OSS-Fuzz data is first imported via a special Pub/Sub pipeline into OSV.
# This data needs to be dumped into a publicly accessible/editable place for
# manual/human editing if required.
#
# This then becomes the source of truth where any edits are imported back
# into OSV.
with tempfile.TemporaryDirectory() as tmp_dir:
callbacks = GitRemoteCallback(oss_fuzz_source.repo_username,
self._ssh_key_public_path,
self._ssh_key_private_path)
repo = osv.clone_with_retries(oss_fuzz_source.repo_url,
tmp_dir,
callbacks=callbacks)
if not repo:
raise RuntimeError('Failed to clone source repo')
vulnerabilities_path = os.path.join(
tmp_dir, oss_fuzz_source.directory_path or '')
# TODO(ochang): Make this more efficient by recording whether or not we
# imported already in Datastore.
for bug in osv.Bug.query(
osv.Bug.status == osv.BugStatus.PROCESSED):
source_name, source_id = osv.parse_source_id(bug.source_id)
if source_name != oss_fuzz_source.name:
continue
project_dir = os.path.join(vulnerabilities_path, bug.project)
os.makedirs(project_dir, exist_ok=True)
vulnerability_path = os.path.join(project_dir,
source_id + '.yaml')
if os.path.exists(vulnerability_path):
continue
logging.info('Writing %s', bug.key.id())
with open(vulnerability_path, 'w') as handle:
data = json_format.MessageToDict(bug.to_vulnerability())
yaml.dump(data,
handle,
sort_keys=False,
Dumper=self.YamlDumper)
# Commit Vulnerability changes back to the oss-fuzz source repository.
logging.info('Commiting and pushing changes')
repo.index.add_all()
repo.index.write()
tree = repo.index.write_tree()
author = _git_author()
repo.create_commit(repo.head.name, author, author,
'Import from OSS-Fuzz', tree,
[repo.head.peel().oid])
# TODO(ochang): Rebase and retry if necessary.
repo.remotes['origin'].push([repo.head.name], callbacks=callbacks)
def import_new_oss_fuzz_entries(self, repo, oss_fuzz_source):
"""Import new entries."""
exported = []
vulnerabilities_path = os.path.join(
osv.repo_path(repo), oss_fuzz_source.directory_path or '')
for bug in osv.Bug.query(
osv.Bug.source_of_truth == osv.SourceOfTruth.INTERNAL):
if bug.status != osv.BugStatus.PROCESSED:
continue
if not bug.public:
continue
source_name, _ = osv.parse_source_id(bug.source_id)
if source_name != oss_fuzz_source.name:
continue
vulnerability_path = os.path.join(vulnerabilities_path,
osv.source_path(bug))
os.makedirs(os.path.dirname(vulnerability_path), exist_ok=True)
if os.path.exists(vulnerability_path):
continue
logging.info('Writing %s', bug.key.id())
osv.vulnerability_to_yaml(bug.to_vulnerability(),
vulnerability_path)
# The source of truth is now this yaml file.
bug.source_of_truth = osv.SourceOfTruth.SOURCE_REPO
exported.append(bug)
# Commit Vulnerability changes back to the oss-fuzz source repository.
repo.index.add_all()
diff = repo.index.diff_to_tree(repo.head.peel().tree)
if not diff:
logging.info('No new entries, skipping committing.')
return
logging.info('Commiting and pushing new entries')
if osv.push_source_changes(repo, 'Import from OSS-Fuzz',
self._git_callbacks(oss_fuzz_source)):
ndb.put_multi(exported)
def import_new_oss_fuzz_entries(self, repo, oss_fuzz_source):
"""Import new entries."""
# TODO(ochang): Make this more efficient by recording whether or not we
# imported already in Datastore.
vulnerabilities_path = os.path.join(
osv.repo_path(repo), oss_fuzz_source.directory_path or '')
for bug in osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED):
if not bug.public:
continue
source_name, source_id = osv.parse_source_id(bug.source_id)
if source_name != oss_fuzz_source.name:
continue
project_dir = os.path.join(vulnerabilities_path, bug.project)
os.makedirs(project_dir, exist_ok=True)
vulnerability_path = os.path.join(
project_dir, source_id + VULNERABILITY_EXTENSION)
if os.path.exists(vulnerability_path):
continue
logging.info('Writing %s', bug.key.id())
osv.vulnerability_to_yaml(bug.to_vulnerability_new(),
vulnerability_path)
# Commit Vulnerability changes back to the oss-fuzz source repository.
repo.index.add_all()
diff = repo.index.diff_to_tree(repo.head.peel().tree)
if not diff:
logging.info('No new entries, skipping committing.')
return
logging.info('Commiting and pushing new entries')
osv.push_source_changes(repo, 'Import from OSS-Fuzz',
self._git_callbacks(oss_fuzz_source))