def export_oss_fuzz_to_bucket(self): """Export OSS-Fuzz vulns to bucket.""" storage_client = storage.Client() bucket = storage_client.get_bucket(self._oss_fuzz_export_bucket) def export_oss_fuzz(vulnerability, testcase_id, issue_id): """Export a single vulnerability.""" try: blob = bucket.blob(f'testcase/{testcase_id}.json') data = json.dumps(json_format.MessageToDict(vulnerability)) blob.upload_from_string(data) if not issue_id: return blob = bucket.blob(f'issue/{issue_id}.json') blob.upload_from_string(data) except Exception as e: logging.error('Failed to export: %s', e) with concurrent.futures.ThreadPoolExecutor( max_workers=_EXPORT_WORKERS) as executor: for bug in osv.Bug.query(osv.Bug.ecosystem == 'OSS-Fuzz'): if not bug.public: continue _, source_id = osv.parse_source_id(bug.source_id) executor.submit(export_oss_fuzz, bug.to_vulnerability(), source_id, bug.issue_id)
def process_oss_fuzz(self, oss_fuzz_source): """Process OSS-Fuzz source data.""" # Export OSS-Fuzz Vulnerability data into source repository. # OSS-Fuzz data is first imported via a special Pub/Sub pipeline into OSV. # This data needs to be dumped into a publicly accessible/editable place for # manual/human editing if required. # # This then becomes the source of truth where any edits are imported back # into OSV. with tempfile.TemporaryDirectory() as tmp_dir: callbacks = GitRemoteCallback(oss_fuzz_source.repo_username, self._ssh_key_public_path, self._ssh_key_private_path) repo = osv.clone_with_retries(oss_fuzz_source.repo_url, tmp_dir, callbacks=callbacks) if not repo: raise RuntimeError('Failed to clone source repo') vulnerabilities_path = os.path.join( tmp_dir, oss_fuzz_source.directory_path or '') # TODO(ochang): Make this more efficient by recording whether or not we # imported already in Datastore. for bug in osv.Bug.query( osv.Bug.status == osv.BugStatus.PROCESSED): source_name, source_id = osv.parse_source_id(bug.source_id) if source_name != oss_fuzz_source.name: continue project_dir = os.path.join(vulnerabilities_path, bug.project) os.makedirs(project_dir, exist_ok=True) vulnerability_path = os.path.join(project_dir, source_id + '.yaml') if os.path.exists(vulnerability_path): continue logging.info('Writing %s', bug.key.id()) with open(vulnerability_path, 'w') as handle: data = json_format.MessageToDict(bug.to_vulnerability()) yaml.dump(data, handle, sort_keys=False, Dumper=self.YamlDumper) # Commit Vulnerability changes back to the oss-fuzz source repository. logging.info('Commiting and pushing changes') repo.index.add_all() repo.index.write() tree = repo.index.write_tree() author = _git_author() repo.create_commit(repo.head.name, author, author, 'Import from OSS-Fuzz', tree, [repo.head.peel().oid]) # TODO(ochang): Rebase and retry if necessary. repo.remotes['origin'].push([repo.head.name], callbacks=callbacks)
def import_new_oss_fuzz_entries(self, repo, oss_fuzz_source): """Import new entries.""" exported = [] vulnerabilities_path = os.path.join( osv.repo_path(repo), oss_fuzz_source.directory_path or '') for bug in osv.Bug.query( osv.Bug.source_of_truth == osv.SourceOfTruth.INTERNAL): if bug.status != osv.BugStatus.PROCESSED: continue if not bug.public: continue source_name, _ = osv.parse_source_id(bug.source_id) if source_name != oss_fuzz_source.name: continue vulnerability_path = os.path.join(vulnerabilities_path, osv.source_path(bug)) os.makedirs(os.path.dirname(vulnerability_path), exist_ok=True) if os.path.exists(vulnerability_path): continue logging.info('Writing %s', bug.key.id()) osv.vulnerability_to_yaml(bug.to_vulnerability(), vulnerability_path) # The source of truth is now this yaml file. bug.source_of_truth = osv.SourceOfTruth.SOURCE_REPO exported.append(bug) # Commit Vulnerability changes back to the oss-fuzz source repository. repo.index.add_all() diff = repo.index.diff_to_tree(repo.head.peel().tree) if not diff: logging.info('No new entries, skipping committing.') return logging.info('Commiting and pushing new entries') if osv.push_source_changes(repo, 'Import from OSS-Fuzz', self._git_callbacks(oss_fuzz_source)): ndb.put_multi(exported)
def import_new_oss_fuzz_entries(self, repo, oss_fuzz_source): """Import new entries.""" # TODO(ochang): Make this more efficient by recording whether or not we # imported already in Datastore. vulnerabilities_path = os.path.join( osv.repo_path(repo), oss_fuzz_source.directory_path or '') for bug in osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED): if not bug.public: continue source_name, source_id = osv.parse_source_id(bug.source_id) if source_name != oss_fuzz_source.name: continue project_dir = os.path.join(vulnerabilities_path, bug.project) os.makedirs(project_dir, exist_ok=True) vulnerability_path = os.path.join( project_dir, source_id + VULNERABILITY_EXTENSION) if os.path.exists(vulnerability_path): continue logging.info('Writing %s', bug.key.id()) osv.vulnerability_to_yaml(bug.to_vulnerability_new(), vulnerability_path) # Commit Vulnerability changes back to the oss-fuzz source repository. repo.index.add_all() diff = repo.index.diff_to_tree(repo.head.peel().tree) if not diff: logging.info('No new entries, skipping committing.') return logging.info('Commiting and pushing new entries') osv.push_source_changes(repo, 'Import from OSS-Fuzz', self._git_callbacks(oss_fuzz_source))