def _analyze_vulnerability(self, source_repo, repo, vulnerability, path, original_sha256): """Analyze vulnerability and push new changes.""" # Add OSS-Fuzz bug = osv.Bug.get_by_id(vulnerability.id) if bug: fix_result = osv.FixResult.get_by_id(bug.source_id) if fix_result: add_fix_information(vulnerability, bug, fix_result) result = osv.analyze(vulnerability, analyze_git=not source_repo.ignore_git, detect_cherrypicks=source_repo.detect_cherrypicks, versions_from_repo=source_repo.versions_from_repo) if not result.has_changes: return result if not source_repo.editable: return result output_path = os.path.join(osv.repo_path(repo), path) if self._push_new_ranges_and_versions(source_repo, repo, vulnerability, output_path, original_sha256): logging.info('Updated range/versions for vulnerability %s.', vulnerability.id) return result logging.warning('Discarding changes for %s due to conflicts.', vulnerability.id) raise UpdateConflictError
def _source_update(self, message): """Source update.""" source = message.attributes['source'] path = message.attributes['path'] original_sha256 = message.attributes['original_sha256'] source_repo = osv.get_source_repository(source) repo = osv.clone_with_retries( source_repo.repo_url, os.path.join(self._sources_dir, source), callbacks=self._git_callbacks(source_repo)) yaml_path = os.path.join(osv.repo_path(repo), path) current_sha256 = osv.sha256(yaml_path) if current_sha256 != original_sha256: logging.warning( 'sha256sum of %s no longer matches (expected=%s vs current=%s).', path, original_sha256, current_sha256) return try: vulnerability = osv.parse_vulnerability(yaml_path) except Exception as e: logging.error('Failed to parse vulnerability %s: %s', yaml_path, e) return self._do_update(source_repo, repo, vulnerability, yaml_path, original_sha256)
def _request_analysis_external(self, source_repo, repo, path): """Request analysis.""" original_sha256 = osv.sha256(os.path.join(osv.repo_path(repo), path)) self._publisher.publish(_TASKS_TOPIC, data=b'', type='update', source=source_repo.name, path=path, original_sha256=original_sha256)
def _source_update(self, message): """Source update.""" source = message.attributes['source'] path = message.attributes['path'] source_repo = osv.get_source_repository(source) repo = osv.clone_with_retries( source_repo.repo_url, os.path.join(self._sources_dir, source), callbacks=self._git_callbacks(source_repo)) yaml_path = os.path.join(osv.repo_path(repo), path) vulnerability = osv.parse_vulnerability(yaml_path) self._do_update(source_repo, repo, vulnerability, yaml_path)
def _request_analysis_external(self, source_repo, repo, path, deleted=False): """Request analysis.""" if deleted: original_sha256 = '' else: original_sha256 = osv.sha256(os.path.join(osv.repo_path(repo), path)) self._publisher.publish( _TASKS_TOPIC, data=b'', type='update', source=source_repo.name, path=path, original_sha256=original_sha256, deleted=str(deleted).lower())
def _request_analysis(self, bug, source_repo, repo): """Request analysis.""" if bug.source_of_truth == osv.SourceOfTruth.SOURCE_REPO: path = osv.source_path(source_repo, bug) file_path = os.path.join(osv.repo_path(repo), path) if not os.path.exists(file_path): logging.info( 'Skipping analysis for %s as the source file no longer exists.', path) return original_sha256 = osv.sha256(file_path) self._request_analysis_external(source_repo, original_sha256, path) else: self._request_internal_analysis(bug)
def import_new_oss_fuzz_entries(self, repo, oss_fuzz_source): """Import new entries.""" exported = [] vulnerabilities_path = os.path.join( osv.repo_path(repo), oss_fuzz_source.directory_path or '') for bug in osv.Bug.query( osv.Bug.source_of_truth == osv.SourceOfTruth.INTERNAL): if bug.status != osv.BugStatus.PROCESSED: continue if not bug.public: continue source_name, _ = osv.parse_source_id(bug.source_id) if source_name != oss_fuzz_source.name: continue vulnerability_path = os.path.join(vulnerabilities_path, osv.source_path(bug)) os.makedirs(os.path.dirname(vulnerability_path), exist_ok=True) if os.path.exists(vulnerability_path): continue logging.info('Writing %s', bug.key.id()) osv.vulnerability_to_yaml(bug.to_vulnerability(), vulnerability_path) # The source of truth is now this yaml file. bug.source_of_truth = osv.SourceOfTruth.SOURCE_REPO exported.append(bug) # Commit Vulnerability changes back to the oss-fuzz source repository. repo.index.add_all() diff = repo.index.diff_to_tree(repo.head.peel().tree) if not diff: logging.info('No new entries, skipping committing.') return logging.info('Commiting and pushing new entries') if osv.push_source_changes(repo, 'Import from OSS-Fuzz', self._git_callbacks(oss_fuzz_source)): ndb.put_multi(exported)
def _source_update(self, message): """Source update.""" source = message.attributes['source'] path = message.attributes['path'] original_sha256 = message.attributes['original_sha256'] deleted = message.attributes['deleted'] == 'true' source_repo = osv.get_source_repository(source) repo = osv.ensure_updated_checkout( source_repo.repo_url, os.path.join(self._sources_dir, source), git_callbacks=self._git_callbacks(source_repo)) yaml_path = os.path.join(osv.repo_path(repo), path) if not os.path.exists(yaml_path): logging.info('%s was deleted.', yaml_path) if deleted: self._handle_deleted(yaml_path) return if deleted: logging.info('Deletion request but source still exists, aborting.') return current_sha256 = osv.sha256(yaml_path) if current_sha256 != original_sha256: logging.warning( 'sha256sum of %s no longer matches (expected=%s vs current=%s).', path, original_sha256, current_sha256) return try: vulnerability = osv.parse_vulnerability(yaml_path) except Exception as e: logging.error('Failed to parse vulnerability %s: %s', yaml_path, e) return self._do_update(source_repo, repo, vulnerability, yaml_path, path, original_sha256)
def import_new_oss_fuzz_entries(self, repo, oss_fuzz_source): """Import new entries.""" # TODO(ochang): Make this more efficient by recording whether or not we # imported already in Datastore. vulnerabilities_path = os.path.join( osv.repo_path(repo), oss_fuzz_source.directory_path or '') for bug in osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED): if not bug.public: continue source_name, source_id = osv.parse_source_id(bug.source_id) if source_name != oss_fuzz_source.name: continue project_dir = os.path.join(vulnerabilities_path, bug.project) os.makedirs(project_dir, exist_ok=True) vulnerability_path = os.path.join( project_dir, source_id + VULNERABILITY_EXTENSION) if os.path.exists(vulnerability_path): continue logging.info('Writing %s', bug.key.id()) osv.vulnerability_to_yaml(bug.to_vulnerability_new(), vulnerability_path) # Commit Vulnerability changes back to the oss-fuzz source repository. repo.index.add_all() diff = repo.index.diff_to_tree(repo.head.peel().tree) if not diff: logging.info('No new entries, skipping committing.') return logging.info('Commiting and pushing new entries') osv.push_source_changes(repo, 'Import from OSS-Fuzz', self._git_callbacks(oss_fuzz_source))
def _source_update(self, message): """Source update.""" source = message.attributes['source'] path = message.attributes['path'] original_sha256 = message.attributes['original_sha256'] deleted = message.attributes['deleted'] == 'true' source_repo = osv.get_source_repository(source) if source_repo.type == osv.SourceRepositoryType.GIT: repo = osv.ensure_updated_checkout( source_repo.repo_url, os.path.join(self._sources_dir, source), git_callbacks=self._git_callbacks(source_repo), branch=source_repo.repo_branch) vuln_path = os.path.join(osv.repo_path(repo), path) if not os.path.exists(vuln_path): logging.info('%s was deleted.', vuln_path) if deleted: self._handle_deleted(source_repo, path) return if deleted: logging.info( 'Deletion request but source still exists, aborting.') return try: vulnerabilities = osv.parse_vulnerabilities( vuln_path, key_path=source_repo.key_path) except Exception as e: logging.error('Failed to parse vulnerability %s: %s', vuln_path, e) return current_sha256 = osv.sha256(vuln_path) elif source_repo.type == osv.SourceRepositoryType.BUCKET: storage_client = storage.Client() bucket = storage_client.bucket(source_repo.bucket) try: blob = bucket.blob(path).download_as_bytes() except google.cloud.exceptions.NotFound: logging.error('Bucket path %s does not exist.', path) return current_sha256 = osv.sha256_bytes(blob) try: vulnerabilities = osv.parse_vulnerabilities_from_data( blob, extension=os.path.splitext(path)[1], key_path=source_repo.key_path) except Exception as e: logging.error('Failed to parse vulnerability %s: %s', path, e) return repo = None else: raise RuntimeError('Unsupported SourceRepository type.') if current_sha256 != original_sha256: logging.warning( 'sha256sum of %s no longer matches (expected=%s vs current=%s).', path, original_sha256, current_sha256) return for vulnerability in vulnerabilities: self._do_update(source_repo, repo, vulnerability, path, original_sha256)
def _process_updates_git(self, source_repo): """Process updates for a git source_repo.""" repo = self.checkout(source_repo) walker = repo.walk(repo.head.target, pygit2.GIT_SORT_TOPOLOGICAL) if source_repo.last_synced_hash: walker.hide(source_repo.last_synced_hash) # Get list of changed files since last sync. changed_entries = set() deleted_entries = set() for commit in walker: if commit.author.email == osv.AUTHOR_EMAIL: continue if _NO_UPDATE_MARKER in commit.message: logging.info('Skipping commit %s as no update marker found.', commit.id) continue logging.info('Processing commit %s from %s', commit.id, commit.author.email) for parent in commit.parents: diff = repo.diff(parent, commit) for delta in diff.deltas: if delta.old_file and _is_vulnerability_file(source_repo, delta.old_file.path): if delta.status == pygit2.GIT_DELTA_DELETED: deleted_entries.add(delta.old_file.path) continue changed_entries.add(delta.old_file.path) if delta.new_file and _is_vulnerability_file(source_repo, delta.new_file.path): changed_entries.add(delta.new_file.path) # Create tasks for changed files. for changed_entry in changed_entries: if source_repo.ignore_file(changed_entry): continue path = os.path.join(osv.repo_path(repo), changed_entry) if not os.path.exists(path): # Path no longer exists. It must have been deleted in another commit. continue logging.info('Re-analysis triggered for %s', changed_entry) original_sha256 = osv.sha256(path) self._request_analysis_external(source_repo, original_sha256, changed_entry) # Mark deleted entries as invalid. for deleted_entry in deleted_entries: if source_repo.ignore_file(deleted_entry): continue path = os.path.join(osv.repo_path(repo), deleted_entry) if os.path.exists(path): # Path still exists. It must have been added back in another commit. continue logging.info('Marking %s as invalid', deleted_entry) original_sha256 = '' self._request_analysis_external( source_repo, original_sha256, deleted_entry, deleted=True) source_repo.last_synced_hash = str(repo.head.target) source_repo.put()