def setUp(self): tests.reset_emulator() tests.mock_clone(self, return_value=pygit2.Repository('osv-test')) osv.Bug( id='2020-1', project='project', affected=['v0.1.1'], public=True).put() osv.Bug( id='2020-2', project='project', affected=['v0.2'], public=False).put()
def setUp(self): tests.reset_emulator() self.clone_repository_patcher = mock.patch('pygit2.clone_repository') mock_clone = self.clone_repository_patcher.start() mock_clone.return_value = pygit2.Repository('osv-test') osv.Bug( id='2020-1', project='project', affected=['v0.1.1'], public=True).put() osv.Bug( id='2020-2', project='project', affected=['v0.2'], public=False).put()
def setUp(self): self.maxDiff = None tests.reset_emulator() self.original_clone = pygit2.clone_repository tests.mock_clone(self, func=self.mock_clone) tests.mock_datetime(self) repo = tests.mock_repository(self) self.remote_source_repo_path = repo.path # Initialise fake source_repo. self.tmp_dir = tempfile.TemporaryDirectory() self.mock_repo = tests.mock_repository(self) self.mock_repo.add_file( 'BLAH-123.yaml', self._load_test_data(os.path.join(TEST_DATA_DIR, 'BLAH-123.yaml'))) self.mock_repo.add_file( 'BLAH-124.yaml', self._load_test_data(os.path.join(TEST_DATA_DIR, 'BLAH-124.yaml'))) self.mock_repo.add_file( 'BLAH-125.yaml', self._load_test_data(os.path.join(TEST_DATA_DIR, 'BLAH-125.yaml'))) self.mock_repo.commit('User', 'user@email') osv.SourceRepository(id='source', name='source', repo_url='file://' + self.remote_source_repo_path, repo_username='').put() osv.Bug(id='BLAH-123', project='blah.com/package', ecosystem='golang', source_id='source:BLAH-123.yaml', source_of_truth=osv.SourceOfTruth.SOURCE_REPO).put() osv.Bug(id='BLAH-124', regressed='eefe8ec3f1f90d0e684890e810f3f21e8500a4cd', project='blah.com/package', ecosystem='golang', source_id='source:BLAH-124.yaml', source_of_truth=osv.SourceOfTruth.SOURCE_REPO).put() osv.Bug(id='BLAH-125', regressed='eefe8ec3f1f90d0e684890e810f3f21e8500a4cd', fixed='8d8242f545e9cec3e6d0d2e3f5bde8be1c659735', project='blah.com/package', ecosystem='golang', source_id='source:BLAH-125.yaml', source_of_truth=osv.SourceOfTruth.SOURCE_REPO).put()
def setUp(self): tests.reset_emulator() self.clone_repository_patcher = mock.patch('pygit2.clone_repository') self.maxDiff = None mock_clone = self.clone_repository_patcher.start() mock_clone.return_value = pygit2.Repository('osv-test') patcher = mock.patch('osv.types.utcnow') mock_utcnow = patcher.start() mock_utcnow.return_value = datetime.datetime(2021, 1, 1) self.addCleanup(patcher.stop) allocated_bug = osv.Bug(id='2020-1337', timestamp=datetime.datetime(2020, 1, 1), source_id='oss-fuzz:123', status=osv.BugStatus.UNPROCESSED, public=False) allocated_bug.put() should_be_deleted = osv.AffectedCommit(id='2020-1337-abcd', bug_id='2020-1337', commit='abcd', confidence=100, project='project', ecosystem='ecosystem', public=False) should_be_deleted.put()
def _do_update(self, source_repo, repo, vulnerability, relative_path, original_sha256): """Process updates on a vulnerability.""" logging.info('Processing update for vulnerability %s', vulnerability.id) try: result = self._analyze_vulnerability(source_repo, repo, vulnerability, relative_path, original_sha256) except UpdateConflictError: # Discard changes due to conflict. return # Update datastore with new information. bug = osv.Bug.get_by_id(vulnerability.id) if not bug: if source_repo.name == 'oss-fuzz': logging.warning('%s not found for OSS-Fuzz source.', vulnerability.id) return bug = osv.Bug(db_id=vulnerability.id, source_id=f'{source_repo.name}:{relative_path}', timestamp=osv.utcnow(), status=osv.BugStatus.PROCESSED, source_of_truth=osv.SourceOfTruth.SOURCE_REPO) bug.update_from_vulnerability(vulnerability) bug.public = True bug.put() osv.update_affected_commits(bug.key.id(), result.commits, bug.project, bug.ecosystem, bug.public)
def setUp(self): self.maxDiff = None # pylint: disable=invalid-name self.tmp_dir = tempfile.mkdtemp() tests.mock_datetime(self) self.mock_repo = tests.mock_repository(self) storage_patcher = mock.patch('google.cloud.storage.Client') self.addCleanup(storage_patcher.stop) self.mock_storage_client = storage_patcher.start() self.remote_source_repo_path = self.mock_repo.path self.source_repo = osv.SourceRepository(id='oss-fuzz', name='oss-fuzz', repo_url='file://' + self.remote_source_repo_path, repo_username='') self.source_repo.put() osv.Bug( id='2017-134', affected=['FILE5_29', 'FILE5_30'], affected_fuzzy=['5-29', '5-30'], details=( 'OSS-Fuzz report: ' 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1064\n\n' 'Crash type: Heap-buffer-overflow READ 1\n' 'Crash state:\ncdf_file_property_info\ncdf_file_summary_info\n' 'cdf_check_summary_info\n'), ecosystem='OSS-Fuzz', fixed='19ccebafb7663c422c714e0c67fa4775abf91c43', has_affected=True, issue_id='1064', project='file', public=True, reference_urls=[ 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1064' ], regressed='17ee4cf670c363de8d2ea4a4897d7a699837873f', repo_url='https://github.com/file/file.git', search_indices=['file', '2017-134', '2017', '134'], severity='MEDIUM', sort_key='2017-0000134', source_id='oss-fuzz:5417710252982272', source_of_truth=osv.SourceOfTruth.INTERNAL, status=1, summary='Heap-buffer-overflow in cdf_file_property_info', timestamp=datetime.datetime(2021, 1, 15, 0, 0, 24, 559102)).put()
def setUp(self): self.maxDiff = None # pylint: disable=invalid-name self.tmp_dir = tempfile.mkdtemp() self.remote_source_repo_path = os.path.join(self.tmp_dir, 'source_repo') # Initialise fake source_repo. repo = pygit2.init_repository(self.remote_source_repo_path, True) tree = repo.TreeBuilder().write() author = pygit2.Signature('OSV', '*****@*****.**') repo.create_commit('HEAD', author, author, 'Initial commit', tree, []) osv.SourceRepository( id='oss-fuzz', name='oss-fuzz', repo_url='file://' + self.remote_source_repo_path, repo_username='').put() osv.Bug( id='2017-134', affected=['FILE5_29', 'FILE5_30'], affected_fuzzy=['5-29', '5-30'], confidence=100, details=( 'OSS-Fuzz report: ' 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1064\n\n' 'Crash type: Heap-buffer-overflow READ 1\n' 'Crash state:\ncdf_file_property_info\ncdf_file_summary_info\n' 'cdf_check_summary_info\n'), ecosystem='', fixed='19ccebafb7663c422c714e0c67fa4775abf91c43', has_affected=True, issue_id='1064', project='file', public=True, reference_urls=[ 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1064' ], regressed='17ee4cf670c363de8d2ea4a4897d7a699837873f', repo_url='https://github.com/file/file.git', search_indices=['file', '2017-134', '2017', '134'], severity='MEDIUM', sort_key='2017-0000134', source_id='oss-fuzz:5417710252982272', status=1, summary='Heap-buffer-overflow in cdf_file_property_info', timestamp=datetime.datetime(2021, 1, 15, 0, 0, 24, 559102)).put()
def test_mark_bug_invalid(self): """Test mark_bug_invalid.""" osv.Bug(id='2021-1', source_id='oss-fuzz:1337').put() osv.AffectedCommit(bug_id='2021-1').put() osv.AffectedCommit(bug_id='2021-1').put() message = mock.Mock() message.attributes = { 'type': 'invalid', 'testcase_id': '1337', 'source_id': '', } worker.mark_bug_invalid(message) bug = ndb.Key(osv.Bug, '2021-1').get() self.assertEqual(osv.BugStatus.INVALID, bug.status) commits = list(osv.AffectedCommit.query()) self.assertEqual(0, len(commits))
def setUp(self): self.maxDiff = None tests.reset_emulator() # TODO(ochang): Refactor out into common test utilities. self.original_clone = pygit2.clone_repository self.clone_repository_patcher = mock.patch('pygit2.clone_repository') mock_clone = self.clone_repository_patcher.start() mock_clone.side_effect = self.mock_clone patcher = mock.patch('osv.types.utcnow') mock_utcnow = patcher.start() mock_utcnow.return_value = datetime.datetime(2021, 1, 1) self.addCleanup(patcher.stop) # Initialise fake source_repo. self.tmp_dir = tempfile.TemporaryDirectory() self.remote_source_repo_path = os.path.join(self.tmp_dir.name, 'source_repo') repo = pygit2.init_repository(self.remote_source_repo_path, True) tree = repo.TreeBuilder().write() author = pygit2.Signature('OSV', '*****@*****.**') repo.create_commit('HEAD', author, author, 'Initial commit', tree, []) # Add a source. oid = repo.write( pygit2.GIT_OBJ_BLOB, self._load_test_data(os.path.join(TEST_DATA_DIR, 'BLAH-123.yaml'))) repo.index.add( pygit2.IndexEntry('BLAH-123.yaml', oid, pygit2.GIT_FILEMODE_BLOB)) repo.index.write() tree = repo.index.write_tree() repo.create_commit('HEAD', author, author, 'Changes', tree, [repo.head.peel().oid]) osv.SourceRepository(id='source', name='source', repo_url='file://' + self.remote_source_repo_path, repo_username='').put() osv.Bug(id='BLAH-123', project='blah.com/package', ecosystem='golang').put()
def setUp(self): tests.reset_emulator() self.maxDiff = None tests.mock_clone(self, return_value=pygit2.Repository('osv-test')) tests.mock_datetime(self) allocated_bug = osv.Bug(id='2020-1337', timestamp=datetime.datetime(2020, 1, 1), source_id='oss-fuzz:123', status=osv.BugStatus.UNPROCESSED, public=False) allocated_bug.put() should_be_deleted = osv.AffectedCommit(id='2020-1337-abcd', bug_id='2020-1337', commit='abcd', project='project', ecosystem='ecosystem', public=False) should_be_deleted.put()
def test_scheduled_updates_already_done(self, mock_publish): """Scheduled updates already done.""" source_repo = osv.SourceRepository.get_by_id('oss-fuzz') source_repo.last_update_date = importer.utcnow().date() source_repo.put() self.mock_repo.add_file('proj/OSV-2021-1337.yaml', '') self.mock_repo.commit('OSV', '*****@*****.**') osv.Bug(id='2021-1337', project='proj', fixed='', status=1, source_id='oss-fuzz:123', source_of_truth=osv.SourceOfTruth.SOURCE_REPO, timestamp=datetime.datetime(2020, 1, 1, 0, 0, 0, 0)).put() imp = importer.Importer('fake_public_key', 'fake_private_key', self.tmp_dir, 'bucket') imp.run() self.assertEqual(0, mock_publish.call_count)
def setUp(self): self.maxDiff = None # pylint: disable=invalid-name self.tmp_dir = tempfile.mkdtemp() self.remote_source_repo_path = os.path.join(self.tmp_dir, 'source_repo') patcher = mock.patch('osv.types.utcnow') mock_utcnow = patcher.start() mock_utcnow.return_value = datetime.datetime(2021, 1, 1) self.addCleanup(patcher.stop) # Initialise fake source_repo. repo = pygit2.init_repository(self.remote_source_repo_path, True) tree = repo.TreeBuilder().write() author = pygit2.Signature('OSV', '*****@*****.**') repo.create_commit('HEAD', author, author, 'Initial commit', tree, []) # Add a fake user change. with open(os.path.join(self.remote_source_repo_path, '2021-111.yaml'), 'w') as f: f.write('') oid = repo.write(pygit2.GIT_OBJ_BLOB, '') repo.index.add( pygit2.IndexEntry('2021-111.yaml', oid, pygit2.GIT_FILEMODE_BLOB)) repo.index.write() tree = repo.index.write_tree() author = pygit2.Signature('User', 'user@email') repo.create_commit('HEAD', author, author, 'Changes', tree, [repo.head.peel().oid]) osv.SourceRepository(id='oss-fuzz', name='oss-fuzz', repo_url='file://' + self.remote_source_repo_path, repo_username='').put() osv.Bug( id='2017-134', affected=['FILE5_29', 'FILE5_30'], affected_fuzzy=['5-29', '5-30'], confidence=100, details=( 'OSS-Fuzz report: ' 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1064\n\n' 'Crash type: Heap-buffer-overflow READ 1\n' 'Crash state:\ncdf_file_property_info\ncdf_file_summary_info\n' 'cdf_check_summary_info\n'), ecosystem='', fixed='19ccebafb7663c422c714e0c67fa4775abf91c43', has_affected=True, issue_id='1064', project='file', public=True, reference_urls=[ 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1064' ], regressed='17ee4cf670c363de8d2ea4a4897d7a699837873f', repo_url='https://github.com/file/file.git', search_indices=['file', '2017-134', '2017', '134'], severity='MEDIUM', sort_key='2017-0000134', source_id='oss-fuzz:5417710252982272', status=1, summary='Heap-buffer-overflow in cdf_file_property_info', timestamp=datetime.datetime(2021, 1, 15, 0, 0, 24, 559102)).put()
def process_results(): """Generate impact requests.""" if not request.headers.get('X-Appengine-Cron'): abort(403) publisher = pubsub_v1.PublisherClient() counters = {} for regress_result in osv.RegressResult.query(): key_id = regress_result.key.id() if not regress_result.commit: logging.info('Missing commit info for %s.', key_id) continue fixed_result = ndb.Key(osv.FixResult, key_id).get() if not fixed_result or not fixed_result.commit: logging.info('Fixed result does not exist for %s.', key_id) bug = osv.Bug.query(osv.Bug.source_id == key_id).get() if bug: logging.info('Bug already exists for %s.', key_id) continue if regress_result.issue_id: bug = osv.Bug.query( osv.Bug.issue_id == regress_result.issue_id).get() if bug: logging.info('Bug already exists for issue %s.', regress_result.issue_id) continue # Get ID counter for the year. if regress_result.timestamp: id_year = regress_result.timestamp.year else: id_year = None counter = counters.get(id_year) if not counter: counter = _get_counter(id_year) counters[id_year] = counter try: cur_id = '{}-{}'.format(counter.key.id(), counter.next_id) logging.info('Allocating OSV-%s.', cur_id) counter.next_id += 1 # Create the Bug now to avoid races when this cron is run again before the # impact task finishes. bug = osv.Bug(id=cur_id, timestamp=datetime.datetime.utcnow(), public=False, source_id=key_id, status=osv.BugStatus.UNPROCESSED) bug.put() logging.info('Requesting impact for %s.', key_id) publisher.publish(_TASKS_TOPIC, data=b'', type='impact', source_id=key_id, allocated_id=cur_id) finally: counter.put() return 'done'
def test_scheduled_updates(self, mock_publish): """Test scheduled updates.""" self.mock_repo.add_file('proj/OSV-2021-1337.yaml', _EMPTY_VULNERABILITY) self.mock_repo.add_file('proj/OSV-2021-1339.yaml', _EMPTY_VULNERABILITY) self.mock_repo.add_file('OSV-2021-1338.yaml', _EMPTY_VULNERABILITY) self.mock_repo.commit('OSV', '*****@*****.**') osv.SourceRepository( type=osv.SourceRepositoryType.GIT, id='source', name='source', repo_url='file://' + self.remote_source_repo_path, repo_username='').put() osv.Bug( db_id='OSV-2021-1337', affected_packages=[ osv.AffectedPackage( package=osv.Package(ecosystem='OSS-Fuzz', name='proj')) ], status=1, source_id='oss-fuzz:123', source_of_truth=osv.SourceOfTruth.SOURCE_REPO, timestamp=datetime.datetime(2020, 1, 1, 0, 0, 0, 0)).put() osv.Bug( db_id='OSV-2021-1338', affected_packages=[ osv.AffectedPackage( package=osv.Package(ecosystem='ecosystem', name='proj'), ranges=[ osv.AffectedRange2( type='GIT', events=[ osv.AffectedEvent(type='introduced', value='0'), osv.AffectedEvent(type='fixed', value='fix'), ]) ]) ], source_id='source:OSV-2021-1338.yaml', status=1, source_of_truth=osv.SourceOfTruth.SOURCE_REPO, timestamp=importer.utcnow()).put() osv.Bug( db_id='OSV-2021-1339', affected_packages=[ osv.AffectedPackage( package=osv.Package(ecosystem='OSS-Fuzz', name='proj')) ], status=1, source_id='oss-fuzz:124', source_of_truth=osv.SourceOfTruth.INTERNAL, timestamp=datetime.datetime(2020, 1, 1, 0, 0, 0, 0)).put() imp = importer.Importer('fake_public_key', 'fake_private_key', self.tmp_dir, 'bucket') imp.run() mock_publish.assert_has_calls([ mock.call( 'projects/oss-vdb/topics/tasks', data=b'', deleted='false', original_sha256=('bd3cc48676794308a58a19c97972a5e5' '42abcc9eb948db5701421616432cc0b9'), path='proj/OSV-2021-1337.yaml', source='oss-fuzz', type='update'), mock.call( 'projects/oss-vdb/topics/tasks', allocated_id='OSV-2021-1339', data=b'', source_id='oss-fuzz:124', type='impact'), ]) source_repo = osv.SourceRepository.get_by_id('oss-fuzz') self.assertEqual(datetime.date(2021, 1, 1), source_repo.last_update_date)
def test_scheduled_updates(self, mock_publish): """Test scheduled updates.""" self.mock_repo.add_file('proj/OSV-2021-1337.yaml', '') self.mock_repo.add_file('proj/OSV-2021-1339.yaml', '') self.mock_repo.add_file('OSV-2021-1338.yaml', '') self.mock_repo.commit('OSV', '*****@*****.**') osv.Bug(id='2021-1337', project='proj', ecosystem='OSS-Fuzz', fixed='', status=1, source_id='oss-fuzz:123', source_of_truth=osv.SourceOfTruth.SOURCE_REPO, timestamp=datetime.datetime(2020, 1, 1, 0, 0, 0, 0)).put() osv.Bug(id='2021-1338', project='proj', fixed='fix', source_id='source:OSV-2021-1338.yaml', status=1, source_of_truth=osv.SourceOfTruth.SOURCE_REPO, timestamp=importer.utcnow()).put() osv.Bug(id='2021-1339', project='proj', ecosystem='OSS-Fuzz', fixed='', status=1, source_id='oss-fuzz:124', source_of_truth=osv.SourceOfTruth.INTERNAL, timestamp=datetime.datetime(2020, 1, 1, 0, 0, 0, 0)).put() imp = importer.Importer('fake_public_key', 'fake_private_key', self.tmp_dir, 'bucket') imp.run() mock_publish.assert_has_calls([ mock.call('projects/oss-vdb/topics/tasks', data=b'', deleted='false', original_sha256=('e3b0c44298fc1c149afbf4c8996fb924' '27ae41e4649b934ca495991b7852b855'), path='proj/OSV-2021-1337.yaml', source='oss-fuzz', type='update'), mock.call('projects/oss-vdb/topics/tasks', allocated_id='2021-1339', data=b'', source_id='oss-fuzz:124', type='impact'), mock.call('projects/oss-vdb/topics/tasks', data=b'', deleted='false', original_sha256=('e3b0c44298fc1c149afbf4c8996fb924' '27ae41e4649b934ca495991b7852b855'), path='OSV-2021-1338.yaml', source='oss-fuzz', type='update'), ]) source_repo = osv.SourceRepository.get_by_id('oss-fuzz') self.assertEqual(datetime.date(2021, 1, 1), source_repo.last_update_date)
def process_results(): """Generate impact requests.""" if not request.headers.get('X-Appengine-Cron'): abort(403) publisher = pubsub_v1.PublisherClient() counters = {} for regress_result in osv.RegressResult.query(): key_id = regress_result.key.id() if not regress_result.commit: logging.info('Missing commit info for %s.', key_id) continue fixed_result = ndb.Key(osv.FixResult, key_id).get() if not fixed_result or not fixed_result.commit: logging.info('Fixed result does not exist for %s.', key_id) bug = osv.Bug.query(osv.Bug.source_id == key_id).get() if bug: logging.info('Bug already exists for %s.', key_id) continue if regress_result.issue_id: bug = osv.Bug.query( osv.Bug.issue_id == regress_result.issue_id).get() if bug: logging.info('Bug already exists for issue %s.', regress_result.issue_id) continue # Get ID counter for the year. if regress_result.timestamp: id_year = regress_result.timestamp.year else: id_year = None counter = counters.get(id_year) if not counter: counter = _get_counter(id_year) counters[id_year] = counter try: cur_id = '{}-{}'.format(counter.key.id(), counter.next_id) logging.info('Allocating OSV-%s.', cur_id) counter.next_id += 1 # Create the Bug now to avoid races when this cron is run again before the # impact task finishes. bug = osv.Bug(id=cur_id, timestamp=datetime.datetime.utcnow(), public=False, source_id=key_id, status=osv.BugStatus.UNPROCESSED) bug.put() logging.info('Requesting impact for %s.', key_id) publisher.publish(_TASKS_TOPIC, data=b'', type='impact', source_id=key_id, allocated_id=cur_id) finally: counter.put() # Re-compute bugs that aren't fixed. for bug in osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED, osv.Bug.fixed == ''): publisher.publish(_TASKS_TOPIC, data=b'', type='impact', source_id=bug.source_id, allocated_id=bug.key.id()) # Re-compute existing Bugs for a period of time, as upstream changes may # affect results. cutoff_time = (datetime.datetime.utcnow() - datetime.timedelta(days=_BUG_REDO_DAYS)) query = osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED, osv.Bug.timestamp >= cutoff_time) for bug in query: logging.info('Re-requesting impact for %s.', bug.key.id()) if not bug.fixed: # Previous query already requested impact tasks for unfixed bugs. continue publisher.publish(_TASKS_TOPIC, data=b'', type='impact', source_id=bug.source_id, allocated_id=bug.key.id()) return 'done'
def test_basic(self, mock_publish): """Test basic run.""" osv.Bug( db_id='OSV-2017-134', affected=['FILE5_29', 'FILE5_30'], affected_fuzzy=['5-29', '5-30'], affected_ranges=[{ 'type': 'GIT', 'repo_url': 'https://github.com/file/file.git', 'introduced': '17ee4cf670c363de8d2ea4a4897d7a699837873f', 'fixed': '19ccebafb7663c422c714e0c67fa4775abf91c43', }], details=( 'OSS-Fuzz report: ' 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1064\n\n' 'Crash type: Heap-buffer-overflow READ 1\n' 'Crash state:\ncdf_file_property_info\ncdf_file_summary_info\n' 'cdf_check_summary_info\n'), ecosystem='OSS-Fuzz', ecosystem_specific={ 'severity': 'MEDIUM', }, fixed='19ccebafb7663c422c714e0c67fa4775abf91c43', has_affected=True, issue_id='1064', project='file', public=True, reference_url_types={ 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1064': 'REPORT' }, regressed='17ee4cf670c363de8d2ea4a4897d7a699837873f', search_indices=['file', '2017-134', '2017', '134'], source_id='oss-fuzz:5417710252982272', source_of_truth=osv.SourceOfTruth.INTERNAL, status=1, summary='Heap-buffer-overflow in cdf_file_property_info', timestamp=datetime.datetime(2021, 1, 15, 0, 0, 24, 559102)).put() self.mock_repo.add_file('2021-111.yaml', '') self.mock_repo.commit('User', 'user@email') imp = importer.Importer('fake_public_key', 'fake_private_key', self.tmp_dir, 'bucket') imp.run() repo = pygit2.Repository(self.remote_source_repo_path) commit = repo.head.peel() self.assertEqual('*****@*****.**', commit.author.email) self.assertEqual('OSV', commit.author.name) self.assertEqual('Import from OSS-Fuzz', commit.message) diff = repo.diff(commit.parents[0], commit) self.assertEqual(self._load_test_data('expected_patch_basic.diff'), diff.patch) mock_publish.assert_has_calls([ mock.call('projects/oss-vdb/topics/tasks', data=b'', deleted='false', original_sha256=('e3b0c44298fc1c149afbf4c8996fb924' '27ae41e4649b934ca495991b7852b855'), path='2021-111.yaml', source='oss-fuzz', type='update') ]) bug = osv.Bug.get_by_id('OSV-2017-134') self.assertEqual(osv.SourceOfTruth.SOURCE_REPO, bug.source_of_truth) source_repo = osv.SourceRepository.get_by_id('oss-fuzz') self.assertEqual(str(commit.id), source_repo.last_synced_hash) self.mock_storage_client().get_bucket.assert_called_with('bucket') bucket = self.mock_storage_client().get_bucket('bucket') expected_upload_contents = self._load_test_data('expected.json') bucket.blob.assert_has_calls([ mock.call('testcase/5417710252982272.json'), mock.call().upload_from_string(expected_upload_contents), mock.call('issue/1064.json'), mock.call().upload_from_string(expected_upload_contents), ])
def _do_update(self, source_repo, repo, vulnerability, yaml_path, relative_path, original_sha256): """Process updates on a vulnerability.""" logging.info('Processing update for vulnerability %s', vulnerability.id) package_repo_dir = tempfile.TemporaryDirectory() package_repo_url = None package_repo = None bug = osv.Bug.get_by_id(vulnerability.id) if bug: fix_result = osv.FixResult.get_by_id(bug.source_id) if fix_result: add_fix_information(vulnerability, bug, fix_result) # Repo -> Git range collectors range_collectors = collections.defaultdict(osv.RangeCollector) versions_with_bug = set() versions_with_fix = set() commits = set() try: for affected_range in vulnerability.affects.ranges: if affected_range.type != vulnerability_pb2.AffectedRange.GIT: continue # Convert empty values ('') to None. introduced = affected_range.introduced or None fixed = affected_range.fixed or None range_collectors[affected_range.repo].add(introduced, fixed) for affected_range in vulnerability.affects.ranges: # Go through existing provided ranges to find additional ranges (via # cherrypicks and branches). if affected_range.type != vulnerability_pb2.AffectedRange.GIT: continue current_repo_url = affected_range.repo if current_repo_url != package_repo_url: # Different repo from previous one. package_repo_dir.cleanup() package_repo_dir = tempfile.TemporaryDirectory() package_repo_url = current_repo_url package_repo = osv.clone_with_retries( package_repo_url, package_repo_dir.name) result = osv.get_affected(package_repo, affected_range.introduced, affected_range.fixed) for introduced, fixed in result.affected_ranges: range_collectors[current_repo_url].add(introduced, fixed) versions_with_fix.update(result.tags_with_fix) versions_with_bug.update(result.tags_with_bug) commits.update(result.commits) finally: package_repo_dir.cleanup() # Enumerate ECOSYSTEM and SEMVER ranges. versions = self._enumerate_versions(vulnerability.package.name, vulnerability.package.ecosystem, vulnerability.affects.ranges) # Add additional versions derived from tags. versions.extend(versions_with_bug - versions_with_fix) if self._push_new_ranges_and_versions(source_repo, repo, vulnerability, yaml_path, original_sha256, range_collectors, versions): logging.info('Updated range/versions for vulnerability %s.', vulnerability.id) else: logging.warning('Discarding changes for %s due to conflicts.', vulnerability.id) return # Update datastore with new information. bug = osv.Bug.get_by_id(vulnerability.id) if not bug: if source_repo.name == 'oss-fuzz': logging.warning('%s not found for OSS-Fuzz source.', vulnerability.id) return bug = osv.Bug(id=vulnerability.id, source_id=f'{source_repo.name}:{relative_path}', timestamp=osv.utcnow(), status=osv.BugStatus.PROCESSED, source_of_truth=osv.SourceOfTruth.SOURCE_REPO) bug.update_from_vulnerability(vulnerability) bug.put() osv.update_affected_commits(bug.key.id(), commits, bug.project, bug.ecosystem, bug.public)