def handle_get(self): queue = taskqueue.Queue('update') if queue.fetch_statistics().tasks > 0: self.response.write('update already in progress') return query = Library.query() cursor = None more = True task_count = 0 while more: keys, cursor, more = query.fetch_page(50, keys_only=True, start_cursor=cursor) for key in keys: task_count = task_count + 1 task_url = util.update_library_task(key.id()) util.new_task(task_url, target='manage', queue_name='update') logging.info('triggered %d library updates', task_count) query = Author.query() cursor = None more = True task_count = 0 while more: keys, cursor, more = query.fetch_page(50, keys_only=True, start_cursor=cursor) for key in keys: task_count = task_count + 1 task_url = util.update_author_task(key.id()) util.new_task(task_url, target='manage', queue_name='update') logging.info('triggered %d author updates', task_count)
def test_update_doesnt_ingest_older_versions(self): library_key = Library(id='org/repo', tags=['v0.1.0', 'v1.0.0', 'v2.0.0'], spdx_identifier='MIT').put() Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put() VersionCache.update(library_key) self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304}) self.respond_to_github( 'https://api.github.com/repos/org/repo/contributors', {'status': 304}) self.respond_to_github( 'https://api.github.com/repos/org/repo/tags', """[ {"name": "v0.5.0", "commit": {"sha": "new"}}, {"name": "v1.0.0", "commit": {"sha": "old"}} ]""") self.respond_to_github( 'https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) tasks = self.tasks.get_filtered_tasks() self.assertEqual([], [task.url for task in tasks])
def test_update_collection(self): library_key = Library(id='org/repo', tags=['v0.0.1'], collection_sequence_number=1, kind='collection', spdx_identifier='MIT').put() Version(id='v0.0.1', parent=library_key, sha="old", status=Status.ready).put() self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', """{ "ref": "refs/heads/master", "object": {"sha": "new-master-sha"} }""") response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = library_key.get() self.assertEqual(library.error, None) self.assertEqual(library.status, Status.ready) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'v0.0.2', 'new-master-sha'), util.ingest_version_task('org', 'repo', 'v0.0.2'), ], [task.url for task in tasks]) version = Version.get_by_id('v0.0.2', parent=library_key) self.assertEqual(version.sha, 'new-master-sha') self.assertEqual(version.status, Status.pending)
def test_update_all(self): library_key = Library(id='owner/repo').put() author_key = Author(id='owner').put() response = self.app.get('/manage/update-all', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.update_library_task(library_key.id()), util.update_author_task(author_key.id()), ], [task.url for task in tasks])
def test_subsequent_update_triggers_version_deletion(self): library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"old","v2.0.0":"old"}').put() Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put() Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put() Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put() VersionCache.update(library_key) self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.delete_task('org', 'repo', 'v0.1.0'), ], [task.url for task in tasks])
def test_update_doesnt_ingest_older_versions(self): library_key = Library(id='org/repo', tags=['v0.1.0', 'v1.0.0', 'v2.0.0'], spdx_identifier='MIT').put() Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put() VersionCache.update(library_key) self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/tags', """[ {"name": "v0.5.0", "commit": {"sha": "new"}}, {"name": "v1.0.0", "commit": {"sha": "old"}} ]""") self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ ], [task.url for task in tasks])