def handle_get(self):
queue = taskqueue.Queue('update')
if queue.fetch_statistics().tasks > 0:
self.response.write('update already in progress')
return
query = Library.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50, keys_only=True, start_cursor=cursor)
for key in keys:
task_count = task_count + 1
task_url = util.update_library_task(key.id())
util.new_task(task_url, target='manage', queue_name='update')
logging.info('triggered %d library updates', task_count)
query = Author.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50, keys_only=True, start_cursor=cursor)
for key in keys:
task_count = task_count + 1
task_url = util.update_author_task(key.id())
util.new_task(task_url, target='manage', queue_name='update')
logging.info('triggered %d author updates', task_count)
def test_update_doesnt_ingest_older_versions(self):
library_key = Library(id='org/repo',
tags=['v0.1.0', 'v1.0.0', 'v2.0.0'],
spdx_identifier='MIT').put()
Version(id='v1.0.0',
parent=library_key,
sha="old",
status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo',
{'status': 304})
self.respond_to_github(
'https://api.github.com/repos/org/repo/contributors',
{'status': 304})
self.respond_to_github(
'https://api.github.com/repos/org/repo/tags', """[
{"name": "v0.5.0", "commit": {"sha": "new"}},
{"name": "v1.0.0", "commit": {"sha": "old"}}
]""")
self.respond_to_github(
'https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'),
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([], [task.url for task in tasks])
def handle_get(self):
queue = taskqueue.Queue('update')
if queue.fetch_statistics().tasks > 0:
self.response.write('update already in progress')
return
query = Library.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50,
keys_only=True,
start_cursor=cursor)
for key in keys:
task_count = task_count + 1
task_url = util.update_library_task(key.id())
util.new_task(task_url, target='manage', queue_name='update')
logging.info('triggered %d library updates', task_count)
query = Author.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50,
keys_only=True,
start_cursor=cursor)
for key in keys:
task_count = task_count + 1
task_url = util.update_author_task(key.id())
util.new_task(task_url, target='manage', queue_name='update')
logging.info('triggered %d author updates', task_count)
def test_update_collection(self):
library_key = Library(id='org/repo', tags=['v0.0.1'], collection_sequence_number=1, kind='collection', spdx_identifier='MIT').put()
Version(id='v0.0.1', parent=library_key, sha="old", status=Status.ready).put()
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', """{
"ref": "refs/heads/master",
"object": {"sha": "new-master-sha"}
}""")
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = library_key.get()
self.assertEqual(library.error, None)
self.assertEqual(library.status, Status.ready)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v0.0.2', 'new-master-sha'),
util.ingest_version_task('org', 'repo', 'v0.0.2'),
], [task.url for task in tasks])
version = Version.get_by_id('v0.0.2', parent=library_key)
self.assertEqual(version.sha, 'new-master-sha')
self.assertEqual(version.status, Status.pending)
def test_update_collection(self):
library_key = Library(id='org/repo', tags=['v0.0.1'], collection_sequence_number=1, kind='collection', spdx_identifier='MIT').put()
Version(id='v0.0.1', parent=library_key, sha="old", status=Status.ready).put()
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', """{
"ref": "refs/heads/master",
"object": {"sha": "new-master-sha"}
}""")
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = library_key.get()
self.assertEqual(library.error, None)
self.assertEqual(library.status, Status.ready)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v0.0.2', 'new-master-sha'),
util.ingest_version_task('org', 'repo', 'v0.0.2'),
], [task.url for task in tasks])
version = Version.get_by_id('v0.0.2', parent=library_key)
self.assertEqual(version.sha, 'new-master-sha')
self.assertEqual(version.status, Status.pending)
def test_update_all(self):
library_key = Library(id='owner/repo').put()
author_key = Author(id='owner').put()
response = self.app.get('/manage/update-all', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.update_library_task(library_key.id()),
util.update_author_task(author_key.id()),
], [task.url for task in tasks])
def test_update_all(self):
library_key = Library(id='owner/repo').put()
author_key = Author(id='owner').put()
response = self.app.get('/manage/update-all', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.update_library_task(library_key.id()),
util.update_author_task(author_key.id()),
], [task.url for task in tasks])
def test_subsequent_update_triggers_version_deletion(self):
library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"old","v2.0.0":"old"}').put()
Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.delete_task('org', 'repo', 'v0.1.0'),
], [task.url for task in tasks])
def test_subsequent_update_triggers_version_deletion(self):
library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"old","v2.0.0":"old"}').put()
Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.delete_task('org', 'repo', 'v0.1.0'),
], [task.url for task in tasks])
def test_update_doesnt_ingest_older_versions(self):
library_key = Library(id='org/repo', tags=['v0.1.0', 'v1.0.0', 'v2.0.0'], spdx_identifier='MIT').put()
Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags', """[
{"name": "v0.5.0", "commit": {"sha": "new"}},
{"name": "v1.0.0", "commit": {"sha": "old"}}
]""")
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
], [task.url for task in tasks])
