def test_version_cache(self):
library_key = ndb.Key(Library, 'a/b')
Version(id='v2.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v1.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v3.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v3.0.X', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v4.0.0', sha='x', status=Status.error,
parent=library_key).put()
Version(id='v5.0.0',
sha='x',
status=Status.pending,
parent=library_key).put()
Version(id='xxx', sha='x', status=Status.ready,
parent=library_key).put()
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, [])
latest_changed = VersionCache.update(library_key)
self.assertTrue(latest_changed)
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0'])
Version(id='v6.0.0', sha='x', status=Status.ready,
parent=library_key).put()
latest_changed = VersionCache.update(library_key)
self.assertTrue(latest_changed)
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions,
['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0', 'v6.0.0'])
def test_analyzer_index_empty(self):
metadata = """{
"full_name": "full-name"
}"""
library_key = Library(id='owner/repo', metadata=metadata).put()
version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
content = Content(id='analysis', parent=version_key, status=Status.pending)
data = {"analyzerData": {}}
content.json = data
content.status = Status.ready
content.put()
VersionCache.update(library_key)
response = self.app.get(util.update_indexes_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
index = search.Index('repo')
document = index.get('owner/repo')
self.assertIsNotNone(document)
self.assertTrue(len(document.fields) > 0)
elements = [field for field in document.fields if field.name == 'element']
self.assertEqual(len(elements), 0)
behaviors = [field for field in document.fields if field.name == 'behavior']
self.assertEqual(len(behaviors), 0)
def test_update_doesnt_ingest_older_versions(self):
library_key = Library(id='org/repo',
tags=['v0.1.0', 'v1.0.0', 'v2.0.0'],
spdx_identifier='MIT').put()
Version(id='v1.0.0',
parent=library_key,
sha="old",
status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo',
{'status': 304})
self.respond_to_github(
'https://api.github.com/repos/org/repo/contributors',
{'status': 304})
self.respond_to_github(
'https://api.github.com/repos/org/repo/tags', """[
{"name": "v0.5.0", "commit": {"sha": "new"}},
{"name": "v1.0.0", "commit": {"sha": "old"}}
]""")
self.respond_to_github(
'https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'),
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([], [task.url for task in tasks])
def test_update_indexes(self):
metadata = """{
"full_name": "full-name"
}"""
collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put()
collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put()
Content(id='bower', parent=collection_version_key, content="""{"dependencies": {
"a": "org/element-1#1.0.0",
"b": "org/element-2#1.0.0"
}}""").put()
VersionCache.update(collection_library_key)
response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
# Triggers ingestions
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_library_task('org', 'element-1'),
util.ensure_library_task('org', 'element-2'),
], [task.url for task in tasks])
# Ensures collection references
ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1"))
self.assertIsNotNone(ref1)
ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2"))
self.assertIsNotNone(ref2)
def test_update_indexes(self):
metadata = """{
"full_name": "full-name"
}"""
collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put()
collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put()
Content(id='bower', parent=collection_version_key, content="""{"dependencies": {
"a": "org/element-1#1.0.0",
"b": "org/element-2#1.0.0"
}}""").put()
VersionCache.update(collection_library_key)
response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
# Triggers ingestions
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_library_task('org', 'element-1'),
util.ensure_library_task('org', 'element-2'),
], [task.url for task in tasks])
# Ensures collection references
ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1"))
self.assertIsNotNone(ref1)
ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2"))
self.assertIsNotNone(ref2)
# Validate search index
index = search.Index('repo')
document = index.get('my/collection')
self.assertIsNotNone(document)
self.assertTrue(len(document.fields) > 0)
def test_delete_version(self):
library_key = ndb.Key(Library, 'owner/repo')
version_key = Version(id='v1.0.0', parent=library_key, sha='1', status=Status.ready).put()
VersionCache.update(library_key)
response = self.app.get('/task/delete/owner/repo/v1.0.0', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = version_key.get()
self.assertIsNone(version)
self.assertEqual(Library.versions_for_key_async(library_key).get_result(), [])
def test_delete_version(self):
library_key = ndb.Key(Library, 'owner/repo')
version_key = Version(id='v1.0.0', parent=library_key, sha='1', status=Status.ready).put()
VersionCache.update(library_key)
response = self.app.get('/task/delete/owner/repo/v1.0.0', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = version_key.get()
self.assertIsNone(version)
self.assertEqual(Library.versions_for_key_async(library_key).get_result(), [])
def handle_get(self, owner, repo, version):
# FIXME: Make deletion transactional with check on library that tag is excluded.
version_key = ndb.Key(Library, Library.id(owner, repo), Version, version)
ndb.delete_multi(ndb.Query(ancestor=version_key).iter(keys_only=True))
if VersionCache.update(version_key.parent()):
task_url = util.update_indexes_task(owner, repo)
util.new_task(task_url, target='manage')
def handle_get(self, scope, package, version):
# FIXME: Make deletion transactional with check on library that tag is excluded.
version_key = ndb.Key(Library, Library.id(scope, package), Version, version)
ndb.delete_multi(ndb.Query(ancestor=version_key).iter(keys_only=True))
if VersionCache.update(version_key.parent()):
task_url = util.update_indexes_task(scope, package)
util.new_task(task_url, target='manage')
def test_analyze_latest(self):
library_key = Library(id='owner/repo').put()
Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
version_key = Version(id='v1.1.2', parent=library_key, sha='sha', status='ready').put()
VersionCache.update(library_key)
response = self.app.get('/task/analyze/owner/repo/True', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
content = Content.get_by_id('analysis', parent=version_key)
self.assertEqual(content.get_json(), None)
self.assertEqual(content.status, Status.pending)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('owner', 'repo', 'v1.1.2'),
], [task.url for task in tasks])
def test_subsequent_update_triggers_version_deletion(self):
library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"old","v2.0.0":"old"}').put()
Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.delete_task('org', 'repo', 'v0.1.0'),
], [task.url for task in tasks])
def test_subsequent_update_triggers_version_deletion(self):
library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"old","v2.0.0":"old"}').put()
Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.delete_task('org', 'repo', 'v0.1.0'),
], [task.url for task in tasks])
def test_update_doesnt_ingest_older_versions(self):
library_key = Library(id='org/repo', tags=['v0.1.0', 'v1.0.0', 'v2.0.0'], spdx_identifier='MIT').put()
Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags', """[
{"name": "v0.5.0", "commit": {"sha": "new"}},
{"name": "v1.0.0", "commit": {"sha": "old"}}
]""")
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
], [task.url for task in tasks])
def test_version_cache(self):
library_key = ndb.Key(Library, 'a/b')
Version(id='v2.0.0', sha='x', status=Status.ready, parent=library_key).put()
Version(id='v1.0.0', sha='x', status=Status.ready, parent=library_key).put()
Version(id='v3.0.0', sha='x', status=Status.ready, parent=library_key).put()
Version(id='v3.0.X', sha='x', status=Status.ready, parent=library_key).put()
Version(id='v4.0.0', sha='x', status=Status.error, parent=library_key).put()
Version(id='v5.0.0', sha='x', status=Status.pending, parent=library_key).put()
Version(id='xxx', sha='x', status=Status.ready, parent=library_key).put()
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, [])
latest_changed = VersionCache.update(library_key)
self.assertTrue(latest_changed)
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0'])
Version(id='v6.0.0', sha='x', status=Status.ready, parent=library_key).put()
latest_changed = VersionCache.update(library_key)
self.assertTrue(latest_changed)
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0', 'v6.0.0'])
def update_versions_and_index(self):
if VersionCache.update(self.version_key.parent()):
task_url = util.update_indexes_task(self.owner, self.repo)
util.new_task(task_url, target='manage', transactional=True)
def update_versions_and_index(self):
if VersionCache.update(self.version_key.parent()):
task_url = util.update_indexes_task(self.owner, self.repo)
util.new_task(task_url, target='manage', transactional=True)
