def test_ingest_collection(self):
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"keywords": ["element-collection"], "license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', '{"ref": "refs/heads/master", "object": {"sha": "master-sha"}}')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}')
self.assertEqual(library.contributors, '["a"]')
self.assertEqual(library.tags, ['v0.0.1'])
version = ndb.Key(Library, 'org/repo', Version, 'v0.0.1').get()
self.assertIsNone(version.error)
self.assertEqual(version.status, Status.pending)
self.assertEqual(version.sha, 'master-sha')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v0.0.1', 'master-sha'),
util.ensure_author_task('org'),
util.ingest_version_task('org', 'repo', 'v0.0.1'),
], [task.url for task in tasks])
def test_update_collection(self):
library_key = Library(id='org/repo', tags=['v0.0.1'], collection_sequence_number=1, kind='collection', spdx_identifier='MIT').put()
Version(id='v0.0.1', parent=library_key, sha="old", status=Status.ready).put()
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', """{
"ref": "refs/heads/master",
"object": {"sha": "new-master-sha"}
}""")
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = library_key.get()
self.assertEqual(library.error, None)
self.assertEqual(library.status, Status.ready)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v0.0.2', 'new-master-sha'),
util.ingest_version_task('org', 'repo', 'v0.0.2'),
], [task.url for task in tasks])
version = Version.get_by_id('v0.0.2', parent=library_key)
self.assertEqual(version.sha, 'new-master-sha')
self.assertEqual(version.status, Status.pending)
def test_ingest_preview(self):
self.respond_to_github('https://api.github.com/repos/org/repo',
'{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github(
'https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github(
'https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github(
'https://raw.githubusercontent.com/org/repo/master/bower.json',
'{"license": "MIT"}')
response = self.app.get(util.ingest_preview_task('org', 'repo'),
params={
'commit': 'commit-sha',
'url': 'url'
},
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertTrue(library.shallow_ingestion)
version = Version.get_by_id('commit-sha', parent=library.key)
self.assertEquals(version.status, Status.pending)
self.assertEquals(version.sha, 'commit-sha')
self.assertEquals(version.url, 'url')
self.assertTrue(version.preview)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 2)
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'commit-sha'),
util.ingest_version_task('org', 'repo', 'commit-sha'),
], [task.url for task in tasks])
def test_analyze_leaves_existing_content_when_reanalyzing(self):
library_key = Library(id='owner/repo').put()
version_key = Version(id='v1.1.1',
parent=library_key,
sha='sha',
status='ready').put()
content = Content(id='analysis',
parent=version_key,
status=Status.pending)
content.content = 'existing data'
content.status = Status.ready
content.put()
response = self.app.get('/task/analyze/owner/repo',
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
content = Content.get_by_id('analysis', parent=version_key)
self.assertEqual(content.content, 'existing data')
self.assertEqual(content.status, Status.ready)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('owner', 'repo', 'v1.1.1'),
], [task.url for task in tasks])
def test_ingest_preview(self):
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}')
response = self.app.get(util.ingest_preview_task('org', 'repo'), params={'commit': 'commit-sha', 'url': 'url'}, headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertTrue(library.shallow_ingestion)
version = Version.get_by_id('commit-sha', parent=library.key)
self.assertEquals(version.status, Status.pending)
self.assertEquals(version.sha, 'commit-sha')
self.assertEquals(version.url, 'url')
self.assertTrue(version.preview)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 2)
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'commit-sha'),
util.ingest_version_task('org', 'repo', 'commit-sha'),
], [task.url for task in tasks])
def test_ingest_collection(self):
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"keywords": ["element-collection"], "license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', '{"ref": "refs/heads/master", "object": {"sha": "master-sha"}}')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}')
self.assertEqual(library.contributors, '["a"]')
self.assertEqual(library.tags, ['v0.0.1'])
version = ndb.Key(Library, 'org/repo', Version, 'v0.0.1').get()
self.assertIsNone(version.error)
self.assertEqual(version.status, Status.pending)
self.assertEqual(version.sha, 'master-sha')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v0.0.1', 'master-sha'),
util.ensure_author_task('org'),
util.ingest_version_task('org', 'repo', 'v0.0.1'),
], [task.url for task in tasks])
def test_ingest_element(self):
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/tags', '''[{"name": "v0.5.0", "commit": {"sha": "old"}},{"name": "v1.0.0", "commit": {"sha": "lol"}}]''')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}')
self.assertEqual(library.contributors, '["a"]')
self.assertEqual(library.tags, ['v0.5.0', 'v1.0.0'])
version = ndb.Key(Library, 'org/repo', Version, 'v1.0.0').get()
self.assertIsNotNone(version)
self.assertIsNone(version.error)
self.assertEqual(version.sha, 'lol')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v1.0.0'),
util.ensure_author_task('org'),
util.ingest_version_task('org', 'repo', 'v1.0.0'),
], [task.url for task in tasks])
def test_update_collection(self):
library_key = Library(id='org/repo', tags=['v0.0.1'], collection_sequence_number=1, kind='collection', spdx_identifier='MIT').put()
Version(id='v0.0.1', parent=library_key, sha="old", status=Status.ready).put()
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', """{
"ref": "refs/heads/master",
"object": {"sha": "new-master-sha"}
}""")
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = library_key.get()
self.assertEqual(library.error, None)
self.assertEqual(library.status, Status.ready)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v0.0.2', 'new-master-sha'),
util.ingest_version_task('org', 'repo', 'v0.0.2'),
], [task.url for task in tasks])
version = Version.get_by_id('v0.0.2', parent=library_key)
self.assertEqual(version.sha, 'new-master-sha')
self.assertEqual(version.status, Status.pending)
def test_ingest_element(self):
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/tags', '''[{"name": "v0.5.0", "commit": {"sha": "old"}},{"name": "v1.0.0", "commit": {"sha": "lol"}}]''')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}')
self.assertEqual(library.contributors, '["a"]')
self.assertEqual(library.tags, ['v0.5.0', 'v1.0.0'])
version = ndb.Key(Library, 'org/repo', Version, 'v1.0.0').get()
self.assertIsNotNone(version)
self.assertIsNone(version.error)
self.assertEqual(version.sha, 'lol')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v1.0.0'),
util.ensure_author_task('org'),
util.ingest_version_task('org', 'repo', 'v1.0.0'),
], [task.url for task in tasks])
def trigger_analysis(self, tag, sha, transactional=False):
analysis_sha = None
if self.library.kind == 'collection':
analysis_sha = sha
version_key = ndb.Key(Library, self.library.key.id(), Version, tag)
Content(id='analysis', parent=version_key, status=Status.pending).put()
task_url = util.ingest_analysis_task(self.owner, self.repo, tag,
analysis_sha)
util.new_task(task_url, target='analysis', transactional=transactional)
def trigger_analysis(self, tag, sha, transactional=False):
analysis_sha = None
if self.library.kind == 'collection':
analysis_sha = sha
version_key = ndb.Key(Library, self.library.key.id(), Version, tag)
content = Content.get_by_id('analysis', parent=version_key)
if content is None or content.status == Status.error:
Content(id='analysis', parent=version_key, status=Status.pending).put()
task_url = util.ingest_analysis_task(self.owner, self.repo, tag, analysis_sha)
util.new_task(task_url, target='analysis', transactional=transactional, queue_name='analysis')
def trigger_analysis(self, tag, sha, transactional=False):
analysis_sha = None
if self.library.kind == 'collection':
analysis_sha = sha
version_key = ndb.Key(Library, self.library.key.id(), Version, tag)
content = Content.get_by_id('analysis', parent=version_key)
if content is None or content.status == Status.error:
Content(id='analysis', parent=version_key, status=Status.pending).put()
task_url = util.ingest_analysis_task(self.scope, self.package, tag, analysis_sha)
util.new_task(task_url, target='analysis', transactional=transactional, queue_name='analysis')
def test_analyze(self):
library_key = Library(id='owner/repo').put()
Version(id='v1.1.1', parent=library_key, sha='sha',
status='ready').put()
response = self.app.get('/task/analyze/owner/repo',
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('owner', 'repo', 'v1.1.1'),
], [task.url for task in tasks])
def test_analyze(self):
library_key = Library(id='owner/repo').put()
version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
response = self.app.get('/task/analyze/owner/repo', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
content = Content.get_by_id('analysis', parent=version_key)
self.assertEqual(content.content, None)
self.assertEqual(content.status, Status.pending)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('owner', 'repo', 'v1.1.1'),
], [task.url for task in tasks])
def test_analyze_latest(self):
library_key = Library(id='owner/repo').put()
Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
version_key = Version(id='v1.1.2', parent=library_key, sha='sha', status='ready').put()
VersionCache.update(library_key)
response = self.app.get('/task/analyze/owner/repo/True', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
content = Content.get_by_id('analysis', parent=version_key)
self.assertEqual(content.get_json(), None)
self.assertEqual(content.status, Status.pending)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('owner', 'repo', 'v1.1.2'),
], [task.url for task in tasks])
def test_subsequent_update_triggers_version_ingestion(self):
library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"new","v2.0.0":"old","v3.0.0":"new"}').put()
Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v3.0.0'),
util.ingest_version_task('org', 'repo', 'v3.0.0'),
], [task.url for task in tasks])
def test_subsequent_update_triggers_version_ingestion(self):
library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"new","v2.0.0":"old","v3.0.0":"new"}').put()
Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v3.0.0'),
util.ingest_version_task('org', 'repo', 'v3.0.0'),
], [task.url for task in tasks])
