def test_ingest_collection(self): self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"keywords": ["element-collection"], "license": "MIT"}') self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', '{"ref": "refs/heads/master", "object": {"sha": "master-sha"}}') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}') self.assertEqual(library.contributors, '["a"]') self.assertEqual(library.tags, ['v0.0.1']) version = ndb.Key(Library, 'org/repo', Version, 'v0.0.1').get() self.assertIsNone(version.error) self.assertEqual(version.status, Status.pending) self.assertEqual(version.sha, 'master-sha') tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'v0.0.1', 'master-sha'), util.ensure_author_task('org'), util.ingest_version_task('org', 'repo', 'v0.0.1'), ], [task.url for task in tasks])
def test_update_collection(self): library_key = Library(id='org/repo', tags=['v0.0.1'], collection_sequence_number=1, kind='collection', spdx_identifier='MIT').put() Version(id='v0.0.1', parent=library_key, sha="old", status=Status.ready).put() self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', """{ "ref": "refs/heads/master", "object": {"sha": "new-master-sha"} }""") response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = library_key.get() self.assertEqual(library.error, None) self.assertEqual(library.status, Status.ready) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'v0.0.2', 'new-master-sha'), util.ingest_version_task('org', 'repo', 'v0.0.2'), ], [task.url for task in tasks]) version = Version.get_by_id('v0.0.2', parent=library_key) self.assertEqual(version.sha, 'new-master-sha') self.assertEqual(version.status, Status.pending)
def test_ingest_preview(self): self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github( 'https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github( 'https://api.github.com/repos/org/repo/stats/participation', '{}') self.respond_to_github( 'https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}') response = self.app.get(util.ingest_preview_task('org', 'repo'), params={ 'commit': 'commit-sha', 'url': 'url' }, headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertTrue(library.shallow_ingestion) version = Version.get_by_id('commit-sha', parent=library.key) self.assertEquals(version.status, Status.pending) self.assertEquals(version.sha, 'commit-sha') self.assertEquals(version.url, 'url') self.assertTrue(version.preview) tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 2) self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'commit-sha'), util.ingest_version_task('org', 'repo', 'commit-sha'), ], [task.url for task in tasks])
def test_analyze_leaves_existing_content_when_reanalyzing(self): library_key = Library(id='owner/repo').put() version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() content = Content(id='analysis', parent=version_key, status=Status.pending) content.content = 'existing data' content.status = Status.ready content.put() response = self.app.get('/task/analyze/owner/repo', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) content = Content.get_by_id('analysis', parent=version_key) self.assertEqual(content.content, 'existing data') self.assertEqual(content.status, Status.ready) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('owner', 'repo', 'v1.1.1'), ], [task.url for task in tasks])
def test_ingest_preview(self): self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}') response = self.app.get(util.ingest_preview_task('org', 'repo'), params={'commit': 'commit-sha', 'url': 'url'}, headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertTrue(library.shallow_ingestion) version = Version.get_by_id('commit-sha', parent=library.key) self.assertEquals(version.status, Status.pending) self.assertEquals(version.sha, 'commit-sha') self.assertEquals(version.url, 'url') self.assertTrue(version.preview) tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 2) self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'commit-sha'), util.ingest_version_task('org', 'repo', 'commit-sha'), ], [task.url for task in tasks])
def test_ingest_element(self): self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}') self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/tags', '''[{"name": "v0.5.0", "commit": {"sha": "old"}},{"name": "v1.0.0", "commit": {"sha": "lol"}}]''') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}') self.assertEqual(library.contributors, '["a"]') self.assertEqual(library.tags, ['v0.5.0', 'v1.0.0']) version = ndb.Key(Library, 'org/repo', Version, 'v1.0.0').get() self.assertIsNotNone(version) self.assertIsNone(version.error) self.assertEqual(version.sha, 'lol') tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'v1.0.0'), util.ensure_author_task('org'), util.ingest_version_task('org', 'repo', 'v1.0.0'), ], [task.url for task in tasks])
def trigger_analysis(self, tag, sha, transactional=False): analysis_sha = None if self.library.kind == 'collection': analysis_sha = sha version_key = ndb.Key(Library, self.library.key.id(), Version, tag) Content(id='analysis', parent=version_key, status=Status.pending).put() task_url = util.ingest_analysis_task(self.owner, self.repo, tag, analysis_sha) util.new_task(task_url, target='analysis', transactional=transactional)
def trigger_analysis(self, tag, sha, transactional=False): analysis_sha = None if self.library.kind == 'collection': analysis_sha = sha version_key = ndb.Key(Library, self.library.key.id(), Version, tag) content = Content.get_by_id('analysis', parent=version_key) if content is None or content.status == Status.error: Content(id='analysis', parent=version_key, status=Status.pending).put() task_url = util.ingest_analysis_task(self.owner, self.repo, tag, analysis_sha) util.new_task(task_url, target='analysis', transactional=transactional, queue_name='analysis')
def trigger_analysis(self, tag, sha, transactional=False): analysis_sha = None if self.library.kind == 'collection': analysis_sha = sha version_key = ndb.Key(Library, self.library.key.id(), Version, tag) content = Content.get_by_id('analysis', parent=version_key) if content is None or content.status == Status.error: Content(id='analysis', parent=version_key, status=Status.pending).put() task_url = util.ingest_analysis_task(self.scope, self.package, tag, analysis_sha) util.new_task(task_url, target='analysis', transactional=transactional, queue_name='analysis')
def test_analyze(self): library_key = Library(id='owner/repo').put() Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() response = self.app.get('/task/analyze/owner/repo', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('owner', 'repo', 'v1.1.1'), ], [task.url for task in tasks])
def test_analyze(self): library_key = Library(id='owner/repo').put() version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() response = self.app.get('/task/analyze/owner/repo', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) content = Content.get_by_id('analysis', parent=version_key) self.assertEqual(content.content, None) self.assertEqual(content.status, Status.pending) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('owner', 'repo', 'v1.1.1'), ], [task.url for task in tasks])
def test_analyze_latest(self): library_key = Library(id='owner/repo').put() Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() version_key = Version(id='v1.1.2', parent=library_key, sha='sha', status='ready').put() VersionCache.update(library_key) response = self.app.get('/task/analyze/owner/repo/True', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) content = Content.get_by_id('analysis', parent=version_key) self.assertEqual(content.get_json(), None) self.assertEqual(content.status, Status.pending) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('owner', 'repo', 'v1.1.2'), ], [task.url for task in tasks])
def test_subsequent_update_triggers_version_ingestion(self): library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"new","v2.0.0":"old","v3.0.0":"new"}').put() Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put() Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put() Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put() VersionCache.update(library_key) self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'v3.0.0'), util.ingest_version_task('org', 'repo', 'v3.0.0'), ], [task.url for task in tasks])