def test_add_element(self):
response = self.app.get('/manage/add/element/org/repo')
self.assertEqual(response.status_int, 200)
self.assertEqual(response.normal_body, 'OK')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].url, util.ingest_library_task('org', 'repo', 'element'))
self.respond_to_github('https://api.github.com/repos/org/repo', 'metadata bits')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/tags', '[{"ref": "refs/tags/v1.0.0", "object": {"sha": "lol"}}]')
response = self.app.get(util.ingest_library_task('org', 'repo', 'element'))
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertEqual(library.kind, 'element')
self.assertEqual(library.metadata, 'metadata bits')
self.assertEqual(library.contributors, '["a"]')
self.assertEqual(library.contributor_count, 1)
version = ndb.Key(Library, 'org/repo', Version, 'v1.0.0').get()
self.assertIsNone(version.error)
self.assertEqual(version.sha, 'lol')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 2)
self.assertEqual(tasks[1].url, util.ingest_version_task('org', 'repo', 'v1.0.0') + '?latestVersion=True')
def test_ingest_element_no_versions(self):
self.respond_to_github(
'https://raw.githubusercontent.com/org/repo/master/bower.json',
'{"license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo',
'{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github(
'https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/tags',
'''[]''')
self.respond_to_github(
'https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'),
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNotNone(library.error)
self.assertIsNotNone(json.loads(library.error).get('code', None))
self.assertEqual(library.metadata,
'{"owner":{"login":"******"},"name":"repo"}')
self.assertEqual(library.contributors, '["a"]')
self.assertEqual(library.tags, [])
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_author_task('org'),
], [task.url for task in tasks])
def test_ingest_collection(self):
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"keywords": ["element-collection"], "license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', '{"ref": "refs/heads/master", "object": {"sha": "master-sha"}}')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}')
self.assertEqual(library.contributors, '["a"]')
self.assertEqual(library.tags, ['v0.0.1'])
version = ndb.Key(Library, 'org/repo', Version, 'v0.0.1').get()
self.assertIsNone(version.error)
self.assertEqual(version.status, Status.pending)
self.assertEqual(version.sha, 'master-sha')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v0.0.1', 'master-sha'),
util.ensure_author_task('org'),
util.ingest_version_task('org', 'repo', 'v0.0.1'),
], [task.url for task in tasks])
def test_ingest_element(self):
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/tags', '''[{"name": "v0.5.0", "commit": {"sha": "old"}},{"name": "v1.0.0", "commit": {"sha": "lol"}}]''')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}')
self.assertEqual(library.contributors, '["a"]')
self.assertEqual(library.tags, ['v0.5.0', 'v1.0.0'])
version = ndb.Key(Library, 'org/repo', Version, 'v1.0.0').get()
self.assertIsNotNone(version)
self.assertIsNone(version.error)
self.assertEqual(version.sha, 'lol')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v1.0.0'),
util.ensure_author_task('org'),
util.ingest_version_task('org', 'repo', 'v1.0.0'),
], [task.url for task in tasks])
def test_github_error_fails_gracefully(self):
self.respond_to_github('https://api.github.com/repos/org/repo',
{'status': '500'})
response = self.app.get(util.ingest_library_task('org', 'repo'),
headers={'X-AppEngine-QueueName': 'default'},
status=502)
self.assertEqual(response.status_int, 502)
def test_ingest_collection(self):
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"keywords": ["element-collection"], "license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', '{"ref": "refs/heads/master", "object": {"sha": "master-sha"}}')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}')
self.assertEqual(library.contributors, '["a"]')
self.assertEqual(library.tags, ['v0.0.1'])
version = ndb.Key(Library, 'org/repo', Version, 'v0.0.1').get()
self.assertIsNone(version.error)
self.assertEqual(version.status, Status.pending)
self.assertEqual(version.sha, 'master-sha')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v0.0.1', 'master-sha'),
util.ensure_author_task('org'),
util.ingest_version_task('org', 'repo', 'v0.0.1'),
], [task.url for task in tasks])
def test_ingest_element(self):
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/tags', '''[{"name": "v0.5.0", "commit": {"sha": "old"}},{"name": "v1.0.0", "commit": {"sha": "lol"}}]''')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}')
self.assertEqual(library.contributors, '["a"]')
self.assertEqual(library.tags, ['v0.5.0', 'v1.0.0'])
version = ndb.Key(Library, 'org/repo', Version, 'v1.0.0').get()
self.assertIsNotNone(version)
self.assertIsNone(version.error)
self.assertEqual(version.sha, 'lol')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v1.0.0'),
util.ensure_author_task('org'),
util.ingest_version_task('org', 'repo', 'v1.0.0'),
], [task.url for task in tasks])
def test_add(self):
token = self.app.get('/manage/token').normal_body
response = self.app.get('/manage/add/org/repo', params={'token': token})
self.assertEqual(response.status_int, 200)
self.assertEqual(response.normal_body, 'OK')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].url, util.ingest_library_task('org', 'repo'))
def test_add(self):
token = self.app.get('/manage/token').normal_body
response = self.app.get('/manage/add/org/repo', params={'token': token})
self.assertEqual(response.status_int, 200)
self.assertEqual(response.normal_body, 'OK')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].url, util.ingest_library_task('org', 'repo'))
def test_ensure_when_missing(self):
response = self.app.get(util.ensure_library_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_library_task('owner', 'repo'),
], [task.url for task in tasks])
def test_add_no_scope(self):
self.respond_to('https://www.google.com/recaptcha/api/siteverify', '{"success": true}')
response = self.app.post('/api/publish/package')
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].url, util.ingest_library_task('@@npm', 'package'))
def test_ensure_when_missing(self):
response = self.app.get(util.ensure_library_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_library_task('owner', 'repo'),
], [task.url for task in tasks])
def test_add_no_scope(self):
self.respond_to('https://www.google.com/recaptcha/api/siteverify',
'{"success": true}')
response = self.app.post('/api/publish/package')
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].url,
util.ingest_library_task('@@npm', 'package'))
def test_ingest_no_license(self):
self.respond_to_github('https://api.github.com/repos/org/repo', '{"license": null, "owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNotNone(library.error)
self.assertEqual(library.status, Status.error)
def test_ingest_no_license(self):
self.respond_to_github('https://api.github.com/repos/org/repo', '{"license": null, "owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNotNone(library.error)
self.assertEqual(library.status, Status.error)
def test_ingest_license_fallback(self):
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo/tags', '[{"name": "v1.0.0", "commit": {"sha": "lol"}}]')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertEqual(library.status, Status.ready)
self.assertEqual(library.spdx_identifier, 'MIT')
def test_ingest_license_fallback(self):
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo/tags', '[{"name": "v1.0.0", "commit": {"sha": "lol"}}]')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertEqual(library.status, Status.ready)
self.assertEqual(library.spdx_identifier, 'MIT')
def post(self, library):
if not validate_captcha(self):
return
split = library.split('/')
if len(split) is 2:
scope = split[0]
package = split[1]
elif len(split) is 1:
scope = '@@npm'
package = library
else:
self.response.set_status(400)
self.response.write('Invalid name')
return
# TODO: validate valid repo and return result
task_url = util.ingest_library_task(scope, package)
util.new_task(task_url, target='manage')
def test_ingest_element_no_versions(self):
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}')
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/tags', '''[]''')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNotNone(library.error)
self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}')
self.assertEqual(library.contributors, '["a"]')
self.assertEqual(library.tags, [])
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_author_task('org'),
], [task.url for task in tasks])
def get(self, owner, repo, version):
logging.info('ingesting version %s/%s/%s', owner, repo, version)
key = ndb.Key(Library, '%s/%s' % (owner, repo), Version, version, Content, 'bower')
bower = json.loads(key.get().content)
ver = key.parent().get()
dependencies = bower.get('dependencies', {})
library_keys = []
dep_list = []
for name in dependencies.keys():
ver.dependencies.append(dependencies[name])
dep = Dependency.from_string(dependencies[name])
dep_list.append(dep)
library_keys.append(ndb.Key(Library, '%s/%s' % (dep.owner.lower(), dep.repo.lower())))
libraries = Library.get_or_create_list(library_keys)
for i, library in enumerate(libraries):
dep = dep_list[i]
library.collections.append(CollectionReference(version=key.parent(), semver=dep.version))
# FIXME: Can't assume this is an element.
task_url = util.ingest_library_task(dep.owner.lower(), dep.repo.lower(), 'element')
util.new_task(task_url)
libraries.append(ver)
ndb.put_multi(libraries)
def get(self, owner, repo, kind):
task_url = util.ingest_library_task(owner, repo, kind)
util.new_task(task_url)
self.response.write('OK')
def handle_get(self, owner, repo):
library = Library.get_by_id(Library.id(owner, repo))
if library is None or library.shallow_ingestion:
task_url = util.ingest_library_task(owner, repo)
util.new_task(task_url, target='manage')
def handle_get(self, owner, repo):
task_url = util.ingest_library_task(owner, repo)
util.new_task(task_url, target='manage')
self.response.write('OK')
def post(self, owner, repo):
if not validate_captcha(self):
return
# TODO: validate valid repo and return result
task_url = util.ingest_library_task(owner, repo)
util.new_task(task_url, target='manage')
def handle_get(self, owner, repo):
library = Library.get_by_id(Library.id(owner, repo))
if library is None or library.shallow_ingestion:
task_url = util.ingest_library_task(owner, repo)
util.new_task(task_url, target='manage')
def handle_get(self, owner, repo):
task_url = util.ingest_library_task(owner, repo)
util.new_task(task_url, target='manage')
self.response.write('OK')
def test_github_error_fails_gracefully(self):
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': '500'})
response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'}, status=502)
self.assertEqual(response.status_int, 502)