def test_analyze_resets_error_content_when_reanalyzing(self):
library_key = Library(id='owner/repo').put()
version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
content = Content(id='analysis', parent=version_key, status=Status.pending)
content.status = Status.error
content.put()
response = self.app.get('/task/analyze/owner/repo', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
content = Content.get_by_id('analysis', parent=version_key)
self.assertEqual(content.status, Status.pending)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('owner', 'repo', 'v1.1.1'),
], [task.url for task in tasks])
def test_compressed(self):
library_key = Library(id='owner/repo').put()
version_key = Version(id='v1.1.1',
parent=library_key,
sha='sha',
status='ready').put()
content = Content(id='analysis',
parent=version_key,
status=Status.pending)
content.json = dict({"analyzerData": "some data"})
content.status = Status.ready
content.put()
response = self.app.get(
'/api/docs/owner/repo/v1.1.1?use_analyzer_data')
self.assertEqual(response.status_int, 200)
self.assertEqual(
json.loads(response.normal_body).get('analysis'), "some data")
def test_subsequent_update_triggers_version_deletion(self):
library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"old","v2.0.0":"old"}').put()
Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.delete_task('org', 'repo', 'v0.1.0'),
], [task.url for task in tasks])
def test_update_doesnt_ingest_older_versions(self):
library_key = Library(id='org/repo', tags=['v0.1.0', 'v1.0.0', 'v2.0.0'], spdx_identifier='MIT').put()
Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags', """[
{"name": "v0.5.0", "commit": {"sha": "new"}},
{"name": "v1.0.0", "commit": {"sha": "old"}}
]""")
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
], [task.url for task in tasks])
def test_update_indexes(self):
metadata = """{
"full_name": "full-name"
}"""
collection_library_key = Library(id='my/collection',
status=Status.ready,
kind='collection',
metadata=metadata).put()
collection_version_key = Version(id='v1.0.0',
parent=collection_library_key,
sha='sha',
status=Status.ready).put()
Content(id='bower',
parent=collection_version_key,
content="""{"dependencies": {
"a": "org/element-1#1.0.0",
"b": "org/element-2#1.0.0"
}}""").put()
VersionCache.update(collection_library_key)
response = self.app.get(util.update_indexes_task('my', 'collection'),
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
# Triggers ingestions
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_library_task('org', 'element-1'),
util.ensure_library_task('org', 'element-2'),
], [task.url for task in tasks])
# Ensures collection references
ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0",
parent=ndb.Key(
Library, "org/element-1"))
self.assertIsNotNone(ref1)
ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0",
parent=ndb.Key(
Library, "org/element-2"))
self.assertIsNotNone(ref2)
def test_update_collection(self):
library_key = Library(id='org/repo',
tags=['v0.0.1'],
collection_sequence_number=1,
kind='collection',
spdx_identifier='MIT').put()
Version(id='v0.0.1',
parent=library_key,
sha="old",
status=Status.ready).put()
self.respond_to_github('https://api.github.com/repos/org/repo',
{'status': 304})
self.respond_to_github(
'https://api.github.com/repos/org/repo/contributors',
{'status': 304})
self.respond_to_github(
'https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github(
'https://api.github.com/repos/org/repo/git/refs/heads/master', """{
"ref": "refs/heads/master",
"object": {"sha": "new-master-sha"}
}""")
response = self.app.get(util.update_library_task('org/repo'),
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = library_key.get()
self.assertEqual(library.error, None)
self.assertEqual(library.status, Status.ready)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v0.0.2',
'new-master-sha'),
util.ingest_version_task('org', 'repo', 'v0.0.2'),
], [task.url for task in tasks])
version = Version.get_by_id('v0.0.2', parent=library_key)
self.assertEqual(version.sha, 'new-master-sha')
self.assertEqual(version.status, Status.pending)
def test_update_deletes_missing_repo(self):
library = Library(id='org/repo',
metadata_etag='a',
contributors_etag='b',
tags_etag='c',
spdx_identifier='MIT')
library.put()
version = Version(parent=library.key, id='v1.0.0', sha='lol')
version.put()
self.respond_to_github('https://api.github.com/repos/org/repo',
{'status': 404})
response = self.app.get('/task/update/org/repo',
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = version.key.get()
library = library.key.get()
self.assertIsNone(library)
self.assertIsNone(version)
def test_ingest_version(self):
library_key = Library(
id='org/repo',
metadata=
'{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}'
).put()
Version(id='v1.0.0', parent=library_key, sha='sha').put()
self.respond_to_github(
r'https://api.github.com/repos/org/repo/readme\?ref=sha',
'{"content":"%s"}' % b64encode('README'))
self.respond_to(
'https://raw.githubusercontent.com/org/repo/sha/bower.json', '{}')
self.respond_to_github('https://api.github.com/markdown',
'<html>README</html>')
response = self.app.get(util.ingest_version_task(
'org', 'repo', 'v1.0.0'),
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = Version.get_by_id('v1.0.0', parent=library_key)
self.assertIsNone(version.error)
self.assertEqual(version.status, Status.ready)
self.assertFalse(version.preview)
versions = Library.versions_for_key_async(library_key).get_result()
self.assertEqual(['v1.0.0'], versions)
readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content,
'readme').get()
self.assertEqual(readme.content, 'README')
readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content,
'readme.html').get()
self.assertEqual(readme_html.content, '<html>README</html>')
bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content,
'bower').get()
self.assertEqual(bower.content, '{}')
def test_update_respects_304(self):
library = Library(id='org/repo',
metadata_etag='a',
contributors_etag='b',
tags_etag='c',
tag_map='{}',
spdx_identifier='MIT')
library.put()
self.respond_to_github('https://api.github.com/repos/org/repo',
{'status': 304})
self.respond_to_github(
'https://api.github.com/repos/org/repo/contributors',
{'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags',
{'status': 304})
self.respond_to_github(
'https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get('/task/update/org/repo',
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 0)
def test_analyzer_index(self):
metadata = """{
"full_name": "full-name"
}"""
library_key = Library(id='owner/repo', metadata=metadata).put()
version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
content = Content(id='analysis', parent=version_key, status=Status.pending)
data = {
"analyzerData": {
"elements": [{"tagname": "my-element"}, {"classname": "another-element"}],
"metadata": {
"polymer": {"behaviors": [{"name": "polymer-behavior"}]}
}
}
}
content.json = data
content.status = Status.ready
content.put()
VersionCache.update(library_key)
response = self.app.get(util.update_indexes_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
index = search.Index('repo')
document = index.get('owner/repo')
self.assertIsNotNone(document)
self.assertTrue(len(document.fields) > 0)
elements = [field for field in document.fields if field.name == 'element']
self.assertEqual(len(elements), 1)
self.assertEqual(len(elements[0].value.split(' ')), 2)
behaviors = [field for field in document.fields if field.name == 'behavior']
self.assertEqual(len(behaviors), 1)
self.assertEqual(len(behaviors[0].value.split(' ')), 1)
def test_normal(self):
headers = {'X-Github-Event': 'pull_request'}
payload = {
'action': 'opened',
'repository': {
'owner': {
'login': '******'
},
'name': 'repo',
'full_name': 'owner/repo'
},
'pull_request': {
'head': {
'sha': 'sha',
'repo': {
'owner': {
'login': '******'
},
'name': 'pull_repo',
'full_name': 'pull_owner/pull_repo'
}
},
'url': 'github_pr_url'
}
}
library = Library(id='owner/repo')
library.put()
self.respond_to('https://api.github.com/repos/owner/repo/statuses',
{'status': 201})
self.app.post('/api/preview-event',
params=json.dumps(payload),
headers=headers,
status=200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 1)
