def handle_get(self, owner, repo, latest=False):
self.init_library(owner, repo)
if self.library is None:
self.response.set_status(404)
self.response.write('could not find library: %s' %
Library.id(owner, repo))
return
if latest:
version_id = Library.default_version_for_key_async(
self.library.key).get_result()
if version_id:
version = Version.get_by_id(version_id,
parent=self.library.key)
if version is not None:
self.trigger_analysis(version_id,
version.sha,
transactional=False)
else:
versions = Version.query(Version.status == Status.ready,
ancestor=self.library.key).fetch()
for version in versions:
self.trigger_analysis(version.key.id(),
version.sha,
transactional=False)
def test_update_collection(self):
library_key = Library(id='org/repo', tags=['v0.0.1'], collection_sequence_number=1, kind='collection', spdx_identifier='MIT').put()
Version(id='v0.0.1', parent=library_key, sha="old", status=Status.ready).put()
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', """{
"ref": "refs/heads/master",
"object": {"sha": "new-master-sha"}
}""")
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = library_key.get()
self.assertEqual(library.error, None)
self.assertEqual(library.status, Status.ready)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v0.0.2', 'new-master-sha'),
util.ingest_version_task('org', 'repo', 'v0.0.2'),
], [task.url for task in tasks])
version = Version.get_by_id('v0.0.2', parent=library_key)
self.assertEqual(version.sha, 'new-master-sha')
self.assertEqual(version.status, Status.pending)
def test_ingest_version(self):
library_key = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}').put()
Version(id='v1.0.0', parent=library_key, sha='sha').put()
self.respond_to_github(r'https://api.github.com/repos/org/repo/readme\?ref=sha', '{"content":"%s"}' % b64encode('README'))
self.respond_to('https://raw.githubusercontent.com/org/repo/sha/bower.json', '{}')
self.respond_to_github('https://api.github.com/markdown', '<html>README</html>')
response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = Version.get_by_id('v1.0.0', parent=library_key)
self.assertIsNone(version.error)
self.assertEqual(version.status, Status.ready)
self.assertFalse(version.preview)
versions = Library.versions_for_key_async(library_key).get_result()
self.assertEqual(['v1.0.0'], versions)
readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme').get()
self.assertEqual(readme.content, 'README')
readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme.html').get()
self.assertEqual(readme_html.content, '<html>README</html>')
bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'bower').get()
self.assertEqual(bower.get_json(), {})
def test_latest_matching_collection_version_is_returned(self):
collection_key = ndb.Key(Library, 'collection/1')
collection_v1 = Version(id='v1.0.0',
sha='x',
status=Status.ready,
parent=collection_key).put()
collection_v2 = Version(id='v2.0.0',
sha='x',
status=Status.ready,
parent=collection_key).put()
collection_v3 = Version(id='v3.0.0',
sha='x',
status=Status.ready,
parent=collection_key).put()
element_key = ndb.Key(Library, 'ele/ment')
element_v1 = Version(id='v1.0.0',
sha='x',
status=Status.ready,
parent=element_key).put()
CollectionReference.ensure(element_key, collection_v1, '^1.0.0')
CollectionReference.ensure(element_key, collection_v2, '^1.0.0')
CollectionReference.ensure(element_key, collection_v3, '^2.0.0')
collections = yield Version.collections_for_key_async(element_v1)
collection_keys = [collection.key for collection in collections]
# Only latest matching version of the collection should be present.
self.assertEqual(collection_keys, [
collection_v2,
])
def test_delete_version(self):
library_key = ndb.Key(Library, 'owner/repo')
version_key = Version(id='v1.0.0', parent=library_key, sha='1', status=Status.ready).put()
VersionCache.update(library_key)
response = self.app.get('/task/delete/owner/repo/v1.0.0', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = version_key.get()
self.assertIsNone(version)
self.assertEqual(Library.versions_for_key_async(library_key).get_result(), [])
def test_delete_version(self):
library_key = ndb.Key(Library, 'owner/repo')
version_key = Version(id='v1.0.0', parent=library_key, sha='1', status=Status.ready).put()
VersionCache.update(library_key)
response = self.app.get('/task/delete/owner/repo/v1.0.0', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = version_key.get()
self.assertIsNone(version)
self.assertEqual(Library.versions_for_key_async(library_key).get_result(), [])
def trigger_version_ingestion(self, tag, sha, url=None, preview=False):
version_object = Version.get_by_id(tag, parent=self.library.key)
if version_object is not None and (version_object.status == Status.ready or version_object.status == Status.pending):
# Version object is already up to date or pending
return False
Version(id=tag, parent=self.library.key, sha=sha, url=url, preview=preview).put()
task_url = util.ingest_version_task(self.scope, self.package, tag)
util.new_task(task_url, target='manage', transactional=True)
self.trigger_analysis(tag, sha, transactional=True)
return True
def test_version_cache(self):
library_key = ndb.Key(Library, 'a/b')
Version(id='v2.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v1.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v3.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v3.0.X', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v4.0.0', sha='x', status=Status.error,
parent=library_key).put()
Version(id='v5.0.0',
sha='x',
status=Status.pending,
parent=library_key).put()
Version(id='xxx', sha='x', status=Status.ready,
parent=library_key).put()
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, [])
latest_changed = VersionCache.update(library_key)
self.assertTrue(latest_changed)
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0'])
Version(id='v6.0.0', sha='x', status=Status.ready,
parent=library_key).put()
latest_changed = VersionCache.update(library_key)
self.assertTrue(latest_changed)
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions,
['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0', 'v6.0.0'])
def test_update_deletes_missing_repo(self):
library = Library(id='org/repo', metadata_etag='a', contributors_etag='b', tags_etag='c', spdx_identifier='MIT')
library.put()
version = Version(parent=library.key, id='v1.0.0', sha='lol')
version.put()
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 404})
response = self.app.get('/task/update/org/repo', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = version.key.get()
library = library.key.get()
self.assertIsNone(library)
self.assertIsNone(version)
def test_update_deletes_missing_repo(self):
library = Library(id='org/repo', metadata_etag='a', contributors_etag='b', tags_etag='c', spdx_identifier='MIT')
library.put()
version = Version(parent=library.key, id='v1.0.0', sha='lol')
version.put()
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 404})
response = self.app.get('/task/update/org/repo', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = version.key.get()
library = library.key.get()
self.assertIsNone(library)
self.assertIsNone(version)
def test_ingest_preview(self):
self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}')
response = self.app.get(util.ingest_preview_task('org', 'repo'), params={'commit': 'commit-sha', 'url': 'url'}, headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertTrue(library.shallow_ingestion)
version = Version.get_by_id('commit-sha', parent=library.key)
self.assertEquals(version.status, Status.pending)
self.assertEquals(version.sha, 'commit-sha')
self.assertEquals(version.url, 'url')
self.assertTrue(version.preview)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 2)
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'commit-sha'),
util.ingest_version_task('org', 'repo', 'commit-sha'),
], [task.url for task in tasks])
def test_ingest_version(self):
library_key = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}').put()
Version(id='v1.0.0', parent=library_key, sha='sha').put()
self.respond_to_github(r'https://api.github.com/repos/org/repo/readme\?ref=sha', '{"content":"%s"}' % b64encode('README'))
self.respond_to('https://raw.githubusercontent.com/org/repo/sha/bower.json', '{}')
self.respond_to_github('https://api.github.com/markdown', '<html>README</html>')
response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = Version.get_by_id('v1.0.0', parent=library_key)
self.assertIsNone(version.error)
self.assertEqual(version.status, Status.ready)
self.assertFalse(version.preview)
versions = Library.versions_for_key_async(library_key).get_result()
self.assertEqual(['v1.0.0'], versions)
readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme').get()
self.assertEqual(readme.content, 'README')
readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme.html').get()
self.assertEqual(readme_html.content, '<html>README</html>')
bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'bower').get()
self.assertEqual(bower.content, '{}')
def test_update_collection(self):
library_key = Library(id='org/repo', tags=['v0.0.1'], collection_sequence_number=1, kind='collection', spdx_identifier='MIT').put()
Version(id='v0.0.1', parent=library_key, sha="old", status=Status.ready).put()
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', """{
"ref": "refs/heads/master",
"object": {"sha": "new-master-sha"}
}""")
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = library_key.get()
self.assertEqual(library.error, None)
self.assertEqual(library.status, Status.ready)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'v0.0.2', 'new-master-sha'),
util.ingest_version_task('org', 'repo', 'v0.0.2'),
], [task.url for task in tasks])
version = Version.get_by_id('v0.0.2', parent=library_key)
self.assertEqual(version.sha, 'new-master-sha')
self.assertEqual(version.status, Status.pending)
def test_analyze_leaves_existing_content_when_reanalyzing(self):
library_key = Library(id='owner/repo').put()
version_key = Version(id='v1.1.1',
parent=library_key,
sha='sha',
status='ready').put()
content = Content(id='analysis',
parent=version_key,
status=Status.pending)
content.content = 'existing data'
content.status = Status.ready
content.put()
response = self.app.get('/task/analyze/owner/repo',
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
content = Content.get_by_id('analysis', parent=version_key)
self.assertEqual(content.content, 'existing data')
self.assertEqual(content.status, Status.ready)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('owner', 'repo', 'v1.1.1'),
], [task.url for task in tasks])
def get(self, owner, repo, version=None):
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Content-Type'] = 'application/json'
library_key = ndb.Key(Library, Library.id(owner, repo))
if version is None:
version = yield Library.default_version_for_key_async(library_key)
if version is None:
self.response.set_status(404)
return
version_key = ndb.Key(Library, library_key.id(), Version, version)
collection_versions = yield Version.collections_for_key_async(version_key)
collection_futures = []
for collection_version in collection_versions:
collection_futures.append(LibraryMetadata.brief_async(collection_version.key.parent(), collection_version.key.id()))
collections = []
for future in collection_futures:
collection_result = yield future
if collection_result is not None:
collections.append(collection_result)
result = {
'results': collections,
'count': len(collections),
}
self.response.write(json.dumps(result))
def test_ingest_preview(self):
self.respond_to_github('https://api.github.com/repos/org/repo',
'{"owner":{"login":"******"},"name":"repo"}')
self.respond_to_github(
'https://api.github.com/repos/org/repo/contributors', '["a"]')
self.respond_to_github(
'https://api.github.com/repos/org/repo/stats/participation', '{}')
self.respond_to_github(
'https://raw.githubusercontent.com/org/repo/master/bower.json',
'{"license": "MIT"}')
response = self.app.get(util.ingest_preview_task('org', 'repo'),
params={
'commit': 'commit-sha',
'url': 'url'
},
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertTrue(library.shallow_ingestion)
version = Version.get_by_id('commit-sha', parent=library.key)
self.assertEquals(version.status, Status.pending)
self.assertEquals(version.sha, 'commit-sha')
self.assertEquals(version.url, 'url')
self.assertTrue(version.preview)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 2)
self.assertEqual([
util.ingest_analysis_task('org', 'repo', 'commit-sha'),
util.ingest_version_task('org', 'repo', 'commit-sha'),
], [task.url for task in tasks])
def test_update_indexes(self):
metadata = """{
"full_name": "full-name"
}"""
collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put()
collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put()
Content(id='bower', parent=collection_version_key, content="""{"dependencies": {
"a": "org/element-1#1.0.0",
"b": "org/element-2#1.0.0"
}}""").put()
VersionCache.update(collection_library_key)
response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
# Triggers ingestions
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_library_task('org', 'element-1'),
util.ensure_library_task('org', 'element-2'),
], [task.url for task in tasks])
# Ensures collection references
ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1"))
self.assertIsNotNone(ref1)
ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2"))
self.assertIsNotNone(ref2)
# Validate search index
index = search.Index('repo')
document = index.get('my/collection')
self.assertIsNotNone(document)
self.assertTrue(len(document.fields) > 0)
def get(self, owner, repo, kind):
commit = self.request.get('commit', None)
url = self.request.get('url', None)
assert commit is not None and url is not None
self.init_library(owner, repo, kind)
is_new = self.library.metadata is None and self.library.error is None
if is_new:
self.library.ingest_versions = False
self.library_dirty = True
self.update_metadata()
version = Version(parent=self.library.key, id=commit, sha=commit, url=url)
version.put()
task_url = util.ingest_version_task(owner, repo, commit)
util.new_task(task_url)
self.commit()
def test_analyzer_index_empty(self):
metadata = """{
"full_name": "full-name"
}"""
library_key = Library(id='owner/repo', metadata=metadata).put()
version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
content = Content(id='analysis', parent=version_key, status=Status.pending)
data = {"analyzerData": {}}
content.json = data
content.status = Status.ready
content.put()
VersionCache.update(library_key)
response = self.app.get(util.update_indexes_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
index = search.Index('repo')
document = index.get('owner/repo')
self.assertIsNotNone(document)
self.assertTrue(len(document.fields) > 0)
elements = [field for field in document.fields if field.name == 'element']
self.assertEqual(len(elements), 0)
behaviors = [field for field in document.fields if field.name == 'behavior']
self.assertEqual(len(behaviors), 0)
def test_update_doesnt_ingest_older_versions(self):
library_key = Library(id='org/repo',
tags=['v0.1.0', 'v1.0.0', 'v2.0.0'],
spdx_identifier='MIT').put()
Version(id='v1.0.0',
parent=library_key,
sha="old",
status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo',
{'status': 304})
self.respond_to_github(
'https://api.github.com/repos/org/repo/contributors',
{'status': 304})
self.respond_to_github(
'https://api.github.com/repos/org/repo/tags', """[
{"name": "v0.5.0", "commit": {"sha": "new"}},
{"name": "v1.0.0", "commit": {"sha": "old"}}
]""")
self.respond_to_github(
'https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'),
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([], [task.url for task in tasks])
def test_stale_ref_is_removed(self):
# Stale since the collection version doesn't actually exist.
collection_v0 = ndb.Key(Library, 'collection/1', Version, 'v0.5.0')
element_key = ndb.Key(Library, 'ele/ment')
element_v1 = Version(id='v1.0.0',
sha='x',
status=Status.ready,
parent=element_key).put()
ref0 = CollectionReference.ensure(element_key, collection_v0, '^1.0.0')
collections = yield Version.collections_for_key_async(element_v1)
collection_keys = [collection.key for collection in collections]
self.assertIsNone(ref0.get())
self.assertEqual(collection_keys, [])
def test_analyze_latest(self):
library_key = Library(id='owner/repo').put()
Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
version_key = Version(id='v1.1.2', parent=library_key, sha='sha', status='ready').put()
VersionCache.update(library_key)
response = self.app.get('/task/analyze/owner/repo/True', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
content = Content.get_by_id('analysis', parent=version_key)
self.assertEqual(content.get_json(), None)
self.assertEqual(content.status, Status.pending)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('owner', 'repo', 'v1.1.2'),
], [task.url for task in tasks])
def handle_get(self, scope, package, latest=False):
self.init_library(scope, package)
if self.library is None:
self.response.set_status(404)
self.response.write('could not find library: %s' % Library.id(scope, package))
return
if latest:
version_id = Library.default_version_for_key_async(self.library.key).get_result()
if version_id:
version = Version.get_by_id(version_id, parent=self.library.key)
if version is not None:
self.trigger_analysis(version_id, version.sha, transactional=False)
else:
versions = Version.query(Version.status == Status.ready, ancestor=self.library.key).fetch()
for version in versions:
self.trigger_analysis(version.key.id(), version.sha, transactional=False)
def test_subsequent_update_triggers_version_deletion(self):
library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"old","v2.0.0":"old"}').put()
Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put()
Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put()
VersionCache.update(library_key)
self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304})
self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}')
response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.delete_task('org', 'repo', 'v0.1.0'),
], [task.url for task in tasks])
def handle_get(self, owner, repo):
self.init_library(owner, repo)
if self.library is None:
self.response.set_status(404)
self.response.write('could not find library: %s' % Library.id(owner, repo))
return
versions = Version.query(Version.status == Status.ready, ancestor=self.library.key).fetch()
for version in versions:
self.trigger_analysis(version.key.id(), version.sha, transactional=False)
def test_npm_unscoped(self):
library_key = Library(id='@@npm/package', status='ready').put()
Version(id='v1.1.1', parent=library_key, sha='sha',
status='ready').put()
response = self.app.get('/api/meta/@@npm/package/v1.1.1')
self.assertEqual(response.status_int, 200)
body = json.loads(response.normal_body)
self.assertEqual(body.get('apiKey'), '@@npm/package')
self.assertEqual(body.get('npmScope'), None)
self.assertEqual(body.get('npmPackage'), 'package')
def trigger_version_ingestion(self, tag, sha, url=None, preview=False):
version_object = Version.get_by_id(tag, parent=self.library.key)
if version_object is not None and (version_object.status == Status.ready or version_object.status == Status.pending):
# Version object is already up to date or pending
return False
Version(id=tag, parent=self.library.key, sha=sha, url=url, preview=preview).put()
task_url = util.ingest_version_task(self.owner, self.repo, tag)
util.new_task(task_url, target='manage', transactional=True)
self.trigger_analysis(tag, sha, transactional=True)
return True
def ingest_versions(self):
if not self.library.ingest_versions:
return
response = self.github.github_resource('repos', self.owner, self.repo, 'git/refs/tags', etag=self.library.tags_etag)
if response.status_code != 304:
if response.status_code != 200:
return self.error('repo tags not found (%d)' % response.status_code)
self.library.tags = response.content
self.library.tags_etag = response.headers.get('ETag', None)
self.library_dirty = True
data = json.loads(response.content)
if not isinstance(data, object):
data = []
data = [d for d in data if versiontag.is_valid(d['ref'][10:])]
if len(data) is 0:
return self.error('repo contains no valid version tags')
data.sort(lambda a, b: versiontag.compare(a['ref'][10:], b['ref'][10:]))
data_refs = [d['ref'][10:] for d in data]
self.library.tags = json.dumps(data_refs)
self.library.tags_etag = response.headers.get('ETag', None)
data.reverse()
is_newest = True
for version in data:
tag = version['ref'][10:]
if not versiontag.is_valid(tag):
continue
sha = version['object']['sha']
params = {}
if is_newest:
params["latestVersion"] = "True"
is_newest = False
version_object = Version(parent=self.library.key, id=tag, sha=sha)
version_object.put()
task_url = util.ingest_version_task(self.owner, self.repo, tag)
util.new_task(task_url, params)
util.publish_analysis_request(self.owner, self.repo, tag)
def test_analyze(self):
library_key = Library(id='owner/repo').put()
Version(id='v1.1.1', parent=library_key, sha='sha',
status='ready').put()
response = self.app.get('/task/analyze/owner/repo',
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_analysis_task('owner', 'repo', 'v1.1.1'),
], [task.url for task in tasks])
def test_stale_ref_is_removed(self):
# Stale since the collection version doesn't actually exist.
collection_v0 = ndb.Key(Library, 'collection/1', Version, 'v0.5.0')
element_key = ndb.Key(Library, 'ele/ment')
element_v1 = Version(id='v1.0.0', sha='x', status=Status.ready, parent=element_key).put()
ref0 = CollectionReference.ensure(element_key, collection_v0, '^1.0.0')
collections = yield Version.collections_for_key_async(element_v1)
collection_keys = [collection.key for collection in collections]
self.assertIsNone(ref0.get())
self.assertEqual(collection_keys, [])
def test_ingest_version(self):
library = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}', contributor_count=417)
version = Version(parent=library.key, id='v1.0.0', sha='lol')
library.put()
version.put()
self.respond_to('https://raw.githubusercontent.com/org/repo/v1.0.0/README.md', 'README')
self.respond_to('https://raw.githubusercontent.com/org/repo/v1.0.0/bower.json', '{}')
self.respond_to_github('https://api.github.com/markdown', '<html>README</html>')
response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'))
self.assertEqual(response.status_int, 200)
version = version.key.get()
self.assertIsNone(version.error)
readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme').get()
self.assertEqual(readme.content, 'README')
readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme.html').get()
self.assertEqual(readme_html.content, '<html>README</html>')
bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'bower').get()
self.assertEqual(bower.content, '{}')
def handle_get(self, owner, repo):
self.init_library(owner, repo)
if self.library is None:
self.response.set_status(404)
self.response.write('could not find library: %s' %
Library.id(owner, repo))
return
versions = Version.query(Version.status == Status.ready,
ancestor=self.library.key).fetch()
for version in versions:
self.trigger_analysis(version.key.id(),
version.sha,
transactional=False)
def test_ingest_version_pages(self):
library_key = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}').put()
Version(id='v1.0.0', parent=library_key, sha='sha').put()
self.respond_to_github(r'https://api.github.com/repos/org/repo/readme\?ref=sha', '{"content":"%s"}' % b64encode('README'))
self.respond_to('https://raw.githubusercontent.com/org/repo/sha/bower.json', '{"pages":{"custom doc":"doc.md"}}')
self.respond_to_github('https://api.github.com/markdown', '<html>README</html>')
self.respond_to_github(r'https://api.github.com/repos/org/repo/contents/doc.md\?ref=sha', '{"content":"%s", "type":"file"}' % b64encode('doc.md'))
self.respond_to_github('https://api.github.com/markdown', '<html>doc.md</html>')
response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
page = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'page-doc.md').get()
self.assertEqual(page.content, '<html>doc.md</html>')
def test_null_homepage(self):
library_key = Library(
id='owner/repo',
status='ready',
metadata=
'{"owner":{"login":"******"},"name":"repo", "license": {"spdx_id": "MIT"}, "homepage": null}'
).put()
Version(id='v1.0.0', parent=library_key, sha='sha',
status='ready').put()
response = self.app.get('/api/meta/owner/repo/v1.0.0')
self.assertEqual(response.status_int, 200)
body = json.loads(response.normal_body)
self.assertEqual(body.get('owner'), 'owner')
self.assertEqual(body.get('repo'), 'repo')
self.assertIsNone(body.get('homepage'))
def handle_get(self, owner, repo, version):
self.owner = owner
self.repo = repo
self.version = version
library_key = ndb.Key(Library, Library.id(owner, repo))
self.version_object = Version.get_by_id(version, parent=library_key)
if self.version_object is None:
return self.error('Version entity does not exist: %s/%s' % (Library.id(owner, repo), version))
self.sha = self.version_object.sha
self.version_key = self.version_object.key
self.update_readme()
self.update_bower()
self.set_ready()
def handle_get(self, owner, repo, version):
self.owner = owner
self.repo = repo
self.version = version
library_key = ndb.Key(Library, Library.id(owner, repo))
self.version_object = Version.get_by_id(version, parent=library_key)
if self.version_object is None:
return self.error('Version entity does not exist: %s/%s' %
(Library.id(owner, repo), version))
self.sha = self.version_object.sha
self.version_key = self.version_object.key
self.update_readme()
self.update_bower()
self.set_ready()
def test_ingest_commit(self):
self.respond_to_github('https://api.github.com/repos/org/repo', 'metadata bits')
self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]')
self.app.get(util.ingest_commit_task('org', 'repo', 'element'), params={'commit': 'commit-sha', 'url': 'url'})
library = Library.get_by_id('org/repo')
self.assertIsNotNone(library)
self.assertIsNone(library.error)
self.assertFalse(library.ingest_versions)
version = Version.get_by_id(parent=library.key, id='commit-sha')
self.assertEqual(version.sha, 'commit-sha')
self.assertEqual(version.url, 'url')
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].url, util.ingest_version_task('org', 'repo', 'commit-sha'))
def test_compressed(self):
library_key = Library(id='owner/repo').put()
version_key = Version(id='v1.1.1',
parent=library_key,
sha='sha',
status='ready').put()
content = Content(id='analysis',
parent=version_key,
status=Status.pending)
content.json = dict({"analyzerData": "some data"})
content.status = Status.ready
content.put()
response = self.app.get(
'/api/docs/owner/repo/v1.1.1?use_analyzer_data')
self.assertEqual(response.status_int, 200)
self.assertEqual(
json.loads(response.normal_body).get('analysis'), "some data")
def test_latest_matching_collection_version_is_returned(self):
collection_key = ndb.Key(Library, 'collection/1')
collection_v1 = Version(id='v1.0.0', sha='x', status=Status.ready, parent=collection_key).put()
collection_v2 = Version(id='v2.0.0', sha='x', status=Status.ready, parent=collection_key).put()
collection_v3 = Version(id='v3.0.0', sha='x', status=Status.ready, parent=collection_key).put()
element_key = ndb.Key(Library, 'ele/ment')
element_v1 = Version(id='v1.0.0', sha='x', status=Status.ready, parent=element_key).put()
CollectionReference.ensure(element_key, collection_v1, '^1.0.0')
CollectionReference.ensure(element_key, collection_v2, '^1.0.0')
CollectionReference.ensure(element_key, collection_v3, '^2.0.0')
collections = yield Version.collections_for_key_async(element_v1)
collection_keys = [collection.key for collection in collections]
# Only latest matching version of the collection should be present.
self.assertEqual(collection_keys, [
collection_v2,
])
def get(self, owner, repo, ver=None):
# TODO: Share all of this boilerplate between GetDataMeta and GetHydroData
self.response.headers['Access-Control-Allow-Origin'] = '*'
owner = owner.lower()
repo = repo.lower()
library_key = ndb.Key(Library, '%s/%s' % (owner, repo))
# TODO: version shouldn't be optional here
if ver is None:
versions = Version.query(ancestor=library_key).map(lambda v: v.key.id())
versions.sort(versiontag.compare)
if versions == []:
self.response.set_status(404)
return
ver = versions[-1]
version_key = ndb.Key(Library, '%s/%s' % (owner, repo), Version, ver)
hydro = Content.get_by_id('hydrolyzer', parent=version_key, read_policy=ndb.EVENTUAL_CONSISTENCY)
if hydro is None:
self.response.set_status(404)
return
self.response.headers['Content-Type'] = 'application/json'
self.response.write(hydro.content)
def test_versions_for_key(self):
library_key = ndb.Key(Library, 'a/b')
Version(id='v2.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v1.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v3.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v3.0.X', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v4.0.0', sha='x', status=Status.error,
parent=library_key).put()
Version(id='v5.0.0',
sha='x',
status=Status.pending,
parent=library_key).put()
Version(id='xxx', sha='x', status=Status.ready,
parent=library_key).put()
versions = yield Library.uncached_versions_for_key_async(library_key)
self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0'])
def post(self):
message_json = json.loads(urllib.unquote(self.request.body).rstrip('='))
message = message_json['message']
data = base64.b64decode(str(message['data']))
attributes = message['attributes']
owner = attributes['owner']
repo = attributes['repo']
version = attributes['version']
logging.info('Ingesting analysis data %s/%s/%s', owner, repo, version)
parent = Version.get_by_id(version, parent=ndb.Key(Library, '%s/%s' % (owner, repo)))
# Don't accept the analysis data unless the version still exists in the datastore
if parent is not None:
content = Content(parent=parent.key, id='analysis', content=data)
try:
content.put()
# TODO: Which exception is this for?
# pylint: disable=bare-except
except:
logging.error(sys.exc_info()[0])
self.response.set_status(200)
def test_ingest_version_falls_back(self):
library = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}', contributor_count=417)
library.tags = json.dumps(["v1.0.0", "v1.0.1"])
library.put()
version1 = Version(parent=library.key, id='v1.0.0', sha='lol')
version1.put()
version2 = Version(parent=library.key, id='v1.0.1', sha='lol')
version2.put()
self.respond_to('https://raw.githubusercontent.com/org/repo/v1.0.1/README.md', chr(248))
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 0)
self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.1'), params={'latestVersion': 'True'})
version2 = version2.key.get()
self.assertEqual(version2.error, "Could not store README.md as a utf-8 string")
tasks = self.tasks.get_filtered_tasks()
self.assertEqual(len(tasks), 1)
self.assertEqual(tasks[0].url, util.ingest_version_task('org', 'repo', 'v1.0.0') + '?latestVersion=True')
def get(self, owner, repo, kind):
if not (kind == 'element' or kind == 'collection'):
self.response.set_status(400)
return
owner = owner.lower()
repo = repo.lower()
library = Library.maybe_create_with_kind(owner, repo, kind)
logging.info('created library')
github = quota.GitHub()
if not github.reserve(3):
self.response.set_status(500)
return
response = github.github_resource('repos', owner, repo)
if not response.status_code == 200:
library.error = 'repo metadata not found'
github.release()
library.put()
return
library.metadata = response.content
response = github.github_resource('repos', owner, repo, 'contributors')
if not response.status_code == 200:
library.error = 'repo contributors not found'
github.release()
library.put()
return
library.contributors = response.content
library.contributor_count = len(json.loads(response.content))
response = github.github_resource('repos', owner, repo, 'git/refs/tags')
if not response.status_code == 200:
library.error = 'repo tags not found'
github.release()
library.put()
return
data = json.loads(response.content)
if not isinstance(data, object):
library.error = 'repo contians no valid version tags'
github.release()
library.put()
return
library.put()
for version in data:
tag = version['ref'][10:]
if not versiontag.is_valid(tag):
continue
sha = version['object']['sha']
version_object = Version(parent=library.key, id=tag, sha=sha)
version_object.put()
util.new_task('ingest/version', owner, repo, detail=tag)
util.publish_hydrolyze_pending(
'/task/ingest/hydrolyzer/%s/%s/%s' % (owner, repo, tag),
owner,
repo,
tag)
def get(self, owner, repo, kind):
if not (kind == 'element' or kind == 'collection'):
self.response.set_status(400)
return
owner = owner.lower()
repo = repo.lower()
library = Library.maybe_create_with_kind(owner, repo, kind)
library_dirty = False
if library.error is not None:
library_dirty = True
library.error = None
logging.info('created library')
github = quota.GitHub()
if not github.reserve(3):
self.response.set_status(500)
return
response = github.github_resource('repos', owner, repo, etag=library.metadata_etag)
if response.status_code != 304:
if response.status_code == 200:
library.metadata = response.content
library.metadata_etag = response.headers.get('ETag', None)
library_dirty = True
else:
library.error = 'repo metadata not found (%d)' % response.status_code
github.release()
library.put()
return
response = github.github_resource('repos', owner, repo, 'contributors', etag=library.contributors_etag)
if response.status_code != 304:
if response.status_code == 200:
library.contributors = response.content
library.contributors_etag = response.headers.get('ETag', None)
library.contributor_count = len(json.loads(response.content))
library_dirty = True
else:
library.error = 'repo contributors not found (%d)' % response.status_code
github.release()
library.put()
return
response = github.github_resource('repos', owner, repo, 'git/refs/tags', etag=library.tags_etag)
if response.status_code != 304:
if response.status_code == 200:
library.tags = response.content
library.tags_etag = response.headers.get('ETag', None)
library_dirty = True
data = json.loads(response.content)
if not isinstance(data, object):
library.error = 'repo contains no valid version tags'
github.release()
library.put()
return
for version in data:
tag = version['ref'][10:]
if not versiontag.is_valid(tag):
continue
sha = version['object']['sha']
version_object = Version(parent=library.key, id=tag, sha=sha)
version_object.put()
task_url = util.ingest_version_task(owner, repo, tag)
util.new_task(task_url)
util.publish_analysis_request(owner, repo, tag)
else:
library.error = 'repo tags not found (%d)' % response.status_code
github.release()
library.put()
return
if library_dirty:
library.put()
github.release()
def get(self, owner, repo, ver=None):
owner = owner.lower()
repo = repo.lower()
library = Library.get_by_id('%s/%s' % (owner, repo), read_policy=ndb.EVENTUAL_CONSISTENCY)
if library is None or library.error is not None:
self.response.write(str(library))
self.response.set_status(404)
return
versions = library.versions()
if ver is None:
ver = versions[-1]
version = Version.get_by_id(ver, parent=library.key, read_policy=ndb.EVENTUAL_CONSISTENCY)
if version is None or version.error is not None:
self.response.write(str(version))
self.response.set_status(404)
return
metadata = json.loads(library.metadata)
dependencies = []
bower = Content.get_by_id('bower', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY)
if bower is not None:
try:
bower_json = json.loads(bower.content)
# TODO: Which exception is this for?
# pylint: disable=bare-except
except:
bower_json = {}
readme = Content.get_by_id('readme.html', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY)
full_name_match = re.match(r'(.*)/(.*)', metadata['full_name'])
result = {
'version': ver,
'versions': versions,
'readme': None if readme is None else readme.content,
'subscribers': metadata['subscribers_count'],
'stars': metadata['stargazers_count'],
'forks': metadata['forks'],
'contributors': library.contributor_count,
'open_issues': metadata['open_issues'],
'updated_at': metadata['updated_at'],
'owner': full_name_match.groups()[0],
'repo': full_name_match.groups()[1],
'bower': None if bower is None else {
'description': bower_json.get('description', ''),
'license': bower_json.get('license', ''),
'dependencies': bower_json.get('dependencies', []),
'keywords': bower_json.get('keywords', []),
},
'collections': []
}
for collection in library.collections:
if not versiontag.match(ver, collection.semver):
continue
collection_version = collection.version.id()
collection_library = collection.version.parent().get()
collection_metadata = json.loads(collection_library.metadata)
collection_name_match = re.match(r'(.*)/(.*)', collection_metadata['full_name'])
result['collections'].append({
'owner': collection_name_match.groups()[0],
'repo': collection_name_match.groups()[1],
'version': collection_version
})
if library.kind == 'collection':
dependencies = []
version_futures = []
for dep in version.dependencies:
parsed_dep = Dependency.fromString(dep)
dep_key = ndb.Key(Library, "%s/%s" % (parsed_dep.owner.lower(), parsed_dep.repo.lower()))
version_futures.append(Library.versions_for_key_async(dep_key))
for i, dep in enumerate(version.dependencies):
parsed_dep = Dependency.fromString(dep)
versions = version_futures[i].get_result()
versions.reverse()
while len(versions) > 0 and not versiontag.match(versions[0], parsed_dep.version):
versions.pop()
if len(versions) == 0:
dependencies.append({
'error': 'unsatisfyable dependency',
'owner': parsed_dep.owner,
'repo': parsed_dep.repo,
'versionSpec': parsed_dep.version
})
else:
dependencies.append(brief_metadata_from_datastore(parsed_dep.owner, parsed_dep.repo, versions[0]))
result['dependencies'] = dependencies
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(result))
