def handle_get(self, owner, repo, latest=False): self.init_library(owner, repo) if self.library is None: self.response.set_status(404) self.response.write('could not find library: %s' % Library.id(owner, repo)) return if latest: version_id = Library.default_version_for_key_async( self.library.key).get_result() if version_id: version = Version.get_by_id(version_id, parent=self.library.key) if version is not None: self.trigger_analysis(version_id, version.sha, transactional=False) else: versions = Version.query(Version.status == Status.ready, ancestor=self.library.key).fetch() for version in versions: self.trigger_analysis(version.key.id(), version.sha, transactional=False)
def test_update_collection(self): library_key = Library(id='org/repo', tags=['v0.0.1'], collection_sequence_number=1, kind='collection', spdx_identifier='MIT').put() Version(id='v0.0.1', parent=library_key, sha="old", status=Status.ready).put() self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', """{ "ref": "refs/heads/master", "object": {"sha": "new-master-sha"} }""") response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = library_key.get() self.assertEqual(library.error, None) self.assertEqual(library.status, Status.ready) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'v0.0.2', 'new-master-sha'), util.ingest_version_task('org', 'repo', 'v0.0.2'), ], [task.url for task in tasks]) version = Version.get_by_id('v0.0.2', parent=library_key) self.assertEqual(version.sha, 'new-master-sha') self.assertEqual(version.status, Status.pending)
def test_ingest_version(self): library_key = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}').put() Version(id='v1.0.0', parent=library_key, sha='sha').put() self.respond_to_github(r'https://api.github.com/repos/org/repo/readme\?ref=sha', '{"content":"%s"}' % b64encode('README')) self.respond_to('https://raw.githubusercontent.com/org/repo/sha/bower.json', '{}') self.respond_to_github('https://api.github.com/markdown', '<html>README</html>') response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) version = Version.get_by_id('v1.0.0', parent=library_key) self.assertIsNone(version.error) self.assertEqual(version.status, Status.ready) self.assertFalse(version.preview) versions = Library.versions_for_key_async(library_key).get_result() self.assertEqual(['v1.0.0'], versions) readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme').get() self.assertEqual(readme.content, 'README') readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme.html').get() self.assertEqual(readme_html.content, '<html>README</html>') bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'bower').get() self.assertEqual(bower.get_json(), {})
def test_latest_matching_collection_version_is_returned(self): collection_key = ndb.Key(Library, 'collection/1') collection_v1 = Version(id='v1.0.0', sha='x', status=Status.ready, parent=collection_key).put() collection_v2 = Version(id='v2.0.0', sha='x', status=Status.ready, parent=collection_key).put() collection_v3 = Version(id='v3.0.0', sha='x', status=Status.ready, parent=collection_key).put() element_key = ndb.Key(Library, 'ele/ment') element_v1 = Version(id='v1.0.0', sha='x', status=Status.ready, parent=element_key).put() CollectionReference.ensure(element_key, collection_v1, '^1.0.0') CollectionReference.ensure(element_key, collection_v2, '^1.0.0') CollectionReference.ensure(element_key, collection_v3, '^2.0.0') collections = yield Version.collections_for_key_async(element_v1) collection_keys = [collection.key for collection in collections] # Only latest matching version of the collection should be present. self.assertEqual(collection_keys, [ collection_v2, ])
def test_delete_version(self): library_key = ndb.Key(Library, 'owner/repo') version_key = Version(id='v1.0.0', parent=library_key, sha='1', status=Status.ready).put() VersionCache.update(library_key) response = self.app.get('/task/delete/owner/repo/v1.0.0', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) version = version_key.get() self.assertIsNone(version) self.assertEqual(Library.versions_for_key_async(library_key).get_result(), [])
def trigger_version_ingestion(self, tag, sha, url=None, preview=False): version_object = Version.get_by_id(tag, parent=self.library.key) if version_object is not None and (version_object.status == Status.ready or version_object.status == Status.pending): # Version object is already up to date or pending return False Version(id=tag, parent=self.library.key, sha=sha, url=url, preview=preview).put() task_url = util.ingest_version_task(self.scope, self.package, tag) util.new_task(task_url, target='manage', transactional=True) self.trigger_analysis(tag, sha, transactional=True) return True
def test_version_cache(self): library_key = ndb.Key(Library, 'a/b') Version(id='v2.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v1.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v3.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v3.0.X', sha='x', status=Status.ready, parent=library_key).put() Version(id='v4.0.0', sha='x', status=Status.error, parent=library_key).put() Version(id='v5.0.0', sha='x', status=Status.pending, parent=library_key).put() Version(id='xxx', sha='x', status=Status.ready, parent=library_key).put() versions = yield Library.versions_for_key_async(library_key) self.assertEqual(versions, []) latest_changed = VersionCache.update(library_key) self.assertTrue(latest_changed) versions = yield Library.versions_for_key_async(library_key) self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0']) Version(id='v6.0.0', sha='x', status=Status.ready, parent=library_key).put() latest_changed = VersionCache.update(library_key) self.assertTrue(latest_changed) versions = yield Library.versions_for_key_async(library_key) self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0', 'v6.0.0'])
def test_update_deletes_missing_repo(self): library = Library(id='org/repo', metadata_etag='a', contributors_etag='b', tags_etag='c', spdx_identifier='MIT') library.put() version = Version(parent=library.key, id='v1.0.0', sha='lol') version.put() self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 404}) response = self.app.get('/task/update/org/repo', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) version = version.key.get() library = library.key.get() self.assertIsNone(library) self.assertIsNone(version)
def test_ingest_preview(self): self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}') response = self.app.get(util.ingest_preview_task('org', 'repo'), params={'commit': 'commit-sha', 'url': 'url'}, headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertTrue(library.shallow_ingestion) version = Version.get_by_id('commit-sha', parent=library.key) self.assertEquals(version.status, Status.pending) self.assertEquals(version.sha, 'commit-sha') self.assertEquals(version.url, 'url') self.assertTrue(version.preview) tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 2) self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'commit-sha'), util.ingest_version_task('org', 'repo', 'commit-sha'), ], [task.url for task in tasks])
def test_ingest_version(self): library_key = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}').put() Version(id='v1.0.0', parent=library_key, sha='sha').put() self.respond_to_github(r'https://api.github.com/repos/org/repo/readme\?ref=sha', '{"content":"%s"}' % b64encode('README')) self.respond_to('https://raw.githubusercontent.com/org/repo/sha/bower.json', '{}') self.respond_to_github('https://api.github.com/markdown', '<html>README</html>') response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) version = Version.get_by_id('v1.0.0', parent=library_key) self.assertIsNone(version.error) self.assertEqual(version.status, Status.ready) self.assertFalse(version.preview) versions = Library.versions_for_key_async(library_key).get_result() self.assertEqual(['v1.0.0'], versions) readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme').get() self.assertEqual(readme.content, 'README') readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme.html').get() self.assertEqual(readme_html.content, '<html>README</html>') bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'bower').get() self.assertEqual(bower.content, '{}')
def test_analyze_leaves_existing_content_when_reanalyzing(self): library_key = Library(id='owner/repo').put() version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() content = Content(id='analysis', parent=version_key, status=Status.pending) content.content = 'existing data' content.status = Status.ready content.put() response = self.app.get('/task/analyze/owner/repo', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) content = Content.get_by_id('analysis', parent=version_key) self.assertEqual(content.content, 'existing data') self.assertEqual(content.status, Status.ready) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('owner', 'repo', 'v1.1.1'), ], [task.url for task in tasks])
def get(self, owner, repo, version=None): self.response.headers['Access-Control-Allow-Origin'] = '*' self.response.headers['Content-Type'] = 'application/json' library_key = ndb.Key(Library, Library.id(owner, repo)) if version is None: version = yield Library.default_version_for_key_async(library_key) if version is None: self.response.set_status(404) return version_key = ndb.Key(Library, library_key.id(), Version, version) collection_versions = yield Version.collections_for_key_async(version_key) collection_futures = [] for collection_version in collection_versions: collection_futures.append(LibraryMetadata.brief_async(collection_version.key.parent(), collection_version.key.id())) collections = [] for future in collection_futures: collection_result = yield future if collection_result is not None: collections.append(collection_result) result = { 'results': collections, 'count': len(collections), } self.response.write(json.dumps(result))
def test_ingest_preview(self): self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github( 'https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github( 'https://api.github.com/repos/org/repo/stats/participation', '{}') self.respond_to_github( 'https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}') response = self.app.get(util.ingest_preview_task('org', 'repo'), params={ 'commit': 'commit-sha', 'url': 'url' }, headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertTrue(library.shallow_ingestion) version = Version.get_by_id('commit-sha', parent=library.key) self.assertEquals(version.status, Status.pending) self.assertEquals(version.sha, 'commit-sha') self.assertEquals(version.url, 'url') self.assertTrue(version.preview) tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 2) self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'commit-sha'), util.ingest_version_task('org', 'repo', 'commit-sha'), ], [task.url for task in tasks])
def test_update_indexes(self): metadata = """{ "full_name": "full-name" }""" collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put() collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put() Content(id='bower', parent=collection_version_key, content="""{"dependencies": { "a": "org/element-1#1.0.0", "b": "org/element-2#1.0.0" }}""").put() VersionCache.update(collection_library_key) response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) # Triggers ingestions tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ensure_library_task('org', 'element-1'), util.ensure_library_task('org', 'element-2'), ], [task.url for task in tasks]) # Ensures collection references ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1")) self.assertIsNotNone(ref1) ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2")) self.assertIsNotNone(ref2) # Validate search index index = search.Index('repo') document = index.get('my/collection') self.assertIsNotNone(document) self.assertTrue(len(document.fields) > 0)
def get(self, owner, repo, kind): commit = self.request.get('commit', None) url = self.request.get('url', None) assert commit is not None and url is not None self.init_library(owner, repo, kind) is_new = self.library.metadata is None and self.library.error is None if is_new: self.library.ingest_versions = False self.library_dirty = True self.update_metadata() version = Version(parent=self.library.key, id=commit, sha=commit, url=url) version.put() task_url = util.ingest_version_task(owner, repo, commit) util.new_task(task_url) self.commit()
def test_analyzer_index_empty(self): metadata = """{ "full_name": "full-name" }""" library_key = Library(id='owner/repo', metadata=metadata).put() version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() content = Content(id='analysis', parent=version_key, status=Status.pending) data = {"analyzerData": {}} content.json = data content.status = Status.ready content.put() VersionCache.update(library_key) response = self.app.get(util.update_indexes_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) index = search.Index('repo') document = index.get('owner/repo') self.assertIsNotNone(document) self.assertTrue(len(document.fields) > 0) elements = [field for field in document.fields if field.name == 'element'] self.assertEqual(len(elements), 0) behaviors = [field for field in document.fields if field.name == 'behavior'] self.assertEqual(len(behaviors), 0)
def test_update_doesnt_ingest_older_versions(self): library_key = Library(id='org/repo', tags=['v0.1.0', 'v1.0.0', 'v2.0.0'], spdx_identifier='MIT').put() Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put() VersionCache.update(library_key) self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304}) self.respond_to_github( 'https://api.github.com/repos/org/repo/contributors', {'status': 304}) self.respond_to_github( 'https://api.github.com/repos/org/repo/tags', """[ {"name": "v0.5.0", "commit": {"sha": "new"}}, {"name": "v1.0.0", "commit": {"sha": "old"}} ]""") self.respond_to_github( 'https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) tasks = self.tasks.get_filtered_tasks() self.assertEqual([], [task.url for task in tasks])
def test_stale_ref_is_removed(self): # Stale since the collection version doesn't actually exist. collection_v0 = ndb.Key(Library, 'collection/1', Version, 'v0.5.0') element_key = ndb.Key(Library, 'ele/ment') element_v1 = Version(id='v1.0.0', sha='x', status=Status.ready, parent=element_key).put() ref0 = CollectionReference.ensure(element_key, collection_v0, '^1.0.0') collections = yield Version.collections_for_key_async(element_v1) collection_keys = [collection.key for collection in collections] self.assertIsNone(ref0.get()) self.assertEqual(collection_keys, [])
def test_analyze_latest(self): library_key = Library(id='owner/repo').put() Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() version_key = Version(id='v1.1.2', parent=library_key, sha='sha', status='ready').put() VersionCache.update(library_key) response = self.app.get('/task/analyze/owner/repo/True', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) content = Content.get_by_id('analysis', parent=version_key) self.assertEqual(content.get_json(), None) self.assertEqual(content.status, Status.pending) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('owner', 'repo', 'v1.1.2'), ], [task.url for task in tasks])
def handle_get(self, scope, package, latest=False): self.init_library(scope, package) if self.library is None: self.response.set_status(404) self.response.write('could not find library: %s' % Library.id(scope, package)) return if latest: version_id = Library.default_version_for_key_async(self.library.key).get_result() if version_id: version = Version.get_by_id(version_id, parent=self.library.key) if version is not None: self.trigger_analysis(version_id, version.sha, transactional=False) else: versions = Version.query(Version.status == Status.ready, ancestor=self.library.key).fetch() for version in versions: self.trigger_analysis(version.key.id(), version.sha, transactional=False)
def test_subsequent_update_triggers_version_deletion(self): library_key = Library(id='org/repo', spdx_identifier='MIT', tag_map='{"v1.0.0":"old","v2.0.0":"old"}').put() Version(id='v0.1.0', parent=library_key, sha="old", status=Status.ready).put() Version(id='v1.0.0', parent=library_key, sha="old", status=Status.ready).put() Version(id='v2.0.0', parent=library_key, sha="old", status=Status.ready).put() VersionCache.update(library_key) self.respond_to_github('https://api.github.com/repos/org/repo', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/contributors', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/tags', {'status': 304}) self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.update_library_task('org/repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.delete_task('org', 'repo', 'v0.1.0'), ], [task.url for task in tasks])
def handle_get(self, owner, repo): self.init_library(owner, repo) if self.library is None: self.response.set_status(404) self.response.write('could not find library: %s' % Library.id(owner, repo)) return versions = Version.query(Version.status == Status.ready, ancestor=self.library.key).fetch() for version in versions: self.trigger_analysis(version.key.id(), version.sha, transactional=False)
def test_npm_unscoped(self): library_key = Library(id='@@npm/package', status='ready').put() Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() response = self.app.get('/api/meta/@@npm/package/v1.1.1') self.assertEqual(response.status_int, 200) body = json.loads(response.normal_body) self.assertEqual(body.get('apiKey'), '@@npm/package') self.assertEqual(body.get('npmScope'), None) self.assertEqual(body.get('npmPackage'), 'package')
def trigger_version_ingestion(self, tag, sha, url=None, preview=False): version_object = Version.get_by_id(tag, parent=self.library.key) if version_object is not None and (version_object.status == Status.ready or version_object.status == Status.pending): # Version object is already up to date or pending return False Version(id=tag, parent=self.library.key, sha=sha, url=url, preview=preview).put() task_url = util.ingest_version_task(self.owner, self.repo, tag) util.new_task(task_url, target='manage', transactional=True) self.trigger_analysis(tag, sha, transactional=True) return True
def ingest_versions(self): if not self.library.ingest_versions: return response = self.github.github_resource('repos', self.owner, self.repo, 'git/refs/tags', etag=self.library.tags_etag) if response.status_code != 304: if response.status_code != 200: return self.error('repo tags not found (%d)' % response.status_code) self.library.tags = response.content self.library.tags_etag = response.headers.get('ETag', None) self.library_dirty = True data = json.loads(response.content) if not isinstance(data, object): data = [] data = [d for d in data if versiontag.is_valid(d['ref'][10:])] if len(data) is 0: return self.error('repo contains no valid version tags') data.sort(lambda a, b: versiontag.compare(a['ref'][10:], b['ref'][10:])) data_refs = [d['ref'][10:] for d in data] self.library.tags = json.dumps(data_refs) self.library.tags_etag = response.headers.get('ETag', None) data.reverse() is_newest = True for version in data: tag = version['ref'][10:] if not versiontag.is_valid(tag): continue sha = version['object']['sha'] params = {} if is_newest: params["latestVersion"] = "True" is_newest = False version_object = Version(parent=self.library.key, id=tag, sha=sha) version_object.put() task_url = util.ingest_version_task(self.owner, self.repo, tag) util.new_task(task_url, params) util.publish_analysis_request(self.owner, self.repo, tag)
def test_analyze(self): library_key = Library(id='owner/repo').put() Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() response = self.app.get('/task/analyze/owner/repo', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('owner', 'repo', 'v1.1.1'), ], [task.url for task in tasks])
def test_ingest_version(self): library = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}', contributor_count=417) version = Version(parent=library.key, id='v1.0.0', sha='lol') library.put() version.put() self.respond_to('https://raw.githubusercontent.com/org/repo/v1.0.0/README.md', 'README') self.respond_to('https://raw.githubusercontent.com/org/repo/v1.0.0/bower.json', '{}') self.respond_to_github('https://api.github.com/markdown', '<html>README</html>') response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0')) self.assertEqual(response.status_int, 200) version = version.key.get() self.assertIsNone(version.error) readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme').get() self.assertEqual(readme.content, 'README') readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme.html').get() self.assertEqual(readme_html.content, '<html>README</html>') bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'bower').get() self.assertEqual(bower.content, '{}')
def test_ingest_version_pages(self): library_key = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}').put() Version(id='v1.0.0', parent=library_key, sha='sha').put() self.respond_to_github(r'https://api.github.com/repos/org/repo/readme\?ref=sha', '{"content":"%s"}' % b64encode('README')) self.respond_to('https://raw.githubusercontent.com/org/repo/sha/bower.json', '{"pages":{"custom doc":"doc.md"}}') self.respond_to_github('https://api.github.com/markdown', '<html>README</html>') self.respond_to_github(r'https://api.github.com/repos/org/repo/contents/doc.md\?ref=sha', '{"content":"%s", "type":"file"}' % b64encode('doc.md')) self.respond_to_github('https://api.github.com/markdown', '<html>doc.md</html>') response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) page = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'page-doc.md').get() self.assertEqual(page.content, '<html>doc.md</html>')
def test_null_homepage(self): library_key = Library( id='owner/repo', status='ready', metadata= '{"owner":{"login":"******"},"name":"repo", "license": {"spdx_id": "MIT"}, "homepage": null}' ).put() Version(id='v1.0.0', parent=library_key, sha='sha', status='ready').put() response = self.app.get('/api/meta/owner/repo/v1.0.0') self.assertEqual(response.status_int, 200) body = json.loads(response.normal_body) self.assertEqual(body.get('owner'), 'owner') self.assertEqual(body.get('repo'), 'repo') self.assertIsNone(body.get('homepage'))
def handle_get(self, owner, repo, version): self.owner = owner self.repo = repo self.version = version library_key = ndb.Key(Library, Library.id(owner, repo)) self.version_object = Version.get_by_id(version, parent=library_key) if self.version_object is None: return self.error('Version entity does not exist: %s/%s' % (Library.id(owner, repo), version)) self.sha = self.version_object.sha self.version_key = self.version_object.key self.update_readme() self.update_bower() self.set_ready()
def test_ingest_commit(self): self.respond_to_github('https://api.github.com/repos/org/repo', 'metadata bits') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.app.get(util.ingest_commit_task('org', 'repo', 'element'), params={'commit': 'commit-sha', 'url': 'url'}) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertFalse(library.ingest_versions) version = Version.get_by_id(parent=library.key, id='commit-sha') self.assertEqual(version.sha, 'commit-sha') self.assertEqual(version.url, 'url') tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].url, util.ingest_version_task('org', 'repo', 'commit-sha'))
def test_compressed(self): library_key = Library(id='owner/repo').put() version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() content = Content(id='analysis', parent=version_key, status=Status.pending) content.json = dict({"analyzerData": "some data"}) content.status = Status.ready content.put() response = self.app.get( '/api/docs/owner/repo/v1.1.1?use_analyzer_data') self.assertEqual(response.status_int, 200) self.assertEqual( json.loads(response.normal_body).get('analysis'), "some data")
def get(self, owner, repo, ver=None): # TODO: Share all of this boilerplate between GetDataMeta and GetHydroData self.response.headers['Access-Control-Allow-Origin'] = '*' owner = owner.lower() repo = repo.lower() library_key = ndb.Key(Library, '%s/%s' % (owner, repo)) # TODO: version shouldn't be optional here if ver is None: versions = Version.query(ancestor=library_key).map(lambda v: v.key.id()) versions.sort(versiontag.compare) if versions == []: self.response.set_status(404) return ver = versions[-1] version_key = ndb.Key(Library, '%s/%s' % (owner, repo), Version, ver) hydro = Content.get_by_id('hydrolyzer', parent=version_key, read_policy=ndb.EVENTUAL_CONSISTENCY) if hydro is None: self.response.set_status(404) return self.response.headers['Content-Type'] = 'application/json' self.response.write(hydro.content)
def test_versions_for_key(self): library_key = ndb.Key(Library, 'a/b') Version(id='v2.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v1.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v3.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v3.0.X', sha='x', status=Status.ready, parent=library_key).put() Version(id='v4.0.0', sha='x', status=Status.error, parent=library_key).put() Version(id='v5.0.0', sha='x', status=Status.pending, parent=library_key).put() Version(id='xxx', sha='x', status=Status.ready, parent=library_key).put() versions = yield Library.uncached_versions_for_key_async(library_key) self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0'])
def post(self): message_json = json.loads(urllib.unquote(self.request.body).rstrip('=')) message = message_json['message'] data = base64.b64decode(str(message['data'])) attributes = message['attributes'] owner = attributes['owner'] repo = attributes['repo'] version = attributes['version'] logging.info('Ingesting analysis data %s/%s/%s', owner, repo, version) parent = Version.get_by_id(version, parent=ndb.Key(Library, '%s/%s' % (owner, repo))) # Don't accept the analysis data unless the version still exists in the datastore if parent is not None: content = Content(parent=parent.key, id='analysis', content=data) try: content.put() # TODO: Which exception is this for? # pylint: disable=bare-except except: logging.error(sys.exc_info()[0]) self.response.set_status(200)
def test_ingest_version_falls_back(self): library = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}', contributor_count=417) library.tags = json.dumps(["v1.0.0", "v1.0.1"]) library.put() version1 = Version(parent=library.key, id='v1.0.0', sha='lol') version1.put() version2 = Version(parent=library.key, id='v1.0.1', sha='lol') version2.put() self.respond_to('https://raw.githubusercontent.com/org/repo/v1.0.1/README.md', chr(248)) tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 0) self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.1'), params={'latestVersion': 'True'}) version2 = version2.key.get() self.assertEqual(version2.error, "Could not store README.md as a utf-8 string") tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].url, util.ingest_version_task('org', 'repo', 'v1.0.0') + '?latestVersion=True')
def get(self, owner, repo, kind): if not (kind == 'element' or kind == 'collection'): self.response.set_status(400) return owner = owner.lower() repo = repo.lower() library = Library.maybe_create_with_kind(owner, repo, kind) logging.info('created library') github = quota.GitHub() if not github.reserve(3): self.response.set_status(500) return response = github.github_resource('repos', owner, repo) if not response.status_code == 200: library.error = 'repo metadata not found' github.release() library.put() return library.metadata = response.content response = github.github_resource('repos', owner, repo, 'contributors') if not response.status_code == 200: library.error = 'repo contributors not found' github.release() library.put() return library.contributors = response.content library.contributor_count = len(json.loads(response.content)) response = github.github_resource('repos', owner, repo, 'git/refs/tags') if not response.status_code == 200: library.error = 'repo tags not found' github.release() library.put() return data = json.loads(response.content) if not isinstance(data, object): library.error = 'repo contians no valid version tags' github.release() library.put() return library.put() for version in data: tag = version['ref'][10:] if not versiontag.is_valid(tag): continue sha = version['object']['sha'] version_object = Version(parent=library.key, id=tag, sha=sha) version_object.put() util.new_task('ingest/version', owner, repo, detail=tag) util.publish_hydrolyze_pending( '/task/ingest/hydrolyzer/%s/%s/%s' % (owner, repo, tag), owner, repo, tag)
def get(self, owner, repo, kind): if not (kind == 'element' or kind == 'collection'): self.response.set_status(400) return owner = owner.lower() repo = repo.lower() library = Library.maybe_create_with_kind(owner, repo, kind) library_dirty = False if library.error is not None: library_dirty = True library.error = None logging.info('created library') github = quota.GitHub() if not github.reserve(3): self.response.set_status(500) return response = github.github_resource('repos', owner, repo, etag=library.metadata_etag) if response.status_code != 304: if response.status_code == 200: library.metadata = response.content library.metadata_etag = response.headers.get('ETag', None) library_dirty = True else: library.error = 'repo metadata not found (%d)' % response.status_code github.release() library.put() return response = github.github_resource('repos', owner, repo, 'contributors', etag=library.contributors_etag) if response.status_code != 304: if response.status_code == 200: library.contributors = response.content library.contributors_etag = response.headers.get('ETag', None) library.contributor_count = len(json.loads(response.content)) library_dirty = True else: library.error = 'repo contributors not found (%d)' % response.status_code github.release() library.put() return response = github.github_resource('repos', owner, repo, 'git/refs/tags', etag=library.tags_etag) if response.status_code != 304: if response.status_code == 200: library.tags = response.content library.tags_etag = response.headers.get('ETag', None) library_dirty = True data = json.loads(response.content) if not isinstance(data, object): library.error = 'repo contains no valid version tags' github.release() library.put() return for version in data: tag = version['ref'][10:] if not versiontag.is_valid(tag): continue sha = version['object']['sha'] version_object = Version(parent=library.key, id=tag, sha=sha) version_object.put() task_url = util.ingest_version_task(owner, repo, tag) util.new_task(task_url) util.publish_analysis_request(owner, repo, tag) else: library.error = 'repo tags not found (%d)' % response.status_code github.release() library.put() return if library_dirty: library.put() github.release()
def get(self, owner, repo, ver=None): owner = owner.lower() repo = repo.lower() library = Library.get_by_id('%s/%s' % (owner, repo), read_policy=ndb.EVENTUAL_CONSISTENCY) if library is None or library.error is not None: self.response.write(str(library)) self.response.set_status(404) return versions = library.versions() if ver is None: ver = versions[-1] version = Version.get_by_id(ver, parent=library.key, read_policy=ndb.EVENTUAL_CONSISTENCY) if version is None or version.error is not None: self.response.write(str(version)) self.response.set_status(404) return metadata = json.loads(library.metadata) dependencies = [] bower = Content.get_by_id('bower', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY) if bower is not None: try: bower_json = json.loads(bower.content) # TODO: Which exception is this for? # pylint: disable=bare-except except: bower_json = {} readme = Content.get_by_id('readme.html', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY) full_name_match = re.match(r'(.*)/(.*)', metadata['full_name']) result = { 'version': ver, 'versions': versions, 'readme': None if readme is None else readme.content, 'subscribers': metadata['subscribers_count'], 'stars': metadata['stargazers_count'], 'forks': metadata['forks'], 'contributors': library.contributor_count, 'open_issues': metadata['open_issues'], 'updated_at': metadata['updated_at'], 'owner': full_name_match.groups()[0], 'repo': full_name_match.groups()[1], 'bower': None if bower is None else { 'description': bower_json.get('description', ''), 'license': bower_json.get('license', ''), 'dependencies': bower_json.get('dependencies', []), 'keywords': bower_json.get('keywords', []), }, 'collections': [] } for collection in library.collections: if not versiontag.match(ver, collection.semver): continue collection_version = collection.version.id() collection_library = collection.version.parent().get() collection_metadata = json.loads(collection_library.metadata) collection_name_match = re.match(r'(.*)/(.*)', collection_metadata['full_name']) result['collections'].append({ 'owner': collection_name_match.groups()[0], 'repo': collection_name_match.groups()[1], 'version': collection_version }) if library.kind == 'collection': dependencies = [] version_futures = [] for dep in version.dependencies: parsed_dep = Dependency.fromString(dep) dep_key = ndb.Key(Library, "%s/%s" % (parsed_dep.owner.lower(), parsed_dep.repo.lower())) version_futures.append(Library.versions_for_key_async(dep_key)) for i, dep in enumerate(version.dependencies): parsed_dep = Dependency.fromString(dep) versions = version_futures[i].get_result() versions.reverse() while len(versions) > 0 and not versiontag.match(versions[0], parsed_dep.version): versions.pop() if len(versions) == 0: dependencies.append({ 'error': 'unsatisfyable dependency', 'owner': parsed_dep.owner, 'repo': parsed_dep.repo, 'versionSpec': parsed_dep.version }) else: dependencies.append(brief_metadata_from_datastore(parsed_dep.owner, parsed_dep.repo, versions[0])) result['dependencies'] = dependencies self.response.headers['Access-Control-Allow-Origin'] = '*' self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(result))