def test_version_cache(self): library_key = ndb.Key(Library, 'a/b') Version(id='v2.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v1.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v3.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v3.0.X', sha='x', status=Status.ready, parent=library_key).put() Version(id='v4.0.0', sha='x', status=Status.error, parent=library_key).put() Version(id='v5.0.0', sha='x', status=Status.pending, parent=library_key).put() Version(id='xxx', sha='x', status=Status.ready, parent=library_key).put() versions = yield Library.versions_for_key_async(library_key) self.assertEqual(versions, []) latest_changed = VersionCache.update(library_key) self.assertTrue(latest_changed) versions = yield Library.versions_for_key_async(library_key) self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0']) Version(id='v6.0.0', sha='x', status=Status.ready, parent=library_key).put() latest_changed = VersionCache.update(library_key) self.assertTrue(latest_changed) versions = yield Library.versions_for_key_async(library_key) self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0', 'v6.0.0'])
def test_ingest_version(self): library_key = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}').put() Version(id='v1.0.0', parent=library_key, sha='sha').put() self.respond_to_github(r'https://api.github.com/repos/org/repo/readme\?ref=sha', '{"content":"%s"}' % b64encode('README')) self.respond_to('https://raw.githubusercontent.com/org/repo/sha/bower.json', '{}') self.respond_to_github('https://api.github.com/markdown', '<html>README</html>') response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) version = Version.get_by_id('v1.0.0', parent=library_key) self.assertIsNone(version.error) self.assertEqual(version.status, Status.ready) self.assertFalse(version.preview) versions = Library.versions_for_key_async(library_key).get_result() self.assertEqual(['v1.0.0'], versions) readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme').get() self.assertEqual(readme.content, 'README') readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme.html').get() self.assertEqual(readme_html.content, '<html>README</html>') bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'bower').get() self.assertEqual(bower.get_json(), {})
def test_ingest_version(self): library_key = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}').put() Version(id='v1.0.0', parent=library_key, sha='sha').put() self.respond_to_github(r'https://api.github.com/repos/org/repo/readme\?ref=sha', '{"content":"%s"}' % b64encode('README')) self.respond_to('https://raw.githubusercontent.com/org/repo/sha/bower.json', '{}') self.respond_to_github('https://api.github.com/markdown', '<html>README</html>') response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) version = Version.get_by_id('v1.0.0', parent=library_key) self.assertIsNone(version.error) self.assertEqual(version.status, Status.ready) self.assertFalse(version.preview) versions = Library.versions_for_key_async(library_key).get_result() self.assertEqual(['v1.0.0'], versions) readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme').get() self.assertEqual(readme.content, 'README') readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme.html').get() self.assertEqual(readme_html.content, '<html>README</html>') bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'bower').get() self.assertEqual(bower.content, '{}')
def get(self, owner, repo, version=None): self.response.headers['Access-Control-Allow-Origin'] = '*' self.response.headers['Content-Type'] = 'application/json' library_key = ndb.Key(Library, Library.id(owner, repo)) if version is None: version = yield Library.default_version_for_key_async(library_key) if version is None: self.response.set_status(404) return version_key = ndb.Key(Library, library_key.id(), Version, version) bower = yield Content.get_by_id_async('bower', parent=version_key) if bower is None: self.response.set_status(404) return bower_json = bower.get_json() bower_dependencies = bower_json.get('dependencies', {}) dependencies = [] version_futures = [] for name in bower_dependencies.keys(): dependency = Dependency.from_string(bower_dependencies[name]) if dependency is None: continue dependencies.append(dependency) dependency_library_key = ndb.Key(Library, Library.id(dependency.owner, dependency.repo)) version_futures.append(Library.versions_for_key_async(dependency_library_key)) dependency_futures = [] for i, dependency in enumerate(dependencies): versions = yield version_futures[i] def matches(version, spec): try: return versiontag.match(version, spec) except ValueError: # FIXME: What other cases do we need to support here? return False while len(versions) > 0 and not matches(versions[-1], dependency.version): versions.pop() if len(versions) > 0: dependency_library_key = ndb.Key(Library, Library.id(dependency.owner.lower(), dependency.repo.lower())) dependency_futures.append(LibraryMetadata.brief_async(dependency_library_key, versions[-1])) results = [] for future in dependency_futures: dependency_result = yield future if dependency_result is not None: results.append(dependency_result) result = { 'results': results, 'count': len(results), } self.response.write(json.dumps(result))
def test_delete_version(self): library_key = ndb.Key(Library, 'owner/repo') version_key = Version(id='v1.0.0', parent=library_key, sha='1', status=Status.ready).put() VersionCache.update(library_key) response = self.app.get('/task/delete/owner/repo/v1.0.0', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) version = version_key.get() self.assertIsNone(version) self.assertEqual(Library.versions_for_key_async(library_key).get_result(), [])
def test_delete_version(self): library_key = ndb.Key(Library, 'owner/repo') version_key = Version(id='v1.0.0', parent=library_key, sha='1', status=Status.ready).put() VersionCache.update(library_key) response = self.app.get('/task/delete/owner/repo/v1.0.0', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) version = version_key.get() self.assertIsNone(version) self.assertEqual(Library.versions_for_key_async(library_key).get_result(), [])
def update_versions(self): if self.library.shallow_ingestion: return if self.library.kind == 'collection': new_tag_map = self.update_collection_tags() else: assert self.library.kind == 'element' new_tag_map = self.update_element_tags() new_tags = new_tag_map.keys() ingested_tags = Library.versions_for_key_async( self.library.key).get_result() logging.info('%d of %d tags ingested', len(ingested_tags), len(new_tags)) tags_to_add = list(set(new_tags) - set(ingested_tags)) tags_to_add.sort(versiontag.compare) if ingested_tags == [] and len(tags_to_add) > 0: # Only ingest the default version if we're doing ingestion for the first time. tags_to_add = [versiontag.default_version(tags_to_add)] else: tags_to_add = [ tag for tag in tags_to_add if versiontag.compare( tag, versiontag.default_version(ingested_tags)) > 0 ] tags_to_delete = list(set(ingested_tags) - set(new_tags)) logging.info('%d adds and %d deletes pending', len(tags_to_add), len(tags_to_delete)) # To avoid running into limits on the number of tasks (5) that can be spawned transactionally # only ingest (2 tasks) or delete (1 task) one version per update. if len(tags_to_add) > 0: # Ingest from newest to oldest. tag = tags_to_add[-1] if self.trigger_version_ingestion(tag, new_tag_map[tag]): if self.library.kind == 'collection': logging.info('ingesting new collection version (%s)', tag) else: logging.info('ingesting new %s version (%s)', versiontag.categorize(tag, ingested_tags), tag) elif len(tags_to_delete) > 0: tag = tags_to_delete[0] self.trigger_version_deletion(tags_to_delete[0]) if len(new_tags) is 0: return self.error("couldn't find any tagged versions", ErrorCodes.Library_no_version)
def test_version_cache(self): library_key = ndb.Key(Library, 'a/b') Version(id='v2.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v1.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v3.0.0', sha='x', status=Status.ready, parent=library_key).put() Version(id='v3.0.X', sha='x', status=Status.ready, parent=library_key).put() Version(id='v4.0.0', sha='x', status=Status.error, parent=library_key).put() Version(id='v5.0.0', sha='x', status=Status.pending, parent=library_key).put() Version(id='xxx', sha='x', status=Status.ready, parent=library_key).put() versions = yield Library.versions_for_key_async(library_key) self.assertEqual(versions, []) latest_changed = VersionCache.update(library_key) self.assertTrue(latest_changed) versions = yield Library.versions_for_key_async(library_key) self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0']) Version(id='v6.0.0', sha='x', status=Status.ready, parent=library_key).put() latest_changed = VersionCache.update(library_key) self.assertTrue(latest_changed) versions = yield Library.versions_for_key_async(library_key) self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0', 'v6.0.0'])
def update_versions(self): if self.library.shallow_ingestion: return if self.library.kind == 'collection': new_tag_map = self.update_collection_tags() elif self.scope.startswith('@'): assert self.library.kind == 'element' new_tag_map = self.update_package_tags() else: assert self.library.kind == 'element' new_tag_map = self.update_element_tags() new_tags = new_tag_map.keys() ingested_tags = Library.versions_for_key_async(self.library.key).get_result() logging.info('%d of %d tags ingested', len(ingested_tags), len(new_tags)) tags_to_add = list(set(new_tags) - set(ingested_tags)) tags_to_add.sort(versiontag.compare) if ingested_tags == [] and len(tags_to_add) > 0: # Only ingest the default version if we're doing ingestion for the first time. tags_to_add = [versiontag.default_version(tags_to_add)] else: tags_to_add = [tag for tag in tags_to_add if versiontag.compare(tag, versiontag.default_version(ingested_tags)) > 0] tags_to_delete = list(set(ingested_tags) - set(new_tags)) logging.info('%d adds and %d deletes pending', len(tags_to_add), len(tags_to_delete)) # To avoid running into limits on the number of tasks (5) that can be spawned transactionally # only ingest (2 tasks) or delete (1 task) one version per update. if len(tags_to_add) > 0: # Ingest from newest to oldest. tag = tags_to_add[-1] if self.trigger_version_ingestion(tag, new_tag_map[tag]): if self.library.kind == 'collection': logging.info('ingesting new collection version (%s)', tag) else: logging.info('ingesting new %s version (%s)', versiontag.categorize(tag, ingested_tags), tag) elif len(tags_to_delete) > 0: tag = tags_to_delete[0] self.trigger_version_deletion(tags_to_delete[0]) if len(new_tags) is 0: return self.error("couldn't find any tagged versions", ErrorCodes.Library_no_version)
def full_async(library_key, tag=None, brief=False, assume_latest=False): if assume_latest: assert tag is not None library_future = library_key.get_async() if tag is None or not brief or not assume_latest: versions_future = Library.versions_for_key_async(library_key) if tag is None: versions = yield versions_future default_version = versiontag.default_version(versions) version_key = None if len(versions) == 0 else ndb.Key(Library, library_key.id(), Version, default_version) else: version_key = ndb.Key(Library, library_key.id(), Version, tag) if version_key is not None: version_future = version_key.get_async() bower_future = Content.get_by_id_async('bower', parent=version_key) if not brief: readme_future = Content.get_by_id_async('readme.html', parent=version_key) library = yield library_future if library is None or library.status == Status.suppressed: raise ndb.Return(None) result = {} # Add NPM package fields key = library_key.string_id() if key.startswith('@'): parts = key.split('/') if parts[0] != '@@npm': result['npmScope'] = parts[0] result['npmFullPackage'] = key else: result['npmFullPackage'] = parts[1] result['npmPackage'] = parts[1] if library.migrated_from_bower: result['migratedFromBower'] = True elif library.npm_package: result['migratedToNpm'] = library.npm_package result['apiKey'] = key result['kind'] = library.kind result['status'] = library.status if library.status != Status.ready: if library.status == Status.error: result['error'] = library.error raise ndb.Return(result) version = None if version_key is not None: version = yield version_future if version is None: raise ndb.Return(None) result['spdx_identifier'] = library.spdx_identifier result['version'] = version.key.id() if version.status != Status.ready: result['status'] = version.status if version.status == Status.error: result['error'] = version.error raise ndb.Return(result) if not brief or not assume_latest: versions = yield versions_future result['versions'] = versions if len(versions) > 0: result['default_version'] = versiontag.default_version(versions) # Remove latest_version once deployed clients all use default_version result['latest_version'] = versiontag.default_version(versions) if not brief and library.participation is not None: result['activity'] = json.loads(library.participation).get('all', []) if not brief and library.contributors is not None: contributors = [] raw = json.loads(library.contributors) for contributor in raw: contributors.append({ 'login': contributor['login'], 'avatar_url': contributor['avatar_url'], 'contributions': contributor['contributions'], }) result['contributors'] = contributors if library.metadata is not None: metadata = json.loads(library.metadata) result['description'] = metadata.get('description', '') result['subscribers'] = metadata.get('subscribers_count', 0) result['stars'] = metadata.get('stargazers_count', 0) result['forks'] = metadata.get('forks', 0) result['open_issues'] = metadata.get('open_issues', 0) result['updated_at'] = metadata.get('updated_at', 0) result['owner'] = metadata['owner']['login'] result['avatar_url'] = metadata['owner'].get('avatar_url', '') result['repo'] = metadata['name'] if metadata.get('homepage') and re.match(r'https?', metadata.get('homepage')): result['homepage'] = metadata['homepage'] result['default_branch'] = metadata.get('default_branch', 'master') if not brief: readme = yield readme_future result['readme'] = None if readme is None else readme.content bower = yield bower_future if bower is not None: bower_json = bower.get_json() dependencies = bower_json.get('dependencies', {}) result['dependency_count'] = len(dependencies) result['bower'] = { 'license': bower_json.get('license', ''), 'dependencies': dependencies, 'keywords': bower_json.get('keywords', []), 'demos': bower_json.get('demos', {}), 'pages': bower_json.get('pages', {}), } if result.get('description', '') == '': result['description'] = bower_json.get('description', '') raise ndb.Return(result)
def full_async(library_key, tag=None, brief=False, assume_latest=False): if assume_latest: assert tag is not None library_future = library_key.get_async() if tag is None or not brief or not assume_latest: versions_future = Library.versions_for_key_async(library_key) if tag is None: versions = yield versions_future default_version = versiontag.default_version(versions) version_key = None if len(versions) == 0 else ndb.Key( Library, library_key.id(), Version, default_version) else: version_key = ndb.Key(Library, library_key.id(), Version, tag) if version_key is not None: version_future = version_key.get_async() bower_future = Content.get_by_id_async('bower', parent=version_key) if not brief: readme_future = Content.get_by_id_async('readme.html', parent=version_key) library = yield library_future if library is None or library.status == Status.suppressed: raise ndb.Return(None) result = {} result['kind'] = library.kind result['status'] = library.status if library.status != Status.ready: if library.status == Status.error: result['error'] = library.error raise ndb.Return(result) version = None if version_key is not None: version = yield version_future if version is None: raise ndb.Return(None) result['spdx_identifier'] = library.spdx_identifier result['version'] = version.key.id() if version.status != Status.ready: result['status'] = version.status if version.status == Status.error: result['error'] = version.error raise ndb.Return(result) if not brief or not assume_latest: versions = yield versions_future result['versions'] = versions if len(versions) > 0: result['default_version'] = versiontag.default_version( versions) # Remove latest_version once deployed clients all use default_version result['latest_version'] = versiontag.default_version(versions) if not brief and library.participation is not None: result['activity'] = json.loads(library.participation).get( 'all', []) if not brief and library.contributors is not None: contributors = [] raw = json.loads(library.contributors) for contributor in raw: contributors.append({ 'login': contributor['login'], 'avatar_url': contributor['avatar_url'], 'contributions': contributor['contributions'], }) result['contributors'] = contributors if library.metadata is not None: metadata = json.loads(library.metadata) result['description'] = metadata['description'] result['subscribers'] = metadata['subscribers_count'] result['stars'] = metadata['stargazers_count'] result['forks'] = metadata['forks'] result['open_issues'] = metadata['open_issues'] result['updated_at'] = metadata['updated_at'] result['owner'] = metadata['owner']['login'] result['avatar_url'] = metadata['owner']['avatar_url'] result['repo'] = metadata['name'] result['homepage'] = metadata['homepage'] result['default_branch'] = metadata['default_branch'] if not brief: readme = yield readme_future result['readme'] = None if readme is None else readme.content bower = yield bower_future if bower is not None: bower_json = json.loads(bower.content) dependencies = bower_json.get('dependencies', []) result['dependency_count'] = len(dependencies) result['bower'] = { 'license': bower_json.get('license', ''), 'dependencies': dependencies, 'keywords': bower_json.get('keywords', []), } if result.get('description', '') == '': result['description'] = bower_json.get('description', '') raise ndb.Return(result)
def get(self, owner, repo, ver=None): owner = owner.lower() repo = repo.lower() library = Library.get_by_id('%s/%s' % (owner, repo), read_policy=ndb.EVENTUAL_CONSISTENCY) if library is None or library.error is not None: self.response.write(str(library)) self.response.set_status(404) return versions = library.versions() if ver is None: ver = versions[-1] version = Version.get_by_id(ver, parent=library.key, read_policy=ndb.EVENTUAL_CONSISTENCY) if version is None or version.error is not None: self.response.write(str(version)) self.response.set_status(404) return metadata = json.loads(library.metadata) dependencies = [] bower = Content.get_by_id('bower', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY) if bower is not None: try: bower_json = json.loads(bower.content) # TODO: Which exception is this for? # pylint: disable=bare-except except: bower_json = {} readme = Content.get_by_id('readme.html', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY) full_name_match = re.match(r'(.*)/(.*)', metadata['full_name']) result = { 'version': ver, 'versions': versions, 'readme': None if readme is None else readme.content, 'subscribers': metadata['subscribers_count'], 'stars': metadata['stargazers_count'], 'forks': metadata['forks'], 'contributors': library.contributor_count, 'open_issues': metadata['open_issues'], 'updated_at': metadata['updated_at'], 'owner': full_name_match.groups()[0], 'repo': full_name_match.groups()[1], 'bower': None if bower is None else { 'description': bower_json.get('description', ''), 'license': bower_json.get('license', ''), 'dependencies': bower_json.get('dependencies', []), 'keywords': bower_json.get('keywords', []), }, 'collections': [] } for collection in library.collections: if not versiontag.match(ver, collection.semver): continue collection_version = collection.version.id() collection_library = collection.version.parent().get() collection_metadata = json.loads(collection_library.metadata) collection_name_match = re.match(r'(.*)/(.*)', collection_metadata['full_name']) result['collections'].append({ 'owner': collection_name_match.groups()[0], 'repo': collection_name_match.groups()[1], 'version': collection_version }) if library.kind == 'collection': dependencies = [] version_futures = [] for dep in version.dependencies: parsed_dep = Dependency.fromString(dep) dep_key = ndb.Key(Library, "%s/%s" % (parsed_dep.owner.lower(), parsed_dep.repo.lower())) version_futures.append(Library.versions_for_key_async(dep_key)) for i, dep in enumerate(version.dependencies): parsed_dep = Dependency.fromString(dep) versions = version_futures[i].get_result() versions.reverse() while len(versions) > 0 and not versiontag.match(versions[0], parsed_dep.version): versions.pop() if len(versions) == 0: dependencies.append({ 'error': 'unsatisfyable dependency', 'owner': parsed_dep.owner, 'repo': parsed_dep.repo, 'versionSpec': parsed_dep.version }) else: dependencies.append(brief_metadata_from_datastore(parsed_dep.owner, parsed_dep.repo, versions[0])) result['dependencies'] = dependencies self.response.headers['Access-Control-Allow-Origin'] = '*' self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(result))