def test_version_cache(self):
library_key = ndb.Key(Library, 'a/b')
Version(id='v2.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v1.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v3.0.0', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v3.0.X', sha='x', status=Status.ready,
parent=library_key).put()
Version(id='v4.0.0', sha='x', status=Status.error,
parent=library_key).put()
Version(id='v5.0.0',
sha='x',
status=Status.pending,
parent=library_key).put()
Version(id='xxx', sha='x', status=Status.ready,
parent=library_key).put()
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, [])
latest_changed = VersionCache.update(library_key)
self.assertTrue(latest_changed)
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0'])
Version(id='v6.0.0', sha='x', status=Status.ready,
parent=library_key).put()
latest_changed = VersionCache.update(library_key)
self.assertTrue(latest_changed)
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions,
['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0', 'v6.0.0'])
def test_ingest_version(self):
library_key = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}').put()
Version(id='v1.0.0', parent=library_key, sha='sha').put()
self.respond_to_github(r'https://api.github.com/repos/org/repo/readme\?ref=sha', '{"content":"%s"}' % b64encode('README'))
self.respond_to('https://raw.githubusercontent.com/org/repo/sha/bower.json', '{}')
self.respond_to_github('https://api.github.com/markdown', '<html>README</html>')
response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = Version.get_by_id('v1.0.0', parent=library_key)
self.assertIsNone(version.error)
self.assertEqual(version.status, Status.ready)
self.assertFalse(version.preview)
versions = Library.versions_for_key_async(library_key).get_result()
self.assertEqual(['v1.0.0'], versions)
readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme').get()
self.assertEqual(readme.content, 'README')
readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme.html').get()
self.assertEqual(readme_html.content, '<html>README</html>')
bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'bower').get()
self.assertEqual(bower.get_json(), {})
def test_ingest_version(self):
library_key = Library(id='org/repo', metadata='{"full_name": "NSS Bob", "stargazers_count": 420, "subscribers_count": 419, "forks": 418, "updated_at": "2011-8-10T13:47:12Z"}').put()
Version(id='v1.0.0', parent=library_key, sha='sha').put()
self.respond_to_github(r'https://api.github.com/repos/org/repo/readme\?ref=sha', '{"content":"%s"}' % b64encode('README'))
self.respond_to('https://raw.githubusercontent.com/org/repo/sha/bower.json', '{}')
self.respond_to_github('https://api.github.com/markdown', '<html>README</html>')
response = self.app.get(util.ingest_version_task('org', 'repo', 'v1.0.0'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = Version.get_by_id('v1.0.0', parent=library_key)
self.assertIsNone(version.error)
self.assertEqual(version.status, Status.ready)
self.assertFalse(version.preview)
versions = Library.versions_for_key_async(library_key).get_result()
self.assertEqual(['v1.0.0'], versions)
readme = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme').get()
self.assertEqual(readme.content, 'README')
readme_html = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'readme.html').get()
self.assertEqual(readme_html.content, '<html>README</html>')
bower = ndb.Key(Library, 'org/repo', Version, 'v1.0.0', Content, 'bower').get()
self.assertEqual(bower.content, '{}')
def get(self, owner, repo, version=None):
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Content-Type'] = 'application/json'
library_key = ndb.Key(Library, Library.id(owner, repo))
if version is None:
version = yield Library.default_version_for_key_async(library_key)
if version is None:
self.response.set_status(404)
return
version_key = ndb.Key(Library, library_key.id(), Version, version)
bower = yield Content.get_by_id_async('bower', parent=version_key)
if bower is None:
self.response.set_status(404)
return
bower_json = bower.get_json()
bower_dependencies = bower_json.get('dependencies', {})
dependencies = []
version_futures = []
for name in bower_dependencies.keys():
dependency = Dependency.from_string(bower_dependencies[name])
if dependency is None:
continue
dependencies.append(dependency)
dependency_library_key = ndb.Key(Library, Library.id(dependency.owner, dependency.repo))
version_futures.append(Library.versions_for_key_async(dependency_library_key))
dependency_futures = []
for i, dependency in enumerate(dependencies):
versions = yield version_futures[i]
def matches(version, spec):
try:
return versiontag.match(version, spec)
except ValueError:
# FIXME: What other cases do we need to support here?
return False
while len(versions) > 0 and not matches(versions[-1], dependency.version):
versions.pop()
if len(versions) > 0:
dependency_library_key = ndb.Key(Library, Library.id(dependency.owner.lower(), dependency.repo.lower()))
dependency_futures.append(LibraryMetadata.brief_async(dependency_library_key, versions[-1]))
results = []
for future in dependency_futures:
dependency_result = yield future
if dependency_result is not None:
results.append(dependency_result)
result = {
'results': results,
'count': len(results),
}
self.response.write(json.dumps(result))
def test_delete_version(self):
library_key = ndb.Key(Library, 'owner/repo')
version_key = Version(id='v1.0.0', parent=library_key, sha='1', status=Status.ready).put()
VersionCache.update(library_key)
response = self.app.get('/task/delete/owner/repo/v1.0.0', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = version_key.get()
self.assertIsNone(version)
self.assertEqual(Library.versions_for_key_async(library_key).get_result(), [])
def test_delete_version(self):
library_key = ndb.Key(Library, 'owner/repo')
version_key = Version(id='v1.0.0', parent=library_key, sha='1', status=Status.ready).put()
VersionCache.update(library_key)
response = self.app.get('/task/delete/owner/repo/v1.0.0', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
version = version_key.get()
self.assertIsNone(version)
self.assertEqual(Library.versions_for_key_async(library_key).get_result(), [])
def update_versions(self):
if self.library.shallow_ingestion:
return
if self.library.kind == 'collection':
new_tag_map = self.update_collection_tags()
else:
assert self.library.kind == 'element'
new_tag_map = self.update_element_tags()
new_tags = new_tag_map.keys()
ingested_tags = Library.versions_for_key_async(
self.library.key).get_result()
logging.info('%d of %d tags ingested', len(ingested_tags),
len(new_tags))
tags_to_add = list(set(new_tags) - set(ingested_tags))
tags_to_add.sort(versiontag.compare)
if ingested_tags == [] and len(tags_to_add) > 0:
# Only ingest the default version if we're doing ingestion for the first time.
tags_to_add = [versiontag.default_version(tags_to_add)]
else:
tags_to_add = [
tag for tag in tags_to_add if versiontag.compare(
tag, versiontag.default_version(ingested_tags)) > 0
]
tags_to_delete = list(set(ingested_tags) - set(new_tags))
logging.info('%d adds and %d deletes pending', len(tags_to_add),
len(tags_to_delete))
# To avoid running into limits on the number of tasks (5) that can be spawned transactionally
# only ingest (2 tasks) or delete (1 task) one version per update.
if len(tags_to_add) > 0:
# Ingest from newest to oldest.
tag = tags_to_add[-1]
if self.trigger_version_ingestion(tag, new_tag_map[tag]):
if self.library.kind == 'collection':
logging.info('ingesting new collection version (%s)', tag)
else:
logging.info('ingesting new %s version (%s)',
versiontag.categorize(tag, ingested_tags),
tag)
elif len(tags_to_delete) > 0:
tag = tags_to_delete[0]
self.trigger_version_deletion(tags_to_delete[0])
if len(new_tags) is 0:
return self.error("couldn't find any tagged versions",
ErrorCodes.Library_no_version)
def test_version_cache(self):
library_key = ndb.Key(Library, 'a/b')
Version(id='v2.0.0', sha='x', status=Status.ready, parent=library_key).put()
Version(id='v1.0.0', sha='x', status=Status.ready, parent=library_key).put()
Version(id='v3.0.0', sha='x', status=Status.ready, parent=library_key).put()
Version(id='v3.0.X', sha='x', status=Status.ready, parent=library_key).put()
Version(id='v4.0.0', sha='x', status=Status.error, parent=library_key).put()
Version(id='v5.0.0', sha='x', status=Status.pending, parent=library_key).put()
Version(id='xxx', sha='x', status=Status.ready, parent=library_key).put()
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, [])
latest_changed = VersionCache.update(library_key)
self.assertTrue(latest_changed)
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0'])
Version(id='v6.0.0', sha='x', status=Status.ready, parent=library_key).put()
latest_changed = VersionCache.update(library_key)
self.assertTrue(latest_changed)
versions = yield Library.versions_for_key_async(library_key)
self.assertEqual(versions, ['v1.0.0', 'v2.0.0', 'v3.0.0', 'v4.0.0', 'v6.0.0'])
def update_versions(self):
if self.library.shallow_ingestion:
return
if self.library.kind == 'collection':
new_tag_map = self.update_collection_tags()
elif self.scope.startswith('@'):
assert self.library.kind == 'element'
new_tag_map = self.update_package_tags()
else:
assert self.library.kind == 'element'
new_tag_map = self.update_element_tags()
new_tags = new_tag_map.keys()
ingested_tags = Library.versions_for_key_async(self.library.key).get_result()
logging.info('%d of %d tags ingested', len(ingested_tags), len(new_tags))
tags_to_add = list(set(new_tags) - set(ingested_tags))
tags_to_add.sort(versiontag.compare)
if ingested_tags == [] and len(tags_to_add) > 0:
# Only ingest the default version if we're doing ingestion for the first time.
tags_to_add = [versiontag.default_version(tags_to_add)]
else:
tags_to_add = [tag for tag in tags_to_add if versiontag.compare(tag, versiontag.default_version(ingested_tags)) > 0]
tags_to_delete = list(set(ingested_tags) - set(new_tags))
logging.info('%d adds and %d deletes pending', len(tags_to_add), len(tags_to_delete))
# To avoid running into limits on the number of tasks (5) that can be spawned transactionally
# only ingest (2 tasks) or delete (1 task) one version per update.
if len(tags_to_add) > 0:
# Ingest from newest to oldest.
tag = tags_to_add[-1]
if self.trigger_version_ingestion(tag, new_tag_map[tag]):
if self.library.kind == 'collection':
logging.info('ingesting new collection version (%s)', tag)
else:
logging.info('ingesting new %s version (%s)', versiontag.categorize(tag, ingested_tags), tag)
elif len(tags_to_delete) > 0:
tag = tags_to_delete[0]
self.trigger_version_deletion(tags_to_delete[0])
if len(new_tags) is 0:
return self.error("couldn't find any tagged versions", ErrorCodes.Library_no_version)
def full_async(library_key, tag=None, brief=False, assume_latest=False):
if assume_latest:
assert tag is not None
library_future = library_key.get_async()
if tag is None or not brief or not assume_latest:
versions_future = Library.versions_for_key_async(library_key)
if tag is None:
versions = yield versions_future
default_version = versiontag.default_version(versions)
version_key = None if len(versions) == 0 else ndb.Key(Library, library_key.id(), Version, default_version)
else:
version_key = ndb.Key(Library, library_key.id(), Version, tag)
if version_key is not None:
version_future = version_key.get_async()
bower_future = Content.get_by_id_async('bower', parent=version_key)
if not brief:
readme_future = Content.get_by_id_async('readme.html', parent=version_key)
library = yield library_future
if library is None or library.status == Status.suppressed:
raise ndb.Return(None)
result = {}
# Add NPM package fields
key = library_key.string_id()
if key.startswith('@'):
parts = key.split('/')
if parts[0] != '@@npm':
result['npmScope'] = parts[0]
result['npmFullPackage'] = key
else:
result['npmFullPackage'] = parts[1]
result['npmPackage'] = parts[1]
if library.migrated_from_bower:
result['migratedFromBower'] = True
elif library.npm_package:
result['migratedToNpm'] = library.npm_package
result['apiKey'] = key
result['kind'] = library.kind
result['status'] = library.status
if library.status != Status.ready:
if library.status == Status.error:
result['error'] = library.error
raise ndb.Return(result)
version = None
if version_key is not None:
version = yield version_future
if version is None:
raise ndb.Return(None)
result['spdx_identifier'] = library.spdx_identifier
result['version'] = version.key.id()
if version.status != Status.ready:
result['status'] = version.status
if version.status == Status.error:
result['error'] = version.error
raise ndb.Return(result)
if not brief or not assume_latest:
versions = yield versions_future
result['versions'] = versions
if len(versions) > 0:
result['default_version'] = versiontag.default_version(versions)
# Remove latest_version once deployed clients all use default_version
result['latest_version'] = versiontag.default_version(versions)
if not brief and library.participation is not None:
result['activity'] = json.loads(library.participation).get('all', [])
if not brief and library.contributors is not None:
contributors = []
raw = json.loads(library.contributors)
for contributor in raw:
contributors.append({
'login': contributor['login'],
'avatar_url': contributor['avatar_url'],
'contributions': contributor['contributions'],
})
result['contributors'] = contributors
if library.metadata is not None:
metadata = json.loads(library.metadata)
result['description'] = metadata.get('description', '')
result['subscribers'] = metadata.get('subscribers_count', 0)
result['stars'] = metadata.get('stargazers_count', 0)
result['forks'] = metadata.get('forks', 0)
result['open_issues'] = metadata.get('open_issues', 0)
result['updated_at'] = metadata.get('updated_at', 0)
result['owner'] = metadata['owner']['login']
result['avatar_url'] = metadata['owner'].get('avatar_url', '')
result['repo'] = metadata['name']
if metadata.get('homepage') and re.match(r'https?', metadata.get('homepage')):
result['homepage'] = metadata['homepage']
result['default_branch'] = metadata.get('default_branch', 'master')
if not brief:
readme = yield readme_future
result['readme'] = None if readme is None else readme.content
bower = yield bower_future
if bower is not None:
bower_json = bower.get_json()
dependencies = bower_json.get('dependencies', {})
result['dependency_count'] = len(dependencies)
result['bower'] = {
'license': bower_json.get('license', ''),
'dependencies': dependencies,
'keywords': bower_json.get('keywords', []),
'demos': bower_json.get('demos', {}),
'pages': bower_json.get('pages', {}),
}
if result.get('description', '') == '':
result['description'] = bower_json.get('description', '')
raise ndb.Return(result)
def full_async(library_key, tag=None, brief=False, assume_latest=False):
if assume_latest:
assert tag is not None
library_future = library_key.get_async()
if tag is None or not brief or not assume_latest:
versions_future = Library.versions_for_key_async(library_key)
if tag is None:
versions = yield versions_future
default_version = versiontag.default_version(versions)
version_key = None if len(versions) == 0 else ndb.Key(
Library, library_key.id(), Version, default_version)
else:
version_key = ndb.Key(Library, library_key.id(), Version, tag)
if version_key is not None:
version_future = version_key.get_async()
bower_future = Content.get_by_id_async('bower', parent=version_key)
if not brief:
readme_future = Content.get_by_id_async('readme.html',
parent=version_key)
library = yield library_future
if library is None or library.status == Status.suppressed:
raise ndb.Return(None)
result = {}
result['kind'] = library.kind
result['status'] = library.status
if library.status != Status.ready:
if library.status == Status.error:
result['error'] = library.error
raise ndb.Return(result)
version = None
if version_key is not None:
version = yield version_future
if version is None:
raise ndb.Return(None)
result['spdx_identifier'] = library.spdx_identifier
result['version'] = version.key.id()
if version.status != Status.ready:
result['status'] = version.status
if version.status == Status.error:
result['error'] = version.error
raise ndb.Return(result)
if not brief or not assume_latest:
versions = yield versions_future
result['versions'] = versions
if len(versions) > 0:
result['default_version'] = versiontag.default_version(
versions)
# Remove latest_version once deployed clients all use default_version
result['latest_version'] = versiontag.default_version(versions)
if not brief and library.participation is not None:
result['activity'] = json.loads(library.participation).get(
'all', [])
if not brief and library.contributors is not None:
contributors = []
raw = json.loads(library.contributors)
for contributor in raw:
contributors.append({
'login':
contributor['login'],
'avatar_url':
contributor['avatar_url'],
'contributions':
contributor['contributions'],
})
result['contributors'] = contributors
if library.metadata is not None:
metadata = json.loads(library.metadata)
result['description'] = metadata['description']
result['subscribers'] = metadata['subscribers_count']
result['stars'] = metadata['stargazers_count']
result['forks'] = metadata['forks']
result['open_issues'] = metadata['open_issues']
result['updated_at'] = metadata['updated_at']
result['owner'] = metadata['owner']['login']
result['avatar_url'] = metadata['owner']['avatar_url']
result['repo'] = metadata['name']
result['homepage'] = metadata['homepage']
result['default_branch'] = metadata['default_branch']
if not brief:
readme = yield readme_future
result['readme'] = None if readme is None else readme.content
bower = yield bower_future
if bower is not None:
bower_json = json.loads(bower.content)
dependencies = bower_json.get('dependencies', [])
result['dependency_count'] = len(dependencies)
result['bower'] = {
'license': bower_json.get('license', ''),
'dependencies': dependencies,
'keywords': bower_json.get('keywords', []),
}
if result.get('description', '') == '':
result['description'] = bower_json.get('description', '')
raise ndb.Return(result)
def get(self, owner, repo, ver=None):
owner = owner.lower()
repo = repo.lower()
library = Library.get_by_id('%s/%s' % (owner, repo), read_policy=ndb.EVENTUAL_CONSISTENCY)
if library is None or library.error is not None:
self.response.write(str(library))
self.response.set_status(404)
return
versions = library.versions()
if ver is None:
ver = versions[-1]
version = Version.get_by_id(ver, parent=library.key, read_policy=ndb.EVENTUAL_CONSISTENCY)
if version is None or version.error is not None:
self.response.write(str(version))
self.response.set_status(404)
return
metadata = json.loads(library.metadata)
dependencies = []
bower = Content.get_by_id('bower', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY)
if bower is not None:
try:
bower_json = json.loads(bower.content)
# TODO: Which exception is this for?
# pylint: disable=bare-except
except:
bower_json = {}
readme = Content.get_by_id('readme.html', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY)
full_name_match = re.match(r'(.*)/(.*)', metadata['full_name'])
result = {
'version': ver,
'versions': versions,
'readme': None if readme is None else readme.content,
'subscribers': metadata['subscribers_count'],
'stars': metadata['stargazers_count'],
'forks': metadata['forks'],
'contributors': library.contributor_count,
'open_issues': metadata['open_issues'],
'updated_at': metadata['updated_at'],
'owner': full_name_match.groups()[0],
'repo': full_name_match.groups()[1],
'bower': None if bower is None else {
'description': bower_json.get('description', ''),
'license': bower_json.get('license', ''),
'dependencies': bower_json.get('dependencies', []),
'keywords': bower_json.get('keywords', []),
},
'collections': []
}
for collection in library.collections:
if not versiontag.match(ver, collection.semver):
continue
collection_version = collection.version.id()
collection_library = collection.version.parent().get()
collection_metadata = json.loads(collection_library.metadata)
collection_name_match = re.match(r'(.*)/(.*)', collection_metadata['full_name'])
result['collections'].append({
'owner': collection_name_match.groups()[0],
'repo': collection_name_match.groups()[1],
'version': collection_version
})
if library.kind == 'collection':
dependencies = []
version_futures = []
for dep in version.dependencies:
parsed_dep = Dependency.fromString(dep)
dep_key = ndb.Key(Library, "%s/%s" % (parsed_dep.owner.lower(), parsed_dep.repo.lower()))
version_futures.append(Library.versions_for_key_async(dep_key))
for i, dep in enumerate(version.dependencies):
parsed_dep = Dependency.fromString(dep)
versions = version_futures[i].get_result()
versions.reverse()
while len(versions) > 0 and not versiontag.match(versions[0], parsed_dep.version):
versions.pop()
if len(versions) == 0:
dependencies.append({
'error': 'unsatisfyable dependency',
'owner': parsed_dep.owner,
'repo': parsed_dep.repo,
'versionSpec': parsed_dep.version
})
else:
dependencies.append(brief_metadata_from_datastore(parsed_dep.owner, parsed_dep.repo, versions[0]))
result['dependencies'] = dependencies
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(result))
