def default_version(self):
self.assertEqual(versiontag.default_version([]), None)
self.assertEqual(versiontag.default_version(['v1.0.0', 'v2.0.0']),
'v2.0.0')
self.assertEqual(versiontag.default_version(['v1.0.0', 'v2.0.0-pre']),
'v1.0.0')
self.assertEqual(
versiontag.default_version(['v1.0.0-pre', 'v2.0.0-pre']),
'v2.0.0-pre')
def update_versions(self):
if self.library.shallow_ingestion:
return
if self.library.kind == 'collection':
new_tag_map = self.update_collection_tags()
else:
assert self.library.kind == 'element'
new_tag_map = self.update_element_tags()
new_tags = new_tag_map.keys()
ingested_tags = Library.versions_for_key_async(
self.library.key).get_result()
logging.info('%d of %d tags ingested', len(ingested_tags),
len(new_tags))
tags_to_add = list(set(new_tags) - set(ingested_tags))
tags_to_add.sort(versiontag.compare)
if ingested_tags == [] and len(tags_to_add) > 0:
# Only ingest the default version if we're doing ingestion for the first time.
tags_to_add = [versiontag.default_version(tags_to_add)]
else:
tags_to_add = [
tag for tag in tags_to_add if versiontag.compare(
tag, versiontag.default_version(ingested_tags)) > 0
]
tags_to_delete = list(set(ingested_tags) - set(new_tags))
logging.info('%d adds and %d deletes pending', len(tags_to_add),
len(tags_to_delete))
# To avoid running into limits on the number of tasks (5) that can be spawned transactionally
# only ingest (2 tasks) or delete (1 task) one version per update.
if len(tags_to_add) > 0:
# Ingest from newest to oldest.
tag = tags_to_add[-1]
if self.trigger_version_ingestion(tag, new_tag_map[tag]):
if self.library.kind == 'collection':
logging.info('ingesting new collection version (%s)', tag)
else:
logging.info('ingesting new %s version (%s)',
versiontag.categorize(tag, ingested_tags),
tag)
elif len(tags_to_delete) > 0:
tag = tags_to_delete[0]
self.trigger_version_deletion(tags_to_delete[0])
if len(new_tags) is 0:
return self.error("couldn't find any tagged versions",
ErrorCodes.Library_no_version)
def update(library_key):
"""Updates the version cache and returns whether the latest version has
changed and an index update is needed.
"""
versions = Library.uncached_versions_for_key(library_key)
version_cache = VersionCache.get_or_insert('versions', parent=library_key)
needs_index_update = False
if version_cache.versions != versions:
old_default = versiontag.default_version(version_cache.versions)
new_default = versiontag.default_version(versions)
needs_index_update = old_default != new_default
version_cache.versions = versions
version_cache.put()
return needs_index_update
def update(library_key):
"""Updates the version cache and returns whether the latest version has
changed and an index update is needed.
"""
versions = Library.uncached_versions_for_key(library_key)
version_cache = VersionCache.get_or_insert('versions', parent=library_key)
needs_index_update = False
if version_cache.versions != versions:
old_default = versiontag.default_version(version_cache.versions)
new_default = versiontag.default_version(versions)
needs_index_update = old_default != new_default
version_cache.versions = versions
version_cache.put()
return needs_index_update
def update_versions(self):
if self.library.shallow_ingestion:
return
if self.library.kind == 'collection':
new_tag_map = self.update_collection_tags()
elif self.scope.startswith('@'):
assert self.library.kind == 'element'
new_tag_map = self.update_package_tags()
else:
assert self.library.kind == 'element'
new_tag_map = self.update_element_tags()
new_tags = new_tag_map.keys()
ingested_tags = Library.versions_for_key_async(self.library.key).get_result()
logging.info('%d of %d tags ingested', len(ingested_tags), len(new_tags))
tags_to_add = list(set(new_tags) - set(ingested_tags))
tags_to_add.sort(versiontag.compare)
if ingested_tags == [] and len(tags_to_add) > 0:
# Only ingest the default version if we're doing ingestion for the first time.
tags_to_add = [versiontag.default_version(tags_to_add)]
else:
tags_to_add = [tag for tag in tags_to_add if versiontag.compare(tag, versiontag.default_version(ingested_tags)) > 0]
tags_to_delete = list(set(ingested_tags) - set(new_tags))
logging.info('%d adds and %d deletes pending', len(tags_to_add), len(tags_to_delete))
# To avoid running into limits on the number of tasks (5) that can be spawned transactionally
# only ingest (2 tasks) or delete (1 task) one version per update.
if len(tags_to_add) > 0:
# Ingest from newest to oldest.
tag = tags_to_add[-1]
if self.trigger_version_ingestion(tag, new_tag_map[tag]):
if self.library.kind == 'collection':
logging.info('ingesting new collection version (%s)', tag)
else:
logging.info('ingesting new %s version (%s)', versiontag.categorize(tag, ingested_tags), tag)
elif len(tags_to_delete) > 0:
tag = tags_to_delete[0]
self.trigger_version_deletion(tags_to_delete[0])
if len(new_tags) is 0:
return self.error("couldn't find any tagged versions", ErrorCodes.Library_no_version)
def default_version_for_key_async(key):
versions = yield Library.versions_for_key_async(key)
if versions == []:
raise ndb.Return(None)
raise ndb.Return(versiontag.default_version(versions))
def full_async(library_key, tag=None, brief=False, assume_latest=False):
if assume_latest:
assert tag is not None
library_future = library_key.get_async()
if tag is None or not brief or not assume_latest:
versions_future = Library.versions_for_key_async(library_key)
if tag is None:
versions = yield versions_future
default_version = versiontag.default_version(versions)
version_key = None if len(versions) == 0 else ndb.Key(Library, library_key.id(), Version, default_version)
else:
version_key = ndb.Key(Library, library_key.id(), Version, tag)
if version_key is not None:
version_future = version_key.get_async()
bower_future = Content.get_by_id_async('bower', parent=version_key)
if not brief:
readme_future = Content.get_by_id_async('readme.html', parent=version_key)
library = yield library_future
if library is None or library.status == Status.suppressed:
raise ndb.Return(None)
result = {}
# Add NPM package fields
key = library_key.string_id()
if key.startswith('@'):
parts = key.split('/')
if parts[0] != '@@npm':
result['npmScope'] = parts[0]
result['npmFullPackage'] = key
else:
result['npmFullPackage'] = parts[1]
result['npmPackage'] = parts[1]
if library.migrated_from_bower:
result['migratedFromBower'] = True
elif library.npm_package:
result['migratedToNpm'] = library.npm_package
result['apiKey'] = key
result['kind'] = library.kind
result['status'] = library.status
if library.status != Status.ready:
if library.status == Status.error:
result['error'] = library.error
raise ndb.Return(result)
version = None
if version_key is not None:
version = yield version_future
if version is None:
raise ndb.Return(None)
result['spdx_identifier'] = library.spdx_identifier
result['version'] = version.key.id()
if version.status != Status.ready:
result['status'] = version.status
if version.status == Status.error:
result['error'] = version.error
raise ndb.Return(result)
if not brief or not assume_latest:
versions = yield versions_future
result['versions'] = versions
if len(versions) > 0:
result['default_version'] = versiontag.default_version(versions)
# Remove latest_version once deployed clients all use default_version
result['latest_version'] = versiontag.default_version(versions)
if not brief and library.participation is not None:
result['activity'] = json.loads(library.participation).get('all', [])
if not brief and library.contributors is not None:
contributors = []
raw = json.loads(library.contributors)
for contributor in raw:
contributors.append({
'login': contributor['login'],
'avatar_url': contributor['avatar_url'],
'contributions': contributor['contributions'],
})
result['contributors'] = contributors
if library.metadata is not None:
metadata = json.loads(library.metadata)
result['description'] = metadata.get('description', '')
result['subscribers'] = metadata.get('subscribers_count', 0)
result['stars'] = metadata.get('stargazers_count', 0)
result['forks'] = metadata.get('forks', 0)
result['open_issues'] = metadata.get('open_issues', 0)
result['updated_at'] = metadata.get('updated_at', 0)
result['owner'] = metadata['owner']['login']
result['avatar_url'] = metadata['owner'].get('avatar_url', '')
result['repo'] = metadata['name']
if metadata.get('homepage') and re.match(r'https?', metadata.get('homepage')):
result['homepage'] = metadata['homepage']
result['default_branch'] = metadata.get('default_branch', 'master')
if not brief:
readme = yield readme_future
result['readme'] = None if readme is None else readme.content
bower = yield bower_future
if bower is not None:
bower_json = bower.get_json()
dependencies = bower_json.get('dependencies', {})
result['dependency_count'] = len(dependencies)
result['bower'] = {
'license': bower_json.get('license', ''),
'dependencies': dependencies,
'keywords': bower_json.get('keywords', []),
'demos': bower_json.get('demos', {}),
'pages': bower_json.get('pages', {}),
}
if result.get('description', '') == '':
result['description'] = bower_json.get('description', '')
raise ndb.Return(result)
def default_version_for_key_async(key):
versions = yield Library.versions_for_key_async(key)
if versions == []:
raise ndb.Return(None)
raise ndb.Return(versiontag.default_version(versions))
def full_async(library_key, tag=None, brief=False, assume_latest=False):
if assume_latest:
assert tag is not None
library_future = library_key.get_async()
if tag is None or not brief or not assume_latest:
versions_future = Library.versions_for_key_async(library_key)
if tag is None:
versions = yield versions_future
default_version = versiontag.default_version(versions)
version_key = None if len(versions) == 0 else ndb.Key(
Library, library_key.id(), Version, default_version)
else:
version_key = ndb.Key(Library, library_key.id(), Version, tag)
if version_key is not None:
version_future = version_key.get_async()
bower_future = Content.get_by_id_async('bower', parent=version_key)
if not brief:
readme_future = Content.get_by_id_async('readme.html',
parent=version_key)
library = yield library_future
if library is None or library.status == Status.suppressed:
raise ndb.Return(None)
result = {}
result['kind'] = library.kind
result['status'] = library.status
if library.status != Status.ready:
if library.status == Status.error:
result['error'] = library.error
raise ndb.Return(result)
version = None
if version_key is not None:
version = yield version_future
if version is None:
raise ndb.Return(None)
result['spdx_identifier'] = library.spdx_identifier
result['version'] = version.key.id()
if version.status != Status.ready:
result['status'] = version.status
if version.status == Status.error:
result['error'] = version.error
raise ndb.Return(result)
if not brief or not assume_latest:
versions = yield versions_future
result['versions'] = versions
if len(versions) > 0:
result['default_version'] = versiontag.default_version(
versions)
# Remove latest_version once deployed clients all use default_version
result['latest_version'] = versiontag.default_version(versions)
if not brief and library.participation is not None:
result['activity'] = json.loads(library.participation).get(
'all', [])
if not brief and library.contributors is not None:
contributors = []
raw = json.loads(library.contributors)
for contributor in raw:
contributors.append({
'login':
contributor['login'],
'avatar_url':
contributor['avatar_url'],
'contributions':
contributor['contributions'],
})
result['contributors'] = contributors
if library.metadata is not None:
metadata = json.loads(library.metadata)
result['description'] = metadata['description']
result['subscribers'] = metadata['subscribers_count']
result['stars'] = metadata['stargazers_count']
result['forks'] = metadata['forks']
result['open_issues'] = metadata['open_issues']
result['updated_at'] = metadata['updated_at']
result['owner'] = metadata['owner']['login']
result['avatar_url'] = metadata['owner']['avatar_url']
result['repo'] = metadata['name']
result['homepage'] = metadata['homepage']
result['default_branch'] = metadata['default_branch']
if not brief:
readme = yield readme_future
result['readme'] = None if readme is None else readme.content
bower = yield bower_future
if bower is not None:
bower_json = json.loads(bower.content)
dependencies = bower_json.get('dependencies', [])
result['dependency_count'] = len(dependencies)
result['bower'] = {
'license': bower_json.get('license', ''),
'dependencies': dependencies,
'keywords': bower_json.get('keywords', []),
}
if result.get('description', '') == '':
result['description'] = bower_json.get('description', '')
raise ndb.Return(result)
def default_version(self): self.assertEqual(versiontag.default_version([]), None) self.assertEqual(versiontag.default_version(['v1.0.0', 'v2.0.0']), 'v2.0.0') self.assertEqual(versiontag.default_version(['v1.0.0', 'v2.0.0-pre']), 'v1.0.0') self.assertEqual(versiontag.default_version(['v1.0.0-pre', 'v2.0.0-pre']), 'v2.0.0-pre')
