def default_version(self): self.assertEqual(versiontag.default_version([]), None) self.assertEqual(versiontag.default_version(['v1.0.0', 'v2.0.0']), 'v2.0.0') self.assertEqual(versiontag.default_version(['v1.0.0', 'v2.0.0-pre']), 'v1.0.0') self.assertEqual( versiontag.default_version(['v1.0.0-pre', 'v2.0.0-pre']), 'v2.0.0-pre')
def update_versions(self): if self.library.shallow_ingestion: return if self.library.kind == 'collection': new_tag_map = self.update_collection_tags() else: assert self.library.kind == 'element' new_tag_map = self.update_element_tags() new_tags = new_tag_map.keys() ingested_tags = Library.versions_for_key_async( self.library.key).get_result() logging.info('%d of %d tags ingested', len(ingested_tags), len(new_tags)) tags_to_add = list(set(new_tags) - set(ingested_tags)) tags_to_add.sort(versiontag.compare) if ingested_tags == [] and len(tags_to_add) > 0: # Only ingest the default version if we're doing ingestion for the first time. tags_to_add = [versiontag.default_version(tags_to_add)] else: tags_to_add = [ tag for tag in tags_to_add if versiontag.compare( tag, versiontag.default_version(ingested_tags)) > 0 ] tags_to_delete = list(set(ingested_tags) - set(new_tags)) logging.info('%d adds and %d deletes pending', len(tags_to_add), len(tags_to_delete)) # To avoid running into limits on the number of tasks (5) that can be spawned transactionally # only ingest (2 tasks) or delete (1 task) one version per update. if len(tags_to_add) > 0: # Ingest from newest to oldest. tag = tags_to_add[-1] if self.trigger_version_ingestion(tag, new_tag_map[tag]): if self.library.kind == 'collection': logging.info('ingesting new collection version (%s)', tag) else: logging.info('ingesting new %s version (%s)', versiontag.categorize(tag, ingested_tags), tag) elif len(tags_to_delete) > 0: tag = tags_to_delete[0] self.trigger_version_deletion(tags_to_delete[0]) if len(new_tags) is 0: return self.error("couldn't find any tagged versions", ErrorCodes.Library_no_version)
def update(library_key): """Updates the version cache and returns whether the latest version has changed and an index update is needed. """ versions = Library.uncached_versions_for_key(library_key) version_cache = VersionCache.get_or_insert('versions', parent=library_key) needs_index_update = False if version_cache.versions != versions: old_default = versiontag.default_version(version_cache.versions) new_default = versiontag.default_version(versions) needs_index_update = old_default != new_default version_cache.versions = versions version_cache.put() return needs_index_update
def update_versions(self): if self.library.shallow_ingestion: return if self.library.kind == 'collection': new_tag_map = self.update_collection_tags() elif self.scope.startswith('@'): assert self.library.kind == 'element' new_tag_map = self.update_package_tags() else: assert self.library.kind == 'element' new_tag_map = self.update_element_tags() new_tags = new_tag_map.keys() ingested_tags = Library.versions_for_key_async(self.library.key).get_result() logging.info('%d of %d tags ingested', len(ingested_tags), len(new_tags)) tags_to_add = list(set(new_tags) - set(ingested_tags)) tags_to_add.sort(versiontag.compare) if ingested_tags == [] and len(tags_to_add) > 0: # Only ingest the default version if we're doing ingestion for the first time. tags_to_add = [versiontag.default_version(tags_to_add)] else: tags_to_add = [tag for tag in tags_to_add if versiontag.compare(tag, versiontag.default_version(ingested_tags)) > 0] tags_to_delete = list(set(ingested_tags) - set(new_tags)) logging.info('%d adds and %d deletes pending', len(tags_to_add), len(tags_to_delete)) # To avoid running into limits on the number of tasks (5) that can be spawned transactionally # only ingest (2 tasks) or delete (1 task) one version per update. if len(tags_to_add) > 0: # Ingest from newest to oldest. tag = tags_to_add[-1] if self.trigger_version_ingestion(tag, new_tag_map[tag]): if self.library.kind == 'collection': logging.info('ingesting new collection version (%s)', tag) else: logging.info('ingesting new %s version (%s)', versiontag.categorize(tag, ingested_tags), tag) elif len(tags_to_delete) > 0: tag = tags_to_delete[0] self.trigger_version_deletion(tags_to_delete[0]) if len(new_tags) is 0: return self.error("couldn't find any tagged versions", ErrorCodes.Library_no_version)
def default_version_for_key_async(key): versions = yield Library.versions_for_key_async(key) if versions == []: raise ndb.Return(None) raise ndb.Return(versiontag.default_version(versions))
def full_async(library_key, tag=None, brief=False, assume_latest=False): if assume_latest: assert tag is not None library_future = library_key.get_async() if tag is None or not brief or not assume_latest: versions_future = Library.versions_for_key_async(library_key) if tag is None: versions = yield versions_future default_version = versiontag.default_version(versions) version_key = None if len(versions) == 0 else ndb.Key(Library, library_key.id(), Version, default_version) else: version_key = ndb.Key(Library, library_key.id(), Version, tag) if version_key is not None: version_future = version_key.get_async() bower_future = Content.get_by_id_async('bower', parent=version_key) if not brief: readme_future = Content.get_by_id_async('readme.html', parent=version_key) library = yield library_future if library is None or library.status == Status.suppressed: raise ndb.Return(None) result = {} # Add NPM package fields key = library_key.string_id() if key.startswith('@'): parts = key.split('/') if parts[0] != '@@npm': result['npmScope'] = parts[0] result['npmFullPackage'] = key else: result['npmFullPackage'] = parts[1] result['npmPackage'] = parts[1] if library.migrated_from_bower: result['migratedFromBower'] = True elif library.npm_package: result['migratedToNpm'] = library.npm_package result['apiKey'] = key result['kind'] = library.kind result['status'] = library.status if library.status != Status.ready: if library.status == Status.error: result['error'] = library.error raise ndb.Return(result) version = None if version_key is not None: version = yield version_future if version is None: raise ndb.Return(None) result['spdx_identifier'] = library.spdx_identifier result['version'] = version.key.id() if version.status != Status.ready: result['status'] = version.status if version.status == Status.error: result['error'] = version.error raise ndb.Return(result) if not brief or not assume_latest: versions = yield versions_future result['versions'] = versions if len(versions) > 0: result['default_version'] = versiontag.default_version(versions) # Remove latest_version once deployed clients all use default_version result['latest_version'] = versiontag.default_version(versions) if not brief and library.participation is not None: result['activity'] = json.loads(library.participation).get('all', []) if not brief and library.contributors is not None: contributors = [] raw = json.loads(library.contributors) for contributor in raw: contributors.append({ 'login': contributor['login'], 'avatar_url': contributor['avatar_url'], 'contributions': contributor['contributions'], }) result['contributors'] = contributors if library.metadata is not None: metadata = json.loads(library.metadata) result['description'] = metadata.get('description', '') result['subscribers'] = metadata.get('subscribers_count', 0) result['stars'] = metadata.get('stargazers_count', 0) result['forks'] = metadata.get('forks', 0) result['open_issues'] = metadata.get('open_issues', 0) result['updated_at'] = metadata.get('updated_at', 0) result['owner'] = metadata['owner']['login'] result['avatar_url'] = metadata['owner'].get('avatar_url', '') result['repo'] = metadata['name'] if metadata.get('homepage') and re.match(r'https?', metadata.get('homepage')): result['homepage'] = metadata['homepage'] result['default_branch'] = metadata.get('default_branch', 'master') if not brief: readme = yield readme_future result['readme'] = None if readme is None else readme.content bower = yield bower_future if bower is not None: bower_json = bower.get_json() dependencies = bower_json.get('dependencies', {}) result['dependency_count'] = len(dependencies) result['bower'] = { 'license': bower_json.get('license', ''), 'dependencies': dependencies, 'keywords': bower_json.get('keywords', []), 'demos': bower_json.get('demos', {}), 'pages': bower_json.get('pages', {}), } if result.get('description', '') == '': result['description'] = bower_json.get('description', '') raise ndb.Return(result)
def full_async(library_key, tag=None, brief=False, assume_latest=False): if assume_latest: assert tag is not None library_future = library_key.get_async() if tag is None or not brief or not assume_latest: versions_future = Library.versions_for_key_async(library_key) if tag is None: versions = yield versions_future default_version = versiontag.default_version(versions) version_key = None if len(versions) == 0 else ndb.Key( Library, library_key.id(), Version, default_version) else: version_key = ndb.Key(Library, library_key.id(), Version, tag) if version_key is not None: version_future = version_key.get_async() bower_future = Content.get_by_id_async('bower', parent=version_key) if not brief: readme_future = Content.get_by_id_async('readme.html', parent=version_key) library = yield library_future if library is None or library.status == Status.suppressed: raise ndb.Return(None) result = {} result['kind'] = library.kind result['status'] = library.status if library.status != Status.ready: if library.status == Status.error: result['error'] = library.error raise ndb.Return(result) version = None if version_key is not None: version = yield version_future if version is None: raise ndb.Return(None) result['spdx_identifier'] = library.spdx_identifier result['version'] = version.key.id() if version.status != Status.ready: result['status'] = version.status if version.status == Status.error: result['error'] = version.error raise ndb.Return(result) if not brief or not assume_latest: versions = yield versions_future result['versions'] = versions if len(versions) > 0: result['default_version'] = versiontag.default_version( versions) # Remove latest_version once deployed clients all use default_version result['latest_version'] = versiontag.default_version(versions) if not brief and library.participation is not None: result['activity'] = json.loads(library.participation).get( 'all', []) if not brief and library.contributors is not None: contributors = [] raw = json.loads(library.contributors) for contributor in raw: contributors.append({ 'login': contributor['login'], 'avatar_url': contributor['avatar_url'], 'contributions': contributor['contributions'], }) result['contributors'] = contributors if library.metadata is not None: metadata = json.loads(library.metadata) result['description'] = metadata['description'] result['subscribers'] = metadata['subscribers_count'] result['stars'] = metadata['stargazers_count'] result['forks'] = metadata['forks'] result['open_issues'] = metadata['open_issues'] result['updated_at'] = metadata['updated_at'] result['owner'] = metadata['owner']['login'] result['avatar_url'] = metadata['owner']['avatar_url'] result['repo'] = metadata['name'] result['homepage'] = metadata['homepage'] result['default_branch'] = metadata['default_branch'] if not brief: readme = yield readme_future result['readme'] = None if readme is None else readme.content bower = yield bower_future if bower is not None: bower_json = json.loads(bower.content) dependencies = bower_json.get('dependencies', []) result['dependency_count'] = len(dependencies) result['bower'] = { 'license': bower_json.get('license', ''), 'dependencies': dependencies, 'keywords': bower_json.get('keywords', []), } if result.get('description', '') == '': result['description'] = bower_json.get('description', '') raise ndb.Return(result)
def default_version(self): self.assertEqual(versiontag.default_version([]), None) self.assertEqual(versiontag.default_version(['v1.0.0', 'v2.0.0']), 'v2.0.0') self.assertEqual(versiontag.default_version(['v1.0.0', 'v2.0.0-pre']), 'v1.0.0') self.assertEqual(versiontag.default_version(['v1.0.0-pre', 'v2.0.0-pre']), 'v2.0.0-pre')