def update_versions(self):
if self.library.shallow_ingestion:
return
if self.library.kind == 'collection':
new_tag_map = self.update_collection_tags()
else:
assert self.library.kind == 'element'
new_tag_map = self.update_element_tags()
new_tags = new_tag_map.keys()
ingested_tags = Library.versions_for_key_async(
self.library.key).get_result()
logging.info('%d of %d tags ingested', len(ingested_tags),
len(new_tags))
tags_to_add = list(set(new_tags) - set(ingested_tags))
tags_to_add.sort(versiontag.compare)
if ingested_tags == [] and len(tags_to_add) > 0:
# Only ingest the default version if we're doing ingestion for the first time.
tags_to_add = [versiontag.default_version(tags_to_add)]
else:
tags_to_add = [
tag for tag in tags_to_add if versiontag.compare(
tag, versiontag.default_version(ingested_tags)) > 0
]
tags_to_delete = list(set(ingested_tags) - set(new_tags))
logging.info('%d adds and %d deletes pending', len(tags_to_add),
len(tags_to_delete))
# To avoid running into limits on the number of tasks (5) that can be spawned transactionally
# only ingest (2 tasks) or delete (1 task) one version per update.
if len(tags_to_add) > 0:
# Ingest from newest to oldest.
tag = tags_to_add[-1]
if self.trigger_version_ingestion(tag, new_tag_map[tag]):
if self.library.kind == 'collection':
logging.info('ingesting new collection version (%s)', tag)
else:
logging.info('ingesting new %s version (%s)',
versiontag.categorize(tag, ingested_tags),
tag)
elif len(tags_to_delete) > 0:
tag = tags_to_delete[0]
self.trigger_version_deletion(tags_to_delete[0])
if len(new_tags) is 0:
return self.error("couldn't find any tagged versions",
ErrorCodes.Library_no_version)
def collections_for_key_async(version_key):
library_key = version_key.parent()
collection_references = yield CollectionReference.query(ancestor=library_key).fetch_async()
collection_version_futures = [ref.collection_version_key().get_async() for ref in collection_references]
# If there are multiple versions of a collection we want to find the most recent one that applies.
result_map = {}
for i, version_future in enumerate(collection_version_futures):
collection_version = yield version_future
if collection_version is None:
# Remove the stale reference.
yield collection_references[i].key.delete_async()
elif versiontag.match(version_key.id(), collection_references[i].semver):
collection_id = collection_version.key.parent().id()
existing_version = result_map.get(collection_id, None)
if existing_version is None or versiontag.compare(collection_version.key.id(), existing_version.key.id()) > 0:
result_map[collection_id] = collection_version
raise ndb.Return(result_map.values())
def collections_for_key_async(version_key):
library_key = version_key.parent()
collection_references = yield CollectionReference.query(ancestor=library_key).fetch_async()
collection_version_futures = [ref.collection_version_key().get_async() for ref in collection_references]
# If there are multiple versions of a collection we want to find the most recent one that applies.
result_map = {}
for i, version_future in enumerate(collection_version_futures):
collection_version = yield version_future
if collection_version is None:
# Remove the stale reference.
yield collection_references[i].key.delete_async()
elif versiontag.match(version_key.id(), collection_references[i].semver):
collection_id = collection_version.key.parent().id()
existing_version = result_map.get(collection_id, None)
if existing_version is None or versiontag.compare(collection_version.key.id(), existing_version.key.id()) > 0:
result_map[collection_id] = collection_version
raise ndb.Return(result_map.values())
def update_versions(self):
if self.library.shallow_ingestion:
return
if self.library.kind == 'collection':
new_tag_map = self.update_collection_tags()
elif self.scope.startswith('@'):
assert self.library.kind == 'element'
new_tag_map = self.update_package_tags()
else:
assert self.library.kind == 'element'
new_tag_map = self.update_element_tags()
new_tags = new_tag_map.keys()
ingested_tags = Library.versions_for_key_async(self.library.key).get_result()
logging.info('%d of %d tags ingested', len(ingested_tags), len(new_tags))
tags_to_add = list(set(new_tags) - set(ingested_tags))
tags_to_add.sort(versiontag.compare)
if ingested_tags == [] and len(tags_to_add) > 0:
# Only ingest the default version if we're doing ingestion for the first time.
tags_to_add = [versiontag.default_version(tags_to_add)]
else:
tags_to_add = [tag for tag in tags_to_add if versiontag.compare(tag, versiontag.default_version(ingested_tags)) > 0]
tags_to_delete = list(set(ingested_tags) - set(new_tags))
logging.info('%d adds and %d deletes pending', len(tags_to_add), len(tags_to_delete))
# To avoid running into limits on the number of tasks (5) that can be spawned transactionally
# only ingest (2 tasks) or delete (1 task) one version per update.
if len(tags_to_add) > 0:
# Ingest from newest to oldest.
tag = tags_to_add[-1]
if self.trigger_version_ingestion(tag, new_tag_map[tag]):
if self.library.kind == 'collection':
logging.info('ingesting new collection version (%s)', tag)
else:
logging.info('ingesting new %s version (%s)', versiontag.categorize(tag, ingested_tags), tag)
elif len(tags_to_delete) > 0:
tag = tags_to_delete[0]
self.trigger_version_deletion(tags_to_delete[0])
if len(new_tags) is 0:
return self.error("couldn't find any tagged versions", ErrorCodes.Library_no_version)
def ingest_versions(self):
if not self.library.ingest_versions:
return
response = self.github.github_resource('repos', self.owner, self.repo, 'git/refs/tags', etag=self.library.tags_etag)
if response.status_code != 304:
if response.status_code != 200:
return self.error('repo tags not found (%d)' % response.status_code)
self.library.tags = response.content
self.library.tags_etag = response.headers.get('ETag', None)
self.library_dirty = True
data = json.loads(response.content)
if not isinstance(data, object):
data = []
data = [d for d in data if versiontag.is_valid(d['ref'][10:])]
if len(data) is 0:
return self.error('repo contains no valid version tags')
data.sort(lambda a, b: versiontag.compare(a['ref'][10:], b['ref'][10:]))
data_refs = [d['ref'][10:] for d in data]
self.library.tags = json.dumps(data_refs)
self.library.tags_etag = response.headers.get('ETag', None)
data.reverse()
is_newest = True
for version in data:
tag = version['ref'][10:]
if not versiontag.is_valid(tag):
continue
sha = version['object']['sha']
params = {}
if is_newest:
params["latestVersion"] = "True"
is_newest = False
version_object = Version(parent=self.library.key, id=tag, sha=sha)
version_object.put()
task_url = util.ingest_version_task(self.owner, self.repo, tag)
util.new_task(task_url, params)
util.publish_analysis_request(self.owner, self.repo, tag)