def update_element_tags(self):
# Transition from when we didn't store tag_map
if self.library.tag_map is None:
self.library.tags_etag = None
response = util.github_get('repos',
self.owner,
self.repo,
'tags',
etag=self.library.tags_etag)
if response.status_code == 304:
return json.loads(self.library.tag_map)
if response.status_code != 200:
return self.retry('could not update git/refs/tags (%d)' %
response.status_code)
try:
data = json.loads(response.content)
except ValueError:
return self.error("could not parse tags",
ErrorCodes.Library_element_parse_tags)
tag_map = dict((tag['name'], tag['commit']['sha']) for tag in data
if versiontag.is_valid(tag['name']))
tags = tag_map.keys()
tags.sort(versiontag.compare)
self.library.library_dirty = True
self.library.tags = tags
self.library.tag_map = json.dumps(tag_map)
self.library.tags_etag = response.headers.get('ETag', None)
self.library.tags_updated = datetime.datetime.now()
return tag_map
def update_element_tags(self):
# Transition from when we didn't store tag_map
if self.library.tag_map is None:
self.library.tags_etag = None
response = util.github_get('repos', self.owner, self.repo, 'tags', etag=self.library.tags_etag)
if response.status_code == 304:
return json.loads(self.library.tag_map)
if response.status_code != 200:
return self.retry('could not update git/refs/tags (%d)' % response.status_code)
try:
data = json.loads(response.content)
except ValueError:
return self.error("could not parse tags")
tag_map = dict((tag['name'], tag['commit']['sha']) for tag in data
if versiontag.is_valid(tag['name']))
tags = tag_map.keys()
tags.sort(versiontag.compare)
self.library.library_dirty = True
self.library.tags = tags
self.library.tag_map = json.dumps(tag_map)
self.library.tags_etag = response.headers.get('ETag', None)
self.library.tags_updated = datetime.datetime.now()
return tag_map
def uncached_versions_for_key(key):
versions = Version.query(Version.status != Status.pending,
ancestor=key).fetch(keys_only=True)
versions = [
key.id() for key in versions if versiontag.is_valid(key.id())
]
versions.sort(versiontag.compare)
return versions
def ingest_versions(self):
if not self.library.ingest_versions:
return
response = self.github.github_resource('repos', self.owner, self.repo, 'git/refs/tags', etag=self.library.tags_etag)
if response.status_code != 304:
if response.status_code != 200:
return self.error('repo tags not found (%d)' % response.status_code)
self.library.tags = response.content
self.library.tags_etag = response.headers.get('ETag', None)
self.library_dirty = True
data = json.loads(response.content)
if not isinstance(data, object):
data = []
data = [d for d in data if versiontag.is_valid(d['ref'][10:])]
if len(data) is 0:
return self.error('repo contains no valid version tags')
data.sort(lambda a, b: versiontag.compare(a['ref'][10:], b['ref'][10:]))
data_refs = [d['ref'][10:] for d in data]
self.library.tags = json.dumps(data_refs)
self.library.tags_etag = response.headers.get('ETag', None)
data.reverse()
is_newest = True
for version in data:
tag = version['ref'][10:]
if not versiontag.is_valid(tag):
continue
sha = version['object']['sha']
params = {}
if is_newest:
params["latestVersion"] = "True"
is_newest = False
version_object = Version(parent=self.library.key, id=tag, sha=sha)
version_object.put()
task_url = util.ingest_version_task(self.owner, self.repo, tag)
util.new_task(task_url, params)
util.publish_analysis_request(self.owner, self.repo, tag)
def update_package_tags(self):
registry_metadata = json.loads(self.library.registry_metadata)
assert registry_metadata is not None
versions = registry_metadata.get('versions', {})
tags = versions.keys()
tags.sort(versiontag.compare)
# Create a tag map of tag to sha
tag_map = dict((tag, versions.get(tag).get('gitHead', '')) for tag in tags
if versiontag.is_valid(tag))
if self.library.tags is None or self.library.tags != tags:
self.library.library_dirty = True
self.library.tags = tags
self.library.tag_map = json.dumps(tag_map)
self.library.tags_updated = datetime.datetime.now()
return tag_map
def update_package_tags(self):
registry_metadata = json.loads(self.library.registry_metadata)
assert registry_metadata is not None
versions = registry_metadata.get('versions', {})
tags = versions.keys()
tags.sort(versiontag.compare)
# Create a tag map of tag to sha
tag_map = dict((tag, versions.get(tag).get('gitHead', '')) for tag in tags
if versiontag.is_valid(tag) and versions.get(tag).get('gitHead'))
if self.library.tags is None or self.library.tags != tags:
self.library.library_dirty = True
self.library.tags = tags
self.library.tag_map = json.dumps(tag_map)
self.library.tags_updated = datetime.datetime.now()
return tag_map
def uncached_versions_for_key(key): versions = Version.query(Version.status != Status.pending, ancestor=key).fetch(keys_only=True) versions = [key.id() for key in versions if versiontag.is_valid(key.id())] versions.sort(versiontag.compare) return versions
def get(self, owner, repo, kind):
if not (kind == 'element' or kind == 'collection'):
self.response.set_status(400)
return
owner = owner.lower()
repo = repo.lower()
library = Library.maybe_create_with_kind(owner, repo, kind)
library_dirty = False
if library.error is not None:
library_dirty = True
library.error = None
logging.info('created library')
github = quota.GitHub()
if not github.reserve(3):
self.response.set_status(500)
return
response = github.github_resource('repos', owner, repo, etag=library.metadata_etag)
if response.status_code != 304:
if response.status_code == 200:
library.metadata = response.content
library.metadata_etag = response.headers.get('ETag', None)
library_dirty = True
else:
library.error = 'repo metadata not found (%d)' % response.status_code
github.release()
library.put()
return
response = github.github_resource('repos', owner, repo, 'contributors', etag=library.contributors_etag)
if response.status_code != 304:
if response.status_code == 200:
library.contributors = response.content
library.contributors_etag = response.headers.get('ETag', None)
library.contributor_count = len(json.loads(response.content))
library_dirty = True
else:
library.error = 'repo contributors not found (%d)' % response.status_code
github.release()
library.put()
return
response = github.github_resource('repos', owner, repo, 'git/refs/tags', etag=library.tags_etag)
if response.status_code != 304:
if response.status_code == 200:
library.tags = response.content
library.tags_etag = response.headers.get('ETag', None)
library_dirty = True
data = json.loads(response.content)
if not isinstance(data, object):
library.error = 'repo contains no valid version tags'
github.release()
library.put()
return
for version in data:
tag = version['ref'][10:]
if not versiontag.is_valid(tag):
continue
sha = version['object']['sha']
version_object = Version(parent=library.key, id=tag, sha=sha)
version_object.put()
task_url = util.ingest_version_task(owner, repo, tag)
util.new_task(task_url)
util.publish_analysis_request(owner, repo, tag)
else:
library.error = 'repo tags not found (%d)' % response.status_code
github.release()
library.put()
return
if library_dirty:
library.put()
github.release()
def get(self, owner, repo, kind):
if not (kind == 'element' or kind == 'collection'):
self.response.set_status(400)
return
owner = owner.lower()
repo = repo.lower()
library = Library.maybe_create_with_kind(owner, repo, kind)
logging.info('created library')
github = quota.GitHub()
if not github.reserve(3):
self.response.set_status(500)
return
response = github.github_resource('repos', owner, repo)
if not response.status_code == 200:
library.error = 'repo metadata not found'
github.release()
library.put()
return
library.metadata = response.content
response = github.github_resource('repos', owner, repo, 'contributors')
if not response.status_code == 200:
library.error = 'repo contributors not found'
github.release()
library.put()
return
library.contributors = response.content
library.contributor_count = len(json.loads(response.content))
response = github.github_resource('repos', owner, repo, 'git/refs/tags')
if not response.status_code == 200:
library.error = 'repo tags not found'
github.release()
library.put()
return
data = json.loads(response.content)
if not isinstance(data, object):
library.error = 'repo contians no valid version tags'
github.release()
library.put()
return
library.put()
for version in data:
tag = version['ref'][10:]
if not versiontag.is_valid(tag):
continue
sha = version['object']['sha']
version_object = Version(parent=library.key, id=tag, sha=sha)
version_object.put()
util.new_task('ingest/version', owner, repo, detail=tag)
util.publish_hydrolyze_pending(
'/task/ingest/hydrolyzer/%s/%s/%s' % (owner, repo, tag),
owner,
repo,
tag)