def init_library(self, owner, repo, kind=None, create=True):
self.owner = owner.lower()
self.repo = repo.lower()
if create:
assert kind is not None
self.library = Library.maybe_create_with_kind(self.owner, self.repo, kind)
else:
self.library = Library.get_by_id('%s/%s' % (owner, repo))
def get(self, owner, repo, kind):
if not (kind == 'element' or kind == 'collection'):
self.response.set_status(400)
return
owner = owner.lower()
repo = repo.lower()
library = Library.maybe_create_with_kind(owner, repo, kind)
library_dirty = False
if library.error is not None:
library_dirty = True
library.error = None
logging.info('created library')
github = quota.GitHub()
if not github.reserve(3):
self.response.set_status(500)
return
response = github.github_resource('repos', owner, repo, etag=library.metadata_etag)
if response.status_code != 304:
if response.status_code == 200:
library.metadata = response.content
library.metadata_etag = response.headers.get('ETag', None)
library_dirty = True
else:
library.error = 'repo metadata not found (%d)' % response.status_code
github.release()
library.put()
return
response = github.github_resource('repos', owner, repo, 'contributors', etag=library.contributors_etag)
if response.status_code != 304:
if response.status_code == 200:
library.contributors = response.content
library.contributors_etag = response.headers.get('ETag', None)
library.contributor_count = len(json.loads(response.content))
library_dirty = True
else:
library.error = 'repo contributors not found (%d)' % response.status_code
github.release()
library.put()
return
response = github.github_resource('repos', owner, repo, 'git/refs/tags', etag=library.tags_etag)
if response.status_code != 304:
if response.status_code == 200:
library.tags = response.content
library.tags_etag = response.headers.get('ETag', None)
library_dirty = True
data = json.loads(response.content)
if not isinstance(data, object):
library.error = 'repo contains no valid version tags'
github.release()
library.put()
return
for version in data:
tag = version['ref'][10:]
if not versiontag.is_valid(tag):
continue
sha = version['object']['sha']
version_object = Version(parent=library.key, id=tag, sha=sha)
version_object.put()
task_url = util.ingest_version_task(owner, repo, tag)
util.new_task(task_url)
util.publish_analysis_request(owner, repo, tag)
else:
library.error = 'repo tags not found (%d)' % response.status_code
github.release()
library.put()
return
if library_dirty:
library.put()
github.release()
def get(self, owner, repo, kind):
if not (kind == 'element' or kind == 'collection'):
self.response.set_status(400)
return
owner = owner.lower()
repo = repo.lower()
library = Library.maybe_create_with_kind(owner, repo, kind)
logging.info('created library')
github = quota.GitHub()
if not github.reserve(3):
self.response.set_status(500)
return
response = github.github_resource('repos', owner, repo)
if not response.status_code == 200:
library.error = 'repo metadata not found'
github.release()
library.put()
return
library.metadata = response.content
response = github.github_resource('repos', owner, repo, 'contributors')
if not response.status_code == 200:
library.error = 'repo contributors not found'
github.release()
library.put()
return
library.contributors = response.content
library.contributor_count = len(json.loads(response.content))
response = github.github_resource('repos', owner, repo, 'git/refs/tags')
if not response.status_code == 200:
library.error = 'repo tags not found'
github.release()
library.put()
return
data = json.loads(response.content)
if not isinstance(data, object):
library.error = 'repo contians no valid version tags'
github.release()
library.put()
return
library.put()
for version in data:
tag = version['ref'][10:]
if not versiontag.is_valid(tag):
continue
sha = version['object']['sha']
version_object = Version(parent=library.key, id=tag, sha=sha)
version_object.put()
util.new_task('ingest/version', owner, repo, detail=tag)
util.publish_hydrolyze_pending(
'/task/ingest/hydrolyzer/%s/%s/%s' % (owner, repo, tag),
owner,
repo,
tag)
