def init_library(self, owner, repo, kind=None, create=True): self.owner = owner.lower() self.repo = repo.lower() if create: assert kind is not None self.library = Library.maybe_create_with_kind(self.owner, self.repo, kind) else: self.library = Library.get_by_id('%s/%s' % (owner, repo))
def get(self, owner, repo, kind): if not (kind == 'element' or kind == 'collection'): self.response.set_status(400) return owner = owner.lower() repo = repo.lower() library = Library.maybe_create_with_kind(owner, repo, kind) library_dirty = False if library.error is not None: library_dirty = True library.error = None logging.info('created library') github = quota.GitHub() if not github.reserve(3): self.response.set_status(500) return response = github.github_resource('repos', owner, repo, etag=library.metadata_etag) if response.status_code != 304: if response.status_code == 200: library.metadata = response.content library.metadata_etag = response.headers.get('ETag', None) library_dirty = True else: library.error = 'repo metadata not found (%d)' % response.status_code github.release() library.put() return response = github.github_resource('repos', owner, repo, 'contributors', etag=library.contributors_etag) if response.status_code != 304: if response.status_code == 200: library.contributors = response.content library.contributors_etag = response.headers.get('ETag', None) library.contributor_count = len(json.loads(response.content)) library_dirty = True else: library.error = 'repo contributors not found (%d)' % response.status_code github.release() library.put() return response = github.github_resource('repos', owner, repo, 'git/refs/tags', etag=library.tags_etag) if response.status_code != 304: if response.status_code == 200: library.tags = response.content library.tags_etag = response.headers.get('ETag', None) library_dirty = True data = json.loads(response.content) if not isinstance(data, object): library.error = 'repo contains no valid version tags' github.release() library.put() return for version in data: tag = version['ref'][10:] if not versiontag.is_valid(tag): continue sha = version['object']['sha'] version_object = Version(parent=library.key, id=tag, sha=sha) version_object.put() task_url = util.ingest_version_task(owner, repo, tag) util.new_task(task_url) util.publish_analysis_request(owner, repo, tag) else: library.error = 'repo tags not found (%d)' % response.status_code github.release() library.put() return if library_dirty: library.put() github.release()
def get(self, owner, repo, kind): if not (kind == 'element' or kind == 'collection'): self.response.set_status(400) return owner = owner.lower() repo = repo.lower() library = Library.maybe_create_with_kind(owner, repo, kind) logging.info('created library') github = quota.GitHub() if not github.reserve(3): self.response.set_status(500) return response = github.github_resource('repos', owner, repo) if not response.status_code == 200: library.error = 'repo metadata not found' github.release() library.put() return library.metadata = response.content response = github.github_resource('repos', owner, repo, 'contributors') if not response.status_code == 200: library.error = 'repo contributors not found' github.release() library.put() return library.contributors = response.content library.contributor_count = len(json.loads(response.content)) response = github.github_resource('repos', owner, repo, 'git/refs/tags') if not response.status_code == 200: library.error = 'repo tags not found' github.release() library.put() return data = json.loads(response.content) if not isinstance(data, object): library.error = 'repo contians no valid version tags' github.release() library.put() return library.put() for version in data: tag = version['ref'][10:] if not versiontag.is_valid(tag): continue sha = version['object']['sha'] version_object = Version(parent=library.key, id=tag, sha=sha) version_object.put() util.new_task('ingest/version', owner, repo, detail=tag) util.publish_hydrolyze_pending( '/task/ingest/hydrolyzer/%s/%s/%s' % (owner, repo, tag), owner, repo, tag)