def _log_rate_limit(self, future): """log GitLab rate limit headers - error if 0 remaining - warn if 10% or less remain - debug otherwise """ try: r = future.result() except HTTPError as e: r = e.response limit_s = r.headers.get('X-RateLimit-Limit', '') remaining_s = r.headers.get('X-RateLimit-Remaining', '') if not remaining_s or not limit_s: if r.code < 300: app_log.warn("No rate limit headers. Did GitLab change? %s", json.dumps(r.headers, indent=1)) return remaining = int(remaining_s) limit = int(limit_s) if remaining == 0: jsondata = response_text(r) data = json.loads(jsondata) app_log.error("GitLab rate limit (%s) exceeded: %s", limit, data.get('message', 'no message')) return if 10 * remaining > limit: log = app_log.debug else: log = app_log.warn log("%i/%i GitLab API requests remaining", remaining, limit)
def _log_rate_limit(self, future): """log GitLab rate limit headers - error if 0 remaining - warn if 10% or less remain - debug otherwise """ try: r = future.result() except HTTPError as e: r = e.response limit_s = r.headers.get('X-RateLimit-Limit', '') remaining_s = r.headers.get('X-RateLimit-Remaining', '') if not remaining_s or not limit_s: if r.code < 300: app_log.warn("No rate limit headers. Did GitLab change? %s", json.dumps(r.headers, indent=1) ) return remaining = int(remaining_s) limit = int(limit_s) if remaining == 0: jsondata = response_text(r) data = json.loads(jsondata) app_log.error("GitLab rate limit (%s) exceeded: %s", limit, data.get('message', 'no message')) return if 10 * remaining > limit: log = app_log.debug else: log = app_log.warn log("%i/%i GitLab API requests remaining", remaining, limit)
def refs(self, user, repo): """get branches and tags for this user/repo""" ref_types = ("branches", "tags") ref_data = [None, None] for i, ref_type in enumerate(ref_types): with self.catch_client_error(): response = yield getattr(self.gitlab_client, "get_%s" % ref_type)(user, repo) ref_data[i] = json.loads(response_text(response)) raise gen.Return(ref_data)
def _extract_tree_entry(self, path, tree_response): """extract a single tree entry from a file list For use as a callback in get_tree_entry raises 404 if not found """ tree_response.rethrow() jsondata = response_text(tree_response) data = json.loads(jsondata) for entry in data['tree']: if entry['path'] == path: return entry raise HTTPError(404, "%s not found among %i files" % (path, len(data['tree'])))
def _extract_tree_entry(self, path, tree_response): """extract a single tree entry from a file list For use as a callback in get_tree_entry raises 404 if not found """ tree_response.rethrow() jsondata = response_text(tree_response) data = json.loads(jsondata) for entry in data['tree']: if entry['path'] == path: return entry raise HTTPError( 404, "%s not found among %i files" % (path, len(data['tree'])))
def get(self, user): page = self.get_argument("page", None) params = {'sort' : 'updated'} if page: params['page'] = page with self.catch_client_error(): response = yield self.gitlab_client.get_repos(user, params=params) prev_url, next_url = self.get_page_links(response) repos = json.loads(response_text(response)) entries = [] for repo in repos: entries.append(dict( url=repo['name'], name=repo['name'], )) provider_url = u"https://gitlab.com/{user}".format(user=user) html = self.render_template("userview.html", entries=entries, provider_url=provider_url, next_url=next_url, prev_url=prev_url, **PROVIDER_CTX ) yield self.cache_and_finish(html)
def get(self, user, repo, ref, path): raw_url = u"https://raw.gitlabusercontent.com/{user}/{repo}/{ref}/{path}".format( user=user, repo=repo, ref=ref, path=quote(path) ) blob_url = u"https://gitlab.com/{user}/{repo}/blob/{ref}/{path}".format( user=user, repo=repo, ref=ref, path=quote(path), ) with self.catch_client_error(): tree_entry = yield self.gitlab_client.get_tree_entry( user, repo, path=path, ref=ref ) if tree_entry['type'] == 'tree': tree_url = "/gitlab/{user}/{repo}/tree/{ref}/{path}/".format( user=user, repo=repo, ref=ref, path=quote(path), ) app_log.info("%s is a directory, redirecting to %s", self.request.path, tree_url) self.redirect(tree_url) return # fetch file data from the blobs API with self.catch_client_error(): response = yield self.gitlab_client.fetch(tree_entry['url']) data = json.loads(response_text(response)) contents = data['content'] if data['encoding'] == 'base64': # filedata will be bytes filedata = base64_decode(contents) else: # filedata will be unicode filedata = contents if path.endswith('.ipynb'): dir_path = path.rsplit('/', 1)[0] base_url = "/gitlab/{user}/{repo}/tree/{ref}".format( user=user, repo=repo, ref=ref, ) breadcrumbs = [{ 'url' : base_url, 'name' : repo, }] breadcrumbs.extend(self.breadcrumbs(dir_path, base_url)) try: # filedata may be bytes, but we need text if isinstance(filedata, bytes): nbjson = filedata.decode('utf-8') else: nbjson = filedata except Exception as e: app_log.error("Failed to decode notebook: %s", raw_url, exc_info=True) raise web.HTTPError(400) yield self.finish_notebook(nbjson, raw_url, provider_url=blob_url, breadcrumbs=breadcrumbs, msg="file from GitLab: %s" % raw_url, public=True, format=self.format, request=self.request, **PROVIDER_CTX ) else: mime, enc = mimetypes.guess_type(path) self.set_header("Content-Type", mime or 'text/plain') self.cache_and_finish(filedata)
def get(self, user, repo, ref, path): if not self.request.uri.endswith('/'): self.redirect(self.request.uri + '/') return path = path.rstrip('/') with self.catch_client_error(): response = yield self.gitlab_client.get_contents(user, repo, path, ref=ref) contents = json.loads(response_text(response)) branches, tags = yield self.refs(user, repo) for nav_ref in branches + tags: nav_ref["url"] = (u"/gitlab/{user}/{repo}/tree/{ref}/{path}" .format( ref=nav_ref["name"], user=user, repo=repo, path=path )) if not isinstance(contents, list): app_log.info( "{format}/{user}/{repo}/{ref}/{path} not tree, redirecting to blob", extra=dict(format=self.format_prefix, user=user, repo=repo, ref=ref, path=path) ) self.redirect( u"{format}/gitlab/{user}/{repo}/blob/{ref}/{path}".format( format=self.format_prefix, user=user, repo=repo, ref=ref, path=path, ) ) return base_url = u"/gitlab/{user}/{repo}/tree/{ref}".format( user=user, repo=repo, ref=ref, ) provider_url = u"https://gitlab.com/{user}/{repo}/tree/{ref}/{path}".format( user=user, repo=repo, ref=ref, path=path, ) breadcrumbs = [{ 'url' : base_url, 'name' : repo, }] breadcrumbs.extend(self.breadcrumbs(path, base_url)) entries = [] dirs = [] ipynbs = [] others = [] for file in contents: e = {} e['name'] = file['name'] if file['type'] == 'dir': e['url'] = u'/gitlab/{user}/{repo}/tree/{ref}/{path}'.format( user=user, repo=repo, ref=ref, path=file['path'] ) e['url'] = quote(e['url']) e['class'] = 'fa-folder-open' dirs.append(e) elif file['name'].endswith('.ipynb'): e['url'] = u'/gitlab/{user}/{repo}/blob/{ref}/{path}'.format( user=user, repo=repo, ref=ref, path=file['path'] ) e['url'] = quote(e['url']) e['class'] = 'fa-book' ipynbs.append(e) elif file['html_url']: e['url'] = file['html_url'] e['class'] = 'fa-share' others.append(e) else: # submodules don't have html_url e['url'] = '' e['class'] = 'fa-folder-close' others.append(e) entries.extend(dirs) entries.extend(ipynbs) entries.extend(others) html = self.render_template("treelist.html", entries=entries, breadcrumbs=breadcrumbs, provider_url=provider_url, user=user, repo=repo, ref=ref, path=path, branches=branches, tags=tags, tree_type="gitlab", **PROVIDER_CTX ) yield self.cache_and_finish(html)