Ejemplo n.º 1
0
    def _log_rate_limit(self, future):
        """log GitLab rate limit headers

        - error if 0 remaining
        - warn if 10% or less remain
        - debug otherwise
        """
        try:
            r = future.result()
        except HTTPError as e:
            r = e.response
        limit_s = r.headers.get('X-RateLimit-Limit', '')
        remaining_s = r.headers.get('X-RateLimit-Remaining', '')
        if not remaining_s or not limit_s:
            if r.code < 300:
                app_log.warn("No rate limit headers. Did GitLab change? %s",
                             json.dumps(r.headers, indent=1))
            return

        remaining = int(remaining_s)
        limit = int(limit_s)
        if remaining == 0:
            jsondata = response_text(r)
            data = json.loads(jsondata)
            app_log.error("GitLab rate limit (%s) exceeded: %s", limit,
                          data.get('message', 'no message'))
            return

        if 10 * remaining > limit:
            log = app_log.debug
        else:
            log = app_log.warn
        log("%i/%i GitLab API requests remaining", remaining, limit)
Ejemplo n.º 2
0
    def _log_rate_limit(self, future):
        """log GitLab rate limit headers

        - error if 0 remaining
        - warn if 10% or less remain
        - debug otherwise
        """
        try:
            r = future.result()
        except HTTPError as e:
            r = e.response
        limit_s = r.headers.get('X-RateLimit-Limit', '')
        remaining_s = r.headers.get('X-RateLimit-Remaining', '')
        if not remaining_s or not limit_s:
            if r.code < 300:
                app_log.warn("No rate limit headers. Did GitLab change? %s",
                    json.dumps(r.headers, indent=1)
                )
            return

        remaining = int(remaining_s)
        limit = int(limit_s)
        if remaining == 0:
            jsondata = response_text(r)
            data = json.loads(jsondata)
            app_log.error("GitLab rate limit (%s) exceeded: %s", limit, data.get('message', 'no message'))
            return

        if 10 * remaining > limit:
            log = app_log.debug
        else:
            log = app_log.warn
        log("%i/%i GitLab API requests remaining", remaining, limit)
Ejemplo n.º 3
0
    def refs(self, user, repo):
        """get branches and tags for this user/repo"""
        ref_types = ("branches", "tags")
        ref_data = [None, None]

        for i, ref_type in enumerate(ref_types):
            with self.catch_client_error():
                response = yield getattr(self.gitlab_client, "get_%s" % ref_type)(user, repo)
            ref_data[i] = json.loads(response_text(response))

        raise gen.Return(ref_data)
Ejemplo n.º 4
0
    def _extract_tree_entry(self, path, tree_response):
        """extract a single tree entry from a file list

        For use as a callback in get_tree_entry
        raises 404 if not found
        """
        tree_response.rethrow()
        jsondata = response_text(tree_response)
        data = json.loads(jsondata)
        for entry in data['tree']:
            if entry['path'] == path:
                return entry

        raise HTTPError(404, "%s not found among %i files" % (path, len(data['tree'])))
Ejemplo n.º 5
0
    def _extract_tree_entry(self, path, tree_response):
        """extract a single tree entry from a file list

        For use as a callback in get_tree_entry
        raises 404 if not found
        """
        tree_response.rethrow()
        jsondata = response_text(tree_response)
        data = json.loads(jsondata)
        for entry in data['tree']:
            if entry['path'] == path:
                return entry

        raise HTTPError(
            404, "%s not found among %i files" % (path, len(data['tree'])))
Ejemplo n.º 6
0
    def get(self, user):
        page = self.get_argument("page", None)
        params = {'sort' : 'updated'}
        if page:
            params['page'] = page
        with self.catch_client_error():
            response = yield self.gitlab_client.get_repos(user, params=params)

        prev_url, next_url = self.get_page_links(response)
        repos = json.loads(response_text(response))

        entries = []
        for repo in repos:
            entries.append(dict(
                url=repo['name'],
                name=repo['name'],
            ))
        provider_url = u"https://gitlab.com/{user}".format(user=user)
        html = self.render_template("userview.html",
            entries=entries, provider_url=provider_url,
            next_url=next_url, prev_url=prev_url,
            **PROVIDER_CTX
        )
        yield self.cache_and_finish(html)
Ejemplo n.º 7
0
    def get(self, user, repo, ref, path):
        raw_url = u"https://raw.gitlabusercontent.com/{user}/{repo}/{ref}/{path}".format(
            user=user, repo=repo, ref=ref, path=quote(path)
        )
        blob_url = u"https://gitlab.com/{user}/{repo}/blob/{ref}/{path}".format(
            user=user, repo=repo, ref=ref, path=quote(path),
        )
        with self.catch_client_error():
            tree_entry = yield self.gitlab_client.get_tree_entry(
                user, repo, path=path, ref=ref
            )

        if tree_entry['type'] == 'tree':
            tree_url = "/gitlab/{user}/{repo}/tree/{ref}/{path}/".format(
                user=user, repo=repo, ref=ref, path=quote(path),
            )
            app_log.info("%s is a directory, redirecting to %s", self.request.path, tree_url)
            self.redirect(tree_url)
            return

        # fetch file data from the blobs API
        with self.catch_client_error():
            response = yield self.gitlab_client.fetch(tree_entry['url'])

        data = json.loads(response_text(response))
        contents = data['content']
        if data['encoding'] == 'base64':
            # filedata will be bytes
            filedata = base64_decode(contents)
        else:
            # filedata will be unicode
            filedata = contents

        if path.endswith('.ipynb'):
            dir_path = path.rsplit('/', 1)[0]
            base_url = "/gitlab/{user}/{repo}/tree/{ref}".format(
                user=user, repo=repo, ref=ref,
            )
            breadcrumbs = [{
                'url' : base_url,
                'name' : repo,
            }]
            breadcrumbs.extend(self.breadcrumbs(dir_path, base_url))

            try:
                # filedata may be bytes, but we need text
                if isinstance(filedata, bytes):
                    nbjson = filedata.decode('utf-8')
                else:
                    nbjson = filedata
            except Exception as e:
                app_log.error("Failed to decode notebook: %s", raw_url, exc_info=True)
                raise web.HTTPError(400)
            yield self.finish_notebook(nbjson, raw_url,
                provider_url=blob_url,
                breadcrumbs=breadcrumbs,
                msg="file from GitLab: %s" % raw_url,
                public=True,
                format=self.format,
                request=self.request,
                **PROVIDER_CTX
            )
        else:
            mime, enc = mimetypes.guess_type(path)
            self.set_header("Content-Type", mime or 'text/plain')
            self.cache_and_finish(filedata)
Ejemplo n.º 8
0
    def get(self, user, repo, ref, path):
        if not self.request.uri.endswith('/'):
            self.redirect(self.request.uri + '/')
            return
        path = path.rstrip('/')
        with self.catch_client_error():
            response = yield self.gitlab_client.get_contents(user, repo, path, ref=ref)

        contents = json.loads(response_text(response))

        branches, tags = yield self.refs(user, repo)

        for nav_ref in branches + tags:
            nav_ref["url"] = (u"/gitlab/{user}/{repo}/tree/{ref}/{path}"
                .format(
                    ref=nav_ref["name"], user=user, repo=repo, path=path
                ))

        if not isinstance(contents, list):
            app_log.info(
                "{format}/{user}/{repo}/{ref}/{path} not tree, redirecting to blob",
                extra=dict(format=self.format_prefix, user=user, repo=repo, ref=ref, path=path)
            )
            self.redirect(
                u"{format}/gitlab/{user}/{repo}/blob/{ref}/{path}".format(
                    format=self.format_prefix, user=user, repo=repo, ref=ref, path=path,
                )
            )
            return

        base_url = u"/gitlab/{user}/{repo}/tree/{ref}".format(
            user=user, repo=repo, ref=ref,
        )
        provider_url = u"https://gitlab.com/{user}/{repo}/tree/{ref}/{path}".format(
            user=user, repo=repo, ref=ref, path=path,
        )

        breadcrumbs = [{
            'url' : base_url,
            'name' : repo,
        }]
        breadcrumbs.extend(self.breadcrumbs(path, base_url))

        entries = []
        dirs = []
        ipynbs = []
        others = []
        for file in contents:
            e = {}
            e['name'] = file['name']
            if file['type'] == 'dir':
                e['url'] = u'/gitlab/{user}/{repo}/tree/{ref}/{path}'.format(
                user=user, repo=repo, ref=ref, path=file['path']
                )
                e['url'] = quote(e['url'])
                e['class'] = 'fa-folder-open'
                dirs.append(e)
            elif file['name'].endswith('.ipynb'):
                e['url'] = u'/gitlab/{user}/{repo}/blob/{ref}/{path}'.format(
                user=user, repo=repo, ref=ref, path=file['path']
                )
                e['url'] = quote(e['url'])
                e['class'] = 'fa-book'
                ipynbs.append(e)
            elif file['html_url']:
                e['url'] = file['html_url']
                e['class'] = 'fa-share'
                others.append(e)
            else:
                # submodules don't have html_url
                e['url'] = ''
                e['class'] = 'fa-folder-close'
                others.append(e)


        entries.extend(dirs)
        entries.extend(ipynbs)
        entries.extend(others)

        html = self.render_template("treelist.html",
            entries=entries, breadcrumbs=breadcrumbs, provider_url=provider_url,
            user=user, repo=repo, ref=ref, path=path,
            branches=branches, tags=tags, tree_type="gitlab",
            **PROVIDER_CTX
        )
        yield self.cache_and_finish(html)