Ejemplo n.º 1
0
    def _fetch_api_url(self, url):
        # fetch the url and parse to json
        '''
        jdata = None
        try:
            resp = self.instance._requester.requestJson(
                'GET',
                url
            )
            data = resp[2]
            jdata = json.loads(data)
        except Exception as e:
            logging.error(e)
        '''

        jdata = None
        while True:
            resp = self.instance._requester.requestJson(
                'GET',
                url
            )
            data = resp[2]
            jdata = json.loads(data)

            if isinstance(jdata, dict) and jdata.get('documentation_url'):
                if C.DEFAULT_BREAKPOINTS:
                    import epdb; epdb.st()
                else:
                    raise RateLimitError("rate limited")
            else:
                break

        return jdata
Ejemplo n.º 2
0
    def get_request(self, url):
        '''Get an arbitrary API endpoint'''

        headers = {
            u'Accept': u','.join(self.accepts_headers),
            u'Authorization': u'Bearer %s' % self.token,
        }

        rr = requests.get(url, headers=headers)
        data = rr.json()

        # handle ratelimits ...
        if isinstance(data, dict) and data.get(u'message'):
            if data[u'message'].lower().startswith(u'api rate limit exceeded'):
                raise RateLimitError()

        # pagination
        if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'):
            _data = self.get_request(rr.links[u'next'][u'url'])
            try:
                if isinstance(data, list):
                    data += _data
                elif isinstance(data, dict):
                    data.update(_data)
            except TypeError as e:
                if C.DEFAULT_BREAKPOINTS:
                    logging.error(u'breakpoint!')
                    import epdb
                    epdb.st()
                else:
                    raise Exception(e)

        return data
Ejemplo n.º 3
0
    def get_request(self, url):
        '''Get an arbitrary API endpoint'''

        accepts = [
            u'application/json',
            u'application/vnd.github.mockingbird-preview',
            u'application/vnd.github.sailor-v-preview+json',
            u'application/vnd.github.starfox-preview+json',
            u'application/vnd.github.squirrel-girl-preview',
            u'application/vnd.github.v3+json',
        ]

        headers = {
            u'Accept': u','.join(accepts),
            u'Authorization': u'Bearer %s' % self.token,
        }

        rr = requests.get(url, headers=headers)
        data = rr.json()

        # handle ratelimits ...
        if isinstance(data, dict) and data.get(u'message'):
            if data[u'message'].lower().startswith(u'api rate limit exceeded'):
                raise RateLimitError()

        if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'):
            _data = self.get_request(rr.links[u'next'][u'url'])
            data += _data

        return data
Ejemplo n.º 4
0
    def get_cached_request(self, url):
        '''Use a combination of sqlite and ondisk caching to GET an api resource'''

        url_parts = url.split('/')

        cdf = os.path.join(self.cached_requests_dir,
                           url.replace('https://', '') + '.json.gz')
        cdd = os.path.dirname(cdf)
        if not os.path.exists(cdd):
            os.makedirs(cdd)

        # FIXME - commits are static and can always be used from cache.
        if url_parts[-2] == 'commits' and os.path.exists(cdf):
            return read_gzip_json_file(cdf)

        headers = {
            u'Accept': u','.join(self.accepts_headers),
            u'Authorization': u'Bearer %s' % self.token,
        }

        meta = ADB.get_github_api_request_meta(url, token=self.token)
        if meta is None:
            meta = {}

        # https://developer.github.com/v3/#conditional-requests
        etag = meta.get('etag')
        if etag and os.path.exists(cdf):
            headers['If-None-Match'] = etag

        rr = requests.get(url, headers=headers)

        if rr.status_code == 304:
            # not modified
            with open(cdf, 'r') as f:
                data = json.loads(f.read())
        else:
            data = rr.json()

            # handle ratelimits ...
            if isinstance(data, dict) and data.get(u'message'):
                if data[u'message'].lower().startswith(
                        u'api rate limit exceeded'):
                    raise RateLimitError()

            # cache data to disk
            logging.debug('write %s' % cdf)
            write_gzip_json_file(cdf, data)

        # save the meta
        ADB.set_github_api_request_meta(url, rr.headers, cdf, token=self.token)

        # pagination
        if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'):
            _data = self.get_request(rr.links[u'next'][u'url'])
            if isinstance(data, list):
                data += _data
            else:
                data.update(_data)

        return data
Ejemplo n.º 5
0
    def get_reviews(self):
        # https://developer.github.com/
        #   early-access/graphql/enum/pullrequestreviewstate/
        # https://developer.github.com/v3/
        #   pulls/reviews/#list-reviews-on-a-pull-request
        reviews_url = self.pullrequest.url + u'/reviews'
        headers = {
            u'Accept': u'application/vnd.github.black-cat-preview+json',
        }

        status, hdrs, body = self.instance._requester.requestJson(
            u'GET', reviews_url, headers=headers)
        jdata = json.loads(body)

        # need to catch rate limit message here
        if isinstance(jdata, dict) and u'rate' in jdata[u'message']:
            raise RateLimitError("rate limited")

        if isinstance(jdata, dict):
            logging.error(
                u'get_reviews | pr_reviews.keys=%s | pr_reviews.len=%s | '
                u'resp.headers=%s | resp.status=%s',
                jdata.keys(),
                len(jdata),
                hdrs,
                status,
            )

        return jdata
Ejemplo n.º 6
0
    def get_request(self, url):
        '''Get an arbitrary API endpoint'''

        headers = {
            'Accept': ','.join(HEADERS),
            'Authorization': 'Bearer %s' % self.token,
        }

        rr = requests.get(url, headers=headers)
        data = rr.json()

        # handle ratelimits ...
        if isinstance(data, dict) and data.get('message'):
            if data['message'].lower().startswith('api rate limit exceeded'):
                raise RateLimitError()

        # pagination
        if hasattr(rr, 'links') and rr.links and rr.links.get('next'):
            _data = self.get_request(rr.links['next']['url'])
            if isinstance(data, list):
                data += _data
            elif isinstance(data, dict):
                data.update(_data)

        return data
Ejemplo n.º 7
0
    def get_reviews(self):
        # https://developer.github.com/
        #   early-access/graphql/enum/pullrequestreviewstate/
        # https://developer.github.com/v3/
        #   pulls/reviews/#list-reviews-on-a-pull-request
        reviews_url = self.pullrequest.url + u'/reviews'
        headers = {
            u'Accept': u'application/vnd.github.black-cat-preview+json',
        }

        status, hdrs, body = self.instance._requester.requestJson(
            u'GET', reviews_url, headers=headers)
        jdata = json.loads(body)

        if isinstance(jdata, dict):
            logging.error(
                u'get_reviews | pr_reviews.keys=%s | pr_reviews.len=%s | '
                u'resp.headers=%s | resp.status=%s',
                jdata.keys(),
                len(jdata),
                hdrs,
                status,
            )

            is_rate_limited = u'rate' in jdata[u'message']
            is_server_error = (u'Server Error' == jdata[u'message']
                               or 500 <= status < 600)

            if is_rate_limited:
                raise RateLimitError("rate limited")

            if is_server_error:
                raise RateLimitError("server error")

            raise RateLimitError("unknown error: GH responded with a dict "
                                 "while a list of reviews was expected")

        return jdata
Ejemplo n.º 8
0
    def get_reviews(self):
        # https://developer.github.com/
        #   early-access/graphql/enum/pullrequestreviewstate/
        # https://developer.github.com/v3/
        #   pulls/reviews/#list-reviews-on-a-pull-request
        reviews_url = self.pullrequest.url + '/reviews'
        headers = dict(
            Accept='application/vnd.github.black-cat-preview+json', )

        resp = self.instance._requester.requestJson('GET',
                                                    reviews_url,
                                                    headers=headers)
        jdata = json.loads(resp[2])

        # need to catch rate limit message here
        if isinstance(jdata, dict) and 'rate' in jdata:
            raise RateLimitError("rate limited")

        return jdata
Ejemplo n.º 9
0
    def get_request(self, url):
        '''Get an arbitrary API endpoint'''

        headers = {
            u'Accept': u','.join(self.accepts_headers),
            u'Authorization': u'Bearer %s' % self.token,
        }

        rr = requests.get(url, headers=headers)
        data = rr.json()

        # handle ratelimits ...
        if isinstance(data, dict) and data.get(u'message'):
            if data[u'message'].lower().startswith(u'api rate limit exceeded'):
                raise RateLimitError()

        # pagination
        if hasattr(rr, u'links') and rr.links and rr.links.get(u'next'):
            _data = self.get_request(rr.links[u'next'][u'url'])
            data += _data

        return data