Example #1
0
    def request(self, method='GET', host=None, uri='/', headers=None, query=None, body=b'',
                region=None, service=None,
                expires=None,
                date=time.gmtime(),
                signmethod=None,
                statusexpected=None,
                xmlexpected=True,
                inputobject=None,
                operationtimeout=None,
                retry=None,
                receptiontimeout=None,
                _inputIOWrapper=None):

        if retry is None: retry = self.HTTP_CONNECTION_RETRY_NUMBER
        if receptiontimeout is None: receptiontimeout = self.HTTP_RECEPTION_TIMEOUT
        if host is None: host = self.endpoint

        _redirectcount = 0
        _retrycount = 0
        _outputiooffset = None

        if headers is None: headers = {}
        if query is None: query = {}
        if statusexpected is None: statusexpected = [200]
        headers['Connection'] = 'keep-alive'
        starttime = time.time()

        while True:
            # if request body is an io like object ensure that we set the pointer back to the start before retrying
            if hasattr(body, 'reset'):
                if _redirectcount > 0:
                    body.reset()
            else:
                try:
                    if _outputiooffset is None:
                        _outputiooffset = body.tell()
                    else:
                        body.seek(_outputiooffset, io.SEEK_SET)
                except:
                    pass

            conn = self.getConnection(destination=host, timeout=receptiontimeout)

            headers, query, body = auth.signRequest(access_key=self.access_key, secret_key=self.secret_key,
                                                         endpoint=host, region=region, service=service,
                                                         signmethod=signmethod, date=date,
                                                         uri=uri, method=method, headers=headers,
                                                         query=query, body=body, expires=expires)

            self.logger.debug("Requesting %s %s %s query=%s headers=%s", method, host, uri, query, headers)

            if query != {}:
                url = "%s?%s" % (uri, auth.canonicalQueryString(query))
            else:
                url = uri

            #counting the requests
            if method not in self.count:
                self.count[method] = 0
            self.count[method] += 1

            try:
                conn.request(method=method, url=url, body=body, headers=headers)
                response = conn.getresponse()
            except Exception as _e:
                if _retrycount <= retry:
                    _retrycount += 1
                    time.sleep(self.COOL_DOWN_BETWEEN_RETRIES)
                    continue
                raise

            data = None

            if xmlexpected or ('Content-Type' in response.headers and
                               response.headers['Content-Type'] in ('application/xml', 'text/xml')):

                if ('Content-Length' not in response.headers) and \
                   ('Transfer-Encoding' not in response.headers or response.headers['Transfer-Encoding'] != 'chunked') and \
                   ('Connection' not in response.headers or response.headers['Connection'] != 'close'):
                #every xml response should have 'Content-Length' set or 'Transfer-Encoding' = chunked
                #take a peek of the data then consume the rest of it
                    try:
                        data = response.read(1024)
                        while response.read(32768) != b'':
                            pass
                    except:
                        pass
                    resultdata = {'status':response.status, 'reason':response.reason, 'headers':dict(response.headers),
                                  'data':data, 'type':'error'}
                    raise AWSDataException('missing content length', resultdata)

                handler = AWSXMLHandler()
                incrementalParser = xml.sax.make_parser()
                incrementalParser.setContentHandler(handler)

                doretry = False
                while True:
                    try:
                        if (operationtimeout is not None) and (time.time() - starttime > operationtimeout):
                            raise AWSTimeout('operation timeout')
                        data = response.read(amt=32768)
                    except:
                        # TODO: not all exception should retry, maybe an exception white list would be the way to go
                        if _retrycount <= retry:
                            _retrycount += 1
                            doretry = True
                            time.sleep(self.COOL_DOWN_BETWEEN_RETRIES)
                            break
                        raise
                    if len(data) == 0:
                        break
                    self.logger.debug("Received >> %s", data)
                    incrementalParser.feed(data)
                if doretry:
                    continue

                awsresponse = handler.getdict()

                if awsresponse is not None:
                    checkResult = self.checkForErrors(awsresponse, response.status, response.reason, response.headers)
                    if checkResult is not None:
                        #TODO: some of the errors should reset/close the connection
                        self.logger.debug("request:checkForErrors %s",checkResult)
                        #the checkResult is redirect host and exception
                        if isinstance(checkResult, dict):
                            if _redirectcount < 3:
                                _redirectcount += 1
                                if "endpoint" in checkResult:
                                    host = checkResult["endpoint"]
                                #TODO: find out who else may redirect in the future
                                continue
                            raise checkResult["exception"]
                        #the checkResult exception
                        if _retrycount <= retry:
                            _retrycount += 1
                            time.sleep(self.COOL_DOWN_BETWEEN_RETRIES)
                            continue
                        raise checkResult

                resultdata = {'status':response.status, 'reason':response.reason, 'headers':dict(response.headers),
                              'awsresponse':awsresponse, 'type':'xmldict'}

                if statusexpected is not True and response.status not in statusexpected:
                    raise AWSStatusException(resultdata)
                else:
                    return resultdata

            if statusexpected is not True and response.status not in statusexpected:
                #take a peek of the data then consume the rest of it
                try:
                    data = response.read(1024)
                    while response.read(32768) != b'':
                        pass
                except:
                    pass
                resultdata = {'status':response.status, 'reason':response.reason, 'headers':dict(response.headers),
                              'data':data, 'type':'error'}
                raise AWSStatusException(resultdata)

            if 'Content-Length' not in response.headers:
                #every non xml response should have  'Content-Length' set
                #take a peek of the data then consume the rest of it
                try:
                    data = response.read(1024)
                    while response.read(32768) != b'':
                        pass
                except:
                    pass
                if data == b'':
                    return {'status':response.status, 'reason':response.reason,
                            'headers':dict(response.headers), 'type':'empty'}

                raise AWSDataException('missing content length', {'status':response.status, 'reason':response.reason,
                                                                  'headers':dict(response.headers),'data':data,
                                                                  'type':'error'})

            #if we are here then most probably we want to download some data
            size = int(response.headers['Content-Length'])
            sizeinfo = {}
            if response.status == 206:
                contentrange = response.headers['Content-Range']
                contentrange = contentrange.split(' ')[1].split('/')
                crange = contentrange[0].split('-')
                sizeinfo = {'size': int(contentrange[1]), 'start': int(crange[0]), 'end': int(crange[1]), 'downloaded': 0}
            elif response.status == 200:
                sizeinfo = {'size': size, 'start': 0, 'end': size - 1, 'downloaded': 0}

            if inputobject is None:
                if size > self.MAX_IN_MEMORY_READ_CHUNK_SIZE_FOR_RAW_DATA:
                    inputobject = tempfile.TemporaryFile(mode="w+b", dir=self.TEMP_DIR, prefix='awstmp-')
                else:
                    inputobject = io.BytesIO()
                if _inputIOWrapper is not None:
                    inputobject = _inputIOWrapper(inputobject)

            ammount = 0

            while True:
                try:
                    if (operationtimeout is not None) and (time.time() - starttime > operationtimeout):
                        raise AWSTimeout('operation timeout')
                    data = response.read(32768)
                    ammount += len(data)

                except Exception as e:
                    if ammount > 0:
                        # don't loose partial data yet, it may be useful even in this situation
                        sizeinfo['downloaded'] = ammount
                        raise AWSPartialReception(status=response.status, reason=response.reason,
                                                  headers=dict(response.headers), data=inputobject, sizeinfo=sizeinfo,
                                                  exception=e)

                    # TODO: not all exception should retry, maybe an exception white list would be the way to go
                    if _retrycount <= retry:
                        _retrycount += 1
                        time.sleep(self.COOL_DOWN_BETWEEN_RETRIES)
                        break
                    raise

                if (data == b'') or (ammount > size):
                    return {'status':response.status, 'reason':response.reason, 'headers':dict(response.headers),
                            'sizeinfo':sizeinfo, 'type':'raw', 'inputobject':inputobject}
                inputobject.write(data)

            # if we are here then we should retry
            continue
Example #2
0
    def request(
        self,
        callback,
        endpoint=None,
        method="GET",
        uri="/",
        query=None,
        headers=None,
        statusexpected=None,
        body=b"",
        signmethod=None,
        region=None,
        service=None,
        date=time.gmtime(),
        xmlexpected=True,
        connect_timeout=2,
        request_timeout=5,
    ):

        if endpoint is None:
            endpoint = self.endpoint
        if statusexpected is None:
            statusexpected = [200]
        headers, query, body = auth.signRequest(
            access_key=self.access_key,
            secret_key=self.secret_key,
            endpoint=endpoint,
            region=region,
            service=service,
            signmethod=signmethod,
            date=date,
            uri=uri,
            method=method,
            headers=headers,
            query=query,
            body=body,
        )

        awsresponse = []
        handler = AWSXMLHandler()
        incrementalParser = xml.sax.make_parser()
        incrementalParser.setContentHandler(handler)
        streamingCallback = functools.partial(self.streamingCallback, incrementalParser, awsresponse)

        protocol = "https" if self.secure else "http"

        # counting the requests
        if method not in self.count:
            self.count[method] = 0
        self.count[method] += 1

        if method != "POST":
            body = None
        request = tornado.httpclient.HTTPRequest(
            "%s://%s%s?%s" % (protocol, endpoint, uri, auth.canonicalQueryString(query)),
            headers=headers,
            body=body,
            streaming_callback=streamingCallback,
            connect_timeout=connect_timeout,
            request_timeout=request_timeout,
            method=method,
        )

        # TODO: timeout handling

        response = yield tornado.gen.Task(self.http_client.fetch, request)

        resultdata = {"status": response.code, "headers": dict(response.headers), "data": None}

        if response.code == 599:
            raise AWSStatusException(resultdata)

        if not hasattr(handler, "exception"):
            awsresponsexml = handler.getdict()
            self.checkForErrors(awsresponsexml, response.code, "", response.headers)
            # TODO: redirect handling
        else:
            if xmlexpected:
                raise AWSDataException("xml-expected")

        if xmlexpected:
            resultdata["data"] = awsresponsexml
        else:
            resultdata["data"] = b"".join(awsresponse)

        if statusexpected is not True and response.code not in statusexpected:
            raise AWSStatusException(resultdata)

        self._ioloop.add_callback(functools.partial(callback, resultdata))
Example #3
0
    def request(self, callback, endpoint=None, method='GET', uri='/', query=None, headers=None, statusexpected=None,
                body=b'', signmethod=None, region=None, service=None, date=time.gmtime(), xmlexpected=True,
                connect_timeout=2, request_timeout=5):

        if query is None:
            query = {}
        if headers is None:
            headers = {}

        if endpoint is None: endpoint = self.endpoint
        if statusexpected is None: statusexpected = [200]
        headers, query, body = auth.signRequest(access_key=self.access_key, secret_key=self.secret_key,
                                                endpoint=endpoint, region=region, service=service,
                                                signmethod=signmethod, date=date,
                                                uri=uri, method=method, headers=headers,
                                                query=query, body=body)

        awsresponse = []
        handler = AWSXMLHandler()
        incrementalParser = xml.sax.make_parser()
        incrementalParser.setContentHandler(handler)
        streamingCallback = functools.partial(self.streamingCallback, incrementalParser, awsresponse)

        protocol = 'https' if self.secure else 'http'

        #counting the requests
        if method not in self.count:
            self.count[method] = 0
        self.count[method] += 1

        if method not in ["POST", "PUT"]:
            body = None
        request = tornado.httpclient.HTTPRequest("%s://%s%s?%s" % (protocol, endpoint, uri, auth.canonicalQueryString(query)),
                                                 headers=headers, body=body, streaming_callback=streamingCallback,
                                                 connect_timeout=connect_timeout, request_timeout=request_timeout, method=method)

        #TODO: timeout & retry handling

        response = yield tornado.gen.Task(self.http_client.fetch, request)

        resultdata = {'status': response.code, 'headers': dict(response.headers), 'data':None}

        if response.code == 599:
            resultdata['reason'] = response.reason
            raise AWSStatusException(resultdata)

        if not hasattr(handler, 'exception'):
            awsresponsexml = handler.getdict()
            if awsresponsexml is not None:
                checkResult = self.checkForErrors(awsresponsexml, response.code, '', response.headers)
                if checkResult is not None:
                    #TODO: redirect handling
                    if isinstance(checkResult, dict): raise checkResult['exception']
                    #TODO: retry handling
                    raise checkResult
        else:
            if xmlexpected:
                raise AWSDataException('xml-expected', '')

        if xmlexpected:
            resultdata['data'] = awsresponsexml
        else:
            resultdata['data'] = b''.join(awsresponse)

        if statusexpected is not True and response.code not in statusexpected:
            raise AWSStatusException(resultdata)

        self._ioloop.add_callback(functools.partial(callback, resultdata))