def request(self, method='GET', host=None, uri='/', headers=None, query=None, body=b'',
region=None, service=None,
expires=None,
date=time.gmtime(),
signmethod=None,
statusexpected=None,
xmlexpected=True,
inputobject=None,
operationtimeout=None,
retry=None,
receptiontimeout=None,
_inputIOWrapper=None):
if retry is None: retry = self.HTTP_CONNECTION_RETRY_NUMBER
if receptiontimeout is None: receptiontimeout = self.HTTP_RECEPTION_TIMEOUT
if host is None: host = self.endpoint
_redirectcount = 0
_retrycount = 0
_outputiooffset = None
if headers is None: headers = {}
if query is None: query = {}
if statusexpected is None: statusexpected = [200]
headers['Connection'] = 'keep-alive'
starttime = time.time()
while True:
# if request body is an io like object ensure that we set the pointer back to the start before retrying
if hasattr(body, 'reset'):
if _redirectcount > 0:
body.reset()
else:
try:
if _outputiooffset is None:
_outputiooffset = body.tell()
else:
body.seek(_outputiooffset, io.SEEK_SET)
except:
pass
conn = self.getConnection(destination=host, timeout=receptiontimeout)
headers, query, body = auth.signRequest(access_key=self.access_key, secret_key=self.secret_key,
endpoint=host, region=region, service=service,
signmethod=signmethod, date=date,
uri=uri, method=method, headers=headers,
query=query, body=body, expires=expires)
self.logger.debug("Requesting %s %s %s query=%s headers=%s", method, host, uri, query, headers)
if query != {}:
url = "%s?%s" % (uri, auth.canonicalQueryString(query))
else:
url = uri
#counting the requests
if method not in self.count:
self.count[method] = 0
self.count[method] += 1
try:
conn.request(method=method, url=url, body=body, headers=headers)
response = conn.getresponse()
except Exception as _e:
if _retrycount <= retry:
_retrycount += 1
time.sleep(self.COOL_DOWN_BETWEEN_RETRIES)
continue
raise
data = None
if xmlexpected or ('Content-Type' in response.headers and
response.headers['Content-Type'] in ('application/xml', 'text/xml')):
if ('Content-Length' not in response.headers) and \
('Transfer-Encoding' not in response.headers or response.headers['Transfer-Encoding'] != 'chunked') and \
('Connection' not in response.headers or response.headers['Connection'] != 'close'):
#every xml response should have 'Content-Length' set or 'Transfer-Encoding' = chunked
#take a peek of the data then consume the rest of it
try:
data = response.read(1024)
while response.read(32768) != b'':
pass
except:
pass
resultdata = {'status':response.status, 'reason':response.reason, 'headers':dict(response.headers),
'data':data, 'type':'error'}
raise AWSDataException('missing content length', resultdata)
handler = AWSXMLHandler()
incrementalParser = xml.sax.make_parser()
incrementalParser.setContentHandler(handler)
doretry = False
while True:
try:
if (operationtimeout is not None) and (time.time() - starttime > operationtimeout):
raise AWSTimeout('operation timeout')
data = response.read(amt=32768)
except:
# TODO: not all exception should retry, maybe an exception white list would be the way to go
if _retrycount <= retry:
_retrycount += 1
doretry = True
time.sleep(self.COOL_DOWN_BETWEEN_RETRIES)
break
raise
if len(data) == 0:
break
self.logger.debug("Received >> %s", data)
incrementalParser.feed(data)
if doretry:
continue
awsresponse = handler.getdict()
if awsresponse is not None:
checkResult = self.checkForErrors(awsresponse, response.status, response.reason, response.headers)
if checkResult is not None:
#TODO: some of the errors should reset/close the connection
self.logger.debug("request:checkForErrors %s",checkResult)
#the checkResult is redirect host and exception
if isinstance(checkResult, dict):
if _redirectcount < 3:
_redirectcount += 1
if "endpoint" in checkResult:
host = checkResult["endpoint"]
#TODO: find out who else may redirect in the future
continue
raise checkResult["exception"]
#the checkResult exception
if _retrycount <= retry:
_retrycount += 1
time.sleep(self.COOL_DOWN_BETWEEN_RETRIES)
continue
raise checkResult
resultdata = {'status':response.status, 'reason':response.reason, 'headers':dict(response.headers),
'awsresponse':awsresponse, 'type':'xmldict'}
if statusexpected is not True and response.status not in statusexpected:
raise AWSStatusException(resultdata)
else:
return resultdata
if statusexpected is not True and response.status not in statusexpected:
#take a peek of the data then consume the rest of it
try:
data = response.read(1024)
while response.read(32768) != b'':
pass
except:
pass
resultdata = {'status':response.status, 'reason':response.reason, 'headers':dict(response.headers),
'data':data, 'type':'error'}
raise AWSStatusException(resultdata)
if 'Content-Length' not in response.headers:
#every non xml response should have 'Content-Length' set
#take a peek of the data then consume the rest of it
try:
data = response.read(1024)
while response.read(32768) != b'':
pass
except:
pass
if data == b'':
return {'status':response.status, 'reason':response.reason,
'headers':dict(response.headers), 'type':'empty'}
raise AWSDataException('missing content length', {'status':response.status, 'reason':response.reason,
'headers':dict(response.headers),'data':data,
'type':'error'})
#if we are here then most probably we want to download some data
size = int(response.headers['Content-Length'])
sizeinfo = {}
if response.status == 206:
contentrange = response.headers['Content-Range']
contentrange = contentrange.split(' ')[1].split('/')
crange = contentrange[0].split('-')
sizeinfo = {'size': int(contentrange[1]), 'start': int(crange[0]), 'end': int(crange[1]), 'downloaded': 0}
elif response.status == 200:
sizeinfo = {'size': size, 'start': 0, 'end': size - 1, 'downloaded': 0}
if inputobject is None:
if size > self.MAX_IN_MEMORY_READ_CHUNK_SIZE_FOR_RAW_DATA:
inputobject = tempfile.TemporaryFile(mode="w+b", dir=self.TEMP_DIR, prefix='awstmp-')
else:
inputobject = io.BytesIO()
if _inputIOWrapper is not None:
inputobject = _inputIOWrapper(inputobject)
ammount = 0
while True:
try:
if (operationtimeout is not None) and (time.time() - starttime > operationtimeout):
raise AWSTimeout('operation timeout')
data = response.read(32768)
ammount += len(data)
except Exception as e:
if ammount > 0:
# don't loose partial data yet, it may be useful even in this situation
sizeinfo['downloaded'] = ammount
raise AWSPartialReception(status=response.status, reason=response.reason,
headers=dict(response.headers), data=inputobject, sizeinfo=sizeinfo,
exception=e)
# TODO: not all exception should retry, maybe an exception white list would be the way to go
if _retrycount <= retry:
_retrycount += 1
time.sleep(self.COOL_DOWN_BETWEEN_RETRIES)
break
raise
if (data == b'') or (ammount > size):
return {'status':response.status, 'reason':response.reason, 'headers':dict(response.headers),
'sizeinfo':sizeinfo, 'type':'raw', 'inputobject':inputobject}
inputobject.write(data)
# if we are here then we should retry
continue
def request(
self,
callback,
endpoint=None,
method="GET",
uri="/",
query=None,
headers=None,
statusexpected=None,
body=b"",
signmethod=None,
region=None,
service=None,
date=time.gmtime(),
xmlexpected=True,
connect_timeout=2,
request_timeout=5,
):
if endpoint is None:
endpoint = self.endpoint
if statusexpected is None:
statusexpected = [200]
headers, query, body = auth.signRequest(
access_key=self.access_key,
secret_key=self.secret_key,
endpoint=endpoint,
region=region,
service=service,
signmethod=signmethod,
date=date,
uri=uri,
method=method,
headers=headers,
query=query,
body=body,
)
awsresponse = []
handler = AWSXMLHandler()
incrementalParser = xml.sax.make_parser()
incrementalParser.setContentHandler(handler)
streamingCallback = functools.partial(self.streamingCallback, incrementalParser, awsresponse)
protocol = "https" if self.secure else "http"
# counting the requests
if method not in self.count:
self.count[method] = 0
self.count[method] += 1
if method != "POST":
body = None
request = tornado.httpclient.HTTPRequest(
"%s://%s%s?%s" % (protocol, endpoint, uri, auth.canonicalQueryString(query)),
headers=headers,
body=body,
streaming_callback=streamingCallback,
connect_timeout=connect_timeout,
request_timeout=request_timeout,
method=method,
)
# TODO: timeout handling
response = yield tornado.gen.Task(self.http_client.fetch, request)
resultdata = {"status": response.code, "headers": dict(response.headers), "data": None}
if response.code == 599:
raise AWSStatusException(resultdata)
if not hasattr(handler, "exception"):
awsresponsexml = handler.getdict()
self.checkForErrors(awsresponsexml, response.code, "", response.headers)
# TODO: redirect handling
else:
if xmlexpected:
raise AWSDataException("xml-expected")
if xmlexpected:
resultdata["data"] = awsresponsexml
else:
resultdata["data"] = b"".join(awsresponse)
if statusexpected is not True and response.code not in statusexpected:
raise AWSStatusException(resultdata)
self._ioloop.add_callback(functools.partial(callback, resultdata))
def request(self, callback, endpoint=None, method='GET', uri='/', query=None, headers=None, statusexpected=None,
body=b'', signmethod=None, region=None, service=None, date=time.gmtime(), xmlexpected=True,
connect_timeout=2, request_timeout=5):
if query is None:
query = {}
if headers is None:
headers = {}
if endpoint is None: endpoint = self.endpoint
if statusexpected is None: statusexpected = [200]
headers, query, body = auth.signRequest(access_key=self.access_key, secret_key=self.secret_key,
endpoint=endpoint, region=region, service=service,
signmethod=signmethod, date=date,
uri=uri, method=method, headers=headers,
query=query, body=body)
awsresponse = []
handler = AWSXMLHandler()
incrementalParser = xml.sax.make_parser()
incrementalParser.setContentHandler(handler)
streamingCallback = functools.partial(self.streamingCallback, incrementalParser, awsresponse)
protocol = 'https' if self.secure else 'http'
#counting the requests
if method not in self.count:
self.count[method] = 0
self.count[method] += 1
if method not in ["POST", "PUT"]:
body = None
request = tornado.httpclient.HTTPRequest("%s://%s%s?%s" % (protocol, endpoint, uri, auth.canonicalQueryString(query)),
headers=headers, body=body, streaming_callback=streamingCallback,
connect_timeout=connect_timeout, request_timeout=request_timeout, method=method)
#TODO: timeout & retry handling
response = yield tornado.gen.Task(self.http_client.fetch, request)
resultdata = {'status': response.code, 'headers': dict(response.headers), 'data':None}
if response.code == 599:
resultdata['reason'] = response.reason
raise AWSStatusException(resultdata)
if not hasattr(handler, 'exception'):
awsresponsexml = handler.getdict()
if awsresponsexml is not None:
checkResult = self.checkForErrors(awsresponsexml, response.code, '', response.headers)
if checkResult is not None:
#TODO: redirect handling
if isinstance(checkResult, dict): raise checkResult['exception']
#TODO: retry handling
raise checkResult
else:
if xmlexpected:
raise AWSDataException('xml-expected', '')
if xmlexpected:
resultdata['data'] = awsresponsexml
else:
resultdata['data'] = b''.join(awsresponse)
if statusexpected is not True and response.code not in statusexpected:
raise AWSStatusException(resultdata)
self._ioloop.add_callback(functools.partial(callback, resultdata))