def closeConn(conn, connHolder):
try:
if conn:
conn.close()
except Exception as ex:
LOG(ERROR, ex)
finally:
if connHolder is not None:
connHolder['lock'].acquire()
try:
connHolder['connSet'].remove(conn)
except Exception as ex1:
LOG(WARNING, ex1)
finally:
connHolder['lock'].release()
def getSignature(self, method, bucket, key, args_path, headers, payload=None):
outPut = 'AWS4-HMAC-SHA256' + '\n'
outPut += self.longDate + '\n'
outPut += self.getScope() + '\n'
cannonicalRequest = self.getCanonicalRequest(method, bucket, key, args_path, headers, payload)
LOG(DEBUG, 'v4 cannonicalRequest: %s' % cannonicalRequest)
if IS_PYTHON2:
stringToSign = outPut + self.__shaCannonicalRequest_python2(cannonicalRequest)
signingKey = self.getSigningKey_python2()
else:
stringToSign = outPut + self.__shaCannonicalRequest_python3(cannonicalRequest)
stringToSign = stringToSign.encode('UTF-8')
signingKey = self.getSigningKey_python3()
return self.hmacSha256(signingKey, stringToSign)
def doClose(result, conn, connHolder):
if not result:
closeConn(conn, connHolder)
elif 'close' == result.getheader('connection', '').lower() or 'close' == result.getheader('Connection', '').lower():
LOG(INFO, 'server inform to close connection')
closeConn(conn, connHolder)
elif toInt(result.status) >= 500 or connHolder is None:
closeConn(conn, connHolder)
else:
if connHolder is not None:
connHolder['lock'].acquire()
try:
return connHolder['connSet'].add(conn)
finally:
connHolder['lock'].release()
def parse_xml(cls, conn, methodName=None, connHolder=None, readable=False):
if not conn:
return cls.getNoneResult('connection is null')
result = None
try:
result = conn.getresponse()
if not result:
return cls.getNoneResult('response is null')
return cls.__parse_xml(result, methodName, readable=readable)
except RedirectException as ex:
raise ex
except Exception as e:
LOG(ERROR, traceback.format_exc())
raise e
finally:
GetResult.doClose(result, conn, connHolder)
def parse_xml(cls, conn, methodName=None, connHolder=None):
if not conn:
return cls.getNoneResult('connection is null')
result = None
try:
result = conn.getresponse()
if not result:
return cls.getNoneResult('response is null')
return cls.__parse_xml(result, methodName)
except RedirectException as re:
raise re
except Exception as e:
LOG(ERROR, traceback.format_exc())
return cls.getNoneResult(common_util.toString(e))
finally:
GetResult.doClose(result, conn, connHolder)
def doClose(result, conn, connHolder):
if not result:
closeConn(conn, connHolder)
elif result.getheader('connection', '').lower() == 'close' or result.getheader('Connection', '').lower() == 'close':
LOG(INFO, 'server inform to close connection')
closeConn(conn, connHolder)
elif toInt(result.status) >= 500 or connHolder is None:
closeConn(conn, connHolder)
elif hasattr(conn, '_redirect') and conn._redirect:
closeConn(conn, connHolder)
else:
if connHolder is not None:
connHolder['lock'].acquire()
try:
connHolder['connSet'].add(conn)
finally:
connHolder['lock'].release()
def getSignature(self,
method,
bucket,
key,
path_args,
headers,
expires=None):
canonical_string = self.__make_canonicalstring(method, bucket, key,
path_args, headers,
expires)
LOG(DEBUG, 'v2 canonical_string: %s' % canonical_string)
if IS_PYTHON2:
hashed = hmac.new(self.sk, canonical_string, hashlib.sha1)
encode_canonical = binascii.b2a_base64(hashed.digest())[:-1]
else:
hashed = hmac.new(self.sk.encode('UTF-8'),
canonical_string.encode('UTF-8'), hashlib.sha1)
encode_canonical = binascii.b2a_base64(
hashed.digest())[:-1].decode('UTF-8')
return encode_canonical
def parse_content(cls,
conn,
objectKey,
downloadPath=None,
chuckSize=65536,
loadStreamInMemory=False,
connHolder=None):
if not conn:
return cls.getNoneResult('connection is null')
closeConn = True
result = None
try:
result = conn.getresponse()
if not result:
return cls.getNoneResult('response is null')
if connHolder and hasattr(connHolder, 'createTimeStamp'):
connHolder.createTimeStamp = time.time()
if not common_util.toInt(result.status) < 300:
return cls.__parse_xml(result)
if loadStreamInMemory:
LOG(DEBUG,
'loadStreamInMemory is True, read stream into memory')
buf = None
while True:
chunk = result.read(chuckSize)
if not chunk:
break
if buf is None:
buf = chunk
else:
buf += chunk
body = ObjectStream(buffer=buf,
size=common_util.toLong(len(buf)))
elif downloadPath is None or common_util.toString(
downloadPath).strip() == '':
LOG(DEBUG, 'DownloadPath is null, return conn directly')
closeConn = False
body = ObjectStream(
response=ResponseWrapper(conn, result, connHolder))
else:
objectKey = common_util.safe_encode(objectKey)
downloadPath = common_util.safe_encode(downloadPath)
file_path = cls.get_data(result, downloadPath, chuckSize)
body = ObjectStream(url=common_util.toString(file_path))
LOG(DEBUG,
'DownloadPath is ' + common_util.toString(file_path))
status = common_util.toInt(result.status)
reason = result.reason
headers = dict(result.getheaders())
header = cls.__parse_headers(headers)
requestId = headers.get('x-amz-request-id')
convert_util.parseGetObject(dict(header), body)
return GetResult(status=status,
reason=reason,
header=header,
body=body,
requestId=requestId)
except RedirectException as ex:
raise ex
except Exception as e:
LOG(ERROR, traceback.format_exc())
raise e
finally:
if closeConn:
GetResult.doClose(result, conn, connHolder)
def __parse_xml(cls,
result,
methodName=None,
chuckSize=65536,
readable=False):
status = common_util.toInt(result.status)
reason = result.reason
code = None
message = None
body = None
requestId = None
hostId = None
resource = None
headers = dict(result.getheaders())
xml = None
while True:
chunk = result.read(chuckSize)
if not chunk:
break
xml = chunk if xml is None else xml + chunk
if status == 307 and not readable and ('location' in headers
or 'Location' in headers):
location = headers.get('location')
if location is None:
location = headers.get('Location')
LOG(WARNING, 'http code is %d, need to redirect to %s', status,
location)
cls.CONTEXT.location = location
raise RedirectException(
'http code is {0}, need to redirect to {1}'.format(
status, location))
else:
header = cls.__parse_headers(headers)
if status < 300:
if methodName is not None:
methodName = 'parse' + methodName[:1].upper(
) + methodName[1:]
parseMethod = getattr(convert_util, methodName)
if parseMethod is not None:
if xml:
xml = xml if IS_PYTHON2 else xml.decode('UTF-8')
LOG(DEBUG, 'recv Msg:%s', xml)
try:
search = cls.PATTERN.search(xml)
xml = xml if search is None else xml.replace(
search.group(), '')
body = parseMethod(xml, dict(header))
except Exception as e:
LOG(ERROR, e)
else:
body = parseMethod(dict(header))
requestId = headers.get('x-amz-request-id')
elif xml:
xml = xml if IS_PYTHON2 else xml.decode('UTF-8')
try:
search = cls.PATTERN.search(xml)
xml = xml if search is None else xml.replace(
search.group(), '')
root = ET.fromstring(xml)
code = root.find('./Code')
code = code.text if code is not None else None
message = root.find('./Message')
message = message.text if message is not None else None
requestId = root.find('./RequestId')
requestId = requestId.text if requestId is not None else None
hostId = root.find('./HostId')
hostId = hostId.text if hostId is not None else None
key = root.find('./Key')
bucket = root.find('./BucketName')
resource = bucket if bucket is not None else key
resource = resource.text if resource is not None else None
except Exception as ee:
LOG(ERROR, common_util.toString(ee))
LOG(ERROR, traceback.format_exc())
LOG(
DEBUG,
'http response result:status:%d,reason:%s,code:%s,message:%s,headers:%s',
status, reason, code, message, header)
return GetResult(code=code,
message=message,
status=status,
reason=reason,
body=body,
requestId=requestId,
hostId=hostId,
resource=resource,
header=header)