def upload_part(self, seq, offset, chunk_size, labels, buffer = '', remote_status = None):
"""
Upload a file chunk
http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html
"""
# TODO implement Content-MD5
debug("Uploading part %i of %r (%s bytes)" % (seq, self.upload_id, chunk_size))
if remote_status is not None:
if int(remote_status['size']) == chunk_size:
checksum = calculateChecksum(buffer, self.file, offset, chunk_size, self.s3.config.send_chunk)
remote_checksum = remote_status['checksum'].strip('"\'')
if remote_checksum == checksum:
warning("MultiPart: size and md5sum match for %s part %d, skipping." % (self.uri, seq))
self.parts[seq] = remote_status['checksum']
return
else:
warning("MultiPart: checksum (%s vs %s) does not match for %s part %d, reuploading."
% (remote_checksum, checksum, self.uri, seq))
else:
warning("MultiPart: size (%d vs %d) does not match for %s part %d, reuploading."
% (int(remote_status['size']), chunk_size, self.uri, seq))
headers = { "content-length": str(chunk_size) }
query_string = "?partNumber=%i&uploadId=%s" % (seq, encode_to_s3(self.upload_id))
request = self.s3.create_request("OBJECT_PUT", uri = self.uri, headers = headers, extra = query_string)
response = self.s3.send_file(request, self.file, labels, buffer, offset = offset, chunk_size = chunk_size)
self.parts[seq] = response["headers"]["etag"]
return response
def sign_string_v4(method='GET', host='', canonical_uri='/', params={}, region='us-east-1', cur_headers={}, body=''):
service = 's3'
cfg = Config.Config()
access_key = cfg.access_key
secret_key = cfg.secret_key
t = datetime.datetime.utcnow()
amzdate = t.strftime('%Y%m%dT%H%M%SZ')
datestamp = t.strftime('%Y%m%d')
canonical_querystring = '&'.join(['%s=%s' % (urllib.quote_plus(p), quote_param(params[p])) for p in sorted(params.keys())])
splits = canonical_uri.split('?')
canonical_uri = quote_param(splits[0], quote_backslashes=False)
canonical_querystring += '&'.join([('%s' if '=' in qs else '%s=') % qs for qs in splits[1:]])
if type(body) == type(sha256('')):
payload_hash = body.hexdigest()
else:
payload_hash = sha256(body).hexdigest()
canonical_headers = {'host' : host,
'x-amz-content-sha256': payload_hash,
'x-amz-date' : amzdate
}
signed_headers = 'host;x-amz-content-sha256;x-amz-date'
for header in cur_headers.keys():
# avoid duplicate headers and previous Authorization
if header == 'Authorization' or header in signed_headers.split(';'):
continue
canonical_headers[header.strip()] = str(cur_headers[header]).strip()
signed_headers += ';' + header.strip()
# sort headers into a string
canonical_headers_str = ''
for k, v in sorted(canonical_headers.items()):
canonical_headers_str += k + ":" + v + "\n"
canonical_headers = canonical_headers_str
debug(u"canonical_headers = %s" % canonical_headers)
signed_headers = ';'.join(sorted(signed_headers.split(';')))
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
debug('Canonical Request:\n%s\n----------------------' % canonical_request)
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope + '\n' + sha256(canonical_request).hexdigest()
signing_key = getSignatureKey(secret_key, datestamp, region, service)
signature = hmac.new(signing_key, encode_to_s3(string_to_sign), sha256).hexdigest()
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ',' + 'SignedHeaders=' + signed_headers + ',' + 'Signature=' + signature
headers = dict(cur_headers.items() + {'x-amz-date':amzdate, 'Authorization':authorization_header, 'x-amz-content-sha256': payload_hash}.items())
debug("signature-v4 headers: %s" % headers)
return headers
def sign_string_v4(method='GET', host='', canonical_uri='/', params={}, region='us-east-1', cur_headers={}, body=''):
service = 's3'
cfg = Config.Config()
access_key = cfg.access_key
secret_key = cfg.secret_key
t = datetime.datetime.utcnow()
amzdate = t.strftime('%Y%m%dT%H%M%SZ')
datestamp = t.strftime('%Y%m%d')
canonical_querystring = '&'.join(['%s=%s' % (urllib.quote_plus(p), quote_param(params[p])) for p in sorted(params.keys())])
splits = canonical_uri.split('?')
canonical_uri = quote_param(splits[0], quote_backslashes=False)
canonical_querystring += '&'.join([('%s' if '=' in qs else '%s=') % qs for qs in splits[1:]])
if type(body) == type(sha256('')):
payload_hash = body.hexdigest()
else:
payload_hash = sha256(body).hexdigest()
canonical_headers = {'host' : host,
'x-amz-content-sha256': payload_hash,
'x-amz-date' : amzdate
}
signed_headers = 'host;x-amz-content-sha256;x-amz-date'
for header in cur_headers.keys():
# avoid duplicate headers and previous Authorization
if header == 'Authorization' or header in signed_headers.split(';'):
continue
canonical_headers[header.strip()] = str(cur_headers[header]).strip()
signed_headers += ';' + header.strip()
# sort headers into a string
canonical_headers_str = ''
for k, v in sorted(canonical_headers.items()):
canonical_headers_str += k + ":" + v + "\n"
canonical_headers = canonical_headers_str
debug(u"canonical_headers = %s" % canonical_headers)
signed_headers = ';'.join(sorted(signed_headers.split(';')))
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
debug('Canonical Request:\n%s\n----------------------' % canonical_request)
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope + '\n' + sha256(canonical_request).hexdigest()
signing_key = getSignatureKey(secret_key, datestamp, region, service)
signature = hmac.new(signing_key, encode_to_s3(string_to_sign), sha256).hexdigest()
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ',' + 'SignedHeaders=' + signed_headers + ',' + 'Signature=' + signature
headers = dict(cur_headers.items() + {'x-amz-date':amzdate, 'Authorization':authorization_header, 'x-amz-content-sha256': payload_hash}.items())
debug("signature-v4 headers: %s" % headers)
return headers
def upload_part(self,
seq,
offset,
chunk_size,
labels,
buffer='',
remote_status=None):
"""
Upload a file chunk
http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html
"""
# TODO implement Content-MD5
debug("Uploading part %i of %r (%s bytes)" %
(seq, self.upload_id, chunk_size))
if remote_status is not None:
if int(remote_status['size']) == chunk_size:
checksum = calculateChecksum(buffer, self.file, offset,
chunk_size,
self.s3.config.send_chunk)
remote_checksum = remote_status['checksum'].strip('"\'')
if remote_checksum == checksum:
warning(
"MultiPart: size and md5sum match for %s part %d, skipping."
% (self.uri, seq))
self.parts[seq] = remote_status['checksum']
return
else:
warning(
"MultiPart: checksum (%s vs %s) does not match for %s part %d, reuploading."
% (remote_checksum, checksum, self.uri, seq))
else:
warning(
"MultiPart: size (%d vs %d) does not match for %s part %d, reuploading."
% (int(remote_status['size']), chunk_size, self.uri, seq))
headers = {"content-length": str(chunk_size)}
query_string = "?partNumber=%i&uploadId=%s" % (
seq, encode_to_s3(self.upload_id))
request = self.s3.create_request("OBJECT_PUT",
uri=self.uri,
headers=headers,
extra=query_string)
response = self.s3.send_file(request,
self.file,
labels,
buffer,
offset=offset,
chunk_size=chunk_size)
self.parts[seq] = response["headers"]["etag"]
return response
def complete_multipart_upload(self):
"""
Finish a multipart upload
http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadComplete.html
"""
debug("MultiPart: Completing upload: %s" % self.upload_id)
parts_xml = []
part_xml = "<Part><PartNumber>%i</PartNumber><ETag>%s</ETag></Part>"
for seq, etag in self.parts.items():
parts_xml.append(part_xml % (seq, etag))
body = "<CompleteMultipartUpload>%s</CompleteMultipartUpload>" % (
"".join(parts_xml))
headers = {"content-length": str(len(body))}
request = self.s3.create_request("OBJECT_POST",
uri=self.uri,
headers=headers,
extra="?uploadId=%s" %
encode_to_s3(self.upload_id),
body=body)
response = self.s3.send_request(request)
return response
def getSignatureKey(key, dateStamp, regionName, serviceName):
kDate = sign(encode_to_s3('AWS4' + key), dateStamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
def sign(key, msg):
return hmac.new(key, encode_to_s3(msg), sha256).digest()
def complete_multipart_upload(self):
"""
Finish a multipart upload
http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadComplete.html
"""
debug("MultiPart: Completing upload: %s" % self.upload_id)
parts_xml = []
part_xml = "<Part><PartNumber>%i</PartNumber><ETag>%s</ETag></Part>"
for seq, etag in self.parts.items():
parts_xml.append(part_xml % (seq, etag))
body = "<CompleteMultipartUpload>%s</CompleteMultipartUpload>" % ("".join(parts_xml))
headers = { "content-length": str(len(body)) }
request = self.s3.create_request("OBJECT_POST", uri = self.uri, headers = headers, extra = "?uploadId=%s" % encode_to_s3(self.upload_id), body = body)
response = self.s3.send_request(request)
return response
def sign_string_v4(method="GET", host="", canonical_uri="/", params={}, region="us-east-1", cur_headers={}, body=""):
service = "s3"
cfg = Config.Config()
access_key = cfg.access_key
secret_key = cfg.secret_key
t = datetime.datetime.utcnow()
amzdate = t.strftime("%Y%m%dT%H%M%SZ")
datestamp = t.strftime("%Y%m%d")
canonical_querystring = "&".join(
["%s=%s" % (urllib.quote_plus(p), quote_param(params[p])) for p in sorted(params.keys())]
)
splits = canonical_uri.split("?")
canonical_uri = quote_param(splits[0], quote_backslashes=False)
canonical_querystring += "&".join([("%s" if "=" in qs else "%s=") % qs for qs in splits[1:]])
if type(body) == type(sha256("")):
payload_hash = body.hexdigest()
else:
payload_hash = sha256(body).hexdigest()
canonical_headers = {"host": host, "x-amz-content-sha256": payload_hash, "x-amz-date": amzdate}
signed_headers = "host;x-amz-content-sha256;x-amz-date"
for header in cur_headers.keys():
# avoid duplicate headers and previous Authorization
if header == "Authorization" or header in signed_headers.split(";"):
continue
canonical_headers[header.strip()] = str(cur_headers[header]).strip()
signed_headers += ";" + header.strip()
# sort headers into a string
canonical_headers_str = ""
for k, v in sorted(canonical_headers.items()):
canonical_headers_str += k + ":" + v + "\n"
canonical_headers = canonical_headers_str
debug(u"canonical_headers = %s" % canonical_headers)
signed_headers = ";".join(sorted(signed_headers.split(";")))
canonical_request = (
method
+ "\n"
+ canonical_uri
+ "\n"
+ canonical_querystring
+ "\n"
+ canonical_headers
+ "\n"
+ signed_headers
+ "\n"
+ payload_hash
)
debug("Canonical Request:\n%s\n----------------------" % canonical_request)
algorithm = "AWS4-HMAC-SHA256"
credential_scope = datestamp + "/" + region + "/" + service + "/" + "aws4_request"
string_to_sign = algorithm + "\n" + amzdate + "\n" + credential_scope + "\n" + sha256(canonical_request).hexdigest()
signing_key = getSignatureKey(secret_key, datestamp, region, service)
signature = hmac.new(signing_key, encode_to_s3(string_to_sign), sha256).hexdigest()
authorization_header = (
algorithm
+ " "
+ "Credential="
+ access_key
+ "/"
+ credential_scope
+ ","
+ "SignedHeaders="
+ signed_headers
+ ","
+ "Signature="
+ signature
)
headers = dict(
cur_headers.items()
+ {"x-amz-date": amzdate, "Authorization": authorization_header, "x-amz-content-sha256": payload_hash}.items()
)
debug("signature-v4 headers: %s" % headers)
return headers
def getSignatureKey(key, dateStamp, regionName, serviceName):
kDate = sign(encode_to_s3("AWS4" + key), dateStamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, "aws4_request")
return kSigning
def sign(key, msg):
return hmac.new(key, encode_to_s3(msg), sha256).digest()