def deleteObject(self, callback, bucketname, objectname):
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
yield tornado.gen.Task(self.request, uri=uri + urlquote(objectname), endpoint=endpoint, statusexpected=[204],
xmlexpected=False, method='DELETE', signmethod=SIGNATURE_S3_REST)
self._ioloop.add_callback(functools.partial(callback, True))
def initiateMultipartUpload(self, bucketname, objectname,
meta=None,
x_amz_server_side_encryption=None,
x_amz_storage_class=None,
x_amz_website_redirect_location=None,
expires=None,
# TODO: not handled yet
x_amz_acl=None,
x_amz_grant_read=None, x_amz_grant_write=None, x_amz_grant_read_acp=None,
x_amz_grant_write_acp=None,
x_amz_grant_full_control=None
):
"""
x_amz_server_side_encryption valid values None, AES256
x_amz_storage_class valid values None (STANDARD), REDUCED_REDUNDANCY
x_amz_website_redirect_location see http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectPUT.html
"""
headers = {}
if x_amz_server_side_encryption:
headers['x-amz-server-side-encryption'] = x_amz_server_side_encryption
if x_amz_storage_class:
headers['x-amz-storage-class'] = x_amz_storage_class
if x_amz_website_redirect_location:
headers['x-amz-website-redirect-location'] = x_amz_website_redirect_location
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
data = self.request(method="POST", uri=uri + urlquote(objectname), headers=headers, host=endpoint,
query={"uploads": None}, signmethod=SIGNATURE_S3_REST)
data = data['awsresponse']['InitiateMultipartUploadResult']
if data['Bucket'] != bucketname or data['Key'] != objectname:
raise IntegrityCheckException('unexpected bucket/key name received', (data['Bucket'],data['Key']),
(bucketname, objectname))
return data
def putObject(self, callback, bucketname, objectname, value):
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
yield tornado.gen.Task(self.request, uri=uri + urlquote(objectname), endpoint=endpoint, body=value,
xmlexpected=False, method='PUT', signmethod=SIGNATURE_S3_REST)
self._ioloop.add_callback(functools.partial(callback, True))
def createQueue(self, qName, delaySeconds=None, maximumMessageSize=None, messageRetentionPeriod=None,
receiveMessageWaitTimeSeconds=None, visibilityTimeout=None, policy=None):
"""
@param qName: the sqs queue name
@type qName: str
"""
query = {'Action': 'CreateQueue', 'QueueName': qName, 'Version': '2012-11-05'}
i = 1
if delaySeconds is not None:
if not isinstance(delaySeconds, int) or not 0 <= delaySeconds <= 900:
raise UserInputException("param delaySeconds sould be an integer between 0 and 900 (15 minutes)")
query["Attribute.%d.Name"%(i,)] = "DelaySeconds"
query["Attribute.%d.Value"%(i,)] = delaySeconds
i += 1
if maximumMessageSize is not None:
if not isinstance(maximumMessageSize, int) or not 1024 <= maximumMessageSize <= 65536:
raise UserInputException("param maximumMessageSize sould be an integer between 1024 and 65536 (64 KiB)")
query["Attribute.%d.Name"%(i,)] = "MaximumMessageSize"
query["Attribute.%d.Value"%(i,)] = maximumMessageSize
i += 1
if messageRetentionPeriod is not None:
if not isinstance(messageRetentionPeriod, int) or not 60 <= messageRetentionPeriod <= 1209600:
raise UserInputException(
"param messageRetentionPeriod sould be an integer between 60 (1 minute) and 1209600 (14 days)")
query["Attribute.%d.Name"%(i,)] = "MessageRetentionPeriod"
query["Attribute.%d.Value"%(i,)] = messageRetentionPeriod
i += 1
if receiveMessageWaitTimeSeconds is not None:
if not isinstance(receiveMessageWaitTimeSeconds, int) or not 0 <= receiveMessageWaitTimeSeconds <= 20:
raise UserInputException(
"param receiveMessageWaitTimeSeconds sould be an integer between 0 and 20 (secconds)")
query["Attribute.%d.Name"%(i,)] = "ReceiveMessageWaitTimeSeconds"
query["Attribute.%d.Value"%(i,)] = receiveMessageWaitTimeSeconds
i += 1
if visibilityTimeout is not None:
if not isinstance(visibilityTimeout, int) or not 0 <= visibilityTimeout <= 43200:
raise UserInputException("param visibilityTimeout sould be an integer between 0 and 43200 (12 hours)")
query["Attribute.%d.Name"%(i,)] = "VisibilityTimeout"
query["Attribute.%d.Value"%(i,)] = visibilityTimeout
i += 1
if policy is not None:
query["Attribute.%d.Name"%(i,)] = "Policy"
query["Attribute.%d.Value"%(i,)] = urlquote(policy)
i += 1
data = self.request(method="GET", signmethod=SIGNATURE_V4_HEADERS, query=query, region=self.endpoint[4:-14],
service='sqs', host=self.endpoint)
url = data['awsresponse']['CreateQueueResponse']['CreateQueueResult']['QueueUrl']
if url[7] == '/': url = url[8:]
else: url = url[7:]
url = url.split('/')
return {'endpoint': url[0], 'accNumber': url[1], 'qName': url[2]}
def getObject(self, bucketname, objectname, inputobject=None, byterange=None, versionID=None,
if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None,
_inputIOWrapper=None):
"""
range = list(start, end)
inputbuffer = be any object implementing write(bytes)
if no inputbuffer is provided then the response will be depending on the response size an io.BytesIO or a
tempfile.TemporaryFile opened to mode w+b
"""
query = {}
if versionID is not None:
query['vesionId'] = versionID
headers = {}
statusexpected = [404, 200]
if byterange is not None:
if len(byterange) > 1:
headers['Range'] = "bytes=%d-%d" % (byterange[0], byterange[1])
else:
headers['Range'] = "bytes=%d-" % (byterange[0],)
statusexpected = [200, 206]
if if_modified_since is not None:
# TODO: implement
headers['If-Modified-Since'] = None
statusexpected.append(304)
if if_unmodified_since is not None:
# TODO: implement
headers['If-Unmodified-Since'] = None
statusexpected.append(412)
if if_match is not None:
headers['If-Match'] = '"' + if_match + '"'
statusexpected.append(412)
if if_none_match is not None:
headers['If-None-Match'] = '"' + if_none_match + '"'
statusexpected.append(304)
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
data = self.request(method="GET", uri=uri + urlquote(objectname),
headers=headers, host=endpoint, statusexpected=statusexpected,
query=query, inputobject=inputobject, xmlexpected=False,
_inputIOWrapper=_inputIOWrapper, signmethod=SIGNATURE_S3_REST)
headers = data['headers']
result = dict((k, v) for k, v in data['headers'].items() if k in ('ETag',
'x-amz-delete-marker', 'x-amz-expiration',
'x-amz-server-side-encryption',
'x-amz-restore', 'x-amz-version-id',
'x-amz-website-redirect-location')
or k.startswith('x-amz-meta-'))
result['status'] = data['status']
if data['status'] in (200, 206):
result['range'] = data['sizeinfo']
result['data'] = data['inputobject']
return result
def listParts(self, bucketname, objectname, uploadId, max_parts=None, part_number_marker=None):
query = {"uploadId": uploadId}
if max_parts is not None:
query['max-parts'] = max_parts
if part_number_marker is not None:
query['part-number-marker'] = part_number_marker
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
data = self.request(method="GET", uri=uri + urlquote(objectname), host=endpoint, query=query,
signmethod=SIGNATURE_S3_REST)
data = data['awsresponse']['ListPartsResult']
if data['Bucket'] != bucketname or data['Key'] != objectname:
raise IntegrityCheckException('unexpected bucket/key name received', (data['Bucket'],data['Key']),
(bucketname, objectname))
return data
def completeMultipartUpload(self, bucketname, objectname, uploadId, parts):
data = ["<CompleteMultipartUpload>"]
for partnumber in parts:
data.append("<Part><PartNumber>%s</PartNumber><ETag>%s</ETag></Part>" % (partnumber, parts[partnumber]))
data.append("</CompleteMultipartUpload>")
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
data = self.request(method="POST", uri=uri + urlquote(objectname), body="".join(data), host=endpoint,
query={"uploadId": uploadId}, signmethod=SIGNATURE_S3_REST)
data = data['awsresponse']['CompleteMultipartUploadResult']
if data['Bucket'] != bucketname or data['Key'] != objectname:
raise IntegrityCheckException('unexpected bucket/key name received', (data['Bucket'],data['Key']),
(bucketname, objectname))
return data
def deleteObject(self, bucketname, objectname, versionID=None, x_amz_mfa=None):
headers = {}
query = {}
if x_amz_mfa is not None:
headers['x-amz-mfa'] = x_amz_mfa
if versionID is not None:
query['vesionId'] = versionID
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
data = self.request(method="DELETE", uri=uri + urlquote(objectname), host=endpoint, query=query,
statusexpected=[204], signmethod=SIGNATURE_S3_REST, xmlexpected=False)
headers = dict((k, v) for k, v in data['headers'].items() if k in ('x-amz-version-id', 'x-amz-delete-marker'))
return headers
def uploadOjectPart(self, bucketname, objectname, partnumber, uploadid, value, objlen=None, md5digest=None):
headers = {}
if objlen is not None:
headers['Content-Length'] = str(objlen)
if md5digest is not None:
headers['Content-MD5'] = base64.b64encode(md5digest).strip().decode()
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
data = self.request(method="PUT", uri=uri + urlquote(objectname), body=value, headers=headers, host=endpoint,
query={"partNumber": partnumber, "uploadId": uploadid},signmethod=SIGNATURE_S3_REST)
headers = data['headers']
if md5digest is not None:
if headers['ETag'][1:-1] != binascii.hexlify(md5digest).decode():
raise IntegrityCheckException('uploadOjectPart returned unexpected ETag value', headers['ETag'][1:-1],
binascii.hexlify(md5digest).decode())
return dict((k, v) for k, v in headers.items() if k in ('ETag'))
def putObject(self, bucketname, objectname, value, objlen=None, md5digest=None,
meta=None,
x_amz_server_side_encryption=None, x_amz_storage_class=None,
x_amz_website_redirect_location=None,
expires=None,
# TODO: not handled yet
x_amz_acl=None,
x_amz_grant_read=None, x_amz_grant_write=None, x_amz_grant_read_acp=None, x_amz_grant_write_acp=None,
x_amz_grant_full_control=None):
"""
expires: int Number of milliseconds before expiration
meta: a list of meta strings
x_amz_server_side_encryption valid values None, AES256
x_amz_storage_class valid values None (STANDARD), REDUCED_REDUNDANCY
x_amz_website_redirect_location see http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectPUT.html
"""
headers = {}
if expires is not None:
headers['Expires'] = expires
if x_amz_server_side_encryption is not None:
headers['x-amz-server-side-encryption'] = x_amz_server_side_encryption
if x_amz_storage_class is not None:
headers['x-amz-storage-class'] = x_amz_storage_class
if x_amz_website_redirect_location is not None:
headers['x-amz-website-redirect-location'] = x_amz_website_redirect_location
# TODO: implement true support for 'Expect':'100-continue'
if objlen is not None:
headers['Content-Length'] = str(objlen)
if md5digest is not None:
headers['Content-MD5'] = base64.b64encode(md5digest).strip().decode()
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
data = self.request(method="PUT", uri=uri + urlquote(objectname), body=value, headers=headers,
host=endpoint, signmethod=SIGNATURE_S3_REST, xmlexpected=False)
headers = data['headers']
if md5digest is not None:
if headers['ETag'][1:-1] != binascii.hexlify(md5digest).decode():
raise IntegrityCheckException('putObject returned unexpected ETag value',
headers['ETag'][1:-1], binascii.hexlify(md5digest).decode())
return dict((k, v) for k, v in headers.items() if k in ('ETag',
'x-amz-expiration', 'x-amz-server-side-encryption',
'x-amz-version-id'))
def getObject(self, callback, bucketname, objectname, byterange=None, versionID=None,
if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None):
query = {}
if versionID is not None:
query['vesionId'] = versionID
headers = {}
statusexpected = [404, 200]
if byterange is not None:
if len(byterange) > 1:
headers['Range'] = "bytes=%d-%d" % (byterange[0], byterange[1])
else:
headers['Range'] = "bytes=%d-" % (byterange[0],)
statusexpected = [200, 206]
if if_modified_since is not None:
# TODO: implement
headers['If-Modified-Since'] = None
statusexpected.append(304)
if if_unmodified_since is not None:
# TODO: implement
headers['If-Unmodified-Since'] = None
statusexpected.append(412)
if if_match is not None:
headers['If-Match'] = '"' + if_match + '"'
statusexpected.append(412)
if if_none_match is not None:
headers['If-None-Match'] = '"' + if_none_match + '"'
statusexpected.append(304)
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
data = yield tornado.gen.Task(self.request, uri=uri + urlquote(objectname), endpoint=endpoint, query=query,
statusexpected=statusexpected, headers=headers, xmlexpected=False,
signmethod=SIGNATURE_S3_REST)
self._ioloop.add_callback(functools.partial(callback, data))
def putObjectCopy(self, bucketname, objectname, sourcebucketname, sourceobjectname, metadata=None,
x_amz_server_side_encryption=None,
x_amz_storage_class=None,
x_amz_website_redirect_location=None,
x_amz_metadata_directive=None,
x_amz_copy_source_if_match=None, x_amz_copy_source_if_none_match=None,
# TODO: not handled yet
x_amz_copy_source_if_unmodified_since=None, x_amz_copy_source_if_modified_since=None,
x_amz_acl=None,
x_amz_grant_read=None, x_amz_grant_write=None, x_amz_grant_read_acp=None,
x_amz_grant_write_acp=None, x_amz_grant_full_control=None):
headers = {'x-amz-copy-source': "/%s/%s" % (sourcebucketname, sourceobjectname)}
if x_amz_server_side_encryption is not None:
headers['x-amz-server-side-encryption'] = x_amz_server_side_encryption
if x_amz_storage_class is not None:
headers['x-amz-storage-class'] = x_amz_storage_class
if x_amz_website_redirect_location is not None:
headers['x-amz-website-redirect-location'] = x_amz_website_redirect_location
if metadata is not None:
for name in metadata:
headers['x-amz-meta-'+ name] = metadata[name]
headers['x-amz-metadata-directive'] = 'REPLACE'
if x_amz_metadata_directive is not None:
if x_amz_metadata_directive not in ['COPY','REPLACE']:
raise UserInputException('x_amz_metadata_directive valid values: COPY|REPLACE')
headers['x-amz-metadata-directive'] = x_amz_metadata_directive
if x_amz_copy_source_if_match is not None:
headers['x-amz-copy-source-if-match'] = x_amz_copy_source_if_match
if x_amz_copy_source_if_none_match is not None:
headers['x-amz-copy-source-if-none-match'] = x_amz_copy_source_if_none_match
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
data = self.request(method="PUT", uri=uri + urlquote(objectname), headers=headers,
host=endpoint, signmethod=SIGNATURE_S3_REST)
return data['awsresponse']['CopyObjectResult']
def abortMultipartUpload(self, bucketname, objectname, uploadId):
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
self.request(method="DELETE", uri=uri + urlquote(objectname), host=endpoint, statusexpected=[204],
query={"uploadId": uploadId}, signmethod=SIGNATURE_S3_REST)
def getObjectAcl(self, bucketname, objectname):
uri, endpoint = self._buketname2PathAndEndpoint(bucketname)
data = self.request(method="GET", uri=uri + urlquote(objectname), host=endpoint, query={'acl': None},
signmethod=SIGNATURE_S3_REST)
return data['awsresponse']['AccessControlPolicy']
def setQueueAttributes(self, qName, delaySeconds=None, maximumMessageSize=None, messageRetentionPeriod=None,
receiveMessageWaitTimeSeconds=None, visibilityTimeout=None, policy=None):
"""
@param qName: the sqs queue name
@type qName: str
@type delaySeconds: int
@type maximumMessageSize: int
@type messageRetentionPeriod: int
@type receiveMessageWaitTimeSeconds: int
@type visibilityTimeout: int
@type policy: str
@rtype: None
"""
query = {'Action': 'SetQueueAttributes', 'Version': '2012-11-05'}
i = 1
if delaySeconds is not None:
if not isinstance(delaySeconds, int) or not 0 <= delaySeconds <= 900:
raise UserInputException("param delaySeconds sould be an integer between 0 and 900 (15 minutes)")
query["Attribute.%d.Name" % (i,)] = "DelaySeconds"
query["Attribute.%d.Value" % (i,)] = delaySeconds
i += 1
if maximumMessageSize is not None:
if not isinstance(maximumMessageSize, int) or not 1024 <= maximumMessageSize <= 65536:
raise UserInputException("param maximumMessageSize sould be an integer between 1024 and 65536 (64 KiB)")
query["Attribute.%d.Name" % (i,)] = "MaximumMessageSize"
query["Attribute.%d.Value" % (i,)] = maximumMessageSize
i += 1
if messageRetentionPeriod is not None:
if not isinstance(messageRetentionPeriod, int) or not 60 <= messageRetentionPeriod <= 1209600:
raise UserInputException(
"param messageRetentionPeriod sould be an integer between 60 (1 minute) and 1209600 (14 days)")
query["Attribute.%d.Name" % (i,)] = "MessageRetentionPeriod"
query["Attribute.%d.Value" % (i,)] = messageRetentionPeriod
i += 1
if receiveMessageWaitTimeSeconds is not None:
if not isinstance(receiveMessageWaitTimeSeconds, int) or not 0 <= receiveMessageWaitTimeSeconds <= 20:
raise UserInputException(
"param receiveMessageWaitTimeSeconds sould be an integer between 0 and 20 (secconds)")
query["Attribute.%d.Name" % (i,)] = "ReceiveMessageWaitTimeSeconds"
query["Attribute.%d.Value" % (i,)] = receiveMessageWaitTimeSeconds
i += 1
if visibilityTimeout is not None:
if not isinstance(visibilityTimeout, int) or not 0 <= visibilityTimeout <= 43200:
raise UserInputException("param visibilityTimeout sould be an integer between 0 and 43200 (12 hours)")
query["Attribute.%d.Name" % (i,)] = "VisibilityTimeout"
query["Attribute.%d.Value" % (i,)] = visibilityTimeout
i += 1
if policy is not None:
query["Attribute.%d.Name" % (i,)] = "Policy"
query["Attribute.%d.Value" % (i,)] = urlquote(policy)
i += 1
data = self.request(method="GET", signmethod=SIGNATURE_V4_HEADERS, query=query, region=self.endpoint[4:-14],
service='sqs', host=self.endpoint, uri="/%s/%s" % (self.accNumber, qName))
data['awsresponse']['SetQueueAttributesResponse']
