def deleteObject(self, callback, bucketname, objectname): uri, endpoint = self._buketname2PathAndEndpoint(bucketname) yield tornado.gen.Task(self.request, uri=uri + urlquote(objectname), endpoint=endpoint, statusexpected=[204], xmlexpected=False, method='DELETE', signmethod=SIGNATURE_S3_REST) self._ioloop.add_callback(functools.partial(callback, True))
def initiateMultipartUpload(self, bucketname, objectname, meta=None, x_amz_server_side_encryption=None, x_amz_storage_class=None, x_amz_website_redirect_location=None, expires=None, # TODO: not handled yet x_amz_acl=None, x_amz_grant_read=None, x_amz_grant_write=None, x_amz_grant_read_acp=None, x_amz_grant_write_acp=None, x_amz_grant_full_control=None ): """ x_amz_server_side_encryption valid values None, AES256 x_amz_storage_class valid values None (STANDARD), REDUCED_REDUNDANCY x_amz_website_redirect_location see http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectPUT.html """ headers = {} if x_amz_server_side_encryption: headers['x-amz-server-side-encryption'] = x_amz_server_side_encryption if x_amz_storage_class: headers['x-amz-storage-class'] = x_amz_storage_class if x_amz_website_redirect_location: headers['x-amz-website-redirect-location'] = x_amz_website_redirect_location uri, endpoint = self._buketname2PathAndEndpoint(bucketname) data = self.request(method="POST", uri=uri + urlquote(objectname), headers=headers, host=endpoint, query={"uploads": None}, signmethod=SIGNATURE_S3_REST) data = data['awsresponse']['InitiateMultipartUploadResult'] if data['Bucket'] != bucketname or data['Key'] != objectname: raise IntegrityCheckException('unexpected bucket/key name received', (data['Bucket'],data['Key']), (bucketname, objectname)) return data
def putObject(self, callback, bucketname, objectname, value): uri, endpoint = self._buketname2PathAndEndpoint(bucketname) yield tornado.gen.Task(self.request, uri=uri + urlquote(objectname), endpoint=endpoint, body=value, xmlexpected=False, method='PUT', signmethod=SIGNATURE_S3_REST) self._ioloop.add_callback(functools.partial(callback, True))
def createQueue(self, qName, delaySeconds=None, maximumMessageSize=None, messageRetentionPeriod=None, receiveMessageWaitTimeSeconds=None, visibilityTimeout=None, policy=None): """ @param qName: the sqs queue name @type qName: str """ query = {'Action': 'CreateQueue', 'QueueName': qName, 'Version': '2012-11-05'} i = 1 if delaySeconds is not None: if not isinstance(delaySeconds, int) or not 0 <= delaySeconds <= 900: raise UserInputException("param delaySeconds sould be an integer between 0 and 900 (15 minutes)") query["Attribute.%d.Name"%(i,)] = "DelaySeconds" query["Attribute.%d.Value"%(i,)] = delaySeconds i += 1 if maximumMessageSize is not None: if not isinstance(maximumMessageSize, int) or not 1024 <= maximumMessageSize <= 65536: raise UserInputException("param maximumMessageSize sould be an integer between 1024 and 65536 (64 KiB)") query["Attribute.%d.Name"%(i,)] = "MaximumMessageSize" query["Attribute.%d.Value"%(i,)] = maximumMessageSize i += 1 if messageRetentionPeriod is not None: if not isinstance(messageRetentionPeriod, int) or not 60 <= messageRetentionPeriod <= 1209600: raise UserInputException( "param messageRetentionPeriod sould be an integer between 60 (1 minute) and 1209600 (14 days)") query["Attribute.%d.Name"%(i,)] = "MessageRetentionPeriod" query["Attribute.%d.Value"%(i,)] = messageRetentionPeriod i += 1 if receiveMessageWaitTimeSeconds is not None: if not isinstance(receiveMessageWaitTimeSeconds, int) or not 0 <= receiveMessageWaitTimeSeconds <= 20: raise UserInputException( "param receiveMessageWaitTimeSeconds sould be an integer between 0 and 20 (secconds)") query["Attribute.%d.Name"%(i,)] = "ReceiveMessageWaitTimeSeconds" query["Attribute.%d.Value"%(i,)] = receiveMessageWaitTimeSeconds i += 1 if visibilityTimeout is not None: if not isinstance(visibilityTimeout, int) or not 0 <= visibilityTimeout <= 43200: raise UserInputException("param visibilityTimeout sould be an integer between 0 and 43200 (12 hours)") query["Attribute.%d.Name"%(i,)] = "VisibilityTimeout" query["Attribute.%d.Value"%(i,)] = visibilityTimeout i += 1 if policy is not None: query["Attribute.%d.Name"%(i,)] = "Policy" query["Attribute.%d.Value"%(i,)] = urlquote(policy) i += 1 data = self.request(method="GET", signmethod=SIGNATURE_V4_HEADERS, query=query, region=self.endpoint[4:-14], service='sqs', host=self.endpoint) url = data['awsresponse']['CreateQueueResponse']['CreateQueueResult']['QueueUrl'] if url[7] == '/': url = url[8:] else: url = url[7:] url = url.split('/') return {'endpoint': url[0], 'accNumber': url[1], 'qName': url[2]}
def getObject(self, bucketname, objectname, inputobject=None, byterange=None, versionID=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, _inputIOWrapper=None): """ range = list(start, end) inputbuffer = be any object implementing write(bytes) if no inputbuffer is provided then the response will be depending on the response size an io.BytesIO or a tempfile.TemporaryFile opened to mode w+b """ query = {} if versionID is not None: query['vesionId'] = versionID headers = {} statusexpected = [404, 200] if byterange is not None: if len(byterange) > 1: headers['Range'] = "bytes=%d-%d" % (byterange[0], byterange[1]) else: headers['Range'] = "bytes=%d-" % (byterange[0],) statusexpected = [200, 206] if if_modified_since is not None: # TODO: implement headers['If-Modified-Since'] = None statusexpected.append(304) if if_unmodified_since is not None: # TODO: implement headers['If-Unmodified-Since'] = None statusexpected.append(412) if if_match is not None: headers['If-Match'] = '"' + if_match + '"' statusexpected.append(412) if if_none_match is not None: headers['If-None-Match'] = '"' + if_none_match + '"' statusexpected.append(304) uri, endpoint = self._buketname2PathAndEndpoint(bucketname) data = self.request(method="GET", uri=uri + urlquote(objectname), headers=headers, host=endpoint, statusexpected=statusexpected, query=query, inputobject=inputobject, xmlexpected=False, _inputIOWrapper=_inputIOWrapper, signmethod=SIGNATURE_S3_REST) headers = data['headers'] result = dict((k, v) for k, v in data['headers'].items() if k in ('ETag', 'x-amz-delete-marker', 'x-amz-expiration', 'x-amz-server-side-encryption', 'x-amz-restore', 'x-amz-version-id', 'x-amz-website-redirect-location') or k.startswith('x-amz-meta-')) result['status'] = data['status'] if data['status'] in (200, 206): result['range'] = data['sizeinfo'] result['data'] = data['inputobject'] return result
def listParts(self, bucketname, objectname, uploadId, max_parts=None, part_number_marker=None): query = {"uploadId": uploadId} if max_parts is not None: query['max-parts'] = max_parts if part_number_marker is not None: query['part-number-marker'] = part_number_marker uri, endpoint = self._buketname2PathAndEndpoint(bucketname) data = self.request(method="GET", uri=uri + urlquote(objectname), host=endpoint, query=query, signmethod=SIGNATURE_S3_REST) data = data['awsresponse']['ListPartsResult'] if data['Bucket'] != bucketname or data['Key'] != objectname: raise IntegrityCheckException('unexpected bucket/key name received', (data['Bucket'],data['Key']), (bucketname, objectname)) return data
def completeMultipartUpload(self, bucketname, objectname, uploadId, parts): data = ["<CompleteMultipartUpload>"] for partnumber in parts: data.append("<Part><PartNumber>%s</PartNumber><ETag>%s</ETag></Part>" % (partnumber, parts[partnumber])) data.append("</CompleteMultipartUpload>") uri, endpoint = self._buketname2PathAndEndpoint(bucketname) data = self.request(method="POST", uri=uri + urlquote(objectname), body="".join(data), host=endpoint, query={"uploadId": uploadId}, signmethod=SIGNATURE_S3_REST) data = data['awsresponse']['CompleteMultipartUploadResult'] if data['Bucket'] != bucketname or data['Key'] != objectname: raise IntegrityCheckException('unexpected bucket/key name received', (data['Bucket'],data['Key']), (bucketname, objectname)) return data
def deleteObject(self, bucketname, objectname, versionID=None, x_amz_mfa=None): headers = {} query = {} if x_amz_mfa is not None: headers['x-amz-mfa'] = x_amz_mfa if versionID is not None: query['vesionId'] = versionID uri, endpoint = self._buketname2PathAndEndpoint(bucketname) data = self.request(method="DELETE", uri=uri + urlquote(objectname), host=endpoint, query=query, statusexpected=[204], signmethod=SIGNATURE_S3_REST, xmlexpected=False) headers = dict((k, v) for k, v in data['headers'].items() if k in ('x-amz-version-id', 'x-amz-delete-marker')) return headers
def uploadOjectPart(self, bucketname, objectname, partnumber, uploadid, value, objlen=None, md5digest=None): headers = {} if objlen is not None: headers['Content-Length'] = str(objlen) if md5digest is not None: headers['Content-MD5'] = base64.b64encode(md5digest).strip().decode() uri, endpoint = self._buketname2PathAndEndpoint(bucketname) data = self.request(method="PUT", uri=uri + urlquote(objectname), body=value, headers=headers, host=endpoint, query={"partNumber": partnumber, "uploadId": uploadid},signmethod=SIGNATURE_S3_REST) headers = data['headers'] if md5digest is not None: if headers['ETag'][1:-1] != binascii.hexlify(md5digest).decode(): raise IntegrityCheckException('uploadOjectPart returned unexpected ETag value', headers['ETag'][1:-1], binascii.hexlify(md5digest).decode()) return dict((k, v) for k, v in headers.items() if k in ('ETag'))
def putObject(self, bucketname, objectname, value, objlen=None, md5digest=None, meta=None, x_amz_server_side_encryption=None, x_amz_storage_class=None, x_amz_website_redirect_location=None, expires=None, # TODO: not handled yet x_amz_acl=None, x_amz_grant_read=None, x_amz_grant_write=None, x_amz_grant_read_acp=None, x_amz_grant_write_acp=None, x_amz_grant_full_control=None): """ expires: int Number of milliseconds before expiration meta: a list of meta strings x_amz_server_side_encryption valid values None, AES256 x_amz_storage_class valid values None (STANDARD), REDUCED_REDUNDANCY x_amz_website_redirect_location see http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectPUT.html """ headers = {} if expires is not None: headers['Expires'] = expires if x_amz_server_side_encryption is not None: headers['x-amz-server-side-encryption'] = x_amz_server_side_encryption if x_amz_storage_class is not None: headers['x-amz-storage-class'] = x_amz_storage_class if x_amz_website_redirect_location is not None: headers['x-amz-website-redirect-location'] = x_amz_website_redirect_location # TODO: implement true support for 'Expect':'100-continue' if objlen is not None: headers['Content-Length'] = str(objlen) if md5digest is not None: headers['Content-MD5'] = base64.b64encode(md5digest).strip().decode() uri, endpoint = self._buketname2PathAndEndpoint(bucketname) data = self.request(method="PUT", uri=uri + urlquote(objectname), body=value, headers=headers, host=endpoint, signmethod=SIGNATURE_S3_REST, xmlexpected=False) headers = data['headers'] if md5digest is not None: if headers['ETag'][1:-1] != binascii.hexlify(md5digest).decode(): raise IntegrityCheckException('putObject returned unexpected ETag value', headers['ETag'][1:-1], binascii.hexlify(md5digest).decode()) return dict((k, v) for k, v in headers.items() if k in ('ETag', 'x-amz-expiration', 'x-amz-server-side-encryption', 'x-amz-version-id'))
def getObject(self, callback, bucketname, objectname, byterange=None, versionID=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None): query = {} if versionID is not None: query['vesionId'] = versionID headers = {} statusexpected = [404, 200] if byterange is not None: if len(byterange) > 1: headers['Range'] = "bytes=%d-%d" % (byterange[0], byterange[1]) else: headers['Range'] = "bytes=%d-" % (byterange[0],) statusexpected = [200, 206] if if_modified_since is not None: # TODO: implement headers['If-Modified-Since'] = None statusexpected.append(304) if if_unmodified_since is not None: # TODO: implement headers['If-Unmodified-Since'] = None statusexpected.append(412) if if_match is not None: headers['If-Match'] = '"' + if_match + '"' statusexpected.append(412) if if_none_match is not None: headers['If-None-Match'] = '"' + if_none_match + '"' statusexpected.append(304) uri, endpoint = self._buketname2PathAndEndpoint(bucketname) data = yield tornado.gen.Task(self.request, uri=uri + urlquote(objectname), endpoint=endpoint, query=query, statusexpected=statusexpected, headers=headers, xmlexpected=False, signmethod=SIGNATURE_S3_REST) self._ioloop.add_callback(functools.partial(callback, data))
def putObjectCopy(self, bucketname, objectname, sourcebucketname, sourceobjectname, metadata=None, x_amz_server_side_encryption=None, x_amz_storage_class=None, x_amz_website_redirect_location=None, x_amz_metadata_directive=None, x_amz_copy_source_if_match=None, x_amz_copy_source_if_none_match=None, # TODO: not handled yet x_amz_copy_source_if_unmodified_since=None, x_amz_copy_source_if_modified_since=None, x_amz_acl=None, x_amz_grant_read=None, x_amz_grant_write=None, x_amz_grant_read_acp=None, x_amz_grant_write_acp=None, x_amz_grant_full_control=None): headers = {'x-amz-copy-source': "/%s/%s" % (sourcebucketname, sourceobjectname)} if x_amz_server_side_encryption is not None: headers['x-amz-server-side-encryption'] = x_amz_server_side_encryption if x_amz_storage_class is not None: headers['x-amz-storage-class'] = x_amz_storage_class if x_amz_website_redirect_location is not None: headers['x-amz-website-redirect-location'] = x_amz_website_redirect_location if metadata is not None: for name in metadata: headers['x-amz-meta-'+ name] = metadata[name] headers['x-amz-metadata-directive'] = 'REPLACE' if x_amz_metadata_directive is not None: if x_amz_metadata_directive not in ['COPY','REPLACE']: raise UserInputException('x_amz_metadata_directive valid values: COPY|REPLACE') headers['x-amz-metadata-directive'] = x_amz_metadata_directive if x_amz_copy_source_if_match is not None: headers['x-amz-copy-source-if-match'] = x_amz_copy_source_if_match if x_amz_copy_source_if_none_match is not None: headers['x-amz-copy-source-if-none-match'] = x_amz_copy_source_if_none_match uri, endpoint = self._buketname2PathAndEndpoint(bucketname) data = self.request(method="PUT", uri=uri + urlquote(objectname), headers=headers, host=endpoint, signmethod=SIGNATURE_S3_REST) return data['awsresponse']['CopyObjectResult']
def abortMultipartUpload(self, bucketname, objectname, uploadId): uri, endpoint = self._buketname2PathAndEndpoint(bucketname) self.request(method="DELETE", uri=uri + urlquote(objectname), host=endpoint, statusexpected=[204], query={"uploadId": uploadId}, signmethod=SIGNATURE_S3_REST)
def getObjectAcl(self, bucketname, objectname): uri, endpoint = self._buketname2PathAndEndpoint(bucketname) data = self.request(method="GET", uri=uri + urlquote(objectname), host=endpoint, query={'acl': None}, signmethod=SIGNATURE_S3_REST) return data['awsresponse']['AccessControlPolicy']
def setQueueAttributes(self, qName, delaySeconds=None, maximumMessageSize=None, messageRetentionPeriod=None, receiveMessageWaitTimeSeconds=None, visibilityTimeout=None, policy=None): """ @param qName: the sqs queue name @type qName: str @type delaySeconds: int @type maximumMessageSize: int @type messageRetentionPeriod: int @type receiveMessageWaitTimeSeconds: int @type visibilityTimeout: int @type policy: str @rtype: None """ query = {'Action': 'SetQueueAttributes', 'Version': '2012-11-05'} i = 1 if delaySeconds is not None: if not isinstance(delaySeconds, int) or not 0 <= delaySeconds <= 900: raise UserInputException("param delaySeconds sould be an integer between 0 and 900 (15 minutes)") query["Attribute.%d.Name" % (i,)] = "DelaySeconds" query["Attribute.%d.Value" % (i,)] = delaySeconds i += 1 if maximumMessageSize is not None: if not isinstance(maximumMessageSize, int) or not 1024 <= maximumMessageSize <= 65536: raise UserInputException("param maximumMessageSize sould be an integer between 1024 and 65536 (64 KiB)") query["Attribute.%d.Name" % (i,)] = "MaximumMessageSize" query["Attribute.%d.Value" % (i,)] = maximumMessageSize i += 1 if messageRetentionPeriod is not None: if not isinstance(messageRetentionPeriod, int) or not 60 <= messageRetentionPeriod <= 1209600: raise UserInputException( "param messageRetentionPeriod sould be an integer between 60 (1 minute) and 1209600 (14 days)") query["Attribute.%d.Name" % (i,)] = "MessageRetentionPeriod" query["Attribute.%d.Value" % (i,)] = messageRetentionPeriod i += 1 if receiveMessageWaitTimeSeconds is not None: if not isinstance(receiveMessageWaitTimeSeconds, int) or not 0 <= receiveMessageWaitTimeSeconds <= 20: raise UserInputException( "param receiveMessageWaitTimeSeconds sould be an integer between 0 and 20 (secconds)") query["Attribute.%d.Name" % (i,)] = "ReceiveMessageWaitTimeSeconds" query["Attribute.%d.Value" % (i,)] = receiveMessageWaitTimeSeconds i += 1 if visibilityTimeout is not None: if not isinstance(visibilityTimeout, int) or not 0 <= visibilityTimeout <= 43200: raise UserInputException("param visibilityTimeout sould be an integer between 0 and 43200 (12 hours)") query["Attribute.%d.Name" % (i,)] = "VisibilityTimeout" query["Attribute.%d.Value" % (i,)] = visibilityTimeout i += 1 if policy is not None: query["Attribute.%d.Name" % (i,)] = "Policy" query["Attribute.%d.Value" % (i,)] = urlquote(policy) i += 1 data = self.request(method="GET", signmethod=SIGNATURE_V4_HEADERS, query=query, region=self.endpoint[4:-14], service='sqs', host=self.endpoint, uri="/%s/%s" % (self.accNumber, qName)) data['awsresponse']['SetQueueAttributesResponse']