def create_invalidation_request(self, distribution_id, paths,
caller_reference=None):
"""Creates a new invalidation request
:see: http://goo.gl/8vECq
"""
# We allow you to pass in either an array or
# an InvalidationBatch object
if not isinstance(paths, InvalidationBatch):
paths = InvalidationBatch(paths)
paths.connection = self
uri = '/%s/distribution/%s/invalidation' % (self.Version,
distribution_id)
response = self.make_request('POST', uri,
{'Content-Type': 'text/xml'},
data=paths.to_xml())
body = response.read()
if response.status == 201:
h = handler.XmlHandler(paths, self)
xml.sax.parseString(body, h)
return paths
else:
raise CloudFrontServerError(response.status, response.reason, body)
def get_lifecycle_config(self, headers=None):
"""
Returns the current lifecycle configuration on the bucket.
:rtype: :class:`boto.gs.lifecycle.LifecycleConfig`
:returns: A LifecycleConfig object that describes all current
lifecycle rules in effect for the bucket.
"""
response = self.connection.make_request('GET',
self.name,
query_args=LIFECYCLE_ARG,
headers=headers)
body = response.read()
boto.log.debug(body)
if response.status == 200:
lifecycle_config = LifecycleConfig()
h = handler.XmlHandler(lifecycle_config, self)
xml.sax.parseString(body, h)
return lifecycle_config
else:
raise self.connection.provider.storage_response_error(
response.status, response.reason, body)
def get_all_parts(self, max_parts=None, part_number_marker=None):
"""
Return the uploaded parts of this MultiPart Upload. This is
a lower-level method that requires you to manually page through
results. To simplify this process, you can just use the
object itself as an iterator and it will automatically handle
all of the paging with S3.
"""
self._parts = []
query_args = 'uploadId=%s' % self.id
if max_parts:
query_args += '&max_parts=%d' % max_parts
if part_number_marker:
query_args += '&part-number-marker=%s' % part_number_marker
response = self.bucket.connection.make_request('GET', self.bucket.name,
self.key_name,
query_args=query_args)
body = response.read()
if response.status == 200:
h = handler.XmlHandler(self, self)
xml.sax.parseString(body, h)
return self._parts
def __init__(self, status, reason, body=None, *args):
StandardError.__init__(self, status, reason, body, *args)
self.status = status
self.reason = reason
self.body = body or ''
self.request_id = None
self.error_code = None
self.error_message = None
self.box_usage = None
# Attempt to parse the error response. If body isn't present,
# then just ignore the error response.
if self.body:
try:
h = handler.XmlHandler(self, self)
xml.sax.parseString(self.body, h)
except xml.sax.SAXParseException, pe:
# Go ahead and clean up anything that may have
# managed to get into the error data so we
# don't get partial garbage.
print "Warning: failed to parse error message from AWS: %s" % pe
self._cleanupParsedProperties()
def test_parse_xml(self):
x = pretty_print_xml
xml_in = """<?xml version="1.0" encoding="UTF-8"?>
<WebsiteConfiguration xmlns='http://s3.amazonaws.com/doc/2006-03-01/'>
<IndexDocument>
<Suffix>index.html</Suffix>
</IndexDocument>
<ErrorDocument>
<Key>error.html</Key>
</ErrorDocument>
<RoutingRules>
<RoutingRule>
<Condition>
<KeyPrefixEquals>docs/</KeyPrefixEquals>
</Condition>
<Redirect>
<Protocol>https</Protocol>
<HostName>www.example.com</HostName>
<ReplaceKeyWith>documents/</ReplaceKeyWith>
<HttpRedirectCode>302</HttpRedirectCode>
</Redirect>
</RoutingRule>
<RoutingRule>
<Condition>
<HttpErrorCodeReturnedEquals>404</HttpErrorCodeReturnedEquals>
</Condition>
<Redirect>
<HostName>example.com</HostName>
<ReplaceKeyPrefixWith>report-404/</ReplaceKeyPrefixWith>
</Redirect>
</RoutingRule>
</RoutingRules>
</WebsiteConfiguration>
"""
webconfig = WebsiteConfiguration()
h = handler.XmlHandler(webconfig, None)
xml.sax.parseString(xml_in, h)
xml_out = webconfig.to_xml()
self.assertEqual(x(xml_in), x(xml_out))
def get_attributes(self,
domain_or_name,
item_name,
attribute_names=None,
item=None):
"""
Retrieve attributes for a given item in a domain.
@type domain_or_name: string or L{Domain<boto.sdb.domain.Domain>} object.
@param domain_or_name: Either the name of a domain or a Domain object
@type item_name: string
@param item_name: The name of the item whose attributes are being retrieved.
@type attribute_names: string or list of strings
@param attribute_names: An attribute name or list of attribute names. This
parameter is optional. If not supplied, all attributes
will be retrieved for the item.
@rtype: L{Item<boto.sdb.item.Item>}
@return: An Item mapping type containing the requested attribute name/values
"""
domain, domain_name = self.get_domain_and_name(domain_or_name)
params = {'DomainName': domain_name, 'ItemName': item_name}
if attribute_names:
if not isinstance(attribute_names, list):
attribute_names = [attribute_names]
self.build_list_params(params, attribute_names, 'AttributeName')
response = self.make_request('GetAttributes', params)
body = response.read()
if response.status == 200:
if item == None:
item = Item(domain, item_name)
h = handler.XmlHandler(item, self)
xml.sax.parseString(body, h)
return item
else:
raise SDBResponseError(response.status, response.reason, body)
def RunCommand(self):
cors_arg = self.args[0]
uri_args = self.args[1:]
# Disallow multi-provider setcors requests.
storage_uri = self.UrisAreForSingleProvider(uri_args)
if not storage_uri:
raise CommandException('"%s" command spanning providers not allowed.' %
self.command_name)
# Open, read and parse file containing XML document.
cors_file = open(cors_arg, 'r')
cors_txt = cors_file.read()
cors_file.close()
cors_obj = Cors()
# Parse XML document and convert into Cors object.
h = handler.XmlHandler(cors_obj, None)
try:
xml.sax.parseString(cors_txt, h)
except xml.sax._exceptions.SAXParseException, e:
raise CommandException('Requested CORS is invalid: %s at line %s, '
'column %s' % (e.getMessage(), e.getLineNumber(),
e.getColumnNumber()))
def __init__(self, status, reason, body=None, *args):
StandardError.__init__(self, status, reason, body, *args)
self.status = status
self.reason = reason
self.body = body or ''
self.request_id = None
self.error_code = None
self.error_message = None
self.box_usage = None
# Attempt to parse the error response. If body isn't present,
# then just ignore the error response.
if self.body:
try:
h = handler.XmlHandler(self, self)
xml.sax.parseString(self.body, h)
except xml.sax.SAXParseException, pe:
# Remove unparsable message body so we don't include garbage
# in exception. But first, save self.body in self.error_message
# because occasionally we get error messages from Eucalyptus
# that are just text strings that we want to preserve.
self.error_message = self.body
self.body = None
def create_distribution(self,
origin,
enabled,
caller_reference='',
cnames=None,
comment=''):
config = DistributionConfig(origin=origin,
enabled=enabled,
caller_reference=caller_reference,
cnames=cnames,
comment=comment)
response = self.make_request('POST',
'/%s/distribution' % self.Version,
{'Content-Type': 'text/xml'},
data=config.to_xml())
body = response.read()
if response.status == 201:
d = Distribution(connection=self)
h = handler.XmlHandler(d, self)
xml.sax.parseString(body, h)
return d
else:
raise CloudFrontServerError(response.status, response.reason, body)
def get_response(self, action, params, page=0, itemSet=None):
"""
Utility method to handle calls to ECS and parsing of responses.
"""
params['Service'] = "AWSECommerceService"
params['Operation'] = action
if page:
params['ItemPage'] = page
response = self.make_request(None, params, "/onca/xml")
body = response.read()
boto.log.debug(body)
if response.status != 200:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
if itemSet == None:
rs = ItemSet(self, action, params, page)
else:
rs = itemSet
h = handler.XmlHandler(rs, self)
xml.sax.parseString(body, h)
return rs
def get_object(self,
action,
params,
cls,
path='/',
parent=None,
verb='GET'):
if not parent:
parent = self
response = self.make_request(action, params, path, verb)
body = response.read()
boto.log.debug(body)
if not body:
boto.log.error('Null body %s' % body)
raise self.ResponseError(response.status, response.reason, body)
elif response.status == 200:
obj = cls(parent)
h = handler.XmlHandler(obj, parent)
xml.sax.parseString(body, h)
return obj
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def _SetLifecycleConfig(self):
lifecycle_arg = self.args[0]
uri_args = self.args[1:]
# Disallow multi-provider setlifecycle requests.
storage_uri = self.UrisAreForSingleProvider(uri_args)
if not storage_uri:
raise CommandException(
'"%s" command spanning providers not allowed.' %
self.command_name)
# Open, read and parse file containing XML document.
lifecycle_file = open(lifecycle_arg, 'r')
lifecycle_txt = lifecycle_file.read()
lifecycle_file.close()
lifecycle_config = LifecycleConfig()
# Parse XML document and convert into LifecycleConfig object.
h = handler.XmlHandler(lifecycle_config, None)
try:
xml.sax.parseString(lifecycle_txt, h)
except xml.sax._exceptions.SAXParseException, e:
raise CommandException(
'Requested lifecycle config is invalid: %s at line %s, column %s'
% (e.getMessage(), e.getLineNumber(), e.getColumnNumber()))
def refund(self, callerReference, transactionId, refundAmount=None,
callerDescription=None):
"""
Refund a transaction. This refunds the full amount by default
unless 'refundAmount' is specified.
"""
params = {}
params['CallerReference'] = callerReference
params['TransactionId'] = transactionId
if(refundAmount != None):
params['RefundAmount.Value'] = refundAmount
params['RefundAmount.CurrencyCode'] = "USD"
if(callerDescription != None):
params['CallerDescription'] = callerDescription
response = self.make_request("Refund", params)
body = response.read()
if(response.status == 200):
rs = ResultSet([("RefundResponse", FPSResponse)])
h = handler.XmlHandler(rs, self)
xml.sax.parseString(body, h)
return rs
else:
raise FPSResponseError(response.status, response.reason, body)
def get_all_rrsets(self,
hosted_zone_id,
type=None,
name=None,
identifier=None,
maxitems=None):
"""
Retrieve the Resource Record Sets defined for this Hosted Zone.
Returns the raw XML data returned by the Route53 call.
:type hosted_zone_id: str
:param hosted_zone_id: The unique identifier for the Hosted Zone
:type type: str
:param type: The type of resource record set to begin the record
listing from. Valid choices are:
* A
* AAAA
* CNAME
* MX
* NS
* PTR
* SOA
* SPF
* SRV
* TXT
Valid values for weighted resource record sets:
* A
* AAAA
* CNAME
* TXT
Valid values for Zone Apex Aliases:
* A
* AAAA
:type name: str
:param name: The first name in the lexicographic ordering of domain
names to be retrieved
:type identifier: str
:param identifier: In a hosted zone that includes weighted resource
record sets (multiple resource record sets with the same DNS
name and type that are differentiated only by SetIdentifier),
if results were truncated for a given DNS name and type,
the value of SetIdentifier for the next resource record
set that has the current DNS name and type
:type maxitems: int
:param maxitems: The maximum number of records
"""
params = {
'type': type,
'name': name,
'Identifier': identifier,
'maxitems': maxitems
}
uri = '/%s/hostedzone/%s/rrset' % (self.Version, hosted_zone_id)
response = self.make_request('GET', uri, params=params)
body = response.read()
boto.log.debug(body)
if response.status >= 300:
raise exception.DNSServerError(response.status, response.reason,
body)
rs = ResourceRecordSets(connection=self, hosted_zone_id=hosted_zone_id)
h = handler.XmlHandler(rs, self)
xml.sax.parseString(body, h)
return rs
def copy_key(self, new_key_name, src_bucket_name,
src_key_name, metadata=None, src_version_id=None,
storage_class='STANDARD', preserve_acl=False):
"""
Create a new key in the bucket by copying another existing key.
:type new_key_name: string
:param new_key_name: The name of the new key
:type src_bucket_name: string
:param src_bucket_name: The name of the source bucket
:type src_key_name: string
:param src_key_name: The name of the source key
:type src_version_id: string
:param src_version_id: The version id for the key. This param
is optional. If not specified, the newest
version of the key will be copied.
:type metadata: dict
:param metadata: Metadata to be associated with new key.
If metadata is supplied, it will replace the
metadata of the source key being copied.
If no metadata is supplied, the source key's
metadata will be copied to the new key.
:type storage_class: string
:param storage_class: The storage class of the new key.
By default, the new key will use the
standard storage class. Possible values are:
STANDARD | REDUCED_REDUNDANCY
:type preserve_acl: bool
:param preserve_acl: If True, the ACL from the source key
will be copied to the destination
key. If False, the destination key
will have the default ACL.
Note that preserving the ACL in the
new key object will require two
additional API calls to S3, one to
retrieve the current ACL and one to
set that ACL on the new object. If
you don't care about the ACL, a value
of False will be significantly more
efficient.
:rtype: :class:`boto.s3.key.Key` or subclass
:returns: An instance of the newly created key object
"""
if preserve_acl:
acl = self.get_xml_acl(src_key_name)
src = '%s/%s' % (src_bucket_name, urllib.quote(src_key_name))
if src_version_id:
src += '?version_id=%s' % src_version_id
provider = self.connection.provider
headers = {provider.copy_source_header : src}
if storage_class != 'STANDARD':
headers[provider.storage_class_header] = storage_class
if metadata:
headers[provider.metadata_directive_header] = 'REPLACE'
headers = boto.utils.merge_meta(headers, metadata)
else:
headers[provider.metadata_directive_header] = 'COPY'
response = self.connection.make_request('PUT', self.name, new_key_name,
headers=headers)
body = response.read()
if response.status == 200:
key = self.new_key(new_key_name)
h = handler.XmlHandler(key, self)
xml.sax.parseString(body, h)
if hasattr(key, 'Error'):
raise provider.storage_copy_error(key.Code, key.Message, body)
key.handle_version_headers(response)
if preserve_acl:
self.set_xml_acl(acl, new_key_name)
return key
else:
raise provider.storage_response_error(response.status, response.reason, body)
def pay(self,
transactionAmount,
senderTokenId,
recipientTokenId=None,
callerTokenId=None,
chargeFeeTo="Recipient",
callerReference=None,
senderReference=None,
recipientReference=None,
senderDescription=None,
recipientDescription=None,
callerDescription=None,
metadata=None,
transactionDate=None,
reserve=False):
"""
Make a payment transaction. You must specify the amount.
This can also perform a Reserve request if 'reserve' is set to True.
"""
params = {}
params['SenderTokenId'] = senderTokenId
# this is for 2008-09-17 specification
params['TransactionAmount.Amount'] = str(transactionAmount)
params['TransactionAmount.CurrencyCode'] = "USD"
#params['TransactionAmount'] = str(transactionAmount)
params['ChargeFeeTo'] = chargeFeeTo
params['RecipientTokenId'] = (recipientTokenId if recipientTokenId
is not None else boto.config.get(
"FPS", "recipient_token"))
params['CallerTokenId'] = (callerTokenId if callerTokenId is not None
else boto.config.get("FPS", "caller_token"))
if (transactionDate != None):
params['TransactionDate'] = transactionDate
if (senderReference != None):
params['SenderReference'] = senderReference
if (recipientReference != None):
params['RecipientReference'] = recipientReference
if (senderDescription != None):
params['SenderDescription'] = senderDescription
if (recipientDescription != None):
params['RecipientDescription'] = recipientDescription
if (callerDescription != None):
params['CallerDescription'] = callerDescription
if (metadata != None):
params['MetaData'] = metadata
if (callerReference == None):
callerReference = uuid.uuid4()
params['CallerReference'] = callerReference
if reserve:
action = "Reserve"
else:
action = "Pay"
response = self.make_request(action, params)
body = response.read()
if (response.status == 200):
rs = ResultSet([("%sResponse" % action, FPSResponse)])
h = handler.XmlHandler(rs, self)
xml.sax.parseString(body, h)
return rs
else:
raise FPSResponseError(response.status, response.reason, body)
def copy_key(self,
new_key_name,
src_bucket_name,
src_key_name,
metadata=None,
src_version_id=None):
"""
Create a new key in the bucket by copying another existing key.
:type new_key_name: string
:param new_key_name: The name of the new key
:type src_bucket_name: string
:param src_bucket_name: The name of the source bucket
:type src_key_name: string
:param src_key_name: The name of the source key
:type src_version_id: string
:param src_version_id: The version id for the key. This param
is optional. If not specified, the newest
version of the key will be copied.
:type metadata: dict
:param metadata: Metadata to be associated with new key.
If metadata is supplied, it will replace the
metadata of the source key being copied.
If no metadata is supplied, the source key's
metadata will be copied to the new key.
:rtype: :class:`boto.s3.key.Key` or subclass
:returns: An instance of the newly created key object
"""
src = '%s/%s' % (src_bucket_name, urllib.quote(src_key_name))
if src_version_id:
src += '?version_id=%s' % src_version_id
if metadata:
headers = {
'x-amz-copy-source': src,
'x-amz-metadata-directive': 'REPLACE'
}
headers = boto.utils.merge_meta(headers, metadata)
else:
headers = {
'x-amz-copy-source': src,
'x-amz-metadata-directive': 'COPY'
}
response = self.connection.make_request('PUT',
self.name,
new_key_name,
headers=headers)
body = response.read()
if response.status == 200:
key = self.new_key(new_key_name)
h = handler.XmlHandler(key, self)
xml.sax.parseString(body, h)
if hasattr(key, 'Error'):
raise S3CopyError(key.Code, key.Message, body)
key.handle_version_headers(response)
return key
else:
raise S3ResponseError(response.status, response.reason, body)
def get_all_buckets_hook(body):
rs = ResultSet([('Bucket', self.bucket_class)])
h = handler.XmlHandler(rs, self)
xml.sax.parseString(body, h)
return rs
def get_all_hook(body):
boto.log.debug(body)
rs = ResultSet(element_map)
h = handler.XmlHandler(rs, self)
xml.sax.parseString(body, h)
return rs
def _parse_xml(self, body, markers):
rs = ResultSet(markers)
h = handler.XmlHandler(rs, None)
xml.sax.parseString(body, h)
return rs
def test_1_basic(self):
"""basic regression test for Google Cloud Storage"""
print '--- running GSConnection tests ---'
c = GSConnection()
# create a new, empty bucket
bucket_name = 'test-%d' % int(time.time())
bucket = c.create_bucket(bucket_name)
# now try a get_bucket call and see if it's really there
bucket = c.get_bucket(bucket_name)
key_name = 'foobar'
k = bucket.new_key(key_name)
s1 = 'This is a test of file upload and download'
s2 = 'This is a second string to test file upload and download'
k.set_contents_from_string(s1)
fp = open(key_name, 'wb')
# now get the contents from s3 to a local file
k.get_contents_to_file(fp)
fp.close()
fp = open(key_name)
# check to make sure content read from s3 is identical to original
assert s1 == fp.read(), 'corrupted file'
fp.close()
bucket.delete_key(k)
# test a few variations on get_all_keys - first load some data
# for the first one, let's override the content type
phony_mimetype = 'application/x-boto-test'
headers = {'Content-Type': phony_mimetype}
k.name = 'foo/bar'
k.set_contents_from_string(s1, headers)
k.name = 'foo/bas'
k.set_contents_from_filename('foobar')
k.name = 'foo/bat'
k.set_contents_from_string(s1)
k.name = 'fie/bar'
k.set_contents_from_string(s1)
k.name = 'fie/bas'
k.set_contents_from_string(s1)
k.name = 'fie/bat'
k.set_contents_from_string(s1)
# try resetting the contents to another value
md5 = k.md5
k.set_contents_from_string(s2)
assert k.md5 != md5
# Test for stream API
fp2 = open('foobar', 'rb')
k.md5 = None
k.base64md5 = None
k.set_contents_from_stream(fp2, headers=headers)
fp = open('foobar1', 'wb')
k.get_contents_to_file(fp)
fp.close()
fp2.seek(0, 0)
fp = open('foobar1', 'rb')
assert (fp2.read() == fp.read()), 'Chunked Transfer corrupted the Data'
fp.close()
fp2.close()
os.unlink('foobar1')
os.unlink('foobar')
all = bucket.get_all_keys()
assert len(all) == 6
rs = bucket.get_all_keys(prefix='foo')
assert len(rs) == 3
rs = bucket.get_all_keys(prefix='', delimiter='/')
assert len(rs) == 2
rs = bucket.get_all_keys(maxkeys=5)
assert len(rs) == 5
# test the lookup method
k = bucket.lookup('foo/bar')
assert isinstance(k, bucket.key_class)
assert k.content_type == phony_mimetype
k = bucket.lookup('notthere')
assert k == None
# try some metadata stuff
key_name = 'has_metadata'
k = bucket.new_key(key_name)
mdkey1 = 'meta1'
mdval1 = 'This is the first metadata value'
k.set_metadata(mdkey1, mdval1)
mdkey2 = 'meta2'
mdval2 = 'This is the second metadata value'
k.set_metadata(mdkey2, mdval2)
# try a unicode metadata value
mdval3 = u'föö'
mdkey3 = 'meta3'
k.set_metadata(mdkey3, mdval3)
k.set_contents_from_string(s1)
k = bucket.lookup(key_name)
assert k.get_metadata(mdkey1) == mdval1
assert k.get_metadata(mdkey2) == mdval2
assert k.get_metadata(mdkey3) == mdval3
k = bucket.new_key(key_name)
k.get_contents_as_string()
assert k.get_metadata(mdkey1) == mdval1
assert k.get_metadata(mdkey2) == mdval2
assert k.get_metadata(mdkey3) == mdval3
bucket.delete_key(k)
# test list and iterator
rs1 = bucket.list()
num_iter = 0
for r in rs1:
num_iter = num_iter + 1
rs = bucket.get_all_keys()
num_keys = len(rs)
assert num_iter == num_keys
# try some acl stuff
bucket.set_acl('public-read')
acl = bucket.get_acl()
assert len(acl.entries.entry_list) == 2
bucket.set_acl('private')
acl = bucket.get_acl()
assert len(acl.entries.entry_list) == 1
k = bucket.lookup('foo/bar')
k.set_acl('public-read')
acl = k.get_acl()
assert len(acl.entries.entry_list) == 2
k.set_acl('private')
acl = k.get_acl()
assert len(acl.entries.entry_list) == 1
#
# Test case-insensitivity of XML ACL parsing.
acl_xml = (
'<ACCESSControlList><EntrIes><Entry>' +
'<Scope type="AllUsers"></Scope><Permission>READ</Permission>' +
'</Entry></EntrIes></ACCESSControlList>')
acl = boto.gs.acl.ACL()
h = handler.XmlHandler(acl, bucket)
xml.sax.parseString(acl_xml, h)
bucket.set_acl(acl)
assert len(acl.entries.entry_list) == 1
#
# try set/get raw logging subresource
empty_logging_str = "<?xml version='1.0' encoding='UTF-8'?><Logging/>"
logging_str = ("<?xml version='1.0' encoding='UTF-8'?><Logging>"
"<LogBucket>log-bucket</LogBucket>" +
"<LogObjectPrefix>example</LogObjectPrefix>" +
"</Logging>")
bucket.set_subresource('logging', logging_str)
assert bucket.get_subresource('logging') == logging_str
# try disable/enable logging
bucket.disable_logging()
assert bucket.get_subresource('logging') == empty_logging_str
bucket.enable_logging('log-bucket', 'example')
assert bucket.get_subresource('logging') == logging_str
# now delete all keys in bucket
for k in bucket:
bucket.delete_key(k)
# now delete bucket
time.sleep(5)
c.delete_bucket(bucket)
