def test_delete_multi_objects_with_quiet(self):
bucket = 'bucket'
put_objects = ['obj']
query = 'delete'
# with Quiet true
quiet = 'true'
self._prepare_test_delete_multi_objects(bucket, put_objects)
xml = self._gen_multi_delete_xml(put_objects, quiet)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), 0)
# with Quiet false
quiet = 'false'
self._prepare_test_delete_multi_objects(bucket, put_objects)
xml = self._gen_multi_delete_xml(put_objects, quiet)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), 1)
def test_delete_multi_objects_with_quiet(self):
bucket = 'bucket'
put_objects = ['obj']
query = 'delete'
# with Quiet true
quiet = 'true'
self._prepare_test_delete_multi_objects(bucket, put_objects)
xml = self._gen_multi_delete_xml(put_objects, quiet)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), 0)
# with Quiet false
quiet = 'false'
self._prepare_test_delete_multi_objects(bucket, put_objects)
xml = self._gen_multi_delete_xml(put_objects, quiet)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), 1)
def test_put_object_content_md5(self):
obj = 'object'
content = b'abcdefghij'
etag = md5(content).hexdigest()
headers = {'Content-MD5': calculate_md5(content)}
status, headers, body = \
self.conn.make_request('PUT', self.bucket, obj, headers, content)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self._assertObjectEtag(self.bucket, obj, etag)
def test_put_object_content_md5(self):
obj = 'object'
content = 'abcdefghij'
etag = md5(content).hexdigest()
headers = {'Content-MD5': calculate_md5(content)}
status, headers, body = \
self.conn.make_request('PUT', self.bucket, obj, headers, content)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self._assertObjectEtag(self.bucket, obj, etag)
def test_delete_bucket_multi_upload_object_exisiting(self):
bucket = 'bucket'
keys = ['obj1']
uploads = []
results_generator = self._initiate_multi_uploads_result_generator(
bucket, keys)
# Initiate Multipart Upload
for expected_key, (status, _, body) in \
zip(keys, results_generator):
self.assertEqual(status, 200) # sanity
elem = fromstring(body, 'InitiateMultipartUploadResult')
key = elem.find('Key').text
self.assertEqual(expected_key, key) # sanity
upload_id = elem.find('UploadId').text
self.assertTrue(upload_id is not None) # sanity
self.assertTrue((key, upload_id) not in uploads)
uploads.append((key, upload_id))
self.assertEqual(len(uploads), len(keys)) # sanity
# Upload Part
key, upload_id = uploads[0]
content = b'a' * self.min_segment_size
status, headers, body = \
self._upload_part(bucket, key, upload_id, content)
self.assertEqual(status, 200)
# Complete Multipart Upload
key, upload_id = uploads[0]
etags = [md5(content, usedforsecurity=False).hexdigest()]
xml = self._gen_comp_xml(etags)
status, headers, body = \
self._complete_multi_upload(bucket, key, upload_id, xml)
self.assertEqual(status, 200) # sanity
# GET multipart object
status, headers, body = \
self.conn.make_request('GET', bucket, key)
self.assertEqual(status, 200) # sanity
self.assertEqual(content, body) # sanity
# DELETE bucket while the object existing
status, headers, body = \
self.conn.make_request('DELETE', bucket)
self.assertEqual(status, 409) # sanity
# The object must still be there.
status, headers, body = \
self.conn.make_request('GET', bucket, key)
self.assertEqual(status, 200) # sanity
self.assertEqual(content, body) # sanity
# Can delete it with DeleteMultipleObjects request
elem = Element('Delete')
SubElement(elem, 'Quiet').text = 'true'
obj_elem = SubElement(elem, 'Object')
SubElement(obj_elem, 'Key').text = key
body = tostring(elem, use_s3ns=False)
status, headers, body = self.conn.make_request(
'POST',
bucket,
body=body,
query='delete',
headers={'Content-MD5': calculate_md5(body)})
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
status, headers, body = \
self.conn.make_request('GET', bucket, key)
self.assertEqual(status, 404) # sanity
# Now we can delete
status, headers, body = \
self.conn.make_request('DELETE', bucket)
self.assertEqual(status, 204) # sanity
def test_delete_multi_objects(self):
bucket = 'bucket'
put_objects = ['obj%s' % var for var in xrange(4)]
self._prepare_test_delete_multi_objects(bucket, put_objects)
query = 'delete'
# Delete an object via MultiDelete API
req_objects = ['obj0']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue(headers['content-type'] is not None)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body)
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
# Delete 2 objects via MultiDelete API
req_objects = ['obj1', 'obj2']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
# Delete 2 objects via MultiDelete API but one (obj4) doesn't exist.
req_objects = ['obj3', 'obj4']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
# S3 assumes a NoSuchKey object as deleted.
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
# Delete 2 objects via MultiDelete API but no objects exist
req_objects = ['obj4', 'obj5']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
def test_delete_multi_objects_error(self):
bucket = 'bucket'
put_objects = ['obj']
self._prepare_test_delete_multi_objects(bucket, put_objects)
xml = self._gen_multi_delete_xml(put_objects)
content_md5 = calculate_md5(xml)
query = 'delete'
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('POST', bucket, body=xml,
headers={
'Content-MD5': content_md5
},
query=query)
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
status, headers, body = \
self.conn.make_request('POST', 'nothing', body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'NoSuchBucket')
# without Object tag
xml = self._gen_invalid_multi_delete_xml()
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'MalformedXML')
# without value of Key tag
xml = self._gen_invalid_multi_delete_xml(hasObjectTag=True)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'UserKeyMustBeSpecified')
max_deletes = tf.cluster_info.get('s3api',
{}).get('max_multi_delete_objects',
1000)
# specified number of objects are over max_multi_delete_objects
# (Default 1000), but xml size is relatively small
req_objects = ['obj%s' for var in xrange(max_deletes + 1)]
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'MalformedXML')
# specified xml size is large, but number of objects are
# smaller than max_multi_delete_objects.
obj = 'a' * 102400
req_objects = [obj + str(var) for var in xrange(max_deletes - 1)]
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'MalformedXML')
def test_delete_multi_objects(self):
bucket = 'bucket'
put_objects = ['obj%s' % var for var in range(4)]
self._prepare_test_delete_multi_objects(bucket, put_objects)
query = 'delete'
# Delete an object via MultiDelete API
req_objects = ['obj0']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
self.assertCommonResponseHeaders(headers)
self.assertTrue(headers['content-type'] is not None)
self.assertEqual(headers['content-length'], str(len(body)))
elem = fromstring(body)
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
# Delete 2 objects via MultiDelete API
req_objects = ['obj1', 'obj2']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
# Delete 2 objects via MultiDelete API but one (obj4) doesn't exist.
req_objects = ['obj3', 'obj4']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
# S3 assumes a NoSuchKey object as deleted.
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
# Delete 2 objects via MultiDelete API but no objects exist
req_objects = ['obj4', 'obj5']
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(status, 200)
elem = fromstring(body, 'DeleteResult')
resp_objects = elem.findall('Deleted')
self.assertEqual(len(resp_objects), len(req_objects))
for o in resp_objects:
self.assertTrue(o.find('Key').text in req_objects)
def test_delete_multi_objects_error(self):
bucket = 'bucket'
put_objects = ['obj']
self._prepare_test_delete_multi_objects(bucket, put_objects)
xml = self._gen_multi_delete_xml(put_objects)
content_md5 = calculate_md5(xml)
query = 'delete'
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('POST', bucket, body=xml,
headers={
'Content-MD5': content_md5
},
query=query)
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
status, headers, body = \
self.conn.make_request('POST', 'nothing', body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'NoSuchBucket')
# without Object tag
xml = self._gen_invalid_multi_delete_xml()
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'MalformedXML')
# without value of Key tag
xml = self._gen_invalid_multi_delete_xml(hasObjectTag=True)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'UserKeyMustBeSpecified')
max_deletes = tf.cluster_info.get('s3api', {}).get(
'max_multi_delete_objects', 1000)
# specified number of objects are over max_multi_delete_objects
# (Default 1000), but xml size is relatively small
req_objects = ['obj%s' for var in range(max_deletes + 1)]
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'MalformedXML')
# specified xml size is large, but number of objects are
# smaller than max_multi_delete_objects.
obj = 'a' * 102400
req_objects = [obj + str(var) for var in range(max_deletes - 1)]
xml = self._gen_multi_delete_xml(req_objects)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'MalformedXML')
def test_delete_multi_objects_error(self):
bucket = 'bucket'
put_objects = ['obj']
self._prepare_test_delete_multi_objects(bucket, put_objects)
xml = self._gen_multi_delete_xml(put_objects)
content_md5 = calculate_md5(xml)
query = 'delete'
auth_error_conn = Connection(aws_secret_key='invalid')
status, headers, body = \
auth_error_conn.make_request('POST', bucket, body=xml,
headers={
'Content-MD5': content_md5
},
query=query)
self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch')
status, headers, body = \
self.conn.make_request('POST', 'nothing', body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'NoSuchBucket')
# without Object tag
xml = self._gen_invalid_multi_delete_xml()
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'MalformedXML')
# without value of Key tag
xml = self._gen_invalid_multi_delete_xml(hasObjectTag=True)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'UserKeyMustBeSpecified')
# specified number of objects are over max_multi_delete_objects
# (Default 1000), but xml size is smaller than 61365 bytes.
req_objects = ['obj%s' for var in xrange(1001)]
xml = self._gen_multi_delete_xml(req_objects)
self.assertTrue(len(xml.encode('utf-8')) <= MAX_MULTI_DELETE_BODY_SIZE)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'MalformedXML')
# specified xml size is over 61365 bytes, but number of objects are
# smaller than max_multi_delete_objects.
obj = 'a' * 1024
req_objects = [obj + str(var) for var in xrange(999)]
xml = self._gen_multi_delete_xml(req_objects)
self.assertTrue(len(xml.encode('utf-8')) > MAX_MULTI_DELETE_BODY_SIZE)
content_md5 = calculate_md5(xml)
status, headers, body = \
self.conn.make_request('POST', bucket, body=xml,
headers={'Content-MD5': content_md5},
query=query)
self.assertEqual(get_error_code(body), 'MalformedXML')
