def test_delete_multi_objects_with_quiet(self): bucket = 'bucket' put_objects = ['obj'] query = 'delete' # with Quiet true quiet = 'true' self._prepare_test_delete_multi_objects(bucket, put_objects) xml = self._gen_multi_delete_xml(put_objects, quiet) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), 0) # with Quiet false quiet = 'false' self._prepare_test_delete_multi_objects(bucket, put_objects) xml = self._gen_multi_delete_xml(put_objects, quiet) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), 1)
def test_put_object_content_md5(self): obj = 'object' content = b'abcdefghij' etag = md5(content).hexdigest() headers = {'Content-MD5': calculate_md5(content)} status, headers, body = \ self.conn.make_request('PUT', self.bucket, obj, headers, content) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self._assertObjectEtag(self.bucket, obj, etag)
def test_put_object_content_md5(self): obj = 'object' content = 'abcdefghij' etag = md5(content).hexdigest() headers = {'Content-MD5': calculate_md5(content)} status, headers, body = \ self.conn.make_request('PUT', self.bucket, obj, headers, content) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self._assertObjectEtag(self.bucket, obj, etag)
def test_delete_bucket_multi_upload_object_exisiting(self): bucket = 'bucket' keys = ['obj1'] uploads = [] results_generator = self._initiate_multi_uploads_result_generator( bucket, keys) # Initiate Multipart Upload for expected_key, (status, _, body) in \ zip(keys, results_generator): self.assertEqual(status, 200) # sanity elem = fromstring(body, 'InitiateMultipartUploadResult') key = elem.find('Key').text self.assertEqual(expected_key, key) # sanity upload_id = elem.find('UploadId').text self.assertTrue(upload_id is not None) # sanity self.assertTrue((key, upload_id) not in uploads) uploads.append((key, upload_id)) self.assertEqual(len(uploads), len(keys)) # sanity # Upload Part key, upload_id = uploads[0] content = b'a' * self.min_segment_size status, headers, body = \ self._upload_part(bucket, key, upload_id, content) self.assertEqual(status, 200) # Complete Multipart Upload key, upload_id = uploads[0] etags = [md5(content, usedforsecurity=False).hexdigest()] xml = self._gen_comp_xml(etags) status, headers, body = \ self._complete_multi_upload(bucket, key, upload_id, xml) self.assertEqual(status, 200) # sanity # GET multipart object status, headers, body = \ self.conn.make_request('GET', bucket, key) self.assertEqual(status, 200) # sanity self.assertEqual(content, body) # sanity # DELETE bucket while the object existing status, headers, body = \ self.conn.make_request('DELETE', bucket) self.assertEqual(status, 409) # sanity # The object must still be there. status, headers, body = \ self.conn.make_request('GET', bucket, key) self.assertEqual(status, 200) # sanity self.assertEqual(content, body) # sanity # Can delete it with DeleteMultipleObjects request elem = Element('Delete') SubElement(elem, 'Quiet').text = 'true' obj_elem = SubElement(elem, 'Object') SubElement(obj_elem, 'Key').text = key body = tostring(elem, use_s3ns=False) status, headers, body = self.conn.make_request( 'POST', bucket, body=body, query='delete', headers={'Content-MD5': calculate_md5(body)}) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) status, headers, body = \ self.conn.make_request('GET', bucket, key) self.assertEqual(status, 404) # sanity # Now we can delete status, headers, body = \ self.conn.make_request('DELETE', bucket) self.assertEqual(status, 204) # sanity
def test_delete_multi_objects(self): bucket = 'bucket' put_objects = ['obj%s' % var for var in xrange(4)] self._prepare_test_delete_multi_objects(bucket, put_objects) query = 'delete' # Delete an object via MultiDelete API req_objects = ['obj0'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue(headers['content-type'] is not None) self.assertEqual(headers['content-length'], str(len(body))) elem = fromstring(body) resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects) # Delete 2 objects via MultiDelete API req_objects = ['obj1', 'obj2'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects) # Delete 2 objects via MultiDelete API but one (obj4) doesn't exist. req_objects = ['obj3', 'obj4'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') # S3 assumes a NoSuchKey object as deleted. self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects) # Delete 2 objects via MultiDelete API but no objects exist req_objects = ['obj4', 'obj5'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects)
def test_delete_multi_objects_error(self): bucket = 'bucket' put_objects = ['obj'] self._prepare_test_delete_multi_objects(bucket, put_objects) xml = self._gen_multi_delete_xml(put_objects) content_md5 = calculate_md5(xml) query = 'delete' auth_error_conn = Connection(aws_secret_key='invalid') status, headers, body = \ auth_error_conn.make_request('POST', bucket, body=xml, headers={ 'Content-MD5': content_md5 }, query=query) self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch') status, headers, body = \ self.conn.make_request('POST', 'nothing', body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'NoSuchBucket') # without Object tag xml = self._gen_invalid_multi_delete_xml() content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'MalformedXML') # without value of Key tag xml = self._gen_invalid_multi_delete_xml(hasObjectTag=True) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'UserKeyMustBeSpecified') max_deletes = tf.cluster_info.get('s3api', {}).get('max_multi_delete_objects', 1000) # specified number of objects are over max_multi_delete_objects # (Default 1000), but xml size is relatively small req_objects = ['obj%s' for var in xrange(max_deletes + 1)] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'MalformedXML') # specified xml size is large, but number of objects are # smaller than max_multi_delete_objects. obj = 'a' * 102400 req_objects = [obj + str(var) for var in xrange(max_deletes - 1)] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'MalformedXML')
def test_delete_multi_objects(self): bucket = 'bucket' put_objects = ['obj%s' % var for var in range(4)] self._prepare_test_delete_multi_objects(bucket, put_objects) query = 'delete' # Delete an object via MultiDelete API req_objects = ['obj0'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) self.assertCommonResponseHeaders(headers) self.assertTrue(headers['content-type'] is not None) self.assertEqual(headers['content-length'], str(len(body))) elem = fromstring(body) resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects) # Delete 2 objects via MultiDelete API req_objects = ['obj1', 'obj2'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects) # Delete 2 objects via MultiDelete API but one (obj4) doesn't exist. req_objects = ['obj3', 'obj4'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') # S3 assumes a NoSuchKey object as deleted. self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects) # Delete 2 objects via MultiDelete API but no objects exist req_objects = ['obj4', 'obj5'] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(status, 200) elem = fromstring(body, 'DeleteResult') resp_objects = elem.findall('Deleted') self.assertEqual(len(resp_objects), len(req_objects)) for o in resp_objects: self.assertTrue(o.find('Key').text in req_objects)
def test_delete_multi_objects_error(self): bucket = 'bucket' put_objects = ['obj'] self._prepare_test_delete_multi_objects(bucket, put_objects) xml = self._gen_multi_delete_xml(put_objects) content_md5 = calculate_md5(xml) query = 'delete' auth_error_conn = Connection(aws_secret_key='invalid') status, headers, body = \ auth_error_conn.make_request('POST', bucket, body=xml, headers={ 'Content-MD5': content_md5 }, query=query) self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch') status, headers, body = \ self.conn.make_request('POST', 'nothing', body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'NoSuchBucket') # without Object tag xml = self._gen_invalid_multi_delete_xml() content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'MalformedXML') # without value of Key tag xml = self._gen_invalid_multi_delete_xml(hasObjectTag=True) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'UserKeyMustBeSpecified') max_deletes = tf.cluster_info.get('s3api', {}).get( 'max_multi_delete_objects', 1000) # specified number of objects are over max_multi_delete_objects # (Default 1000), but xml size is relatively small req_objects = ['obj%s' for var in range(max_deletes + 1)] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'MalformedXML') # specified xml size is large, but number of objects are # smaller than max_multi_delete_objects. obj = 'a' * 102400 req_objects = [obj + str(var) for var in range(max_deletes - 1)] xml = self._gen_multi_delete_xml(req_objects) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'MalformedXML')
def test_delete_multi_objects_error(self): bucket = 'bucket' put_objects = ['obj'] self._prepare_test_delete_multi_objects(bucket, put_objects) xml = self._gen_multi_delete_xml(put_objects) content_md5 = calculate_md5(xml) query = 'delete' auth_error_conn = Connection(aws_secret_key='invalid') status, headers, body = \ auth_error_conn.make_request('POST', bucket, body=xml, headers={ 'Content-MD5': content_md5 }, query=query) self.assertEqual(get_error_code(body), 'SignatureDoesNotMatch') status, headers, body = \ self.conn.make_request('POST', 'nothing', body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'NoSuchBucket') # without Object tag xml = self._gen_invalid_multi_delete_xml() content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'MalformedXML') # without value of Key tag xml = self._gen_invalid_multi_delete_xml(hasObjectTag=True) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'UserKeyMustBeSpecified') # specified number of objects are over max_multi_delete_objects # (Default 1000), but xml size is smaller than 61365 bytes. req_objects = ['obj%s' for var in xrange(1001)] xml = self._gen_multi_delete_xml(req_objects) self.assertTrue(len(xml.encode('utf-8')) <= MAX_MULTI_DELETE_BODY_SIZE) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'MalformedXML') # specified xml size is over 61365 bytes, but number of objects are # smaller than max_multi_delete_objects. obj = 'a' * 1024 req_objects = [obj + str(var) for var in xrange(999)] xml = self._gen_multi_delete_xml(req_objects) self.assertTrue(len(xml.encode('utf-8')) > MAX_MULTI_DELETE_BODY_SIZE) content_md5 = calculate_md5(xml) status, headers, body = \ self.conn.make_request('POST', bucket, body=xml, headers={'Content-MD5': content_md5}, query=query) self.assertEqual(get_error_code(body), 'MalformedXML')