class ObjectTestSuite(EutesterTestCase):
data_size = 1000
def __init__(self):
self.setuptestcase()
self.setup_parser()
self.parser.add_argument("--s3endpoint", default=None)
self.get_args()
# Setup basic eutester object
if self.args.s3endpoint:
self.tester = S3ops( credpath=self.args.credpath, endpoint=self.args.endpoint)
else:
self.tester = Eucaops( credpath=self.args.credpath, config_file=self.args.config, password=self.args.password)
self.bucket_prefix = "eutester-bucket-test-suite-" + str(int(time.time())) + "-"
self.buckets_used = set()
random.seed(time.time())
self.test_bucket_name = self.bucket_prefix + str(random.randint(0,100))
self.test_bucket = self.tester.create_bucket(self.test_bucket_name)
self.buckets_used.add(self.test_bucket_name)
#Create some test data for the objects
self.test_object_data = ""
for i in range(0, self.data_size):
self.test_object_data += chr(random.randint(32,126))
print "Generated data for objects: " + self.test_object_data
def print_key_info(self, keys=None):
for key in keys:
self.tester.info("Key=" + str(key.key) + " -- version= " + str(key.version_id) + " -- eTag= " + str(key.etag)
+ " -- ACL= " + str(key.get_xml_acl()))
def put_object(self, bucket=None, object_key=None, object_data=None):
"""Puts an object with the specified name and data in the specified bucket"""
if bucket == None:
raise Exception("Cannot put object without proper bucket reference")
try :
key = Key(bucket=bucket,name=object_key)
key.set_contents_from_string(object_data)
return key.etag
except Exception as e:
self.tester.info("Exception occured during 'PUT' of object " + object_key + " into bucket " + bucket.name + ": " + e.message)
return None
def enable_versioning(self, bucket):
"""Enable versioning on the bucket, checking that it is not already enabled and that the operation succeeds."""
vstatus = bucket.get_versioning_status()
if vstatus != None and len(vstatus.keys()) > 0 and vstatus['Versioning'] != None and vstatus['Versioning'] != 'Disabled':
self.tester.info("Versioning status should be null/Disabled, found: " + vstatus['Versioning'])
return False
else:
self.tester.info("Bucket versioning is Disabled")
#Enable versioning
bucket.configure_versioning(True)
if bucket.get_versioning_status()['Versioning'] == 'Enabled':
self.tester.info("Versioning status correctly set to enabled")
return True
else:
self.tester.info("Versioning status not enabled, should be.")
return False
return False
def suspend_versioning(self, bucket):
"""Suspend versioning on the bucket, checking that it is previously enabled and that the operation succeeds."""
if bucket.get_versioning_status()['Versioning'] == 'Enabled':
self.tester.info("Versioning status correctly set to enabled")
else:
self.tester.info("Versioning status not enabled, should be. Can't suspend if not enabled....")
return False
#Enable versioning
bucket.configure_versioning(False)
if bucket.get_versioning_status()['Versioning'] == 'Suspended':
self.tester.info("Versioning status correctly set to suspended")
return True
else:
self.tester.info("Versioning status not suspended.")
return False
return False
def check_version_listing(self, version_list, total_expected_length):
"""Checks a version listing for both completeness and ordering as well as pagination if required"""
self.tester.info("Checking bucket version listing. Listing is " + str(len(version_list)) + " entries long")
if total_expected_length >= 1000:
assert(len(version_list) == 999)
else:
assert(len(version_list) == total_expected_length)
prev_obj = None
should_fail = None
for obj in version_list:
if isinstance(obj,Key):
self.tester.info("Key: " + obj.name + " -- " + obj.version_id + "--" + obj.last_modified)
if prev_obj != None:
if self.compare_versions(prev_obj, obj) > 0:
should_fail = obj
prev_obj = obj
else:
self.tester.info("Not a key, skipping: " + str(obj))
return should_fail
def compare_versions(self, key1, key2):
"""
Returns -1 if key1 < key2, 0 if equal, and 1 if key1 > key2.
Compares names lexicographically, if equal, compares date_modified if versions are different.
If version_id and name are equal then key1 = key2
If an error occurs or something is wrong, returns None
"""
if key1.name < key2.name:
#self.debug("Key1: " + key1.name + " is less than " + key2.name)
return 1
elif key1.name > key2.name:
#self.debug("Key1: " + key1.name + " is greater than " + key2.name)
return -1
else:
if key1.version_id == key2.version_id:
#self.debug("Key1: " + key1.name + " is the same version as " + key2.name)
return 0
else:
if dateutil.parser.parse(key1.last_modified) > dateutil.parser.parse(key2.last_modified):
#self.debug("Key1: " + key1.last_modified + " last modified is greater than " + key2.last_modified)
return 1
elif dateutil.parser.parse(key1.last_modified) < dateutil.parser.parse(key2.last_modified):
#self.debug("Key1: " + key1.last_modified + " last modified is less than " + key2.last_modified)
return -1
return None
def test_object_basic_ops(self):
"""
Tests basic operations on objects: simple GET,PUT,HEAD,DELETE.
"""
self.tester.info("Basic Object Operations Test (GET/PUT/HEAD)")
if self.test_bucket == None:
self.fail("Error: test_bucket not set, cannot run test")
#Test PUT & GET
testkey="testkey1-" + str(int(time.time()))
self.put_object(bucket=self.test_bucket, object_key=testkey, object_data=self.test_object_data)
ret_key = self.test_bucket.get_key(testkey)
ret_content = ret_key.get_contents_as_string()
if ret_content == self.test_object_data:
self.tester.info("Set content = get content, put passed")
else:
if ret_content != None:
self.tester.info("Got content: " + ret_content)
else:
self.tester.info("No content returned")
self.tester.info("Expected content: " + self.test_object_data)
self.fail("Put content not the same as what was returned")
#Test HEAD
key_meta = self.test_bucket.get_key(testkey)
if key_meta != ret_key:
self.tester.info("Something is wrong, the HEAD operation returned different metadata than the GET operation")
else:
self.tester.info("HEAD meta = GET meta, all is good")
#Test copy operation (GET w/source headers)
new_key = "testkey2"
self.test_bucket.copy_key(new_key, self.test_bucket_name,testkey)
keylist = self.test_bucket.list()
counter = 0
for k in keylist:
if isinstance(k, Prefix):
self.tester.info("Prefix: " + "NULL" if k == None else k.name)
else:
self.tester.info("Key: " + k.name + " Etag: " + k.etag)
counter += 1
if counter != 2:
self.fail("Expected 2 keys after copy operation, found only: " + len(keylist))
try:
ret_key = self.test_bucket.get_key(new_key)
except:
self.fail("Could not get object copy")
if ret_key == None:
self.fail("Could not get object copy")
if self.test_bucket.get_key(testkey).get_contents_as_string() != ret_key.get_contents_as_string():
self.fail("Contents of original key and copy don't match")
else:
self.tester.info("Copy key contents match original!")
#Test DELETE
self.test_bucket.delete_key(testkey)
ret_key = None
ret_key = self.test_bucket.get_key(testkey)
if ret_key:
self.tester.info("Erroneously got: " + ret_key.name)
raise S3ResponseError(404, "Should have thrown exception for getting a non-existent object")
self.tester.info("Finishing basic ops test")
def test_object_byte_offset_read(self):
"""Tests fetching specific byte offsets of the object"""
self.tester.info("Byte-range Offset GET Test")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
testkey = "rangetestkey-" + str(int(time.time()))
source_bytes = bytearray(self.test_object_data)
#Put the object initially
self.put_object(bucket=self.test_bucket, object_key=testkey, object_data=self.test_object_data)
#Test range for first 100 bytes of object
print "Trying start-range object get"
try:
data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=0-99"})
except:
self.fail("Failed range object get first 100 bytes")
startrangedata = bytearray(data_str)
print "Got: " + startrangedata
print "Expected: " + str(source_bytes[:100])
start = 0
for i in range(0,100):
if startrangedata[i-start] != source_bytes[i]:
print "Byte: " + startrangedata[i] + " differs!"
self.fail("Start-range Ranged-get failed")
print "Trying mid-object range"
try:
data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=500-599"})
except:
self.fail("Failed range object get for middle 100 bytes")
midrangedata = bytearray(data_str)
start = 500
for i in range(start,start+100):
if midrangedata[i-start] != source_bytes[i]:
print "Byte: " + midrangedata[i] + "differs!"
self.fail("Mid-range Ranged-get failed")
print "Trying end-range object get"
#Test range for last 100 bytes of object
try:
data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=800-899"})
except:
self.fail("Failed range object get for last 100 bytes")
endrangedata = bytearray(data_str)
print "Got: " + str(endrangedata)
start = 800
try:
for i in range(start,start+100):
if endrangedata[i-start] != source_bytes[i]:
print "Byte: " + endrangedata[i] + "differs!"
self.fail("End-range Ranged-get failed")
except Exception as e:
print "Exception! Received: " + e
print "Range test complete"
def test_object_post(self):
"""Test the POST method for putting objects, requires a pre-signed upload policy and url"""
self.fail("Test not implemented")
def test_object_large_objects(self):
"""Test operations on large objects (>1MB), but not so large that we must use the multi-part upload interface"""
self.tester.info("Testing large-ish objects over 1MB in size on bucket" + self.test_bucket_name)
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
test_data = ""
large_obj_size_bytes = 5 * 1024 * 1024 #5MB
self.tester.info("Generating " + str(large_obj_size_bytes) + " bytes of data")
#Create some test data
for i in range(0, large_obj_size_bytes):
test_data += chr(random.randint(32,126))
self.tester.info("Uploading object content of size: " + str(large_obj_size_bytes) + " bytes")
keyname = "largeobj-" + str(int(time.time()))
self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=test_data)
self.tester.info("Done uploading object")
ret_key = self.test_bucket.get_key(keyname)
ret_data = ret_key.get_contents_as_string()
if ret_data != test_data:
self.fail("Fetched data and generated data don't match")
else:
self.tester.info("Data matches!")
self.tester.info("Removing large object")
self.test_bucket.delete_key(ret_key)
self.tester.info("Complete large object test")
pass
def test_object_multipart(self):
"""Test the multipart upload interface"""
self.fail("Feature not implemented")
def test_object_versioning_enabled(self):
"""Tests object versioning for get/put/delete on a versioned bucket"""
self.tester.info("Testing bucket Versioning-Enabled")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
if not self.enable_versioning(self.test_bucket):
self.fail("Could not properly enable versioning")
#Create some keys
keyname = "versionkey-" + str(int(time.time()))
#Multiple versions of the data
v1data = self.test_object_data + "--version1"
v2data = self.test_object_data + "--version2"
v3data = self.test_object_data + "--version3"
#Test sequence: put v1, get v1, put v2, put v3, get v3, delete v3, restore with v1 (copy), put v3 again, delete v2 explicitly
self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=v1data)
#Get v1
obj_v1 = self.test_bucket.get_key(keyname)
self.tester.check_md5(eTag=obj_v1.etag,data=v1data)
self.tester.info("Initial bucket state after object uploads without versioning:")
self.print_key_info(keys=[obj_v1])
#Put v2 (and get/head to confirm success)
self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v2data)
obj_v2 = self.test_bucket.get_key(keyname)
self.tester.check_md5(eTag=obj_v2.etag,data=v2data)
self.print_key_info(keys=[obj_v1, obj_v2])
#Put v3 (and get/head to confirm success)
self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v3data)
obj_v3 = self.test_bucket.get_key(keyname)
self.tester.check_md5(eTag=obj_v3.etag,data=v3data)
self.print_key_info(keys=[obj_v1, obj_v2, obj_v3])
#Get a specific version, v1
v1_return = self.test_bucket.get_key(key_name=keyname,version_id=obj_v1.version_id)
self.print_key_info(keys=[v1_return])
#Delete current latest version (v3)
self.test_bucket.delete_key(keyname)
del_obj = self.test_bucket.get_key(keyname)
if del_obj:
self.tester.info("Erroneously got: " + del_obj.name)
raise S3ResponseError(404, "Should have thrown this exception for getting a non-existent object")
#Restore v1 using copy
try:
self.test_bucket.copy_key(new_key_name=obj_v1.key,src_bucket_name=self.test_bucket_name,src_key_name=keyname,src_version_id=obj_v1.version_id)
except S3ResponseError as e:
self.fail("Failed to restore key from previous version using copy got error: " + str(e.status))
restored_obj = self.test_bucket.get_key(keyname)
self.tester.check_md5(eTag=restored_obj.etag,data=v1data)
self.print_key_info(keys=[restored_obj])
#Put v3 again
self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v3data)
self.tester.check_md5(eTag=obj_v3.etag,data=v3data)
self.print_key_info([self.test_bucket.get_key(keyname)])
#Delete v2 explicitly
self.test_bucket.delete_key(key_name=obj_v2.key,version_id=obj_v2.version_id)
del_obj = self.test_bucket.get_key(keyname,version_id=obj_v2.version_id)
if del_obj:
raise S3ResponseError("Should have gotten 404 not-found error, but got: " + del_obj.key + " instead",404)
#Show what's on top
top_obj = self.test_bucket.get_key(keyname)
self.print_key_info([top_obj])
self.tester.check_md5(eTag=top_obj.etag,data=v3data)
self.tester.info("Finished the versioning enabled test. Success!!")
def clear_and_rebuild_bucket(self, bucket_name):
self.tester.clear_bucket(bucket_name)
return self.tester.create_bucket(bucket_name)
def test_object_versionlisting(self):
"""
Tests object version listing from a bucket
"""
version_max = 3
keyrange = 20
self.tester.info("Testing listing versions in a bucket and pagination using " + str(keyrange) + " keys with " + str(version_max) + " versions per key")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
if not self.enable_versioning(self.test_bucket):
self.fail("Could not enable versioning properly. Failing")
key = "testkey-" + str(int(time.time()))
keys = [ key + str(k) for k in range(0,keyrange)]
contents = [ self.test_object_data + "--v" + str(v) for v in range(0,version_max)]
try:
for keyname in keys:
#Put version_max versions of each key
for v in range(0,version_max):
self.tester.info("Putting: " + keyname + " version " + str(v))
self.test_bucket.new_key(keyname).set_contents_from_string(contents[v])
except S3ResponseError as e:
self.fail("Failed putting object versions for test: " + str(e.status))
listing = self.test_bucket.get_all_versions()
self.tester.info("Bucket version listing is " + str(len(listing)) + " entries long")
if keyrange * version_max >= 1000:
if not len(listing) == 999:
self.test_bucket.configure_versioning(False)
self.tester.debug(str(listing))
raise Exception("Bucket version listing did not limit the response to 999. Instead: " + str(len(listing)))
else:
if not len(listing) == keyrange * version_max:
self.test_bucket.configure_versioning(False)
self.tester.debug(str(listing))
raise Exception("Bucket version listing did not equal the number uploaded. Instead: " + str(len(listing)))
prev_obj = None
for obj in listing:
if isinstance(obj,Key):
self.tester.info("Key: " + obj.name + " -- " + obj.version_id + "--" + obj.last_modified)
if prev_obj != None:
if self.compare_versions(prev_obj, obj) <= 0:
raise Exception("Version listing not sorted correctly, offending key: " + obj.name + " version: " + obj.version_id + " date: " + obj.last_modified)
prev_obj = obj
else:
self.tester.info("Not a key, skipping: " + str(obj))
def test_object_versioning_suspended(self):
"""Tests object versioning on a suspended bucket, a more complicated test than the Enabled test"""
self.tester.info("Testing bucket Versioning-Suspended")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
#Create some keys
keyname1 = "versionkey1-" + str(int(time.time()))
keyname2 = "versionkey2-" + str(int(time.time()))
keyname3 = "versionkey3-" + str(int(time.time()))
keyname4 = "versionkey4-" + str(int(time.time()))
keyname5 = "versionkey5-" + str(int(time.time()))
v1data = self.test_object_data + "--version1"
v2data = self.test_object_data + "--version2"
v3data = self.test_object_data + "--version3"
vstatus = self.test_bucket.get_versioning_status()
if vstatus:
self.fail("Versioning status should be null/Disabled but was: " + str(vstatus))
else:
self.tester.info("Bucket versioning is Disabled")
self.put_object(bucket=self.test_bucket, object_key=keyname1, object_data=v1data)
self.put_object(bucket=self.test_bucket, object_key=keyname2, object_data=v1data)
self.put_object(bucket=self.test_bucket, object_key=keyname3, object_data=v1data)
self.put_object(bucket=self.test_bucket, object_key=keyname4, object_data=v1data)
self.put_object(bucket=self.test_bucket, object_key=keyname5, object_data=v1data)
key1 = self.test_bucket.get_key(keyname1)
key2 = self.test_bucket.get_key(keyname2)
key3 = self.test_bucket.get_key(keyname3)
key4 = self.test_bucket.get_key(keyname4)
key5 = self.test_bucket.get_key(keyname5)
self.tester.info("Initial bucket state after object uploads without versioning:")
self.print_key_info(keys=[key1,key2,key3,key4,key5])
#Enable versioning
self.test_bucket.configure_versioning(True)
if self.test_bucket.get_versioning_status():
self.tester.info("Versioning status correctly set to enabled")
else:
self.tester.info("Versionign status not enabled, should be.")
#Update a subset of the keys
key1_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname1,object_data=v2data)
key2_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname2,object_data=v2data)
key3_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname3,object_data=v2data)
key3_etag3=self.put_object(bucket=self.test_bucket, object_key=keyname3,object_data=v3data)
#Delete a key
self.test_bucket.delete_key(keyname5)
#Suspend versioning
self.test_bucket.configure_versioning(False)
#Get latest of each key
key1=self.test_bucket.get_key(keyname1)
key2=self.test_bucket.get_key(keyname2)
key3=self.test_bucket.get_key(keyname3)
key4=self.test_bucket.get_key(keyname4)
key5=self.test_bucket.get_key(keyname5)
#Delete a key
#Add a key
#Add same key again
#Fetch each key
def test_object_acl(self):
"""Tests object acl get/set and manipulation"""
self.fail("Test not implemented")
#TODO: test custom and canned acls that are both valid an invalid
def test_object_torrent(self):
"""Tests object torrents"""
self.fail("Feature not implemented yet")
def clean_method(self):
'''This is the teardown method'''
#Delete the testing bucket if it is left-over
self.tester.info('Deleting the buckets used for testing')
for bucket in self.buckets_used:
try:
self.tester.info('Checking bucket ' + bucket + ' for possible cleaning/delete')
if self.tester.s3.bucket_exists(bucket):
self.tester.info('Found bucket exists, cleaning it')
self.tester.clear_bucket(bucket)
self.buckets_used.remove(bucket)
else:
self.tester.info('Bucket ' + bucket + ' not found, skipping')
except:
self.tester.info('Exception checking bucket ' + bucket)
return
class ObjectTestSuite(EutesterTestCase):
data_size = 1000
def __init__(self):
self.setuptestcase()
self.setup_parser()
self.parser.add_argument("--endpoint", default=None)
self.get_args()
# Setup basic eutester object
if self.args.endpoint:
self.tester = S3ops(credpath=self.args.credpath, endpoint=self.args.endpoint)
else:
self.tester = Eucaops( credpath=self.args.credpath, config_file=self.args.config, password=self.args.password)
self.bucket_prefix = "eutester-" + str(int(time.time())) + "-"
self.buckets_used = set()
random.seed(time.time())
self.test_bucket_name = self.bucket_prefix + str(random.randint(0,100))
self.test_bucket = self.tester.create_bucket(self.test_bucket_name)
self.buckets_used.add(self.test_bucket_name)
#Create some test data for the objects
def ensure_bucket_exists():
try:
self.tester.s3.get_bucket(self.test_bucket_name)
return True
except Exception:
return False
self.tester.wait_for_result(ensure_bucket_exists, True)
self.test_object_data = ""
for i in range(0, self.data_size):
self.test_object_data += chr(random.randint(32,126))
print "Generated data for objects: " + self.test_object_data
def print_key_info(self, keys=None):
for key in keys:
self.tester.info("Key=" + str(key.key) + " -- version= " + str(key.version_id) + " -- eTag= " + str(key.etag)
+ " -- ACL= " + str(key.get_xml_acl()))
def put_object(self, bucket=None, object_key=None, object_data=None):
"""Puts an object with the specified name and data in the specified bucket"""
if bucket == None:
raise Exception("Cannot put object without proper bucket reference")
try :
key = Key(bucket=bucket,name=object_key)
key.set_contents_from_string(object_data)
return key.etag
except Exception as e:
self.tester.debug("Exception occured during 'PUT' of object " + object_key + " into bucket " + bucket.name + ": " + e.message)
raise e
def post_object(self, bucket_name=None, object_key=None, object_data=None, policy=None, acl=None):
"""Uploads an object using POST + form upload"""
fields = {
'key' : object_key,
'acl' : acl,
'AWSAccessKeyId': self.tester.get_access_key(),
'Policy' : policy,
'Signature': self.sign_policy(sak=self.tester.get_secret_key(), b64_policy_json=policy)
}
self.tester.info('Fields: ' + str(fields))
url = 'http://' + self.tester.s3.host + ':' + str(self.tester.s3.port) \
+ '/' + self.tester.s3.path + '/' + bucket_name
self.tester.debug('Sending POST request to: ' + url)
response = requests.post(url, data=fields, files={'file': BytesIO(object_data)})
return response
#return None
def generate_default_policy_b64(self, bucket, key, acl):
delta = timedelta(hours=1)
expire_time = (datetime.utcnow() + delta).replace(microsecond=0)
policy = {'conditions': [{'acl': acl},
{'bucket': bucket},
{'key': key},
],
'expiration': time.strftime('%Y-%m-%dT%H:%M:%SZ',
expire_time.timetuple())}
policy_json = json.dumps(policy)
self.tester.info('generated default policy: %s', policy_json)
return base64.b64encode(policy_json)
def sign_policy(self, sak=None, b64_policy_json=None):
my_hmac = hmac.new(sak, digestmod=hashlib.sha1)
my_hmac.update(b64_policy_json)
return base64.b64encode(my_hmac.digest())
def enable_versioning(self, bucket):
"""Enable versioning on the bucket, checking that it is not already enabled and that the operation succeeds."""
vstatus = bucket.get_versioning_status()
if vstatus != None and len(vstatus.keys()) > 0 and vstatus['Versioning'] != None and vstatus['Versioning'] != 'Disabled':
self.tester.info("Versioning status should be null/Disabled, found: " + vstatus['Versioning'])
return False
else:
self.tester.info("Bucket versioning is Disabled")
#Enable versioning
bucket.configure_versioning(True)
if bucket.get_versioning_status()['Versioning'] == 'Enabled':
self.tester.info("Versioning status correctly set to enabled")
return True
else:
self.tester.info("Versioning status not enabled, should be.")
return False
return False
def suspend_versioning(self, bucket):
"""Suspend versioning on the bucket, checking that it is previously enabled and that the operation succeeds."""
if bucket.get_versioning_status()['Versioning'] == 'Enabled':
self.tester.info("Versioning status correctly set to enabled")
else:
self.tester.info("Versioning status not enabled, should be. Can't suspend if not enabled....")
return False
#Enable versioning
bucket.configure_versioning(False)
if bucket.get_versioning_status()['Versioning'] == 'Suspended':
self.tester.info("Versioning status correctly set to suspended")
return True
else:
self.tester.info("Versioning status not suspended.")
return False
return False
def check_version_listing(self, version_list, total_expected_length):
"""Checks a version listing for both completeness and ordering as well as pagination if required"""
self.tester.info("Checking bucket version listing. Listing is " + str(len(version_list)) + " entries long")
if total_expected_length >= 1000:
assert(len(version_list) == 999)
else:
assert(len(version_list) == total_expected_length)
prev_obj = None
should_fail = None
for obj in version_list:
if isinstance(obj,Key):
self.tester.info("Key: " + obj.name + " -- " + obj.version_id + "--" + obj.last_modified)
if prev_obj != None:
if self.compare_versions(prev_obj, obj) > 0:
should_fail = obj
prev_obj = obj
else:
self.tester.info("Not a key, skipping: " + str(obj))
return should_fail
def compare_versions(self, key1, key2):
"""
Returns -1 if key1 < key2, 0 if equal, and 1 if key1 > key2.
Compares names lexicographically, if equal, compares date_modified if versions are different.
If version_id and name are equal then key1 = key2
If an error occurs or something is wrong, returns None
"""
if key1.name < key2.name:
#self.debug("Key1: " + key1.name + " is less than " + key2.name)
return 1
elif key1.name > key2.name:
#self.debug("Key1: " + key1.name + " is greater than " + key2.name)
return -1
else:
if key1.version_id == key2.version_id:
#self.debug("Key1: " + key1.name + " is the same version as " + key2.name)
return 0
else:
if dateutil.parser.parse(key1.last_modified) > dateutil.parser.parse(key2.last_modified):
#self.debug("Key1: " + key1.last_modified + " last modified is greater than " + key2.last_modified)
return 1
elif dateutil.parser.parse(key1.last_modified) < dateutil.parser.parse(key2.last_modified):
#self.debug("Key1: " + key1.last_modified + " last modified is less than " + key2.last_modified)
return -1
return None
def test_object_basic_ops(self):
"""
Tests basic operations on objects: simple GET,PUT,HEAD,DELETE.
"""
self.tester.info("Basic Object Operations Test (GET/PUT/HEAD)")
if self.test_bucket == None:
self.fail("Error: test_bucket not set, cannot run test")
#Test PUT & GET
testkey="testkey1-" + str(int(time.time()))
self.put_object(bucket=self.test_bucket, object_key=testkey, object_data=self.test_object_data)
ret_key = self.test_bucket.get_key(testkey)
ret_content = ret_key.get_contents_as_string()
if ret_content == self.test_object_data:
self.tester.info("Set content = get content, put passed")
else:
if ret_content != None:
self.tester.info("Got content: " + ret_content)
else:
self.tester.info("No content returned")
self.tester.info("Expected content: " + self.test_object_data)
self.fail("Put content not the same as what was returned")
#Test HEAD
key_meta = self.test_bucket.get_key(testkey)
if key_meta.key != ret_key.key or key_meta.etag != ret_key.etag or key_meta.size != ret_key.size:
self.tester.info("Something is wrong, the HEAD operation returned different metadata than the GET operation")
self.tester.info("Expected key " + ret_key.key + " etag: " + ret_key.etag + " Got: " + key_meta.key + " etag: " + key_meta.etag)
else:
self.tester.info("HEAD meta = GET meta, all is good")
#Test copy operation (GET w/source headers)
self.tester.info("Testing COPY object")
new_key = "testkey2"
self.test_bucket.copy_key(new_key_name=new_key, src_bucket_name=self.test_bucket_name, src_key_name=testkey)
keylist = self.test_bucket.list()
counter = 0
for k in keylist:
if isinstance(k, Prefix):
self.tester.info("Prefix: " + "NULL" if k == None else k.name)
else:
self.tester.info("Key: " + k.name + " Etag: " + k.etag)
counter += 1
if counter != 2:
self.fail("Expected 2 keys after copy operation, found only: " + len(keylist))
try:
ret_key = self.test_bucket.get_key(new_key)
except:
self.fail("Could not get object copy")
if ret_key == None:
self.fail("Could not get object copy")
if self.test_bucket.get_key(testkey).get_contents_as_string() != ret_key.get_contents_as_string():
self.fail("Contents of original key and copy don't match")
else:
self.tester.info("Copy key contents match original!")
#Test DELETE
self.test_bucket.delete_key(testkey)
ret_key = None
ret_key = self.test_bucket.get_key(testkey)
if ret_key:
self.tester.info("Erroneously got: " + ret_key.name)
raise S3ResponseError(404, "Should have thrown exception for getting a non-existent object")
self.tester.info("Finishing basic ops test")
def test_object_byte_offset_read(self):
"""Tests fetching specific byte offsets of the object"""
self.tester.info("Byte-range Offset GET Test")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
testkey = "rangetestkey-" + str(int(time.time()))
source_bytes = bytearray(self.test_object_data)
#Put the object initially
self.put_object(bucket=self.test_bucket, object_key=testkey, object_data=self.test_object_data)
#Test range for first 100 bytes of object
print "Trying start-range object get"
try:
data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=0-99"})
except:
self.fail("Failed range object get first 100 bytes")
startrangedata = bytearray(data_str)
print "Got: " + startrangedata
print "Expected: " + str(source_bytes[:100])
start = 0
for i in range(0,100):
if startrangedata[i-start] != source_bytes[i]:
print "Byte: " + startrangedata[i] + " differs!"
self.fail("Start-range Ranged-get failed")
print "Trying mid-object range"
try:
data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=500-599"})
except:
self.fail("Failed range object get for middle 100 bytes")
midrangedata = bytearray(data_str)
start = 500
for i in range(start,start+100):
if midrangedata[i-start] != source_bytes[i]:
print "Byte: " + midrangedata[i] + "differs!"
self.fail("Mid-range Ranged-get failed")
print "Trying end-range object get"
#Test range for last 100 bytes of object
try:
data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=800-899"})
except:
self.fail("Failed range object get for last 100 bytes")
endrangedata = bytearray(data_str)
print "Got: " + str(endrangedata)
start = 800
try:
for i in range(start,start+100):
if endrangedata[i-start] != source_bytes[i]:
print "Byte: " + endrangedata[i] + "differs!"
self.fail("End-range Ranged-get failed")
except Exception as e:
print "Exception! Received: " + e
print "Range test complete"
def test_object_post(self):
"""Test the POST method for putting objects, requires a pre-signed upload policy and url"""
self.tester.info("Testing POST form upload on bucket" + self.test_bucket_name)
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
itr = 1
self.tester.info('Doing ' + str(itr) + ' POST upload iterations')
acl = 'ec2-bundle-read'
for k in xrange(0, itr):
key = 'postkey1' + str(k)
data = os.urandom(512)
computed_md5 = '"' + hashlib.md5(data).hexdigest() + '"'
self.tester.info('Data md5: ' + computed_md5 + ' data length: ' + str(len(computed_md5)))
self.tester.info('Uploading object ' + self.test_bucket_name + '/' + key + ' via POST with acl : ' + acl)
response = self.post_object(bucket_name=self.test_bucket_name,
object_key=key,
object_data=data,
acl=acl,
policy=self.generate_default_policy_b64(self.test_bucket_name, key, acl=acl))
self.tester.info('Got response for POST: ' + str(response.status_code) + ': ' + str(response.text))
assert(response.status_code == 204)
fetched_key = self.test_bucket.get_key(key)
fetched_content = fetched_key.get_contents_as_string()
self.tester.info('Got fetched md5: ' + fetched_key.etag)
self.tester.info('Calculated md5: ' + computed_md5 + ' recieved md5 ' + fetched_key.etag)
assert(fetched_key.etag == computed_md5)
assert(fetched_content == data)
self.tester.info("Done with upload test")
def test_object_post_large(self):
"""Test the POST method for putting objects, requires a pre-signed upload policy and url"""
self.tester.info("Testing POST form upload on bucket" + self.test_bucket_name)
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
self.tester.info("Testing POST form upload of 10MB data on bucket" + self.test_bucket_name)
itr = 1
large_obj_size_bytes = 10 * 1024 * 1024 #10MB content
self.tester.info('Doing ' + str(itr) + ' iterations of large object of size ' + str(large_obj_size_bytes) + ' with POST')
acl = 'ec2-bundle-read'
for i in xrange(0, itr):
key = 'postkey_10mb_' + str(i)
self.tester.info('Generating ' + str(large_obj_size_bytes) + ' bytes for upload')
#Create some test data
data = str(os.urandom(large_obj_size_bytes))
self.tester.info("Data length: " + str(len(data)))
computed_md5 = '"' + hashlib.md5(data).hexdigest() + '"'
self.tester.info('Data md5 is: ' + computed_md5)
self.tester.info('Uploading object via POST using acl: ' + acl)
response = self.post_object(bucket_name=self.test_bucket.name,
object_key=key,
object_data=data,
policy=self.generate_default_policy_b64(self.test_bucket.name, key, acl=acl),
acl=acl)
self.tester.info('Got response for POST: ' + str(response.status_code) + ': ' + str(response.text))
assert(response.status_code == 204)
self.tester.info('Fetching the content for verification')
fetched_key = self.test_bucket.get_key(key_name=key)
self.tester.info('Got fetched content length : ' + str(fetched_key.size) + ' Expected ' + str(len(data)))
assert(fetched_key.size == len(data))
self.tester.info('Got fetched md5: ' + fetched_key.etag)
self.tester.info('Calculated md5: ' + computed_md5 + ' recieved md5 ' + fetched_key.etag)
assert(fetched_key.etag == computed_md5)
fetched_content = fetched_key.get_contents_as_string()
assert(fetched_content == data)
def test_object_large_objects(self):
"""Test operations on large objects (>1MB), but not so large that we must use the multi-part upload interface"""
self.tester.info("Testing large-ish objects over 1MB in size on bucket" + self.test_bucket_name)
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
test_data = None
large_obj_size_bytes = 5 * 1024 * 1024 #5MB
self.tester.info("Generating " + str(large_obj_size_bytes) + " bytes of data")
#Create some test data
#for i in range(0, large_obj_size_bytes):
# test_data += chr(random.randint(32,126))
test_data = bytearray(os.urandom(large_obj_size_bytes))
self.tester.info("Uploading object content of size: " + str(large_obj_size_bytes) + " bytes")
keyname = "largeobj-" + str(int(time.time()))
self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=test_data)
self.tester.info("Done uploading object")
ret_key = self.test_bucket.get_key(keyname)
ret_data = ret_key.get_contents_as_string()
if ret_data != test_data:
self.fail("Fetched data and generated data don't match")
else:
self.tester.info("Data matches!")
self.tester.info("Removing large object")
self.test_bucket.delete_key(ret_key)
self.tester.info("Complete large object test")
pass
def test_object_multipart(self):
"""Tests basic multipart upload functionality"""
self.tester.info("Testing Multipart")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
self.test_multipart_upload()
self.test_abort_multipart_upload()
def test_multipart_upload(self):
'''Basic multipart upload'''
self.tester.info("Testing multipart upload")
self.tester.info("Creating random file representing part...")
temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart")
temp_file.write(os.urandom(5 * 1024 * 1024))
keyname="multi-" + str(int(time.time()))
self.tester.info("Initiating multipart upload...much upload")
reply = self.initiate_multipart_upload(keyname)
self.tester.info("Uploading parts...Such Parts")
for partnum in range(1, 11):
temp_file.seek(0, os.SEEK_SET)
reply.upload_part_from_file(temp_file, partnum)
self.tester.info("Listing parts...")
self.test_bucket.get_all_multipart_uploads()
self.tester.info("Completing upload...So OSG")
reply.complete_upload()
temp_file.close()
self.tester.info("HEAD request...");
returned_key = self.test_bucket.get_key(keyname, validate=True);
download_temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="mpu-download")
self.tester.info("Downloading object...very mpu");
returned_key.get_contents_to_file(download_temp_file);
self.tester.info("Deleting object...WOW")
self.test_bucket.delete_key(keyname)
def test_abort_multipart_upload(self):
'''Basic multipart upload'''
self.tester.info("Testing abort multipart upload")
temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart")
temp_file.write(os.urandom(5 * 1024 * 1024))
keyname="multi-" + str(int(time.time()))
reply = self.initiate_multipart_upload(keyname)
for partnum in range(1, 11):
temp_file.seek(0, os.SEEK_SET)
reply.upload_part_from_file(temp_file, partnum)
self.test_bucket.get_all_multipart_uploads()
self.tester.info("Canceling upload")
reply.cancel_upload()
temp_file.close()
def initiate_multipart_upload(self, keyname):
self.tester.info("Initiating multipart upload " + keyname)
return self.test_bucket.initiate_multipart_upload(keyname)
def test_object_versioning_enabled(self):
"""Tests object versioning for get/put/delete on a versioned bucket"""
self.tester.info("Testing bucket Versioning-Enabled")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
if not self.enable_versioning(self.test_bucket):
self.fail("Could not properly enable versioning")
#Create some keys
keyname = "versionkey-" + str(int(time.time()))
#Multiple versions of the data
v1data = self.test_object_data + "--version1"
v2data = self.test_object_data + "--version2"
v3data = self.test_object_data + "--version3"
#Test sequence: put v1, get v1, put v2, put v3, get v3, delete v3, restore with v1 (copy), put v3 again, delete v2 explicitly
self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=v1data)
#Get v1
obj_v1 = self.test_bucket.get_key(keyname)
self.tester.check_md5(eTag=obj_v1.etag,data=v1data)
self.tester.info("Initial bucket state after object uploads with versioning enabled:")
self.print_key_info(keys=[obj_v1])
self.tester.info("Adding another version")
#Put v2 (and get/head to confirm success)
self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v2data)
obj_v2 = self.test_bucket.get_key(keyname)
self.tester.check_md5(eTag=obj_v2.etag,data=v2data)
self.print_key_info(keys=[obj_v1, obj_v2])
self.tester.info("Adding another version")
#Put v3 (and get/head to confirm success)
self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v3data)
obj_v3 = self.test_bucket.get_key(keyname)
self.tester.check_md5(eTag=obj_v3.etag,data=v3data)
self.print_key_info(keys=[obj_v1, obj_v2, obj_v3])
self.tester.info("Getting specific version")
#Get a specific version, v1
v1_return = self.test_bucket.get_key(key_name=keyname,version_id=obj_v1.version_id)
self.print_key_info(keys=[v1_return])
#Delete current latest version (v3)
self.test_bucket.delete_key(keyname)
del_obj = self.test_bucket.get_key(keyname)
if del_obj:
self.tester.info("Erroneously got: " + del_obj.name)
raise S3ResponseError(404, "Should have thrown this exception for getting a non-existent object")
#Restore v1 using copy
self.tester.info("Restoring version")
try:
self.test_bucket.copy_key(new_key_name=obj_v1.key,src_bucket_name=self.test_bucket_name,src_key_name=keyname,src_version_id=obj_v1.version_id)
except S3ResponseError as e:
self.fail("Failed to restore key from previous version using copy got error: " + str(e.status))
restored_obj = self.test_bucket.get_key(keyname)
assert(restored_obj != None)
self.tester.check_md5(eTag=restored_obj.etag,data=v1data)
self.print_key_info(keys=[restored_obj])
#Put v3 again
self.tester.info("Adding another version")
self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v3data)
self.tester.check_md5(eTag=obj_v3.etag,data=v3data)
self.print_key_info([self.test_bucket.get_key(keyname)])
#Delete v2 explicitly
self.test_bucket.delete_key(key_name=obj_v2.key,version_id=obj_v2.version_id)
del_obj = self.test_bucket.get_key(keyname,version_id=obj_v2.version_id)
if del_obj:
raise S3ResponseError("Should have gotten 404 not-found error, but got: " + del_obj.key + " instead",404)
#Show what's on top
top_obj = self.test_bucket.get_key(keyname)
self.print_key_info([top_obj])
self.tester.check_md5(eTag=top_obj.etag,data=v3data)
self.tester.info("Finished the versioning enabled test. Success!!")
def clear_and_rebuild_bucket(self, bucket_name):
self.tester.clear_bucket(bucket_name)
return self.tester.create_bucket(bucket_name)
def test_object_versionlisting(self):
"""
Tests object version listing from a bucket
"""
version_max = 3
keyrange = 20
self.tester.info("Testing listing versions in a bucket and pagination using " + str(keyrange) + " keys with " + str(version_max) + " versions per key")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
if not self.enable_versioning(self.test_bucket):
self.fail("Could not enable versioning properly. Failing")
key = "testkey-" + str(int(time.time()))
keys = [ key + str(k) for k in range(0,keyrange)]
contents = [ self.test_object_data + "--v" + str(v) for v in range(0,version_max)]
try:
for keyname in keys:
#Put version_max versions of each key
for v in range(0,version_max):
self.tester.info("Putting: " + keyname + " version " + str(v))
self.test_bucket.new_key(keyname).set_contents_from_string(contents[v])
except S3ResponseError as e:
self.fail("Failed putting object versions for test: " + str(e.status))
listing = self.test_bucket.get_all_versions()
self.tester.info("Bucket version listing is " + str(len(listing)) + " entries long")
if keyrange * version_max >= 1000:
if not len(listing) == 999:
self.test_bucket.configure_versioning(False)
self.tester.debug(str(listing))
raise Exception("Bucket version listing did not limit the response to 999. Instead: " + str(len(listing)))
else:
if not len(listing) == keyrange * version_max:
self.test_bucket.configure_versioning(False)
self.tester.debug(str(listing))
raise Exception("Bucket version listing did not equal the number uploaded. Instead: " + str(len(listing)))
prev_obj = None
for obj in listing:
if isinstance(obj,Key):
self.tester.info("Key: " + obj.name + " -- " + obj.version_id + "--" + obj.last_modified)
if prev_obj != None:
if self.compare_versions(prev_obj, obj) <= 0:
raise Exception("Version listing not sorted correctly, offending key: " + obj.name + " version: " + obj.version_id + " date: " + obj.last_modified)
prev_obj = obj
else:
self.tester.info("Not a key, skipping: " + str(obj))
def test_object_versioning_suspended(self):
"""Tests object versioning on a suspended bucket, a more complicated test than the Enabled test"""
self.tester.info("Testing bucket Versioning-Suspended")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
#Create some keys
keyname1 = "versionkey1-" + str(int(time.time()))
keyname2 = "versionkey2-" + str(int(time.time()))
keyname3 = "versionkey3-" + str(int(time.time()))
keyname4 = "versionkey4-" + str(int(time.time()))
keyname5 = "versionkey5-" + str(int(time.time()))
v1data = self.test_object_data + "--version1"
v2data = self.test_object_data + "--version2"
v3data = self.test_object_data + "--version3"
vstatus = self.test_bucket.get_versioning_status()
if vstatus:
self.fail("Versioning status should be null/Disabled but was: " + str(vstatus))
else:
self.tester.info("Bucket versioning is Disabled")
self.put_object(bucket=self.test_bucket, object_key=keyname1, object_data=v1data)
self.put_object(bucket=self.test_bucket, object_key=keyname2, object_data=v1data)
self.put_object(bucket=self.test_bucket, object_key=keyname3, object_data=v1data)
self.put_object(bucket=self.test_bucket, object_key=keyname4, object_data=v1data)
self.put_object(bucket=self.test_bucket, object_key=keyname5, object_data=v1data)
key1 = self.test_bucket.get_key(keyname1)
key2 = self.test_bucket.get_key(keyname2)
key3 = self.test_bucket.get_key(keyname3)
key4 = self.test_bucket.get_key(keyname4)
key5 = self.test_bucket.get_key(keyname5)
self.tester.info("Initial bucket state after object uploads without versioning:")
self.print_key_info(keys=[key1,key2,key3,key4,key5])
#Enable versioning
self.test_bucket.configure_versioning(True)
if self.test_bucket.get_versioning_status():
self.tester.info("Versioning status correctly set to enabled")
else:
self.tester.info("Versionign status not enabled, should be.")
#Update a subset of the keys
key1_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname1,object_data=v2data)
key2_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname2,object_data=v2data)
key3_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname3,object_data=v2data)
key3_etag3=self.put_object(bucket=self.test_bucket, object_key=keyname3,object_data=v3data)
#Delete a key
self.test_bucket.delete_key(keyname5)
#Suspend versioning
self.test_bucket.configure_versioning(False)
#Get latest of each key
key1=self.test_bucket.get_key(keyname1)
key2=self.test_bucket.get_key(keyname2)
key3=self.test_bucket.get_key(keyname3)
key4=self.test_bucket.get_key(keyname4)
key5=self.test_bucket.get_key(keyname5)
#Delete a key
#Add a key
#Add same key again
#Fetch each key
def test_object_acl(self):
"""Tests object acl get/set and manipulation"""
self.fail("Test not implemented")
#TODO: test custom and canned acls that are both valid an invalid
def test_object_torrent(self):
"""Tests object torrents"""
self.fail("Feature not implemented yet")
def clean_method(self):
'''This is the teardown method'''
#Delete the testing bucket if it is left-over
self.tester.info('Deleting the buckets used for testing')
for bucket in self.buckets_used:
try:
self.tester.info('Checking bucket ' + bucket + ' for possible cleaning/delete')
if self.tester.s3.head_bucket(bucket) != None:
self.tester.info('Found bucket exists, cleaning it')
self.tester.clear_bucket(bucket)
else:
self.tester.info('Bucket ' + bucket + ' not found, skipping')
except Exception as e:
self.tester.info('Exception checking bucket ' + bucket + ' Exception msg: ' + e.message)
return
def test_multipart_upload(self):
'''Basic multipart upload'''
self.tester.info("Testing multipart upload")
self.tester.info("Creating random file representing part...")
temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart")
temp_file.write(os.urandom(5 * 1024 * 1024))
keyname="multi-" + str(int(time.time()))
self.tester.info("Initiating multipart upload...much upload")
reply = self.initiate_multipart_upload(keyname)
self.tester.info("Uploading parts...Such Parts")
for partnum in range(1, 11):
temp_file.seek(0, os.SEEK_SET)
reply.upload_part_from_file(temp_file, partnum)
self.tester.info("Listing parts...")
self.test_bucket.get_all_multipart_uploads()
self.tester.info("Completing upload...So OSG")
reply.complete_upload()
temp_file.close()
self.tester.info("HEAD request...");
returned_key = self.test_bucket.get_key(keyname, validate=True);
download_temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="mpu-download")
self.tester.info("Downloading object...very mpu");
returned_key.get_contents_to_file(download_temp_file);
self.tester.info("Deleting object...WOW")
self.test_bucket.delete_key(keyname)
def test_abort_multipart_upload(self):
'''Basic multipart upload'''
self.tester.info("Testing abort multipart upload")
temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart")
temp_file.write(os.urandom(5 * 1024 * 1024))
keyname="multi-" + str(int(time.time()))
reply = self.initiate_multipart_upload(keyname)
for partnum in range(1, 11):
temp_file.seek(0, os.SEEK_SET)
reply.upload_part_from_file(temp_file, partnum)
self.test_bucket.get_all_multipart_uploads()
self.tester.info("!!!!!!!!!!!!!!!!!!!!!!!!!!DO NOT WANT!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1")
self.tester.info("Canceling upload")
reply.cancel_upload()
temp_file.close()
def initiate_multipart_upload(self, keyname):
self.tester.info("Initiating multipart upload " + keyname)
return self.test_bucket.initiate_multipart_upload(keyname)
class ObjectTestSuite(EutesterTestCase):
data_size = 1000
def __init__(self):
self.setuptestcase()
self.setup_parser()
self.parser.add_argument("--endpoint", default=None)
self.get_args()
# Setup basic eutester object
if self.args.endpoint:
self.tester = S3ops(credpath=self.args.credpath,
endpoint=self.args.endpoint)
else:
self.tester = Eucaops(credpath=self.args.credpath,
config_file=self.args.config,
password=self.args.password)
self.bucket_prefix = "eutester-" + str(int(time.time())) + "-"
self.buckets_used = set()
random.seed(time.time())
self.test_bucket_name = self.bucket_prefix + str(random.randint(
0, 100))
self.test_bucket = self.tester.create_bucket(self.test_bucket_name)
self.buckets_used.add(self.test_bucket_name)
#Create some test data for the objects
def ensure_bucket_exists():
try:
self.tester.s3.get_bucket(self.test_bucket_name)
return True
except Exception:
return False
self.tester.wait_for_result(ensure_bucket_exists, True)
self.test_object_data = ""
for i in range(0, self.data_size):
self.test_object_data += chr(random.randint(32, 126))
print "Generated data for objects: " + self.test_object_data
def print_key_info(self, keys=None):
for key in keys:
self.tester.info("Key=" + str(key.key) + " -- version= " +
str(key.version_id) + " -- eTag= " +
str(key.etag) + " -- ACL= " +
str(key.get_xml_acl()))
def put_object(self, bucket=None, object_key=None, object_data=None):
"""Puts an object with the specified name and data in the specified bucket"""
if bucket == None:
raise Exception(
"Cannot put object without proper bucket reference")
try:
key = Key(bucket=bucket, name=object_key)
key.set_contents_from_string(object_data)
return key.etag
except Exception as e:
self.tester.debug("Exception occured during 'PUT' of object " +
object_key + " into bucket " + bucket.name +
": " + e.message)
raise e
def post_object(self,
bucket_name=None,
object_key=None,
object_data=None,
policy=None,
acl=None):
"""Uploads an object using POST + form upload"""
fields = {
'key':
object_key,
'acl':
acl,
'AWSAccessKeyId':
self.tester.get_access_key(),
'Policy':
policy,
'Signature':
self.sign_policy(sak=self.tester.get_secret_key(),
b64_policy_json=policy)
}
self.tester.info('Fields: ' + str(fields))
url = 'http://' + self.tester.s3.host + ':' + str(self.tester.s3.port) \
+ '/' + self.tester.s3.path + '/' + bucket_name
self.tester.debug('Sending POST request to: ' + url)
response = requests.post(url,
data=fields,
files={'file': BytesIO(object_data)})
return response
#return None
def generate_default_policy_b64(self, bucket, key, acl):
delta = timedelta(hours=1)
expire_time = (datetime.utcnow() + delta).replace(microsecond=0)
policy = {
'conditions': [
{
'acl': acl
},
{
'bucket': bucket
},
{
'key': key
},
],
'expiration':
time.strftime('%Y-%m-%dT%H:%M:%SZ', expire_time.timetuple())
}
policy_json = json.dumps(policy)
self.tester.info('generated default policy: %s', policy_json)
return base64.b64encode(policy_json)
def sign_policy(self, sak=None, b64_policy_json=None):
my_hmac = hmac.new(sak, digestmod=hashlib.sha1)
my_hmac.update(b64_policy_json)
return base64.b64encode(my_hmac.digest())
def enable_versioning(self, bucket):
"""Enable versioning on the bucket, checking that it is not already enabled and that the operation succeeds."""
vstatus = bucket.get_versioning_status()
if vstatus != None and len(vstatus.keys()) > 0 and vstatus[
'Versioning'] != None and vstatus['Versioning'] != 'Disabled':
self.tester.info(
"Versioning status should be null/Disabled, found: " +
vstatus['Versioning'])
return False
else:
self.tester.info("Bucket versioning is Disabled")
#Enable versioning
bucket.configure_versioning(True)
if bucket.get_versioning_status()['Versioning'] == 'Enabled':
self.tester.info("Versioning status correctly set to enabled")
return True
else:
self.tester.info("Versioning status not enabled, should be.")
return False
return False
def suspend_versioning(self, bucket):
"""Suspend versioning on the bucket, checking that it is previously enabled and that the operation succeeds."""
if bucket.get_versioning_status()['Versioning'] == 'Enabled':
self.tester.info("Versioning status correctly set to enabled")
else:
self.tester.info(
"Versioning status not enabled, should be. Can't suspend if not enabled...."
)
return False
#Enable versioning
bucket.configure_versioning(False)
if bucket.get_versioning_status()['Versioning'] == 'Suspended':
self.tester.info("Versioning status correctly set to suspended")
return True
else:
self.tester.info("Versioning status not suspended.")
return False
return False
def check_version_listing(self, version_list, total_expected_length):
"""Checks a version listing for both completeness and ordering as well as pagination if required"""
self.tester.info("Checking bucket version listing. Listing is " +
str(len(version_list)) + " entries long")
if total_expected_length >= 1000:
assert (len(version_list) == 999)
else:
assert (len(version_list) == total_expected_length)
prev_obj = None
should_fail = None
for obj in version_list:
if isinstance(obj, Key):
self.tester.info("Key: " + obj.name + " -- " + obj.version_id +
"--" + obj.last_modified)
if prev_obj != None:
if self.compare_versions(prev_obj, obj) > 0:
should_fail = obj
prev_obj = obj
else:
self.tester.info("Not a key, skipping: " + str(obj))
return should_fail
def compare_versions(self, key1, key2):
"""
Returns -1 if key1 < key2, 0 if equal, and 1 if key1 > key2.
Compares names lexicographically, if equal, compares date_modified if versions are different.
If version_id and name are equal then key1 = key2
If an error occurs or something is wrong, returns None
"""
if key1.name < key2.name:
#self.debug("Key1: " + key1.name + " is less than " + key2.name)
return 1
elif key1.name > key2.name:
#self.debug("Key1: " + key1.name + " is greater than " + key2.name)
return -1
else:
if key1.version_id == key2.version_id:
#self.debug("Key1: " + key1.name + " is the same version as " + key2.name)
return 0
else:
if dateutil.parser.parse(
key1.last_modified) > dateutil.parser.parse(
key2.last_modified):
#self.debug("Key1: " + key1.last_modified + " last modified is greater than " + key2.last_modified)
return 1
elif dateutil.parser.parse(
key1.last_modified) < dateutil.parser.parse(
key2.last_modified):
#self.debug("Key1: " + key1.last_modified + " last modified is less than " + key2.last_modified)
return -1
return None
def test_object_basic_ops(self):
"""
Tests basic operations on objects: simple GET,PUT,HEAD,DELETE.
"""
self.tester.info("Basic Object Operations Test (GET/PUT/HEAD)")
if self.test_bucket == None:
self.fail("Error: test_bucket not set, cannot run test")
#Test PUT & GET
testkey = "testkey1-" + str(int(time.time()))
self.put_object(bucket=self.test_bucket,
object_key=testkey,
object_data=self.test_object_data)
ret_key = self.test_bucket.get_key(testkey)
ret_content = ret_key.get_contents_as_string()
if ret_content == self.test_object_data:
self.tester.info("Set content = get content, put passed")
else:
if ret_content != None:
self.tester.info("Got content: " + ret_content)
else:
self.tester.info("No content returned")
self.tester.info("Expected content: " + self.test_object_data)
self.fail("Put content not the same as what was returned")
#Test HEAD
key_meta = self.test_bucket.get_key(testkey)
if key_meta.key != ret_key.key or key_meta.etag != ret_key.etag or key_meta.size != ret_key.size:
self.tester.info(
"Something is wrong, the HEAD operation returned different metadata than the GET operation"
)
self.tester.info("Expected key " + ret_key.key + " etag: " +
ret_key.etag + " Got: " + key_meta.key +
" etag: " + key_meta.etag)
else:
self.tester.info("HEAD meta = GET meta, all is good")
#Test copy operation (GET w/source headers)
self.tester.info("Testing COPY object")
new_key = "testkey2"
self.test_bucket.copy_key(new_key_name=new_key,
src_bucket_name=self.test_bucket_name,
src_key_name=testkey)
keylist = self.test_bucket.list()
counter = 0
for k in keylist:
if isinstance(k, Prefix):
self.tester.info("Prefix: " + "NULL" if k == None else k.name)
else:
self.tester.info("Key: " + k.name + " Etag: " + k.etag)
counter += 1
if counter != 2:
self.fail("Expected 2 keys after copy operation, found only: " +
len(keylist))
try:
ret_key = self.test_bucket.get_key(new_key)
except:
self.fail("Could not get object copy")
if ret_key == None:
self.fail("Could not get object copy")
if self.test_bucket.get_key(testkey).get_contents_as_string(
) != ret_key.get_contents_as_string():
self.fail("Contents of original key and copy don't match")
else:
self.tester.info("Copy key contents match original!")
#Test DELETE
self.test_bucket.delete_key(testkey)
ret_key = None
ret_key = self.test_bucket.get_key(testkey)
if ret_key:
self.tester.info("Erroneously got: " + ret_key.name)
raise S3ResponseError(
404,
"Should have thrown exception for getting a non-existent object"
)
self.tester.info("Finishing basic ops test")
def test_object_byte_offset_read(self):
"""Tests fetching specific byte offsets of the object"""
self.tester.info("Byte-range Offset GET Test")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
testkey = "rangetestkey-" + str(int(time.time()))
source_bytes = bytearray(self.test_object_data)
#Put the object initially
self.put_object(bucket=self.test_bucket,
object_key=testkey,
object_data=self.test_object_data)
#Test range for first 100 bytes of object
print "Trying start-range object get"
try:
data_str = Key(bucket=self.test_bucket,
name=testkey).get_contents_as_string(
headers={"Range": "bytes=0-99"})
except:
self.fail("Failed range object get first 100 bytes")
startrangedata = bytearray(data_str)
print "Got: " + startrangedata
print "Expected: " + str(source_bytes[:100])
start = 0
for i in range(0, 100):
if startrangedata[i - start] != source_bytes[i]:
print "Byte: " + startrangedata[i] + " differs!"
self.fail("Start-range Ranged-get failed")
print "Trying mid-object range"
try:
data_str = Key(bucket=self.test_bucket,
name=testkey).get_contents_as_string(
headers={"Range": "bytes=500-599"})
except:
self.fail("Failed range object get for middle 100 bytes")
midrangedata = bytearray(data_str)
start = 500
for i in range(start, start + 100):
if midrangedata[i - start] != source_bytes[i]:
print "Byte: " + midrangedata[i] + "differs!"
self.fail("Mid-range Ranged-get failed")
print "Trying end-range object get"
#Test range for last 100 bytes of object
try:
data_str = Key(bucket=self.test_bucket,
name=testkey).get_contents_as_string(
headers={"Range": "bytes=800-899"})
except:
self.fail("Failed range object get for last 100 bytes")
endrangedata = bytearray(data_str)
print "Got: " + str(endrangedata)
start = 800
try:
for i in range(start, start + 100):
if endrangedata[i - start] != source_bytes[i]:
print "Byte: " + endrangedata[i] + "differs!"
self.fail("End-range Ranged-get failed")
except Exception as e:
print "Exception! Received: " + e
print "Range test complete"
def test_object_post(self):
"""Test the POST method for putting objects, requires a pre-signed upload policy and url"""
self.tester.info("Testing POST form upload on bucket" +
self.test_bucket_name)
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
itr = 1
self.tester.info('Doing ' + str(itr) + ' POST upload iterations')
acl = 'ec2-bundle-read'
for k in xrange(0, itr):
key = 'postkey1' + str(k)
data = os.urandom(512)
computed_md5 = '"' + hashlib.md5(data).hexdigest() + '"'
self.tester.info('Data md5: ' + computed_md5 + ' data length: ' +
str(len(computed_md5)))
self.tester.info('Uploading object ' + self.test_bucket_name +
'/' + key + ' via POST with acl : ' + acl)
response = self.post_object(
bucket_name=self.test_bucket_name,
object_key=key,
object_data=data,
acl=acl,
policy=self.generate_default_policy_b64(self.test_bucket_name,
key,
acl=acl))
self.tester.info('Got response for POST: ' +
str(response.status_code) + ': ' +
str(response.text))
assert (response.status_code == 204)
fetched_key = self.test_bucket.get_key(key)
fetched_content = fetched_key.get_contents_as_string()
self.tester.info('Got fetched md5: ' + fetched_key.etag)
self.tester.info('Calculated md5: ' + computed_md5 +
' recieved md5 ' + fetched_key.etag)
assert (fetched_key.etag == computed_md5)
assert (fetched_content == data)
self.tester.info("Done with upload test")
def test_object_post_large(self):
"""Test the POST method for putting objects, requires a pre-signed upload policy and url"""
self.tester.info("Testing POST form upload on bucket" +
self.test_bucket_name)
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
self.tester.info("Testing POST form upload of 10MB data on bucket" +
self.test_bucket_name)
itr = 1
large_obj_size_bytes = 10 * 1024 * 1024 #10MB content
self.tester.info('Doing ' + str(itr) +
' iterations of large object of size ' +
str(large_obj_size_bytes) + ' with POST')
acl = 'ec2-bundle-read'
for i in xrange(0, itr):
key = 'postkey_10mb_' + str(i)
self.tester.info('Generating ' + str(large_obj_size_bytes) +
' bytes for upload')
#Create some test data
data = str(os.urandom(large_obj_size_bytes))
self.tester.info("Data length: " + str(len(data)))
computed_md5 = '"' + hashlib.md5(data).hexdigest() + '"'
self.tester.info('Data md5 is: ' + computed_md5)
self.tester.info('Uploading object via POST using acl: ' + acl)
response = self.post_object(
bucket_name=self.test_bucket.name,
object_key=key,
object_data=data,
policy=self.generate_default_policy_b64(self.test_bucket.name,
key,
acl=acl),
acl=acl)
self.tester.info('Got response for POST: ' +
str(response.status_code) + ': ' +
str(response.text))
assert (response.status_code == 204)
self.tester.info('Fetching the content for verification')
fetched_key = self.test_bucket.get_key(key_name=key)
self.tester.info('Got fetched content length : ' +
str(fetched_key.size) + ' Expected ' +
str(len(data)))
assert (fetched_key.size == len(data))
self.tester.info('Got fetched md5: ' + fetched_key.etag)
self.tester.info('Calculated md5: ' + computed_md5 +
' recieved md5 ' + fetched_key.etag)
assert (fetched_key.etag == computed_md5)
fetched_content = fetched_key.get_contents_as_string()
assert (fetched_content == data)
def test_object_large_objects(self):
"""Test operations on large objects (>1MB), but not so large that we must use the multi-part upload interface"""
self.tester.info(
"Testing large-ish objects over 1MB in size on bucket" +
self.test_bucket_name)
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
test_data = None
large_obj_size_bytes = 5 * 1024 * 1024 #5MB
self.tester.info("Generating " + str(large_obj_size_bytes) +
" bytes of data")
#Create some test data
#for i in range(0, large_obj_size_bytes):
# test_data += chr(random.randint(32,126))
test_data = bytearray(os.urandom(large_obj_size_bytes))
self.tester.info("Uploading object content of size: " +
str(large_obj_size_bytes) + " bytes")
keyname = "largeobj-" + str(int(time.time()))
self.put_object(bucket=self.test_bucket,
object_key=keyname,
object_data=test_data)
self.tester.info("Done uploading object")
ret_key = self.test_bucket.get_key(keyname)
ret_data = ret_key.get_contents_as_string()
if ret_data != test_data:
self.fail("Fetched data and generated data don't match")
else:
self.tester.info("Data matches!")
self.tester.info("Removing large object")
self.test_bucket.delete_key(ret_key)
self.tester.info("Complete large object test")
pass
def test_object_multipart(self):
"""Tests basic multipart upload functionality"""
self.tester.info("Testing Multipart")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
self.test_multipart_upload()
self.test_abort_multipart_upload()
def test_multipart_upload(self):
'''Basic multipart upload'''
self.tester.info("Testing multipart upload")
self.tester.info("Creating random file representing part...")
temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart")
temp_file.write(os.urandom(5 * 1024 * 1024))
keyname = "multi-" + str(int(time.time()))
self.tester.info("Initiating multipart upload...much upload")
reply = self.initiate_multipart_upload(keyname)
self.tester.info("Uploading parts...Such Parts")
for partnum in range(1, 11):
temp_file.seek(0, os.SEEK_SET)
reply.upload_part_from_file(temp_file, partnum)
self.tester.info("Listing parts...")
self.test_bucket.get_all_multipart_uploads()
self.tester.info("Completing upload...So OSG")
reply.complete_upload()
temp_file.close()
self.tester.info("HEAD request...")
returned_key = self.test_bucket.get_key(keyname, validate=True)
download_temp_file = tempfile.NamedTemporaryFile(mode="w+b",
prefix="mpu-download")
self.tester.info("Downloading object...very mpu")
returned_key.get_contents_to_file(download_temp_file)
self.tester.info("Deleting object...WOW")
self.test_bucket.delete_key(keyname)
def test_abort_multipart_upload(self):
'''Basic multipart upload'''
self.tester.info("Testing abort multipart upload")
temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart")
temp_file.write(os.urandom(5 * 1024 * 1024))
keyname = "multi-" + str(int(time.time()))
reply = self.initiate_multipart_upload(keyname)
for partnum in range(1, 11):
temp_file.seek(0, os.SEEK_SET)
reply.upload_part_from_file(temp_file, partnum)
self.test_bucket.get_all_multipart_uploads()
self.tester.info("Canceling upload")
reply.cancel_upload()
temp_file.close()
def initiate_multipart_upload(self, keyname):
self.tester.info("Initiating multipart upload " + keyname)
return self.test_bucket.initiate_multipart_upload(keyname)
def test_object_versioning_enabled(self):
"""Tests object versioning for get/put/delete on a versioned bucket"""
self.tester.info("Testing bucket Versioning-Enabled")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
if not self.enable_versioning(self.test_bucket):
self.fail("Could not properly enable versioning")
#Create some keys
keyname = "versionkey-" + str(int(time.time()))
#Multiple versions of the data
v1data = self.test_object_data + "--version1"
v2data = self.test_object_data + "--version2"
v3data = self.test_object_data + "--version3"
#Test sequence: put v1, get v1, put v2, put v3, get v3, delete v3, restore with v1 (copy), put v3 again, delete v2 explicitly
self.put_object(bucket=self.test_bucket,
object_key=keyname,
object_data=v1data)
#Get v1
obj_v1 = self.test_bucket.get_key(keyname)
self.tester.check_md5(eTag=obj_v1.etag, data=v1data)
self.tester.info(
"Initial bucket state after object uploads with versioning enabled:"
)
self.print_key_info(keys=[obj_v1])
self.tester.info("Adding another version")
#Put v2 (and get/head to confirm success)
self.put_object(bucket=self.test_bucket,
object_key=keyname,
object_data=v2data)
obj_v2 = self.test_bucket.get_key(keyname)
self.tester.check_md5(eTag=obj_v2.etag, data=v2data)
self.print_key_info(keys=[obj_v1, obj_v2])
self.tester.info("Adding another version")
#Put v3 (and get/head to confirm success)
self.put_object(bucket=self.test_bucket,
object_key=keyname,
object_data=v3data)
obj_v3 = self.test_bucket.get_key(keyname)
self.tester.check_md5(eTag=obj_v3.etag, data=v3data)
self.print_key_info(keys=[obj_v1, obj_v2, obj_v3])
self.tester.info("Getting specific version")
#Get a specific version, v1
v1_return = self.test_bucket.get_key(key_name=keyname,
version_id=obj_v1.version_id)
self.print_key_info(keys=[v1_return])
#Delete current latest version (v3)
self.test_bucket.delete_key(keyname)
del_obj = self.test_bucket.get_key(keyname)
if del_obj:
self.tester.info("Erroneously got: " + del_obj.name)
raise S3ResponseError(
404,
"Should have thrown this exception for getting a non-existent object"
)
#Restore v1 using copy
self.tester.info("Restoring version")
try:
self.test_bucket.copy_key(new_key_name=obj_v1.key,
src_bucket_name=self.test_bucket_name,
src_key_name=keyname,
src_version_id=obj_v1.version_id)
except S3ResponseError as e:
self.fail(
"Failed to restore key from previous version using copy got error: "
+ str(e.status))
restored_obj = self.test_bucket.get_key(keyname)
assert (restored_obj != None)
self.tester.check_md5(eTag=restored_obj.etag, data=v1data)
self.print_key_info(keys=[restored_obj])
#Put v3 again
self.tester.info("Adding another version")
self.put_object(bucket=self.test_bucket,
object_key=keyname,
object_data=v3data)
self.tester.check_md5(eTag=obj_v3.etag, data=v3data)
self.print_key_info([self.test_bucket.get_key(keyname)])
#Delete v2 explicitly
self.test_bucket.delete_key(key_name=obj_v2.key,
version_id=obj_v2.version_id)
del_obj = self.test_bucket.get_key(keyname,
version_id=obj_v2.version_id)
if del_obj:
raise S3ResponseError(
"Should have gotten 404 not-found error, but got: " +
del_obj.key + " instead", 404)
#Show what's on top
top_obj = self.test_bucket.get_key(keyname)
self.print_key_info([top_obj])
self.tester.check_md5(eTag=top_obj.etag, data=v3data)
self.tester.info("Finished the versioning enabled test. Success!!")
def clear_and_rebuild_bucket(self, bucket_name):
self.tester.clear_bucket(bucket_name)
return self.tester.create_bucket(bucket_name)
def test_object_versionlisting(self):
"""
Tests object version listing from a bucket
"""
version_max = 3
keyrange = 20
self.tester.info(
"Testing listing versions in a bucket and pagination using " +
str(keyrange) + " keys with " + str(version_max) +
" versions per key")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
if not self.enable_versioning(self.test_bucket):
self.fail("Could not enable versioning properly. Failing")
key = "testkey-" + str(int(time.time()))
keys = [key + str(k) for k in range(0, keyrange)]
contents = [
self.test_object_data + "--v" + str(v)
for v in range(0, version_max)
]
try:
for keyname in keys:
#Put version_max versions of each key
for v in range(0, version_max):
self.tester.info("Putting: " + keyname + " version " +
str(v))
self.test_bucket.new_key(keyname).set_contents_from_string(
contents[v])
except S3ResponseError as e:
self.fail("Failed putting object versions for test: " +
str(e.status))
listing = self.test_bucket.get_all_versions()
self.tester.info("Bucket version listing is " + str(len(listing)) +
" entries long")
if keyrange * version_max >= 1000:
if not len(listing) == 999:
self.test_bucket.configure_versioning(False)
self.tester.debug(str(listing))
raise Exception(
"Bucket version listing did not limit the response to 999. Instead: "
+ str(len(listing)))
else:
if not len(listing) == keyrange * version_max:
self.test_bucket.configure_versioning(False)
self.tester.debug(str(listing))
raise Exception(
"Bucket version listing did not equal the number uploaded. Instead: "
+ str(len(listing)))
prev_obj = None
for obj in listing:
if isinstance(obj, Key):
self.tester.info("Key: " + obj.name + " -- " + obj.version_id +
"--" + obj.last_modified)
if prev_obj != None:
if self.compare_versions(prev_obj, obj) <= 0:
raise Exception(
"Version listing not sorted correctly, offending key: "
+ obj.name + " version: " + obj.version_id +
" date: " + obj.last_modified)
prev_obj = obj
else:
self.tester.info("Not a key, skipping: " + str(obj))
def test_object_versioning_suspended(self):
"""Tests object versioning on a suspended bucket, a more complicated test than the Enabled test"""
self.tester.info("Testing bucket Versioning-Suspended")
self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name)
#Create some keys
keyname1 = "versionkey1-" + str(int(time.time()))
keyname2 = "versionkey2-" + str(int(time.time()))
keyname3 = "versionkey3-" + str(int(time.time()))
keyname4 = "versionkey4-" + str(int(time.time()))
keyname5 = "versionkey5-" + str(int(time.time()))
v1data = self.test_object_data + "--version1"
v2data = self.test_object_data + "--version2"
v3data = self.test_object_data + "--version3"
vstatus = self.test_bucket.get_versioning_status()
if vstatus:
self.fail("Versioning status should be null/Disabled but was: " +
str(vstatus))
else:
self.tester.info("Bucket versioning is Disabled")
self.put_object(bucket=self.test_bucket,
object_key=keyname1,
object_data=v1data)
self.put_object(bucket=self.test_bucket,
object_key=keyname2,
object_data=v1data)
self.put_object(bucket=self.test_bucket,
object_key=keyname3,
object_data=v1data)
self.put_object(bucket=self.test_bucket,
object_key=keyname4,
object_data=v1data)
self.put_object(bucket=self.test_bucket,
object_key=keyname5,
object_data=v1data)
key1 = self.test_bucket.get_key(keyname1)
key2 = self.test_bucket.get_key(keyname2)
key3 = self.test_bucket.get_key(keyname3)
key4 = self.test_bucket.get_key(keyname4)
key5 = self.test_bucket.get_key(keyname5)
self.tester.info(
"Initial bucket state after object uploads without versioning:")
self.print_key_info(keys=[key1, key2, key3, key4, key5])
#Enable versioning
self.test_bucket.configure_versioning(True)
if self.test_bucket.get_versioning_status():
self.tester.info("Versioning status correctly set to enabled")
else:
self.tester.info("Versionign status not enabled, should be.")
#Update a subset of the keys
key1_etag2 = self.put_object(bucket=self.test_bucket,
object_key=keyname1,
object_data=v2data)
key2_etag2 = self.put_object(bucket=self.test_bucket,
object_key=keyname2,
object_data=v2data)
key3_etag2 = self.put_object(bucket=self.test_bucket,
object_key=keyname3,
object_data=v2data)
key3_etag3 = self.put_object(bucket=self.test_bucket,
object_key=keyname3,
object_data=v3data)
#Delete a key
self.test_bucket.delete_key(keyname5)
#Suspend versioning
self.test_bucket.configure_versioning(False)
#Get latest of each key
key1 = self.test_bucket.get_key(keyname1)
key2 = self.test_bucket.get_key(keyname2)
key3 = self.test_bucket.get_key(keyname3)
key4 = self.test_bucket.get_key(keyname4)
key5 = self.test_bucket.get_key(keyname5)
#Delete a key
#Add a key
#Add same key again
#Fetch each key
def test_object_acl(self):
"""Tests object acl get/set and manipulation"""
self.fail("Test not implemented")
#TODO: test custom and canned acls that are both valid an invalid
def test_object_torrent(self):
"""Tests object torrents"""
self.fail("Feature not implemented yet")
def clean_method(self):
'''This is the teardown method'''
#Delete the testing bucket if it is left-over
self.tester.info('Deleting the buckets used for testing')
for bucket in self.buckets_used:
try:
self.tester.info('Checking bucket ' + bucket +
' for possible cleaning/delete')
if self.tester.s3.head_bucket(bucket) != None:
self.tester.info('Found bucket exists, cleaning it')
self.tester.clear_bucket(bucket)
else:
self.tester.info('Bucket ' + bucket +
' not found, skipping')
except Exception as e:
self.tester.info('Exception checking bucket ' + bucket +
' Exception msg: ' + e.message)
return
def test_multipart_upload(self):
'''Basic multipart upload'''
self.tester.info("Testing multipart upload")
self.tester.info("Creating random file representing part...")
temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart")
temp_file.write(os.urandom(5 * 1024 * 1024))
keyname = "multi-" + str(int(time.time()))
self.tester.info("Initiating multipart upload...much upload")
reply = self.initiate_multipart_upload(keyname)
self.tester.info("Uploading parts...Such Parts")
for partnum in range(1, 11):
temp_file.seek(0, os.SEEK_SET)
reply.upload_part_from_file(temp_file, partnum)
self.tester.info("Listing parts...")
self.test_bucket.get_all_multipart_uploads()
self.tester.info("Completing upload...So OSG")
reply.complete_upload()
temp_file.close()
self.tester.info("HEAD request...")
returned_key = self.test_bucket.get_key(keyname, validate=True)
download_temp_file = tempfile.NamedTemporaryFile(mode="w+b",
prefix="mpu-download")
self.tester.info("Downloading object...very mpu")
returned_key.get_contents_to_file(download_temp_file)
self.tester.info("Deleting object...WOW")
self.test_bucket.delete_key(keyname)
def test_abort_multipart_upload(self):
'''Basic multipart upload'''
self.tester.info("Testing abort multipart upload")
temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart")
temp_file.write(os.urandom(5 * 1024 * 1024))
keyname = "multi-" + str(int(time.time()))
reply = self.initiate_multipart_upload(keyname)
for partnum in range(1, 11):
temp_file.seek(0, os.SEEK_SET)
reply.upload_part_from_file(temp_file, partnum)
self.test_bucket.get_all_multipart_uploads()
self.tester.info(
"!!!!!!!!!!!!!!!!!!!!!!!!!!DO NOT WANT!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1"
)
self.tester.info("Canceling upload")
reply.cancel_upload()
temp_file.close()
def initiate_multipart_upload(self, keyname):
self.tester.info("Initiating multipart upload " + keyname)
return self.test_bucket.initiate_multipart_upload(keyname)
