class ObjectTestSuite(EutesterTestCase): data_size = 1000 def __init__(self): self.setuptestcase() self.setup_parser() self.parser.add_argument("--s3endpoint", default=None) self.get_args() # Setup basic eutester object if self.args.s3endpoint: self.tester = S3ops( credpath=self.args.credpath, endpoint=self.args.endpoint) else: self.tester = Eucaops( credpath=self.args.credpath, config_file=self.args.config, password=self.args.password) self.bucket_prefix = "eutester-bucket-test-suite-" + str(int(time.time())) + "-" self.buckets_used = set() random.seed(time.time()) self.test_bucket_name = self.bucket_prefix + str(random.randint(0,100)) self.test_bucket = self.tester.create_bucket(self.test_bucket_name) self.buckets_used.add(self.test_bucket_name) #Create some test data for the objects self.test_object_data = "" for i in range(0, self.data_size): self.test_object_data += chr(random.randint(32,126)) print "Generated data for objects: " + self.test_object_data def print_key_info(self, keys=None): for key in keys: self.tester.info("Key=" + str(key.key) + " -- version= " + str(key.version_id) + " -- eTag= " + str(key.etag) + " -- ACL= " + str(key.get_xml_acl())) def put_object(self, bucket=None, object_key=None, object_data=None): """Puts an object with the specified name and data in the specified bucket""" if bucket == None: raise Exception("Cannot put object without proper bucket reference") try : key = Key(bucket=bucket,name=object_key) key.set_contents_from_string(object_data) return key.etag except Exception as e: self.tester.info("Exception occured during 'PUT' of object " + object_key + " into bucket " + bucket.name + ": " + e.message) return None def enable_versioning(self, bucket): """Enable versioning on the bucket, checking that it is not already enabled and that the operation succeeds.""" vstatus = bucket.get_versioning_status() if vstatus != None and len(vstatus.keys()) > 0 and vstatus['Versioning'] != None and vstatus['Versioning'] != 'Disabled': self.tester.info("Versioning status should be null/Disabled, found: " + vstatus['Versioning']) return False else: self.tester.info("Bucket versioning is Disabled") #Enable versioning bucket.configure_versioning(True) if bucket.get_versioning_status()['Versioning'] == 'Enabled': self.tester.info("Versioning status correctly set to enabled") return True else: self.tester.info("Versioning status not enabled, should be.") return False return False def suspend_versioning(self, bucket): """Suspend versioning on the bucket, checking that it is previously enabled and that the operation succeeds.""" if bucket.get_versioning_status()['Versioning'] == 'Enabled': self.tester.info("Versioning status correctly set to enabled") else: self.tester.info("Versioning status not enabled, should be. Can't suspend if not enabled....") return False #Enable versioning bucket.configure_versioning(False) if bucket.get_versioning_status()['Versioning'] == 'Suspended': self.tester.info("Versioning status correctly set to suspended") return True else: self.tester.info("Versioning status not suspended.") return False return False def check_version_listing(self, version_list, total_expected_length): """Checks a version listing for both completeness and ordering as well as pagination if required""" self.tester.info("Checking bucket version listing. Listing is " + str(len(version_list)) + " entries long") if total_expected_length >= 1000: assert(len(version_list) == 999) else: assert(len(version_list) == total_expected_length) prev_obj = None should_fail = None for obj in version_list: if isinstance(obj,Key): self.tester.info("Key: " + obj.name + " -- " + obj.version_id + "--" + obj.last_modified) if prev_obj != None: if self.compare_versions(prev_obj, obj) > 0: should_fail = obj prev_obj = obj else: self.tester.info("Not a key, skipping: " + str(obj)) return should_fail def compare_versions(self, key1, key2): """ Returns -1 if key1 < key2, 0 if equal, and 1 if key1 > key2. Compares names lexicographically, if equal, compares date_modified if versions are different. If version_id and name are equal then key1 = key2 If an error occurs or something is wrong, returns None """ if key1.name < key2.name: #self.debug("Key1: " + key1.name + " is less than " + key2.name) return 1 elif key1.name > key2.name: #self.debug("Key1: " + key1.name + " is greater than " + key2.name) return -1 else: if key1.version_id == key2.version_id: #self.debug("Key1: " + key1.name + " is the same version as " + key2.name) return 0 else: if dateutil.parser.parse(key1.last_modified) > dateutil.parser.parse(key2.last_modified): #self.debug("Key1: " + key1.last_modified + " last modified is greater than " + key2.last_modified) return 1 elif dateutil.parser.parse(key1.last_modified) < dateutil.parser.parse(key2.last_modified): #self.debug("Key1: " + key1.last_modified + " last modified is less than " + key2.last_modified) return -1 return None def test_object_basic_ops(self): """ Tests basic operations on objects: simple GET,PUT,HEAD,DELETE. """ self.tester.info("Basic Object Operations Test (GET/PUT/HEAD)") if self.test_bucket == None: self.fail("Error: test_bucket not set, cannot run test") #Test PUT & GET testkey="testkey1-" + str(int(time.time())) self.put_object(bucket=self.test_bucket, object_key=testkey, object_data=self.test_object_data) ret_key = self.test_bucket.get_key(testkey) ret_content = ret_key.get_contents_as_string() if ret_content == self.test_object_data: self.tester.info("Set content = get content, put passed") else: if ret_content != None: self.tester.info("Got content: " + ret_content) else: self.tester.info("No content returned") self.tester.info("Expected content: " + self.test_object_data) self.fail("Put content not the same as what was returned") #Test HEAD key_meta = self.test_bucket.get_key(testkey) if key_meta != ret_key: self.tester.info("Something is wrong, the HEAD operation returned different metadata than the GET operation") else: self.tester.info("HEAD meta = GET meta, all is good") #Test copy operation (GET w/source headers) new_key = "testkey2" self.test_bucket.copy_key(new_key, self.test_bucket_name,testkey) keylist = self.test_bucket.list() counter = 0 for k in keylist: if isinstance(k, Prefix): self.tester.info("Prefix: " + "NULL" if k == None else k.name) else: self.tester.info("Key: " + k.name + " Etag: " + k.etag) counter += 1 if counter != 2: self.fail("Expected 2 keys after copy operation, found only: " + len(keylist)) try: ret_key = self.test_bucket.get_key(new_key) except: self.fail("Could not get object copy") if ret_key == None: self.fail("Could not get object copy") if self.test_bucket.get_key(testkey).get_contents_as_string() != ret_key.get_contents_as_string(): self.fail("Contents of original key and copy don't match") else: self.tester.info("Copy key contents match original!") #Test DELETE self.test_bucket.delete_key(testkey) ret_key = None ret_key = self.test_bucket.get_key(testkey) if ret_key: self.tester.info("Erroneously got: " + ret_key.name) raise S3ResponseError(404, "Should have thrown exception for getting a non-existent object") self.tester.info("Finishing basic ops test") def test_object_byte_offset_read(self): """Tests fetching specific byte offsets of the object""" self.tester.info("Byte-range Offset GET Test") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) testkey = "rangetestkey-" + str(int(time.time())) source_bytes = bytearray(self.test_object_data) #Put the object initially self.put_object(bucket=self.test_bucket, object_key=testkey, object_data=self.test_object_data) #Test range for first 100 bytes of object print "Trying start-range object get" try: data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=0-99"}) except: self.fail("Failed range object get first 100 bytes") startrangedata = bytearray(data_str) print "Got: " + startrangedata print "Expected: " + str(source_bytes[:100]) start = 0 for i in range(0,100): if startrangedata[i-start] != source_bytes[i]: print "Byte: " + startrangedata[i] + " differs!" self.fail("Start-range Ranged-get failed") print "Trying mid-object range" try: data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=500-599"}) except: self.fail("Failed range object get for middle 100 bytes") midrangedata = bytearray(data_str) start = 500 for i in range(start,start+100): if midrangedata[i-start] != source_bytes[i]: print "Byte: " + midrangedata[i] + "differs!" self.fail("Mid-range Ranged-get failed") print "Trying end-range object get" #Test range for last 100 bytes of object try: data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=800-899"}) except: self.fail("Failed range object get for last 100 bytes") endrangedata = bytearray(data_str) print "Got: " + str(endrangedata) start = 800 try: for i in range(start,start+100): if endrangedata[i-start] != source_bytes[i]: print "Byte: " + endrangedata[i] + "differs!" self.fail("End-range Ranged-get failed") except Exception as e: print "Exception! Received: " + e print "Range test complete" def test_object_post(self): """Test the POST method for putting objects, requires a pre-signed upload policy and url""" self.fail("Test not implemented") def test_object_large_objects(self): """Test operations on large objects (>1MB), but not so large that we must use the multi-part upload interface""" self.tester.info("Testing large-ish objects over 1MB in size on bucket" + self.test_bucket_name) self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) test_data = "" large_obj_size_bytes = 5 * 1024 * 1024 #5MB self.tester.info("Generating " + str(large_obj_size_bytes) + " bytes of data") #Create some test data for i in range(0, large_obj_size_bytes): test_data += chr(random.randint(32,126)) self.tester.info("Uploading object content of size: " + str(large_obj_size_bytes) + " bytes") keyname = "largeobj-" + str(int(time.time())) self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=test_data) self.tester.info("Done uploading object") ret_key = self.test_bucket.get_key(keyname) ret_data = ret_key.get_contents_as_string() if ret_data != test_data: self.fail("Fetched data and generated data don't match") else: self.tester.info("Data matches!") self.tester.info("Removing large object") self.test_bucket.delete_key(ret_key) self.tester.info("Complete large object test") pass def test_object_multipart(self): """Test the multipart upload interface""" self.fail("Feature not implemented") def test_object_versioning_enabled(self): """Tests object versioning for get/put/delete on a versioned bucket""" self.tester.info("Testing bucket Versioning-Enabled") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) if not self.enable_versioning(self.test_bucket): self.fail("Could not properly enable versioning") #Create some keys keyname = "versionkey-" + str(int(time.time())) #Multiple versions of the data v1data = self.test_object_data + "--version1" v2data = self.test_object_data + "--version2" v3data = self.test_object_data + "--version3" #Test sequence: put v1, get v1, put v2, put v3, get v3, delete v3, restore with v1 (copy), put v3 again, delete v2 explicitly self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=v1data) #Get v1 obj_v1 = self.test_bucket.get_key(keyname) self.tester.check_md5(eTag=obj_v1.etag,data=v1data) self.tester.info("Initial bucket state after object uploads without versioning:") self.print_key_info(keys=[obj_v1]) #Put v2 (and get/head to confirm success) self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v2data) obj_v2 = self.test_bucket.get_key(keyname) self.tester.check_md5(eTag=obj_v2.etag,data=v2data) self.print_key_info(keys=[obj_v1, obj_v2]) #Put v3 (and get/head to confirm success) self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v3data) obj_v3 = self.test_bucket.get_key(keyname) self.tester.check_md5(eTag=obj_v3.etag,data=v3data) self.print_key_info(keys=[obj_v1, obj_v2, obj_v3]) #Get a specific version, v1 v1_return = self.test_bucket.get_key(key_name=keyname,version_id=obj_v1.version_id) self.print_key_info(keys=[v1_return]) #Delete current latest version (v3) self.test_bucket.delete_key(keyname) del_obj = self.test_bucket.get_key(keyname) if del_obj: self.tester.info("Erroneously got: " + del_obj.name) raise S3ResponseError(404, "Should have thrown this exception for getting a non-existent object") #Restore v1 using copy try: self.test_bucket.copy_key(new_key_name=obj_v1.key,src_bucket_name=self.test_bucket_name,src_key_name=keyname,src_version_id=obj_v1.version_id) except S3ResponseError as e: self.fail("Failed to restore key from previous version using copy got error: " + str(e.status)) restored_obj = self.test_bucket.get_key(keyname) self.tester.check_md5(eTag=restored_obj.etag,data=v1data) self.print_key_info(keys=[restored_obj]) #Put v3 again self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v3data) self.tester.check_md5(eTag=obj_v3.etag,data=v3data) self.print_key_info([self.test_bucket.get_key(keyname)]) #Delete v2 explicitly self.test_bucket.delete_key(key_name=obj_v2.key,version_id=obj_v2.version_id) del_obj = self.test_bucket.get_key(keyname,version_id=obj_v2.version_id) if del_obj: raise S3ResponseError("Should have gotten 404 not-found error, but got: " + del_obj.key + " instead",404) #Show what's on top top_obj = self.test_bucket.get_key(keyname) self.print_key_info([top_obj]) self.tester.check_md5(eTag=top_obj.etag,data=v3data) self.tester.info("Finished the versioning enabled test. Success!!") def clear_and_rebuild_bucket(self, bucket_name): self.tester.clear_bucket(bucket_name) return self.tester.create_bucket(bucket_name) def test_object_versionlisting(self): """ Tests object version listing from a bucket """ version_max = 3 keyrange = 20 self.tester.info("Testing listing versions in a bucket and pagination using " + str(keyrange) + " keys with " + str(version_max) + " versions per key") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) if not self.enable_versioning(self.test_bucket): self.fail("Could not enable versioning properly. Failing") key = "testkey-" + str(int(time.time())) keys = [ key + str(k) for k in range(0,keyrange)] contents = [ self.test_object_data + "--v" + str(v) for v in range(0,version_max)] try: for keyname in keys: #Put version_max versions of each key for v in range(0,version_max): self.tester.info("Putting: " + keyname + " version " + str(v)) self.test_bucket.new_key(keyname).set_contents_from_string(contents[v]) except S3ResponseError as e: self.fail("Failed putting object versions for test: " + str(e.status)) listing = self.test_bucket.get_all_versions() self.tester.info("Bucket version listing is " + str(len(listing)) + " entries long") if keyrange * version_max >= 1000: if not len(listing) == 999: self.test_bucket.configure_versioning(False) self.tester.debug(str(listing)) raise Exception("Bucket version listing did not limit the response to 999. Instead: " + str(len(listing))) else: if not len(listing) == keyrange * version_max: self.test_bucket.configure_versioning(False) self.tester.debug(str(listing)) raise Exception("Bucket version listing did not equal the number uploaded. Instead: " + str(len(listing))) prev_obj = None for obj in listing: if isinstance(obj,Key): self.tester.info("Key: " + obj.name + " -- " + obj.version_id + "--" + obj.last_modified) if prev_obj != None: if self.compare_versions(prev_obj, obj) <= 0: raise Exception("Version listing not sorted correctly, offending key: " + obj.name + " version: " + obj.version_id + " date: " + obj.last_modified) prev_obj = obj else: self.tester.info("Not a key, skipping: " + str(obj)) def test_object_versioning_suspended(self): """Tests object versioning on a suspended bucket, a more complicated test than the Enabled test""" self.tester.info("Testing bucket Versioning-Suspended") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) #Create some keys keyname1 = "versionkey1-" + str(int(time.time())) keyname2 = "versionkey2-" + str(int(time.time())) keyname3 = "versionkey3-" + str(int(time.time())) keyname4 = "versionkey4-" + str(int(time.time())) keyname5 = "versionkey5-" + str(int(time.time())) v1data = self.test_object_data + "--version1" v2data = self.test_object_data + "--version2" v3data = self.test_object_data + "--version3" vstatus = self.test_bucket.get_versioning_status() if vstatus: self.fail("Versioning status should be null/Disabled but was: " + str(vstatus)) else: self.tester.info("Bucket versioning is Disabled") self.put_object(bucket=self.test_bucket, object_key=keyname1, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname2, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname3, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname4, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname5, object_data=v1data) key1 = self.test_bucket.get_key(keyname1) key2 = self.test_bucket.get_key(keyname2) key3 = self.test_bucket.get_key(keyname3) key4 = self.test_bucket.get_key(keyname4) key5 = self.test_bucket.get_key(keyname5) self.tester.info("Initial bucket state after object uploads without versioning:") self.print_key_info(keys=[key1,key2,key3,key4,key5]) #Enable versioning self.test_bucket.configure_versioning(True) if self.test_bucket.get_versioning_status(): self.tester.info("Versioning status correctly set to enabled") else: self.tester.info("Versionign status not enabled, should be.") #Update a subset of the keys key1_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname1,object_data=v2data) key2_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname2,object_data=v2data) key3_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname3,object_data=v2data) key3_etag3=self.put_object(bucket=self.test_bucket, object_key=keyname3,object_data=v3data) #Delete a key self.test_bucket.delete_key(keyname5) #Suspend versioning self.test_bucket.configure_versioning(False) #Get latest of each key key1=self.test_bucket.get_key(keyname1) key2=self.test_bucket.get_key(keyname2) key3=self.test_bucket.get_key(keyname3) key4=self.test_bucket.get_key(keyname4) key5=self.test_bucket.get_key(keyname5) #Delete a key #Add a key #Add same key again #Fetch each key def test_object_acl(self): """Tests object acl get/set and manipulation""" self.fail("Test not implemented") #TODO: test custom and canned acls that are both valid an invalid def test_object_torrent(self): """Tests object torrents""" self.fail("Feature not implemented yet") def clean_method(self): '''This is the teardown method''' #Delete the testing bucket if it is left-over self.tester.info('Deleting the buckets used for testing') for bucket in self.buckets_used: try: self.tester.info('Checking bucket ' + bucket + ' for possible cleaning/delete') if self.tester.s3.bucket_exists(bucket): self.tester.info('Found bucket exists, cleaning it') self.tester.clear_bucket(bucket) self.buckets_used.remove(bucket) else: self.tester.info('Bucket ' + bucket + ' not found, skipping') except: self.tester.info('Exception checking bucket ' + bucket) return
class ObjectTestSuite(EutesterTestCase): data_size = 1000 def __init__(self): self.setuptestcase() self.setup_parser() self.parser.add_argument("--endpoint", default=None) self.get_args() # Setup basic eutester object if self.args.endpoint: self.tester = S3ops(credpath=self.args.credpath, endpoint=self.args.endpoint) else: self.tester = Eucaops( credpath=self.args.credpath, config_file=self.args.config, password=self.args.password) self.bucket_prefix = "eutester-" + str(int(time.time())) + "-" self.buckets_used = set() random.seed(time.time()) self.test_bucket_name = self.bucket_prefix + str(random.randint(0,100)) self.test_bucket = self.tester.create_bucket(self.test_bucket_name) self.buckets_used.add(self.test_bucket_name) #Create some test data for the objects def ensure_bucket_exists(): try: self.tester.s3.get_bucket(self.test_bucket_name) return True except Exception: return False self.tester.wait_for_result(ensure_bucket_exists, True) self.test_object_data = "" for i in range(0, self.data_size): self.test_object_data += chr(random.randint(32,126)) print "Generated data for objects: " + self.test_object_data def print_key_info(self, keys=None): for key in keys: self.tester.info("Key=" + str(key.key) + " -- version= " + str(key.version_id) + " -- eTag= " + str(key.etag) + " -- ACL= " + str(key.get_xml_acl())) def put_object(self, bucket=None, object_key=None, object_data=None): """Puts an object with the specified name and data in the specified bucket""" if bucket == None: raise Exception("Cannot put object without proper bucket reference") try : key = Key(bucket=bucket,name=object_key) key.set_contents_from_string(object_data) return key.etag except Exception as e: self.tester.debug("Exception occured during 'PUT' of object " + object_key + " into bucket " + bucket.name + ": " + e.message) raise e def post_object(self, bucket_name=None, object_key=None, object_data=None, policy=None, acl=None): """Uploads an object using POST + form upload""" fields = { 'key' : object_key, 'acl' : acl, 'AWSAccessKeyId': self.tester.get_access_key(), 'Policy' : policy, 'Signature': self.sign_policy(sak=self.tester.get_secret_key(), b64_policy_json=policy) } self.tester.info('Fields: ' + str(fields)) url = 'http://' + self.tester.s3.host + ':' + str(self.tester.s3.port) \ + '/' + self.tester.s3.path + '/' + bucket_name self.tester.debug('Sending POST request to: ' + url) response = requests.post(url, data=fields, files={'file': BytesIO(object_data)}) return response #return None def generate_default_policy_b64(self, bucket, key, acl): delta = timedelta(hours=1) expire_time = (datetime.utcnow() + delta).replace(microsecond=0) policy = {'conditions': [{'acl': acl}, {'bucket': bucket}, {'key': key}, ], 'expiration': time.strftime('%Y-%m-%dT%H:%M:%SZ', expire_time.timetuple())} policy_json = json.dumps(policy) self.tester.info('generated default policy: %s', policy_json) return base64.b64encode(policy_json) def sign_policy(self, sak=None, b64_policy_json=None): my_hmac = hmac.new(sak, digestmod=hashlib.sha1) my_hmac.update(b64_policy_json) return base64.b64encode(my_hmac.digest()) def enable_versioning(self, bucket): """Enable versioning on the bucket, checking that it is not already enabled and that the operation succeeds.""" vstatus = bucket.get_versioning_status() if vstatus != None and len(vstatus.keys()) > 0 and vstatus['Versioning'] != None and vstatus['Versioning'] != 'Disabled': self.tester.info("Versioning status should be null/Disabled, found: " + vstatus['Versioning']) return False else: self.tester.info("Bucket versioning is Disabled") #Enable versioning bucket.configure_versioning(True) if bucket.get_versioning_status()['Versioning'] == 'Enabled': self.tester.info("Versioning status correctly set to enabled") return True else: self.tester.info("Versioning status not enabled, should be.") return False return False def suspend_versioning(self, bucket): """Suspend versioning on the bucket, checking that it is previously enabled and that the operation succeeds.""" if bucket.get_versioning_status()['Versioning'] == 'Enabled': self.tester.info("Versioning status correctly set to enabled") else: self.tester.info("Versioning status not enabled, should be. Can't suspend if not enabled....") return False #Enable versioning bucket.configure_versioning(False) if bucket.get_versioning_status()['Versioning'] == 'Suspended': self.tester.info("Versioning status correctly set to suspended") return True else: self.tester.info("Versioning status not suspended.") return False return False def check_version_listing(self, version_list, total_expected_length): """Checks a version listing for both completeness and ordering as well as pagination if required""" self.tester.info("Checking bucket version listing. Listing is " + str(len(version_list)) + " entries long") if total_expected_length >= 1000: assert(len(version_list) == 999) else: assert(len(version_list) == total_expected_length) prev_obj = None should_fail = None for obj in version_list: if isinstance(obj,Key): self.tester.info("Key: " + obj.name + " -- " + obj.version_id + "--" + obj.last_modified) if prev_obj != None: if self.compare_versions(prev_obj, obj) > 0: should_fail = obj prev_obj = obj else: self.tester.info("Not a key, skipping: " + str(obj)) return should_fail def compare_versions(self, key1, key2): """ Returns -1 if key1 < key2, 0 if equal, and 1 if key1 > key2. Compares names lexicographically, if equal, compares date_modified if versions are different. If version_id and name are equal then key1 = key2 If an error occurs or something is wrong, returns None """ if key1.name < key2.name: #self.debug("Key1: " + key1.name + " is less than " + key2.name) return 1 elif key1.name > key2.name: #self.debug("Key1: " + key1.name + " is greater than " + key2.name) return -1 else: if key1.version_id == key2.version_id: #self.debug("Key1: " + key1.name + " is the same version as " + key2.name) return 0 else: if dateutil.parser.parse(key1.last_modified) > dateutil.parser.parse(key2.last_modified): #self.debug("Key1: " + key1.last_modified + " last modified is greater than " + key2.last_modified) return 1 elif dateutil.parser.parse(key1.last_modified) < dateutil.parser.parse(key2.last_modified): #self.debug("Key1: " + key1.last_modified + " last modified is less than " + key2.last_modified) return -1 return None def test_object_basic_ops(self): """ Tests basic operations on objects: simple GET,PUT,HEAD,DELETE. """ self.tester.info("Basic Object Operations Test (GET/PUT/HEAD)") if self.test_bucket == None: self.fail("Error: test_bucket not set, cannot run test") #Test PUT & GET testkey="testkey1-" + str(int(time.time())) self.put_object(bucket=self.test_bucket, object_key=testkey, object_data=self.test_object_data) ret_key = self.test_bucket.get_key(testkey) ret_content = ret_key.get_contents_as_string() if ret_content == self.test_object_data: self.tester.info("Set content = get content, put passed") else: if ret_content != None: self.tester.info("Got content: " + ret_content) else: self.tester.info("No content returned") self.tester.info("Expected content: " + self.test_object_data) self.fail("Put content not the same as what was returned") #Test HEAD key_meta = self.test_bucket.get_key(testkey) if key_meta.key != ret_key.key or key_meta.etag != ret_key.etag or key_meta.size != ret_key.size: self.tester.info("Something is wrong, the HEAD operation returned different metadata than the GET operation") self.tester.info("Expected key " + ret_key.key + " etag: " + ret_key.etag + " Got: " + key_meta.key + " etag: " + key_meta.etag) else: self.tester.info("HEAD meta = GET meta, all is good") #Test copy operation (GET w/source headers) self.tester.info("Testing COPY object") new_key = "testkey2" self.test_bucket.copy_key(new_key_name=new_key, src_bucket_name=self.test_bucket_name, src_key_name=testkey) keylist = self.test_bucket.list() counter = 0 for k in keylist: if isinstance(k, Prefix): self.tester.info("Prefix: " + "NULL" if k == None else k.name) else: self.tester.info("Key: " + k.name + " Etag: " + k.etag) counter += 1 if counter != 2: self.fail("Expected 2 keys after copy operation, found only: " + len(keylist)) try: ret_key = self.test_bucket.get_key(new_key) except: self.fail("Could not get object copy") if ret_key == None: self.fail("Could not get object copy") if self.test_bucket.get_key(testkey).get_contents_as_string() != ret_key.get_contents_as_string(): self.fail("Contents of original key and copy don't match") else: self.tester.info("Copy key contents match original!") #Test DELETE self.test_bucket.delete_key(testkey) ret_key = None ret_key = self.test_bucket.get_key(testkey) if ret_key: self.tester.info("Erroneously got: " + ret_key.name) raise S3ResponseError(404, "Should have thrown exception for getting a non-existent object") self.tester.info("Finishing basic ops test") def test_object_byte_offset_read(self): """Tests fetching specific byte offsets of the object""" self.tester.info("Byte-range Offset GET Test") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) testkey = "rangetestkey-" + str(int(time.time())) source_bytes = bytearray(self.test_object_data) #Put the object initially self.put_object(bucket=self.test_bucket, object_key=testkey, object_data=self.test_object_data) #Test range for first 100 bytes of object print "Trying start-range object get" try: data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=0-99"}) except: self.fail("Failed range object get first 100 bytes") startrangedata = bytearray(data_str) print "Got: " + startrangedata print "Expected: " + str(source_bytes[:100]) start = 0 for i in range(0,100): if startrangedata[i-start] != source_bytes[i]: print "Byte: " + startrangedata[i] + " differs!" self.fail("Start-range Ranged-get failed") print "Trying mid-object range" try: data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=500-599"}) except: self.fail("Failed range object get for middle 100 bytes") midrangedata = bytearray(data_str) start = 500 for i in range(start,start+100): if midrangedata[i-start] != source_bytes[i]: print "Byte: " + midrangedata[i] + "differs!" self.fail("Mid-range Ranged-get failed") print "Trying end-range object get" #Test range for last 100 bytes of object try: data_str = Key(bucket=self.test_bucket,name=testkey).get_contents_as_string(headers={"Range":"bytes=800-899"}) except: self.fail("Failed range object get for last 100 bytes") endrangedata = bytearray(data_str) print "Got: " + str(endrangedata) start = 800 try: for i in range(start,start+100): if endrangedata[i-start] != source_bytes[i]: print "Byte: " + endrangedata[i] + "differs!" self.fail("End-range Ranged-get failed") except Exception as e: print "Exception! Received: " + e print "Range test complete" def test_object_post(self): """Test the POST method for putting objects, requires a pre-signed upload policy and url""" self.tester.info("Testing POST form upload on bucket" + self.test_bucket_name) self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) itr = 1 self.tester.info('Doing ' + str(itr) + ' POST upload iterations') acl = 'ec2-bundle-read' for k in xrange(0, itr): key = 'postkey1' + str(k) data = os.urandom(512) computed_md5 = '"' + hashlib.md5(data).hexdigest() + '"' self.tester.info('Data md5: ' + computed_md5 + ' data length: ' + str(len(computed_md5))) self.tester.info('Uploading object ' + self.test_bucket_name + '/' + key + ' via POST with acl : ' + acl) response = self.post_object(bucket_name=self.test_bucket_name, object_key=key, object_data=data, acl=acl, policy=self.generate_default_policy_b64(self.test_bucket_name, key, acl=acl)) self.tester.info('Got response for POST: ' + str(response.status_code) + ': ' + str(response.text)) assert(response.status_code == 204) fetched_key = self.test_bucket.get_key(key) fetched_content = fetched_key.get_contents_as_string() self.tester.info('Got fetched md5: ' + fetched_key.etag) self.tester.info('Calculated md5: ' + computed_md5 + ' recieved md5 ' + fetched_key.etag) assert(fetched_key.etag == computed_md5) assert(fetched_content == data) self.tester.info("Done with upload test") def test_object_post_large(self): """Test the POST method for putting objects, requires a pre-signed upload policy and url""" self.tester.info("Testing POST form upload on bucket" + self.test_bucket_name) self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) self.tester.info("Testing POST form upload of 10MB data on bucket" + self.test_bucket_name) itr = 1 large_obj_size_bytes = 10 * 1024 * 1024 #10MB content self.tester.info('Doing ' + str(itr) + ' iterations of large object of size ' + str(large_obj_size_bytes) + ' with POST') acl = 'ec2-bundle-read' for i in xrange(0, itr): key = 'postkey_10mb_' + str(i) self.tester.info('Generating ' + str(large_obj_size_bytes) + ' bytes for upload') #Create some test data data = str(os.urandom(large_obj_size_bytes)) self.tester.info("Data length: " + str(len(data))) computed_md5 = '"' + hashlib.md5(data).hexdigest() + '"' self.tester.info('Data md5 is: ' + computed_md5) self.tester.info('Uploading object via POST using acl: ' + acl) response = self.post_object(bucket_name=self.test_bucket.name, object_key=key, object_data=data, policy=self.generate_default_policy_b64(self.test_bucket.name, key, acl=acl), acl=acl) self.tester.info('Got response for POST: ' + str(response.status_code) + ': ' + str(response.text)) assert(response.status_code == 204) self.tester.info('Fetching the content for verification') fetched_key = self.test_bucket.get_key(key_name=key) self.tester.info('Got fetched content length : ' + str(fetched_key.size) + ' Expected ' + str(len(data))) assert(fetched_key.size == len(data)) self.tester.info('Got fetched md5: ' + fetched_key.etag) self.tester.info('Calculated md5: ' + computed_md5 + ' recieved md5 ' + fetched_key.etag) assert(fetched_key.etag == computed_md5) fetched_content = fetched_key.get_contents_as_string() assert(fetched_content == data) def test_object_large_objects(self): """Test operations on large objects (>1MB), but not so large that we must use the multi-part upload interface""" self.tester.info("Testing large-ish objects over 1MB in size on bucket" + self.test_bucket_name) self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) test_data = None large_obj_size_bytes = 5 * 1024 * 1024 #5MB self.tester.info("Generating " + str(large_obj_size_bytes) + " bytes of data") #Create some test data #for i in range(0, large_obj_size_bytes): # test_data += chr(random.randint(32,126)) test_data = bytearray(os.urandom(large_obj_size_bytes)) self.tester.info("Uploading object content of size: " + str(large_obj_size_bytes) + " bytes") keyname = "largeobj-" + str(int(time.time())) self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=test_data) self.tester.info("Done uploading object") ret_key = self.test_bucket.get_key(keyname) ret_data = ret_key.get_contents_as_string() if ret_data != test_data: self.fail("Fetched data and generated data don't match") else: self.tester.info("Data matches!") self.tester.info("Removing large object") self.test_bucket.delete_key(ret_key) self.tester.info("Complete large object test") pass def test_object_multipart(self): """Tests basic multipart upload functionality""" self.tester.info("Testing Multipart") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) self.test_multipart_upload() self.test_abort_multipart_upload() def test_multipart_upload(self): '''Basic multipart upload''' self.tester.info("Testing multipart upload") self.tester.info("Creating random file representing part...") temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart") temp_file.write(os.urandom(5 * 1024 * 1024)) keyname="multi-" + str(int(time.time())) self.tester.info("Initiating multipart upload...much upload") reply = self.initiate_multipart_upload(keyname) self.tester.info("Uploading parts...Such Parts") for partnum in range(1, 11): temp_file.seek(0, os.SEEK_SET) reply.upload_part_from_file(temp_file, partnum) self.tester.info("Listing parts...") self.test_bucket.get_all_multipart_uploads() self.tester.info("Completing upload...So OSG") reply.complete_upload() temp_file.close() self.tester.info("HEAD request..."); returned_key = self.test_bucket.get_key(keyname, validate=True); download_temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="mpu-download") self.tester.info("Downloading object...very mpu"); returned_key.get_contents_to_file(download_temp_file); self.tester.info("Deleting object...WOW") self.test_bucket.delete_key(keyname) def test_abort_multipart_upload(self): '''Basic multipart upload''' self.tester.info("Testing abort multipart upload") temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart") temp_file.write(os.urandom(5 * 1024 * 1024)) keyname="multi-" + str(int(time.time())) reply = self.initiate_multipart_upload(keyname) for partnum in range(1, 11): temp_file.seek(0, os.SEEK_SET) reply.upload_part_from_file(temp_file, partnum) self.test_bucket.get_all_multipart_uploads() self.tester.info("Canceling upload") reply.cancel_upload() temp_file.close() def initiate_multipart_upload(self, keyname): self.tester.info("Initiating multipart upload " + keyname) return self.test_bucket.initiate_multipart_upload(keyname) def test_object_versioning_enabled(self): """Tests object versioning for get/put/delete on a versioned bucket""" self.tester.info("Testing bucket Versioning-Enabled") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) if not self.enable_versioning(self.test_bucket): self.fail("Could not properly enable versioning") #Create some keys keyname = "versionkey-" + str(int(time.time())) #Multiple versions of the data v1data = self.test_object_data + "--version1" v2data = self.test_object_data + "--version2" v3data = self.test_object_data + "--version3" #Test sequence: put v1, get v1, put v2, put v3, get v3, delete v3, restore with v1 (copy), put v3 again, delete v2 explicitly self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=v1data) #Get v1 obj_v1 = self.test_bucket.get_key(keyname) self.tester.check_md5(eTag=obj_v1.etag,data=v1data) self.tester.info("Initial bucket state after object uploads with versioning enabled:") self.print_key_info(keys=[obj_v1]) self.tester.info("Adding another version") #Put v2 (and get/head to confirm success) self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v2data) obj_v2 = self.test_bucket.get_key(keyname) self.tester.check_md5(eTag=obj_v2.etag,data=v2data) self.print_key_info(keys=[obj_v1, obj_v2]) self.tester.info("Adding another version") #Put v3 (and get/head to confirm success) self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v3data) obj_v3 = self.test_bucket.get_key(keyname) self.tester.check_md5(eTag=obj_v3.etag,data=v3data) self.print_key_info(keys=[obj_v1, obj_v2, obj_v3]) self.tester.info("Getting specific version") #Get a specific version, v1 v1_return = self.test_bucket.get_key(key_name=keyname,version_id=obj_v1.version_id) self.print_key_info(keys=[v1_return]) #Delete current latest version (v3) self.test_bucket.delete_key(keyname) del_obj = self.test_bucket.get_key(keyname) if del_obj: self.tester.info("Erroneously got: " + del_obj.name) raise S3ResponseError(404, "Should have thrown this exception for getting a non-existent object") #Restore v1 using copy self.tester.info("Restoring version") try: self.test_bucket.copy_key(new_key_name=obj_v1.key,src_bucket_name=self.test_bucket_name,src_key_name=keyname,src_version_id=obj_v1.version_id) except S3ResponseError as e: self.fail("Failed to restore key from previous version using copy got error: " + str(e.status)) restored_obj = self.test_bucket.get_key(keyname) assert(restored_obj != None) self.tester.check_md5(eTag=restored_obj.etag,data=v1data) self.print_key_info(keys=[restored_obj]) #Put v3 again self.tester.info("Adding another version") self.put_object(bucket=self.test_bucket, object_key=keyname,object_data=v3data) self.tester.check_md5(eTag=obj_v3.etag,data=v3data) self.print_key_info([self.test_bucket.get_key(keyname)]) #Delete v2 explicitly self.test_bucket.delete_key(key_name=obj_v2.key,version_id=obj_v2.version_id) del_obj = self.test_bucket.get_key(keyname,version_id=obj_v2.version_id) if del_obj: raise S3ResponseError("Should have gotten 404 not-found error, but got: " + del_obj.key + " instead",404) #Show what's on top top_obj = self.test_bucket.get_key(keyname) self.print_key_info([top_obj]) self.tester.check_md5(eTag=top_obj.etag,data=v3data) self.tester.info("Finished the versioning enabled test. Success!!") def clear_and_rebuild_bucket(self, bucket_name): self.tester.clear_bucket(bucket_name) return self.tester.create_bucket(bucket_name) def test_object_versionlisting(self): """ Tests object version listing from a bucket """ version_max = 3 keyrange = 20 self.tester.info("Testing listing versions in a bucket and pagination using " + str(keyrange) + " keys with " + str(version_max) + " versions per key") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) if not self.enable_versioning(self.test_bucket): self.fail("Could not enable versioning properly. Failing") key = "testkey-" + str(int(time.time())) keys = [ key + str(k) for k in range(0,keyrange)] contents = [ self.test_object_data + "--v" + str(v) for v in range(0,version_max)] try: for keyname in keys: #Put version_max versions of each key for v in range(0,version_max): self.tester.info("Putting: " + keyname + " version " + str(v)) self.test_bucket.new_key(keyname).set_contents_from_string(contents[v]) except S3ResponseError as e: self.fail("Failed putting object versions for test: " + str(e.status)) listing = self.test_bucket.get_all_versions() self.tester.info("Bucket version listing is " + str(len(listing)) + " entries long") if keyrange * version_max >= 1000: if not len(listing) == 999: self.test_bucket.configure_versioning(False) self.tester.debug(str(listing)) raise Exception("Bucket version listing did not limit the response to 999. Instead: " + str(len(listing))) else: if not len(listing) == keyrange * version_max: self.test_bucket.configure_versioning(False) self.tester.debug(str(listing)) raise Exception("Bucket version listing did not equal the number uploaded. Instead: " + str(len(listing))) prev_obj = None for obj in listing: if isinstance(obj,Key): self.tester.info("Key: " + obj.name + " -- " + obj.version_id + "--" + obj.last_modified) if prev_obj != None: if self.compare_versions(prev_obj, obj) <= 0: raise Exception("Version listing not sorted correctly, offending key: " + obj.name + " version: " + obj.version_id + " date: " + obj.last_modified) prev_obj = obj else: self.tester.info("Not a key, skipping: " + str(obj)) def test_object_versioning_suspended(self): """Tests object versioning on a suspended bucket, a more complicated test than the Enabled test""" self.tester.info("Testing bucket Versioning-Suspended") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) #Create some keys keyname1 = "versionkey1-" + str(int(time.time())) keyname2 = "versionkey2-" + str(int(time.time())) keyname3 = "versionkey3-" + str(int(time.time())) keyname4 = "versionkey4-" + str(int(time.time())) keyname5 = "versionkey5-" + str(int(time.time())) v1data = self.test_object_data + "--version1" v2data = self.test_object_data + "--version2" v3data = self.test_object_data + "--version3" vstatus = self.test_bucket.get_versioning_status() if vstatus: self.fail("Versioning status should be null/Disabled but was: " + str(vstatus)) else: self.tester.info("Bucket versioning is Disabled") self.put_object(bucket=self.test_bucket, object_key=keyname1, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname2, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname3, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname4, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname5, object_data=v1data) key1 = self.test_bucket.get_key(keyname1) key2 = self.test_bucket.get_key(keyname2) key3 = self.test_bucket.get_key(keyname3) key4 = self.test_bucket.get_key(keyname4) key5 = self.test_bucket.get_key(keyname5) self.tester.info("Initial bucket state after object uploads without versioning:") self.print_key_info(keys=[key1,key2,key3,key4,key5]) #Enable versioning self.test_bucket.configure_versioning(True) if self.test_bucket.get_versioning_status(): self.tester.info("Versioning status correctly set to enabled") else: self.tester.info("Versionign status not enabled, should be.") #Update a subset of the keys key1_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname1,object_data=v2data) key2_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname2,object_data=v2data) key3_etag2=self.put_object(bucket=self.test_bucket, object_key=keyname3,object_data=v2data) key3_etag3=self.put_object(bucket=self.test_bucket, object_key=keyname3,object_data=v3data) #Delete a key self.test_bucket.delete_key(keyname5) #Suspend versioning self.test_bucket.configure_versioning(False) #Get latest of each key key1=self.test_bucket.get_key(keyname1) key2=self.test_bucket.get_key(keyname2) key3=self.test_bucket.get_key(keyname3) key4=self.test_bucket.get_key(keyname4) key5=self.test_bucket.get_key(keyname5) #Delete a key #Add a key #Add same key again #Fetch each key def test_object_acl(self): """Tests object acl get/set and manipulation""" self.fail("Test not implemented") #TODO: test custom and canned acls that are both valid an invalid def test_object_torrent(self): """Tests object torrents""" self.fail("Feature not implemented yet") def clean_method(self): '''This is the teardown method''' #Delete the testing bucket if it is left-over self.tester.info('Deleting the buckets used for testing') for bucket in self.buckets_used: try: self.tester.info('Checking bucket ' + bucket + ' for possible cleaning/delete') if self.tester.s3.head_bucket(bucket) != None: self.tester.info('Found bucket exists, cleaning it') self.tester.clear_bucket(bucket) else: self.tester.info('Bucket ' + bucket + ' not found, skipping') except Exception as e: self.tester.info('Exception checking bucket ' + bucket + ' Exception msg: ' + e.message) return def test_multipart_upload(self): '''Basic multipart upload''' self.tester.info("Testing multipart upload") self.tester.info("Creating random file representing part...") temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart") temp_file.write(os.urandom(5 * 1024 * 1024)) keyname="multi-" + str(int(time.time())) self.tester.info("Initiating multipart upload...much upload") reply = self.initiate_multipart_upload(keyname) self.tester.info("Uploading parts...Such Parts") for partnum in range(1, 11): temp_file.seek(0, os.SEEK_SET) reply.upload_part_from_file(temp_file, partnum) self.tester.info("Listing parts...") self.test_bucket.get_all_multipart_uploads() self.tester.info("Completing upload...So OSG") reply.complete_upload() temp_file.close() self.tester.info("HEAD request..."); returned_key = self.test_bucket.get_key(keyname, validate=True); download_temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="mpu-download") self.tester.info("Downloading object...very mpu"); returned_key.get_contents_to_file(download_temp_file); self.tester.info("Deleting object...WOW") self.test_bucket.delete_key(keyname) def test_abort_multipart_upload(self): '''Basic multipart upload''' self.tester.info("Testing abort multipart upload") temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart") temp_file.write(os.urandom(5 * 1024 * 1024)) keyname="multi-" + str(int(time.time())) reply = self.initiate_multipart_upload(keyname) for partnum in range(1, 11): temp_file.seek(0, os.SEEK_SET) reply.upload_part_from_file(temp_file, partnum) self.test_bucket.get_all_multipart_uploads() self.tester.info("!!!!!!!!!!!!!!!!!!!!!!!!!!DO NOT WANT!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1") self.tester.info("Canceling upload") reply.cancel_upload() temp_file.close() def initiate_multipart_upload(self, keyname): self.tester.info("Initiating multipart upload " + keyname) return self.test_bucket.initiate_multipart_upload(keyname)
class ObjectTestSuite(EutesterTestCase): data_size = 1000 def __init__(self): self.setuptestcase() self.setup_parser() self.parser.add_argument("--endpoint", default=None) self.get_args() # Setup basic eutester object if self.args.endpoint: self.tester = S3ops(credpath=self.args.credpath, endpoint=self.args.endpoint) else: self.tester = Eucaops(credpath=self.args.credpath, config_file=self.args.config, password=self.args.password) self.bucket_prefix = "eutester-" + str(int(time.time())) + "-" self.buckets_used = set() random.seed(time.time()) self.test_bucket_name = self.bucket_prefix + str(random.randint( 0, 100)) self.test_bucket = self.tester.create_bucket(self.test_bucket_name) self.buckets_used.add(self.test_bucket_name) #Create some test data for the objects def ensure_bucket_exists(): try: self.tester.s3.get_bucket(self.test_bucket_name) return True except Exception: return False self.tester.wait_for_result(ensure_bucket_exists, True) self.test_object_data = "" for i in range(0, self.data_size): self.test_object_data += chr(random.randint(32, 126)) print "Generated data for objects: " + self.test_object_data def print_key_info(self, keys=None): for key in keys: self.tester.info("Key=" + str(key.key) + " -- version= " + str(key.version_id) + " -- eTag= " + str(key.etag) + " -- ACL= " + str(key.get_xml_acl())) def put_object(self, bucket=None, object_key=None, object_data=None): """Puts an object with the specified name and data in the specified bucket""" if bucket == None: raise Exception( "Cannot put object without proper bucket reference") try: key = Key(bucket=bucket, name=object_key) key.set_contents_from_string(object_data) return key.etag except Exception as e: self.tester.debug("Exception occured during 'PUT' of object " + object_key + " into bucket " + bucket.name + ": " + e.message) raise e def post_object(self, bucket_name=None, object_key=None, object_data=None, policy=None, acl=None): """Uploads an object using POST + form upload""" fields = { 'key': object_key, 'acl': acl, 'AWSAccessKeyId': self.tester.get_access_key(), 'Policy': policy, 'Signature': self.sign_policy(sak=self.tester.get_secret_key(), b64_policy_json=policy) } self.tester.info('Fields: ' + str(fields)) url = 'http://' + self.tester.s3.host + ':' + str(self.tester.s3.port) \ + '/' + self.tester.s3.path + '/' + bucket_name self.tester.debug('Sending POST request to: ' + url) response = requests.post(url, data=fields, files={'file': BytesIO(object_data)}) return response #return None def generate_default_policy_b64(self, bucket, key, acl): delta = timedelta(hours=1) expire_time = (datetime.utcnow() + delta).replace(microsecond=0) policy = { 'conditions': [ { 'acl': acl }, { 'bucket': bucket }, { 'key': key }, ], 'expiration': time.strftime('%Y-%m-%dT%H:%M:%SZ', expire_time.timetuple()) } policy_json = json.dumps(policy) self.tester.info('generated default policy: %s', policy_json) return base64.b64encode(policy_json) def sign_policy(self, sak=None, b64_policy_json=None): my_hmac = hmac.new(sak, digestmod=hashlib.sha1) my_hmac.update(b64_policy_json) return base64.b64encode(my_hmac.digest()) def enable_versioning(self, bucket): """Enable versioning on the bucket, checking that it is not already enabled and that the operation succeeds.""" vstatus = bucket.get_versioning_status() if vstatus != None and len(vstatus.keys()) > 0 and vstatus[ 'Versioning'] != None and vstatus['Versioning'] != 'Disabled': self.tester.info( "Versioning status should be null/Disabled, found: " + vstatus['Versioning']) return False else: self.tester.info("Bucket versioning is Disabled") #Enable versioning bucket.configure_versioning(True) if bucket.get_versioning_status()['Versioning'] == 'Enabled': self.tester.info("Versioning status correctly set to enabled") return True else: self.tester.info("Versioning status not enabled, should be.") return False return False def suspend_versioning(self, bucket): """Suspend versioning on the bucket, checking that it is previously enabled and that the operation succeeds.""" if bucket.get_versioning_status()['Versioning'] == 'Enabled': self.tester.info("Versioning status correctly set to enabled") else: self.tester.info( "Versioning status not enabled, should be. Can't suspend if not enabled...." ) return False #Enable versioning bucket.configure_versioning(False) if bucket.get_versioning_status()['Versioning'] == 'Suspended': self.tester.info("Versioning status correctly set to suspended") return True else: self.tester.info("Versioning status not suspended.") return False return False def check_version_listing(self, version_list, total_expected_length): """Checks a version listing for both completeness and ordering as well as pagination if required""" self.tester.info("Checking bucket version listing. Listing is " + str(len(version_list)) + " entries long") if total_expected_length >= 1000: assert (len(version_list) == 999) else: assert (len(version_list) == total_expected_length) prev_obj = None should_fail = None for obj in version_list: if isinstance(obj, Key): self.tester.info("Key: " + obj.name + " -- " + obj.version_id + "--" + obj.last_modified) if prev_obj != None: if self.compare_versions(prev_obj, obj) > 0: should_fail = obj prev_obj = obj else: self.tester.info("Not a key, skipping: " + str(obj)) return should_fail def compare_versions(self, key1, key2): """ Returns -1 if key1 < key2, 0 if equal, and 1 if key1 > key2. Compares names lexicographically, if equal, compares date_modified if versions are different. If version_id and name are equal then key1 = key2 If an error occurs or something is wrong, returns None """ if key1.name < key2.name: #self.debug("Key1: " + key1.name + " is less than " + key2.name) return 1 elif key1.name > key2.name: #self.debug("Key1: " + key1.name + " is greater than " + key2.name) return -1 else: if key1.version_id == key2.version_id: #self.debug("Key1: " + key1.name + " is the same version as " + key2.name) return 0 else: if dateutil.parser.parse( key1.last_modified) > dateutil.parser.parse( key2.last_modified): #self.debug("Key1: " + key1.last_modified + " last modified is greater than " + key2.last_modified) return 1 elif dateutil.parser.parse( key1.last_modified) < dateutil.parser.parse( key2.last_modified): #self.debug("Key1: " + key1.last_modified + " last modified is less than " + key2.last_modified) return -1 return None def test_object_basic_ops(self): """ Tests basic operations on objects: simple GET,PUT,HEAD,DELETE. """ self.tester.info("Basic Object Operations Test (GET/PUT/HEAD)") if self.test_bucket == None: self.fail("Error: test_bucket not set, cannot run test") #Test PUT & GET testkey = "testkey1-" + str(int(time.time())) self.put_object(bucket=self.test_bucket, object_key=testkey, object_data=self.test_object_data) ret_key = self.test_bucket.get_key(testkey) ret_content = ret_key.get_contents_as_string() if ret_content == self.test_object_data: self.tester.info("Set content = get content, put passed") else: if ret_content != None: self.tester.info("Got content: " + ret_content) else: self.tester.info("No content returned") self.tester.info("Expected content: " + self.test_object_data) self.fail("Put content not the same as what was returned") #Test HEAD key_meta = self.test_bucket.get_key(testkey) if key_meta.key != ret_key.key or key_meta.etag != ret_key.etag or key_meta.size != ret_key.size: self.tester.info( "Something is wrong, the HEAD operation returned different metadata than the GET operation" ) self.tester.info("Expected key " + ret_key.key + " etag: " + ret_key.etag + " Got: " + key_meta.key + " etag: " + key_meta.etag) else: self.tester.info("HEAD meta = GET meta, all is good") #Test copy operation (GET w/source headers) self.tester.info("Testing COPY object") new_key = "testkey2" self.test_bucket.copy_key(new_key_name=new_key, src_bucket_name=self.test_bucket_name, src_key_name=testkey) keylist = self.test_bucket.list() counter = 0 for k in keylist: if isinstance(k, Prefix): self.tester.info("Prefix: " + "NULL" if k == None else k.name) else: self.tester.info("Key: " + k.name + " Etag: " + k.etag) counter += 1 if counter != 2: self.fail("Expected 2 keys after copy operation, found only: " + len(keylist)) try: ret_key = self.test_bucket.get_key(new_key) except: self.fail("Could not get object copy") if ret_key == None: self.fail("Could not get object copy") if self.test_bucket.get_key(testkey).get_contents_as_string( ) != ret_key.get_contents_as_string(): self.fail("Contents of original key and copy don't match") else: self.tester.info("Copy key contents match original!") #Test DELETE self.test_bucket.delete_key(testkey) ret_key = None ret_key = self.test_bucket.get_key(testkey) if ret_key: self.tester.info("Erroneously got: " + ret_key.name) raise S3ResponseError( 404, "Should have thrown exception for getting a non-existent object" ) self.tester.info("Finishing basic ops test") def test_object_byte_offset_read(self): """Tests fetching specific byte offsets of the object""" self.tester.info("Byte-range Offset GET Test") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) testkey = "rangetestkey-" + str(int(time.time())) source_bytes = bytearray(self.test_object_data) #Put the object initially self.put_object(bucket=self.test_bucket, object_key=testkey, object_data=self.test_object_data) #Test range for first 100 bytes of object print "Trying start-range object get" try: data_str = Key(bucket=self.test_bucket, name=testkey).get_contents_as_string( headers={"Range": "bytes=0-99"}) except: self.fail("Failed range object get first 100 bytes") startrangedata = bytearray(data_str) print "Got: " + startrangedata print "Expected: " + str(source_bytes[:100]) start = 0 for i in range(0, 100): if startrangedata[i - start] != source_bytes[i]: print "Byte: " + startrangedata[i] + " differs!" self.fail("Start-range Ranged-get failed") print "Trying mid-object range" try: data_str = Key(bucket=self.test_bucket, name=testkey).get_contents_as_string( headers={"Range": "bytes=500-599"}) except: self.fail("Failed range object get for middle 100 bytes") midrangedata = bytearray(data_str) start = 500 for i in range(start, start + 100): if midrangedata[i - start] != source_bytes[i]: print "Byte: " + midrangedata[i] + "differs!" self.fail("Mid-range Ranged-get failed") print "Trying end-range object get" #Test range for last 100 bytes of object try: data_str = Key(bucket=self.test_bucket, name=testkey).get_contents_as_string( headers={"Range": "bytes=800-899"}) except: self.fail("Failed range object get for last 100 bytes") endrangedata = bytearray(data_str) print "Got: " + str(endrangedata) start = 800 try: for i in range(start, start + 100): if endrangedata[i - start] != source_bytes[i]: print "Byte: " + endrangedata[i] + "differs!" self.fail("End-range Ranged-get failed") except Exception as e: print "Exception! Received: " + e print "Range test complete" def test_object_post(self): """Test the POST method for putting objects, requires a pre-signed upload policy and url""" self.tester.info("Testing POST form upload on bucket" + self.test_bucket_name) self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) itr = 1 self.tester.info('Doing ' + str(itr) + ' POST upload iterations') acl = 'ec2-bundle-read' for k in xrange(0, itr): key = 'postkey1' + str(k) data = os.urandom(512) computed_md5 = '"' + hashlib.md5(data).hexdigest() + '"' self.tester.info('Data md5: ' + computed_md5 + ' data length: ' + str(len(computed_md5))) self.tester.info('Uploading object ' + self.test_bucket_name + '/' + key + ' via POST with acl : ' + acl) response = self.post_object( bucket_name=self.test_bucket_name, object_key=key, object_data=data, acl=acl, policy=self.generate_default_policy_b64(self.test_bucket_name, key, acl=acl)) self.tester.info('Got response for POST: ' + str(response.status_code) + ': ' + str(response.text)) assert (response.status_code == 204) fetched_key = self.test_bucket.get_key(key) fetched_content = fetched_key.get_contents_as_string() self.tester.info('Got fetched md5: ' + fetched_key.etag) self.tester.info('Calculated md5: ' + computed_md5 + ' recieved md5 ' + fetched_key.etag) assert (fetched_key.etag == computed_md5) assert (fetched_content == data) self.tester.info("Done with upload test") def test_object_post_large(self): """Test the POST method for putting objects, requires a pre-signed upload policy and url""" self.tester.info("Testing POST form upload on bucket" + self.test_bucket_name) self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) self.tester.info("Testing POST form upload of 10MB data on bucket" + self.test_bucket_name) itr = 1 large_obj_size_bytes = 10 * 1024 * 1024 #10MB content self.tester.info('Doing ' + str(itr) + ' iterations of large object of size ' + str(large_obj_size_bytes) + ' with POST') acl = 'ec2-bundle-read' for i in xrange(0, itr): key = 'postkey_10mb_' + str(i) self.tester.info('Generating ' + str(large_obj_size_bytes) + ' bytes for upload') #Create some test data data = str(os.urandom(large_obj_size_bytes)) self.tester.info("Data length: " + str(len(data))) computed_md5 = '"' + hashlib.md5(data).hexdigest() + '"' self.tester.info('Data md5 is: ' + computed_md5) self.tester.info('Uploading object via POST using acl: ' + acl) response = self.post_object( bucket_name=self.test_bucket.name, object_key=key, object_data=data, policy=self.generate_default_policy_b64(self.test_bucket.name, key, acl=acl), acl=acl) self.tester.info('Got response for POST: ' + str(response.status_code) + ': ' + str(response.text)) assert (response.status_code == 204) self.tester.info('Fetching the content for verification') fetched_key = self.test_bucket.get_key(key_name=key) self.tester.info('Got fetched content length : ' + str(fetched_key.size) + ' Expected ' + str(len(data))) assert (fetched_key.size == len(data)) self.tester.info('Got fetched md5: ' + fetched_key.etag) self.tester.info('Calculated md5: ' + computed_md5 + ' recieved md5 ' + fetched_key.etag) assert (fetched_key.etag == computed_md5) fetched_content = fetched_key.get_contents_as_string() assert (fetched_content == data) def test_object_large_objects(self): """Test operations on large objects (>1MB), but not so large that we must use the multi-part upload interface""" self.tester.info( "Testing large-ish objects over 1MB in size on bucket" + self.test_bucket_name) self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) test_data = None large_obj_size_bytes = 5 * 1024 * 1024 #5MB self.tester.info("Generating " + str(large_obj_size_bytes) + " bytes of data") #Create some test data #for i in range(0, large_obj_size_bytes): # test_data += chr(random.randint(32,126)) test_data = bytearray(os.urandom(large_obj_size_bytes)) self.tester.info("Uploading object content of size: " + str(large_obj_size_bytes) + " bytes") keyname = "largeobj-" + str(int(time.time())) self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=test_data) self.tester.info("Done uploading object") ret_key = self.test_bucket.get_key(keyname) ret_data = ret_key.get_contents_as_string() if ret_data != test_data: self.fail("Fetched data and generated data don't match") else: self.tester.info("Data matches!") self.tester.info("Removing large object") self.test_bucket.delete_key(ret_key) self.tester.info("Complete large object test") pass def test_object_multipart(self): """Tests basic multipart upload functionality""" self.tester.info("Testing Multipart") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) self.test_multipart_upload() self.test_abort_multipart_upload() def test_multipart_upload(self): '''Basic multipart upload''' self.tester.info("Testing multipart upload") self.tester.info("Creating random file representing part...") temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart") temp_file.write(os.urandom(5 * 1024 * 1024)) keyname = "multi-" + str(int(time.time())) self.tester.info("Initiating multipart upload...much upload") reply = self.initiate_multipart_upload(keyname) self.tester.info("Uploading parts...Such Parts") for partnum in range(1, 11): temp_file.seek(0, os.SEEK_SET) reply.upload_part_from_file(temp_file, partnum) self.tester.info("Listing parts...") self.test_bucket.get_all_multipart_uploads() self.tester.info("Completing upload...So OSG") reply.complete_upload() temp_file.close() self.tester.info("HEAD request...") returned_key = self.test_bucket.get_key(keyname, validate=True) download_temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="mpu-download") self.tester.info("Downloading object...very mpu") returned_key.get_contents_to_file(download_temp_file) self.tester.info("Deleting object...WOW") self.test_bucket.delete_key(keyname) def test_abort_multipart_upload(self): '''Basic multipart upload''' self.tester.info("Testing abort multipart upload") temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart") temp_file.write(os.urandom(5 * 1024 * 1024)) keyname = "multi-" + str(int(time.time())) reply = self.initiate_multipart_upload(keyname) for partnum in range(1, 11): temp_file.seek(0, os.SEEK_SET) reply.upload_part_from_file(temp_file, partnum) self.test_bucket.get_all_multipart_uploads() self.tester.info("Canceling upload") reply.cancel_upload() temp_file.close() def initiate_multipart_upload(self, keyname): self.tester.info("Initiating multipart upload " + keyname) return self.test_bucket.initiate_multipart_upload(keyname) def test_object_versioning_enabled(self): """Tests object versioning for get/put/delete on a versioned bucket""" self.tester.info("Testing bucket Versioning-Enabled") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) if not self.enable_versioning(self.test_bucket): self.fail("Could not properly enable versioning") #Create some keys keyname = "versionkey-" + str(int(time.time())) #Multiple versions of the data v1data = self.test_object_data + "--version1" v2data = self.test_object_data + "--version2" v3data = self.test_object_data + "--version3" #Test sequence: put v1, get v1, put v2, put v3, get v3, delete v3, restore with v1 (copy), put v3 again, delete v2 explicitly self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=v1data) #Get v1 obj_v1 = self.test_bucket.get_key(keyname) self.tester.check_md5(eTag=obj_v1.etag, data=v1data) self.tester.info( "Initial bucket state after object uploads with versioning enabled:" ) self.print_key_info(keys=[obj_v1]) self.tester.info("Adding another version") #Put v2 (and get/head to confirm success) self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=v2data) obj_v2 = self.test_bucket.get_key(keyname) self.tester.check_md5(eTag=obj_v2.etag, data=v2data) self.print_key_info(keys=[obj_v1, obj_v2]) self.tester.info("Adding another version") #Put v3 (and get/head to confirm success) self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=v3data) obj_v3 = self.test_bucket.get_key(keyname) self.tester.check_md5(eTag=obj_v3.etag, data=v3data) self.print_key_info(keys=[obj_v1, obj_v2, obj_v3]) self.tester.info("Getting specific version") #Get a specific version, v1 v1_return = self.test_bucket.get_key(key_name=keyname, version_id=obj_v1.version_id) self.print_key_info(keys=[v1_return]) #Delete current latest version (v3) self.test_bucket.delete_key(keyname) del_obj = self.test_bucket.get_key(keyname) if del_obj: self.tester.info("Erroneously got: " + del_obj.name) raise S3ResponseError( 404, "Should have thrown this exception for getting a non-existent object" ) #Restore v1 using copy self.tester.info("Restoring version") try: self.test_bucket.copy_key(new_key_name=obj_v1.key, src_bucket_name=self.test_bucket_name, src_key_name=keyname, src_version_id=obj_v1.version_id) except S3ResponseError as e: self.fail( "Failed to restore key from previous version using copy got error: " + str(e.status)) restored_obj = self.test_bucket.get_key(keyname) assert (restored_obj != None) self.tester.check_md5(eTag=restored_obj.etag, data=v1data) self.print_key_info(keys=[restored_obj]) #Put v3 again self.tester.info("Adding another version") self.put_object(bucket=self.test_bucket, object_key=keyname, object_data=v3data) self.tester.check_md5(eTag=obj_v3.etag, data=v3data) self.print_key_info([self.test_bucket.get_key(keyname)]) #Delete v2 explicitly self.test_bucket.delete_key(key_name=obj_v2.key, version_id=obj_v2.version_id) del_obj = self.test_bucket.get_key(keyname, version_id=obj_v2.version_id) if del_obj: raise S3ResponseError( "Should have gotten 404 not-found error, but got: " + del_obj.key + " instead", 404) #Show what's on top top_obj = self.test_bucket.get_key(keyname) self.print_key_info([top_obj]) self.tester.check_md5(eTag=top_obj.etag, data=v3data) self.tester.info("Finished the versioning enabled test. Success!!") def clear_and_rebuild_bucket(self, bucket_name): self.tester.clear_bucket(bucket_name) return self.tester.create_bucket(bucket_name) def test_object_versionlisting(self): """ Tests object version listing from a bucket """ version_max = 3 keyrange = 20 self.tester.info( "Testing listing versions in a bucket and pagination using " + str(keyrange) + " keys with " + str(version_max) + " versions per key") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) if not self.enable_versioning(self.test_bucket): self.fail("Could not enable versioning properly. Failing") key = "testkey-" + str(int(time.time())) keys = [key + str(k) for k in range(0, keyrange)] contents = [ self.test_object_data + "--v" + str(v) for v in range(0, version_max) ] try: for keyname in keys: #Put version_max versions of each key for v in range(0, version_max): self.tester.info("Putting: " + keyname + " version " + str(v)) self.test_bucket.new_key(keyname).set_contents_from_string( contents[v]) except S3ResponseError as e: self.fail("Failed putting object versions for test: " + str(e.status)) listing = self.test_bucket.get_all_versions() self.tester.info("Bucket version listing is " + str(len(listing)) + " entries long") if keyrange * version_max >= 1000: if not len(listing) == 999: self.test_bucket.configure_versioning(False) self.tester.debug(str(listing)) raise Exception( "Bucket version listing did not limit the response to 999. Instead: " + str(len(listing))) else: if not len(listing) == keyrange * version_max: self.test_bucket.configure_versioning(False) self.tester.debug(str(listing)) raise Exception( "Bucket version listing did not equal the number uploaded. Instead: " + str(len(listing))) prev_obj = None for obj in listing: if isinstance(obj, Key): self.tester.info("Key: " + obj.name + " -- " + obj.version_id + "--" + obj.last_modified) if prev_obj != None: if self.compare_versions(prev_obj, obj) <= 0: raise Exception( "Version listing not sorted correctly, offending key: " + obj.name + " version: " + obj.version_id + " date: " + obj.last_modified) prev_obj = obj else: self.tester.info("Not a key, skipping: " + str(obj)) def test_object_versioning_suspended(self): """Tests object versioning on a suspended bucket, a more complicated test than the Enabled test""" self.tester.info("Testing bucket Versioning-Suspended") self.test_bucket = self.clear_and_rebuild_bucket(self.test_bucket_name) #Create some keys keyname1 = "versionkey1-" + str(int(time.time())) keyname2 = "versionkey2-" + str(int(time.time())) keyname3 = "versionkey3-" + str(int(time.time())) keyname4 = "versionkey4-" + str(int(time.time())) keyname5 = "versionkey5-" + str(int(time.time())) v1data = self.test_object_data + "--version1" v2data = self.test_object_data + "--version2" v3data = self.test_object_data + "--version3" vstatus = self.test_bucket.get_versioning_status() if vstatus: self.fail("Versioning status should be null/Disabled but was: " + str(vstatus)) else: self.tester.info("Bucket versioning is Disabled") self.put_object(bucket=self.test_bucket, object_key=keyname1, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname2, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname3, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname4, object_data=v1data) self.put_object(bucket=self.test_bucket, object_key=keyname5, object_data=v1data) key1 = self.test_bucket.get_key(keyname1) key2 = self.test_bucket.get_key(keyname2) key3 = self.test_bucket.get_key(keyname3) key4 = self.test_bucket.get_key(keyname4) key5 = self.test_bucket.get_key(keyname5) self.tester.info( "Initial bucket state after object uploads without versioning:") self.print_key_info(keys=[key1, key2, key3, key4, key5]) #Enable versioning self.test_bucket.configure_versioning(True) if self.test_bucket.get_versioning_status(): self.tester.info("Versioning status correctly set to enabled") else: self.tester.info("Versionign status not enabled, should be.") #Update a subset of the keys key1_etag2 = self.put_object(bucket=self.test_bucket, object_key=keyname1, object_data=v2data) key2_etag2 = self.put_object(bucket=self.test_bucket, object_key=keyname2, object_data=v2data) key3_etag2 = self.put_object(bucket=self.test_bucket, object_key=keyname3, object_data=v2data) key3_etag3 = self.put_object(bucket=self.test_bucket, object_key=keyname3, object_data=v3data) #Delete a key self.test_bucket.delete_key(keyname5) #Suspend versioning self.test_bucket.configure_versioning(False) #Get latest of each key key1 = self.test_bucket.get_key(keyname1) key2 = self.test_bucket.get_key(keyname2) key3 = self.test_bucket.get_key(keyname3) key4 = self.test_bucket.get_key(keyname4) key5 = self.test_bucket.get_key(keyname5) #Delete a key #Add a key #Add same key again #Fetch each key def test_object_acl(self): """Tests object acl get/set and manipulation""" self.fail("Test not implemented") #TODO: test custom and canned acls that are both valid an invalid def test_object_torrent(self): """Tests object torrents""" self.fail("Feature not implemented yet") def clean_method(self): '''This is the teardown method''' #Delete the testing bucket if it is left-over self.tester.info('Deleting the buckets used for testing') for bucket in self.buckets_used: try: self.tester.info('Checking bucket ' + bucket + ' for possible cleaning/delete') if self.tester.s3.head_bucket(bucket) != None: self.tester.info('Found bucket exists, cleaning it') self.tester.clear_bucket(bucket) else: self.tester.info('Bucket ' + bucket + ' not found, skipping') except Exception as e: self.tester.info('Exception checking bucket ' + bucket + ' Exception msg: ' + e.message) return def test_multipart_upload(self): '''Basic multipart upload''' self.tester.info("Testing multipart upload") self.tester.info("Creating random file representing part...") temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart") temp_file.write(os.urandom(5 * 1024 * 1024)) keyname = "multi-" + str(int(time.time())) self.tester.info("Initiating multipart upload...much upload") reply = self.initiate_multipart_upload(keyname) self.tester.info("Uploading parts...Such Parts") for partnum in range(1, 11): temp_file.seek(0, os.SEEK_SET) reply.upload_part_from_file(temp_file, partnum) self.tester.info("Listing parts...") self.test_bucket.get_all_multipart_uploads() self.tester.info("Completing upload...So OSG") reply.complete_upload() temp_file.close() self.tester.info("HEAD request...") returned_key = self.test_bucket.get_key(keyname, validate=True) download_temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="mpu-download") self.tester.info("Downloading object...very mpu") returned_key.get_contents_to_file(download_temp_file) self.tester.info("Deleting object...WOW") self.test_bucket.delete_key(keyname) def test_abort_multipart_upload(self): '''Basic multipart upload''' self.tester.info("Testing abort multipart upload") temp_file = tempfile.NamedTemporaryFile(mode="w+b", prefix="multipart") temp_file.write(os.urandom(5 * 1024 * 1024)) keyname = "multi-" + str(int(time.time())) reply = self.initiate_multipart_upload(keyname) for partnum in range(1, 11): temp_file.seek(0, os.SEEK_SET) reply.upload_part_from_file(temp_file, partnum) self.test_bucket.get_all_multipart_uploads() self.tester.info( "!!!!!!!!!!!!!!!!!!!!!!!!!!DO NOT WANT!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1" ) self.tester.info("Canceling upload") reply.cancel_upload() temp_file.close() def initiate_multipart_upload(self, keyname): self.tester.info("Initiating multipart upload " + keyname) return self.test_bucket.initiate_multipart_upload(keyname)