def multipartObjectUpload(): result = 0 dssSanityLib.whisper("Making bucket and listing...") userObj = dssSanityLib.getConnection() bucketpref = dssSanityLib.getsNewBucketName() b = userObj.create_bucket(bucketpref) #dssSanityLib.listBucket(userObj, "User") source_path = dssSanityLib.MULTIPART_LARGE_FILE source_size = os.stat(source_path).st_size chunk_size = 5242880 ## 5 mb #chunk_size = 1048576 ## 1 mb chunk_count = int(math.ceil(source_size / float(chunk_size))) b1 = userObj.get_bucket(bucketpref) dssSanityLib.whisper("Got bucket: " + str(b1)) try: mp = b1.initiate_multipart_upload(os.path.basename(source_path)) for i in range(chunk_count): dssSanityLib.whisper("Uploading chunk: " + str(i)) offset = chunk_size * i bytes = min(chunk_size, source_size - offset) with FileChunkIO(source_path, 'r', offset=offset, bytes=bytes) as fp: mp.upload_part_from_file(fp, part_num=i + 1) mp.complete_upload() except: print "Unexpected error during multipart upload: ", sys.exc_info() result = -1 dssSanityLib.cleanupUser(userObj, bucketpref) return result
def bucketSanity(): ## Create five buckets dssSanityLib.whisper("Creating five test buckets and putting objects in them...") bucketpref = dssSanityLib.getsNewBucketName() dssSanityLib.createMaxBuckets(12, bucketpref) ## Bucket name conflict during creation #dssSanityLib.whisper("Trying to create a bucket with name conflict...") #userObj = dssSanityLib.getConnection(1) ## Different user #buck_str = bucketpref + '1' #try: #b = userObj.create_bucket(buck_str) #print "Error: Unexpectedly created bucket " + buck_str #return -1 #except: # print "Expected failure: " + str(sys.exc_info()) ## Delete all buckets try: userObj = dssSanityLib.getConnection() dssSanityLib.whisper("Deleting the test buckets...") dssSanityLib.cleanupUser(userObj, bucketpref) except: print "Unexpected failure: " + str(sys.exc_info()) return -1 return 0
def bucketSanity(): ## Create five buckets dssSanityLib.whisper( "Creating five test buckets and putting objects in them...") bucketpref = dssSanityLib.getsNewBucketName() dssSanityLib.createMaxBuckets(12, bucketpref) ## Bucket name conflict during creation #dssSanityLib.whisper("Trying to create a bucket with name conflict...") #userObj = dssSanityLib.getConnection(1) ## Different user #buck_str = bucketpref + '1' #try: #b = userObj.create_bucket(buck_str) #print "Error: Unexpectedly created bucket " + buck_str #return -1 #except: # print "Expected failure: " + str(sys.exc_info()) ## Delete all buckets try: userObj = dssSanityLib.getConnection() dssSanityLib.whisper("Deleting the test buckets...") dssSanityLib.cleanupUser(userObj, bucketpref) except: print "Unexpected failure: " + str(sys.exc_info()) return -1 return 0
def deleteBucket(targ): if (not targ): print "Need target to delete!" return -1 if (dssSanityLib.CLI_USER): userObj = dssSanityLib.getConnection(int(dssSanityLib.CLI_USER)) else: userObj = dssSanityLib.getConnection() dssSanityLib.cleanupUser(userObj, targ) return
def multipartObjectUpload(): result = 0 dssSanityLib.whisper("Making bucket and listing...") userObj = dssSanityLib.getConnection() bucketpref = dssSanityLib.getsNewBucketName() b = userObj.create_bucket(bucketpref) #b.set_acl('public-read-write') ##dssSanityLib.listBucket(userObj, "User") source_path = dssSanityLib.MULTIPART_LARGE_FILE source_size = os.stat(source_path).st_size chunk_size = 5242880 ## 5 mb #chunk_size = 1048576 ## 1 mb chunk_count = int(math.ceil(source_size / float(chunk_size))) b1 = userObj.get_bucket(bucketpref) dssSanityLib.whisper("Got bucket: " + str(b1)) try: mp = b1.initiate_multipart_upload(os.path.basename(source_path)) for i in range(chunk_count): dssSanityLib.whisper("Uploading chunk: " + str(i)) offset = chunk_size * i bytes = min(chunk_size, source_size - offset) with FileChunkIO(source_path, 'r', offset=offset, bytes=bytes) as fp: mp.upload_part_from_file(fp, part_num=i + 1) time.sleep(2) for i in b1.list_multipart_uploads(): print "list_multipart_uploads: " + str(i) for i in b1.get_all_multipart_uploads(): print "get_all_multipart_uploads: " + str(i) print("\n\nCompleting uploads") mp.complete_upload() #print("\n\nCancelling uploads") #mp.cancel_upload() except: print "Unexpected error during multipart upload: ", sys.exc_info() result = -1 dssSanityLib.cleanupUser(userObj, bucketpref) return result
def main(argv): ## PARAM OVERRIDES dssSanityLib.MULTIPART_LARGE_FILE = '/boot/initrd.img-3.19.0-25-generic' # Need a large file to upload in multiparts. dssSanityLib.GLOBAL_DEBUG = 1 # The lib supresses debug logs by default. Override here. ##dssSanityLib.RADOSHOST = '127.0.0.1' # The lib points to DSS staging endpoint by default. Override here. ##dssSanityLib.RADOSPORT = 7480 # The lib points to DSS staging endpoint by default. Override here. ret = dssSanityLib.fetchArgs(argv) if(ret == -1): sys.exit(2) ## TESTCASES dssSanityLib.callTest(bucketSanity(), "Create buckets and objects then delete them") dssSanityLib.callTest(multipartObjectUpload(), "Upload object in Multiparts") dssSanityLib.callTest(dnsNamesTest(), "Check various DNS name rules") dssSanityLib.callTest(publicUrlTest(), "Public URL test") ## CLEANUP userObj = dssSanityLib.getConnection() dssSanityLib.cleanupUser(userObj, 'rjilbucketsanity') return