def multipartObjectUpload():
result = 0
dssSanityLib.whisper("Making bucket and listing...")
userObj = dssSanityLib.getConnection()
bucketpref = dssSanityLib.getsNewBucketName()
b = userObj.create_bucket(bucketpref)
#dssSanityLib.listBucket(userObj, "User")
source_path = dssSanityLib.MULTIPART_LARGE_FILE
source_size = os.stat(source_path).st_size
chunk_size = 5242880 ## 5 mb
#chunk_size = 1048576 ## 1 mb
chunk_count = int(math.ceil(source_size / float(chunk_size)))
b1 = userObj.get_bucket(bucketpref)
dssSanityLib.whisper("Got bucket: " + str(b1))
try:
mp = b1.initiate_multipart_upload(os.path.basename(source_path))
for i in range(chunk_count):
dssSanityLib.whisper("Uploading chunk: " + str(i))
offset = chunk_size * i
bytes = min(chunk_size, source_size - offset)
with FileChunkIO(source_path, 'r', offset=offset, bytes=bytes) as fp:
mp.upload_part_from_file(fp, part_num=i + 1)
mp.complete_upload()
except:
print "Unexpected error during multipart upload: ", sys.exc_info()
result = -1
dssSanityLib.cleanupUser(userObj, bucketpref)
return result
def bucketSanity():
## Create five buckets
dssSanityLib.whisper("Creating five test buckets and putting objects in them...")
bucketpref = dssSanityLib.getsNewBucketName()
dssSanityLib.createMaxBuckets(12, bucketpref)
## Bucket name conflict during creation
#dssSanityLib.whisper("Trying to create a bucket with name conflict...")
#userObj = dssSanityLib.getConnection(1) ## Different user
#buck_str = bucketpref + '1'
#try:
#b = userObj.create_bucket(buck_str)
#print "Error: Unexpectedly created bucket " + buck_str
#return -1
#except:
# print "Expected failure: " + str(sys.exc_info())
## Delete all buckets
try:
userObj = dssSanityLib.getConnection()
dssSanityLib.whisper("Deleting the test buckets...")
dssSanityLib.cleanupUser(userObj, bucketpref)
except:
print "Unexpected failure: " + str(sys.exc_info())
return -1
return 0
def bucketSanity():
## Create five buckets
dssSanityLib.whisper(
"Creating five test buckets and putting objects in them...")
bucketpref = dssSanityLib.getsNewBucketName()
dssSanityLib.createMaxBuckets(12, bucketpref)
## Bucket name conflict during creation
#dssSanityLib.whisper("Trying to create a bucket with name conflict...")
#userObj = dssSanityLib.getConnection(1) ## Different user
#buck_str = bucketpref + '1'
#try:
#b = userObj.create_bucket(buck_str)
#print "Error: Unexpectedly created bucket " + buck_str
#return -1
#except:
# print "Expected failure: " + str(sys.exc_info())
## Delete all buckets
try:
userObj = dssSanityLib.getConnection()
dssSanityLib.whisper("Deleting the test buckets...")
dssSanityLib.cleanupUser(userObj, bucketpref)
except:
print "Unexpected failure: " + str(sys.exc_info())
return -1
return 0
def deleteBucket(targ):
if (not targ):
print "Need target to delete!"
return -1
if (dssSanityLib.CLI_USER):
userObj = dssSanityLib.getConnection(int(dssSanityLib.CLI_USER))
else:
userObj = dssSanityLib.getConnection()
dssSanityLib.cleanupUser(userObj, targ)
return
def deleteBucket(targ):
if (not targ):
print "Need target to delete!"
return -1
if (dssSanityLib.CLI_USER):
userObj = dssSanityLib.getConnection(int(dssSanityLib.CLI_USER))
else:
userObj = dssSanityLib.getConnection()
dssSanityLib.cleanupUser(userObj, targ)
return
def multipartObjectUpload():
result = 0
dssSanityLib.whisper("Making bucket and listing...")
userObj = dssSanityLib.getConnection()
bucketpref = dssSanityLib.getsNewBucketName()
b = userObj.create_bucket(bucketpref)
#b.set_acl('public-read-write')
##dssSanityLib.listBucket(userObj, "User")
source_path = dssSanityLib.MULTIPART_LARGE_FILE
source_size = os.stat(source_path).st_size
chunk_size = 5242880 ## 5 mb
#chunk_size = 1048576 ## 1 mb
chunk_count = int(math.ceil(source_size / float(chunk_size)))
b1 = userObj.get_bucket(bucketpref)
dssSanityLib.whisper("Got bucket: " + str(b1))
try:
mp = b1.initiate_multipart_upload(os.path.basename(source_path))
for i in range(chunk_count):
dssSanityLib.whisper("Uploading chunk: " + str(i))
offset = chunk_size * i
bytes = min(chunk_size, source_size - offset)
with FileChunkIO(source_path, 'r', offset=offset,
bytes=bytes) as fp:
mp.upload_part_from_file(fp, part_num=i + 1)
time.sleep(2)
for i in b1.list_multipart_uploads():
print "list_multipart_uploads: " + str(i)
for i in b1.get_all_multipart_uploads():
print "get_all_multipart_uploads: " + str(i)
print("\n\nCompleting uploads")
mp.complete_upload()
#print("\n\nCancelling uploads")
#mp.cancel_upload()
except:
print "Unexpected error during multipart upload: ", sys.exc_info()
result = -1
dssSanityLib.cleanupUser(userObj, bucketpref)
return result
def main(argv):
## PARAM OVERRIDES
dssSanityLib.MULTIPART_LARGE_FILE = '/boot/initrd.img-3.19.0-25-generic' # Need a large file to upload in multiparts.
dssSanityLib.GLOBAL_DEBUG = 1 # The lib supresses debug logs by default. Override here.
##dssSanityLib.RADOSHOST = '127.0.0.1' # The lib points to DSS staging endpoint by default. Override here.
##dssSanityLib.RADOSPORT = 7480 # The lib points to DSS staging endpoint by default. Override here.
ret = dssSanityLib.fetchArgs(argv)
if(ret == -1):
sys.exit(2)
## TESTCASES
dssSanityLib.callTest(bucketSanity(), "Create buckets and objects then delete them")
dssSanityLib.callTest(multipartObjectUpload(), "Upload object in Multiparts")
dssSanityLib.callTest(dnsNamesTest(), "Check various DNS name rules")
dssSanityLib.callTest(publicUrlTest(), "Public URL test")
## CLEANUP
userObj = dssSanityLib.getConnection()
dssSanityLib.cleanupUser(userObj, 'rjilbucketsanity')
return
