def test_blobfs_upload_1Kb_file(
explictFromTo=False,
forceOAuthLogin=False,
tenantID="",
aadEndpoint=""):
# create file of size 1KB
filename = "test_blob_1kb_file.txt"
file_path = util.create_test_file(filename, 1024)
# upload the file using Azcopy
cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info")
util.process_oauth_command(
cmd,
"LocalBlobFS" if explictFromTo else "",
forceOAuthLogin,
tenantID,
aadEndpoint)
if forceOAuthLogin:
result = cmd.execute_azcopy_command_interactive()
else:
result = cmd.execute_azcopy_copy_command()
if not result:
print("test_blobfs_upload_1Kb_file failed uploading the 1kb file ", filename, " to the filesystem")
sys.exit(1)
# Validate the file uploaded file
fileUrl = util.test_bfs_account_url + filename
result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify()
if not result:
print("test_blobfs_upload_1Kb_file failed while validating the 1kb file upload")
sys.exit(1)
print("test_blobfs_upload_1Kb_file successfully passed")
def util_test_download_1kb_blob_with_oauth(self):
# create file of size 1KB.
filename = "test_1kb_blob_upload.txt"
file_path = util.create_test_file(filename, 1024)
# Upload 1KB file using azcopy.
src = file_path
dest = util.test_oauth_container_url
cmd = util.Command("copy").add_arguments(src).add_arguments(dest). \
add_flags("log-level", "info").add_flags("recursive", "true")
util.process_oauth_command(cmd, "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded blob.
# the resource local path should be the first argument for the azcopy validator.
# the resource sas should be the second argument for azcopy validator.
dest_validate = util.get_resource_from_oauth_container_validate(filename)
result = util.Command("testBlob").add_arguments(file_path).add_arguments(dest_validate).execute_azcopy_verify()
self.assertTrue(result)
# downloading the uploaded file
src = util.get_resource_from_oauth_container(filename)
src_validate = util.get_resource_from_oauth_container_validate(filename)
dest = util.test_directory_path + "/test_1kb_blob_download.txt"
cmd = util.Command("copy").add_arguments(src).add_arguments(dest).add_flags("log-level", "info")
util.process_oauth_command(cmd, "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the downloaded blob
result = util.Command("testBlob").add_arguments(dest).add_arguments(src_validate).execute_azcopy_verify()
self.assertTrue(result)
def util_test_blobfs_download_100_1Kb_file(self, explictFromTo=False):
# create dir with 100 1KB files inside it
dir_name = "dir_blobfs_d_100_1K"
dir_n_file_path = util.create_test_n_files(1024, 100, dir_name)
# Upload the directory with 100 files inside it
cmd = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info").add_flags("recursive","true")
util.process_oauth_command(cmd, "LocalBlobFS" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Validate the uploaded directory
dirUrl = util.test_bfs_account_url + dir_name
result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
# delete the local directory created
try:
shutil.rmtree(dir_n_file_path)
except:
self.fail('error deleting the directory ' + dir_n_file_path)
# download the directory
cmd = util.Command("copy").add_arguments(dirUrl).add_arguments(util.test_directory_path).\
add_flags("log-level", "Info").add_flags("recursive", "true")
util.process_oauth_command(cmd, "BlobFSLocal" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# validate the downloaded directory
result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl).\
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
def test_blobfs_upload_100_1Kb_file(
explictFromTo=False,
forceOAuthLogin=False,
tenantID="",
aadEndpoint=""):
# create dir with 100 1KB files inside it
dir_name = "dir_blobfs_100_1K"
dir_n_file_path = util.create_test_n_files(1024, 100, dir_name)
# Upload the directory with 100 files inside it
cmd = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info").add_flags("recursive","true")
util.process_oauth_command(
cmd,
"LocalBlobFS" if explictFromTo else "",
forceOAuthLogin,
tenantID,
aadEndpoint)
if forceOAuthLogin:
result = cmd.execute_azcopy_command_interactive()
else:
result = cmd.execute_azcopy_copy_command()
if not result:
print("test_blobfs_upload_100_1Kb_file failed uploading the dir ", dir_name, " to the filesystem")
sys.exit(1)
# Validate the uploaded directory
dirUrl = util.test_bfs_account_url + dir_name
result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl).\
add_flags("is-object-dir", "true").execute_azcopy_verify()
if not result:
print("test_blobfs_upload_100_1Kb_file failed while validating the file upload")
sys.exit(1)
print("test_blobfs_upload_100_1Kb_file successfully passed")
def util_test_blobfs_download_1Kb_file(self, explictFromTo=False):
# create file of size 1KB
filename = "test_blob_d_1kb_file.txt"
file_path = util.create_test_file(filename, 1024)
# upload the file using Azcopy
cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info")
util.process_oauth_command(cmd, "LocalBlobFS" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Validate the file uploaded file
fileUrl = util.test_bfs_account_url + filename
result = util.Command("testBlobFS").add_arguments(
file_path).add_arguments(fileUrl).execute_azcopy_verify()
self.assertTrue(result)
# delete the file locally
try:
os.remove(file_path)
except:
self.fail('error deleting the file ' + file_path)
# download the file using Azcopy
cmd = util.Command("copy").add_arguments(fileUrl).add_arguments(util.test_directory_path). \
add_flags("log-level", "Info")
util.process_oauth_command(cmd, "BlobFSLocal" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# validate the downloaded file
result = util.Command("testBlobFS").add_arguments(
file_path).add_arguments(fileUrl).execute_azcopy_verify()
self.assertTrue(result)
def test_recursive_download_blob_oauth(forceOAuthLogin=False,
tenantID="",
aadEndpoint=""):
# create directory and 5 files of 1KB inside that directory.
dir_name = "dir_" + str(10) + "_files"
dir1_path = util.create_test_n_files(1024, 5, dir_name)
dest = util.test_oauth_container_url
# upload the directory to container through azcopy with recursive set to true.
cmd = util.Command("copy").add_arguments(dir1_path).add_arguments(dest).add_flags("log-level", "info") \
.add_flags("recursive", "true")
util.process_oauth_command(cmd, "", forceOAuthLogin, tenantID, aadEndpoint)
if forceOAuthLogin:
result = cmd.execute_azcopy_command_interactive()
else:
result = cmd.execute_azcopy_copy_command()
if not result:
print("error uploading recursive dir ", dir1_path)
return
# verify the uploaded file.
dest_validate = util.get_resource_from_oauth_container_validate(dir_name)
result = util.Command("testBlob").add_arguments(dir1_path).add_arguments(
dest_validate).add_flags("is-object-dir",
"true").execute_azcopy_verify()
if not result:
print("error verify the recursive dir ", dir1_path, " upload")
return
try:
shutil.rmtree(dir1_path)
except OSError as e:
print("error removing the uploaded files. ", e)
return
src_download = util.get_resource_from_oauth_container(dir_name)
# downloading the directory created from container through azcopy with recursive flag to true.
cmd = util.Command("copy").add_arguments(src_download).add_arguments(
util.test_directory_path).add_flags("log-level", "info").add_flags(
"recursive", "true")
util.process_oauth_command(cmd, "", forceOAuthLogin, tenantID, aadEndpoint)
if forceOAuthLogin:
result = cmd.execute_azcopy_command_interactive()
else:
result = cmd.execute_azcopy_copy_command()
if not result:
print("error download recursive dir ", dir1_path)
return
# verify downloaded blob.
result = util.Command("testBlob").add_arguments(dir1_path).add_arguments(
dest_validate).add_flags("is-object-dir",
"true").execute_azcopy_verify()
if not result:
print("error verifying the recursive download ")
return
print("test_recursive_download_blob successfully passed")
def test_download_1kb_blob_oauth(forceOAuthLogin=False,
tenantID="",
aadEndpoint=""):
# create file of size 1KB.
filename = "test_1kb_blob_upload.txt"
file_path = util.create_test_file(filename, 1024)
# Upload 1KB file using azcopy.
src = file_path
dest = util.test_oauth_container_url
cmd = util.Command("copy").add_arguments(src).add_arguments(dest). \
add_flags("log-level", "info").add_flags("recursive", "true")
util.process_oauth_command(cmd, "", forceOAuthLogin, tenantID, aadEndpoint)
if forceOAuthLogin:
result = cmd.execute_azcopy_command_interactive()
else:
result = cmd.execute_azcopy_copy_command()
if not result:
print("failed uploading 1KB file to the container")
return
# Verifying the uploaded blob.
# the resource local path should be the first argument for the azcopy validator.
# the resource sas should be the second argument for azcopy validator.
dest_validate = util.get_resource_from_oauth_container_validate(filename)
result = util.Command("testBlob").add_arguments(file_path).add_arguments(
dest_validate).execute_azcopy_verify()
if not result:
print("test_1kb_file test failed")
return
time.sleep(5)
# downloading the uploaded file
src = util.get_resource_from_oauth_container(filename)
src_validate = util.get_resource_from_oauth_container_validate(filename)
dest = util.test_directory_path + "/test_1kb_blob_download.txt"
cmd = util.Command("copy").add_arguments(src).add_arguments(
dest).add_flags("log-level", "info")
util.process_oauth_command(cmd, "", forceOAuthLogin, tenantID, aadEndpoint)
if forceOAuthLogin:
result = cmd.execute_azcopy_command_interactive()
else:
result = cmd.execute_azcopy_copy_command()
if not result:
print("test_download_1kb_blob test case failed")
return
# Verifying the downloaded blob
result = util.Command("testBlob").add_arguments(dest).add_arguments(
src_validate).execute_azcopy_verify()
if not result:
print("test_download_1kb_blob test case failed")
return
print("test_download_1kb_blob successfully passed")
def test_blobfs_download_100_1Kb_file(explictFromTo=False):
# create dir with 100 1KB files inside it
dir_name = "dir_blobfs_d_100_1K"
dir_n_file_path = util.create_test_n_files(1024, 100, dir_name)
# Upload the directory with 100 files inside it
cmd = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info").add_flags("recursive","true")
util.process_oauth_command(cmd, "LocalBlobFS" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
if not result:
print("test_blobfs_download_100_1Kb_file failed uploading the dir ",
dir_name, " to the filesystem")
sys.exit(1)
# Validate the uploaded directory
dirUrl = util.test_bfs_account_url + dir_name
result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
if not result:
print(
"test_blobfs_download_100_1Kb_file failed while validating the uploaded directory"
)
sys.exit(1)
# delete the local directory created
try:
shutil.rmtree(dir_n_file_path)
except:
print(
"test_blobfs_download_100_1Kb_file failed while deleting the local directory"
)
sys.exit(1)
# download the directory
cmd = util.Command("copy").add_arguments(dirUrl).add_arguments(util.test_directory_path).\
add_flags("log-level", "Info").add_flags("recursive", "true")
util.process_oauth_command(cmd, "BlobFSLocal" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
if not result:
print(
"test_blobfs_download_100_1Kb_file failed while downloading the directory"
)
sys.exit(1)
# validate the downloaded directory
result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl).\
add_flags("is-object-dir", "true").execute_azcopy_verify()
if not result:
print(
"test_blobfs_download_100_1Kb_file failed while validating the downloaded directory"
)
sys.exit(1)
print("test_blobfs_download_100_1Kb_file successfully passed")
def test_blobfs_download_64MB_file(explictFromTo=False):
# create test file of size 64MB
filename = "test_blob_d_64MB_file.txt"
file_path = util.create_test_file(filename, 64 * 1024 * 1024)
# Upload the file using Azcopy
cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info")
util.process_oauth_command(cmd, "LocalBlobFS" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
if not result:
print("test_blobfs_download_64MB_file failed uploading the 64MB file ",
filename, " to the filesystem")
sys.exit(1)
# Validate the file uploaded
fileUrl = util.test_bfs_account_url + filename
result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(
fileUrl).execute_azcopy_verify()
if not result:
print(
"test_blobfs_download_64MB_file failed while validating the 64MB file upload"
)
sys.exit(1)
# delete the file locally
try:
os.remove(file_path)
except:
print(
"test_blobfs_download_64MB_file failed while deleting the file locally"
)
sys.exit(1)
# download the file using azcopy
cmd = util.Command("copy").add_arguments(fileUrl).add_arguments(util.test_directory_path). \
add_flags("log-level", "Info")
util.process_oauth_command(cmd, "BlobFSLocal" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
if not result:
print(
"test_blobfs_download_64MB_file failed downloading the 64MB file ",
filename)
sys.exit(1)
# validate the downloaded file
result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(
fileUrl).execute_azcopy_verify()
if not result:
print(
"test_blobfs_download_64MB_file failed while validating the 64MB downloaded file"
)
sys.exit(1)
print("test_blobfs_download_64MB_file successfully passed")
def util_test_blobfs_upload_64MB_file(self, explictFromTo=False):
# create test file of size 64MB
filename = "test_blob_64MB_file.txt"
file_path = util.create_test_file(filename, 64 * 1024 * 1024)
# Upload the file using Azcopy
cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info")
util.process_oauth_command(cmd, "LocalBlobFS" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Validate the file uploaded
fileUrl = util.test_bfs_account_url + filename
result = util.Command("testBlobFS").add_arguments(
file_path).add_arguments(fileUrl).execute_azcopy_verify()
self.assertTrue(result)
def util_test_blobfs_download_64MB_file(self,
explictFromTo=False,
forceOAuthLogin=False,
tenantID="",
aadEndpoint=""):
# create test file of size 64MB
filename = "test_blob_d_64MB_file.txt"
file_path = util.create_test_file(filename, 64 * 1024 * 1024)
# Upload the file using Azcopy
cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info")
util.process_oauth_command(cmd, "LocalBlobFS" if explictFromTo else "",
forceOAuthLogin, tenantID, aadEndpoint)
if forceOAuthLogin:
result = cmd.execute_azcopy_command_interactive()
else:
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Validate the file uploaded
fileUrl = util.test_bfs_account_url + filename
result = util.Command("testBlobFS").add_arguments(
file_path).add_arguments(fileUrl).execute_azcopy_verify()
self.assertTrue(result)
# delete the file locally
try:
os.remove(file_path)
except:
self.fail('error deleting the file ' + file_path)
# download the file using azcopy
cmd = util.Command("copy").add_arguments(fileUrl).add_arguments(util.test_directory_path). \
add_flags("log-level", "Info")
util.process_oauth_command(cmd, "BlobFSLocal" if explictFromTo else "",
forceOAuthLogin, tenantID, aadEndpoint)
if forceOAuthLogin:
result = cmd.execute_azcopy_command_interactive()
else:
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# validate the downloaded file
result = util.Command("testBlobFS").add_arguments(
file_path).add_arguments(fileUrl).execute_azcopy_verify()
self.assertTrue(result)
def util_test_blobfs_upload_uneven_multiflush_file(self,
explicitFromTo=False):
# create test file of size 64MB
filename = "test_uneven_multiflush_64MB_file.txt"
file_path = util.create_test_file(filename, 64 * 1024 * 1024)
# Upload the file using AzCopy @ 1MB blocks, 15 block flushes (5 flushes, 4 15 blocks, 1 4 blocks)
cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \
add_flags("block-size-mb", "1").add_flags("flush-threshold", "15").add_flags("log-level", "Info")
util.process_oauth_command(cmd,
"LocalBlobFS" if explicitFromTo else "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Validate the file uploaded
fileUrl = util.test_bfs_account_url + filename
result = util.Command("testBlobFS").add_arguments(
file_path).add_arguments(fileUrl).execute_azcopy_verify()
self.assertTrue(result)
def util_test_recursive_download_blob_with_oauth(self):
# create directory and 5 files of 1KB inside that directory.
dir_name = "util_test_recursive_download_blob_with_oauth_dir_" + str(
10) + "_files"
dir1_path = util.create_test_n_files(1024, 5, dir_name)
dest = util.test_oauth_container_url
# upload the directory to container through azcopy with recursive set to true.
cmd = util.Command("copy").add_arguments(dir1_path).add_arguments(dest).add_flags("log-level", "info") \
.add_flags("recursive", "true")
util.process_oauth_command(cmd, "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# verify the uploaded file.
dest_validate = util.get_resource_from_oauth_container_validate(
dir_name)
result = util.Command("testBlob").add_arguments(
dir1_path).add_arguments(dest_validate).add_flags(
"is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
try:
shutil.rmtree(dir1_path)
except OSError as e:
self.fail("error removing the upload files. " + e)
src_download = util.get_resource_from_oauth_container(dir_name)
# downloading the directory created from container through azcopy with recursive flag to true.
cmd = util.Command("copy").add_arguments(src_download).add_arguments(
util.test_directory_path).add_flags("log-level", "info").add_flags(
"recursive", "true")
util.process_oauth_command(cmd, "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# verify downloaded blob.
result = util.Command("testBlob").add_arguments(
dir1_path).add_arguments(dest_validate).add_flags(
"is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
def util_test_blobfs_upload_1Kb_file(self,
explictFromTo=False,
forceOAuthLogin=False,
tenantID="",
aadEndpoint=""):
# create file of size 1KB
filename = "test_blob_1kb_file.txt"
file_path = util.create_test_file(filename, 1024)
# upload the file using Azcopy
cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info")
util.process_oauth_command(cmd, "LocalBlobFS" if explictFromTo else "",
forceOAuthLogin, tenantID, aadEndpoint)
if forceOAuthLogin:
result = cmd.execute_azcopy_command_interactive()
else:
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Validate the file uploaded file
fileUrl = util.test_bfs_account_url + filename
result = util.Command("testBlobFS").add_arguments(
file_path).add_arguments(fileUrl).execute_azcopy_verify()
self.assertTrue(result)