def test_piping_upload_and_download_large_file(self):
# large means azcopy has to rotate buffers when uploading.
# create file of size 8MB
filename = "test_8mb_blob_piping_upload.txt"
source_file_path = util.create_test_file(filename, 8 * 1024 * 1024)
# compute source md5 to compare later
source_file_md5 = compute_md5(source_file_path)
# uploadfile using azcopy
# TODO reviewers please note, this used to use a 4MB file, with a 1 KiB block size, but now we don't support block sizes
# smaller than 1 MB. I've compensated slightly by changing it to an 8 MB file
destination_url = util.get_resource_sas(filename)
azcopy_cmd = util.Command("copy").add_arguments(destination_url).add_flags("block-size-mb", '1').add_flags("from-to", "PipeBlob").string()
self.assertTrue(execute_command_with_pipe(azcopy_cmd, source_file_to_pipe=source_file_path))
# downloading the uploaded file
azcopy_cmd = util.Command("copy").add_arguments(destination_url).add_flags("block-size-mb", '1').add_flags("from-to", "BlobPipe").string()
destination_file_path = util.test_directory_path + "/test_8mb_blob_piping_download.txt"
self.assertTrue(execute_command_with_pipe(azcopy_cmd, destination_file_to_pipe=destination_file_path))
# compute destination md5 to compare
destination_file_md5 = compute_md5(destination_file_path)
# verifying the downloaded blob
self.assertEqual(source_file_md5, destination_file_md5)
def test_blob_download_preserve_last_modified_time(self):
# create a file of 2KB
filename = "test_upload_preserve_last_mtime.txt"
file_path = util.create_test_file(filename, 2048)
# upload file through azcopy.
destination_sas = util.get_resource_sas(filename)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \
add_flags("log-level", "info").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded blob
result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas).execute_azcopy_verify()
self.assertTrue(result)
time.sleep(5)
# download file through azcopy with flag preserve-last-modified-time set to true
download_file_name = util.test_directory_path + "/test_download_preserve_last_mtime.txt"
result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file_name)\
.add_flags("log-level","info").add_flags("preserve-last-modified-time", "true").\
execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the downloaded blob and its modified with the modified time of blob.
result = util.Command("testBlob").add_arguments(download_file_name).add_arguments(destination_sas).add_flags(
"preserve-last-modified-time", "true").execute_azcopy_verify()
self.assertTrue(result)
def test_piping_upload_and_download_small_file(self):
# small means azcopy doesn't have to rotate buffers when uploading.
# it is assumed that the default block size is much larger than 1KB.
# create file of size 1KB
filename = "test_1kb_blob_piping_upload.txt"
source_file_path = util.create_test_file(filename, 1024)
# compute source md5 to compare later
source_file_md5 = compute_md5(source_file_path)
# upload 1KB file using azcopy
destination_url = util.get_resource_sas(filename)
azcopy_cmd = util.Command("copy").add_arguments(
destination_url).add_flags("from-to", "PipeBlob").add_flags(
"log-level", "DEBUG").string()
self.assertTrue(
execute_command_with_pipe(azcopy_cmd,
source_file_to_pipe=source_file_path))
# downloading the uploaded file
azcopy_cmd = util.Command("copy").add_arguments(
destination_url).add_flags("from-to", "BlobPipe").add_flags(
"log-level", "DEBUG").string()
destination_file_path = util.test_directory_path + "/test_1kb_blob_piping_download.txt"
self.assertTrue(
execute_command_with_pipe(
azcopy_cmd, destination_file_to_pipe=destination_file_path))
# compute destination md5 to compare
destination_file_md5 = compute_md5(destination_file_path)
# verifying the downloaded blob
self.assertEqual(source_file_md5, destination_file_md5)
def test_blobfs_upload_1Kb_file(
explictFromTo=False,
forceOAuthLogin=False,
tenantID="",
aadEndpoint=""):
# create file of size 1KB
filename = "test_blob_1kb_file.txt"
file_path = util.create_test_file(filename, 1024)
# upload the file using Azcopy
cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info")
util.process_oauth_command(
cmd,
"LocalBlobFS" if explictFromTo else "",
forceOAuthLogin,
tenantID,
aadEndpoint)
if forceOAuthLogin:
result = cmd.execute_azcopy_command_interactive()
else:
result = cmd.execute_azcopy_copy_command()
if not result:
print("test_blobfs_upload_1Kb_file failed uploading the 1kb file ", filename, " to the filesystem")
sys.exit(1)
# Validate the file uploaded file
fileUrl = util.test_bfs_account_url + filename
result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify()
if not result:
print("test_blobfs_upload_1Kb_file failed while validating the 1kb file upload")
sys.exit(1)
print("test_blobfs_upload_1Kb_file successfully passed")
def test_sync_blob_download_without_wildcards(self):
# created a directory and created 10 files inside the directory
dir_name = "sync_download_without_wildcards"
dir_n_files_path = util.create_test_n_files(1024, 10, dir_name)
# upload the directory
# execute azcopy command
result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \
add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator.
dir_sas = util.get_resource_sas(dir_name)
result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
# download the destination to the source to match the last modified time
result = util.Command("copy").add_arguments(dir_sas).add_arguments(util.test_directory_path). \
add_flags("log-level", "info").add_flags("recursive", "true").add_flags("output", "json"). \
add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command_get_output()
self.assertNotEquals(result, None)
# execute the validator and verify the downloaded dir
result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
# sync the source and destination
result = util.Command("sync").add_arguments(dir_sas).add_arguments(dir_n_files_path). \
add_flags("log-level", "info").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertFalse(result)
def util_test_1GB_blob_upload(self, use_oauth_session=False):
# create 1Gb file
filename = "test_1G_blob.txt"
file_path = util.create_test_file(filename, 1 * 1024 * 1024 * 1024)
# execute azcopy upload.
if not use_oauth_session:
dest = util.get_resource_sas(filename)
dest_validate = dest
else:
dest = util.get_resource_from_oauth_container(filename)
dest_validate = util.get_resource_from_oauth_container_validate(
filename)
result = util.Command("copy").add_arguments(file_path).add_arguments(dest).add_flags("log-level", "info"). \
add_flags("block-size", "104857600").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded blob.
# adding local file path as first argument.
# adding file sas as local argument.
# calling the testBlob validator to verify whether blob has been successfully uploaded or not.
result = util.Command("testBlob").add_arguments(
file_path).add_arguments(dest_validate).execute_azcopy_verify()
self.assertTrue(result)
def util_test_n_1kb_blob_upload(self,
number_of_files,
use_oauth_session=False):
# create dir dir_n_files and 1 kb files inside the dir.
dir_name = "dir_" + str(number_of_files) + "_files"
dir_n_files_path = util.create_test_n_files(1024, number_of_files,
dir_name)
if not use_oauth_session:
dest = util.test_container_url
dest_validate = util.get_resource_sas(dir_name)
else:
dest = util.test_oauth_container_url
dest_validate = util.get_resource_from_oauth_container_validate(
dir_name)
# execute azcopy command
result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(dest). \
add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator.
result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dest_validate). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
def test_sync_single_blob_with_local(self):
# create file of size 1KB.
filename = "test_1kb_blob_sync.txt"
file_path = util.create_test_file(filename, 1024)
blob_path = util.get_resource_sas(filename)
# Upload 1KB file using azcopy.
src = file_path
dest = blob_path
result = util.Command("cp").add_arguments(src).add_arguments(dest). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded blob.
# the resource local path should be the first argument for the azcopy validator.
# the resource sas should be the second argument for azcopy validator.
resource_url = util.get_resource_sas(filename)
result = util.Command("testBlob").add_arguments(file_path).add_arguments(resource_url).execute_azcopy_verify()
self.assertTrue(result)
# Sync 1KB file to local using azcopy.
src = blob_path
dest = file_path
result = util.Command("sync").add_arguments(src).add_arguments(dest). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# Sync 1KB file to blob using azcopy.
# reset local file lmt first
util.create_test_file(filename, 1024)
src = file_path
dest = blob_path
result = util.Command("sync").add_arguments(src).add_arguments(dest). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
def test_sync_entire_directory_with_local(self):
dir_name = "dir_sync_test"
dir_path = util.create_test_n_files(1024, 10, dir_name)
# create sub-directory inside directory
sub_dir_name = os.path.join(dir_name, "sub_dir_sync_test")
util.create_test_n_files(1024, 10, sub_dir_name)
# upload the directory with 20 files
# upload the directory
# execute azcopy command
result = util.Command("copy").add_arguments(dir_path).add_arguments(util.test_container_url). \
add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator.
vdir_sas = util.get_resource_sas(dir_name)
result = util.Command("testBlob").add_arguments(dir_path).add_arguments(vdir_sas). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
# sync to local
src = vdir_sas
dst = dir_path
result = util.Command("sync").add_arguments(src).add_arguments(dst).add_flags("log-level", "info")\
.execute_azcopy_copy_command()
self.assertTrue(result)
# sync back to blob after recreating the files
util.create_test_n_files(1024, 10, sub_dir_name)
src = dir_path
dst = vdir_sas
result = util.Command("sync").add_arguments(src).add_arguments(dst).add_flags("log-level", "info") \
.execute_azcopy_copy_command()
self.assertTrue(result)
def util_test_copy_n_files_from_gcp_bucket_to_blob_account(
self, srcBucketURL, dstAccountURL, n=10, sizeInKB=1):
srcType = "GCP"
result = util.Command("create").add_arguments(srcBucketURL).add_flags("serviceType", srcType). \
add_flags("resourceType", "Bucket").execute_azcopy_create()
self.assertTrue(result)
src_dir_name = "copy_%d_%dKB_files_from_gcp_bucket_to_blob_account" % (
n, sizeInKB)
src_dir_path = util.create_test_n_files(sizeInKB * 1024, n,
src_dir_name)
self.util_upload_to_src(src_dir_path, srcType, srcBucketURL, True)
result = util.Command("copy").add_arguments(srcBucketURL).add_arguments(dstAccountURL). \
add_flags("log-level", "info").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
validate_dir_name = "validate_copy_%d_%dKB_files_from_gcp_bucket_to_blob_account" % (
n, sizeInKB)
local_validate_dest = util.create_test_dir(validate_dir_name)
validateDstBucketURL = util.get_object_sas(dstAccountURL,
self.bucket_name)
dst_directory_url = util.get_object_sas(validateDstBucketURL,
src_dir_name)
result = util.Command("copy").add_arguments(dst_directory_url).add_arguments(local_validate_dest). \
add_flags("log-level", "info").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
result = self.util_are_dir_trees_equal(
src_dir_path, os.path.join(local_validate_dest, src_dir_name))
self.assertTrue(result)
def test_sync_entire_vdir_to_vdir(self):
content_dir_name = "dir_sync_test"
content_dir_path = util.create_test_n_files(1024, 10, content_dir_name)
src_vdir_path = util.get_resource_sas("srcdir")
dst_vdir_path = util.get_resource_sas("dstdir")
# create sub-directory inside directory
sub_dir_name = os.path.join(content_dir_name, "sub_dir_sync_test")
util.create_test_n_files(1024, 10, sub_dir_name)
# upload the directory with 20 files
# upload the directory
# execute azcopy command
result = util.Command("copy").add_arguments(content_dir_path).add_arguments(src_vdir_path). \
add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator.
result = util.Command("testBlob").add_arguments(content_dir_path).add_arguments(src_vdir_path). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
# sync to destination
result = util.Command("sync").add_arguments(src_vdir_path).add_arguments(dst_vdir_path)\
.add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
def util_upload_to_src(self,
localFilePath,
srcType,
srcURLForCopy,
recursive=False,
blobType="",
blobTier=""):
if srcType == "GCP":
cmd = util.Command("upload").add_arguments(
localFilePath).add_arguments(srcURLForCopy).add_flags(
"serviceType", "GCP")
else:
cmd = util.Command("copy").add_arguments(
localFilePath).add_arguments(srcURLForCopy).add_flags(
"log-level", "info")
if blobType != "":
cmd.add_flags("blob-type", blobType)
if blobType == "PageBlob" and blobTier != "":
cmd.add_flags("page-blob-tier", blobTier)
if blobType == "BlockBlob" and blobTier != "":
cmd.add_flags("block-blob-tier", blobTier)
if recursive:
cmd.add_flags("recursive", True)
if srcType == "GCP":
result = cmd.execute_testsuite_upload()
else:
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
def test_copy_single_file_to_file(self):
content_file_name_src = "test_1kb_file_copy_src.txt"
content_file_path_src = util.create_test_file(content_file_name_src,
1024)
content_file_name_dst = "test_1kb_file_copy_dst.txt"
# create source file of size 1KB.
remote_src_file_path = util.get_resource_sas_from_share(
content_file_name_src)
remote_dst_file_path = util.get_resource_sas_from_share(
content_file_name_dst)
result = util.Command("cp").add_arguments(content_file_path_src).add_arguments(remote_src_file_path). \
add_flags("log-level", "debug").execute_azcopy_copy_command()
self.assertTrue(result)
# perform the single file copy using azcopy.
result = util.Command("copy").add_arguments(remote_src_file_path).add_arguments(remote_dst_file_path). \
add_flags("log-level", "debug").execute_azcopy_copy_command()
self.assertTrue(result)
# verifying the copy worked, both remote source and destination should be identical to the local source
result = util.Command("testFile").add_arguments(
content_file_path_src).add_arguments(
remote_src_file_path).execute_azcopy_verify()
self.assertTrue(result)
result = util.Command("testFile").add_arguments(
content_file_path_src).add_arguments(
remote_dst_file_path).execute_azcopy_verify()
self.assertTrue(result)
def test_copy_entire_dir_to_dir(self):
content_dir_name_src = "dir_file_copy_test_src"
content_dir_path_src = util.create_test_n_files(
1024, 10, content_dir_name_src)
content_dir_name_dst = "dir_file_copy_test_dst"
# create sub-directory inside directory
sub_dir_name = os.path.join(content_dir_name_src, "sub_dir_copy_test")
util.create_test_n_files(1024, 10, sub_dir_name)
# upload to the source
result = util.Command("copy").add_arguments(content_dir_path_src).add_arguments(util.test_share_url). \
add_flags("recursive", "true").add_flags("log-level", "debug").execute_azcopy_copy_command()
self.assertTrue(result)
# copy to destination
remote_src_dir_path = util.get_resource_sas_from_share(
content_dir_name_src)
remote_dst_dir_path = util.get_resource_sas_from_share(
content_dir_name_dst)
result = util.Command("copy").add_arguments(util.get_resource_sas_from_share(content_dir_name_src+"/*"))\
.add_arguments(remote_dst_dir_path).add_flags("log-level", "debug").add_flags("recursive", "true")\
.execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator to make sure the copy worked, both remote src and dst should match local src
result = util.Command("testFile").add_arguments(content_dir_path_src).add_arguments(remote_src_dir_path). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
result = util.Command("testFile").add_arguments(content_dir_path_src).add_arguments(remote_dst_dir_path). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
def test_blob_download_with_special_characters(self):
filename_special_characters = "abc|>rd*"
# encode filename beforehand to avoid erroring out
resource_url = util.get_resource_sas(
filename_special_characters.replace("*", "%2A"))
# creating the file with random characters and with file name having special characters.
result = util.Command("create").add_arguments(resource_url).add_flags(
"serviceType",
"Blob").add_flags("resourceType", "SingleFile").add_flags(
"blob-size", "1024").execute_azcopy_verify()
self.assertTrue(result)
# downloading the blob created above.
result = util.Command("copy").add_arguments(
resource_url).add_arguments(util.test_directory_path).add_flags(
"log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
expected_filename = filename_special_characters
if os.name == "nt":
# Windows will encode special characters.
expected_filename = urllib.parse.quote_plus(
filename_special_characters)
# verify if the downloaded file exists or not.
filepath = util.test_directory_path + "/" + expected_filename
self.assertTrue(os.path.isfile(filepath))
# verify the downloaded blob.
result = util.Command("testBlob").add_arguments(
filepath).add_arguments(resource_url).execute_azcopy_verify()
self.assertTrue(result)
def test_sync_single_blob_to_blob(self):
content_file_name = "test_1kb_blob_sync.txt"
content_file_path = util.create_test_file(content_file_name, 1024)
# create source and destination blobs of size 1KB.
# make sure to create the destination first so that it has an older lmt
src_blob_path = util.get_resource_sas("test_1kb_blob_sync_src.txt")
dst_blob_path = util.get_resource_sas("test_1kb_blob_sync_dst.txt")
result = util.Command("cp").add_arguments(content_file_path).add_arguments(dst_blob_path). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
result = util.Command("cp").add_arguments(content_file_path).add_arguments(src_blob_path). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# verifying the uploaded blobs.
# the resource local path should be the first argument for the azcopy validator.
# the resource sas should be the second argument for azcopy validator.
result = util.Command("testBlob").add_arguments(content_file_path).add_arguments(src_blob_path).execute_azcopy_verify()
self.assertTrue(result)
result = util.Command("testBlob").add_arguments(content_file_path).add_arguments(dst_blob_path).execute_azcopy_verify()
self.assertTrue(result)
# perform the single blob sync using azcopy.
result = util.Command("sync").add_arguments(src_blob_path).add_arguments(dst_blob_path). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
def util_test_1kb_blob_upload(self, use_oauth_session=False):
# Creating a single File Of size 1 KB
filename = "test1KB.txt"
file_path = util.create_test_file(filename, 1024)
# executing the azcopy command to upload the 1KB file.
src = file_path
if not use_oauth_session:
dest = util.get_resource_sas(filename)
dest_validate = dest
else:
dest = util.get_resource_from_oauth_container(filename)
dest_validate = util.get_resource_from_oauth_container_validate(
filename)
result = util.Command("copy").add_arguments(src).add_arguments(dest). \
add_flags("log-level", "info").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded blob.
# the resource local path should be the first argument for the azcopy validator.
# the resource sas should be the second argument for azcopy validator.
result = util.Command("testBlob").add_arguments(
file_path).add_arguments(dest_validate).execute_azcopy_verify()
self.assertTrue(result)
def test_upload_download_1kb_file_fullname(self):
# create file of size 1KB.
filename = "test_upload_download_1kb_file_fullname.txt"
file_path = util.create_test_file(filename, 1024)
# Upload 1KB file using azcopy.
src = file_path
dest = util.test_share_url
result = util.Command("copy").add_arguments(src).add_arguments(dest). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded file.
# the resource local path should be the first argument for the azcopy validator.
# the resource sas should be the second argument for azcopy validator.
resource_url = util.get_resource_sas_from_share(filename)
result = util.Command("testFile").add_arguments(file_path).add_arguments(resource_url).execute_azcopy_verify()
self.assertTrue(result)
time.sleep(5)
# downloading the uploaded file
src = util.get_resource_sas_from_share(filename)
dest = util.test_directory_path + "/test_1kb_file_download.txt"
result = util.Command("copy").add_arguments(src).add_arguments(dest).add_flags("log-level",
"info").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the downloaded file
result = util.Command("testFile").add_arguments(dest).add_arguments(src).execute_azcopy_verify()
self.assertTrue(result)
def test_block_size(self):
block_size = 4 * 1024 * 1024
# create file of size 63 Mb
filename = "test63Mb_blob.txt"
file_path = util.create_test_file(filename, 63 * 1024 * 1024)
# execute azcopy upload of 63 Mb file.
destination_sas = util.get_resource_sas(filename)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level", "info"). \
add_flags("block-size", str(block_size)).add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded blob
# calling the testBlob validator to verify whether blob has been successfully uploaded or not
if (63 * 1024 * 1024) % block_size == 0:
number_of_blocks = int(63 * 1024 * 1024 / block_size)
else:
number_of_blocks = int(63 * 1024 * 1024 / block_size) + 1
result = util.Command("testBlob").add_arguments(
file_path).add_arguments(destination_sas).add_flags(
"verify-block-size", "true").add_flags(
"number-blocks-or-pages",
str(number_of_blocks)).execute_azcopy_verify()
self.assertTrue(result)
def test_sync_single_file_to_file(self):
content_file_name_src = "test_1kb_file_sync_src.txt"
content_file_path_src = util.create_test_file(content_file_name_src,
1024)
content_file_name_dst = "test_1kb_file_sync_dst.txt"
content_file_path_dst = util.create_test_file(content_file_name_src,
1024)
# create source and destination files of size 1KB.
# make sure to create the destination first so that it has an older lmt
remote_src_file_path = util.get_resource_sas_from_share(
content_file_name_src)
remote_dst_file_path = util.get_resource_sas_from_share(
content_file_name_dst)
result = util.Command("cp").add_arguments(content_file_path_dst).add_arguments(remote_dst_file_path). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
result = util.Command("cp").add_arguments(content_file_path_src).add_arguments(remote_src_file_path). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# perform the single file sync using azcopy.
result = util.Command("sync").add_arguments(remote_src_file_path).add_arguments(remote_dst_file_path). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# verifying the sync worked, both remote source and destination should be identical to the local source
result = util.Command("testFile").add_arguments(
content_file_path_src).add_arguments(
remote_src_file_path).execute_azcopy_verify()
self.assertTrue(result)
result = util.Command("testFile").add_arguments(
content_file_path_src).add_arguments(
remote_dst_file_path).execute_azcopy_verify()
self.assertTrue(result)
def test_blobfs_upload_100_1Kb_file(
explictFromTo=False,
forceOAuthLogin=False,
tenantID="",
aadEndpoint=""):
# create dir with 100 1KB files inside it
dir_name = "dir_blobfs_100_1K"
dir_n_file_path = util.create_test_n_files(1024, 100, dir_name)
# Upload the directory with 100 files inside it
cmd = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info").add_flags("recursive","true")
util.process_oauth_command(
cmd,
"LocalBlobFS" if explictFromTo else "",
forceOAuthLogin,
tenantID,
aadEndpoint)
if forceOAuthLogin:
result = cmd.execute_azcopy_command_interactive()
else:
result = cmd.execute_azcopy_copy_command()
if not result:
print("test_blobfs_upload_100_1Kb_file failed uploading the dir ", dir_name, " to the filesystem")
sys.exit(1)
# Validate the uploaded directory
dirUrl = util.test_bfs_account_url + dir_name
result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl).\
add_flags("is-object-dir", "true").execute_azcopy_verify()
if not result:
print("test_blobfs_upload_100_1Kb_file failed while validating the file upload")
sys.exit(1)
print("test_blobfs_upload_100_1Kb_file successfully passed")
def util_test_blobfs_download_100_1Kb_file(self, explictFromTo=False):
# create dir with 100 1KB files inside it
dir_name = "dir_blobfs_d_100_1K"
dir_n_file_path = util.create_test_n_files(1024, 100, dir_name)
# Upload the directory with 100 files inside it
cmd = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info").add_flags("recursive","true")
util.process_oauth_command(cmd, "LocalBlobFS" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Validate the uploaded directory
dirUrl = util.test_bfs_account_url + dir_name
result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
# delete the local directory created
try:
shutil.rmtree(dir_n_file_path)
except:
self.fail('error deleting the directory ' + dir_n_file_path)
# download the directory
cmd = util.Command("copy").add_arguments(dirUrl).add_arguments(util.test_directory_path).\
add_flags("log-level", "Info").add_flags("recursive", "true")
util.process_oauth_command(cmd, "BlobFSLocal" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# validate the downloaded directory
result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl).\
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
def recursive_download_blob(self):
# create directory and 5 files of 1KB inside that directory.
dir_name = "dir_" + str(10) + "_files"
dir1_path = util.create_test_n_files(1024, 5, dir_name)
# upload the directory to container through azcopy with recursive set to true.
result = util.Command("copy").add_arguments(dir1_path).add_arguments(util.test_container_url).\
add_flags("log-level","info").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# verify the uploaded file.
destination_sas = util.get_resource_sas(dir_name)
result = util.Command("testBlob").add_arguments(dir1_path).add_arguments(destination_sas).\
add_flags("is-object-dir","true").execute_azcopy_verify()
self.assertTrue(result)
try:
shutil.rmtree(dir1_path)
except OSError as e:
self.fail('error removing the file ' + dir1_path)
# downloading the directory created from container through azcopy with recursive flag to true.
result = util.Command("copy").add_arguments(destination_sas).add_arguments(util.test_directory_path).add_flags(
"log-level", "info").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# verify downloaded blob.
result = util.Command("testBlob").add_arguments(dir1_path).add_arguments(destination_sas).\
add_flags("is-object-dir","true").execute_azcopy_verify()
self.assertTrue(result)
def util_test_blobfs_download_1Kb_file(self, explictFromTo=False):
# create file of size 1KB
filename = "test_blob_d_1kb_file.txt"
file_path = util.create_test_file(filename, 1024)
# upload the file using Azcopy
cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \
add_flags("log-level", "Info")
util.process_oauth_command(cmd, "LocalBlobFS" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Validate the file uploaded file
fileUrl = util.test_bfs_account_url + filename
result = util.Command("testBlobFS").add_arguments(
file_path).add_arguments(fileUrl).execute_azcopy_verify()
self.assertTrue(result)
# delete the file locally
try:
os.remove(file_path)
except:
self.fail('error deleting the file ' + file_path)
# download the file using Azcopy
cmd = util.Command("copy").add_arguments(fileUrl).add_arguments(util.test_directory_path). \
add_flags("log-level", "Info")
util.process_oauth_command(cmd, "BlobFSLocal" if explictFromTo else "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# validate the downloaded file
result = util.Command("testBlobFS").add_arguments(
file_path).add_arguments(fileUrl).execute_azcopy_verify()
self.assertTrue(result)
def util_test_download_1kb_blob_with_oauth(self):
# create file of size 1KB.
filename = "test_1kb_blob_upload.txt"
file_path = util.create_test_file(filename, 1024)
# Upload 1KB file using azcopy.
src = file_path
dest = util.test_oauth_container_url
cmd = util.Command("copy").add_arguments(src).add_arguments(dest). \
add_flags("log-level", "info").add_flags("recursive", "true")
util.process_oauth_command(cmd, "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded blob.
# the resource local path should be the first argument for the azcopy validator.
# the resource sas should be the second argument for azcopy validator.
dest_validate = util.get_resource_from_oauth_container_validate(filename)
result = util.Command("testBlob").add_arguments(file_path).add_arguments(dest_validate).execute_azcopy_verify()
self.assertTrue(result)
# downloading the uploaded file
src = util.get_resource_from_oauth_container(filename)
src_validate = util.get_resource_from_oauth_container_validate(filename)
dest = util.test_directory_path + "/test_1kb_blob_download.txt"
cmd = util.Command("copy").add_arguments(src).add_arguments(dest).add_flags("log-level", "info")
util.process_oauth_command(cmd, "")
result = cmd.execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the downloaded blob
result = util.Command("testBlob").add_arguments(dest).add_arguments(src_validate).execute_azcopy_verify()
self.assertTrue(result)
def test_login_with_default():
# execute the azcopy login.
output = util.Command("login").execute_azcopy_command_interactive()
if output is None:
print("error login")
print("test_login_with_default test failed")
return
# for windows, further check access token file, for other os, report success if login succeeded.
if os.name == 'nt':
# get the job Id of new job started by parsing the azcopy console output.
output = util.Command("info").add_arguments(
"AzCopyAppPath").execute_azcopy_info()
if output is None:
print("error get info")
print(
"test_login_with_default test internal error, fail to validate login"
)
token_file_path = os.path.join(output, "AccessToken.json")
if not os.path.isfile(token_file_path):
print("cannot find cached AccessToken.json")
print("test_login_with_default test failed")
return
# check access token should be refreshed. 5 minutes should be enough for manual operations.
if time.time() - os.stat(token_file_path).st_mtime < 30:
print("test_login_with_default passed successfully")
else:
print("test_login_with_default test failed")
else:
print("test_login_with_default passed successfully")
def test_blob_download_63mb_in_4mb(self):
# create file of 63mb
file_name = "test_63mb_in4mb_upload.txt"
file_path = util.create_test_file(file_name, 63 * 1024 * 1024)
# uploading file through azcopy with flag block-size set to 4194304 i.e 4mb
destination_sas = util.get_resource_sas(file_name)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).\
add_flags("log-level","info").add_flags("block-size", "4194304").execute_azcopy_copy_command()
self.assertTrue(result)
# verify the uploaded file.
result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas).execute_azcopy_verify()
self.assertTrue(result)
# downloading the created parallely in blocks of 4mb file through azcopy.
download_file = util.test_directory_path + "/test_63mb_in4mb_download.txt"
result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file)\
.add_flags("log-level","info").add_flags("block-size", "4194304").execute_azcopy_copy_command()
self.assertTrue(result)
# verify the downloaded file
result = util.Command("testBlob").add_arguments(download_file).add_arguments(
destination_sas).execute_azcopy_verify()
self.assertTrue(result)
def test_logout():
print("test_logout")
# execute the azcopy login.
output = util.Command("logout").execute_azcopy_copy_command_get_output()
if output is None:
print("error logout")
print("test_logout test failed")
return
# for windows, further check access token file, for other os, report success if login succeeded.
if os.name == 'nt':
# get the job Id of new job started by parsing the azcopy console output.
output = util.Command("info").add_arguments(
"AzCopyAppPath").execute_azcopy_info()
if output is None:
print("error get info")
print("test_logout test internal error, fail to validate logout")
print("test_logout AzCopyAppPath detected ", output)
token_file_path = os.path.join(output, "AccessToken.json")
if os.path.isfile(token_file_path):
print("find cached AccessToken.json after logout")
print("test_logout test failed")
else:
print("test_logout passed successfully")
else:
print("test_logout passed successfully")
def util_test_n_1kb_file_in_dir_upload_to_azure_directory(
self, number_of_files, recursive):
# create dir dir_n_files and 1 kb files inside the dir.
dir_name = "dir_" + str(number_of_files) + "_files"
sub_dir_name = "dir_subdir_" + str(number_of_files) + "_files"
# create n test files in dir
src_dir = util.create_test_n_files(1024, number_of_files, dir_name)
# create n test files in subdir, subdir is contained in dir
util.create_test_n_files(1024, number_of_files,
os.path.join(dir_name, sub_dir_name))
# prepare destination directory.
# TODO: note azcopy v2 currently only support existing directory and share.
dest_azure_dir_name = "dest azure_dir_name"
dest_azure_dir = util.get_resource_sas_from_share(dest_azure_dir_name)
result = util.Command("create").add_arguments(dest_azure_dir).add_flags("serviceType", "File"). \
add_flags("resourceType", "Bucket").execute_azcopy_create()
self.assertTrue(result)
# execute azcopy command
result = util.Command("copy").add_arguments(src_dir).add_arguments(dest_azure_dir). \
add_flags("recursive", recursive).add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator.
dest_azure_dir_to_compare = util.get_resource_sas_from_share(
dest_azure_dir_name + "/" + dir_name)
result = util.Command("testFile").add_arguments(src_dir).add_arguments(dest_azure_dir_to_compare). \
add_flags("is-object-dir", "true").add_flags("is-recursive", recursive).execute_azcopy_verify()
self.assertTrue(result)
def test_blobfs_sas_download_100_1Kb_file(self):
# Create dir with 100 1kb files inside it
dir_name = "dir_blobfs_sas_d_100_1K"
dir_n_file_path = util.create_test_n_files(1024, 100, dir_name)
# Upload the directory with 100 files inside it
result = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_sas_account_url). \
add_flags("log-level", "Info").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# Validate the uploaded directory
dirUrl = util.get_resource_sas_from_bfs(dir_name)
dirUrl_nosas = util.test_bfs_account_url + dir_name
result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
# Delete the local files
try:
shutil.rmtree(dir_n_file_path)
except:
self.fail('error deleting the directory ' + dir_n_file_path)
# Download the directory
result = util.Command("copy").add_arguments(dirUrl).add_arguments(util.test_directory_path). \
add_flags("log-level", "Info").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# Validate the downloaded directory
result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
