def test_sync_single_file_to_file(self):
content_file_name_src = "test_1kb_file_sync_src.txt"
content_file_path_src = util.create_test_file(content_file_name_src,
1024)
content_file_name_dst = "test_1kb_file_sync_dst.txt"
content_file_path_dst = util.create_test_file(content_file_name_src,
1024)
# create source and destination files of size 1KB.
# make sure to create the destination first so that it has an older lmt
remote_src_file_path = util.get_resource_sas_from_share(
content_file_name_src)
remote_dst_file_path = util.get_resource_sas_from_share(
content_file_name_dst)
result = util.Command("cp").add_arguments(content_file_path_dst).add_arguments(remote_dst_file_path). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
result = util.Command("cp").add_arguments(content_file_path_src).add_arguments(remote_src_file_path). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# perform the single file sync using azcopy.
result = util.Command("sync").add_arguments(remote_src_file_path).add_arguments(remote_dst_file_path). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# verifying the sync worked, both remote source and destination should be identical to the local source
result = util.Command("testFile").add_arguments(
content_file_path_src).add_arguments(
remote_src_file_path).execute_azcopy_verify()
self.assertTrue(result)
result = util.Command("testFile").add_arguments(
content_file_path_src).add_arguments(
remote_dst_file_path).execute_azcopy_verify()
self.assertTrue(result)
def test_copy_single_file_to_file(self):
content_file_name_src = "test_1kb_file_copy_src.txt"
content_file_path_src = util.create_test_file(content_file_name_src,
1024)
content_file_name_dst = "test_1kb_file_copy_dst.txt"
# create source file of size 1KB.
remote_src_file_path = util.get_resource_sas_from_share(
content_file_name_src)
remote_dst_file_path = util.get_resource_sas_from_share(
content_file_name_dst)
result = util.Command("cp").add_arguments(content_file_path_src).add_arguments(remote_src_file_path). \
add_flags("log-level", "debug").execute_azcopy_copy_command()
self.assertTrue(result)
# perform the single file copy using azcopy.
result = util.Command("copy").add_arguments(remote_src_file_path).add_arguments(remote_dst_file_path). \
add_flags("log-level", "debug").execute_azcopy_copy_command()
self.assertTrue(result)
# verifying the copy worked, both remote source and destination should be identical to the local source
result = util.Command("testFile").add_arguments(
content_file_path_src).add_arguments(
remote_src_file_path).execute_azcopy_verify()
self.assertTrue(result)
result = util.Command("testFile").add_arguments(
content_file_path_src).add_arguments(
remote_dst_file_path).execute_azcopy_verify()
self.assertTrue(result)
def util_test_n_1kb_file_in_dir_upload_to_azure_directory(
self, number_of_files, recursive):
# create dir dir_n_files and 1 kb files inside the dir.
dir_name = "dir_" + str(number_of_files) + "_files"
sub_dir_name = "dir_subdir_" + str(number_of_files) + "_files"
# create n test files in dir
src_dir = util.create_test_n_files(1024, number_of_files, dir_name)
# create n test files in subdir, subdir is contained in dir
util.create_test_n_files(1024, number_of_files,
os.path.join(dir_name, sub_dir_name))
# prepare destination directory.
# TODO: note azcopy v2 currently only support existing directory and share.
dest_azure_dir_name = "dest azure_dir_name"
dest_azure_dir = util.get_resource_sas_from_share(dest_azure_dir_name)
result = util.Command("create").add_arguments(dest_azure_dir).add_flags("serviceType", "File"). \
add_flags("resourceType", "Bucket").execute_azcopy_create()
self.assertTrue(result)
# execute azcopy command
result = util.Command("copy").add_arguments(src_dir).add_arguments(dest_azure_dir). \
add_flags("recursive", recursive).add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator.
dest_azure_dir_to_compare = util.get_resource_sas_from_share(
dest_azure_dir_name + "/" + dir_name)
result = util.Command("testFile").add_arguments(src_dir).add_arguments(dest_azure_dir_to_compare). \
add_flags("is-object-dir", "true").add_flags("is-recursive", recursive).execute_azcopy_verify()
self.assertTrue(result)
def test_upload_download_1kb_file_fullname(self):
# create file of size 1KB.
filename = "test_upload_download_1kb_file_fullname.txt"
file_path = util.create_test_file(filename, 1024)
# Upload 1KB file using azcopy.
src = file_path
dest = util.test_share_url
result = util.Command("copy").add_arguments(src).add_arguments(dest). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded file.
# the resource local path should be the first argument for the azcopy validator.
# the resource sas should be the second argument for azcopy validator.
resource_url = util.get_resource_sas_from_share(filename)
result = util.Command("testFile").add_arguments(file_path).add_arguments(resource_url).execute_azcopy_verify()
self.assertTrue(result)
time.sleep(5)
# downloading the uploaded file
src = util.get_resource_sas_from_share(filename)
dest = util.test_directory_path + "/test_1kb_file_download.txt"
result = util.Command("copy").add_arguments(src).add_arguments(dest).add_flags("log-level",
"info").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the downloaded file
result = util.Command("testFile").add_arguments(dest).add_arguments(src).execute_azcopy_verify()
self.assertTrue(result)
def test_copy_entire_dir_to_dir(self):
content_dir_name_src = "dir_file_copy_test_src"
content_dir_path_src = util.create_test_n_files(
1024, 10, content_dir_name_src)
content_dir_name_dst = "dir_file_copy_test_dst"
# create sub-directory inside directory
sub_dir_name = os.path.join(content_dir_name_src, "sub_dir_copy_test")
util.create_test_n_files(1024, 10, sub_dir_name)
# upload to the source
result = util.Command("copy").add_arguments(content_dir_path_src).add_arguments(util.test_share_url). \
add_flags("recursive", "true").add_flags("log-level", "debug").execute_azcopy_copy_command()
self.assertTrue(result)
# copy to destination
remote_src_dir_path = util.get_resource_sas_from_share(
content_dir_name_src)
remote_dst_dir_path = util.get_resource_sas_from_share(
content_dir_name_dst)
result = util.Command("copy").add_arguments(util.get_resource_sas_from_share(content_dir_name_src+"/*"))\
.add_arguments(remote_dst_dir_path).add_flags("log-level", "debug").add_flags("recursive", "true")\
.execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator to make sure the copy worked, both remote src and dst should match local src
result = util.Command("testFile").add_arguments(content_dir_path_src).add_arguments(remote_src_dir_path). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
result = util.Command("testFile").add_arguments(content_dir_path_src).add_arguments(remote_dst_dir_path). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
def util_test_n_1kb_file_in_dir_upload_download_azure_directory(
self, number_of_files, recursive):
# create dir dir_n_files and 1 kb files inside the dir.
dir_name = "util_test_n_1kb_file_in_dir_upload_download_azure_directory_" + recursive + "_" + str(
number_of_files) + "_files"
sub_dir_name = "dir_subdir_" + str(number_of_files) + "_files"
# create n test files in dir
src_dir = util.create_test_n_files(1024, number_of_files, dir_name)
# create n test files in subdir, subdir is contained in dir
util.create_test_n_files(1024, number_of_files,
os.path.join(dir_name, sub_dir_name))
# prepare destination directory.
# TODO: note azcopy v2 currently only support existing directory and share.
dest_azure_dir_name = "dest azure_dir_name"
dest_azure_dir = util.get_resource_sas_from_share(dest_azure_dir_name)
result = util.Command("create").add_arguments(dest_azure_dir).add_flags("serviceType", "File"). \
add_flags("resourceType", "Bucket").execute_azcopy_create()
self.assertTrue(result)
# execute azcopy command
result = util.Command("copy").add_arguments(src_dir).add_arguments(dest_azure_dir). \
add_flags("recursive", recursive).add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator.
dest_azure_dir_to_compare = util.get_resource_sas_from_share(
dest_azure_dir_name + "/" + dir_name)
result = util.Command("testFile").add_arguments(src_dir).add_arguments(dest_azure_dir_to_compare). \
add_flags("is-object-dir", "true").add_flags("is-recursive", recursive).execute_azcopy_verify()
self.assertTrue(result)
download_azure_src_dir = dest_azure_dir_to_compare
download_local_dest_dir = src_dir + "_download"
try:
if os.path.exists(download_local_dest_dir) and os.path.isdir(
download_local_dest_dir):
shutil.rmtree(download_local_dest_dir)
except:
print("catch error for removing " + download_local_dest_dir)
finally:
os.makedirs(download_local_dest_dir)
# downloading the directory created from azure file directory through azcopy with recursive flag to true.
result = util.Command("copy").add_arguments(download_azure_src_dir).add_arguments(
download_local_dest_dir).add_flags("log-level", "info"). \
add_flags("recursive", recursive).execute_azcopy_copy_command()
self.assertTrue(result)
# verify downloaded file.
# todo: ensure the comparing here
result = util.Command("testFile").add_arguments(os.path.join(download_local_dest_dir, dir_name)).add_arguments(
download_azure_src_dir). \
add_flags("is-object-dir", "true").add_flags("is-recursive", recursive).execute_azcopy_verify()
self.assertTrue(result)
def test_recursive_download_file(self):
# create directory and 5 files of 1KB inside that directory.
dir_name = "dir_" + str(10) + "_files"
dir1_path = util.create_test_n_files(1024, 5, dir_name)
# upload the directory to share through azcopy with recursive set to true.
result = util.Command("copy").add_arguments(dir1_path).add_arguments(util.test_share_url).add_flags("log-level",
"info").add_flags(
"recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# verify the uploaded file.
destination_sas = util.get_resource_sas_from_share(dir_name)
result = util.Command("testFile").add_arguments(dir1_path).add_arguments(destination_sas).add_flags("is-object-dir",
"true").execute_azcopy_verify()
self.assertTrue(result)
try:
shutil.rmtree(dir1_path)
except OSError as e:
self.fail("error removing the uploaded files. " + str(e))
# downloading the directory created from share through azcopy with recursive flag to true.
result = util.Command("copy").add_arguments(destination_sas).add_arguments(util.test_directory_path).add_flags(
"log-level", "info").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# verify downloaded file.
result = util.Command("testFile").add_arguments(dir1_path).add_arguments(destination_sas).add_flags("is-object-dir",
"true").execute_azcopy_verify()
self.assertTrue(result)
def test_file_download_63mb_in_4mb(self):
# create file of 63mb
file_name = "test_63mb_in4mb_upload.txt"
file_path = util.create_test_file(file_name, 63 * 1024 * 1024)
# uploading file through azcopy with flag block-size set to 4194304 i.e 4mb
destination_sas = util.get_resource_sas_from_share(file_name)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level",
"info").add_flags(
"block-size", "4194304").execute_azcopy_copy_command()
self.assertTrue(result)
# verify the uploaded file.
result = util.Command("testFile").add_arguments(file_path).add_arguments(destination_sas).execute_azcopy_verify()
self.assertTrue(result)
# downloading the created parallely in blocks of 4mb file through azcopy.
download_file = util.test_directory_path + "/test_63mb_in4mb_download.txt"
result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file).add_flags("log-level",
"info").add_flags(
"block-size", "4194304").execute_azcopy_copy_command()
self.assertTrue(result)
# verify the downloaded file
result = util.Command("testFile").add_arguments(download_file).add_arguments(
destination_sas).execute_azcopy_verify()
self.assertTrue(result)
def test_download_perserve_last_modified_time(self):
# create a file of 2KB
filename = "test_upload_preserve_last_mtime.txt"
file_path = util.create_test_file(filename, 2048)
# upload file through azcopy.
destination_sas = util.get_resource_sas_from_share(filename)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \
add_flags("log-level", "info").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded file
result = util.Command("testFile").add_arguments(file_path).add_arguments(destination_sas).execute_azcopy_verify()
self.assertTrue(result)
time.sleep(5)
# download file through azcopy with flag preserve-last-modified-time set to true
download_file_name = util.test_directory_path + "/test_download_preserve_last_mtime.txt"
result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file_name).add_flags("log-level",
"info").add_flags(
"preserve-last-modified-time", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the downloaded file and its modified with the modified time of file.
result = util.Command("testFile").add_arguments(download_file_name).add_arguments(destination_sas).add_flags(
"preserve-last-modified-time", "true").execute_azcopy_verify()
self.assertTrue(result)
def test_upload_download_1kb_file_wildcard_several_files(self):
# create file of size 1KB.
filename = "test_upload_download_1kb_file_wildcard_several_files.txt"
prefix = "test_upload_download_1kb_file_wildcard_several*"
file_path = util.create_test_file(filename, 1024)
wildcard_path = file_path.replace(filename, prefix)
# Upload 1KB file using azcopy.
result = util.Command("copy").add_arguments(wildcard_path).add_arguments(util.test_share_url). \
add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded file.
# the resource local path should be the first argument for the azcopy validator.
# the resource sas should be the second argument for azcopy validator.
resource_url = util.get_resource_sas_from_share(filename)
result = util.Command("testFile").add_arguments(file_path).add_arguments(resource_url).execute_azcopy_verify()
self.assertTrue(result)
time.sleep(5)
# downloading the uploaded file
src = util.get_resource_sas_from_share(filename)
wildcardSrc = util.get_resource_sas_from_share(prefix)
dest = util.test_directory_path + "/test_upload_download_1kb_file_wildcard_several_files"
try:
if os.path.exists(dest) and os.path.isdir(dest):
shutil.rmtree(dest)
except:
self.fail('error removing directory ' + dest)
finally:
os.makedirs(dest)
result = util.Command("copy").add_arguments(wildcardSrc).add_arguments(dest).add_flags("log-level",
"info").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the downloaded file
result = util.Command("testFile").add_arguments(os.path.join(dest, filename)).add_arguments(
src).execute_azcopy_verify()
self.assertTrue(result)
def util_test_n_1kb_file_in_dir_upload_download_share(
self, number_of_files):
# create dir dir_n_files and 1 kb files inside the dir.
dir_name = "dir_test_n_1kb_file_in_dir_upload_download_share_" + str(
number_of_files) + "_files"
sub_dir_name = "dir subdir_" + str(number_of_files) + "_files"
# create n test files in dir
src_dir = util.create_test_n_files(1024, number_of_files, dir_name)
# create n test files in subdir, subdir is contained in dir
util.create_test_n_files(1024, number_of_files,
os.path.join(dir_name, sub_dir_name))
# execute azcopy command
dest_share = util.test_share_url
result = util.Command("copy").add_arguments(src_dir).add_arguments(dest_share). \
add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator.
dest_azure_dir = util.get_resource_sas_from_share(dir_name)
result = util.Command("testFile").add_arguments(src_dir).add_arguments(dest_azure_dir). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
download_azure_src_dir = dest_azure_dir
download_local_dest_dir = src_dir + "_download"
try:
if os.path.exists(download_local_dest_dir) and os.path.isdir(
download_local_dest_dir):
shutil.rmtree(download_local_dest_dir)
except:
self.fail("error removing " + download_local_dest_dir)
finally:
os.makedirs(download_local_dest_dir)
# downloading the directory created from azure file share through azcopy with recursive flag to true.
result = util.Command("copy").add_arguments(
download_azure_src_dir).add_arguments(
download_local_dest_dir).add_flags(
"log-level",
"info").add_flags("recursive",
"true").execute_azcopy_copy_command()
self.assertTrue(result)
# verify downloaded file.
result = util.Command("testFile").add_arguments(
os.path.join(
download_local_dest_dir,
dir_name)).add_arguments(download_azure_src_dir).add_flags(
"is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)
def util_test_file_upload_size_n_fullname(self, sizeInBytes=1):
# create the test file.
file_name = "test_file_upload_%dB_fullname.vhd" % (sizeInBytes)
file_path = util.create_test_file(file_name, sizeInBytes)
# execute azcopy upload.
destination = util.get_resource_sas_from_share(file_name)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination).add_flags("log-level", "debug"). \
add_flags("block-size-mb", "4").execute_azcopy_copy_command()
self.assertTrue(result)
# execute validator.
result = util.Command("testFile").add_arguments(file_path).add_arguments(destination).execute_azcopy_verify()
self.assertTrue(result)
def test_file_upload_1mb_wildcard(self):
# create the test file.
file_name = "test_file_upload_1mb_wildcard.vhd"
file_path = util.create_test_file(file_name, 1024 * 1024)
# execute azcopy upload.
destination = util.get_resource_sas_from_share(file_name)
wildcard_path = file_path.replace(file_name, "test_file_upload_1mb_wildcard*")
result = util.Command("copy").add_arguments(wildcard_path).add_arguments(util.test_share_url).add_flags("log-level", "info"). \
add_flags("block-size-mb", "4").execute_azcopy_copy_command()
self.assertTrue(result)
# execute validator.
result = util.Command("testFile").add_arguments(file_path).add_arguments(destination).execute_azcopy_verify()
self.assertTrue(result)
def test_file_upload_1mb_fullname(self):
# create the test file.
file_name = "test_file_upload_1mb_fullname.vhd"
file_path = util.create_test_file(file_name, 1024 * 1024)
# execute azcopy upload.
destination = util.get_resource_sas_from_share(file_name)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination).add_flags("log-level", "info"). \
add_flags("block-size", "4194304").execute_azcopy_copy_command()
self.assertTrue(result)
# execute validator.
result = util.Command("testFile").add_arguments(
file_path).add_arguments(destination).execute_azcopy_verify()
self.assertTrue(result)
def test_guess_mime_type(self):
# create a test html file
filename = "test_guessmimetype.html"
file_path = util.create_test_html_file(filename)
# execute azcopy upload of html file.
destination_sas = util.get_resource_sas_from_share(filename)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level", "info"). \
add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator to verify the content-type.
result = util.Command("testFile").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level",
"info"). \
add_flags("recursive", "true")
self.assertTrue(result)
def test_file_range_for_complete_sparse_file(self):
# create test file.
file_name = "sparse_file.vhd"
file_path = util.create_complete_sparse_file(file_name, 4 * 1024 * 1024)
# execute azcopy file upload.
destination_sas = util.get_resource_sas_from_share(file_name)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level", "info"). \
add_flags("block-size-mb", "4").execute_azcopy_copy_command()
self.assertTrue(result)
# execute validator.
# no of ranges should be 0 for the empty sparse file.
result = util.Command("testFile").add_arguments(file_path).add_arguments(destination_sas).add_flags(
"verify-block-size", "true").add_flags("number-blocks-or-pages", "0").execute_azcopy_verify()
self.assertTrue(result)
def test_1GB_file_upload(self):
# create 1Gb file
filename = "test_1G_file.txt"
file_path = util.create_test_file(filename, 1 * 1024 * 1024 * 1024)
# execute azcopy upload.
destination_sas = util.get_resource_sas_from_share(filename)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level", "info"). \
add_flags("block-size-mb", "100").add_flags("recursive", "true").execute_azcopy_copy_command()
self.assertTrue(result)
# Verifying the uploaded file.
# adding local file path as first argument.
# adding file sas as local argument.
# calling the testFile validator to verify whether file has been successfully uploaded or not.
result = util.Command("testFile").add_arguments(file_path).add_arguments(destination_sas).execute_azcopy_verify()
self.assertTrue(result)
def test_file_upload_partial_sparse_file(self):
# create test file.
file_name = "test_partial_sparse_file.vhd"
file_path = util.create_partial_sparse_file(file_name, 16 * 1024 * 1024)
# execute azcopy file upload.
destination_sas = util.get_resource_sas_from_share(file_name)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level", "info"). \
add_flags("block-size-mb", "4").execute_azcopy_copy_command()
self.assertTrue(result)
# number of range for partial sparse created above will be (size/2)
number_of_ranges = int((16 * 1024 * 1024 / (4 * 1024 * 1024)) / 2)
# execute validator to verify the number of range for uploaded file.
result = util.Command("testFile").add_arguments(file_path).add_arguments(destination_sas). \
add_flags("verify-block-size", "true"). \
add_flags("number-blocks-or-pages", str(number_of_ranges)).execute_azcopy_verify()
self.assertTrue(result)
def test_metaData_content_encoding_content_type(self):
# create 2kb file test_mcect.txt
filename = "test_mcect.txt"
file_path = util.create_test_file(filename, 2048)
# execute azcopy upload command.
destination_sas = util.get_resource_sas_from_share(filename)
result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \
add_flags("log-level", "info").add_flags("recursive", "true").add_flags("metadata",
"author=jiac;viewport=width;description=test file"). \
add_flags("content-type", "testctype").add_flags("content-encoding", "testenc").add_flags("no-guess-mime-type",
"true").execute_azcopy_copy_command()
self.assertTrue(result)
# execute azcopy validate order.
# adding the source in validator as first argument.
# adding the destination in validator as second argument.
result = util.Command("testFile").add_arguments(file_path).add_arguments(destination_sas).add_flags("metadata",
"author=jiac;viewport=width;description=test file"). \
add_flags("content-type", "testctype").add_flags("content-encoding", "testenc").add_flags("no-guess-mime-type",
"true").execute_azcopy_verify()
self.assertTrue(result)
def util_test_n_1kb_file_in_dir_upload_to_share(self, number_of_files):
# create dir dir_n_files and 1 kb files inside the dir.
dir_name = "dir_" + str(number_of_files) + "_files"
sub_dir_name = "dir subdir_" + str(number_of_files) + "_files"
# create n test files in dir
src_dir = util.create_test_n_files(1024, number_of_files, dir_name)
# create n test files in subdir, subdir is contained in dir
util.create_test_n_files(1024, number_of_files, os.path.join(dir_name, sub_dir_name))
# execute azcopy command
dest_share = util.test_share_url
result = util.Command("copy").add_arguments(src_dir).add_arguments(dest_share). \
add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command()
self.assertTrue(result)
# execute the validator.
dest_azure_dir = util.get_resource_sas_from_share(dir_name)
result = util.Command("testFile").add_arguments(src_dir).add_arguments(dest_azure_dir). \
add_flags("is-object-dir", "true").execute_azcopy_verify()
self.assertTrue(result)