def helperChgrp(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
"""
Test `zstash chgrp`.
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path)
print("Testing chgrp")
self.assertWorkspace()
GROUP = "e3sm"
for option in ["-v", ""]:
print("Running zstash chgrp {}".format(option))
cmd = "{}zstash chgrp {} -R {} {}".format(zstash_path, option,
GROUP, self.hpss_path)
output, err = run_cmd(cmd)
if use_hpss:
self.check_strings(cmd, output + err, [], ["ERROR"])
print("Now check that the files are in the {} group".format(
GROUP))
cmd = "hsi ls -l {}".format(self.hpss_path)
output, err = run_cmd(cmd)
expected_present = "e3sm"
else:
expected_present = "chgrp: HPSS is unavailable"
self.check_strings(cmd, output + err, expected_present, ["ERROR"])
def helperLsTarsUpdate(self,
test_name,
hpss_path,
zstash_path=ZSTASH_PATH):
"""
Test `zstash ls --tars` when the database was initially created without the tars table
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
# Create without a tar table -- simulate a user updating an existing database
self.create(use_hpss, zstash_path, no_tars_md5=True)
self.assertWorkspace()
os.chdir(self.test_dir)
print_starred("Testing zstash ls")
cmd = "{}zstash ls --hpss={}".format(zstash_path, self.hpss_path)
output, err = run_cmd(cmd)
# tars should not be listed
self.check_strings(cmd, output + err, [],
["tars table does not exist", ".tar", "ERROR"])
print_starred("Testing zstash ls --tars")
cmd = "{}zstash ls --tars --hpss={}".format(zstash_path,
self.hpss_path)
output, err = run_cmd(cmd)
# tars should not be listed
self.check_strings(cmd, output + err, ["tars table does not exist"],
[".tar", "ERROR"])
os.chdir(TOP_LEVEL)
# Updating should create the tars table
self.add_files(use_hpss, zstash_path)
os.chdir(self.test_dir)
print_starred("Testing zstash ls --tars")
cmd = "{}zstash ls --tars --hpss={}".format(zstash_path,
self.hpss_path)
output, err = run_cmd(cmd)
# tar should be listed now
self.check_strings(cmd, output + err, ["000001.tar"], ["ERROR"])
print_starred("Testing zstash ls --tars -l")
cmd = "{}zstash ls --tars -l --hpss={}".format(zstash_path,
self.hpss_path)
output, err = run_cmd(cmd)
self.check_strings(cmd, output + err, ["000001.tar"], ["ERROR"])
os.chdir(TOP_LEVEL)
def helperCheckVerboseMismatch(self,
test_name,
hpss_path,
zstash_path=ZSTASH_PATH):
"""
Test `zstash check` with MD5 mismatch errors.
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path)
self.add_files(use_hpss, zstash_path)
self.extract(use_hpss, zstash_path)
print_starred("Causing MD5 mismatch errors and checking the files.")
self.assertWorkspace()
os.chdir(self.test_dir)
shutil.copy("{}/index.db".format(self.cache),
"{}/index_old.db".format(self.cache))
print("Messing up the MD5 of all of the files with an even id.")
sqlite_cmd = [
"sqlite3",
"{}/index.db".format(self.cache),
"UPDATE files SET md5 = 0 WHERE id % 2 = 0;",
]
run_cmd(sqlite_cmd)
zstash_cmd = "{}zstash check -v --hpss={}".format(
zstash_path, self.hpss_path)
output, err = run_cmd(zstash_cmd)
# These files have an even `id` in the sqlite3 table.
expected_present = [
"md5 mismatch for: dir/file1.txt",
"md5 mismatch for: file3.txt",
"ERROR: 000001.tar",
"ERROR: 000004.tar",
"ERROR: 000002.tar",
]
# These files have an odd `id` in the sqlite3 table.
expected_absent = [
"ERROR: 000000.tar",
"ERROR: 000003.tar",
"ERROR: 000005.tar",
]
self.check_strings(zstash_cmd, output + err, expected_present,
expected_absent)
# Put the original index.db back.
os.remove("{}/index.db".format(self.cache))
shutil.copy("{}/index_old.db".format(self.cache),
"{}/index.db".format(self.cache))
os.chdir(TOP_LEVEL)
def helperCheckParallel(
self, test_name, hpss_path, zstash_path=ZSTASH_PATH, verbose=False
):
"""
Test `zstash check` in parallel.
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path)
self.add_files(use_hpss, zstash_path)
self.extract(use_hpss, zstash_path)
print_starred("Checking the files in parallel.")
self.assertWorkspace()
os.chdir(self.test_dir)
v_option = " -v" if verbose else ""
cmd = "{}zstash check{} --hpss={} --workers=3".format(
zstash_path, v_option, self.hpss_path
)
output, err = run_cmd(cmd)
os.chdir(TOP_LEVEL)
expected_present = [
"Checking file0.txt",
"Checking file0_hard.txt",
"Checking file0_soft.txt",
"Checking file_empty.txt",
"Checking dir/file1.txt",
"Checking empty_dir",
"Checking dir2/file2.txt",
"Checking file3.txt",
"Checking file4.txt",
"Checking file5.txt",
]
expected_absent = ["ERROR"]
self.check_strings(cmd, output + err, expected_present, expected_absent)
def helperLs(self,
test_name,
hpss_path,
cache=None,
zstash_path=ZSTASH_PATH):
"""
Test `zstash ls`.
"""
self.hpss_path = hpss_path
if cache:
# Override default cache
self.cache = cache
cache_option = " --cache={}".format(self.cache)
else:
cache_option = ""
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path, cache=self.cache)
self.assertWorkspace()
os.chdir(self.test_dir)
for option in ["", "-v", "-l"]:
print_starred("Testing zstash ls {}".format(option))
cmd = "{}zstash ls{} {} --hpss={}".format(zstash_path,
cache_option, option,
self.hpss_path)
output, err = run_cmd(cmd)
self.check_strings(cmd, output + err, ["file0.txt"], ["ERROR"])
os.chdir(TOP_LEVEL)
def helperCreateExcludeFile(self,
test_name,
hpss_path,
zstash_path=ZSTASH_PATH):
"""
Test `zstash --exclude`, excluding a file.
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
if use_hpss:
description_str = "Adding files to HPSS"
else:
description_str = "Adding files to local archive"
print_starred(description_str)
self.assertWorkspace()
self.writeExtraFiles()
excluded_files = "not_exclude_dir/file_b.txt"
cmd = "{}zstash create --exclude={} --hpss={} {}".format(
zstash_path, excluded_files, self.hpss_path, self.test_dir)
output, err = run_cmd(cmd)
expected_present = [
"Archiving exclude_dir/file_a.txt",
"Archiving not_exclude_dir/file_c.txt",
]
if use_hpss:
expected_present += ["Transferring file to HPSS"]
else:
expected_present += ["put: HPSS is unavailable"]
expected_absent = ["ERROR", "Archiving not_exclude_dir/file_b.txt"]
self.check_strings(cmd, output + err, expected_present,
expected_absent)
def helperCheck(self,
test_name,
hpss_path,
cache=None,
zstash_path=ZSTASH_PATH):
"""
Test `zstash check`.
"""
self.hpss_path = hpss_path
if cache:
# Override default cache
self.cache = cache
cache_option = " --cache={}".format(self.cache)
else:
cache_option = ""
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path, cache=self.cache)
self.add_files(use_hpss, zstash_path, cache=self.cache)
print_starred("Testing the checking functionality")
self.assertWorkspace()
os.chdir(self.test_dir)
cmd = "{}zstash check{} --hpss={}".format(zstash_path, cache_option,
self.hpss_path)
output, err = run_cmd(cmd)
expected_present = [
"Checking file0.txt",
"Checking file0_hard.txt",
"Checking file0_soft.txt",
"Checking file_empty.txt",
"Checking dir/file1.txt",
"Checking empty_dir",
"Checking dir2/file2.txt",
"Checking file3.txt",
"Checking file4.txt",
"Checking file5.txt",
]
expected_absent = ["ERROR"]
self.check_strings(cmd, output + err, expected_present,
expected_absent)
cmd = "{}zstash check{} -v --hpss={}".format(zstash_path, cache_option,
self.hpss_path)
output, err = run_cmd(cmd)
self.check_strings(cmd, output + err, expected_present,
expected_absent)
os.chdir(TOP_LEVEL)
def helperExtractKeep(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
"""
Test `zstash extract` with `--keep`.
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path)
self.add_files(use_hpss, zstash_path)
self.extract(use_hpss, zstash_path)
msg = "Deleting the extracted files and doing it again without verbose option, "
msg += "while making sure the tars are kept."
print_starred(msg)
self.assertWorkspace()
shutil.rmtree(self.test_dir)
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
if not use_hpss:
shutil.copytree(
"{}/{}/{}".format(TOP_LEVEL, self.backup_dir, self.cache),
self.copy_dir)
cmd = "{}zstash extract --hpss={} --keep".format(
zstash_path, self.hpss_path)
output, err = run_cmd(cmd)
if not compare(
os.listdir(self.cache),
[
"index.db",
"000000.tar",
"000001.tar",
"000002.tar",
"000003.tar",
"000004.tar",
],
):
error_message = "The zstash directory does not contain expected files.\nIt has: {}".format(
os.listdir(self.cache))
self.stop(error_message)
os.chdir(TOP_LEVEL)
expected_present = [
"Extracting file0.txt",
"Extracting file0_hard.txt",
"Extracting file0_soft.txt",
"Extracting file_empty.txt",
"Extracting dir/file1.txt",
"Extracting empty_dir",
"Extracting dir2/file2.txt",
"Extracting file3.txt",
"Extracting file4.txt",
"Extracting file5.txt",
]
if use_hpss:
expected_present.append("Transferring file from HPSS")
expected_absent = ["ERROR", "Not extracting"]
self.check_strings(cmd, output + err, expected_present,
expected_absent)
def helperCheckParallelKeepTars(
self, test_name, hpss_path, zstash_path=ZSTASH_PATH
):
"""
Test `zstash check` in parallel when hpss is set in `zstash create`.
"""
self.assertWorkspace()
self.hpss_path = hpss_path
self.setupDirs(test_name)
if self.hpss_path.lower() != "none":
keep_option = " --keep"
else:
keep_option = ""
# Run `zstash create`
run_cmd(
"{}zstash create --hpss={}{} --maxsize 128 {}".format(
zstash_path, self.hpss_path, keep_option, self.test_dir
)
)
files = os.listdir("{}/{}".format(self.test_dir, self.cache))
if not compare(files, ["000000.tar", "index.db"]):
error_message = (
"The zstash cache does not contain expected files.\nIt has: {}".format(
files
)
)
self.stop(error_message)
# Run `zstash check` without specifying hpss
os.chdir(self.test_dir)
run_cmd("{}zstash check{} --workers=2".format(zstash_path, keep_option))
os.chdir(TOP_LEVEL)
files = os.listdir("{}/{}".format(self.test_dir, self.cache))
if not compare(files, ["000000.tar", "index.db"]):
error_message = (
"The zstash cache does not contain expected files.\nIt has: {}".format(
files
)
)
self.stop(error_message)
def testCheckKeepTars(self):
"""
Test that `zstash check` does not delete tars if `--hpss=none`.
"""
print_in_box("testKeepTars")
if os.path.exists(self.test_dir):
shutil.rmtree(self.test_dir)
os.mkdir(self.test_dir)
write_file("{}/file1.txt".format(self.test_dir), "")
write_file("{}/file2.txt".format(self.test_dir), "")
self.hpss_path = "none"
zstash_path = ZSTASH_PATH
# Run `zstash create`
run_cmd("{}zstash create --hpss={} {}".format(zstash_path,
self.hpss_path,
self.test_dir))
files = os.listdir("{}/{}/".format(self.test_dir, self.cache))
if not compare(files, ["000000.tar", "index.db"]):
error_message = (
"The zstash cache does not contain expected files.\nIt has: {}"
.format(files))
self.stop(error_message)
os.chdir(self.test_dir)
# Delete txt files
run_cmd("rm file1.txt file2.txt")
# Run `zstash extract`
output, err = run_cmd("{}zstash extract --hpss={}".format(
zstash_path, self.hpss_path))
# Run `zstash check`
output, err = run_cmd("{}zstash check --hpss={}".format(
zstash_path, self.hpss_path))
self.assertEqualOrStop(
output + err,
"INFO: Opening tar archive {}/000000.tar\nINFO: Checking file1.txt\nINFO: Checking file2.txt\nINFO: No failures detected when checking the files.\n"
.format(self.cache),
)
# Check that tar and db files were not deleted
files = os.listdir("{}/".format(self.cache))
if not compare(files, ["000000.tar", "index.db"]):
error_message = (
"The zstash cache does not contain expected files.\nIt has: {}"
.format(files))
self.stop(error_message)
# Check that tar file is read-only
# https://stackoverflow.com/questions/1861836/checking-file-permissions-in-linux-with-python
stat = os.stat("{}/000000.tar".format(self.cache))
oct_mode = str(oct(stat.st_mode))[-3:]
# https://en.wikipedia.org/wiki/Chmod#Numerical_permissions
# Write mode is permitted when any of 2,3,6,7 are included
# That is, in binary, the numbers with middle digit of 1: 010, 011, 110, 111.
invalid_permissions = [2, 3, 6, 7]
# https://stackoverflow.com/questions/3697432/how-to-find-list-intersection
# Get all characters from `oct_mode` that are also in the `invalid_permissions` list.
intersection = [n for n in oct_mode if int(n) in invalid_permissions]
if intersection:
error_message = "oct_mode={} includes {}".format(
oct_mode, intersection)
self.stop(error_message)
os.chdir(TOP_LEVEL)
def helperLsTars(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
"""
Test `zstash ls --tars`
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path)
self.assertWorkspace()
os.chdir(self.test_dir)
print_starred("Testing zstash ls --tars")
cmd = "{}zstash ls --tars --hpss={}".format(zstash_path,
self.hpss_path)
output, err = run_cmd(cmd)
self.check_strings(cmd, output + err, ["000000.tar"], ["ERROR"])
print_starred("Testing zstash ls --tars -l")
cmd = "{}zstash ls --tars -l --hpss={}".format(zstash_path,
self.hpss_path)
output, err = run_cmd(cmd)
self.check_strings(cmd, output + err, ["000000.tar"], ["ERROR"])
os.chdir(TOP_LEVEL)
def helperUpdate(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
"""
Test `zstash update`.
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path)
print_starred(
"Running update on the newly created directory, nothing should happen"
)
self.assertWorkspace()
os.chdir(self.test_dir)
cmd = "{}zstash update -v --hpss={}".format(zstash_path,
self.hpss_path)
output, err = run_cmd(cmd)
os.chdir(TOP_LEVEL)
self.check_strings(cmd, output + err, ["Nothing to update"], ["ERROR"])
def helperExtractTars(tester,
test_name,
hpss_path,
worker_str="",
zstash_path=ZSTASH_PATH):
"""
Test `zstash extract --tars`
"""
tester.hpss_path = hpss_path
use_hpss = tester.setupDirs(test_name)
tester.create(use_hpss, zstash_path)
tester.add_files(use_hpss, zstash_path)
os.rename(tester.test_dir, tester.backup_dir)
os.mkdir(tester.test_dir)
os.chdir(tester.test_dir)
if not use_hpss:
shutil.copytree(
"{}/{}/{}".format(TOP_LEVEL, tester.backup_dir, tester.cache),
tester.copy_dir,
)
zstash_cmd = (
'{}zstash extract --hpss={} --tars="000001-00002,000003,000004-"{}'.
format(zstash_path, tester.hpss_path, worker_str))
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: Opening tar archive zstash/000004.tar",
"INFO: No failures detected when extracting the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present,
expected_absent)
os.chdir(TOP_LEVEL)
def helperUpdateDryRun(self,
test_name,
hpss_path,
zstash_path=ZSTASH_PATH):
"""
Test `zstash update --dry-run`.
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path)
print_starred("Testing update with an actual change")
self.assertWorkspace()
if not os.path.exists("{}/dir2".format(self.test_dir)):
os.mkdir("{}/dir2".format(self.test_dir))
write_file("{}/dir2/file2.txt".format(self.test_dir), "file2 stuff")
write_file("{}/dir/file1.txt".format(self.test_dir),
"file1 stuff with changes")
os.chdir(self.test_dir)
cmd = "{}zstash update --dry-run --hpss={}".format(
zstash_path, self.hpss_path)
output, err = run_cmd(cmd)
os.chdir(TOP_LEVEL)
expected_present = [
"List of files to be updated",
"dir/file1.txt",
"dir2/file2.txt",
]
# Make sure none of the old files or directories are moved.
expected_absent = [
"ERROR",
"file0",
"file_empty",
"empty_dir",
"INFO: Creating new tar archive",
]
self.check_strings(cmd, output + err, expected_present,
expected_absent)
def helperExtractParallel(self, test_name, hpss_path, zstash_path=ZSTASH_PATH):
"""
Test `zstash extract` in parallel.
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path)
self.add_files(use_hpss, zstash_path)
self.extract(use_hpss, zstash_path)
print_starred("Deleting the extracted files and doing it again in parallel.")
self.assertWorkspace()
shutil.rmtree(self.test_dir)
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
if not use_hpss:
shutil.copytree(
"{}/{}/{}".format(TOP_LEVEL, self.backup_dir, self.cache), self.copy_dir
)
cmd = "{}zstash extract -v --hpss={} --workers=3".format(
zstash_path, self.hpss_path
)
output, err = run_cmd(cmd)
os.chdir(TOP_LEVEL)
expected_present = [
"Extracting file0.txt",
"Extracting file0_hard.txt",
"Extracting file0_soft.txt",
"Extracting file_empty.txt",
"Extracting dir/file1.txt",
"Extracting empty_dir",
"Extracting dir2/file2.txt",
"Extracting file3.txt",
"Extracting file4.txt",
"Extracting file5.txt",
]
if use_hpss:
expected_present.append("Transferring file from HPSS")
expected_absent = ["ERROR", "Not extracting"]
self.check_strings(cmd, output + err, expected_present, expected_absent)
# Checking that the printing was done in order.
tar_order = []
console_output = output + err
for word in console_output.replace("\n", " ").split(" "):
if ".tar" in word:
word = word.replace("{}/".format(self.cache), "")
tar_order.append(word)
if tar_order != sorted(tar_order):
error_message = "The tars were printed in this order: {}\nWhen it should have been in this order: {}".format(
tar_order, sorted(tar_order)
)
self.stop(error_message)
# Run again, without verbose option.
shutil.rmtree(self.test_dir)
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
if not use_hpss:
shutil.copytree(
"{}/{}/{}".format(TOP_LEVEL, self.backup_dir, self.cache), self.copy_dir
)
cmd = "{}zstash extract --hpss={} --workers=3".format(
zstash_path, self.hpss_path
)
output, err = run_cmd(cmd)
os.chdir(TOP_LEVEL)
self.check_strings(cmd, output + err, expected_present, expected_absent)
# Checking that the printing was done in order.
tar_order = []
console_output = output + err
for word in console_output.replace("\n", " ").split(" "):
if ".tar" in word:
word = word.replace("{}/".format(self.cache), "")
tar_order.append(word)
if tar_order != sorted(tar_order):
error_message = "The tars were printed in this order: {}\nWhen it should have been in this order: {}".format(
tar_order, sorted(tar_order)
)
self.stop(error_message)
def helperCheckParallelVerboseMismatch(
self, test_name, hpss_path, zstash_path=ZSTASH_PATH
):
"""
Test `zstash check -v` in parallel with MD5 mismatch.
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path)
self.add_files(use_hpss, zstash_path)
self.extract(use_hpss, zstash_path)
print_starred("Causing MD5 mismatch errors and checking the files in parallel.")
self.assertWorkspace()
os.chdir(self.test_dir)
shutil.copy(
"{}/index.db".format(self.cache), "{}/index_old.db".format(self.cache)
)
print("Messing up the MD5 of all of the files with an even id.")
sqlite_cmd = [
"sqlite3",
"{}/index.db".format(self.cache),
"UPDATE files SET md5 = 0 WHERE id % 2 = 0;",
]
run_cmd(sqlite_cmd)
zstash_cmd = "{}zstash check -v --hpss={} --workers=3".format(
zstash_path, self.hpss_path
)
output, err = run_cmd(zstash_cmd)
# These files have an even `id` in the sqlite3 table.
expected_present = [
"md5 mismatch for: dir/file1.txt",
"md5 mismatch for: file3.txt",
"ERROR: 000001.tar",
"ERROR: 000004.tar",
"ERROR: 000002.tar",
]
# These files have an odd `id` in the sqlite3 table.
expected_absent = [
"ERROR: 000000.tar",
"ERROR: 000003.tar",
"ERROR: 000005.tar",
]
self.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Put the original index.db back.
os.remove("{}/index.db".format(self.cache))
shutil.copy(
"{}/index_old.db".format(self.cache), "{}/index.db".format(self.cache)
)
os.chdir(TOP_LEVEL)
print("Verifying the data from database with the actual files")
# Checksums from HPSS
sqlite_cmd = [
"sqlite3",
"{}/{}/index.db".format(self.test_dir, self.cache),
"SELECT md5, name FROM files;",
]
output_hpss, err_hpss = run_cmd(sqlite_cmd)
hpss_dict = {}
for checksum in output_hpss.split("\n"):
checksum = checksum.split("|")
if len(checksum) >= 2:
f_name = checksum[1]
f_hash = checksum[0]
hpss_dict[f_name] = f_hash
# Checksums from local files
find_cmd = "find {} ".format(self.backup_dir)
find_cmd += (
r"""-regex .*\.txt.* -exec md5sum {} + """ # Literal {}, not for formatting
)
output_local, err_local = run_cmd(find_cmd)
local_dict = {}
for checksum in output_local.split("\n"):
checksum = checksum.split(" ")
if len(checksum) >= 2:
f_name = checksum[1].split("/") # remove the backup_dir
f_name = "/".join(f_name[1:])
f_hash = checksum[0]
local_dict[f_name] = f_hash
print("filename|HPSS hash|local file hash")
for k in local_dict:
print("{}|{}|{}".format(k, hpss_dict[k], local_dict[k]))
def helperExtractVerbose(self,
test_name,
hpss_path,
zstash_path=ZSTASH_PATH):
"""
Test `zstash extract -v`.
"""
self.hpss_path = hpss_path
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path)
self.add_files(use_hpss, zstash_path)
self.extract(use_hpss, zstash_path)
print_starred(
"Testing that nothing happens when extracting a second time")
self.assertWorkspace()
os.chdir(self.test_dir)
cmd = "{}zstash extract -v --hpss={}".format(zstash_path,
self.hpss_path)
output, err = run_cmd(cmd)
if use_hpss:
# Check that self.copy_dir only contains `index.db`.
if not compare(os.listdir(self.copy_dir), ["index.db"]):
error_message = (
"The zstash directory should not have any tars.\nIt has: {}"
.format(os.listdir(self.copy_dir)))
self.stop(error_message)
os.chdir(TOP_LEVEL)
expected_present = [
"Not extracting file0.txt",
"Not extracting file0_hard.txt",
"Not extracting file_empty.txt",
"Not extracting dir/file1.txt",
"Not extracting dir2/file2.txt",
"Not extracting file3.txt",
"Not extracting file4.txt",
"Not extracting file5.txt",
]
expected_absent = [
"Not extracting file0_soft.txt", # It's okay to extract the symlinks.
"ERROR",
]
if use_hpss:
# It's okay to extract empty dirs.
expected_absent.append("Not extracting empty_dir")
self.check_strings(cmd, output + err, expected_present,
expected_absent)
msg = "Deleting the extracted files and doing it again, "
msg += "while making sure the tars are kept."
print(msg)
shutil.rmtree(self.test_dir)
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
if not use_hpss:
shutil.copytree(
"{}/{}/{}".format(TOP_LEVEL, self.backup_dir, self.cache),
self.copy_dir)
cmd = "{}zstash extract -v --hpss={} --keep".format(
zstash_path, self.hpss_path)
output, err = run_cmd(cmd)
# Check that self.copy_dir contains all expected files
if not compare(
os.listdir(self.copy_dir),
[
"index.db",
"000000.tar",
"000001.tar",
"000002.tar",
"000003.tar",
"000004.tar",
],
):
error_message = "The zstash directory does not contain expected files.\nIt has: {}".format(
os.listdir(self.copy_dir))
self.stop(error_message)
os.chdir(TOP_LEVEL)
expected_present = [
"Extracting file0.txt",
"Extracting file0_hard.txt",
"Extracting file0_soft.txt",
"Extracting file_empty.txt",
"Extracting dir/file1.txt",
"Extracting empty_dir",
"Extracting dir2/file2.txt",
"Extracting file3.txt",
"Extracting file4.txt",
"Extracting file5.txt",
]
if use_hpss:
expected_present.append("Transferring file from HPSS")
expected_absent = ["ERROR", "Not extracting"]
self.check_strings(cmd, output + err, expected_present,
expected_absent)
def helperCheckTars(
tester, test_name, hpss_path, worker_str="", zstash_path=ZSTASH_PATH
):
"""
Test `zstash check --tars`
"""
tester.hpss_path = hpss_path
use_hpss = tester.setupDirs(test_name)
tester.create(use_hpss, zstash_path)
tester.add_files(use_hpss, zstash_path)
tester.assertWorkspace()
os.chdir(tester.test_dir)
# Starting at 000001 until the end
zstash_cmd = '{}zstash check --hpss={} --tars="000001-"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: Opening tar archive zstash/000004.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Starting from the beginning to 00003 (included)
zstash_cmd = '{}zstash check --hpss={} --tars="-000003"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000000.tar",
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000004.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Specific range
zstash_cmd = '{}zstash check --hpss={} --tars="000001-000003"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
"INFO: Opening tar archive zstash/000004.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Selected tar files
zstash_cmd = '{}zstash check --hpss={} --tars="000001,000003"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000004.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Mix and match
zstash_cmd = (
'{}zstash check --hpss={} --tars="000001-00002,000003,000004-"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: Opening tar archive zstash/000004.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Ending with nonexistent tar
zstash_cmd = '{}zstash check --hpss={} --tars="000001-00007"{}'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = [
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: Opening tar archive zstash/000004.tar",
"INFO: No failures detected when checking the files.",
]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
"INFO: Opening tar archive zstash/000005.tar",
"INFO: Opening tar archive zstash/000006.tar",
"INFO: Opening tar archive zstash/000007.tar",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
# Ending with nonexistent tar
zstash_cmd = '{}zstash check --hpss={} --tars="000001-00003"{} file'.format(
zstash_path, tester.hpss_path, worker_str
)
output, err = run_cmd(zstash_cmd)
expected_present = ["ValueError: If --tars is used, <files> should not be listed."]
expected_absent = [
"INFO: Opening tar archive zstash/000000.tar",
"INFO: Opening tar archive zstash/000001.tar",
"INFO: Opening tar archive zstash/000002.tar",
"INFO: Opening tar archive zstash/000003.tar",
"INFO: Opening tar archive zstash/000004.tar",
"INFO: No failures detected when checking the files.",
]
tester.check_strings(zstash_cmd, output + err, expected_present, expected_absent)
os.chdir(TOP_LEVEL)