def testCheckKeepTars(self): """ Test that `zstash check` does not delete tars if `--hpss=none`. """ print_in_box("testKeepTars") if os.path.exists(self.test_dir): shutil.rmtree(self.test_dir) os.mkdir(self.test_dir) write_file("{}/file1.txt".format(self.test_dir), "") write_file("{}/file2.txt".format(self.test_dir), "") self.hpss_path = "none" zstash_path = ZSTASH_PATH # Run `zstash create` run_cmd("{}zstash create --hpss={} {}".format(zstash_path, self.hpss_path, self.test_dir)) files = os.listdir("{}/{}/".format(self.test_dir, self.cache)) if not compare(files, ["000000.tar", "index.db"]): error_message = ( "The zstash cache does not contain expected files.\nIt has: {}" .format(files)) self.stop(error_message) os.chdir(self.test_dir) # Delete txt files run_cmd("rm file1.txt file2.txt") # Run `zstash extract` output, err = run_cmd("{}zstash extract --hpss={}".format( zstash_path, self.hpss_path)) # Run `zstash check` output, err = run_cmd("{}zstash check --hpss={}".format( zstash_path, self.hpss_path)) self.assertEqualOrStop( output + err, "INFO: Opening tar archive {}/000000.tar\nINFO: Checking file1.txt\nINFO: Checking file2.txt\nINFO: No failures detected when checking the files.\n" .format(self.cache), ) # Check that tar and db files were not deleted files = os.listdir("{}/".format(self.cache)) if not compare(files, ["000000.tar", "index.db"]): error_message = ( "The zstash cache does not contain expected files.\nIt has: {}" .format(files)) self.stop(error_message) # Check that tar file is read-only # https://stackoverflow.com/questions/1861836/checking-file-permissions-in-linux-with-python stat = os.stat("{}/000000.tar".format(self.cache)) oct_mode = str(oct(stat.st_mode))[-3:] # https://en.wikipedia.org/wiki/Chmod#Numerical_permissions # Write mode is permitted when any of 2,3,6,7 are included # That is, in binary, the numbers with middle digit of 1: 010, 011, 110, 111. invalid_permissions = [2, 3, 6, 7] # https://stackoverflow.com/questions/3697432/how-to-find-list-intersection # Get all characters from `oct_mode` that are also in the `invalid_permissions` list. intersection = [n for n in oct_mode if int(n) in invalid_permissions] if intersection: error_message = "oct_mode={} includes {}".format( oct_mode, intersection) self.stop(error_message) os.chdir(TOP_LEVEL)
def helperExtractCache(self, test_name, hpss_path, zstash_path=ZSTASH_PATH): """ Test `zstash extract --cache`. """ self.hpss_path = hpss_path self.cache = "my_cache" use_hpss = self.setupDirs(test_name) if not use_hpss: self.copy_dir = self.cache self.create(use_hpss, zstash_path, cache=self.cache) self.add_files(use_hpss, zstash_path, cache=self.cache) self.extract(use_hpss, zstash_path, cache=self.cache) files = os.listdir("{}/{}".format(self.test_dir, self.cache)) if use_hpss: expected_files = ["index.db"] else: expected_files = [ "index.db", "000003.tar", "000004.tar", "000000.tar", "000001.tar", "000002.tar", ] if not compare(files, expected_files): error_message = ( "The zstash cache does not contain expected files.\nIt has: {}" .format(files)) self.stop(error_message)
def helperUpdateKeep(self, test_name, hpss_path, zstash_path=ZSTASH_PATH): """ Test `zstash update --keep`. """ self.hpss_path = hpss_path use_hpss = self.setupDirs(test_name) # Not keeping the tar from `create`. self.create(use_hpss, zstash_path) self.add_files(use_hpss, zstash_path, keep=True) files = os.listdir("{}/{}".format(self.test_dir, self.cache)) if use_hpss: expected_files = [ "index.db", "000003.tar", "000004.tar", "000001.tar", "000002.tar", ] else: expected_files = [ "index.db", "000003.tar", "000004.tar", "000000.tar", "000001.tar", "000002.tar", ] if not compare(files, expected_files): error_message = ( "The zstash cache does not contain expected files.\nIt has: {}" .format(files)) self.stop(error_message) os.chdir(TOP_LEVEL)
def helperExtractKeep(self, test_name, hpss_path, zstash_path=ZSTASH_PATH): """ Test `zstash extract` with `--keep`. """ self.hpss_path = hpss_path use_hpss = self.setupDirs(test_name) self.create(use_hpss, zstash_path) self.add_files(use_hpss, zstash_path) self.extract(use_hpss, zstash_path) msg = "Deleting the extracted files and doing it again without verbose option, " msg += "while making sure the tars are kept." print_starred(msg) self.assertWorkspace() shutil.rmtree(self.test_dir) os.mkdir(self.test_dir) os.chdir(self.test_dir) if not use_hpss: shutil.copytree( "{}/{}/{}".format(TOP_LEVEL, self.backup_dir, self.cache), self.copy_dir) cmd = "{}zstash extract --hpss={} --keep".format( zstash_path, self.hpss_path) output, err = run_cmd(cmd) if not compare( os.listdir(self.cache), [ "index.db", "000000.tar", "000001.tar", "000002.tar", "000003.tar", "000004.tar", ], ): error_message = "The zstash directory does not contain expected files.\nIt has: {}".format( os.listdir(self.cache)) self.stop(error_message) os.chdir(TOP_LEVEL) expected_present = [ "Extracting file0.txt", "Extracting file0_hard.txt", "Extracting file0_soft.txt", "Extracting file_empty.txt", "Extracting dir/file1.txt", "Extracting empty_dir", "Extracting dir2/file2.txt", "Extracting file3.txt", "Extracting file4.txt", "Extracting file5.txt", ] if use_hpss: expected_present.append("Transferring file from HPSS") expected_absent = ["ERROR", "Not extracting"] self.check_strings(cmd, output + err, expected_present, expected_absent)
def helperCheckParallelKeepTars( self, test_name, hpss_path, zstash_path=ZSTASH_PATH ): """ Test `zstash check` in parallel when hpss is set in `zstash create`. """ self.assertWorkspace() self.hpss_path = hpss_path self.setupDirs(test_name) if self.hpss_path.lower() != "none": keep_option = " --keep" else: keep_option = "" # Run `zstash create` run_cmd( "{}zstash create --hpss={}{} --maxsize 128 {}".format( zstash_path, self.hpss_path, keep_option, self.test_dir ) ) files = os.listdir("{}/{}".format(self.test_dir, self.cache)) if not compare(files, ["000000.tar", "index.db"]): error_message = ( "The zstash cache does not contain expected files.\nIt has: {}".format( files ) ) self.stop(error_message) # Run `zstash check` without specifying hpss os.chdir(self.test_dir) run_cmd("{}zstash check{} --workers=2".format(zstash_path, keep_option)) os.chdir(TOP_LEVEL) files = os.listdir("{}/{}".format(self.test_dir, self.cache)) if not compare(files, ["000000.tar", "index.db"]): error_message = ( "The zstash cache does not contain expected files.\nIt has: {}".format( files ) ) self.stop(error_message)
def helperCreateKeep(self, test_name, hpss_path, zstash_path=ZSTASH_PATH): """ Test `zstash create --keep`. """ self.hpss_path = hpss_path use_hpss = self.setupDirs(test_name) self.create(use_hpss, zstash_path, keep=True) files = os.listdir("{}/{}".format(self.test_dir, self.cache)) if not compare(files, ["index.db", "000000.tar"]): error_message = ( "The zstash cache does not contain expected files.\nIt has: {}" .format(files)) self.stop(error_message) os.chdir(TOP_LEVEL)
def helperExtractVerbose(self, test_name, hpss_path, zstash_path=ZSTASH_PATH): """ Test `zstash extract -v`. """ self.hpss_path = hpss_path use_hpss = self.setupDirs(test_name) self.create(use_hpss, zstash_path) self.add_files(use_hpss, zstash_path) self.extract(use_hpss, zstash_path) print_starred( "Testing that nothing happens when extracting a second time") self.assertWorkspace() os.chdir(self.test_dir) cmd = "{}zstash extract -v --hpss={}".format(zstash_path, self.hpss_path) output, err = run_cmd(cmd) if use_hpss: # Check that self.copy_dir only contains `index.db`. if not compare(os.listdir(self.copy_dir), ["index.db"]): error_message = ( "The zstash directory should not have any tars.\nIt has: {}" .format(os.listdir(self.copy_dir))) self.stop(error_message) os.chdir(TOP_LEVEL) expected_present = [ "Not extracting file0.txt", "Not extracting file0_hard.txt", "Not extracting file_empty.txt", "Not extracting dir/file1.txt", "Not extracting dir2/file2.txt", "Not extracting file3.txt", "Not extracting file4.txt", "Not extracting file5.txt", ] expected_absent = [ "Not extracting file0_soft.txt", # It's okay to extract the symlinks. "ERROR", ] if use_hpss: # It's okay to extract empty dirs. expected_absent.append("Not extracting empty_dir") self.check_strings(cmd, output + err, expected_present, expected_absent) msg = "Deleting the extracted files and doing it again, " msg += "while making sure the tars are kept." print(msg) shutil.rmtree(self.test_dir) os.mkdir(self.test_dir) os.chdir(self.test_dir) if not use_hpss: shutil.copytree( "{}/{}/{}".format(TOP_LEVEL, self.backup_dir, self.cache), self.copy_dir) cmd = "{}zstash extract -v --hpss={} --keep".format( zstash_path, self.hpss_path) output, err = run_cmd(cmd) # Check that self.copy_dir contains all expected files if not compare( os.listdir(self.copy_dir), [ "index.db", "000000.tar", "000001.tar", "000002.tar", "000003.tar", "000004.tar", ], ): error_message = "The zstash directory does not contain expected files.\nIt has: {}".format( os.listdir(self.copy_dir)) self.stop(error_message) os.chdir(TOP_LEVEL) expected_present = [ "Extracting file0.txt", "Extracting file0_hard.txt", "Extracting file0_soft.txt", "Extracting file_empty.txt", "Extracting dir/file1.txt", "Extracting empty_dir", "Extracting dir2/file2.txt", "Extracting file3.txt", "Extracting file4.txt", "Extracting file5.txt", ] if use_hpss: expected_present.append("Transferring file from HPSS") expected_absent = ["ERROR", "Not extracting"] self.check_strings(cmd, output + err, expected_present, expected_absent)