def __init__(self):
self.test_data_folder_structure = "/home/samuel/wksp/swestore/ArchiveOnSwestoreNG/tests/testdata/for_drophere_folder"
self.test_proj_name = "b2222222"
self.proj_folder = "/home/samuel/bubofake_auto/proj"
self.drop_folder = os.path.join(self.proj_folder, self.test_proj_name, "swestore/drophere2archive")
self.uploadcache_folder = "/home/samuel/bubofake_auto/uploadcache"
self.uploadcache_proj_folder = os.path.join(self.uploadcache_folder, self.test_proj_name)
self.confirm_folder = os.path.join(self.proj_folder, self.test_proj_name, "swestore/moved2swestore")
self.verification_folder = "/home/samuel/bubofake_auto/verify/"
self.verification_proj_folder = self.verification_folder + self.test_proj_name
self.uploadcache_proj_folder = os.path.join(self.uploadcache_folder, self.test_proj_name)
self.app = Application(configfile_path="/home/samuel/wksp/swestore/ArchiveOnSwestoreNG/src/settings.autotest.cfg")
class testRoundTrip():
def __init__(self):
self.test_data_folder_structure = "/home/samuel/wksp/swestore/ArchiveOnSwestoreNG/tests/testdata/for_drophere_folder"
self.test_proj_name = "b2222222"
self.proj_folder = "/home/samuel/bubofake_auto/proj"
self.drop_folder = os.path.join(self.proj_folder, self.test_proj_name, "swestore/drophere2archive")
self.uploadcache_folder = "/home/samuel/bubofake_auto/uploadcache"
self.uploadcache_proj_folder = os.path.join(self.uploadcache_folder, self.test_proj_name)
self.confirm_folder = os.path.join(self.proj_folder, self.test_proj_name, "swestore/moved2swestore")
self.verification_folder = "/home/samuel/bubofake_auto/verify/"
self.verification_proj_folder = self.verification_folder + self.test_proj_name
self.uploadcache_proj_folder = os.path.join(self.uploadcache_folder, self.test_proj_name)
self.app = Application(configfile_path="/home/samuel/wksp/swestore/ArchiveOnSwestoreNG/src/settings.autotest.cfg")
# ---- Main test function ----------------------------------------------------
def test_roundtrip(self):
# Execution
self.prepare_for_upload()
print("Done preparing!")
self.upload()
self.create_confirm_files()
self.download()
self.unpack()
self.verify_file_count()
self.verify_adler32_sums()
# ---- Verification functions ----------------------------------------------------
def verify_file_count(self):
assert self.folder_content_count(self.unpacked_folder1) == 7
assert self.folder_content_count(self.unpacked_folder2) == 12
def verify_adler32_sums(self):
for unpacked_folder in self.unpacked_folders:
for filename in os.listdir(unpacked_folder):
file_path = os.path.join(unpacked_folder, filename)
adler32sum_cmd = ["jacksum", "-a", "adler32", "-F", "#CHECKSUM", file_path]
adler32sum_output = self.exec_command(adler32sum_cmd)
adler32sum = adler32sum_output.split(" ")[0]
print("Testing if filename %s equals its adler32sum %s ..." % (filename, adler32sum))
assert filename == adler32sum
# ---- Execution functions ----------------------------------------------------
def prepare_for_upload(self):
self.app.init_action("prepare")
self.app.run()
def upload(self):
self.app.init_action("upload")
self.app.run()
def create_confirm_files(self):
self.app.init_action("createconfirmfiles")
self.app.run()
def download(self):
download_cmd = ["ngcp", "-r", "2", "srm://srm.swegrid.se/snic/uppnex/test/" + self.test_proj_name + "/", self.verification_proj_folder + "/"]
print("Downloading files for verification ...")
self.exec_command(download_cmd)
def unpack(self):
files_in_verification_folder = os.listdir(self.verification_proj_folder)
assert len(files_in_verification_folder) == 1
arch_mssn_folder_name = files_in_verification_folder[0]
assert re.match("arch_mssn.*", arch_mssn_folder_name)
self.arch_mssn_folder = os.path.join(self.verification_proj_folder, arch_mssn_folder_name)
meta_files = self.get_files_in_folder_matching_pattern(self.arch_mssn_folder, ".*\.tar\.meta")
for meta_file in meta_files:
meta_file_path = os.path.join(self.arch_mssn_folder, meta_file)
tar_file_basename = self.rchop(meta_file, ".tar.meta")
tar_file = "%s.tar" % tar_file_basename
tar_file_path = os.path.join(self.arch_mssn_folder, tar_file)
tar_file_first_split_file_path = "%s.split00000" % tar_file_path
if os.path.exists(tar_file_first_split_file_path):
split_files = self.get_files_in_folder_matching_pattern(self.arch_mssn_folder, "%s\.split\d{5}" % tar_file)
split_file_paths = []
for split_file in split_files:
split_file_path = os.path.join(self.arch_mssn_folder, split_file)
split_file_paths.append(split_file_path)
split_file_paths.sort()
concat_cmd = ["cat"]
concat_cmd.extend(split_file_paths)
concat_cmd.extend([">", tar_file_path])
concat_cmd_string = " ".join(concat_cmd)
self.exec_command_string(concat_cmd_string)
assert os.path.exists(tar_file_path)
untar_cmd = ["tar", "-xvf", tar_file_path, "-C", self.arch_mssn_folder]
self.exec_command(untar_cmd)
self.unpacked_folder1 = os.path.join(self.arch_mssn_folder, "biologin")
self.unpacked_folder2 = os.path.join(self.arch_mssn_folder, "biologin2", "24028")
self.unpacked_folders = [self.unpacked_folder1, self.unpacked_folder2]
# ---- Setup and Teardown functions ----------------------------------------------------
def setUp(self):
if os.path.exists(self.drop_folder):
shutil.rmtree(self.drop_folder)
self.remove_folders()
# Remove folders on SweStore:
self.rm_recurs_remote("srm://srm.swegrid.se/snic/uppnex/test/" + self.test_proj_name)
if not os.path.exists(self.drop_folder):
try:
os.makedirs(self.drop_folder)
except:
print("Folder already exists: %s" % self.drop_folder)
if not os.path.exists(self.uploadcache_proj_folder):
try:
os.makedirs(self.uploadcache_proj_folder)
except:
print("Folder already exists: %s" % self.uploadcache_proj_folder)
if not os.path.exists(self.confirm_folder):
try:
os.makedirs(self.confirm_folder)
except:
print("Folder already exists: %s" % self.confirm_folder)
if not os.path.exists(self.verification_proj_folder):
try:
os.makedirs(self.verification_proj_folder)
except:
print("Folder already exists: %s" % self.verification_folder)
shutil.copytree(os.path.join(self.test_data_folder_structure, "biologin"), os.path.join(self.drop_folder, "biologin"))
shutil.copytree(os.path.join(self.test_data_folder_structure, "biologin2"), os.path.join(self.drop_folder, "biologin2"))
shutil.copytree(os.path.join(self.test_data_folder_structure, "emptyfolder"), os.path.join(self.drop_folder, "emptyfolder"))
shutil.copyfile(os.path.join(self.test_data_folder_structure, "archiving_state.new"), os.path.join(self.drop_folder, "archiving_state.new"))
def tearDown(self):
self.remove_folders()
# Remove folders on SweStore:
self.rm_recurs_remote("srm://srm.swegrid.se/snic/uppnex/test/" + self.test_proj_name)
# ---- Helper functions ----------------------------------------------------
def exec_command(self, command):
proc = subprocess.Popen(command, stdout=subprocess.PIPE)
output = proc.communicate()[0]
output = string.strip(output)
return output
def exec_command_string(self, command):
subprocess.call(command, shell=True)
def rm_recurs_remote(self, dir_to_remove):
output = self.exec_command(["ngls", "-r", "0", "-l", dir_to_remove])
lines = string.split(output, "\n")
for line in lines:
if line is not "":
bits = line.split(" ")
if len(bits) > 1:
item = bits[0]
type = bits[1]
item_path = os.path.join(dir_to_remove, item)
if type == "dir":
print("Navigating down in subdir: %s" % item_path)
self.rm_recurs_remote(item_path)
elif type == "file":
# print "Removing file: %s" % filepath
self.exec_command(["ngrm", item_path])
print("No more subfolders here, so deleting folder: %s" % dir_to_remove)
self.exec_command(["ngrm", dir_to_remove])
def remove_folders(self):
if os.path.exists(self.proj_folder):
shutil.rmtree(self.proj_folder)
if os.path.exists(self.uploadcache_folder):
shutil.rmtree(self.uploadcache_folder)
if os.path.exists(self.verification_folder):
shutil.rmtree(self.verification_folder)
def get_files_in_folder_matching_pattern(self, folder, pattern, antipattern=""):
result_files = []
for file in os.listdir(folder):
if (re.match(pattern, file) and antipattern == "") or (re.match(pattern, file) and not re.match(antipattern, file)):
result_files.append(file)
return result_files
def rchop(self, thestring, ending):
if thestring.endswith(ending):
return thestring[:-len(ending)]
return thestring
def folder_content_count(self, folder):
folder_content = os.listdir(folder)
folder_content_cnt = len(folder_content)
return folder_content_cnt
#for config_option in app.config.option_names:
# parser.add_option("--%s" % config_option, dest=config_option)
(options, args) = parser.parse_args()
# Require that the settings file param is set
if not options.configfile:
sys.exit("No settings file specified. Please use the -h flag to view command line options")
if not os.path.isfile(options.configfile):
sys.exit("Specified settings file not a valid file!")
# Require that the action parameter is set, otherwise stop the script
if options.action not in ['prepare', 'upload', 'createconfirmfiles']:
sys.exit("No action specified. Please use the -h flag to view command line options")
app = Application(configfile_path=options.configfile)
#option_list = vars(options)
#for option_name in option_list:
# option_value = option_list[option_name]
# #print "Reading cli option: %s = %s" % (option_name, option_value)
# if type(option_value) in (str, int, bool):
# app.config.options[option_name] = option_value
#options.
# \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
# Execute Application
# \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
