def test_irsync_r_nested_dir_to_coll(self): # test settings depth = 10 files_per_level = 100 file_size = 100 # make local nested dirs base_name = "test_irsync_r_nested_dir_to_coll" local_dir = os.path.join(self.testing_tmp_dir, base_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # sync dir to coll self.user0.assert_icommand("irsync -r {local_dir} i:{base_name}".format(**locals()), "EMPTY") # compare files at each level for dir, files in local_dirs.iteritems(): partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1) # run ils on subcollection self.user0.assert_icommand(['ils', partial_path], 'STDOUT_SINGLELINE') ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', partial_path])[1]) # compare local files with irods objects local_files = set(files) rods_files = set(lib.files_in_ils_output(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files))
def test_irsync_r_nested_dir_to_coll_large_files(self): # test settings depth = 4 files_per_level = 4 file_size = 1024*1024*40 # make local nested dirs base_name = "test_irsync_r_nested_dir_to_coll" local_dir = os.path.join(self.testing_tmp_dir, base_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # sync dir to coll self.user0.assert_icommand("irsync -r {local_dir} i:{base_name}".format(**locals()), "EMPTY") # compare files at each level for dir, files in local_dirs.iteritems(): partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1) # run ils on subcollection self.user0.assert_icommand(['ils', partial_path], 'STDOUT_SINGLELINE') ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', partial_path])[1]) # compare local files with irods objects local_files = set(files) rods_files = set(lib.files_in_ils_output(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files)) # compare local files with files in vault files_in_vault = set(lib.files_in_dir(os.path.join(lib.get_vault_session_path(self.user0), partial_path))) self.assertTrue(local_files == files_in_vault, msg="Files missing from vault:\n" + str(local_files - files_in_vault) + "\n\n" + "Extra files in vault:\n" + str(files_in_vault - local_files))
def test_irsync_r_nested_coll_to_dir(self): # test settings depth = 10 files_per_level = 100 file_size = 100 # make local nested dirs base_name = "test_irsync_r_nested_dir_to_coll" local_dir = os.path.join(self.testing_tmp_dir, base_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # sync dir to coll self.user0.assert_icommand("irsync -r {local_dir} i:{base_name}".format(**locals()), "EMPTY") # remove local coll shutil.rmtree(local_dir) # now sync back coll to dir self.user0.assert_icommand("irsync -r i:{base_name} {local_dir}".format(**locals()), "EMPTY") # compare files at each level for dir, files in local_dirs.iteritems(): partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1) # run ils on subcollection self.user0.assert_icommand(['ils', partial_path], 'STDOUT_SINGLELINE') ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', partial_path])[1]) # compare local files with irods objects local_files = set(files) rods_files = set(lib.files_in_ils_output(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files))
def test_ireg_dir_exclude_from(self): # test settings depth = 10 files_per_level = 10 file_size = 100 # make a local test dir under /tmp/ test_dir_name = 'test_ireg_dir_exclude_from' local_dir = os.path.join(self.testing_tmp_dir, test_dir_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # arbitrary list of files to exclude excluded = set(['junk0001', 'junk0002', 'junk0003']) # make exclusion file exclude_file_path = os.path.join(self.testing_tmp_dir, 'exclude.txt') with open(exclude_file_path, 'w') as exclude_file: for filename in excluded: print >> exclude_file, filename # register local dir target_collection = self.admin.home_collection + '/' + test_dir_name self.admin.assert_icommand( "ireg --exclude-from {exclude_file_path} -C {local_dir} {target_collection}" .format(**locals()), "EMPTY") # compare files at each level for dir, files in local_dirs.iteritems(): subcollection = dir.replace(local_dir, target_collection, 1) # run ils on subcollection self.admin.assert_icommand(['ils', subcollection], 'STDOUT_SINGLELINE') ils_out = lib.ils_output_to_entries( self.admin.run_icommand(['ils', subcollection])[1]) # compare irods objects with local files, minus the excluded ones local_files = set(files) rods_files = set(lib.files_in_ils_output(ils_out)) self.assertSetEqual( rods_files, local_files - excluded, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files)) # unregister collection self.admin.assert_icommand( "irm -rfU {target_collection}".format(**locals())) # remove local test dir and exclusion file os.remove(exclude_file_path) shutil.rmtree(local_dir)
def test_ireg_dir_exclude_from(self): # test settings depth = 10 files_per_level = 10 file_size = 100 # make a local test dir under /tmp/ test_dir_name = 'test_ireg_dir_exclude_from' local_dir = os.path.join(self.testing_tmp_dir, test_dir_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # arbitrary list of files to exclude excluded = set(['junk0001', 'junk0002', 'junk0003']) # make exclusion file exclude_file_path = os.path.join(self.testing_tmp_dir, 'exclude.txt') with open(exclude_file_path, 'w') as exclude_file: for filename in excluded: print >>exclude_file, filename # register local dir target_collection = self.admin.home_collection + '/' + test_dir_name self.admin.assert_icommand("ireg --exclude-from {exclude_file_path} -C {local_dir} {target_collection}".format(**locals()), "EMPTY") # compare files at each level for dir, files in local_dirs.iteritems(): subcollection = dir.replace(local_dir, target_collection, 1) # run ils on subcollection self.admin.assert_icommand(['ils', subcollection], 'STDOUT_SINGLELINE') ils_out = lib.ils_output_to_entries(self.admin.run_icommand(['ils', subcollection])[1]) # compare irods objects with local files, minus the excluded ones local_files = set(files) rods_files = set(lib.files_in_ils_output(ils_out)) self.assertSetEqual(rods_files, local_files - excluded, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files)) # unregister collection self.admin.assert_icommand("irm -rfU {target_collection}".format(**locals())) # remove local test dir and exclusion file os.remove(exclude_file_path) shutil.rmtree(local_dir)
def test_irsync_r_nested_coll_to_coll_large_files(self): # test settings depth = 4 files_per_level = 4 file_size = 1024*1024*40 # make local nested dirs source_base_name = "test_irsync_r_nested_coll_to_coll_source" dest_base_name = "test_irsync_r_nested_coll_to_coll_dest" local_dir = os.path.join(self.testing_tmp_dir, source_base_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # iput dir self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "EMPTY") # sync collections self.user0.assert_icommand("irsync -r i:{source_base_name} i:{dest_base_name}".format(**locals()), "EMPTY") # compare files at each level for dir, files in local_dirs.iteritems(): source_partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1) dest_partial_path = dir.replace(self.testing_tmp_dir+'/'+source_base_name, dest_base_name, 1) # run ils on source subcollection self.user0.assert_icommand(['ils', source_partial_path], 'STDOUT_SINGLELINE') ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', source_partial_path])[1]) # compare local files with irods objects local_files = set(files) rods_files = set(lib.files_in_ils_output(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files)) # now the same thing with dest subcollection self.user0.assert_icommand(['ils', dest_partial_path], 'STDOUT_SINGLELINE') ils_out = lib.ils_output_to_entries(self.user0.run_icommand(['ils', dest_partial_path])[1]) # compare local files with irods objects local_files = set(files) rods_files = set(lib.files_in_ils_output(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files))
def test_irsync_r_nested_dir_to_coll(self): # test settings depth = 10 files_per_level = 100 file_size = 100 # make local nested dirs base_name = "test_irsync_r_nested_dir_to_coll" local_dir = os.path.join(self.testing_tmp_dir, base_name) local_dirs = lib.make_deep_local_tmp_dir( local_dir, depth, files_per_level, file_size) # set client env env = os.environ.copy() env['X509_USER_PROXY'] = self.config['client_user_proxy'] # sync dir to coll self.gsi_user.assert_icommand( "irsync -r {local_dir} i:{base_name}".format(**locals()), "EMPTY", env=env) # compare files at each level for directory, files in local_dirs.iteritems(): partial_path = directory.replace(self.testing_tmp_dir + '/', '', 1) # run ils on subcollection self.gsi_user.assert_icommand( ['ils', partial_path], 'STDOUT_SINGLELINE', env=env) ils_out = lib.ils_output_to_entries( self.gsi_user.run_icommand(['ils', partial_path], env=env)[1]) # compare local files with irods objects local_files = set(files) rods_files = set(lib.files_in_ils_output(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files)) # cleanup self.gsi_user.assert_icommand( "irm -rf {base_name}".format(**locals()), "EMPTY", env=env)
def test_irm_rf_nested_coll(self): # test settings depth = 50 files_per_level = 5 file_size = 5 # make local nested dirs coll_name = "test_irm_r_nested_coll" local_dir = os.path.join(self.testing_tmp_dir, coll_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # iput dir self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "EMPTY") # force remove collection self.user0.assert_icommand("irm -rf {coll_name}".format(**locals()), "EMPTY") self.user0.assert_icommand("ils {coll_name}".format(**locals()), 'STDERR_SINGLELINE', "does not exist") # make sure no files are left in the vault user_vault_dir = os.path.join(lib.get_vault_session_path(self.user0), coll_name) cmd_out = lib.run_command('find {user_vault_dir} -type f'.format(**locals())) self.assertEqual(cmd_out[1], '')
def test_irsync_checksum_behavior(self): # set user0's checksum scheme to MD5 to mismatch with server scheme user0_env_backup = copy.deepcopy(self.user0.environment_file_contents) self.user0.environment_file_contents['irods_default_hash_scheme'] = 'MD5' self.user0.environment_file_contents['irods_match_hash_policy'] = 'compatible' # test settings depth = 1 files_per_level = 5 file_size = 1024*1024*40 # make local nested dirs base_name = 'test_irsync_checksum_behavior' local_dir = os.path.join(self.testing_tmp_dir, base_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # sync dir to coll self.user0.assert_icommand("irsync -r -K {local_dir} i:{base_name}".format(**locals()), "EMPTY") self.user0.assert_icommand("ils -L {base_name}".format(**locals()), "STDOUT_SINGLELINE", "ec8bb3b24d5b0f1b5bdf8c8f0f541ee6") self.user0.assert_icommand("ichksum -r -K {base_name}".format(**locals()), "STDOUT_SINGLELINE", "Total checksum performed = 5, Failed checksum = 0") self.user0.assert_icommand("irsync -v -r -K -l {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 39.999 MB --- a match no sync required") self.user0.assert_icommand("irm -f {base_name}/junk0001".format(**locals()), "EMPTY") self.user0.assert_icommand_fail("ils -L {base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001") self.user0.assert_icommand("irsync -v -r -K -l {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 41943040 N") self.user0.assert_icommand("irsync -v -r -K {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 39.999 MB ") self.user0.assert_icommand("irsync -v -r -K -l {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 39.999 MB --- a match no sync required") lib.assert_command("rm -f {local_dir}/junk0001".format(**locals()), "EMPTY") self.user0.assert_icommand("irsync -v -r -K -l i:{base_name} {local_dir}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 41943040 N") self.user0.assert_icommand("irsync -v -r -K i:{base_name} {local_dir}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 39.999 MB ") self.user0.assert_icommand("irsync -v -r -K -l i:{base_name} {local_dir}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 39.999 MB --- a match no sync required") self.user0.environment_file_contents = user0_env_backup