def test_iput_recursive_bulk_upload__4657(self): bulk_str = 'Bulk upload' source_path = tempfile.mkdtemp() tempfile.mkstemp(dir=source_path) try: _,out,_ = self.admin.assert_icommand( ['iput', '-v', '-r', source_path, 'v-r'], 'STDOUT', ustrings.recurse_ok_string()) self.assertNotIn(bulk_str, out, 'Bulk upload performed with no bulk flag') _,out,_ = self.admin.assert_icommand( ['iput', '-v', '-r', '-f', source_path, 'v-r-f'], 'STDOUT', ustrings.recurse_ok_string()) self.assertNotIn(bulk_str, out, 'Bulk upload performed with no bulk flag') _,out,_ = self.admin.assert_icommand( ['iput', '-v', '-r', '-b', source_path, 'v-r-b'], 'STDOUT', ustrings.recurse_ok_string()) self.assertIn(bulk_str, out, 'Bulk upload not performed when requested') _,out,_ = self.admin.assert_icommand( ['iput', '-v', '-r', '-b', '-f', source_path, 'v-r-b-f'], 'STDOUT', ustrings.recurse_ok_string()) self.assertIn(bulk_str, out, 'Bulk upload not performed when requested') finally: shutil.rmtree(source_path, ignore_errors=True)
def test_icp_closes_file_descriptors__4862(self): test_dir_path = 'test_icp_closes_file_descriptors__4862' logical_put_path = 'iput_large_dir' logical_cp_path = 'icp_large_dir' try: with lib.file_backed_up(self.alice._environment_file_path): del self.alice.environment_file_contents[ 'irods_default_resource'] filepath = core_file.CoreFile(self.plugin_name).filepath with lib.file_backed_up(filepath): if 'python' not in self.plugin_name: os.unlink(filepath) with open(filepath, 'a') as f: f.write(rule_texts[self.plugin_name][self.class_name][ inspect.currentframe().f_code.co_name]) lib.make_large_local_tmp_dir(test_dir_path, 1024, 1024) self.alice.assert_icommand( ['iput', os.path.join(test_dir_path, 'junk0001')], 'STDERR', 'SYS_INVALID_RESC_INPUT') self.alice.assert_icommand([ 'iput', '-R', 'demoResc', '-r', test_dir_path, logical_put_path ], 'STDOUT', ustrings.recurse_ok_string()) _, _, err = self.alice.assert_icommand( ['icp', '-r', logical_put_path, logical_cp_path], 'STDERR', 'SYS_INVALID_RESC_INPUT') self.assertNotIn('SYS_OUT_OF_FILE_DESC', err, 'SYS_OUT_OF_FILE_DESC found in output.') finally: shutil.rmtree(test_dir_path, ignore_errors=True)
def test_rebalance_invocation_timestamp__3665(self): # prepare out of balance tree with enough objects to trigger rebalance paging (>500) localdir = '3665_tmpdir' shutil.rmtree(localdir, ignore_errors=True) lib.make_large_local_tmp_dir(dir_name=localdir, file_count=600, file_size=5) self.admin.assert_icommand(['iput', '-r', localdir], "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) self.admin.assert_icommand(['iadmin', 'mkresc', 'newchild', 'unixfilesystem', test.settings.HOSTNAME_1+':/tmp/newchildVault'], 'STDOUT_SINGLELINE', 'unixfilesystem') self.admin.assert_icommand(['iadmin','addchildtoresc','demoResc','newchild']) # run rebalance with concurrent, interleaved put/trim of new file self.admin.assert_icommand(['ichmod','-r','own','rods',self.admin.session_collection]) self.admin.assert_icommand(['ichmod','-r','inherit',self.admin.session_collection]) laterfilesize = 300 laterfile = '3665_laterfile' lib.make_file(laterfile, laterfilesize) put_thread = Timer(2, subprocess.check_call, [('iput', '-R', 'demoResc', laterfile, self.admin.session_collection)]) trim_thread = Timer(3, subprocess.check_call, [('itrim', '-n3', self.admin.session_collection + '/' + laterfile)]) put_thread.start() trim_thread.start() self.admin.assert_icommand(['iadmin','modresc','demoResc','rebalance']) put_thread.join() trim_thread.join() # new file should not be balanced (rebalance should have skipped it due to it being newer) self.admin.assert_icommand(['ils', '-l', laterfile], 'STDOUT_SINGLELINE', [str(laterfilesize), ' 0 ', laterfile]) self.admin.assert_icommand(['ils', '-l', laterfile], 'STDOUT_SINGLELINE', [str(laterfilesize), ' 1 ', laterfile]) self.admin.assert_icommand(['ils', '-l', laterfile], 'STDOUT_SINGLELINE', [str(laterfilesize), ' 2 ', laterfile]) self.admin.assert_icommand_fail(['ils', '-l', laterfile], 'STDOUT_SINGLELINE', [str(laterfilesize), ' 3 ', laterfile]) # cleanup os.unlink(laterfile) shutil.rmtree(localdir, ignore_errors=True) self.admin.assert_icommand(['iadmin','rmchildfromresc','demoResc','newchild']) self.admin.assert_icommand(['itrim', '-Snewchild', '-r', '/tempZone'], 'STDOUT_SINGLELINE', 'Total size trimmed') self.admin.assert_icommand(['iadmin','rmresc','newchild'])
def iput_r_large_collection(self, user_session, base_name, file_count, file_size): local_dir = os.path.join(self.testing_tmp_dir, base_name) local_files = lib.make_large_local_tmp_dir(local_dir, file_count, file_size) user_session.assert_icommand(['iput', '-r', local_dir], "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) rods_files = set(user_session.get_entries_in_collection(base_name)) self.assertTrue(set(local_files) == rods_files, msg="Files missing:\n" + str(set(local_files) - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - set(local_files))) vault_files = set(os.listdir(os.path.join(user_session.get_vault_session_path(), base_name))) self.assertTrue(set(local_files) == vault_files, msg="Files missing from vault:\n" + str(set(local_files) - vault_files) + "\n\n" + "Extra files in vault:\n" + str(vault_files - set(local_files))) return (local_dir, local_files)
def test_iput_recursive_with_period__issue_2010(self): try: new_dir = 'dir_for_test_iput_recursive_with_period_2010' home_dir = IrodsConfig().irods_directory save_dir = os.getcwd() os.chdir(home_dir) lib.make_deep_local_tmp_dir(new_dir, depth=5, files_per_level=30, file_size=57) self.new_paths.append(os.path.abspath(new_dir)) os.chdir(new_dir) self.user0.assert_icommand(['iput', '-r', './'], 'STDOUT_SINGLELINE', ustrings.recurse_ok_string()) self.user0.assert_icommand_fail('ils -l', 'STDOUT_SINGLELINE', '/.') finally: os.chdir(save_dir)
def test_get_null_perms__2833(self): l = logging.getLogger(__name__) base_name = 'test_dir_for_perms' local_dir = os.path.join(self.testing_tmp_dir, base_name) local_files = lib.make_large_local_tmp_dir(local_dir, 30, 10) self.admin.assert_icommand(['iput', '-r', local_dir], "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) ils_out, _, _ = self.admin.run_icommand(['ils', base_name]) rods_files = [f for f in lib.get_object_names_from_entries(ils_out)] self.admin.assert_icommand(['ichmod','-r','null',self.admin.username,base_name]) self.admin.assert_icommand(['ichmod','read',self.admin.username,base_name+'/'+rods_files[-1]]) self.admin.assert_icommand(['iget','-r',base_name,self.admin.local_session_dir],'STDERR_SINGLELINE','CAT_NO_ACCESS_PERMISSION') assert os.path.isfile(os.path.join(self.admin.local_session_dir,base_name,rods_files[-1])) self.admin.assert_icommand(['ichmod','-r','own',self.admin.username,base_name])
def test_irsync_r_nested_dir_to_coll_large_files(self): # test settings depth = 4 files_per_level = 4 file_size = 1024 * 1024 * 40 # make local nested dirs base_name = "test_irsync_r_nested_dir_to_coll" local_dir = os.path.join(self.testing_tmp_dir, base_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # set client env env = os.environ.copy() env['KRB5CCNAME'] = self.config['client_user_ticket_cache'] # sync dir to coll self.krb_user.assert_icommand( "irsync -r {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string(), env=env) # compare files at each level for directory, files in local_dirs.iteritems(): partial_path = directory.replace(self.testing_tmp_dir + '/', '', 1) # run ils on subcollection self.krb_user.assert_icommand(['ils', partial_path], 'STDOUT_SINGLELINE', env=env) ils_out = ils_output_to_entries( self.krb_user.run_icommand(['ils', partial_path], env=env)[0]) # compare local files with irods objects local_files = set(files) rods_files = set(files_in_ils_output(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files)) # cleanup self.krb_user.assert_icommand("irm -rf {base_name}".format(**locals()), "EMPTY", env=env)
def test_large_files_from_devtest(self): # build expected variables with similar devtest names test_file = os.path.join(self.admin.local_session_dir, 'test_file') lib.make_file(test_file, 4000, contents='arbitrary') myssize = str(os.stat(test_file).st_size) username = self.admin.username irodszone = self.admin.zone_name testuser1 = self.user0.username irodshome = self.admin.session_collection irodsdefresource = self.admin.default_resource dir_w = "." sfile2 = dir_w + "/sfile2" cat_file_into_file_n_times(test_file, sfile2, 2) mysdir = "/tmp/irodssdir" myldir = dir_w + "/ldir" if os.path.exists(myldir): shutil.rmtree(myldir) self.admin.assert_icommand("imkdir icmdtest") # make a directory of 2 large files and 2 small files lfile = dir_w + "/lfile" lfile1 = dir_w + "/lfile1" with open(lfile, 'wt') as f: print('012345678901234567890123456789012345678901234567890123456789012', file=f) for i in range(6): cat_file_into_file_n_times(lfile, lfile1, 9) shutil.move(lfile1, lfile) os.mkdir(myldir) for i in range(1, 3): mylfile = myldir + "/lfile" + str(i) mysfile = myldir + "/sfile" + str(i) if i != 2: shutil.copyfile(lfile, mylfile) else: shutil.move(lfile, mylfile) shutil.copyfile(test_file, mysfile) # do the large files tests lrsfile = dir_w + "/lrsfile" rsfile = dir_w + "/rsfile" if os.path.isfile(lrsfile): os.unlink(lrsfile) if os.path.isfile(rsfile): os.unlink(rsfile) self.admin.assert_icommand("iput -vbPKr --retries 10 --wlock -X " + rsfile + " --lfrestart " + lrsfile + " -N 2 " + myldir + " " + irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "New restartFile") self.admin.assert_icommand("ichksum -rK " + irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "Total checksum performed") if os.path.isfile(lrsfile): os.unlink(lrsfile) if os.path.isfile(rsfile): os.unlink(rsfile) self.admin.assert_icommand("irepl -BvrPT -R " + self.testresc + " --rlock " + irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "icmdtest/testy") self.admin.assert_icommand("itrim -vrS " + irodsdefresource + " --dryrun --age 1 -N 1 " + irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "This is a DRYRUN") self.admin.assert_icommand("itrim -vrS " + irodsdefresource + " -N 1 " + irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "a copy trimmed") self.admin.assert_icommand("icp -vKPTr -N 2 " + irodshome + "/icmdtest/testy " + irodshome + "/icmdtest/testz", 'STDOUT_SINGLELINE', "Processing lfile1") self.admin.assert_icommand("irsync -r i:" + irodshome + "/icmdtest/testy i:" + irodshome + "/icmdtest/testz") self.admin.assert_icommand("irm -vrf " + irodshome + "/icmdtest/testy") self.admin.assert_icommand("iphymv -vrS " + irodsdefresource + " -R " + self.testresc + " " + irodshome + "/icmdtest/testz", 'STDOUT_SINGLELINE', "icmdtest/testz") if os.path.isfile(lrsfile): os.unlink(lrsfile) if os.path.isfile(rsfile): os.unlink(rsfile) if os.path.exists(dir_w + "/testz"): shutil.rmtree(dir_w + "/testz") self.admin.assert_icommand("iget -vPKr --retries 10 -X " + rsfile + " --lfrestart " + lrsfile + " --rlock -N 2 " + irodshome + "/icmdtest/testz " + dir_w + "/testz", 'STDOUT_SINGLELINE', "testz") self.admin.assert_icommand("irsync -r " + dir_w + "/testz i:" + irodshome + "/icmdtest/testz", "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) self.admin.assert_icommand("irsync -r i:" + irodshome + "/icmdtest/testz " + dir_w + "/testz") if os.path.isfile(lrsfile): os.unlink(lrsfile) if os.path.isfile(rsfile): os.unlink(rsfile) compare_dirs = filecmp.dircmp(os.path.join(dir_w, 'testz'), myldir) assert (not compare_dirs.right_only and not compare_dirs.left_only and not compare_dirs.diff_files), "Directories differ" # test -N0 transfer self.admin.assert_icommand("iput -N0 -R " + self.testresc + " " + myldir + "/lfile1 " + irodshome + "/icmdtest/testz/lfoo100") if os.path.isfile(dir_w + "/lfoo100"): os.unlink(dir_w + "/lfoo100") self.admin.assert_icommand("iget -N0 " + irodshome + "/icmdtest/testz/lfoo100 " + dir_w + "/lfoo100") assert filecmp.cmp(os.path.join(myldir, 'lfile1'), os.path.join(dir_w, 'lfoo100')) shutil.rmtree(dir_w + "/testz") os.unlink(dir_w + "/lfoo100") self.admin.assert_icommand("irm -vrf " + irodshome + "/icmdtest/testz") # cleanup os.unlink(sfile2) if os.path.exists(myldir): shutil.rmtree(myldir)
def test_phybun_from_devtest(self): with session.make_session_for_existing_admin() as rods_admin: rods_admin.run_icommand(['ichmod', 'own', '-r', self.admin.username, '/' + self.admin.zone_name]) # build expected variables with similar devtest names test_file = os.path.join(self.admin.local_session_dir, 'test_file') lib.make_file(test_file, 4000, contents='arbitrary') myssize = str(os.stat(test_file).st_size) username = self.admin.username irodszone = self.admin.zone_name testuser1 = self.user0.username irodshome = self.admin.session_collection irodsdefresource = self.admin.default_resource dir_w = "." sfile2 = dir_w + "/sfile2" cat_file_into_file_n_times(test_file, sfile2, 2) mysdir = "/tmp/irodssdir" myldir = dir_w + "/ldir" if os.path.exists(myldir): shutil.rmtree(myldir) self.admin.assert_icommand("imkdir icmdtest") # make a directory containing 20 small files if not os.path.isdir(mysdir): os.mkdir(mysdir) for i in range(20): mysfile = mysdir + "/sfile" + str(i) shutil.copyfile(test_file, mysfile) # make a directory containing 20 small files if not os.path.isdir(mysdir): os.mkdir(mysdir) for i in range(20): mysfile = mysdir + "/sfile" + str(i) shutil.copyfile(progname, mysfile) # iphybun test self.admin.assert_icommand("iput -rR " + self.testresc + " " + mysdir + " " + irodshome + "/icmdtestp", "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) self.admin.assert_icommand("iphybun -KR " + self.anotherresc + " " + irodshome + "/icmdtestp") self.admin.assert_icommand("itrim -rS " + self.testresc + " -N1 " + irodshome + "/icmdtestp", 'STDOUT_SINGLELINE', "files trimmed") out, _ = lib.execute_command(['ils', '-L', os.path.join(irodshome, 'icmdtestp', 'sfile1')]) bunfile = out.split()[-1] print(bunfile) self.admin.assert_icommand("irepl --purgec -R " + self.anotherresc + " " + bunfile) self.admin.assert_icommand("itrim -rS " + self.testresc + " -N1 " + irodshome + "/icmdtestp", 'STDOUT_SINGLELINE', "files trimmed") # get the name of bundle file self.admin.assert_icommand("irm -f --empty " + bunfile) # should not be able to remove it because it is not empty self.admin.assert_icommand("ils " + bunfile, 'STDOUT_SINGLELINE', bunfile) self.admin.assert_icommand("irm -rvf " + irodshome + "/icmdtestp", 'STDOUT_SINGLELINE', "num files done") self.admin.assert_icommand("irm -f --empty " + bunfile) if os.path.exists(dir_w + "/testp"): shutil.rmtree(dir_w + "/testp") shutil.rmtree(mysdir) # cleanup os.unlink(sfile2) if os.path.exists(myldir): shutil.rmtree(myldir) if os.path.exists(mysdir): shutil.rmtree(mysdir)
def test_irsync_r_nested_coll_to_coll_large_files(self): # test settings depth = 4 files_per_level = 4 file_size = 1024*1024*40 # make local nested dirs source_base_name = "test_irsync_r_nested_coll_to_coll_source" dest_base_name = "test_irsync_r_nested_coll_to_coll_dest" local_dir = os.path.join(self.testing_tmp_dir, source_base_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # iput dir self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) # sync collections self.user0.assert_icommand("irsync -r i:{source_base_name} i:{dest_base_name}".format(**locals()), "EMPTY") # compare files at each level for dir, files in local_dirs.items(): source_partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1) dest_partial_path = dir.replace(self.testing_tmp_dir+'/'+source_base_name, dest_base_name, 1) # run ils on source subcollection self.user0.assert_icommand(['ils', source_partial_path], 'STDOUT_SINGLELINE') ils_out = self.user0.get_entries_in_collection(source_partial_path) # compare local files with irods objects local_files = set(files) rods_files = set(lib.get_object_names_from_entries(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files)) # now the same thing with dest subcollection self.user0.assert_icommand(['ils', dest_partial_path], 'STDOUT_SINGLELINE') ils_out = self.user0.get_entries_in_collection(dest_partial_path) # compare local files with irods objects local_files = set(files) rods_files = set(lib.get_object_names_from_entries(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files))
def test_iput_recursive_with_period__issue_2010(self): self.user0.assert_icommand(['iput', '-r', './'], 'STDOUT_SINGLELINE', ustrings.recurse_ok_string()) self.user0.assert_icommand_fail('ils -l', 'STDOUT_SINGLELINE', '/.')
def test_irsync_checksum_behavior(self): # set user0's checksum scheme to MD5 to mismatch with server scheme user0_env_backup = copy.deepcopy(self.user0.environment_file_contents) self.user0.environment_file_contents['irods_default_hash_scheme'] = 'MD5' self.user0.environment_file_contents['irods_match_hash_policy'] = 'compatible' # test settings depth = 1 files_per_level = 5 file_size = 1024*1024*40 # make local nested dirs base_name = 'test_irsync_checksum_behavior' local_dir = os.path.join(self.testing_tmp_dir, base_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # sync dir to coll self.user0.assert_icommand("irsync -r -K {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) self.user0.assert_icommand("ils -L {base_name}".format(**locals()), "STDOUT_SINGLELINE", "ec8bb3b24d5b0f1b5bdf8c8f0f541ee6") self.user0.assert_icommand("ichksum -r -K {base_name}".format(**locals()), "STDOUT_SINGLELINE", "Total checksum performed = 5, Failed checksum = 0") self.user0.assert_icommand("irsync -v -r -K -l {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 40.000 MB --- a match no sync required") self.user0.assert_icommand("irm -f {base_name}/junk0001".format(**locals()), "EMPTY") self.user0.assert_icommand_fail("ils -L {base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001") self.user0.assert_icommand("irsync -v -r -K -l {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 41943040 N") self.user0.assert_icommand("irsync -v -r -K {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 40.000 MB ") self.user0.assert_icommand("irsync -v -r -K -l {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 40.000 MB --- a match no sync required") path = '{local_dir}/junk0001'.format(**locals()) if os.path.exists(path): os.unlink(path) self.user0.assert_icommand("irsync -v -r -K -l i:{base_name} {local_dir}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 41943040 N") self.user0.assert_icommand("irsync -v -r -K i:{base_name} {local_dir}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 40.000 MB ") self.user0.assert_icommand("irsync -v -r -K -l i:{base_name} {local_dir}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 40.000 MB --- a match no sync required") self.user0.environment_file_contents = user0_env_backup
def test_irsync_r_nested_coll_to_coll(self): # test settings depth = 10 files_per_level = 100 file_size = 100 # make local nested dirs source_base_name = "test_irsync_r_nested_coll_to_coll_source" dest_base_name = "test_irsync_r_nested_coll_to_coll_dest" local_dir = os.path.join(self.testing_tmp_dir, source_base_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # iput dir self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) # sync collections self.user0.assert_icommand("irsync -r i:{source_base_name} i:{dest_base_name}".format(**locals()), "EMPTY") # compare files at each level for dir, files in local_dirs.items(): source_partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1) dest_partial_path = dir.replace(self.testing_tmp_dir+'/'+source_base_name, dest_base_name, 1) # run ils on source subcollection self.user0.assert_icommand(['ils', source_partial_path], 'STDOUT_SINGLELINE') ils_out = self.user0.get_entries_in_collection(source_partial_path) # compare local files with irods objects local_files = set(files) rods_files = set(lib.get_object_names_from_entries(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files)) # compare local files with files in vault files_in_vault = set(lib.files_in_dir(os.path.join(self.user0.get_vault_session_path(), source_partial_path))) self.assertTrue(local_files == files_in_vault, msg="Files missing from vault:\n" + str(local_files - files_in_vault) + "\n\n" + "Extra files in vault:\n" + str(files_in_vault - local_files)) # now the same thing with dest subcollection self.user0.assert_icommand(['ils', dest_partial_path], 'STDOUT_SINGLELINE') ils_out = self.user0.get_entries_in_collection(dest_partial_path) # compare local files with irods objects local_files = set(files) rods_files = set(lib.get_object_names_from_entries(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files)) # compare local files with files in vault files_in_vault = set(lib.files_in_dir(os.path.join(self.user0.get_vault_session_path(), dest_partial_path))) self.assertTrue(local_files == files_in_vault, msg="Files missing from vault:\n" + str(local_files - files_in_vault) + "\n\n" + "Extra files in vault:\n" + str(files_in_vault - local_files))
def test_irsync_r_dir_to_coll(self): base_name = "test_irsync_r_dir_to_coll" local_dir = os.path.join(self.testing_tmp_dir, base_name) file_names = set(lib.make_large_local_tmp_dir(local_dir, file_count=1000, file_size=100)) self.user0.assert_icommand("irsync -r " + local_dir + " i:" + base_name, "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE', base_name) rods_files = set(self.user0.get_entries_in_collection(base_name)) self.assertTrue(file_names == rods_files, msg="Files missing:\n" + str(file_names - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - file_names)) vault_files_post_irsync = set(os.listdir(os.path.join(self.user0.get_vault_session_path(), base_name))) self.assertTrue(file_names == vault_files_post_irsync, msg="Files missing from vault:\n" + str(file_names - vault_files_post_irsync) + "\n\n" + "Extra files in vault:\n" + str(vault_files_post_irsync - file_names))
def test_iput_r_with_kw(self): # test settings depth = 50 files_per_level = 5 file_size = 5 # make local nested dirs coll_name = "test_iput_r_with_kw" local_dir = os.path.join(self.testing_tmp_dir, coll_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # restart server with LOG_DEBUG env = os.environ.copy() env['spLogLevel'] = '7' IrodsController(IrodsConfig(injected_environment=env)).restart() # get log offset initial_size_of_server_log = lib.get_file_size_by_path(IrodsConfig().server_log_path) # iput dir self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) # look for occurences of debug sequences in the log rec_op_kw_string = 'DEBUG: unix_file_resolve_hierarchy: recursiveOpr = [1]' rec_op_kw_string_count = lib.count_occurrences_of_string_in_log(IrodsConfig().server_log_path, rec_op_kw_string, start_index=initial_size_of_server_log) # assertions self.assertEqual(rec_op_kw_string_count, files_per_level * depth) # restart server with original environment del(env['spLogLevel']) IrodsController(IrodsConfig(injected_environment=env)).restart()
def test_irm_rf_nested_coll(self): # test settings depth = 50 files_per_level = 5 file_size = 5 # make local nested dirs coll_name = "test_irm_r_nested_coll" local_dir = os.path.join(self.testing_tmp_dir, coll_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # iput dir self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) # force remove collection self.user0.assert_icommand("irm -rf {coll_name}".format(**locals()), "EMPTY") self.user0.assert_icommand("ils {coll_name}".format(**locals()), 'STDERR_SINGLELINE', "does not exist") # make sure no files are left in the vault user_vault_dir = os.path.join(self.user0.get_vault_session_path(), coll_name) out, _ = lib.execute_command('find {user_vault_dir} -type f'.format(**locals())) self.assertEqual(out, '')
def test_irsync_r_nested_dir_to_coll(self): # test settings depth = 10 files_per_level = 100 file_size = 100 # make local nested dirs base_name = "test_irsync_r_nested_dir_to_coll" local_dir = os.path.join(self.testing_tmp_dir, base_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # sync dir to coll self.user0.assert_icommand("irsync -r {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) # compare files at each level for dir, files in local_dirs.items(): partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1) # run ils on subcollection self.user0.assert_icommand(['ils', partial_path], 'STDOUT_SINGLELINE') ils_out = self.user0.get_entries_in_collection(partial_path) # compare local files with irods objects local_files = set(files) rods_files = set(lib.get_object_names_from_entries(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files))
def test_irsync_r_nested_dir_to_coll(self): # test settings depth = 10 files_per_level = 100 file_size = 100 # make local nested dirs base_name = "test_irsync_r_nested_dir_to_coll" local_dir = os.path.join(self.testing_tmp_dir, base_name) local_dirs = lib.make_deep_local_tmp_dir( local_dir, depth, files_per_level, file_size) # set client env env = os.environ.copy() env['KRB5CCNAME'] = self.config['client_user_ticket_cache'] # sync dir to coll self.krb_user.assert_icommand( "irsync -r {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string(), env=env) # compare files at each level for directory, files in local_dirs.iteritems(): partial_path = directory.replace(self.testing_tmp_dir + '/', '', 1) # run ils on subcollection self.krb_user.assert_icommand( ['ils', partial_path], 'STDOUT_SINGLELINE', env=env) ils_out = ils_output_to_entries( self.krb_user.run_icommand(['ils', partial_path], env=env)[0]) # compare local files with irods objects local_files = set(files) rods_files = set(files_in_ils_output(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files)) # cleanup self.krb_user.assert_icommand( "irm -rf {base_name}".format(**locals()), "EMPTY", env=env)
def test_irsync_r_nested_coll_to_dir_large_files(self): # test settings depth = 4 files_per_level = 4 file_size = 1024*1024*40 # make local nested dirs base_name = "test_irsync_r_nested_dir_to_coll" local_dir = os.path.join(self.testing_tmp_dir, base_name) local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size) # sync dir to coll self.user0.assert_icommand("irsync -r {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) # remove local coll shutil.rmtree(local_dir) # now sync back coll to dir self.user0.assert_icommand("irsync -r i:{base_name} {local_dir}".format(**locals()), "EMPTY") # compare files at each level for dir, files in local_dirs.items(): partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1) # run ils on subcollection self.user0.assert_icommand(['ils', partial_path], 'STDOUT_SINGLELINE') ils_out = self.user0.get_entries_in_collection(partial_path) # compare local files with irods objects local_files = set(files) rods_files = set(lib.get_object_names_from_entries(ils_out)) self.assertTrue(local_files == rods_files, msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - local_files)) # compare local files with files in vault files_in_vault = set(lib.files_in_dir(os.path.join(self.user0.get_vault_session_path(), partial_path))) self.assertTrue(local_files == files_in_vault, msg="Files missing from vault:\n" + str(local_files - files_in_vault) + "\n\n" + "Extra files in vault:\n" + str(files_in_vault - local_files))
def test_iput_bulk_check_acpostprocforput__2841(self): # prepare test directory number_of_files = 5 dirname = self.admin.local_session_dir + '/files' # files less than 4200000 were failing to trigger the writeLine for filesize in range(5000, 6000000, 500000): lib.make_large_local_tmp_dir(dirname, number_of_files, filesize) # manipulate core.re and check the server log with temporary_core_file() as core: time.sleep(1) # remove once file hash fix is committed #2279 core.add_rule(rule_texts[self.plugin_name][self.class_name][inspect.currentframe().f_code.co_name]) time.sleep(1) # remove once file hash fix is committed #2279 initial_size_of_server_log = lib.get_file_size_by_path(paths.server_log_path()) self.admin.assert_icommand(['iput', '-frb', dirname], "STDOUT_SINGLELINE", ustrings.recurse_ok_string()) assert number_of_files == lib.count_occurrences_of_string_in_log(paths.server_log_path(), 'writeLine: inString = acPostProcForPut called for', start_index=initial_size_of_server_log) shutil.rmtree(dirname)
def test_irsync_r_symlink(self): # make local dir base_name = "test_irsync_r_symlink" local_dir = os.path.join(self.testing_tmp_dir, base_name) lib.make_dir_p(local_dir) # make file file_name = os.path.join(local_dir, 'the_file') lib.make_file(file_name, 10) # make symlink with relative path link_path_1 = os.path.join(local_dir, 'link1') lib.execute_command(['ln', '-s', 'the_file', link_path_1]) # make symlink with fully qualified path link_path_2 = os.path.join(local_dir, 'link2') lib.execute_command(['ln', '-s', file_name, link_path_2]) # sync dir to coll self.user0.assert_icommand("irsync -r {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string())