def test_iput_recursive_bulk_upload__4657(self):
bulk_str = 'Bulk upload'
source_path = tempfile.mkdtemp()
tempfile.mkstemp(dir=source_path)
try:
_,out,_ = self.admin.assert_icommand(
['iput', '-v', '-r', source_path, 'v-r'],
'STDOUT', ustrings.recurse_ok_string())
self.assertNotIn(bulk_str, out, 'Bulk upload performed with no bulk flag')
_,out,_ = self.admin.assert_icommand(
['iput', '-v', '-r', '-f', source_path, 'v-r-f'],
'STDOUT', ustrings.recurse_ok_string())
self.assertNotIn(bulk_str, out, 'Bulk upload performed with no bulk flag')
_,out,_ = self.admin.assert_icommand(
['iput', '-v', '-r', '-b', source_path, 'v-r-b'],
'STDOUT', ustrings.recurse_ok_string())
self.assertIn(bulk_str, out, 'Bulk upload not performed when requested')
_,out,_ = self.admin.assert_icommand(
['iput', '-v', '-r', '-b', '-f', source_path, 'v-r-b-f'],
'STDOUT', ustrings.recurse_ok_string())
self.assertIn(bulk_str, out, 'Bulk upload not performed when requested')
finally:
shutil.rmtree(source_path, ignore_errors=True)
def test_icp_closes_file_descriptors__4862(self):
test_dir_path = 'test_icp_closes_file_descriptors__4862'
logical_put_path = 'iput_large_dir'
logical_cp_path = 'icp_large_dir'
try:
with lib.file_backed_up(self.alice._environment_file_path):
del self.alice.environment_file_contents[
'irods_default_resource']
filepath = core_file.CoreFile(self.plugin_name).filepath
with lib.file_backed_up(filepath):
if 'python' not in self.plugin_name:
os.unlink(filepath)
with open(filepath, 'a') as f:
f.write(rule_texts[self.plugin_name][self.class_name][
inspect.currentframe().f_code.co_name])
lib.make_large_local_tmp_dir(test_dir_path, 1024, 1024)
self.alice.assert_icommand(
['iput',
os.path.join(test_dir_path, 'junk0001')], 'STDERR',
'SYS_INVALID_RESC_INPUT')
self.alice.assert_icommand([
'iput', '-R', 'demoResc', '-r', test_dir_path,
logical_put_path
], 'STDOUT', ustrings.recurse_ok_string())
_, _, err = self.alice.assert_icommand(
['icp', '-r', logical_put_path, logical_cp_path],
'STDERR', 'SYS_INVALID_RESC_INPUT')
self.assertNotIn('SYS_OUT_OF_FILE_DESC', err,
'SYS_OUT_OF_FILE_DESC found in output.')
finally:
shutil.rmtree(test_dir_path, ignore_errors=True)
def test_rebalance_invocation_timestamp__3665(self):
# prepare out of balance tree with enough objects to trigger rebalance paging (>500)
localdir = '3665_tmpdir'
shutil.rmtree(localdir, ignore_errors=True)
lib.make_large_local_tmp_dir(dir_name=localdir, file_count=600, file_size=5)
self.admin.assert_icommand(['iput', '-r', localdir], "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
self.admin.assert_icommand(['iadmin', 'mkresc', 'newchild', 'unixfilesystem', test.settings.HOSTNAME_1+':/tmp/newchildVault'], 'STDOUT_SINGLELINE', 'unixfilesystem')
self.admin.assert_icommand(['iadmin','addchildtoresc','demoResc','newchild'])
# run rebalance with concurrent, interleaved put/trim of new file
self.admin.assert_icommand(['ichmod','-r','own','rods',self.admin.session_collection])
self.admin.assert_icommand(['ichmod','-r','inherit',self.admin.session_collection])
laterfilesize = 300
laterfile = '3665_laterfile'
lib.make_file(laterfile, laterfilesize)
put_thread = Timer(2, subprocess.check_call, [('iput', '-R', 'demoResc', laterfile, self.admin.session_collection)])
trim_thread = Timer(3, subprocess.check_call, [('itrim', '-n3', self.admin.session_collection + '/' + laterfile)])
put_thread.start()
trim_thread.start()
self.admin.assert_icommand(['iadmin','modresc','demoResc','rebalance'])
put_thread.join()
trim_thread.join()
# new file should not be balanced (rebalance should have skipped it due to it being newer)
self.admin.assert_icommand(['ils', '-l', laterfile], 'STDOUT_SINGLELINE', [str(laterfilesize), ' 0 ', laterfile])
self.admin.assert_icommand(['ils', '-l', laterfile], 'STDOUT_SINGLELINE', [str(laterfilesize), ' 1 ', laterfile])
self.admin.assert_icommand(['ils', '-l', laterfile], 'STDOUT_SINGLELINE', [str(laterfilesize), ' 2 ', laterfile])
self.admin.assert_icommand_fail(['ils', '-l', laterfile], 'STDOUT_SINGLELINE', [str(laterfilesize), ' 3 ', laterfile])
# cleanup
os.unlink(laterfile)
shutil.rmtree(localdir, ignore_errors=True)
self.admin.assert_icommand(['iadmin','rmchildfromresc','demoResc','newchild'])
self.admin.assert_icommand(['itrim', '-Snewchild', '-r', '/tempZone'], 'STDOUT_SINGLELINE', 'Total size trimmed')
self.admin.assert_icommand(['iadmin','rmresc','newchild'])
def iput_r_large_collection(self, user_session, base_name, file_count, file_size):
local_dir = os.path.join(self.testing_tmp_dir, base_name)
local_files = lib.make_large_local_tmp_dir(local_dir, file_count, file_size)
user_session.assert_icommand(['iput', '-r', local_dir], "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
rods_files = set(user_session.get_entries_in_collection(base_name))
self.assertTrue(set(local_files) == rods_files,
msg="Files missing:\n" + str(set(local_files) - rods_files) + "\n\n" +
"Extra files:\n" + str(rods_files - set(local_files)))
vault_files = set(os.listdir(os.path.join(user_session.get_vault_session_path(), base_name)))
self.assertTrue(set(local_files) == vault_files,
msg="Files missing from vault:\n" + str(set(local_files) - vault_files) + "\n\n" +
"Extra files in vault:\n" + str(vault_files - set(local_files)))
return (local_dir, local_files)
def test_iput_recursive_with_period__issue_2010(self):
try:
new_dir = 'dir_for_test_iput_recursive_with_period_2010'
home_dir = IrodsConfig().irods_directory
save_dir = os.getcwd()
os.chdir(home_dir)
lib.make_deep_local_tmp_dir(new_dir, depth=5, files_per_level=30, file_size=57)
self.new_paths.append(os.path.abspath(new_dir))
os.chdir(new_dir)
self.user0.assert_icommand(['iput', '-r', './'], 'STDOUT_SINGLELINE', ustrings.recurse_ok_string())
self.user0.assert_icommand_fail('ils -l', 'STDOUT_SINGLELINE', '/.')
finally:
os.chdir(save_dir)
def test_iput_recursive_with_period__issue_2010(self):
try:
new_dir = 'dir_for_test_iput_recursive_with_period_2010'
home_dir = IrodsConfig().irods_directory
save_dir = os.getcwd()
os.chdir(home_dir)
lib.make_deep_local_tmp_dir(new_dir, depth=5, files_per_level=30, file_size=57)
self.new_paths.append(os.path.abspath(new_dir))
os.chdir(new_dir)
self.user0.assert_icommand(['iput', '-r', './'], 'STDOUT_SINGLELINE', ustrings.recurse_ok_string())
self.user0.assert_icommand_fail('ils -l', 'STDOUT_SINGLELINE', '/.')
finally:
os.chdir(save_dir)
def test_get_null_perms__2833(self):
l = logging.getLogger(__name__)
base_name = 'test_dir_for_perms'
local_dir = os.path.join(self.testing_tmp_dir, base_name)
local_files = lib.make_large_local_tmp_dir(local_dir, 30, 10)
self.admin.assert_icommand(['iput', '-r', local_dir], "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
ils_out, _, _ = self.admin.run_icommand(['ils', base_name])
rods_files = [f for f in lib.get_object_names_from_entries(ils_out)]
self.admin.assert_icommand(['ichmod','-r','null',self.admin.username,base_name])
self.admin.assert_icommand(['ichmod','read',self.admin.username,base_name+'/'+rods_files[-1]])
self.admin.assert_icommand(['iget','-r',base_name,self.admin.local_session_dir],'STDERR_SINGLELINE','CAT_NO_ACCESS_PERMISSION')
assert os.path.isfile(os.path.join(self.admin.local_session_dir,base_name,rods_files[-1]))
self.admin.assert_icommand(['ichmod','-r','own',self.admin.username,base_name])
def test_irsync_r_nested_dir_to_coll_large_files(self):
# test settings
depth = 4
files_per_level = 4
file_size = 1024 * 1024 * 40
# make local nested dirs
base_name = "test_irsync_r_nested_dir_to_coll"
local_dir = os.path.join(self.testing_tmp_dir, base_name)
local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth,
files_per_level, file_size)
# set client env
env = os.environ.copy()
env['KRB5CCNAME'] = self.config['client_user_ticket_cache']
# sync dir to coll
self.krb_user.assert_icommand(
"irsync -r {local_dir} i:{base_name}".format(**locals()),
"STDOUT_SINGLELINE",
ustrings.recurse_ok_string(),
env=env)
# compare files at each level
for directory, files in local_dirs.iteritems():
partial_path = directory.replace(self.testing_tmp_dir + '/', '', 1)
# run ils on subcollection
self.krb_user.assert_icommand(['ils', partial_path],
'STDOUT_SINGLELINE',
env=env)
ils_out = ils_output_to_entries(
self.krb_user.run_icommand(['ils', partial_path], env=env)[0])
# compare local files with irods objects
local_files = set(files)
rods_files = set(files_in_ils_output(ils_out))
self.assertTrue(local_files == rods_files,
msg="Files missing:\n" +
str(local_files - rods_files) + "\n\n" +
"Extra files:\n" + str(rods_files - local_files))
# cleanup
self.krb_user.assert_icommand("irm -rf {base_name}".format(**locals()),
"EMPTY",
env=env)
def test_large_files_from_devtest(self):
# build expected variables with similar devtest names
test_file = os.path.join(self.admin.local_session_dir, 'test_file')
lib.make_file(test_file, 4000, contents='arbitrary')
myssize = str(os.stat(test_file).st_size)
username = self.admin.username
irodszone = self.admin.zone_name
testuser1 = self.user0.username
irodshome = self.admin.session_collection
irodsdefresource = self.admin.default_resource
dir_w = "."
sfile2 = dir_w + "/sfile2"
cat_file_into_file_n_times(test_file, sfile2, 2)
mysdir = "/tmp/irodssdir"
myldir = dir_w + "/ldir"
if os.path.exists(myldir):
shutil.rmtree(myldir)
self.admin.assert_icommand("imkdir icmdtest")
# make a directory of 2 large files and 2 small files
lfile = dir_w + "/lfile"
lfile1 = dir_w + "/lfile1"
with open(lfile, 'wt') as f:
print('012345678901234567890123456789012345678901234567890123456789012', file=f)
for i in range(6):
cat_file_into_file_n_times(lfile, lfile1, 9)
shutil.move(lfile1, lfile)
os.mkdir(myldir)
for i in range(1, 3):
mylfile = myldir + "/lfile" + str(i)
mysfile = myldir + "/sfile" + str(i)
if i != 2:
shutil.copyfile(lfile, mylfile)
else:
shutil.move(lfile, mylfile)
shutil.copyfile(test_file, mysfile)
# do the large files tests
lrsfile = dir_w + "/lrsfile"
rsfile = dir_w + "/rsfile"
if os.path.isfile(lrsfile):
os.unlink(lrsfile)
if os.path.isfile(rsfile):
os.unlink(rsfile)
self.admin.assert_icommand("iput -vbPKr --retries 10 --wlock -X " + rsfile + " --lfrestart " +
lrsfile + " -N 2 " + myldir + " " + irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "New restartFile")
self.admin.assert_icommand("ichksum -rK " + irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "Total checksum performed")
if os.path.isfile(lrsfile):
os.unlink(lrsfile)
if os.path.isfile(rsfile):
os.unlink(rsfile)
self.admin.assert_icommand("irepl -BvrPT -R " + self.testresc + " --rlock " +
irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "icmdtest/testy")
self.admin.assert_icommand("itrim -vrS " + irodsdefresource + " --dryrun --age 1 -N 1 " +
irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "This is a DRYRUN")
self.admin.assert_icommand("itrim -vrS " + irodsdefresource + " -N 1 " +
irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "a copy trimmed")
self.admin.assert_icommand("icp -vKPTr -N 2 " + irodshome + "/icmdtest/testy " +
irodshome + "/icmdtest/testz", 'STDOUT_SINGLELINE', "Processing lfile1")
self.admin.assert_icommand("irsync -r i:" + irodshome + "/icmdtest/testy i:" + irodshome + "/icmdtest/testz")
self.admin.assert_icommand("irm -vrf " + irodshome + "/icmdtest/testy")
self.admin.assert_icommand("iphymv -vrS " + irodsdefresource + " -R " +
self.testresc + " " + irodshome + "/icmdtest/testz", 'STDOUT_SINGLELINE', "icmdtest/testz")
if os.path.isfile(lrsfile):
os.unlink(lrsfile)
if os.path.isfile(rsfile):
os.unlink(rsfile)
if os.path.exists(dir_w + "/testz"):
shutil.rmtree(dir_w + "/testz")
self.admin.assert_icommand("iget -vPKr --retries 10 -X " + rsfile + " --lfrestart " + lrsfile +
" --rlock -N 2 " + irodshome + "/icmdtest/testz " + dir_w + "/testz", 'STDOUT_SINGLELINE', "testz")
self.admin.assert_icommand("irsync -r " + dir_w + "/testz i:" + irodshome + "/icmdtest/testz", "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
self.admin.assert_icommand("irsync -r i:" + irodshome + "/icmdtest/testz " + dir_w + "/testz")
if os.path.isfile(lrsfile):
os.unlink(lrsfile)
if os.path.isfile(rsfile):
os.unlink(rsfile)
compare_dirs = filecmp.dircmp(os.path.join(dir_w, 'testz'), myldir)
assert (not compare_dirs.right_only and not compare_dirs.left_only and not compare_dirs.diff_files), "Directories differ"
# test -N0 transfer
self.admin.assert_icommand("iput -N0 -R " + self.testresc + " " +
myldir + "/lfile1 " + irodshome + "/icmdtest/testz/lfoo100")
if os.path.isfile(dir_w + "/lfoo100"):
os.unlink(dir_w + "/lfoo100")
self.admin.assert_icommand("iget -N0 " + irodshome + "/icmdtest/testz/lfoo100 " + dir_w + "/lfoo100")
assert filecmp.cmp(os.path.join(myldir, 'lfile1'), os.path.join(dir_w, 'lfoo100'))
shutil.rmtree(dir_w + "/testz")
os.unlink(dir_w + "/lfoo100")
self.admin.assert_icommand("irm -vrf " + irodshome + "/icmdtest/testz")
# cleanup
os.unlink(sfile2)
if os.path.exists(myldir):
shutil.rmtree(myldir)
def test_phybun_from_devtest(self):
with session.make_session_for_existing_admin() as rods_admin:
rods_admin.run_icommand(['ichmod', 'own', '-r', self.admin.username, '/' + self.admin.zone_name])
# build expected variables with similar devtest names
test_file = os.path.join(self.admin.local_session_dir, 'test_file')
lib.make_file(test_file, 4000, contents='arbitrary')
myssize = str(os.stat(test_file).st_size)
username = self.admin.username
irodszone = self.admin.zone_name
testuser1 = self.user0.username
irodshome = self.admin.session_collection
irodsdefresource = self.admin.default_resource
dir_w = "."
sfile2 = dir_w + "/sfile2"
cat_file_into_file_n_times(test_file, sfile2, 2)
mysdir = "/tmp/irodssdir"
myldir = dir_w + "/ldir"
if os.path.exists(myldir):
shutil.rmtree(myldir)
self.admin.assert_icommand("imkdir icmdtest")
# make a directory containing 20 small files
if not os.path.isdir(mysdir):
os.mkdir(mysdir)
for i in range(20):
mysfile = mysdir + "/sfile" + str(i)
shutil.copyfile(test_file, mysfile)
# make a directory containing 20 small files
if not os.path.isdir(mysdir):
os.mkdir(mysdir)
for i in range(20):
mysfile = mysdir + "/sfile" + str(i)
shutil.copyfile(progname, mysfile)
# iphybun test
self.admin.assert_icommand("iput -rR " + self.testresc + " " + mysdir + " " + irodshome + "/icmdtestp", "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
self.admin.assert_icommand("iphybun -KR " + self.anotherresc + " " + irodshome + "/icmdtestp")
self.admin.assert_icommand("itrim -rS " + self.testresc + " -N1 " +
irodshome + "/icmdtestp", 'STDOUT_SINGLELINE', "files trimmed")
out, _ = lib.execute_command(['ils', '-L', os.path.join(irodshome, 'icmdtestp', 'sfile1')])
bunfile = out.split()[-1]
print(bunfile)
self.admin.assert_icommand("irepl --purgec -R " + self.anotherresc + " " + bunfile)
self.admin.assert_icommand("itrim -rS " + self.testresc + " -N1 " +
irodshome + "/icmdtestp", 'STDOUT_SINGLELINE', "files trimmed")
# get the name of bundle file
self.admin.assert_icommand("irm -f --empty " + bunfile)
# should not be able to remove it because it is not empty
self.admin.assert_icommand("ils " + bunfile, 'STDOUT_SINGLELINE', bunfile)
self.admin.assert_icommand("irm -rvf " + irodshome + "/icmdtestp", 'STDOUT_SINGLELINE', "num files done")
self.admin.assert_icommand("irm -f --empty " + bunfile)
if os.path.exists(dir_w + "/testp"):
shutil.rmtree(dir_w + "/testp")
shutil.rmtree(mysdir)
# cleanup
os.unlink(sfile2)
if os.path.exists(myldir):
shutil.rmtree(myldir)
if os.path.exists(mysdir):
shutil.rmtree(mysdir)
def test_irsync_r_nested_coll_to_coll_large_files(self):
# test settings
depth = 4
files_per_level = 4
file_size = 1024*1024*40
# make local nested dirs
source_base_name = "test_irsync_r_nested_coll_to_coll_source"
dest_base_name = "test_irsync_r_nested_coll_to_coll_dest"
local_dir = os.path.join(self.testing_tmp_dir, source_base_name)
local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)
# iput dir
self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
# sync collections
self.user0.assert_icommand("irsync -r i:{source_base_name} i:{dest_base_name}".format(**locals()), "EMPTY")
# compare files at each level
for dir, files in local_dirs.items():
source_partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1)
dest_partial_path = dir.replace(self.testing_tmp_dir+'/'+source_base_name, dest_base_name, 1)
# run ils on source subcollection
self.user0.assert_icommand(['ils', source_partial_path], 'STDOUT_SINGLELINE')
ils_out = self.user0.get_entries_in_collection(source_partial_path)
# compare local files with irods objects
local_files = set(files)
rods_files = set(lib.get_object_names_from_entries(ils_out))
self.assertTrue(local_files == rods_files,
msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
"Extra files:\n" + str(rods_files - local_files))
# now the same thing with dest subcollection
self.user0.assert_icommand(['ils', dest_partial_path], 'STDOUT_SINGLELINE')
ils_out = self.user0.get_entries_in_collection(dest_partial_path)
# compare local files with irods objects
local_files = set(files)
rods_files = set(lib.get_object_names_from_entries(ils_out))
self.assertTrue(local_files == rods_files,
msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
"Extra files:\n" + str(rods_files - local_files))
def test_iput_recursive_with_period__issue_2010(self):
self.user0.assert_icommand(['iput', '-r', './'], 'STDOUT_SINGLELINE',
ustrings.recurse_ok_string())
self.user0.assert_icommand_fail('ils -l', 'STDOUT_SINGLELINE', '/.')
def test_irsync_checksum_behavior(self):
# set user0's checksum scheme to MD5 to mismatch with server scheme
user0_env_backup = copy.deepcopy(self.user0.environment_file_contents)
self.user0.environment_file_contents['irods_default_hash_scheme'] = 'MD5'
self.user0.environment_file_contents['irods_match_hash_policy'] = 'compatible'
# test settings
depth = 1
files_per_level = 5
file_size = 1024*1024*40
# make local nested dirs
base_name = 'test_irsync_checksum_behavior'
local_dir = os.path.join(self.testing_tmp_dir, base_name)
local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)
# sync dir to coll
self.user0.assert_icommand("irsync -r -K {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
self.user0.assert_icommand("ils -L {base_name}".format(**locals()), "STDOUT_SINGLELINE", "ec8bb3b24d5b0f1b5bdf8c8f0f541ee6")
self.user0.assert_icommand("ichksum -r -K {base_name}".format(**locals()), "STDOUT_SINGLELINE", "Total checksum performed = 5, Failed checksum = 0")
self.user0.assert_icommand("irsync -v -r -K -l {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 40.000 MB --- a match no sync required")
self.user0.assert_icommand("irm -f {base_name}/junk0001".format(**locals()), "EMPTY")
self.user0.assert_icommand_fail("ils -L {base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001")
self.user0.assert_icommand("irsync -v -r -K -l {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 41943040 N")
self.user0.assert_icommand("irsync -v -r -K {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 40.000 MB ")
self.user0.assert_icommand("irsync -v -r -K -l {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 40.000 MB --- a match no sync required")
path = '{local_dir}/junk0001'.format(**locals())
if os.path.exists(path):
os.unlink(path)
self.user0.assert_icommand("irsync -v -r -K -l i:{base_name} {local_dir}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 41943040 N")
self.user0.assert_icommand("irsync -v -r -K i:{base_name} {local_dir}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 40.000 MB ")
self.user0.assert_icommand("irsync -v -r -K -l i:{base_name} {local_dir}".format(**locals()), "STDOUT_SINGLELINE", "junk0001 40.000 MB --- a match no sync required")
self.user0.environment_file_contents = user0_env_backup
def test_large_files_from_devtest(self):
# build expected variables with similar devtest names
test_file = os.path.join(self.admin.local_session_dir, 'test_file')
lib.make_file(test_file, 4000, contents='arbitrary')
myssize = str(os.stat(test_file).st_size)
username = self.admin.username
irodszone = self.admin.zone_name
testuser1 = self.user0.username
irodshome = self.admin.session_collection
irodsdefresource = self.admin.default_resource
dir_w = "."
sfile2 = dir_w + "/sfile2"
cat_file_into_file_n_times(test_file, sfile2, 2)
mysdir = "/tmp/irodssdir"
myldir = dir_w + "/ldir"
if os.path.exists(myldir):
shutil.rmtree(myldir)
self.admin.assert_icommand("imkdir icmdtest")
# make a directory of 2 large files and 2 small files
lfile = dir_w + "/lfile"
lfile1 = dir_w + "/lfile1"
with open(lfile, 'wt') as f:
print('012345678901234567890123456789012345678901234567890123456789012', file=f)
for i in range(6):
cat_file_into_file_n_times(lfile, lfile1, 9)
shutil.move(lfile1, lfile)
os.mkdir(myldir)
for i in range(1, 3):
mylfile = myldir + "/lfile" + str(i)
mysfile = myldir + "/sfile" + str(i)
if i != 2:
shutil.copyfile(lfile, mylfile)
else:
shutil.move(lfile, mylfile)
shutil.copyfile(test_file, mysfile)
# do the large files tests
lrsfile = dir_w + "/lrsfile"
rsfile = dir_w + "/rsfile"
if os.path.isfile(lrsfile):
os.unlink(lrsfile)
if os.path.isfile(rsfile):
os.unlink(rsfile)
self.admin.assert_icommand("iput -vbPKr --retries 10 --wlock -X " + rsfile + " --lfrestart " +
lrsfile + " -N 2 " + myldir + " " + irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "New restartFile")
self.admin.assert_icommand("ichksum -rK " + irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "Total checksum performed")
if os.path.isfile(lrsfile):
os.unlink(lrsfile)
if os.path.isfile(rsfile):
os.unlink(rsfile)
self.admin.assert_icommand("irepl -BvrPT -R " + self.testresc + " --rlock " +
irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "icmdtest/testy")
self.admin.assert_icommand("itrim -vrS " + irodsdefresource + " --dryrun --age 1 -N 1 " +
irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "This is a DRYRUN")
self.admin.assert_icommand("itrim -vrS " + irodsdefresource + " -N 1 " +
irodshome + "/icmdtest/testy", 'STDOUT_SINGLELINE', "a copy trimmed")
self.admin.assert_icommand("icp -vKPTr -N 2 " + irodshome + "/icmdtest/testy " +
irodshome + "/icmdtest/testz", 'STDOUT_SINGLELINE', "Processing lfile1")
self.admin.assert_icommand("irsync -r i:" + irodshome + "/icmdtest/testy i:" + irodshome + "/icmdtest/testz")
self.admin.assert_icommand("irm -vrf " + irodshome + "/icmdtest/testy")
self.admin.assert_icommand("iphymv -vrS " + irodsdefresource + " -R " +
self.testresc + " " + irodshome + "/icmdtest/testz", 'STDOUT_SINGLELINE', "icmdtest/testz")
if os.path.isfile(lrsfile):
os.unlink(lrsfile)
if os.path.isfile(rsfile):
os.unlink(rsfile)
if os.path.exists(dir_w + "/testz"):
shutil.rmtree(dir_w + "/testz")
self.admin.assert_icommand("iget -vPKr --retries 10 -X " + rsfile + " --lfrestart " + lrsfile +
" --rlock -N 2 " + irodshome + "/icmdtest/testz " + dir_w + "/testz", 'STDOUT_SINGLELINE', "testz")
self.admin.assert_icommand("irsync -r " + dir_w + "/testz i:" + irodshome + "/icmdtest/testz", "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
self.admin.assert_icommand("irsync -r i:" + irodshome + "/icmdtest/testz " + dir_w + "/testz")
if os.path.isfile(lrsfile):
os.unlink(lrsfile)
if os.path.isfile(rsfile):
os.unlink(rsfile)
compare_dirs = filecmp.dircmp(os.path.join(dir_w, 'testz'), myldir)
assert (not compare_dirs.right_only and not compare_dirs.left_only and not compare_dirs.diff_files), "Directories differ"
# test -N0 transfer
self.admin.assert_icommand("iput -N0 -R " + self.testresc + " " +
myldir + "/lfile1 " + irodshome + "/icmdtest/testz/lfoo100")
if os.path.isfile(dir_w + "/lfoo100"):
os.unlink(dir_w + "/lfoo100")
self.admin.assert_icommand("iget -N0 " + irodshome + "/icmdtest/testz/lfoo100 " + dir_w + "/lfoo100")
assert filecmp.cmp(os.path.join(myldir, 'lfile1'), os.path.join(dir_w, 'lfoo100'))
shutil.rmtree(dir_w + "/testz")
os.unlink(dir_w + "/lfoo100")
self.admin.assert_icommand("irm -vrf " + irodshome + "/icmdtest/testz")
# cleanup
os.unlink(sfile2)
if os.path.exists(myldir):
shutil.rmtree(myldir)
def test_irsync_r_nested_coll_to_coll(self):
# test settings
depth = 10
files_per_level = 100
file_size = 100
# make local nested dirs
source_base_name = "test_irsync_r_nested_coll_to_coll_source"
dest_base_name = "test_irsync_r_nested_coll_to_coll_dest"
local_dir = os.path.join(self.testing_tmp_dir, source_base_name)
local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)
# iput dir
self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
# sync collections
self.user0.assert_icommand("irsync -r i:{source_base_name} i:{dest_base_name}".format(**locals()), "EMPTY")
# compare files at each level
for dir, files in local_dirs.items():
source_partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1)
dest_partial_path = dir.replace(self.testing_tmp_dir+'/'+source_base_name, dest_base_name, 1)
# run ils on source subcollection
self.user0.assert_icommand(['ils', source_partial_path], 'STDOUT_SINGLELINE')
ils_out = self.user0.get_entries_in_collection(source_partial_path)
# compare local files with irods objects
local_files = set(files)
rods_files = set(lib.get_object_names_from_entries(ils_out))
self.assertTrue(local_files == rods_files,
msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
"Extra files:\n" + str(rods_files - local_files))
# compare local files with files in vault
files_in_vault = set(lib.files_in_dir(os.path.join(self.user0.get_vault_session_path(),
source_partial_path)))
self.assertTrue(local_files == files_in_vault,
msg="Files missing from vault:\n" + str(local_files - files_in_vault) + "\n\n" +
"Extra files in vault:\n" + str(files_in_vault - local_files))
# now the same thing with dest subcollection
self.user0.assert_icommand(['ils', dest_partial_path], 'STDOUT_SINGLELINE')
ils_out = self.user0.get_entries_in_collection(dest_partial_path)
# compare local files with irods objects
local_files = set(files)
rods_files = set(lib.get_object_names_from_entries(ils_out))
self.assertTrue(local_files == rods_files,
msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
"Extra files:\n" + str(rods_files - local_files))
# compare local files with files in vault
files_in_vault = set(lib.files_in_dir(os.path.join(self.user0.get_vault_session_path(),
dest_partial_path)))
self.assertTrue(local_files == files_in_vault,
msg="Files missing from vault:\n" + str(local_files - files_in_vault) + "\n\n" +
"Extra files in vault:\n" + str(files_in_vault - local_files))
def test_irsync_r_dir_to_coll(self):
base_name = "test_irsync_r_dir_to_coll"
local_dir = os.path.join(self.testing_tmp_dir, base_name)
file_names = set(lib.make_large_local_tmp_dir(local_dir, file_count=1000, file_size=100))
self.user0.assert_icommand("irsync -r " + local_dir + " i:" + base_name, "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE', base_name)
rods_files = set(self.user0.get_entries_in_collection(base_name))
self.assertTrue(file_names == rods_files,
msg="Files missing:\n" + str(file_names - rods_files) + "\n\n" +
"Extra files:\n" + str(rods_files - file_names))
vault_files_post_irsync = set(os.listdir(os.path.join(self.user0.get_vault_session_path(),
base_name)))
self.assertTrue(file_names == vault_files_post_irsync,
msg="Files missing from vault:\n" + str(file_names - vault_files_post_irsync) + "\n\n" +
"Extra files in vault:\n" + str(vault_files_post_irsync - file_names))
def test_iput_r_with_kw(self):
# test settings
depth = 50
files_per_level = 5
file_size = 5
# make local nested dirs
coll_name = "test_iput_r_with_kw"
local_dir = os.path.join(self.testing_tmp_dir, coll_name)
local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)
# restart server with LOG_DEBUG
env = os.environ.copy()
env['spLogLevel'] = '7'
IrodsController(IrodsConfig(injected_environment=env)).restart()
# get log offset
initial_size_of_server_log = lib.get_file_size_by_path(IrodsConfig().server_log_path)
# iput dir
self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
# look for occurences of debug sequences in the log
rec_op_kw_string = 'DEBUG: unix_file_resolve_hierarchy: recursiveOpr = [1]'
rec_op_kw_string_count = lib.count_occurrences_of_string_in_log(IrodsConfig().server_log_path, rec_op_kw_string, start_index=initial_size_of_server_log)
# assertions
self.assertEqual(rec_op_kw_string_count, files_per_level * depth)
# restart server with original environment
del(env['spLogLevel'])
IrodsController(IrodsConfig(injected_environment=env)).restart()
def test_irm_rf_nested_coll(self):
# test settings
depth = 50
files_per_level = 5
file_size = 5
# make local nested dirs
coll_name = "test_irm_r_nested_coll"
local_dir = os.path.join(self.testing_tmp_dir, coll_name)
local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)
# iput dir
self.user0.assert_icommand("iput -r {local_dir}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
# force remove collection
self.user0.assert_icommand("irm -rf {coll_name}".format(**locals()), "EMPTY")
self.user0.assert_icommand("ils {coll_name}".format(**locals()), 'STDERR_SINGLELINE', "does not exist")
# make sure no files are left in the vault
user_vault_dir = os.path.join(self.user0.get_vault_session_path(), coll_name)
out, _ = lib.execute_command('find {user_vault_dir} -type f'.format(**locals()))
self.assertEqual(out, '')
def test_irsync_r_nested_dir_to_coll(self):
# test settings
depth = 10
files_per_level = 100
file_size = 100
# make local nested dirs
base_name = "test_irsync_r_nested_dir_to_coll"
local_dir = os.path.join(self.testing_tmp_dir, base_name)
local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)
# sync dir to coll
self.user0.assert_icommand("irsync -r {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
# compare files at each level
for dir, files in local_dirs.items():
partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1)
# run ils on subcollection
self.user0.assert_icommand(['ils', partial_path], 'STDOUT_SINGLELINE')
ils_out = self.user0.get_entries_in_collection(partial_path)
# compare local files with irods objects
local_files = set(files)
rods_files = set(lib.get_object_names_from_entries(ils_out))
self.assertTrue(local_files == rods_files,
msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
"Extra files:\n" + str(rods_files - local_files))
def test_irsync_r_nested_dir_to_coll(self):
# test settings
depth = 10
files_per_level = 100
file_size = 100
# make local nested dirs
base_name = "test_irsync_r_nested_dir_to_coll"
local_dir = os.path.join(self.testing_tmp_dir, base_name)
local_dirs = lib.make_deep_local_tmp_dir(
local_dir, depth, files_per_level, file_size)
# set client env
env = os.environ.copy()
env['KRB5CCNAME'] = self.config['client_user_ticket_cache']
# sync dir to coll
self.krb_user.assert_icommand(
"irsync -r {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string(), env=env)
# compare files at each level
for directory, files in local_dirs.iteritems():
partial_path = directory.replace(self.testing_tmp_dir + '/', '', 1)
# run ils on subcollection
self.krb_user.assert_icommand(
['ils', partial_path], 'STDOUT_SINGLELINE', env=env)
ils_out = ils_output_to_entries(
self.krb_user.run_icommand(['ils', partial_path], env=env)[0])
# compare local files with irods objects
local_files = set(files)
rods_files = set(files_in_ils_output(ils_out))
self.assertTrue(local_files == rods_files,
msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
"Extra files:\n" + str(rods_files - local_files))
# cleanup
self.krb_user.assert_icommand(
"irm -rf {base_name}".format(**locals()), "EMPTY", env=env)
def test_irsync_r_nested_coll_to_dir_large_files(self):
# test settings
depth = 4
files_per_level = 4
file_size = 1024*1024*40
# make local nested dirs
base_name = "test_irsync_r_nested_dir_to_coll"
local_dir = os.path.join(self.testing_tmp_dir, base_name)
local_dirs = lib.make_deep_local_tmp_dir(local_dir, depth, files_per_level, file_size)
# sync dir to coll
self.user0.assert_icommand("irsync -r {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
# remove local coll
shutil.rmtree(local_dir)
# now sync back coll to dir
self.user0.assert_icommand("irsync -r i:{base_name} {local_dir}".format(**locals()), "EMPTY")
# compare files at each level
for dir, files in local_dirs.items():
partial_path = dir.replace(self.testing_tmp_dir+'/', '', 1)
# run ils on subcollection
self.user0.assert_icommand(['ils', partial_path], 'STDOUT_SINGLELINE')
ils_out = self.user0.get_entries_in_collection(partial_path)
# compare local files with irods objects
local_files = set(files)
rods_files = set(lib.get_object_names_from_entries(ils_out))
self.assertTrue(local_files == rods_files,
msg="Files missing:\n" + str(local_files - rods_files) + "\n\n" +
"Extra files:\n" + str(rods_files - local_files))
# compare local files with files in vault
files_in_vault = set(lib.files_in_dir(os.path.join(self.user0.get_vault_session_path(),
partial_path)))
self.assertTrue(local_files == files_in_vault,
msg="Files missing from vault:\n" + str(local_files - files_in_vault) + "\n\n" +
"Extra files in vault:\n" + str(files_in_vault - local_files))
def test_iput_bulk_check_acpostprocforput__2841(self):
# prepare test directory
number_of_files = 5
dirname = self.admin.local_session_dir + '/files'
# files less than 4200000 were failing to trigger the writeLine
for filesize in range(5000, 6000000, 500000):
lib.make_large_local_tmp_dir(dirname, number_of_files, filesize)
# manipulate core.re and check the server log
with temporary_core_file() as core:
time.sleep(1) # remove once file hash fix is committed #2279
core.add_rule(rule_texts[self.plugin_name][self.class_name][inspect.currentframe().f_code.co_name])
time.sleep(1) # remove once file hash fix is committed #2279
initial_size_of_server_log = lib.get_file_size_by_path(paths.server_log_path())
self.admin.assert_icommand(['iput', '-frb', dirname], "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
assert number_of_files == lib.count_occurrences_of_string_in_log(paths.server_log_path(), 'writeLine: inString = acPostProcForPut called for', start_index=initial_size_of_server_log)
shutil.rmtree(dirname)
def test_phybun_from_devtest(self):
with session.make_session_for_existing_admin() as rods_admin:
rods_admin.run_icommand(['ichmod', 'own', '-r', self.admin.username, '/' + self.admin.zone_name])
# build expected variables with similar devtest names
test_file = os.path.join(self.admin.local_session_dir, 'test_file')
lib.make_file(test_file, 4000, contents='arbitrary')
myssize = str(os.stat(test_file).st_size)
username = self.admin.username
irodszone = self.admin.zone_name
testuser1 = self.user0.username
irodshome = self.admin.session_collection
irodsdefresource = self.admin.default_resource
dir_w = "."
sfile2 = dir_w + "/sfile2"
cat_file_into_file_n_times(test_file, sfile2, 2)
mysdir = "/tmp/irodssdir"
myldir = dir_w + "/ldir"
if os.path.exists(myldir):
shutil.rmtree(myldir)
self.admin.assert_icommand("imkdir icmdtest")
# make a directory containing 20 small files
if not os.path.isdir(mysdir):
os.mkdir(mysdir)
for i in range(20):
mysfile = mysdir + "/sfile" + str(i)
shutil.copyfile(test_file, mysfile)
# make a directory containing 20 small files
if not os.path.isdir(mysdir):
os.mkdir(mysdir)
for i in range(20):
mysfile = mysdir + "/sfile" + str(i)
shutil.copyfile(progname, mysfile)
# iphybun test
self.admin.assert_icommand("iput -rR " + self.testresc + " " + mysdir + " " + irodshome + "/icmdtestp", "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
self.admin.assert_icommand("iphybun -KR " + self.anotherresc + " " + irodshome + "/icmdtestp")
self.admin.assert_icommand("itrim -rS " + self.testresc + " -N1 " +
irodshome + "/icmdtestp", 'STDOUT_SINGLELINE', "files trimmed")
out, _ = lib.execute_command(['ils', '-L', os.path.join(irodshome, 'icmdtestp', 'sfile1')])
bunfile = out.split()[-1]
print(bunfile)
self.admin.assert_icommand("irepl --purgec -R " + self.anotherresc + " " + bunfile)
self.admin.assert_icommand("itrim -rS " + self.testresc + " -N1 " +
irodshome + "/icmdtestp", 'STDOUT_SINGLELINE', "files trimmed")
# get the name of bundle file
self.admin.assert_icommand("irm -f --empty " + bunfile)
# should not be able to remove it because it is not empty
self.admin.assert_icommand("ils " + bunfile, 'STDOUT_SINGLELINE', bunfile)
self.admin.assert_icommand("irm -rvf " + irodshome + "/icmdtestp", 'STDOUT_SINGLELINE', "num files done")
self.admin.assert_icommand("irm -f --empty " + bunfile)
if os.path.exists(dir_w + "/testp"):
shutil.rmtree(dir_w + "/testp")
shutil.rmtree(mysdir)
# cleanup
os.unlink(sfile2)
if os.path.exists(myldir):
shutil.rmtree(myldir)
if os.path.exists(mysdir):
shutil.rmtree(mysdir)
def test_irsync_r_symlink(self):
# make local dir
base_name = "test_irsync_r_symlink"
local_dir = os.path.join(self.testing_tmp_dir, base_name)
lib.make_dir_p(local_dir)
# make file
file_name = os.path.join(local_dir, 'the_file')
lib.make_file(file_name, 10)
# make symlink with relative path
link_path_1 = os.path.join(local_dir, 'link1')
lib.execute_command(['ln', '-s', 'the_file', link_path_1])
# make symlink with fully qualified path
link_path_2 = os.path.join(local_dir, 'link2')
lib.execute_command(['ln', '-s', file_name, link_path_2])
# sync dir to coll
self.user0.assert_icommand("irsync -r {local_dir} i:{base_name}".format(**locals()), "STDOUT_SINGLELINE", ustrings.recurse_ok_string())
