def test_cancel_large_iput(self): base_name = 'test_cancel_large_put' local_dir = os.path.join(self.testing_tmp_dir, base_name) file_size = pow(2, 30) file_name = lib.make_large_local_tmp_dir(local_dir, file_count=1, file_size=file_size)[0] file_local_full_path = os.path.join(local_dir, file_name) iput_cmd = "iput '" + file_local_full_path + "'" file_vault_full_path = os.path.join( lib.get_vault_session_path(self.user0), file_name) self.user0.interrupt_icommand(iput_cmd, file_vault_full_path, 10) # multiple threads could still be writing on the server side, so we need to wait for # the size in the vault to converge - then were done. old_size = None new_size = os.path.getsize(file_vault_full_path) while old_size != new_size: time.sleep(1) old_size = new_size new_size = os.path.getsize(file_vault_full_path) # wait for select() call to timeout, set to "SELECT_TIMEOUT_FOR_CONN", which is 60 seconds time.sleep(63) self.user0.assert_icommand('ils -l', 'STDOUT_SINGLELINE', [file_name, str(new_size)])
def test_irsync_r_coll_to_dir(self): # pick session(s) for the test test_session = self.user_sessions[0] # make test dir dir_name = "irsync_test_dir" dir_path = os.path.join(self.local_test_dir_path, dir_name) local_files = lib.make_large_local_tmp_dir( dir_path, self.config["test_file_count"], self.config["test_file_size"] ) # test specific parameters parameters = self.config.copy() parameters["dir_path"] = dir_path parameters["dir_name"] = dir_name parameters["user_name"] = test_session.username parameters["remote_home_collection"] = "/{remote_zone}/home/{user_name}#{local_zone}".format(**parameters) # sync dir with remote collection test_session.assert_icommand("irsync -r {dir_path} i:{remote_home_collection}/{dir_name}".format(**parameters)) # remove local test dir shutil.rmtree(dir_path) # sync remote collection back with local dir test_session.assert_icommand("irsync -r i:{remote_home_collection}/{dir_name} {dir_path}".format(**parameters)) # compare list of files received_files = os.listdir(dir_path) self.assertTrue(set(local_files) == set(received_files)) # cleanup test_session.assert_icommand("irm -rf {remote_home_collection}/{dir_name}".format(**parameters)) shutil.rmtree(dir_path)
def test_irsync_r_dir_to_coll(self): base_name = "test_irsync_r_dir_to_coll" local_dir = os.path.join(self.testing_tmp_dir, base_name) file_names = set( lib.make_large_local_tmp_dir(local_dir, file_count=1000, file_size=100)) self.user0.assert_icommand( "irsync -r " + local_dir + " i:" + base_name, "EMPTY") self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE', base_name) rods_files = set( lib.ils_output_to_entries( self.user0.run_icommand(['ils', base_name])[1])) self.assertTrue(file_names == rods_files, msg="Files missing:\n" + str(file_names - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - file_names)) vault_files_post_irsync = set( os.listdir( os.path.join(lib.get_vault_session_path(self.user0), base_name))) self.assertTrue(file_names == vault_files_post_irsync, msg="Files missing from vault:\n" + str(file_names - vault_files_post_irsync) + "\n\n" + "Extra files in vault:\n" + str(vault_files_post_irsync - file_names))
def test_irsync_r_coll_to_dir(self): # make test dir dir_name = 'irsync_test_dir' dir_path = os.path.abspath(dir_name) local_files = lib.make_large_local_tmp_dir( dir_path, self.config['test_file_count'], self.config['test_file_size']) kwargs = {'dir_name': dir_name, 'collection': self.config['remote_home_coll']} # sync dir with remote collection self.admin_sessions[0].assert_icommand( "irsync -r {dir_name} i:{collection}/{dir_name}".format(**kwargs)) # remove local test dir shutil.rmtree(dir_path) # sync remote collection back with local dir self.admin_sessions[0].assert_icommand( "irsync -r i:{collection}/{dir_name} {dir_name}".format(**kwargs)) # compare list of files received_files = os.listdir(dir_path) self.assertTrue(set(local_files) == set(received_files)) # cleanup self.admin_sessions[0].assert_icommand( "irm -rf {collection}/{dir_name}".format(**kwargs)) shutil.rmtree(dir_path)
def test_irsync_r_dir_to_coll(self): # make test dir dir_name = 'irsync_test_dir' dir_path = os.path.abspath(dir_name) local_files = lib.make_large_local_tmp_dir( dir_path, self.config['test_file_count'], self.config['test_file_size']) kwargs = {'dir_name': dir_name, 'collection': self.config['remote_home_coll']} # sync dir with remote collection self.admin_sessions[0].assert_icommand( "irsync -r {dir_name} i:{collection}/{dir_name}".format(**kwargs)) # new collection should be there self.admin_sessions[0].assert_icommand( "ils -L {collection}/{dir_name}".format(**kwargs), 'STDOUT_SINGLELINE', dir_name) # files should be there rods_files = set(lib.ils_output_to_entries( self.admin_sessions[0].run_icommand(['ils', "{collection}/{dir_name}".format(**kwargs)])[1])) self.assertTrue(set(local_files) == rods_files) # cleanup self.admin_sessions[0].assert_icommand( "irm -rf {collection}/{dir_name}".format(**kwargs)) shutil.rmtree(dir_path)
def test_irsync_r_coll_to_coll(self): # pick session(s) for the test test_session = self.user_sessions[0] # make test dir dir_name = "irsync_test_dir" dir_path = os.path.join(self.local_test_dir_path, dir_name) local_files = lib.make_large_local_tmp_dir( dir_path, self.config["test_file_count"], self.config["test_file_size"] ) # test specific parameters parameters = self.config.copy() parameters["dir_path"] = dir_path parameters["dir_name"] = dir_name parameters["user_name"] = test_session.username parameters["local_home_collection"] = test_session.home_collection parameters["remote_home_collection"] = "/{remote_zone}/home/{user_name}#{local_zone}".format(**parameters) # put dir in local collection test_session.assert_icommand("iput -r {dir_path} {local_home_collection}/".format(**parameters)) # remove local test dir shutil.rmtree(dir_path) # sync local collection with remote collection test_session.assert_icommand( "irsync -r i:{local_home_collection}/{dir_name} i:{remote_home_collection}/{dir_name}".format(**parameters) ) # collection should be there test_session.assert_icommand( "ils -L {remote_home_collection}/{dir_name}".format(**parameters), "STDOUT_SINGLELINE", dir_name ) # files should be there rods_files = set( lib.ils_output_to_entries( test_session.run_icommand(["ils", "{remote_home_collection}/{dir_name}".format(**parameters)])[1] ) ) self.assertTrue(set(local_files) == rods_files) # get collection back test_session.assert_icommand("iget -r {remote_home_collection}/{dir_name} {dir_path}".format(**parameters)) # compare list of files received_files = os.listdir(dir_path) self.assertTrue(set(local_files) == set(received_files)) # cleanup test_session.assert_icommand("irm -rf {local_home_collection}/{dir_name}".format(**parameters)) test_session.assert_icommand("irm -rf {remote_home_collection}/{dir_name}".format(**parameters)) shutil.rmtree(dir_path)
def iput_r_large_collection(self, session, base_name, file_count, file_size): local_dir = os.path.join(self.testing_tmp_dir, base_name) local_files = lib.make_large_local_tmp_dir(local_dir, file_count, file_size) session.assert_icommand(['iput', '-r', local_dir]) rods_files = set(lib.ils_output_to_entries(session.run_icommand(['ils', base_name])[1])) self.assertTrue(set(local_files) == rods_files, msg="Files missing:\n" + str(set(local_files) - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - set(local_files))) vault_files = set(os.listdir(os.path.join(lib.get_vault_session_path(session), base_name))) self.assertTrue(set(local_files) == vault_files, msg="Files missing from vault:\n" + str(set(local_files) - vault_files) + "\n\n" + "Extra files in vault:\n" + str(vault_files - set(local_files))) return (local_dir, local_files)
def test_msirmcoll(self): # pick session(s) for the test test_session = self.user_sessions[0] # make test dir dir_name = "msiRmColl_test_dir" dir_path = os.path.join(self.local_test_dir_path, dir_name) local_files = lib.make_large_local_tmp_dir( dir_path, self.config["test_file_count"], self.config["test_file_size"] ) # test specific parameters parameters = self.config.copy() parameters["dir_path"] = dir_path parameters["dir_name"] = dir_name parameters["user_name"] = test_session.username parameters["remote_home_collection"] = "/{remote_zone}/home/{user_name}#{local_zone}".format(**parameters) # put dir in remote collection test_session.assert_icommand("iput -r {dir_path} {remote_home_collection}/".format(**parameters)) # new collection should be there test_session.assert_icommand( "ils -L {remote_home_collection}/{dir_name}".format(**parameters), "STDOUT_SINGLELINE", dir_name ) # files should be there rods_files = set( lib.ils_output_to_entries( test_session.run_icommand(["ils", "{remote_home_collection}/{dir_name}".format(**parameters)])[1] ) ) self.assertTrue(set(local_files) == rods_files) # prepare irule sequence # the rule is simple enough not to need a rule file irule_str = '''irule "msiRmColl(*coll, 'forceFlag=', *status); writeLine('stdout', *status)" "*coll={remote_home_collection}/{dir_name}" "ruleExecOut"'''.format( **parameters ) # invoke msiRmColl() and checks that it returns 0 test_session.assert_icommand(irule_str, "STDOUT", "^0$", use_regex=True) # collection should be gone test_session.assert_icommand( "ils -L {remote_home_collection}/{dir_name}".format(**parameters), "STDERR_SINGLELINE", "does not exist" ) # cleanup shutil.rmtree(dir_path)
def test_get_null_perms__2833(self): base_name = 'test_dir_for_perms' local_dir = os.path.join(self.testing_tmp_dir, base_name) local_files = lib.make_large_local_tmp_dir(local_dir, 30, 10) self.admin.assert_icommand(['iput', '-r', local_dir]) rods_files = lib.ils_output_to_entries(self.admin.run_icommand(['ils', base_name])[1]) test_dir = 'test_get_null_perms__2833_dir' self.admin.assert_icommand(['ichmod','-r','null',self.admin.username,base_name]) self.admin.assert_icommand(['ichmod','read',self.admin.username,base_name+'/'+rods_files[-1]]) self.admin.assert_icommand(['iget','-r',base_name,test_dir],'STDERR_SINGLELINE','CAT_NO_ACCESS_PERMISSION') assert os.path.isfile(os.path.join(test_dir,'junk0029')) self.admin.assert_icommand(['ichmod','-r','own',self.admin.username,base_name]) lib.run_command(['rm','-rf',test_dir])
def test_iget_from_bundle(self): """ WIP """ # make test dir dir_name = "iphybun_test_dir" file_count = 20 file_size = 4 dir_path = os.path.join(self.local_test_dir_path, dir_name) local_files = lib.make_large_local_tmp_dir(dir_path, file_count, file_size) # make session for existing *remote* user user, password = "******", "rods" remote_session = lib.make_session_for_existing_user( user, password, configuration.FEDERATION.REMOTE_HOST, configuration.FEDERATION.REMOTE_ZONE ) # test specific parameters parameters = self.config.copy() parameters["dir_path"] = dir_path parameters["dir_name"] = dir_name parameters["user_name"] = remote_session.username parameters["remote_home_collection"] = "/{remote_zone}/home/{user_name}".format(**parameters) # put dir in remote collection remote_session.assert_icommand("iput -fr {dir_path} {remote_home_collection}/".format(**parameters)) # new collection should be there remote_session.assert_icommand( "ils -L {remote_home_collection}/{dir_name}".format(**parameters), "STDOUT_SINGLELINE", dir_name ) # list remote home collection remote_session.assert_icommand( "ils -L {remote_home_collection}".format(**parameters), "STDOUT_SINGLELINE", parameters["remote_home_collection"], ) # cleanup remote_session.assert_icommand("irm -rf {remote_home_collection}/{dir_name}".format(**parameters)) shutil.rmtree(dir_path) # close remote session remote_session.__exit__()
def test_irsync_r_coll_to_coll(self): # make test dir dir_name = 'irsync_test_dir' dir_path = os.path.abspath(dir_name) local_files = lib.make_large_local_tmp_dir( dir_path, self.config['test_file_count'], self.config['test_file_size']) kwargs = {'dir_name': dir_name, 'remote_collection': self.config['remote_home_coll'], 'local_collection': self.config['local_home_coll']} # put dir in local collection self.admin_sessions[0].assert_icommand( "iput -r {dir_name} {local_collection}/".format(**kwargs)) # remove local test dir shutil.rmtree(dir_path) # sync local collection with remote collection self.admin_sessions[0].assert_icommand( "irsync -r i:{local_collection}/{dir_name} i:{remote_collection}/{dir_name}".format(**kwargs)) # collection should be there self.admin_sessions[0].assert_icommand( "ils -L {remote_collection}/{dir_name}".format(**kwargs), 'STDOUT_SINGLELINE', dir_name) # files should be there rods_files = set(lib.ils_output_to_entries( self.admin_sessions[0].run_icommand(['ils', "{remote_collection}/{dir_name}".format(**kwargs)])[1])) self.assertTrue(set(local_files) == rods_files) # get collection back self.admin_sessions[0].assert_icommand( "iget -r {remote_collection}/{dir_name}".format(**kwargs)) # compare list of files received_files = os.listdir(dir_path) self.assertTrue(set(local_files) == set(received_files)) # cleanup self.admin_sessions[0].assert_icommand( "irm -rf {local_collection}/{dir_name}".format(**kwargs)) self.admin_sessions[0].assert_icommand( "irm -rf {remote_collection}/{dir_name}".format(**kwargs)) shutil.rmtree(dir_path)
def test_irsync_r_dir_to_coll(self): base_name = "test_irsync_r_dir_to_coll" local_dir = os.path.join(self.testing_tmp_dir, base_name) file_names = set(lib.make_large_local_tmp_dir(local_dir, file_count=1000, file_size=100)) self.user0.assert_icommand("irsync -r " + local_dir + " i:" + base_name, "EMPTY") self.user0.assert_icommand("ils", 'STDOUT_SINGLELINE', base_name) rods_files = set(lib.ils_output_to_entries(self.user0.run_icommand(['ils', base_name])[1])) self.assertTrue(file_names == rods_files, msg="Files missing:\n" + str(file_names - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - file_names)) vault_files_post_irsync = set(os.listdir(os.path.join(lib.get_vault_session_path(self.user0), base_name))) self.assertTrue(file_names == vault_files_post_irsync, msg="Files missing from vault:\n" + str(file_names - vault_files_post_irsync) + "\n\n" + "Extra files in vault:\n" + str(vault_files_post_irsync - file_names))
def test_irm_rf(self): # pick session(s) for the test test_session = self.user_sessions[0] # make test dir dir_name = "irm_test_dir" dir_path = os.path.join(self.local_test_dir_path, dir_name) local_files = lib.make_large_local_tmp_dir( dir_path, self.config["test_file_count"], self.config["test_file_size"] ) # test specific parameters parameters = self.config.copy() parameters["dir_path"] = dir_path parameters["dir_name"] = dir_name parameters["user_name"] = test_session.username parameters["remote_home_collection"] = "/{remote_zone}/home/{user_name}#{local_zone}".format(**parameters) # put dir in remote collection test_session.assert_icommand("iput -r {dir_path} {remote_home_collection}/".format(**parameters)) # new collection should be there test_session.assert_icommand( "ils -L {remote_home_collection}/{dir_name}".format(**parameters), "STDOUT_SINGLELINE", dir_name ) # files should be there rods_files = set( lib.ils_output_to_entries( test_session.run_icommand(["ils", "{remote_home_collection}/{dir_name}".format(**parameters)])[1] ) ) self.assertTrue(set(local_files) == rods_files) # remove remote coll test_session.assert_icommand("irm -rf {remote_home_collection}/{dir_name}".format(**parameters)) # coll should be gone test_session.assert_icommand( "ils -L {remote_home_collection}/{dir_name}".format(**parameters), "STDERR_SINGLELINE", "does not exist" ) # cleanup shutil.rmtree(dir_path)
def test_iput_bulk_check_acpostprocforput__2841(self): # prepare test directory number_of_files = 5 dirname = self.admin.local_session_dir + '/files' corefile = lib.get_core_re_dir() + "/core.re" # files less than 4200000 were failing to trigger the writeLine for filesize in range(5000, 6000000, 500000): files = lib.make_large_local_tmp_dir(dirname, number_of_files, filesize) # manipulate core.re and check the server log with lib.file_backed_up(corefile): initial_size_of_server_log = lib.get_log_size('server') rules_to_prepend = ''' acBulkPutPostProcPolicy { msiSetBulkPutPostProcPolicy("on"); } acPostProcForPut { writeLine("serverLog", "acPostProcForPut called for $objPath"); } ''' time.sleep(1) # remove once file hash fix is committed #2279 lib.prepend_string_to_file(rules_to_prepend, corefile) time.sleep(1) # remove once file hash fix is committed #2279 self.admin.assert_icommand(['iput', '-frb', dirname]) assert number_of_files == lib.count_occurrences_of_string_in_log( 'server', 'writeLine: inString = acPostProcForPut called for', start_index=initial_size_of_server_log) shutil.rmtree(dirname)
def test_cancel_large_iput(self): base_name = 'test_cancel_large_put' local_dir = os.path.join(self.testing_tmp_dir, base_name) file_size = pow(2, 30) file_name = lib.make_large_local_tmp_dir(local_dir, file_count=1, file_size=file_size)[0] file_local_full_path = os.path.join(local_dir, file_name) iput_cmd = "iput '" + file_local_full_path + "'" file_vault_full_path = os.path.join(lib.get_vault_session_path(self.user0), file_name) self.user0.interrupt_icommand(iput_cmd, file_vault_full_path, 10) # multiple threads could still be writing on the server side, so we need to wait for # the size in the vault to converge - then were done. old_size = None new_size = os.path.getsize(file_vault_full_path) while old_size != new_size: time.sleep(1) old_size = new_size new_size = os.path.getsize(file_vault_full_path) # wait for select() call to timeout, set to "SELECT_TIMEOUT_FOR_CONN", which is 60 seconds time.sleep(63) self.user0.assert_icommand('ils -l', 'STDOUT_SINGLELINE', [file_name, str(new_size)])
def iput_r_large_collection(self, session, base_name, file_count, file_size): local_dir = os.path.join(self.testing_tmp_dir, base_name) local_files = lib.make_large_local_tmp_dir(local_dir, file_count, file_size) session.assert_icommand(['iput', '-r', local_dir]) rods_files = set( lib.ils_output_to_entries( session.run_icommand(['ils', base_name])[1])) self.assertTrue(set(local_files) == rods_files, msg="Files missing:\n" + str(set(local_files) - rods_files) + "\n\n" + "Extra files:\n" + str(rods_files - set(local_files))) vault_files = set( os.listdir( os.path.join(lib.get_vault_session_path(session), base_name))) self.assertTrue(set(local_files) == vault_files, msg="Files missing from vault:\n" + str(set(local_files) - vault_files) + "\n\n" + "Extra files in vault:\n" + str(vault_files - set(local_files))) return (local_dir, local_files)