def test_irepl_over_existing_bad_replica__ticket_1705(self):
# local setup
filename = "reploverwritebad.txt"
filepath = lib.create_local_testfile(filename)
doublefile = "doublefile.txt"
os.system("cat %s %s > %s" % (filename, filename, doublefile))
doublesize = str(os.stat(doublefile).st_size)
# assertions
self.admin.assert_icommand("ils -L " + filename, 'STDERR', "does not exist") # should not be listed
self.admin.assert_icommand("iput " + filename) # put file
self.admin.assert_icommand("ils -L " + filename, 'STDOUT', filename) # for debugging
self.admin.assert_icommand("irepl -R " + self.testresc + " " + filename) # replicate to test resource
self.admin.assert_icommand("ils -L " + filename, 'STDOUT', filename) # for debugging
# overwrite default repl with different data
self.admin.assert_icommand("iput -f %s %s" % (doublefile, filename))
# default resource should have clean copy
self.admin.assert_icommand("ils -L " + filename, 'STDOUT', [" 0 ", " & " + filename])
# default resource should have new double clean copy
self.admin.assert_icommand("ils -L " + filename, 'STDOUT', [" 0 ", " " + doublesize + " ", " & " + filename])
# test resource should not have doublesize file
self.admin.assert_icommand_fail("ils -L " + filename, 'STDOUT',
[" 1 " + self.testresc, " " + doublesize + " ", " " + filename])
# replicate back onto test resource
self.admin.assert_icommand("irepl -R " + self.testresc + " " + filename)
# test resource should have new clean doublesize file
self.admin.assert_icommand("ils -L " + filename, 'STDOUT',
[" 1 " + self.testresc, " " + doublesize + " ", " & " + filename])
# should not have a replica 2
self.admin.assert_icommand_fail("ils -L " + filename, 'STDOUT', [" 2 ", " & " + filename])
# local cleanup
os.remove(filepath)
os.remove(doublefile)
def test_ssl_iput_with_rods_env(self):
lib.run_command('openssl genrsa -out server.key')
lib.run_command('openssl req -batch -new -key server.key -out server.csr')
lib.run_command('openssl req -batch -new -x509 -key server.key -out chain.pem -days 365')
lib.run_command('openssl dhparam -2 -out dhparams.pem 100') # normally 2048, but smaller size here for speed
service_account_environment_file_path = os.path.expanduser('~/.irods/irods_environment.json')
with lib.file_backed_up(service_account_environment_file_path):
server_update = {
'irods_ssl_certificate_chain_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/chain.pem'),
'irods_ssl_certificate_key_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/server.key'),
'irods_ssl_dh_params_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/dhparams.pem'),
}
lib.update_json_file_from_dict(service_account_environment_file_path, server_update)
client_update = {
'irods_client_server_policy': 'CS_NEG_REQUIRE',
'irods_ssl_verify_server': 'none',
}
session_env_backup = copy.deepcopy(self.admin.environment_file_contents)
self.admin.environment_file_contents.update(client_update)
filename = 'encryptedfile.txt'
filepath = lib.create_local_testfile(filename)
self.admin.assert_icommand(['iinit', self.admin.password])
self.admin.assert_icommand(['iput', filename])
self.admin.assert_icommand(['ils', '-L', filename], 'STDOUT', filename)
self.admin.environment_file_contents = session_env_backup
for f in ['server.key', 'server.csr', 'chain.pem', 'dhparams.pem']:
os.unlink(f)
lib.restart_irods_server()
def test_irepl_over_existing_third_replica__ticket_1705(self):
# local setup
filename = "thirdreplicatest.txt"
filepath = lib.create_local_testfile(filename)
hostname = lib.get_hostname()
hostuser = getpass.getuser()
# assertions
self.admin.assert_icommand("iadmin mkresc thirdresc unixfilesystem %s:/tmp/%s/thirdrescVault" %
(hostname, hostuser), 'STDOUT', "Creating") # create third resource
self.admin.assert_icommand("ils -L " + filename, 'STDERR', "does not exist") # should not be listed
self.admin.assert_icommand("iput " + filename) # put file
self.admin.assert_icommand("irepl -R " + self.testresc + " " + filename) # replicate to test resource
self.admin.assert_icommand("irepl -R thirdresc " + filename) # replicate to third resource
self.admin.assert_icommand("irepl " + filename) # replicate overtop default resource
self.admin.assert_icommand("ils -L " + filename, 'STDOUT', filename) # for debugging
self.admin.assert_icommand("irepl -R " + self.testresc + " " + filename) # replicate overtop test resource
self.admin.assert_icommand("ils -L " + filename, 'STDOUT', filename) # for debugging
self.admin.assert_icommand("irepl -R thirdresc " + filename) # replicate overtop third resource
self.admin.assert_icommand("ils -L " + filename, 'STDOUT', filename) # for debugging
# should not have a replica 3
self.admin.assert_icommand_fail("ils -L " + filename, 'STDOUT', [" 3 ", " & " + filename])
# should not have a replica 4
self.admin.assert_icommand_fail("ils -L " + filename, 'STDOUT', [" 4 ", " & " + filename])
self.admin.assert_icommand("irm -f " + filename) # cleanup file
self.admin.assert_icommand("iadmin rmresc thirdresc") # remove third resource
# local cleanup
os.remove(filepath)
def test_iput_overwrite_others_file__ticket_2086(self):
# pre state
self.admin.assert_icommand("ils -L", 'STDOUT', self.testfile) # should be listed
# local setup
filename = "overwritefile.txt"
filepath = lib.create_local_testfile(filename)
# alice tries to put
homepath = "/home/" + self.admin.username + "/" + self.admin._session_id + "/" + self.testfile
logicalpath = "/" + self.admin.zone_name + homepath
self.user0.assert_icommand("iput " + filepath + " " + logicalpath, 'STDERR', "CAT_NO_ACCESS_PERMISSION") # iput
# check physicalpaths (of all replicas)
cmdout = self.admin.run_icommand(['ils', '-L'])
print "[ils -L]:"
print "[" + cmdout[1] + "]"
lines = cmdout[1].splitlines()
for i in range(0, len(lines) - 1):
if "0 demoResc" in lines[i]:
if "/session-" in lines[i + 1]:
l = lines[i + 1]
physicalpath = l.split()[1]
# check file is on disk
print "[ls -l " + physicalpath + "]:"
os.system("ls -l " + physicalpath)
assert os.path.exists(physicalpath)
# local cleanup
output = commands.getstatusoutput('rm ' + filepath)
def test_force_iput_to_diff_resc(self):
filename = "original.txt"
filepath = lib.create_local_testfile(filename)
self.admin.assert_icommand("iput " + filename) # put file
self.admin.assert_icommand("iput -f " + filename) # put file
self.admin.assert_icommand("iput -fR " + self.testresc + " " +
filename, 'STDERR_SINGLELINE',
'HIERARCHY_ERROR') # fail
def test_ssl_iput_small_and_large_files(self):
# set up client and server side for ssl handshake
# server side certificate setup
os.system("openssl genrsa -out server.key 2> /dev/null")
os.system("openssl req -batch -new -key server.key -out server.csr")
os.system("openssl req -batch -new -x509 -key server.key -out server.crt -days 365")
os.system("mv server.crt chain.pem")
# normally 2048, but smaller size here for speed
os.system("openssl dhparam -2 -out dhparams.pem 100 2> /dev/null")
# server side environment variables
os.environ['irodsSSLCertificateChainFile'] = lib.get_irods_top_level_dir() + "/tests/pydevtest/chain.pem"
os.environ['irodsSSLCertificateKeyFile'] = lib.get_irods_top_level_dir() + "/tests/pydevtest/server.key"
os.environ['irodsSSLDHParamsFile'] = lib.get_irods_top_level_dir() + "/tests/pydevtest/dhparams.pem"
# client side environment variables
os.environ['irodsSSLVerifyServer'] = "none"
# add client irodsEnv settings
clientEnvFile = self.admin.local_session_dir + "/irods_environment.json"
os.system("cp %s %sOrig" % (clientEnvFile, clientEnvFile))
env = {}
env['irods_client_server_policy'] = 'CS_NEG_REQUIRE'
lib.update_json_file_from_dict(clientEnvFile, env)
# server reboot to pick up new irodsEnv settings
lib.restart_irods_server()
# do the encrypted put
filename = "encryptedfile.txt"
filepath = lib.create_local_testfile(filename)
self.admin.assert_icommand(['iinit', self.admin.password]) # reinitialize
# small file
self.admin.assert_icommand("iput " + filename) # encrypted put - small file
self.admin.assert_icommand("ils -L " + filename, 'STDOUT', filename) # should be listed
# large file
largefilename = "BIGencryptedfile.txt"
output = commands.getstatusoutput('dd if=/dev/zero of=' + largefilename + ' bs=1M count=60')
assert output[0] == 0, "dd did not successfully exit"
#os.system("ls -al "+largefilename)
self.admin.assert_icommand("iput " + largefilename) # encrypted put - large file
self.admin.assert_icommand("ils -L " + largefilename, 'STDOUT', largefilename) # should be listed
# reset client environment to not require SSL
os.system("mv %sOrig %s" % (clientEnvFile, clientEnvFile))
# clean up
os.system("rm server.key server.csr chain.pem dhparams.pem")
os.remove(filename)
os.remove(largefilename)
# restart iRODS server without altered environment
lib.restart_irods_server()
def test_iget_with_purgec(self):
# local setup
filename = "purgecgetfile.txt"
filepath = lib.create_local_testfile(filename)
# assertions
self.admin.assert_icommand("ils -L " + filename, 'STDERR', "does not exist") # should not be listed
self.admin.assert_icommand("iput " + filename) # put file
self.admin.assert_icommand("iget -f --purgec " + filename) # get file
self.admin.assert_icommand("ils -L " + filename, 'STDOUT', [" 0 ", filename]) # should be listed once
self.admin.assert_icommand_fail("ils -L " + filename, 'STDOUT', [" 1 ", filename]) # should be listed only once
self.admin.assert_icommand_fail("ils -L " + filename, 'STDOUT', [" 2 ", filename]) # should be listed only once
# local cleanup
output = commands.getstatusoutput('rm ' + filepath)
def test_iget_with_dirty_replica(self):
# local setup
filename = "original.txt"
filepath = lib.create_local_testfile(filename)
updated_filename = "updated_file_with_longer_filename.txt"
updated_filepath = lib.create_local_testfile(updated_filename)
retrievedfile = "retrievedfile.txt"
# assertions
self.admin.assert_icommand("ils -L " + filename, 'STDERR', "does not exist") # should not be listed
self.admin.assert_icommand("iput " + filename) # put file
self.admin.assert_icommand("irepl -R " + self.testresc + " " + filename) # replicate file
# force new put on second resource
self.admin.assert_icommand("iput -f -R " + self.testresc + " " + updated_filename + " " + filename)
self.admin.assert_icommand("ils -L " + filename, "STDOUT", filename) # debugging
# should get orig file (replica 0)
self.admin.assert_icommand("iget -f -n 0 " + filename + " " + retrievedfile)
assert 0 == os.system("diff %s %s" % (filename, retrievedfile)) # confirm retrieved is correct
self.admin.assert_icommand("iget -f " + filename + " " + retrievedfile) # should get updated file
assert 0 == os.system("diff %s %s" % (updated_filename, retrievedfile)) # confirm retrieved is correct
# local cleanup
output = commands.getstatusoutput('rm ' + filename)
output = commands.getstatusoutput('rm ' + updated_filename)
output = commands.getstatusoutput('rm ' + retrievedfile)
def test_irepl_update_replicas(self):
# local setup
filename = "updatereplicasfile.txt"
filepath = lib.create_local_testfile(filename)
hostname = lib.get_hostname()
doublefile = "doublefile.txt"
os.system("cat %s %s > %s" % (filename, filename, doublefile))
doublesize = str(os.stat(doublefile).st_size)
# assertions
self.admin.assert_icommand("iadmin mkresc thirdresc unixfilesystem %s:/tmp/thirdrescVault" % hostname, 'STDOUT_SINGLELINE', "Creating") # create third resource
self.admin.assert_icommand("iadmin mkresc fourthresc unixfilesystem %s:/tmp/fourthrescVault" % hostname, 'STDOUT_SINGLELINE', "Creating") # create fourth resource
self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed
self.admin.assert_icommand("iput "+filename) # put file
self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource
self.admin.assert_icommand("irepl -R thirdresc "+filename) # replicate to third resource
self.admin.assert_icommand("irepl -R fourthresc "+filename) # replicate to fourth resource
self.admin.assert_icommand("iput -f -R "+self.testresc+" "+doublefile+" "+filename) # repave overtop test resource
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand("irepl -U "+filename) # update last replica
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("irepl -aU "+filename) # update all replicas
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("irm -f "+filename) # cleanup file
self.admin.assert_icommand("iadmin rmresc thirdresc") # remove third resource
self.admin.assert_icommand("iadmin rmresc fourthresc") # remove third resource
# local cleanup
os.remove(filepath)
os.remove(doublefile)
def test_irepl_invalid_input(self):
# local setup
filename = "somefile.txt"
filepath = lib.create_local_testfile(filename)
# assertions
# should not be listed
self.admin.assert_icommand("ils -L " + filename, "STDERR", "does not exist")
self.admin.assert_icommand("iput " + filename) # put file
# for debugging
self.admin.assert_icommand("ils -L " + filename, "STDOUT", filename)
# replicate to bad resource
self.admin.assert_icommand("irepl -R nonresc " + filename, "STDERR", "SYS_RESC_DOES_NOT_EXIST")
self.admin.assert_icommand("irm -f " + filename) # cleanup file
# local cleanup
os.remove(filepath)
def test_empty_files(self):
# set up
filename = "some_test_file.txt"
filepath = lib.create_local_testfile(filename)
emptyfile = "emptyfile.txt"
lib.assert_command("touch "+emptyfile)
# test it
self.admin.assert_icommand("iput "+emptyfile)
self.admin.assert_icommand("iput -f "+filepath+" "+emptyfile)
self.admin.assert_icommand("ils -L "+" "+emptyfile,"STDOUT_MULTILINE",[" 0 demoResc;cacheResc 63 ", " 1 demoResc;archiveResc 63 "])
os.remove(filepath)
os.remove(emptyfile)
def test_irepl_over_existing_second_replica__ticket_1705(self):
# local setup
filename = "secondreplicatest.txt"
filepath = lib.create_local_testfile(filename)
# assertions
self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed
self.admin.assert_icommand("iput -R "+self.testresc+" "+filename) # put file
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging
self.admin.assert_icommand("irepl "+filename) # replicate to default resource
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging
self.admin.assert_icommand("irepl "+filename) # replicate overtop default resource
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should not have a replica 3
self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate overtop test resource
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should not have a replica 3
# local cleanup
os.remove(filepath)
def test_retry_for_put(self):
# set up
self.admin.assert_icommand("iadmin modresc archiveResc context wos_host=XXXX;wos_policy=Howard;retry_count=2;connect_timeout=5")
filename = "some_test_file.txt"
filepath = lib.create_local_testfile(filename)
# test it
self.admin.assert_icommand( "iput -f "+filepath, 'STDERR_SINGLELINE', "WOS_PUT_ERR")
# verify it
p = subprocess.Popen(['grep "WOS_PUT_ERR" ../../iRODS/server/log/rodsLog.* | grep "2 of 2"'], shell=True, stdout=subprocess.PIPE)
result = p.communicate()[0]
assert( -1 != result.find( "2 of 2" ) )
# clean up
self.admin.assert_icommand("iadmin modresc archiveResc context wos_host=http://wos.edc.renci.org;wos_policy=Howard")
os.remove(filepath)
def test_local_iput_with_force_and_destination_resource__ticket_1706(self):
# local setup
filename = "iputwithforceanddestination.txt"
filepath = lib.create_local_testfile(filename)
doublefile = "doublefile.txt"
os.system("cat %s %s > %s" % (filename, filename, doublefile))
doublesize = str(os.stat(doublefile).st_size)
# assertions
self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed
self.admin.assert_icommand("iput "+filename) # put file
self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) #
self.admin.assert_icommand("iput -f -R %s %s %s" % (self.testresc, doublefile, filename) ) # overwrite test repl with different data
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," "+filename]) # default resource cache should have dirty copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," "+filename]) # default resource archive should have dirty copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," "+doublesize+" "," "+filename]) # default resource cache should not have doublesize file
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," "+doublesize+" "," "+filename]) # default resource archive should not have doublesize file
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," "+doublesize+" ","& "+filename]) # targeted resource should have new double clean copy
# local cleanup
os.remove(filepath)
os.remove(doublefile)
def test_force_iput_to_diff_resc(self):
filename = "original.txt"
filepath = lib.create_local_testfile(filename)
self.admin.assert_icommand("iput " + filename) # put file
self.admin.assert_icommand("iput -f " + filename) # put file
self.admin.assert_icommand("iput -fR " + self.testresc + " " + filename, 'STDERR_SINGLELINE', 'HIERARCHY_ERROR') # fail
def iput_to_root_collection(self):
filename = 'original.txt'
filepath = lib.create_local_testfile(filename)
self.admin.assert_icommand('iput ' + filename + ' /nopes', 'STDERR_SINGLELINE', 'SYS_INVALID_INPUT_PARAM')
