def test_configuration_schema_validation_from_file(self):
schemas_git_dir = tempfile.mkdtemp(prefix='irods-test_configuration_schema_validation_from_file-git')
with lib.directory_deleter(schemas_git_dir):
schemas_repo = 'https://github.com/irods/irods_schema_configuration'
lib.run_command(['git', 'clone', schemas_repo, schemas_git_dir])
schemas_branch = 'v3'
lib.run_command(['git', 'checkout', schemas_branch], cwd=schemas_git_dir)
schemas_deploy_dir = tempfile.mkdtemp(prefix='irods-test_configuration_schema_validation_from_file-schemas')
with lib.directory_deleter(schemas_deploy_dir):
lib.assert_command(['python', os.path.join(schemas_git_dir, 'deploy_schemas_locally.py'), '--output_directory_base', schemas_deploy_dir])
with lib.file_backed_up(os.path.join(lib.get_irods_config_dir(), 'server_config.json')) as server_config_filename:
with open(server_config_filename) as f:
server_config = json.load(f)
server_config['schema_validation_base_uri'] = 'file://' + schemas_deploy_dir
lib.update_json_file_from_dict(server_config_filename, server_config)
irodsctl_fullpath = os.path.join(lib.get_irods_top_level_dir(), 'iRODS', 'irodsctl')
if lib.is_jsonschema_installed():
expected_lines = ['Validating [{0}]... Success'.format(os.path.expanduser('~/.irods/irods_environment.json')),
'Validating [{0}/server_config.json]... Success'.format(lib.get_irods_config_dir()),
'Validating [{0}/VERSION.json]... Success'.format(lib.get_irods_top_level_dir()),
'Validating [{0}/hosts_config.json]... Success'.format(lib.get_irods_config_dir()),
'Validating [{0}/host_access_control_config.json]... Success'.format(lib.get_irods_config_dir())]
if not configuration.TOPOLOGY_FROM_RESOURCE_SERVER:
expected_lines.append('Validating [{0}/database_config.json]... Success'.format(lib.get_irods_config_dir()))
lib.assert_command([irodsctl_fullpath, 'restart'], 'STDOUT_MULTILINE', expected_lines)
else:
lib.assert_command([irodsctl_fullpath, 'restart'], 'STDERR_SINGLELINE', 'jsonschema not installed', desired_rc=0)
def test_authentication_PAM_with_server_params(self):
## set up client and server side for ssl handshake
# server side certificate setup
os.system('openssl genrsa -out server.key')
os.system('openssl req -batch -new -x509 -key server.key -out server.crt -days 365')
os.system('mv server.crt chain.pem')
os.system('openssl dhparam -2 -out dhparams.pem 100') # normally 2048, but smaller size here for speed
# server side environment variables
os.environ['irodsSSLCertificateChainFile'] = lib.get_irods_top_level_dir() + '/tests/pydevtest/chain.pem'
os.environ['irodsSSLCertificateKeyFile'] = lib.get_irods_top_level_dir() + '/tests/pydevtest/server.key'
os.environ['irodsSSLDHParamsFile'] = lib.get_irods_top_level_dir() + '/tests/pydevtest/dhparams.pem'
# client side environment variables
backup_env_contents = copy.deepcopy(self.auth_session.environment_file_contents)
self.auth_session.environment_file_contents['irods_ssl_verify_server'] = 'none'
self.auth_session.environment_file_contents['irods_client_server_policy'] = 'CS_NEG_REQUIRE'
self.auth_session.environment_file_contents['irods_authentication_scheme'] = 'PaM'
# add server_config.json settings
serverConfigFile = lib.get_irods_config_dir() + "/server_config.json"
with open(serverConfigFile) as f:
contents = json.load(f)
os.system("cp %s %sOrig" % (serverConfigFile, serverConfigFile))
contents['pam_password_length'] = 20
contents['pam_no_extend'] = False
contents['pam_password_min_time'] = 121
contents['pam_password_max_time'] = 1209600
with open(serverConfigFile, 'w') as f:
json.dump(contents, f)
# server reboot to pick up new irodsEnv and server settings
lib.restart_irods_server()
# do the reauth
self.auth_session.assert_icommand(['iinit', self.auth_session.password])
# connect and list some files
self.auth_session.assert_icommand("icd")
self.auth_session.assert_icommand("ils -L", 'STDOUT', "home")
# reset client environment to original
self.auth_session.environment_file_contents = backup_env_contents
# clean up
for file in ['server.key', 'chain.pem', 'dhparams.pem']:
os.unlink(file)
# reset server_config.json to original
os.system('mv %sOrig %s' % (serverConfigFile, serverConfigFile))
# server reboot to revert to previous server configuration
os.system(lib.get_irods_top_level_dir() + '/iRODS/irodsctl stop')
os.system(lib.get_irods_top_level_dir() + '/tests/zombiereaper.sh')
os.system(lib.get_irods_top_level_dir() + '/iRODS/irodsctl start')
def test_rulebase_update__2585(self):
rule_file = 'my_rule.r'
test_re = os.path.join(lib.get_core_re_dir(), 'test.re')
my_rule = """
my_rule {
delay("<PLUSET>1s</PLUSET>") {
do_some_stuff();
}
}
INPUT null
OUTPUT ruleExecOut
"""
with open(rule_file, 'w') as f:
f.write(my_rule)
server_config_filename = lib.get_irods_config_dir() + '/server_config.json'
with lib.file_backed_up(server_config_filename):
# write new rule file to config dir
test_rule = 'do_some_stuff() { writeLine( "serverLog", "TEST_STRING_TO_FIND_1_2585" ); }'
with open(test_re, 'w') as f:
f.write(test_rule)
# update server config with additional rule file
server_config_update = {
"re_rulebase_set": [{"filename": "test"}, {"filename": "core"}]
}
lib.update_json_file_from_dict(server_config_filename, server_config_update)
time.sleep(35) # wait for delay rule engine to wake
# checkpoint log to know where to look for the string
initial_log_size = lib.get_log_size('re')
self.admin.assert_icommand('irule -F ' + rule_file)
time.sleep(35) # wait for test to fire
assert lib.count_occurrences_of_string_in_log('re', 'TEST_STRING_TO_FIND_1_2585', start_index=initial_log_size)
# repave rule with new string
test_rule = 'do_some_stuff() { writeLine( "serverLog", "TEST_STRING_TO_FIND_2_2585" ); }'
os.unlink(test_re)
with open(test_re, 'w') as f:
f.write(test_rule)
time.sleep(35) # wait for delay rule engine to wake
# checkpoint log to know where to look for the string
initial_log_size = lib.get_log_size('re')
self.admin.assert_icommand('irule -F ' + rule_file)
time.sleep(35) # wait for test to fire
assert lib.count_occurrences_of_string_in_log('re', 'TEST_STRING_TO_FIND_2_2585', start_index=initial_log_size)
# cleanup
os.unlink(test_re)
os.unlink(rule_file)
def test_authentication_PAM_with_server_params(self):
lib.run_command('openssl genrsa -out server.key')
lib.run_command('openssl req -batch -new -key server.key -out server.csr')
lib.run_command('openssl req -batch -new -x509 -key server.key -out chain.pem -days 365')
lib.run_command('openssl dhparam -2 -out dhparams.pem 1024') # normally 2048, but smaller size here for speed
service_account_environment_file_path = os.path.expanduser('~/.irods/irods_environment.json')
with lib.file_backed_up(service_account_environment_file_path):
server_update = {
'irods_ssl_certificate_chain_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/chain.pem'),
'irods_ssl_certificate_key_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/server.key'),
'irods_ssl_dh_params_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/dhparams.pem'),
'irods_ssl_verify_server': 'none',
}
lib.update_json_file_from_dict(service_account_environment_file_path, server_update)
client_update = {
'irods_ssl_certificate_chain_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/chain.pem'),
'irods_ssl_certificate_key_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/server.key'),
'irods_ssl_dh_params_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/dhparams.pem'),
'irods_ssl_verify_server': 'none',
'irods_authentication_scheme': 'PaM',
'irods_client_server_policy': 'CS_NEG_REQUIRE',
}
auth_session_env_backup = copy.deepcopy(self.auth_session.environment_file_contents)
self.auth_session.environment_file_contents.update(client_update)
server_config_filename = lib.get_irods_config_dir() + '/server_config.json'
with lib.file_backed_up(server_config_filename):
server_config_update = {
'pam_password_length': 20,
'pam_no_extend': False,
'pam_password_min_time': 121,
'pam_password_max_time': 1209600,
}
lib.update_json_file_from_dict(server_config_filename, server_config_update)
lib.restart_irods_server()
# the test
self.auth_session.assert_icommand(['iinit', self.auth_session.password])
self.auth_session.assert_icommand("icd")
self.auth_session.assert_icommand("ils -L", 'STDOUT_SINGLELINE', "home")
self.auth_session.environment_file_contents = auth_session_env_backup
for file in ['tests/pydevtest/server.key', 'tests/pydevtest/chain.pem', 'tests/pydevtest/dhparams.pem']:
os.unlink(os.path.join(lib.get_irods_top_level_dir(), file))
lib.restart_irods_server()
def test_authentication_PAM_with_server_params(self):
lib.run_command('openssl genrsa -out server.key')
lib.run_command('openssl req -batch -new -key server.key -out server.csr')
lib.run_command('openssl req -batch -new -x509 -key server.key -out chain.pem -days 365')
lib.run_command('openssl dhparam -2 -out dhparams.pem 100') # normally 2048, but smaller size here for speed
service_account_environment_file_path = os.path.expanduser('~/.irods/irods_environment.json')
with lib.file_backed_up(service_account_environment_file_path):
server_update = {
'irods_ssl_certificate_chain_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/chain.pem'),
'irods_ssl_certificate_key_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/server.key'),
'irods_ssl_dh_params_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/dhparams.pem'),
'irods_ssl_verify_server': 'none',
}
lib.update_json_file_from_dict(service_account_environment_file_path, server_update)
client_update = {
'irods_ssl_certificate_chain_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/chain.pem'),
'irods_ssl_certificate_key_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/server.key'),
'irods_ssl_dh_params_file': os.path.join(lib.get_irods_top_level_dir(), 'tests/pydevtest/dhparams.pem'),
'irods_ssl_verify_server': 'none',
'irods_authentication_scheme': 'PaM',
'irods_client_server_policy': 'CS_NEG_REQUIRE',
}
auth_session_env_backup = copy.deepcopy(self.auth_session.environment_file_contents)
self.auth_session.environment_file_contents.update(client_update)
server_config_filename = lib.get_irods_config_dir() + '/server_config.json'
with lib.file_backed_up(server_config_filename):
server_config_update = {
'pam_password_length': 20,
'pam_no_extend': False,
'pam_password_min_time': 121,
'pam_password_max_time': 1209600,
}
lib.update_json_file_from_dict(server_config_filename, server_config_update)
lib.restart_irods_server()
# the test
self.auth_session.assert_icommand(['iinit', self.auth_session.password])
self.auth_session.assert_icommand("icd")
self.auth_session.assert_icommand("ils -L", 'STDOUT_SINGLELINE', "home")
self.auth_session.environment_file_contents = auth_session_env_backup
for file in ['tests/pydevtest/server.key', 'tests/pydevtest/chain.pem', 'tests/pydevtest/dhparams.pem']:
os.unlink(os.path.join(lib.get_irods_top_level_dir(), file))
lib.restart_irods_server()
def test_configuration_schema_validation_from_file(self):
schemas_git_dir = tempfile.mkdtemp(
prefix='irods-test_configuration_schema_validation_from_file-git')
with lib.directory_deleter(schemas_git_dir):
schemas_repo = 'https://github.com/irods/irods_schema_configuration'
lib.run_command(['git', 'clone', schemas_repo, schemas_git_dir])
schemas_branch = 'v3'
lib.run_command(['git', 'checkout', schemas_branch],
cwd=schemas_git_dir)
schemas_deploy_dir = tempfile.mkdtemp(
prefix=
'irods-test_configuration_schema_validation_from_file-schemas')
with lib.directory_deleter(schemas_deploy_dir):
lib.assert_command([
'python',
os.path.join(schemas_git_dir, 'deploy_schemas_locally.py'),
'--output_directory_base', schemas_deploy_dir
])
with lib.file_backed_up(
os.path.join(
lib.get_irods_config_dir(),
'server_config.json')) as server_config_filename:
with open(server_config_filename) as f:
server_config = json.load(f)
server_config[
'schema_validation_base_uri'] = 'file://' + schemas_deploy_dir
lib.update_json_file_from_dict(server_config_filename,
server_config)
irodsctl_fullpath = os.path.join(
lib.get_irods_top_level_dir(), 'iRODS', 'irodsctl')
if lib.is_jsonschema_installed():
expected_lines = [
'Validating [/var/lib/irods/.irods/irods_environment.json]... Success',
'Validating [/etc/irods/server_config.json]... Success',
'Validating [/var/lib/irods/VERSION.json]... Success',
'Validating [/etc/irods/hosts_config.json]... Success',
'Validating [/etc/irods/host_access_control_config.json]... Success'
]
if not configuration.TOPOLOGY_FROM_RESOURCE_SERVER:
expected_lines.append(
'Validating [/etc/irods/database_config.json]... Success'
)
lib.assert_command([irodsctl_fullpath, 'restart'],
'STDOUT_MULTILINE', expected_lines)
else:
lib.assert_command([irodsctl_fullpath, 'restart'],
'STDERR_SINGLELINE',
'jsonschema not installed',
desired_rc=0)
class Test_Xmsg(unittest.TestCase):
serverConfigFile = lib.get_irods_config_dir() + '/server_config.json'
serverConfigFileBackup = serverConfigFile + '_orig'
xmsgHost = 'localhost'
xmsgPort = 1279
def setUp(self):
# add Xmsg settings to server_config.json
shutil.copyfile(self.serverConfigFile, self.serverConfigFileBackup)
contents = lib.open_and_load_json_ascii(self.serverConfigFile)
update = {
'xmsg_host': self.xmsgHost,
'xmsg_port': self.xmsgPort,
}
lib.update_json_file_from_dict(self.serverConfigFile, update)
# apparently needed by the server too...
my_env = os.environ.copy()
my_env['XMSG_HOST'] = self.xmsgHost
my_env['XMSG_PORT'] = str(self.xmsgPort)
lib.restart_irods_server(env=my_env)
def tearDown(self):
os.rename(self.serverConfigFileBackup, self.serverConfigFile)
lib.restart_irods_server()
@unittest.skipIf(configuration.TOPOLOGY_FROM_RESOURCE_SERVER
or configuration.USE_SSL,
"Skip for topology testing from resource server or SSL")
def test_send_and_receive_one_xmsg(self):
message = 'Hello World!'
# set up Xmsg in client environment
my_env = os.environ.copy()
my_env['XMSG_HOST'] = self.xmsgHost
my_env['XMSG_PORT'] = str(self.xmsgPort)
# send msg
args = ['/usr/bin/ixmsg', 's', '-M "{0}"'.format(message)]
subprocess.Popen(args, env=my_env).communicate()
# receive msg
args = ['/usr/bin/ixmsg', 'r', '-n 1']
res = subprocess.Popen(args, env=my_env,
stdout=subprocess.PIPE).communicate()
# assertion
print 'looking for "{0}" in "{1}"'.format(message, res[0].rstrip())
assert res[0].find(message) >= 0
def test_authentication_OSAuth(self):
self.auth_session.environment_file_contents['irods_authentication_scheme'] = 'OSAuth'
# setup the irods.key file necessary for OSAuth
keyfile_path = os.path.join(lib.get_irods_config_dir(), 'irods.key')
with open(keyfile_path, 'w') as f:
f.write('gibberish\n')
# do the reauth
self.auth_session.assert_icommand('iexit full')
self.auth_session.assert_icommand(['iinit', self.auth_session.password])
# connect and list some files
self.auth_session.assert_icommand('icd')
self.auth_session.assert_icommand('ils -L', 'STDOUT_SINGLELINE', 'home')
# reset client environment to original
del self.auth_session.environment_file_contents['irods_authentication_scheme']
# clean up keyfile
os.unlink(keyfile_path)
def test_authentication_OSAuth(self):
self.auth_session.environment_file_contents['irods_authentication_scheme'] = 'OSAuth'
# setup the irods.key file necessary for OSAuth
keyfile_path = os.path.join(lib.get_irods_config_dir(), 'irods.key')
with open(keyfile_path, 'w') as f:
f.write('gibberish\n')
# do the reauth
self.auth_session.assert_icommand('iexit full')
self.auth_session.assert_icommand(['iinit', self.auth_session.password])
# connect and list some files
self.auth_session.assert_icommand('icd')
self.auth_session.assert_icommand('ils -L', 'STDOUT_SINGLELINE', 'home')
# reset client environment to original
del self.auth_session.environment_file_contents['irods_authentication_scheme']
# clean up keyfile
os.unlink(keyfile_path)
def test_rulebase_update__2585(self):
rule_file = 'my_rule.r'
test_re = os.path.join(lib.get_core_re_dir(), 'test.re')
my_rule = """
my_rule {
delay("<PLUSET>1s</PLUSET>") {
do_some_stuff();
}
}
INPUT null
OUTPUT ruleExecOut
"""
with open(rule_file, 'w') as f:
f.write(my_rule)
server_config_filename = lib.get_irods_config_dir(
) + '/server_config.json'
with lib.file_backed_up(server_config_filename):
# write new rule file to config dir
test_rule = 'do_some_stuff() { writeLine( "serverLog", "TEST_STRING_TO_FIND_1_2585" ); }'
with open(test_re, 'w') as f:
f.write(test_rule)
# update server config with additional rule file
server_config_update = {
"re_rulebase_set": [{
"filename": "test"
}, {
"filename": "core"
}]
}
lib.update_json_file_from_dict(server_config_filename,
server_config_update)
time.sleep(35) # wait for delay rule engine to wake
# checkpoint log to know where to look for the string
initial_log_size = lib.get_log_size('re')
self.admin.assert_icommand('irule -F ' + rule_file)
time.sleep(35) # wait for test to fire
assert lib.count_occurrences_of_string_in_log(
're',
'TEST_STRING_TO_FIND_1_2585',
start_index=initial_log_size)
# repave rule with new string
test_rule = 'do_some_stuff() { writeLine( "serverLog", "TEST_STRING_TO_FIND_2_2585" ); }'
os.unlink(test_re)
with open(test_re, 'w') as f:
f.write(test_rule)
time.sleep(35) # wait for delay rule engine to wake
# checkpoint log to know where to look for the string
initial_log_size = lib.get_log_size('re')
self.admin.assert_icommand('irule -F ' + rule_file)
time.sleep(35) # wait for test to fire
assert lib.count_occurrences_of_string_in_log(
're',
'TEST_STRING_TO_FIND_2_2585',
start_index=initial_log_size)
# cleanup
os.unlink(test_re)
os.unlink(rule_file)
def test_acPostProcForOpen__3024(self):
test_re = os.path.join(lib.get_core_re_dir(), 'test.re')
server_config_filename = lib.get_irods_config_dir() + '/server_config.json'
# get PEP name from function name
pep_name = inspect.stack()[0][3].split('_')[1]
# user session
sesh = self.user0
testfile = self.testfile
target_obj = os.path.join(sesh.session_collection, testfile)
# prepare rule file
rule_file = "test_rule_file.r"
rule_string = '''
test_acPostProcForCreate__3024 {{
msiDataObjOpen("{target_obj}",*FD);
msiDataObjClose(*FD,*Status);
}}
INPUT null
OUTPUT ruleExecOut
'''.format(**locals())
with open(rule_file, 'w') as f:
f.write(rule_string)
# query for resource properties
columns = ('RESC_ZONE_NAME, '
'RESC_FREE_SPACE, '
'RESC_STATUS, '
'RESC_ID, '
'RESC_NAME, '
'RESC_TYPE_NAME, '
'RESC_LOC, '
'RESC_CLASS_NAME, '
'RESC_VAULT_PATH, '
'RESC_INFO, '
'RESC_COMMENT, '
'RESC_CREATE_TIME, '
'RESC_MODIFY_TIME')
resource = sesh.default_resource
query = '''iquest "SELECT {columns} WHERE RESC_NAME ='{resource}'"'''.format(**locals())
result = sesh.run_icommand(query)[1]
# last line is iquest default formatting separator
resource_property_list = result.splitlines()[:-1]
with lib.file_backed_up(server_config_filename):
# prepare rule
# rule will write PEP name as well as
# resource related rule session vars to server log
rule_body = 'writeLine("serverLog", "{pep_name}");'.format(**locals())
rule_body += ('writeLine("serverLog", $KVPairs.zoneName);'
'writeLine("serverLog", $KVPairs.freeSpace);'
'writeLine("serverLog", $KVPairs.quotaLimit);'
'writeLine("serverLog", $KVPairs.rescStatus);'
'writeLine("serverLog", $KVPairs.rescId);'
'writeLine("serverLog", $KVPairs.rescName);'
'writeLine("serverLog", $KVPairs.rescType);'
'writeLine("serverLog", $KVPairs.rescLoc);'
'writeLine("serverLog", $KVPairs.rescClass);'
'writeLine("serverLog", $KVPairs.rescVaultPath);'
'writeLine("serverLog", $KVPairs.rescInfo);'
'writeLine("serverLog", $KVPairs.rescComments);'
'writeLine("serverLog", $KVPairs.rescCreate);'
'writeLine("serverLog", $KVPairs.rescModify);')
test_rule = '{pep_name} {{ {rule_body} }}'.format(**locals())
# write new rule file
with open(test_re, 'w') as f:
f.write(test_rule)
# update server config with additional rule file
server_config_update = {
"re_rulebase_set": [{"filename": "test"}, {"filename": "core"}]
}
lib.update_json_file_from_dict(server_config_filename, server_config_update)
# iput test file
sesh.assert_icommand('iput -f {testfile}'.format(**locals()))
# checkpoint log to know where to look for the string
initial_log_size = lib.get_log_size('server')
# invoke irule to trigger PEP
sesh.assert_icommand('irule -F {rule_file}'.format(**locals()))
# confirm that PEP was hit by looking for pep name in server log
assert lib.count_occurrences_of_string_in_log('server', pep_name, start_index=initial_log_size)
# check that resource session vars were written to the server log
for line in resource_property_list:
column = line.rsplit('=', 1)[0].strip()
property = line.rsplit('=', 1)[1].strip()
if property:
if column != 'RESC_MODIFY_TIME':
assert lib.count_occurrences_of_string_in_log('server', property, start_index=initial_log_size)
else:
# for resource modify time skip last 2 second digits
assert lib.count_occurrences_of_string_in_log('server', property[:-2], start_index=initial_log_size)
# cleanup
sesh.run_icommand('irm -f {target_obj}'.format(**locals()))
os.unlink(rule_file)
os.unlink(test_re)
class Test_AllRules(resource_suite.ResourceBase, unittest.TestCase):
__metaclass__ = metaclass_unittest_test_case_generator.MetaclassUnittestTestCaseGenerator
global rules30dir
currentdir = os.path.dirname(os.path.realpath(__file__))
rules30dir = currentdir + "/../../iRODS/clients/icommands/test/rules3.0/"
conf_dir = lib.get_irods_config_dir()
def setUp(self):
super(Test_AllRules, self).setUp()
self.rods_session = lib.make_session_for_existing_admin(
) # some rules hardcode 'rods' and 'tempZone'
hostname = socket.gethostname()
hostuser = getpass.getuser()
progname = __file__
dir_w = rules30dir + ".."
self.rods_session.assert_icommand(
'icd'
) # to get into the home directory (for testallrules assumption)
self.rods_session.assert_icommand('iadmin mkuser devtestuser rodsuser')
self.rods_session.assert_icommand(
'iadmin mkresc testallrulesResc unixfilesystem ' + hostname +
':/tmp/' + hostuser + '/pydevtest_testallrulesResc',
'STDOUT_SINGLELINE', 'unixfilesystem')
self.rods_session.assert_icommand('imkdir sub1')
self.rods_session.assert_icommand('imkdir sub3')
self.rods_session.assert_icommand('imkdir forphymv')
self.rods_session.assert_icommand('imkdir ruletest')
self.rods_session.assert_icommand('imkdir test')
self.rods_session.assert_icommand('imkdir test/phypathreg')
self.rods_session.assert_icommand('imkdir ruletest/subforrmcoll')
self.rods_session.assert_icommand('iput ' + progname + ' test/foo1')
self.rods_session.assert_icommand(
'icp test/foo1 sub1/dcmetadatatarget')
self.rods_session.assert_icommand('icp test/foo1 sub1/mdcopysource')
self.rods_session.assert_icommand('icp test/foo1 sub1/mdcopydest')
self.rods_session.assert_icommand('icp test/foo1 sub1/foo1')
self.rods_session.assert_icommand('icp test/foo1 sub1/foo2')
self.rods_session.assert_icommand('icp test/foo1 sub1/foo3')
self.rods_session.assert_icommand('icp test/foo1 forphymv/phymvfile')
self.rods_session.assert_icommand('icp test/foo1 sub1/objunlink1')
self.rods_session.assert_icommand(
'irm sub1/objunlink1') # put it in the trash
self.rods_session.assert_icommand('icp test/foo1 sub1/objunlink2')
self.rods_session.assert_icommand(
'irepl -R testallrulesResc sub1/objunlink2')
self.rods_session.assert_icommand('icp test/foo1 sub1/freebuffer')
self.rods_session.assert_icommand('icp test/foo1 sub1/automove')
self.rods_session.assert_icommand('icp test/foo1 test/versiontest.txt')
self.rods_session.assert_icommand(
'icp test/foo1 test/metadata-target.txt')
self.rods_session.assert_icommand('icp test/foo1 test/ERAtestfile.txt')
self.rods_session.assert_icommand(
'ichmod read devtestuser test/ERAtestfile.txt')
self.rods_session.assert_icommand(
'imeta add -d test/ERAtestfile.txt Fun 99 Balloons')
self.rods_session.assert_icommand(
'icp test/foo1 sub1/for_versioning.txt')
self.rods_session.assert_icommand('imkdir sub1/SaveVersions')
self.rods_session.assert_icommand(
'iput ' + dir_w + '/misc/devtestuser-account-ACL.txt test')
self.rods_session.assert_icommand('iput ' + dir_w +
'/misc/load-metadata.txt test')
self.rods_session.assert_icommand('iput ' + dir_w +
'/misc/load-usermods.txt test')
self.rods_session.assert_icommand('iput ' + dir_w +
'/misc/sample.email test')
self.rods_session.assert_icommand('iput ' + dir_w +
'/misc/email.tag test')
self.rods_session.assert_icommand(
'iput ' + dir_w + '/misc/sample.email test/sample2.email')
self.rods_session.assert_icommand('iput ' + dir_w +
'/misc/email.tag test/email2.tag')
# setup for rulemsiAdmChangeCoreRE and the likes
empty_core_file_name = 'empty.test.re'
new_core_file_name = 'new.test.re'
with open(self.conf_dir + '/' + empty_core_file_name, 'w'):
pass
shutil.copy(self.conf_dir + "/core.re",
self.conf_dir + "/core.re.bckp") # back up core.re
shutil.copy(self.conf_dir + "/core.re",
self.conf_dir + "/" + new_core_file_name) # copy core.re
def tearDown(self):
self.rods_session.assert_icommand(
'icd') # for home directory assumption
self.rods_session.assert_icommand(
['ichmod', '-r', 'own', self.rods_session.username, '.'])
self.rods_session.run_icommand([
'imcoll', '-U',
self.rods_session.home_collection + '/test/phypathreg'
])
self.rods_session.run_icommand(
'irm -rf test ruletest forphymv sub1 sub2 sub3 bagit rules bagit.tar /'
+ self.rods_session.zone_name + '/bundle/home/' +
self.rods_session.username)
self.rods_session.assert_icommand('iadmin rmresc testallrulesResc')
self.rods_session.assert_icommand('iadmin rmuser devtestuser')
self.rods_session.assert_icommand(
'iqdel -a') # remove all/any queued rules
# cleanup mods in iRODS config dir
lib.run_command('mv -f {0}/core.re.bckp {0}/core.re'.format(
self.conf_dir, self.conf_dir))
lib.run_command('rm -f %s/*.test.re' % self.conf_dir)
self.rods_session.__exit__()
super(Test_AllRules, self).tearDown()
def generate_tests_allrules():
def filter_rulefiles(rulefile):
# skip rules that handle .irb files
names_to_skip = [
"rulemsiAdmAppendToTopOfCoreIRB",
"rulemsiAdmChangeCoreIRB",
"rulemsiGetRulesFromDBIntoStruct",
]
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- RE"
return False
# skip rules that fail by design
names_to_skip = ["GoodFailure"]
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- failbydesign"
return False
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- failbydesign"
return False
# skip if an action (run in the core.re), not enough input/output for irule
names_to_skip = [
"rulemsiAclPolicy",
"rulemsiAddUserToGroup",
"rulemsiCheckHostAccessControl",
"rulemsiCheckOwner",
"rulemsiCheckPermission",
"rulemsiCommit",
"rulemsiCreateCollByAdmin",
"rulemsiCreateUser",
"rulemsiDeleteCollByAdmin",
"rulemsiDeleteDisallowed",
"rulemsiDeleteUser",
"rulemsiExtractNaraMetadata",
"rulemsiOprDisallowed",
"rulemsiRegisterData",
"rulemsiRenameCollection",
"rulemsiRenameLocalZone",
"rulemsiRollback",
"rulemsiSetBulkPutPostProcPolicy",
"rulemsiSetDataObjAvoidResc",
"rulemsiSetDataObjPreferredResc",
"rulemsiSetDataTypeFromExt",
"rulemsiSetDefaultResc",
"rulemsiSetGraftPathScheme",
"rulemsiSetMultiReplPerResc",
"rulemsiSetNoDirectRescInp",
"rulemsiSetNumThreads",
"rulemsiSetPublicUserOpr",
"rulemsiSetRandomScheme",
"rulemsiSetRescQuotaPolicy",
"rulemsiSetRescSortScheme",
"rulemsiSetReServerNumProc",
"rulemsiSetResource",
"rulemsiSortDataObj",
"rulemsiStageDataObj",
"rulemsiSysChksumDataObj",
"rulemsiSysMetaModify",
"rulemsiSysReplDataObj",
"rulemsiNoChkFilePathPerm",
"rulemsiNoTrashCan",
]
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- input/output"
return False
# skip rules we are not yet supporting
names_to_skip = [
"rulemsiobj",
]
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- msiobj"
return False
# ERA
names_to_skip = [
"rulemsiFlagInfectedObjs",
"rulemsiGetAuditTrailInfoByActionID",
"rulemsiGetAuditTrailInfoByKeywords",
"rulemsiGetAuditTrailInfoByObjectID",
"rulemsiGetAuditTrailInfoByTimeStamp",
"rulemsiGetAuditTrailInfoByUserID",
"rulemsiMergeDataCopies",
"rulemsiGetCollectionPSmeta-null" # marked for removal - iquest now handles this natively
]
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- ERA"
return False
# XMSG
names_to_skip = [
"rulemsiCreateXmsgInp",
"rulemsiRcvXmsg",
"rulemsiSendXmsg",
"rulemsiXmsgCreateStream",
"rulemsiXmsgServerConnect",
"rulemsiXmsgServerDisConnect",
"rulereadXMsg",
"rulewriteXMsg",
]
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- XMSG"
return False
# FTP
names_to_skip = [
"rulemsiFtpGet",
"rulemsiTwitterPost",
]
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- FTP"
return False
# webservices
names_to_skip = [
"rulemsiConvertCurrency",
"rulemsiGetQuote",
"rulemsiIp2location",
"rulemsiObjByName",
"rulemsiSdssImgCutout_GetJpeg",
]
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- webservices"
return False
# XML
names_to_skip = [
"rulemsiLoadMetadataFromXml",
"rulemsiXmlDocSchemaValidate",
"rulemsiXsltApply",
]
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- XML"
return False
# transition to core microservices only
names_to_skip = [
"rulemsiAddKeyVal.r",
"rulemsiApplyDCMetadataTemplate.r",
"rulemsiAssociateKeyValuePairsToObj.r",
"rulemsiCollectionSpider.r",
"rulemsiCopyAVUMetadata.r",
"rulemsiExportRecursiveCollMeta.r",
"rulemsiFlagDataObjwithAVU.r",
"rulemsiGetCollectionACL.r",
"rulemsiGetCollectionContentsReport.r",
"rulemsiGetCollectionPSmeta.r",
"rulemsiGetCollectionSize.r",
"rulemsiGetDataObjACL.r",
"rulemsiGetDataObjAIP.r",
"rulemsiGetDataObjAVUs.r",
"rulemsiGetDataObjPSmeta.r",
"rulemsiGetObjectPath.r",
"rulemsiGetUserACL.r",
"rulemsiGetUserInfo.r",
"rulemsiGuessDataType.r",
"rulemsiIsColl.r",
"rulemsiIsData.r",
"rulemsiLoadACLFromDataObj.r",
"rulemsiLoadMetadataFromDataObj.r",
"rulemsiLoadUserModsFromDataObj.r",
"rulemsiPropertiesAdd.r",
"rulemsiPropertiesClear.r",
"rulemsiPropertiesClone.r",
"rulemsiPropertiesExists.r",
"rulemsiPropertiesFromString.r",
"rulemsiPropertiesGet.r",
"rulemsiPropertiesNew.r",
"rulemsiPropertiesRemove.r",
"rulemsiPropertiesSet.r",
"rulemsiRecursiveCollCopy.r",
"rulemsiRemoveKeyValuePairsFromObj.r",
"rulemsiSetDataType.r",
"rulemsiString2KeyValPair.r",
"rulemsiStripAVUs.r",
"rulemsiStructFileBundle.r",
"rulewriteKeyValPairs.r",
]
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- transition to core"
return False
# skipping rules requiring additional .re files in community code
names_to_skip = [
"rulemsiAdmAddAppRuleStruct.r",
"rulemsiAdmClearAppRuleStruct.r",
"rulemsiAdmInsertRulesFromStructIntoDB.r",
"rulemsiAdmReadRulesFromFileIntoStruct.r",
"rulemsiAdmRetrieveRulesFromDBIntoStruct.r",
"rulemsiAdmWriteRulesFromStructIntoFile.r",
]
for n in names_to_skip:
if n in rulefile:
# print "skipping " + rulefile + " ----- community"
return False
# skipping for now, not sure why it's throwing a stacktrace at the moment
if "rulemsiPropertiesToString" in rulefile:
# print "skipping " + rulefile + " ----- b/c of stacktrace"
return False
# misc / other
if "ruleintegrity" in rulefile:
# print "skipping " + rulefile + " ----- integrityChecks"
return False
if "z3950" in rulefile:
# print "skipping " + rulefile + " ----- z3950"
return False
if "rulemsiImage" in rulefile:
# print "skipping " + rulefile + " ----- image"
return False
if "rulemsiRda" in rulefile:
# print "skipping " + rulefile + " ----- RDA"
return False
if "rulemsiCollRepl" in rulefile:
# print "skipping " + rulefile + " ----- deprecated"
return False
if "rulemsiTarFileExtract" in rulefile:
# print "skipping " + rulefile + " ----- CAT_NO_ROWS_FOUND - failed in
# call to getDataObjInfoIncSpecColl"
return False
if "rulemsiDataObjRsync" in rulefile:
# print "skipping " + rulefile + " ----- tested separately"
return False
return True
for rulefile in filter(filter_rulefiles,
sorted(os.listdir(rules30dir))):
def make_test(rulefile):
def test(self):
self.rods_session.assert_icommand("icd")
self.rods_session.assert_icommand(
"irule -vF " + rules30dir + rulefile,
'STDOUT_SINGLELINE', "completed successfully")
return test
yield 'test_' + rulefile.replace('.', '_'), make_test(rulefile)
def test_rulemsiDataObjRsync(self):
rulefile = 'rulemsiDataObjRsync.r'
src_filename = 'source.txt'
dest_filename = 'dest.txt'
test_dir = '/tmp'
test_coll = self.rods_session.home_collection + '/synctest'
src_file = os.path.join(test_dir, src_filename)
src_obj = test_coll + '/' + src_filename
dest_obj = test_coll + '/' + dest_filename
# create test collection
self.rods_session.run_icommand(['imkdir', test_coll])
# create source test file
with open(src_file, 'a') as f:
f.write('blah\n')
# upload source test file
self.rods_session.run_icommand(['iput', src_file, test_coll])
# first rsync rule test
self.rods_session.assert_icommand("irule -F " + rules30dir + rulefile,
'STDOUT_SINGLELINE',
"status = 99999992")
# modify the source and try again
for i in range(1, 5):
with open(src_file, 'a') as f:
f.write('blah_' + str(i) + '\n')
# force upload source
self.rods_session.run_icommand(['iput', '-f', src_file, test_coll])
# sync test
self.rods_session.assert_icommand(
"irule -F " + rules30dir + rulefile, 'STDOUT_SINGLELINE',
"status = 99999992")
# cleanup
self.rods_session.run_icommand(['irm', '-rf', test_coll])
os.remove(src_file)
def test_rulemsiPhyBundleColl(self):
rulefile = 'rulemsiPhyBundleColl.r'
# rule test
self.rods_session.assert_icommand(
"irule -F " + rules30dir + rulefile, 'STDOUT_SINGLELINE',
"Create tar file of collection /tempZone/home/rods/test on resource testallrulesResc"
)
# look for the bundle
bundle_path = '/tempZone/bundle/home/' + self.rods_session.username
output = self.rods_session.run_icommand(['ils', '-L', bundle_path])
# last token in stdout should be the bundle file's full physical path
bundlefile = output[1].split()[-1]
# check on the bundle file's name
assert bundlefile.find('test.') >= 0
# check physical path on resource
assert os.path.isfile(bundlefile)
# now try as a normal user (expect err msg)
self.user0.assert_icommand("irule -F " + rules30dir + rulefile,
'STDERR_SINGLELINE', "SYS_NO_API_PRIV")
# cleanup
self.rods_session.run_icommand(['irm', '-rf', bundle_path])
def test_str_2528(self):
self.rods_session.assert_icommand(
'''irule "*a.a = 'A'; *a.b = 'B'; writeLine('stdout', str(*a))" null ruleExecOut''',
'STDOUT_SINGLELINE', "a=A++++b=B")
def test_acPostProcForFilePathReg__3024(self):
test_re = os.path.join(lib.get_core_re_dir(), 'test.re')
server_config_filename = lib.get_irods_config_dir() + '/server_config.json'
# get PEP name from function name
pep_name = inspect.stack()[0][3].split('_')[1]
# user session
# use admin to be allowed to register stuff
sesh = self.admin
# test file for ireg
username = sesh.username
resc_vault_path = lib.get_vault_path(sesh)
testfile = '{resc_vault_path}/home/{username}/foo.txt'.format(**locals())
open(testfile, 'a').close()
# query for resource properties
columns = ('RESC_ZONE_NAME, '
'RESC_FREE_SPACE, '
'RESC_STATUS, '
'RESC_ID, '
'RESC_NAME, '
'RESC_TYPE_NAME, '
'RESC_LOC, '
'RESC_CLASS_NAME, '
'RESC_VAULT_PATH, '
'RESC_INFO, '
'RESC_COMMENT, '
'RESC_CREATE_TIME, '
'RESC_MODIFY_TIME')
resource = sesh.default_resource
query = '''iquest "SELECT {columns} WHERE RESC_NAME ='{resource}'"'''.format(**locals())
result = sesh.run_icommand(query)[1]
# last line is iquest default formatting separator
resource_property_list = result.splitlines()[:-1]
with lib.file_backed_up(server_config_filename):
# prepare rule
# rule will write PEP name as well as
# resource related rule session vars to server log
rule_body = 'writeLine("serverLog", "{pep_name}");'.format(**locals())
rule_body += ('writeLine("serverLog", $KVPairs.zoneName);'
'writeLine("serverLog", $KVPairs.freeSpace);'
'writeLine("serverLog", $KVPairs.quotaLimit);'
'writeLine("serverLog", $KVPairs.rescStatus);'
'writeLine("serverLog", $KVPairs.rescId);'
'writeLine("serverLog", $KVPairs.rescName);'
'writeLine("serverLog", $KVPairs.rescType);'
'writeLine("serverLog", $KVPairs.rescLoc);'
'writeLine("serverLog", $KVPairs.rescClass);'
'writeLine("serverLog", $KVPairs.rescVaultPath);'
'writeLine("serverLog", $KVPairs.rescInfo);'
'writeLine("serverLog", $KVPairs.rescComments);'
'writeLine("serverLog", $KVPairs.rescCreate);'
'writeLine("serverLog", $KVPairs.rescModify);')
test_rule = '{pep_name} {{ {rule_body} }}'.format(**locals())
# write new rule file
with open(test_re, 'w') as f:
f.write(test_rule)
# update server config with additional rule file
server_config_update = {
"re_rulebase_set": [{"filename": "test"}, {"filename": "core"}]
}
lib.update_json_file_from_dict(server_config_filename, server_config_update)
# checkpoint log to know where to look for the string
initial_log_size = lib.get_log_size('server')
# ireg test file to trigger PEP
target_obj = os.path.join(sesh.home_collection, os.path.basename(testfile))
sesh.assert_icommand('ireg {testfile} {target_obj}'.format(**locals()))
# confirm that PEP was hit by looking for pep name in server log
assert lib.count_occurrences_of_string_in_log('server', pep_name, start_index=initial_log_size)
# check that resource session vars were written to the server log
for line in resource_property_list:
column = line.rsplit('=', 1)[0].strip()
property = line.rsplit('=', 1)[1].strip()
if property:
if column != 'RESC_MODIFY_TIME':
assert lib.count_occurrences_of_string_in_log('server', property, start_index=initial_log_size)
# cleanup
sesh.run_icommand('irm -f {target_obj}'.format(**locals()))
os.unlink(test_re)
