def copy_output_packages(irods_build_dir, icommands_build_dir, output_root_directory):
irods_python_ci_utilities.gather_files_satisfying_predicate(
irods_build_dir,
irods_python_ci_utilities.append_os_specific_directory(output_root_directory),
lambda s:s.endswith(irods_python_ci_utilities.get_package_suffix()))
irods_python_ci_utilities.gather_files_satisfying_predicate(
icommands_build_dir,
irods_python_ci_utilities.append_os_specific_directory(output_root_directory),
lambda s:s.endswith(irods_python_ci_utilities.get_package_suffix()))
def copy_output_packages(irods_build_dir, icommands_build_dir, output_root_directory):
irods_python_ci_utilities.gather_files_satisfying_predicate(
irods_build_dir,
irods_python_ci_utilities.append_os_specific_directory(output_root_directory),
lambda s:s.endswith(irods_python_ci_utilities.get_package_suffix()))
irods_python_ci_utilities.gather_files_satisfying_predicate(
icommands_build_dir,
irods_python_ci_utilities.append_os_specific_directory(output_root_directory),
lambda s:s.endswith(irods_python_ci_utilities.get_package_suffix()))
def install_irods_dev_and_runtime(irods_build_dir):
irods_python_ci_utilities.install_os_packages_from_files(
itertools.chain(
glob.glob(
os.path.join(
irods_build_dir, 'irods-dev*.{0}'.format(
irods_python_ci_utilities.get_package_suffix()))),
glob.glob(
os.path.join(
irods_build_dir, 'irods-runtime*.{0}'.format(
irods_python_ci_utilities.get_package_suffix())))))
def install_building_dependencies(externals_directory):
externals_list = [
"irods-externals-cmake3.11.4-0", "irods-externals-avro1.9.0-0",
"irods-externals-boost1.67.0-0", "irods-externals-clang-runtime6.0-0",
"irods-externals-clang6.0-0", "irods-externals-cppzmq4.2.3-0",
"irods-externals-cpr1.3.0-1", "irods-externals-elasticlient0.1.0-1",
"irods-externals-jansson2.7-0", "irods-externals-json3.7.3-0",
"irods-externals-libarchive3.3.2-1", "irods-externals-zeromq4-14.1.6-0"
]
if externals_directory == 'None' or externals_directory is None:
irods_python_ci_utilities.install_irods_core_dev_repository()
irods_python_ci_utilities.install_os_packages(externals_list)
else:
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(
externals_directory)
externals = []
for irods_externals in externals_list:
externals.append(
glob.glob(
os.path.join(
os_specific_directory,
irods_externals + '*.{0}'.format(package_suffix)))[0])
irods_python_ci_utilities.install_os_packages_from_files(externals)
add_cmake_to_front_of_path()
install_os_specific_dependencies()
def main():
parser = optparse.OptionParser()
parser.add_option('--output_root_directory')
parser.add_option('--built_packages_root_directory')
options, _ = parser.parse_args()
output_root_directory = options.output_root_directory
built_packages_root_directory = options.built_packages_root_directory
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory)
irods_python_ci_utilities.install_os_packages_from_files(glob.glob(os.path.join(os_specific_directory, 'irods-resource-plugin-s3*.{0}'.format(package_suffix))))
install_build_prerequisites()
time.sleep(10)
try:
test_output_file = 'log/test_output.log'
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s test_irods_resource_plugin_s3 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True)
finally:
if output_root_directory:
irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_root_directory, lambda x: True)
shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory)
shutil.copytree('/var/lib/irods/test-reports', os.path.join(output_root_directory, 'test-reports'))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--output_root_directory', type=str, required=True)
parser.add_argument('--built_packages_root_directory',
type=str,
required=True)
parser.add_argument('--munge_path',
type=str,
default=None,
help='munge externals path')
parser.add_argument('--test_unified_storage_tiering',
type=str,
default=None,
help='should be either True or False')
args = parser.parse_args()
output_root_directory = args.output_root_directory
built_packages_root_directory = args.built_packages_root_directory
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(
built_packages_root_directory)
irods_python_ci_utilities.subprocess_get_output(
['sudo', '-EH', 'pip', 'install', 'unittest-xml-reporting==1.14.0'])
irods_python_ci_utilities.install_os_packages_from_files(
glob.glob(
os.path.join(os_specific_directory,
'irods-rule-engine-plugin-unified-storage-tiering*')))
test_name = 'test_plugin_unified_storage_tiering'
time.sleep(10)
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'chmod', 'g+rwx', '/dev/fuse'], check_rc=True)
time.sleep(10)
try:
test_output_file = '/var/lib/irods/log/test_output.log'
if args.munge_path is not None or args.munge_path != '':
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'cd scripts; {0}; python2 run_tests.py --xml_output --run_s {1} 2>&1 | tee {2}; exit $PIPESTATUS'
.format(args.munge_path, test_name, test_output_file)
],
check_rc=True)
else:
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'python2 scripts/run_tests.py --xml_output --run_s {0} 2>&1 | tee {1}; exit $PIPESTATUS'
.format(test_name, test_output_file)
],
check_rc=True)
finally:
if output_root_directory:
irods_python_ci_utilities.gather_files_satisfying_predicate(
'/var/lib/irods/log', output_root_directory, lambda x: True)
shutil.copy('/var/lib/irods/log/test_output.log',
output_root_directory)
def main():
parser = optparse.OptionParser()
parser.add_option('--output_root_directory')
parser.add_option('--built_packages_root_directory')
parser.add_option('--message_broker', default='apache-activemq-5.14.1', help='MQ server package name that needs to be tested')
options, _ = parser.parse_args()
output_root_directory = options.output_root_directory
built_packages_root_directory = options.built_packages_root_directory
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory)
irods_python_ci_utilities.install_os_packages_from_files(glob.glob(os.path.join(os_specific_directory, 'irods-rule-engine-plugin-audit-amqp*.{0}'.format(package_suffix))))
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/add_audit_rule_engine_to_rule_engines.py'], check_rc=True)
install_build_prerequisites()
install_messaging_package(options.message_broker)
install_qpid_proton()
time.sleep(10)
try:
test_output_file = 'log/test_output.log'
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s=test_audit_plugin 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s=test_resource_types.Test_Resource_Unixfilesystem 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True)
finally:
if output_root_directory:
irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_root_directory, lambda x: True)
shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory)
def main():
parser = optparse.OptionParser()
parser.add_option('--output_root_directory')
parser.add_option('--built_packages_root_directory')
options, _ = parser.parse_args()
output_root_directory = options.output_root_directory
built_packages_root_directory = options.built_packages_root_directory
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory)
irods_python_ci_utilities.install_os_packages_from_files(glob.glob(os.path.join(os_specific_directory, 'irods-resource-plugin-s3*.{0}'.format(package_suffix))))
install_build_prerequisites()
minio_processes = download_and_start_minio_server()
time.sleep(10)
try:
test_output_file = 'log/test_output.log'
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s test_irods_resource_plugin_s3_minio 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True)
minio_processes[0].terminate()
minio_processes[1].terminate()
finally:
if output_root_directory:
irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_root_directory, lambda x: True)
shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory)
shutil.copytree('/var/lib/irods/test-reports', os.path.join(output_root_directory, 'test-reports'))
def install_building_dependencies(externals_directory):
externals_list = [
'irods-externals-avro1.11.0-0', 'irods-externals-boost1.78.0-0',
'irods-externals-clang-runtime13.0.0-0',
'irods-externals-clang13.0.0-0', 'irods-externals-cmake3.21.4-0',
'irods-externals-fmt8.1.1-0', 'irods-externals-json3.10.4-0',
'irods-externals-libarchive3.5.2-0'
]
if externals_directory == 'None' or externals_directory is None:
irods_python_ci_utilities.install_irods_core_dev_repository()
irods_python_ci_utilities.install_os_packages(externals_list)
else:
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(
externals_directory)
externals = []
for irods_externals in externals_list:
externals.append(
glob.glob(
os.path.join(
os_specific_directory,
irods_externals + '*.{0}'.format(package_suffix)))[0])
irods_python_ci_utilities.install_os_packages_from_files(externals)
add_cmake_to_front_of_path()
install_os_specific_dependencies()
def copy_output_packages(irods_build_dir, icommands_build_dir,
output_root_directory):
# Packages
irods_python_ci_utilities.gather_files_satisfying_predicate(
irods_build_dir,
irods_python_ci_utilities.append_os_specific_directory(
output_root_directory),
lambda s: s.endswith(irods_python_ci_utilities.get_package_suffix()))
# Unit-test binaries
irods_python_ci_utilities.gather_files_satisfying_predicate(
os.path.join(irods_build_dir, 'unit_tests'),
irods_python_ci_utilities.append_os_specific_directory(
output_root_directory),
lambda s: os.path.basename(s).startswith('irods_'))
# iCommands
irods_python_ci_utilities.gather_files_satisfying_predicate(
icommands_build_dir,
irods_python_ci_utilities.append_os_specific_directory(
output_root_directory),
lambda s: s.endswith(irods_python_ci_utilities.get_package_suffix()))
def install_externals(externals_dir, externals_to_install):
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(
externals_dir)
externals = []
externals_list = externals_to_install.split(',')
for irods_externals in externals_list:
externals.append(
glob.glob(
os.path.join(os_specific_directory, irods_externals +
'.{0}'.format(package_suffix)))[0])
irods_python_ci_utilities.install_os_packages_from_files(externals)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--output_root_directory', type=str)
parser.add_argument('--built_packages_root_directory',
type=str,
required=True)
parser.add_argument('--test', metavar='dotted name', type=str)
parser.add_argument('--skip-setup',
action='store_false',
dest='do_setup',
default=True)
args = parser.parse_args()
built_packages_root_directory = args.built_packages_root_directory
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(
built_packages_root_directory)
if args.do_setup:
irods_python_ci_utilities.subprocess_get_output([
'sudo', '-EH', 'python3', '-m', 'pip', 'install',
'unittest-xml-reporting==1.14.0'
])
irods_python_ci_utilities.install_os_packages_from_files(
glob.glob(
os.path.join(
os_specific_directory,
f'irods-rule-engine-plugin-unified-storage-tiering*.{package_suffix}'
)))
test = args.test or 'test_plugin_unified_storage_tiering'
try:
test_output_file = '/var/lib/irods/log/test_output.log'
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
f'python3 scripts/run_tests.py --xml_output --run_s {test} 2>&1 | tee {test_output_file}; exit $PIPESTATUS'
],
check_rc=True)
finally:
output_root_directory = args.output_root_directory
if output_root_directory:
irods_python_ci_utilities.gather_files_satisfying_predicate(
'/var/lib/irods/log', output_root_directory, lambda x: True)
shutil.copy('/var/lib/irods/log/test_output.log',
output_root_directory)
def main():
parser = optparse.OptionParser()
parser.add_option('--output_root_directory')
parser.add_option('--built_packages_root_directory')
parser.add_option('--indexing_engine',
default='elasticsearch',
help='Index/Search Platform needed for plugin test')
options, _ = parser.parse_args()
output_root_directory = options.output_root_directory
built_packages_root_directory = options.built_packages_root_directory
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(
built_packages_root_directory)
install_build_prerequisites()
irods_python_ci_utilities.subprocess_get_output([
'sudo', '-EH', 'pip', 'install', 'unittest-xml-reporting==1.14.0',
'python-qpid-proton==0.30.0'
])
install_indexing_engine(options.indexing_engine)
# Packages are put either in top level or os-specific subdirectory.
# For indexing it seems to be top level for now. But just in case, we check both.
for directory in (built_packages_root_directory, os_specific_directory):
pkgs = get_matching_packages(directory, package_suffix)
if pkgs:
irods_python_ci_utilities.install_os_packages_from_files(pkgs)
break
test_output_file = 'log/test_output.log'
try:
irods_python_ci_utilities.subprocess_get_output( ['sudo', 'su', '-', 'irods', '-c',
'python2 scripts/run_tests.py --xml_output --run_s=test_plugin_indexing 2>&1 |'\
' tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True)
finally:
if output_root_directory:
irods_python_ci_utilities.gather_files_satisfying_predicate(
'/var/lib/irods/log', output_root_directory, lambda x: True)
test_output_file = os.path.join('/var/lib/irods', test_output_file)
if os.path.exists(test_output_file):
shutil.copy(test_output_file, output_root_directory)
def main():
parser = optparse.OptionParser()
parser.add_option('--output_root_directory')
parser.add_option('--built_packages_root_directory')
parser.add_option('--mungefs_packages_directory')
options, _ = parser.parse_args()
output_root_directory = options.output_root_directory
built_packages_root_directory = options.built_packages_root_directory
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(
built_packages_root_directory)
irods_python_ci_utilities.install_os_packages_from_files(
glob.glob(
os.path.join(
os_specific_directory,
'irods-rule-engine-plugin-storage-tiering*.{0}'.format(
package_suffix))))
irods_python_ci_utilities.install_irods_core_dev_repository()
install_cmake_and_add_to_front_of_path()
install_build_prerequisites()
time.sleep(10)
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'chmod', 'g+rwx', '/dev/fuse'], check_rc=True)
time.sleep(10)
try:
test_output_file = 'log/test_output.log'
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'python2 scripts/run_tests.py --xml_output --run_s test_plugin_storage_tiering 2>&1 | tee {0}; exit $PIPESTATUS'
.format(test_output_file)
],
check_rc=True)
finally:
if output_root_directory:
irods_python_ci_utilities.gather_files_satisfying_predicate(
'/var/lib/irods/log', output_root_directory, lambda x: True)
shutil.copy('/var/lib/irods/log/test_output.log',
output_root_directory)
def main():
parser = optparse.OptionParser()
parser.add_option('--output_root_directory')
parser.add_option('--built_packages_root_directory')
options, _ = parser.parse_args()
output_root_directory = options.output_root_directory
built_packages_root_directory = options.built_packages_root_directory
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(
built_packages_root_directory)
install_testing_dependencies()
irods_python_ci_utilities.install_os_packages_from_files(
glob.glob(
os.path.join(os_specific_directory,
'irods-auth-plugin-gsi*.{0}'.format(package_suffix))))
do_globus_config()
time.sleep(10)
try:
test_output_file = 'log/test_output.log'
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'python2 scripts/run_tests.py --xml_output --run_s=test_irods_auth_plugin_gsi 2>&1 | tee {0}; exit $PIPESTATUS'
.format(test_output_file)
],
check_rc=True)
finally:
if output_root_directory:
irods_python_ci_utilities.gather_files_satisfying_predicate(
'/var/lib/irods/log', output_root_directory, lambda x: True)
shutil.copy('/var/lib/irods/log/test_output.log',
output_root_directory)
shutil.copytree(
'/var/lib/irods/test-reports',
os.path.join(output_root_directory, 'test-reports'))
def main():
parser = optparse.OptionParser()
parser.add_option(
'--output_root_directory',
help='Path to the directory where logs and other files are written to.'
)
parser.add_option(
'--built_packages_root_directory',
help='Path to directory containing the audit plugin package.')
parser.add_option('--message_broker',
default='apache-activemq-5.14.1',
help='MQ server package name that needs to be tested.')
options, _ = parser.parse_args()
output_root_directory = os.path.join(options.output_root_directory,
options.message_broker)
built_packages_root_directory = options.built_packages_root_directory
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(
built_packages_root_directory)
install_build_prerequisites()
irods_python_ci_utilities.subprocess_get_output([
'sudo', '-EH', 'python3', '-m', 'pip', 'install',
'unittest-xml-reporting==1.14.0', 'python-qpid-proton==0.36.0'
])
install_messaging_package(options.message_broker)
irods_python_ci_utilities.install_os_packages_from_files(
glob.glob(
os.path.join(
os_specific_directory,
'irods-rule-engine-plugin-audit-amqp*.{0}'.format(
package_suffix))))
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'python scripts/add_audit_rule_engine_to_rule_engines.py'
],
check_rc=True)
time.sleep(10)
test_audit_log = 'log/test_audit_plugin.log'
test_output_file = 'log/test_output.log'
try:
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'python3 scripts/run_tests.py --run_s=test_audit_plugin 2>&1 | tee {0}; exit $PIPESTATUS'
.format(test_audit_log)
],
check_rc=True)
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'python3 scripts/run_tests.py --run_s=test_resource_types.Test_Resource_Unixfilesystem 2>&1 | tee {0}; exit $PIPESTATUS'
.format(test_output_file)
],
check_rc=True)
finally:
if output_root_directory:
irods_python_ci_utilities.gather_files_satisfying_predicate(
'/var/log/irods', output_root_directory, lambda x: True)
test_output_file = os.path.join('/var/lib/irods', test_output_file)
if os.path.exists(test_output_file):
shutil.copy(test_output_file, output_root_directory)
test_audit_log = os.path.join('/var/lib/irods', test_audit_log)
if os.path.exists(test_audit_log):
shutil.copy(test_audit_log, output_root_directory)
def install_irods_dev_and_runtime(irods_build_dir):
irods_python_ci_utilities.install_os_packages_from_files(
itertools.chain(
glob.glob(os.path.join(irods_build_dir, 'irods-dev*.{0}'.format(irods_python_ci_utilities.get_package_suffix()))),
glob.glob(os.path.join(irods_build_dir, 'irods-runtime*.{0}'.format(irods_python_ci_utilities.get_package_suffix())))))