def main(): parser = optparse.OptionParser() parser.add_option('--output_root_directory') parser.add_option('--built_packages_root_directory') options, _ = parser.parse_args() output_root_directory = options.output_root_directory built_packages_root_directory = options.built_packages_root_directory package_suffix = irods_python_ci_utilities.get_package_suffix() os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory) irods_python_ci_utilities.install_os_packages_from_files(glob.glob(os.path.join(os_specific_directory, 'irods-resource-plugin-s3*.{0}'.format(package_suffix)))) install_build_prerequisites() minio_processes = download_and_start_minio_server() time.sleep(10) try: test_output_file = 'log/test_output.log' irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s test_irods_resource_plugin_s3_minio 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True) minio_processes[0].terminate() minio_processes[1].terminate() finally: if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_root_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory) shutil.copytree('/var/lib/irods/test-reports', os.path.join(output_root_directory, 'test-reports'))
def main(): parser = optparse.OptionParser() parser.add_option('--output_root_directory') parser.add_option('--built_packages_root_directory') options, _ = parser.parse_args() output_root_directory = options.output_root_directory built_packages_root_directory = options.built_packages_root_directory package_suffix = irods_python_ci_utilities.get_package_suffix() os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory) irods_python_ci_utilities.install_os_packages_from_files(glob.glob(os.path.join(os_specific_directory, 'irods-resource-plugin-s3*.{0}'.format(package_suffix)))) install_build_prerequisites() time.sleep(10) try: test_output_file = 'log/test_output.log' irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s test_irods_resource_plugin_s3 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True) finally: if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_root_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory) shutil.copytree('/var/lib/irods/test-reports', os.path.join(output_root_directory, 'test-reports'))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--output_root_directory', type=str, required=True) parser.add_argument('--built_packages_root_directory', type=str, required=True) parser.add_argument('--munge_path', type=str, default=None, help='munge externals path') parser.add_argument('--test_unified_storage_tiering', type=str, default=None, help='should be either True or False') args = parser.parse_args() output_root_directory = args.output_root_directory built_packages_root_directory = args.built_packages_root_directory package_suffix = irods_python_ci_utilities.get_package_suffix() os_specific_directory = irods_python_ci_utilities.append_os_specific_directory( built_packages_root_directory) irods_python_ci_utilities.subprocess_get_output( ['sudo', '-EH', 'pip', 'install', 'unittest-xml-reporting==1.14.0']) irods_python_ci_utilities.install_os_packages_from_files( glob.glob( os.path.join(os_specific_directory, 'irods-rule-engine-plugin-unified-storage-tiering*'))) test_name = 'test_plugin_unified_storage_tiering' time.sleep(10) irods_python_ci_utilities.subprocess_get_output( ['sudo', 'chmod', 'g+rwx', '/dev/fuse'], check_rc=True) time.sleep(10) try: test_output_file = '/var/lib/irods/log/test_output.log' if args.munge_path is not None or args.munge_path != '': irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'cd scripts; {0}; python2 run_tests.py --xml_output --run_s {1} 2>&1 | tee {2}; exit $PIPESTATUS' .format(args.munge_path, test_name, test_output_file) ], check_rc=True) else: irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s {0} 2>&1 | tee {1}; exit $PIPESTATUS' .format(test_name, test_output_file) ], check_rc=True) finally: if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate( '/var/lib/irods/log', output_root_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory)
def main(): parser = optparse.OptionParser() parser.add_option('--output_root_directory') parser.add_option('--built_packages_root_directory') parser.add_option('--message_broker', default='apache-activemq-5.14.1', help='MQ server package name that needs to be tested') options, _ = parser.parse_args() output_root_directory = options.output_root_directory built_packages_root_directory = options.built_packages_root_directory package_suffix = irods_python_ci_utilities.get_package_suffix() os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory) irods_python_ci_utilities.install_os_packages_from_files(glob.glob(os.path.join(os_specific_directory, 'irods-rule-engine-plugin-audit-amqp*.{0}'.format(package_suffix)))) irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/add_audit_rule_engine_to_rule_engines.py'], check_rc=True) install_build_prerequisites() install_messaging_package(options.message_broker) install_qpid_proton() time.sleep(10) try: test_output_file = 'log/test_output.log' irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s=test_audit_plugin 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s=test_resource_types.Test_Resource_Unixfilesystem 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True) finally: if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_root_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory)
def gather_logs(database_type): import socket output_directory = '/irods_test_env/{0}/{1}/{2}'.format( irods_python_ci_utilities.get_irods_platform_string(), database_type, socket.gethostname()) irods_python_ci_utilities.gather_files_satisfying_predicate( '/var/lib/irods/log', output_directory, lambda x: True)
def copy_output_packages(irods_build_dir, icommands_build_dir, output_root_directory): irods_python_ci_utilities.gather_files_satisfying_predicate( irods_build_dir, irods_python_ci_utilities.append_os_specific_directory(output_root_directory), lambda s:s.endswith(irods_python_ci_utilities.get_package_suffix())) irods_python_ci_utilities.gather_files_satisfying_predicate( icommands_build_dir, irods_python_ci_utilities.append_os_specific_directory(output_root_directory), lambda s:s.endswith(irods_python_ci_utilities.get_package_suffix()))
def copy_output_packages(irods_build_dir, icommands_build_dir, output_root_directory): irods_python_ci_utilities.gather_files_satisfying_predicate( irods_build_dir, irods_python_ci_utilities.append_os_specific_directory(output_root_directory), lambda s:s.endswith(irods_python_ci_utilities.get_package_suffix())) irods_python_ci_utilities.gather_files_satisfying_predicate( icommands_build_dir, irods_python_ci_utilities.append_os_specific_directory(output_root_directory), lambda s:s.endswith(irods_python_ci_utilities.get_package_suffix()))
def run_unit_test(test_name): report_style= 'junit' report_filename = test_name + '_junit_report.xml' unit_test_binary = os.path.join(get_irods_packages_directory(), test_name) unit_test_cmd = "'{0}' -r {1} -o {2}".format(unit_test_binary, report_style, report_filename) cmd = ['su', '-', 'irods', '-c', unit_test_cmd] try: return subprocess.call(cmd) finally: src_dir = '/var/lib/irods' dst_dir = os.path.join('/irods_test_env', irods_python_ci_utilities.get_irods_platform_string(), 'unit_tests') irods_python_ci_utilities.gather_files_satisfying_predicate(src_dir, dst_dir, lambda f: os.path.basename(f).endswith('_junit_report.xml'))
def main(): parser = optparse.OptionParser() parser.add_option('--output_root_directory') parser.add_option('--built_packages_root_directory') parser.add_option('--mds_ip_address') parser.add_option('--lustre_filesystem_name') options, _ = parser.parse_args() output_root_directory = options.output_root_directory built_packages_root_directory = options.built_packages_root_directory mds_ip_address = options.mds_ip_address lustre_filesystem_name = options.lustre_filesystem_name # start lcap lcap_process = subprocess.Popen( ['/lcap/src/lcapd/lcapd', '-c', '/etc/lcapd.conf'], shell=False) # remove existing mount and all lustre files irods_python_ci_utilities.subprocess_get_output( ['rm', '-rf', '/lustreResc/lustre01/']) # mount the lustre file system with index 0 irods_python_ci_utilities.subprocess_get_output([ 'mount', '-o', 'user_fid2path', '-t', 'lustre', '%s@tcp1:/%s' % (mds_ip_address, lustre_filesystem_name), '/lustreResc/lustre01' ]) # create a directory that goes to MTD 1 irods_python_ci_utilities.subprocess_get_output( ['lfs', 'mkdir', '-i', '1', '/lustreResc/lustre01/MDT0001dir']) irods_python_ci_utilities.subprocess_get_output( ['chown', 'irods:irods', '/lustreResc/lustre01/MDT0001dir']) try: test_output_file = 'log/test_output.log' irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s=test_irods_tools_lustre 2>&1 | tee {0}; exit $PIPESTATUS' .format(test_output_file) ], check_rc=True) finally: if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate( '/var/lib/irods/log', output_root_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory) lcap_process.send_signal(signal.SIGINT)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--output_root_directory', type=str) parser.add_argument('--built_packages_root_directory', type=str, required=True) parser.add_argument('--test', metavar='dotted name', type=str) parser.add_argument('--skip-setup', action='store_false', dest='do_setup', default=True) args = parser.parse_args() built_packages_root_directory = args.built_packages_root_directory package_suffix = irods_python_ci_utilities.get_package_suffix() os_specific_directory = irods_python_ci_utilities.append_os_specific_directory( built_packages_root_directory) if args.do_setup: irods_python_ci_utilities.subprocess_get_output([ 'sudo', '-EH', 'python3', '-m', 'pip', 'install', 'unittest-xml-reporting==1.14.0' ]) irods_python_ci_utilities.install_os_packages_from_files( glob.glob( os.path.join( os_specific_directory, f'irods-rule-engine-plugin-unified-storage-tiering*.{package_suffix}' ))) test = args.test or 'test_plugin_unified_storage_tiering' try: test_output_file = '/var/lib/irods/log/test_output.log' irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', f'python3 scripts/run_tests.py --xml_output --run_s {test} 2>&1 | tee {test_output_file}; exit $PIPESTATUS' ], check_rc=True) finally: output_root_directory = args.output_root_directory if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate( '/var/lib/irods/log', output_root_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory)
def run_test(test_name, database_type): try: test_output_file = '/var/lib/irods/log/test_output.log' if database_type == 'oracle': cmd = 'cd scripts; export PATH={0}:$PATH; python2 run_tests.py --xml_output --run_s={1} 2>&1 | tee {2}; exit $PIPESTATUS'.format(get_mungefs_directory(), test_name, test_output_file) rc, _, _ = irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', cmd]) else: cmd = 'cd scripts; export PATH={0}:$PATH; python2 run_tests.py --use_mungefs --xml_output --run_s={1} 2>&1 | tee {2}; exit $PIPESTATUS'.format(get_mungefs_directory(), test_name, test_output_file) rc, _, _ = irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', cmd]) return rc finally: output_directory = '/irods_test_env/{0}/{1}/{2}'.format(irods_python_ci_utilities.get_irods_platform_string(), database_type, test_name) irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_directory)
def main(): parser = optparse.OptionParser() parser.add_option('--output_root_directory') parser.add_option('--built_packages_root_directory') parser.add_option('--indexing_engine', default='elasticsearch', help='Index/Search Platform needed for plugin test') options, _ = parser.parse_args() output_root_directory = options.output_root_directory built_packages_root_directory = options.built_packages_root_directory package_suffix = irods_python_ci_utilities.get_package_suffix() os_specific_directory = irods_python_ci_utilities.append_os_specific_directory( built_packages_root_directory) install_build_prerequisites() irods_python_ci_utilities.subprocess_get_output([ 'sudo', '-EH', 'pip', 'install', 'unittest-xml-reporting==1.14.0', 'python-qpid-proton==0.30.0' ]) install_indexing_engine(options.indexing_engine) # Packages are put either in top level or os-specific subdirectory. # For indexing it seems to be top level for now. But just in case, we check both. for directory in (built_packages_root_directory, os_specific_directory): pkgs = get_matching_packages(directory, package_suffix) if pkgs: irods_python_ci_utilities.install_os_packages_from_files(pkgs) break test_output_file = 'log/test_output.log' try: irods_python_ci_utilities.subprocess_get_output( ['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s=test_plugin_indexing 2>&1 |'\ ' tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True) finally: if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate( '/var/lib/irods/log', output_root_directory, lambda x: True) test_output_file = os.path.join('/var/lib/irods', test_output_file) if os.path.exists(test_output_file): shutil.copy(test_output_file, output_root_directory)
def main(): parser = optparse.OptionParser() parser.add_option('--output_root_directory') parser.add_option('--built_packages_root_directory') parser.add_option('--mungefs_packages_directory') options, _ = parser.parse_args() output_root_directory = options.output_root_directory built_packages_root_directory = options.built_packages_root_directory package_suffix = irods_python_ci_utilities.get_package_suffix() os_specific_directory = irods_python_ci_utilities.append_os_specific_directory( built_packages_root_directory) irods_python_ci_utilities.install_os_packages_from_files( glob.glob( os.path.join( os_specific_directory, 'irods-rule-engine-plugin-storage-tiering*.{0}'.format( package_suffix)))) irods_python_ci_utilities.install_irods_core_dev_repository() install_cmake_and_add_to_front_of_path() install_build_prerequisites() time.sleep(10) irods_python_ci_utilities.subprocess_get_output( ['sudo', 'chmod', 'g+rwx', '/dev/fuse'], check_rc=True) time.sleep(10) try: test_output_file = 'log/test_output.log' irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s test_plugin_storage_tiering 2>&1 | tee {0}; exit $PIPESTATUS' .format(test_output_file) ], check_rc=True) finally: if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate( '/var/lib/irods/log', output_root_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory)
def main(): parser = optparse.OptionParser() parser.add_option('--output_root_directory') parser.add_option('--built_packages_root_directory') options, _ = parser.parse_args() output_root_directory = options.output_root_directory built_packages_root_directory = options.built_packages_root_directory package_suffix = irods_python_ci_utilities.get_package_suffix() os_specific_directory = irods_python_ci_utilities.append_os_specific_directory( built_packages_root_directory) install_testing_dependencies() irods_python_ci_utilities.install_os_packages_from_files( glob.glob( os.path.join(os_specific_directory, 'irods-auth-plugin-gsi*.{0}'.format(package_suffix)))) do_globus_config() time.sleep(10) try: test_output_file = 'log/test_output.log' irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s=test_irods_auth_plugin_gsi 2>&1 | tee {0}; exit $PIPESTATUS' .format(test_output_file) ], check_rc=True) finally: if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate( '/var/lib/irods/log', output_root_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory) shutil.copytree( '/var/lib/irods/test-reports', os.path.join(output_root_directory, 'test-reports'))
def copy_output_packages(irods_build_dir, icommands_build_dir, output_root_directory): # Packages irods_python_ci_utilities.gather_files_satisfying_predicate( irods_build_dir, irods_python_ci_utilities.append_os_specific_directory( output_root_directory), lambda s: s.endswith(irods_python_ci_utilities.get_package_suffix())) # Unit-test binaries irods_python_ci_utilities.gather_files_satisfying_predicate( os.path.join(irods_build_dir, 'unit_tests'), irods_python_ci_utilities.append_os_specific_directory( output_root_directory), lambda s: os.path.basename(s).startswith('irods_')) # iCommands irods_python_ci_utilities.gather_files_satisfying_predicate( icommands_build_dir, irods_python_ci_utilities.append_os_specific_directory( output_root_directory), lambda s: s.endswith(irods_python_ci_utilities.get_package_suffix()))
def run_tests(test_type, specific_test, federation_args, database_type, use_ssl): if test_type != 'federation': create_irodsauthuser_account() if use_ssl: ssl_string = '--use_ssl' test_type_dict = { 'topology_icat': '--run_python_suite --include_auth_tests --include_timing_tests --topology_test=icat', 'topology_resource': '--run_python_suite --include_auth_tests --include_timing_tests --topology_test=resource', 'federation': '--run_specific_test test_federation --federation {0}'.format( federation_args) } else: ssl_string = '' test_type_dict = { 'topology_icat': '--run_python_suite --include_timing_tests --topology_test=icat', 'topology_resource': '--run_python_suite --include_timing_tests --topology_test=resource', 'federation': '--run_specific_test test_federation --federation {0}'.format( federation_args) } if specific_test is None or specific_test == 'None': test_type_argument = test_type_dict[test_type] else: if test_type == 'topology_icat': test_type_argument = '--run_specific_test=' + specific_test + ' --topology_test=icat' elif test_type == 'topology_resource': test_type_argument = '--run_specific_test=' + specific_test + ' --topology_test=resource' else: test_type_argument = '--run_specific_test=' + specific_test + ' --federation {0}'.format( federation_args) if test_type == 'federation': irods_version = irods_python_ci_utilities.get_irods_version() if irods_version < (4, 0): # we are running copied code on an old zone irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'mkdir -p /var/lib/irods/tests' ], check_rc=True) if irods_version < (4, 2): irods_python_ci_utilities.subprocess_get_output( ['sudo', 'su', '-', 'irods', '-c', 'mkdir /var/lib/irods/log'], check_rc=True) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'echo "" > /var/lib/irods/scripts/irods/database_connect.py' ], check_rc=True) try: test_output_file = '/var/lib/irods/log/test_output.log' run_test_cmd = [ 'su', '-', 'irods', '-c', 'cd scripts; python2 run_tests.py --xml_output {0} {1} 2>&1 | tee {2}; exit $PIPESTATUS' .format(test_type_argument, ssl_string, test_output_file) ] print(run_test_cmd) rc, stdout, stderr = irods_python_ci_utilities.subprocess_get_output( run_test_cmd) return rc finally: output_directory = '/irods_test_env/{0}/{1}/{2}'.format( irods_python_ci_utilities.get_irods_platform_string(), database_type, socket.gethostname()) irods_python_ci_utilities.gather_files_satisfying_predicate( '/var/lib/irods/log', output_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_directory)
def main(): parser = optparse.OptionParser() parser.add_option( '--output_root_directory', help='Path to the directory where logs and other files are written to.' ) parser.add_option( '--built_packages_root_directory', help='Path to directory containing the audit plugin package.') parser.add_option('--message_broker', default='apache-activemq-5.14.1', help='MQ server package name that needs to be tested.') options, _ = parser.parse_args() output_root_directory = os.path.join(options.output_root_directory, options.message_broker) built_packages_root_directory = options.built_packages_root_directory package_suffix = irods_python_ci_utilities.get_package_suffix() os_specific_directory = irods_python_ci_utilities.append_os_specific_directory( built_packages_root_directory) install_build_prerequisites() irods_python_ci_utilities.subprocess_get_output([ 'sudo', '-EH', 'python3', '-m', 'pip', 'install', 'unittest-xml-reporting==1.14.0', 'python-qpid-proton==0.36.0' ]) install_messaging_package(options.message_broker) irods_python_ci_utilities.install_os_packages_from_files( glob.glob( os.path.join( os_specific_directory, 'irods-rule-engine-plugin-audit-amqp*.{0}'.format( package_suffix)))) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'python scripts/add_audit_rule_engine_to_rule_engines.py' ], check_rc=True) time.sleep(10) test_audit_log = 'log/test_audit_plugin.log' test_output_file = 'log/test_output.log' try: irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'python3 scripts/run_tests.py --run_s=test_audit_plugin 2>&1 | tee {0}; exit $PIPESTATUS' .format(test_audit_log) ], check_rc=True) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'python3 scripts/run_tests.py --run_s=test_resource_types.Test_Resource_Unixfilesystem 2>&1 | tee {0}; exit $PIPESTATUS' .format(test_output_file) ], check_rc=True) finally: if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate( '/var/log/irods', output_root_directory, lambda x: True) test_output_file = os.path.join('/var/lib/irods', test_output_file) if os.path.exists(test_output_file): shutil.copy(test_output_file, output_root_directory) test_audit_log = os.path.join('/var/lib/irods', test_audit_log) if os.path.exists(test_audit_log): shutil.copy(test_audit_log, output_root_directory)