def install_messaging_package(message_broker): if 'apache-activemq-' in message_broker: version_number = message_broker.split('-')[2] tarfile = message_broker + '-bin.tar.gz' url = 'http://archive.apache.org/dist/activemq/' + version_number + '/' + tarfile activemq_dir = message_broker + '/bin/activemq' irods_python_ci_utilities.subprocess_get_output(['wget', url]) irods_python_ci_utilities.subprocess_get_output(['tar', 'xvfz', tarfile]) irods_python_ci_utilities.subprocess_get_output([activemq_dir, 'start']) if 'rabbitmq' in message_broker: if irods_python_ci_utilities.get_distribution() == 'Ubuntu': irods_python_ci_utilities.subprocess_get_output('curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | sudo bash', shell=True) irods_python_ci_utilities.subprocess_get_output(['wget', 'https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'dpkg', '-i', 'erlang-solutions_1.0_all.deb']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'update', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'esl-erlang=1:19.3.6', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'rabbitmq-server', '-y']) if irods_python_ci_utilities.get_distribution() == 'Centos' or irods_python_ci_utilities.get_distribution() == 'Centos linux': irods_python_ci_utilities.subprocess_get_output('curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh | sudo bash', shell=True) irods_python_ci_utilities.subprocess_get_output('curl -s https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh | sudo bash', shell=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'erlang', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'rabbitmq-server', '-y']) if irods_python_ci_utilities.get_distribution_version_major() == '6': irods_python_ci_utilities.subprocess_get_output(['sudo', 'update-rc.d', 'rabbitmq-server', 'defaults']) else: irods_python_ci_utilities.subprocess_get_output(['sudo', 'systemctl', 'enable', 'rabbitmq-server']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'service', 'rabbitmq-server', 'start']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'rabbitmq-plugins', 'enable', 'rabbitmq_amqp1_0'])
def install_build_prerequisites(): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages(get_build_prerequisites())
def install_build_prerequisites_apt(): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages(get_build_prerequisites())
def install_os_specific_dependencies_apt(): if irods_python_ci_utilities.get_distribution( ) == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages( ['python-software-properties']) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test' ], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages([ 'fakeroot', 'help2man', 'libbz2-dev', 'libcurl4-gnutls-dev', 'libkrb5-dev', 'libpam0g-dev', 'libssl-dev', 'make', 'python-dev', 'unixodbc', 'unixodbc-dev', 'zlib1g-dev', ])
def install_build_prerequisites(): global Java_Home irods_python_ci_utilities.install_os_packages(get_build_prerequisites()) java_version_check = re.compile('openjdk version[^\d]*1\.8\.', re.MULTILINE) java_version_text = '\n'.join( irods_python_ci_utilities.subprocess_get_output( ['/usr/bin/java', '-version'])[1:3]) java_real_bin = os.path.realpath('/usr/bin/java') Java_Home = os.path.sep.join((java_real_bin.split(os.path.sep))[:-2]) if not java_version_check.search(java_version_text): raise WrongJavaAsDefault if irods_python_ci_utilities.get_distribution( ) == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages( ['python-software-properties']) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test' ], check_rc=True) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'update-java-alternatives', '--set', 'java-1.8.0-openjdk-amd64' ]) irods_python_ci_utilities.install_os_packages(['libstdc++6'])
def install_build_prerequisites(): dispatch_map = { 'Centos': install_build_prerequisites_yum, } try: return dispatch_map[irods_python_ci_utilities.get_distribution()]() except KeyError: irods_python_ci_utilities.raise_not_implemented_for_distribution()
def install_testing_dependencies(): dispatch_map = { 'Ubuntu': install_testing_dependencies_apt, } try: return dispatch_map[irods_python_ci_utilities.get_distribution()]() except KeyError: irods_python_ci_utilities.raise_not_implemented_for_distribution()
def install_build_prerequisites_apt(): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages(get_build_prerequisites()) if irods_python_ci_utilities.get_distribution() == 'Ubuntu': if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/libglobus_callout.so.0', '/usr/lib/libglobus_callout.so') irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/libglobus_gss_assist.so.3', '/usr/lib/libglobus_gss_assist.so') elif irods_python_ci_utilities.get_distribution_version_major() == '14': irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_callout.so.0', '/usr/lib/x86_64-linux-gnu/libglobus_callout.so') irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so.3', '/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so') elif irods_python_ci_utilities.get_distribution_version_major() == '16': irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_callout.so.0', '/usr/lib/x86_64-linux-gnu/libglobus_callout.so') irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so.3', '/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so')
def install_os_specific_dependencies_apt(): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'update'], check_rc=True) if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages(['make', 'libssl-dev', 'libxml2-dev', 'libcurl4-gnutls-dev', 'gcc'])
def install_build_prerequisites(): irods_python_ci_utilities.subprocess_get_output(['sudo', 'pip', 'install', 'boto3', '--upgrade'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'pip', 'install', 'minio==6.0.2', '--upgrade'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', '-EH', 'pip', 'install', 'unittest-xml-reporting==1.14.0']) if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6'])
def enable_admin_privileges(): dispatch_map = { 'Ubuntu': enable_admin_privileges_apt, 'Centos': enable_admin_privileges_yum, 'Centos linux': enable_admin_privileges_yum } try: return dispatch_map[irods_python_ci_utilities.get_distribution()]() except KeyError: irods_python_ci_utilities.raise_not_implemented_for_distribution()
def configure_realm_and_domain(): dispatch_map = { 'Ubuntu': configure_realm_and_domain_apt, 'Centos': configure_realm_and_domain_yum, 'Centos linux': configure_realm_and_domain_yum } try: return dispatch_map[irods_python_ci_utilities.get_distribution()]() except KeyError: irods_python_ci_utilities.raise_not_implemented_for_distribution()
def install_build_prerequisites(): dispatch_map = { 'Ubuntu': install_build_prerequisites_apt, 'Centos': install_build_prerequisites_yum, 'Centos linux': install_build_prerequisites_yum, } try: return dispatch_map[irods_python_ci_utilities.get_distribution()]() except KeyError: irods_python_ci_utilities.raise_not_implemented_for_distribution()
def install_os_specific_dependencies(): dispatch_map = { 'Ubuntu': install_os_specific_dependencies_apt, 'Centos': install_os_specific_dependencies_yum, 'Centos linux': install_os_specific_dependencies_yum, } try: return dispatch_map[irods_python_ci_utilities.get_distribution()]() except KeyError: irods_python_ci_utilities.raise_not_implemented_for_distribution()
def get_build_prerequisites(): dispatch_map = { 'Ubuntu': get_build_prerequisites_apt, 'Centos': get_build_prerequisites_yum, 'Centos linux': get_build_prerequisites_yum, } try: return dispatch_map[irods_python_ci_utilities.get_distribution()]() except KeyError: irods_python_ci_utilities.raise_not_implemented_for_distribution()
def restart_kerberos(): dispatch_map = { 'Ubuntu': restart_kerberos_apt, 'Centos': restart_kerberos_yum, 'Centos linux': restart_kerberos_yum } try: return dispatch_map[irods_python_ci_utilities.get_distribution()]() except KeyError: irods_python_ci_utilities.raise_not_implemented_for_distribution()
def install_building_dependencies(): dispatch_map = { 'Ubuntu': install_building_dependencies_debian, 'Centos': install_building_dependencies_redhat, 'Centos linux': install_building_dependencies_redhat, 'Opensuse ': install_building_dependencies_suse, } try: dispatch_map[ci.get_distribution()]() except KeyError: ci.raise_not_implemented_for_distribution()
def checkout_git_repo_and_run_test_hook(git_repo, git_commitish, passthrough_arguments, install_externals, database_type): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': irods_python_ci_utilities.subprocess_get_output(['apt-get', 'update'], check_rc=True) _git_repo = git_repo.split('/') plugin_name = _git_repo[len(_git_repo) - 1] git_sha = ci_utilities.get_sha_from_commitish(git_repo, git_commitish) git_checkout_dir = irods_python_ci_utilities.git_clone(git_repo, git_sha) plugin_build_dir = '/plugin_mount_dir/{0}'.format(plugin_name) passthru_args = [] if passthrough_arguments is not None: for args in passthrough_arguments.split(','): arg1 = args.split(' ') passthru_args = passthru_args + arg1 if 'kerberos' in plugin_name: plugin_name = plugin_name.replace('kerberos', 'krb') if 'capability_storage_tiering' in plugin_name: if len(passthru_args) > 0: plugin_name = 'irods_rule_engine_plugin_unified_storage_tiering' passthru_args.extend([ '--munge_path', 'export PATH={}:$PATH'.format(get_mungefs_directory()) ]) plugin_basename = plugin_name.replace('_', '-') + '*' plugin_package = glob.glob( os.path.join(plugin_build_dir, irods_python_ci_utilities.get_irods_platform_string(), plugin_basename)) if install_externals: externals_list = ci_utilities.get_package_dependencies( ''.join(plugin_package)) if len(externals_list) > 0: ci_utilities.install_externals_from_list(externals_list, get_externals_directory()) python_script = 'irods_consortium_continuous_integration_test_hook.py' output_directory = '/irods_test_env/{0}/{1}/{2}'.format( plugin_name, irods_python_ci_utilities.get_irods_platform_string(), database_type) time.sleep(10) cmd = [ 'python', python_script, '--output_root_directory', output_directory, '--built_packages_root_directory', plugin_build_dir ] + passthru_args print(cmd) return irods_python_ci_utilities.subprocess_get_output( cmd, cwd=git_checkout_dir, check_rc=True)
def install_build_prerequisites(): irods_python_ci_utilities.subprocess_get_output(['sudo', '-EH', 'pip', 'install', 'unittest-xml-reporting==1.14.0']) if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'update-java-alternatives', '--set', 'java-1.7.0-openjdk-amd64']) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages(get_build_prerequisites())
def install_os_specific_dependencies_apt(): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages([ 'fakeroot', 'help2man', 'libbz2-dev', 'libcurl4-gnutls-dev', 'libkrb5-dev', 'libpam0g-dev', 'libssl-dev', 'make', 'python-dev', 'unixodbc', 'unixodbc-dev', 'zlib1g-dev', ])
def install_irods_repository(): dispatch_map = { 'Ubuntu': install_irods_repository_apt, 'Centos': install_irods_repository_yum, 'Centos linux': install_irods_repository_yum, 'Opensuse ': install_irods_repository_zypper, } try: return dispatch_map[irods_python_ci_utilities.get_distribution()]() except KeyError: irods_python_ci_utilities.raise_not_implemented_for_distribution()
def configure_client(database): dispatch_map = { 'Ubuntu': configure_client_apt, 'Centos': configure_client_yum, 'Centos linux': configure_client_yum, 'Opensuse': configure_client_zypper, } try: return dispatch_map[irods_python_ci_utilities.get_distribution()]( database) except KeyError: irods_python_ci_utilities.raise_not_implemented_for_distribution()
def install_qpid_proton(): local_qpid_proton_dir = tempfile.mkdtemp(prefix='qpid_proton_dir') irods_python_ci_utilities.git_clone('https://github.com/apache/qpid-proton.git', '0.17.0', local_qpid_proton_dir) if irods_python_ci_utilities.get_distribution() == 'Ubuntu': irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'gcc', 'cmake', 'cmake-curses-gui', 'uuid-dev', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'libssl-dev', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'libsasl2-2','libsasl2-dev', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'swig', 'python-dev', 'ruby-dev', 'libperl-dev', '-y']) else: irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'gcc', 'make', 'cmake', 'libuuid-devel', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'openssl-devel', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'cyrus-sasl-devel', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'swig', 'python-devel', 'ruby-devel', 'rubygem-minitest', 'php-devel', 'perl-devel', '-y']) qpid_proton_build_dir = local_qpid_proton_dir + '/build' if not os.path.exists(qpid_proton_build_dir): os.makedirs(qpid_proton_build_dir) os.chdir(qpid_proton_build_dir) irods_python_ci_utilities.subprocess_get_output(['sudo', 'cmake', '..', '-DCMAKE_INSTALL_PREFIX=/usr', '-DSYSINSTALL_BINDINGS=ON']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'make', 'install'])
def main(): parser = argparse.ArgumentParser() parser.add_argument('--federation_name', type=str, required=False) parser.add_argument('-d', '--database_type', default='postgres', help='database type', required=True) parser.add_argument('--database_machine', help='database container name', default=None) parser.add_argument('--install_externals', action='store_true', default=False) parser.add_argument('--test_type', type=str) parser.add_argument('--test_name', type=str, default=None) parser.add_argument('--zone_name', type=str, required=True) parser.add_argument('--remote_zone', type=str, required=True) parser.add_argument('--alias_name', type=str, required=True) args = parser.parse_args() distribution = irods_python_ci_utilities.get_distribution() ci_utilities.install_irods_packages(args.database_type, args.database_machine, args.install_externals, get_irods_packages_directory(), get_externals_directory(), is_provider=True) ci_utilities.setup_irods(args.database_type, args.zone_name, args.database_machine) ci_utilities.start_server(ci_utilities.get_irods_version()) rc = configure_federation(args.zone_name) if args.zone_name == 'tempZone': check_fed_state(args.remote_zone, args.database_type) else: sys.exit(rc)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--is_provider', action='store_true', default=False) parser.add_argument('--upgrade_test', action='store_true', default=False) parser.add_argument('--consumer_name', type=str) parser.add_argument('--provider_name', type=str, required=False) parser.add_argument('-d', '--database_type', default='postgres', help='database type', required=True) parser.add_argument('--database_machine', help='database container name', default=None) parser.add_argument('--install_externals', action='store_true', default=False) parser.add_argument('--test_type', type=str) parser.add_argument('--test_name', type=str) parser.add_argument('--network_name', type=str, required=False) parser.add_argument('--alias_name', type=str, required=False) parser.add_argument('--consumer_list', type=str) parser.add_argument('--use_ssl', action='store_true', default=False) args = parser.parse_args() distribution = irods_python_ci_utilities.get_distribution() ci_utilities.install_irods_packages(args.database_type, args.database_machine, args.install_externals, get_irods_packages_directory(), get_externals_directory(), is_provider = args.is_provider) set_univmss() if not args.is_provider: setup_consumer() if args.use_ssl: enable_pam() if args.upgrade_test: check_ports_open('icat.example.org') check_ports_open('resource2.example.org') check_ports_open('resource3.example.org') ci_utilities.upgrade(get_upgrade_packages_directory(), args.database_type, args.install_externals, get_externals_directory(), is_provider = args.is_provider) # TODO: wait for provider to enable ssl... if args.use_ssl: import enable_ssl time.sleep(60) print('enabling SSL on [' + args.alias_name + ']') enable_ssl.enable_ssl() if args.test_type == 'topology_resource' and args.alias_name == 'resource1.example.org': status = check_ports_open('icat.example.org') if status == 'open': rc = run_tests(args.test_type, args.test_name, args.database_type, args.use_ssl) sys.exit(rc) else: print('waiting on topology to be up:[' + args.alias_name + ']') print('[{0}] waiting for [{1}] to stand up irods'.format(args.alias_name, 'icat.example.org')) check_ports_open('icat.example.org') print('checking topo state on [' + args.alias_name + ']') check_topo_state('icat.example.org', args.database_type) else: ci_utilities.setup_irods(args.database_type, 'tempZone', args.database_machine) if args.use_ssl: enable_pam() if args.upgrade_test: check_ports_open('resource1.example.org') check_ports_open('resource2.example.org') check_ports_open('resource3.example.org') ci_utilities.upgrade(get_upgrade_packages_directory(), args.database_type, args.install_externals, get_externals_directory(), is_provider = args.is_provider) # Do not enable SSL before consumers have had a chance to set up check_ports_open('resource1.example.org') check_ports_open('resource2.example.org') check_ports_open('resource3.example.org') if args.use_ssl: import enable_ssl # TODO: Remove timing-based solution time.sleep(60) print('enabling ssl on [' + args.alias_name + ']') enable_ssl.enable_ssl() time.sleep(100) if args.test_type == 'topology_icat': print('[{0}] waiting for [{1}] to stand up irods'.format(args.alias_name, 'resource1.example.org')) status = check_ports_open('resource1.example.org') print('[{0}] waiting for [{1}] to stand up irods'.format(args.alias_name, 'resource2.example.org')) check_ports_open('resource2.example.org') print('[{0}] waiting for [{1}] to stand up irods'.format(args.alias_name, 'resource3.example.org')) check_ports_open('resource3.example.org') if status == 'open': rc = run_tests(args.test_type, args.test_name, args.database_type, args.use_ssl) sys.exit(rc) else: check_ports_open('resource1.example.org') check_ports_open('resource2.example.org') check_ports_open('resource3.example.org') check_topo_state('resource1.example.org', args.database_type)
def install_database_plugin_dependencies(): packages = { 'Ubuntu': ['unixodbc', 'odbc-postgresql', 'postgresql-client', 'super', 'libc6'], } ci.install_os_packages(packages[ci.get_distribution()])
def install_testing_dependencies(): ci.subprocess_get_output(['sudo', '-E', 'pip2', 'install', '--upgrade', 'unittest-xml-reporting==1.14.0'], check_rc=True) if not (ci.get_distribution() == 'Ubuntu' and ci.get_distribution_version_major() == '12'): ci.install_os_packages(['python-jsonschema'])