def make_irods_readable_copy_of_irodsbuild_proxy(): uid = pwd.getpwnam('irodsbuild').pw_uid proxy_file = '/tmp/x509up_u' + str(uid) irods_copy_of_proxy = '/tmp/irods_copy_of_irodsbuild_gsi_proxy' irods_python_ci_utilities.subprocess_get_output(['sudo', 'cp', proxy_file, irods_copy_of_proxy], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'chown', 'irods:irods', irods_copy_of_proxy], check_rc=True) return irods_copy_of_proxy
def main(): parser = optparse.OptionParser() parser.add_option('--output_root_directory') parser.add_option('--built_packages_root_directory') options, _ = parser.parse_args() output_root_directory = options.output_root_directory built_packages_root_directory = options.built_packages_root_directory package_suffix = irods_python_ci_utilities.get_package_suffix() os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory) irods_python_ci_utilities.install_os_packages_from_files(glob.glob(os.path.join(os_specific_directory, 'irods-resource-plugin-s3*.{0}'.format(package_suffix)))) install_build_prerequisites() minio_processes = download_and_start_minio_server() time.sleep(10) try: test_output_file = 'log/test_output.log' irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s test_irods_resource_plugin_s3_minio 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True) minio_processes[0].terminate() minio_processes[1].terminate() finally: if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_root_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory) shutil.copytree('/var/lib/irods/test-reports', os.path.join(output_root_directory, 'test-reports'))
def install_build_prerequisites(): global Java_Home irods_python_ci_utilities.install_os_packages(get_build_prerequisites()) java_version_check = re.compile('openjdk version[^\d]*1\.8\.', re.MULTILINE) java_version_text = '\n'.join( irods_python_ci_utilities.subprocess_get_output( ['/usr/bin/java', '-version'])[1:3]) java_real_bin = os.path.realpath('/usr/bin/java') Java_Home = os.path.sep.join((java_real_bin.split(os.path.sep))[:-2]) if not java_version_check.search(java_version_text): raise WrongJavaAsDefault if irods_python_ci_utilities.get_distribution( ) == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages( ['python-software-properties']) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test' ], check_rc=True) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'update-java-alternatives', '--set', 'java-1.8.0-openjdk-amd64' ]) irods_python_ci_utilities.install_os_packages(['libstdc++6'])
def main(): parser = optparse.OptionParser() parser.add_option('--output_root_directory') parser.add_option('--built_packages_root_directory') options, _ = parser.parse_args() output_root_directory = options.output_root_directory built_packages_root_directory = options.built_packages_root_directory package_suffix = irods_python_ci_utilities.get_package_suffix() os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory) irods_python_ci_utilities.install_os_packages_from_files(glob.glob(os.path.join(os_specific_directory, 'irods-resource-plugin-s3*.{0}'.format(package_suffix)))) install_build_prerequisites() time.sleep(10) try: test_output_file = 'log/test_output.log' irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s test_irods_resource_plugin_s3 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True) finally: if output_root_directory: irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_root_directory, lambda x: True) shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory) shutil.copytree('/var/lib/irods/test-reports', os.path.join(output_root_directory, 'test-reports'))
def create_test_configuration_json(irodsbuild_proxy_copy, irodsbuild_distinguished_name): config = {'client_user_proxy': irodsbuild_proxy_copy, 'client_user_DN': irodsbuild_distinguished_name} config_file = '/tmp/gsi_test_cfg.json' with open(config_file, 'w') as f: json.dump(config, f) irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', '777', config_file], check_rc=True)
def install_build_prerequisites_apt(): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages(get_build_prerequisites())
def install_kerberos_packages_yum(): irods_python_ci_utilities.install_os_packages( ['krb5-server', 'krb5-libs', 'krb5-workstation']) irods_python_ci_utilities.subprocess_get_output( ['sudo', 'systemctl', 'enable', 'krb5kdc.service'], check_rc=True) irods_python_ci_utilities.subprocess_get_output( ['sudo', 'systemctl', 'enable', 'kadmin.service'], check_rc=True)
def install_r_packages(): install_testing_dependencies() irods_python_ci_utilities.install_os_packages(['r-base']) # copy source into tmp dir, build it, install it as irods user build_dir = os.getcwd() irods_python_ci_utilities.subprocess_get_output( ['sudo', 'chmod', '-R', '777', build_dir], check_rc=True) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'export R_LIBS={0} && cd {0} && R CMD BATCH install-Rcpp.R'.format( build_dir) ], check_rc=True) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'export R_LIBS={0} && cd {0} && make r_cmd_pkg'.format(build_dir) ], check_rc=True) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'export R_LIBS={0} && cd {0} && R CMD BATCH install-packages.R'.format( build_dir) ], check_rc=True) # sync test files irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'rsync -r {0}/tests/ /var/lib/irods/tests/pydevtest/'.format(build_dir) ], check_rc=True)
def install_build_prerequisites(): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages(get_build_prerequisites())
def install_testing_dependencies_apt(): irods_python_ci_utilities.install_os_packages([ 'git', 'python-prettytable', 'python-coverage', 'python-dev', 'libkrb5-dev' ]) irods_python_ci_utilities.subprocess_get_output( ['sudo', '-H', 'pip', 'install', 'gssapi'], check_rc=True)
def install_os_specific_dependencies_apt(): if irods_python_ci_utilities.get_distribution( ) == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages( ['python-software-properties']) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test' ], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages([ 'fakeroot', 'help2man', 'libbz2-dev', 'libcurl4-gnutls-dev', 'libkrb5-dev', 'libpam0g-dev', 'libssl-dev', 'make', 'python-dev', 'unixodbc', 'unixodbc-dev', 'zlib1g-dev', ])
def run_baton_tests(baton_source_dir): env = os.environ.copy() env['CK_DEFAULT_TIMEOUT'] = '20' # the 'check' unit test framework has a default timeout that is too short for three of the baton tests. If you see 'Test timeout expired', may need to increase this irods_python_ci_utilities.subprocess_get_output(['make', 'check'], env=env, cwd=baton_source_dir, check_rc=True)
def run_tests(): # install_testing_dependencies() irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'export R_LIBS={0} && cd /var/lib/irods/tests/pydevtest && python run_tests.py --xml_output --run_specific_test test_client_rirods' .format(os.getcwd()) ], check_rc=False)
def create_privileged_principal(): stdin = '''addprinc root/admin krbtest krbtest ''' irods_python_ci_utilities.subprocess_get_output(['sudo', 'kadmin.local'], data=stdin, check_rc=True)
def install_os_specific_dependencies_apt(): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'update'], check_rc=True) if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages(['make', 'libssl-dev', 'libxml2-dev', 'libcurl4-gnutls-dev', 'gcc'])
def gather_logs(output_root_directory): irods_python_ci_utilities.subprocess_get_output( ['sudo', 'chmod', '-R', '777', '/var/lib/irods'], check_rc=True) irods_python_ci_utilities.subprocess_get_output([ 'rsync', '-r', '/var/lib/irods/iRODS/server/log/', output_root_directory + '/' ], check_rc=True)
def do_globus_config(): irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', 'o+rx', '/home/irodsbuild'], check_rc=True) # so user simpleca can read ~irodsbuild/.globus/usercert_request.pem irodsbuild_password = create_irodsbuild_certificate() create_irods_certificate() generate_proxy('irodsbuild', irodsbuild_password) generate_proxy('irods', None) irodsbuild_proxy_copy = make_irods_readable_copy_of_irodsbuild_proxy() irodsbuild_distinguished_name = get_irodsbuild_distinguished_name() create_test_configuration_json(irodsbuild_proxy_copy, irodsbuild_distinguished_name)
def change_server_hash_scheme_to_md5(): server_config_path = '/etc/irods/server_config.json' irods_python_ci_utilities.subprocess_get_output( ['sudo', 'chmod', 'o+rw', server_config_path], check_rc=True) with open(server_config_path) as f: d = json.load(f) d['default_hash_scheme'] = 'MD5' with open(server_config_path, 'w') as f: json.dump(d, f, ensure_ascii=True, sort_keys=True, indent=4)
def main(output_root_directory, irods_packages_root_directory, externals_directory): install_building_dependencies(externals_directory) if irods_packages_root_directory: install_irods_dev_and_runtime_packages(irods_packages_root_directory) build_directory = tempfile.mkdtemp(prefix='irods_gsi_plugin_build_directory') irods_python_ci_utilities.subprocess_get_output(['cmake', os.path.dirname(os.path.realpath(__file__))], check_rc=True, cwd=build_directory) irods_python_ci_utilities.subprocess_get_output(['make', '-j', str(multiprocessing.cpu_count()), 'package'], check_rc=True, cwd=build_directory) if output_root_directory: copy_output_packages(build_directory, output_root_directory)
def make_irods_readable_copy_of_irodsbuild_proxy(): uid = pwd.getpwnam('irodsbuild').pw_uid proxy_file = '/tmp/x509up_u' + str(uid) irods_copy_of_proxy = '/tmp/irods_copy_of_irodsbuild_gsi_proxy' irods_python_ci_utilities.subprocess_get_output( ['sudo', 'cp', proxy_file, irods_copy_of_proxy], check_rc=True) irods_python_ci_utilities.subprocess_get_output( ['sudo', 'chown', 'irods:irods', irods_copy_of_proxy], check_rc=True) return irods_copy_of_proxy
def create_keytab(): stdin = '''krbtest ktadd -k /var/lib/irods/irods.keytab irods/[email protected] ''' irods_python_ci_utilities.subprocess_get_output( ['sudo', 'kadmin', '-p', 'root/admin'], data=stdin, check_rc=True) irods_python_ci_utilities.subprocess_get_output( ['sudo', 'chown', 'irods:irods', '/var/lib/irods/irods.keytab'], check_rc=True)
def create_unprivileged_principal(principal): stdin = '''{0} addprinc {1} {2} {2} '''.format(kdc_database_master_key(), principal, unprivileged_principal_password()) irods_python_ci_utilities.subprocess_get_output( ['sudo', 'kadmin', '-p', 'root/admin'], data=stdin, check_rc=True)
def install_irods_repository_apt(): irods_python_ci_utilities.subprocess_get_output( 'wget -qO - https://core-dev.irods.org/irods-core-dev-signing-key.asc | sudo apt-key add -', shell=True, check_rc=True) irods_python_ci_utilities.subprocess_get_output( 'echo "deb [arch=amd64] https://core-dev.irods.org/apt/ $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/renci-irods-core-dev.list', shell=True, check_rc=True) subprocess.check_call('apt-get clean && apt-get update', shell=True)
def install_irods_repository_zypper(): irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'rpm', '--import', 'https://core-dev.irods.org/irods-core-dev-signing-key.asc' ], check_rc=True) irods_python_ci_utilities.subprocess_get_output( 'wget -qO - https://core-dev.irods.org/renci-irods-core-dev.zypp.repo | sudo tee /etc/zypp/repos.d/renci-irods-core-dev.zypp.repo', shell=True, check_rc=True)
def checkout_git_repo_and_run_test_hook(git_repo, git_commitish, passthrough_arguments, install_externals, database_type): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': irods_python_ci_utilities.subprocess_get_output(['apt-get', 'update'], check_rc=True) _git_repo = git_repo.split('/') plugin_name = _git_repo[len(_git_repo) - 1] git_sha = ci_utilities.get_sha_from_commitish(git_repo, git_commitish) git_checkout_dir = irods_python_ci_utilities.git_clone(git_repo, git_sha) plugin_build_dir = '/plugin_mount_dir/{0}'.format(plugin_name) passthru_args = [] if passthrough_arguments is not None: for args in passthrough_arguments.split(','): arg1 = args.split(' ') passthru_args = passthru_args + arg1 if 'kerberos' in plugin_name: plugin_name = plugin_name.replace('kerberos', 'krb') if 'capability_storage_tiering' in plugin_name: if len(passthru_args) > 0: plugin_name = 'irods_rule_engine_plugin_unified_storage_tiering' passthru_args.extend([ '--munge_path', 'export PATH={}:$PATH'.format(get_mungefs_directory()) ]) plugin_basename = plugin_name.replace('_', '-') + '*' plugin_package = glob.glob( os.path.join(plugin_build_dir, irods_python_ci_utilities.get_irods_platform_string(), plugin_basename)) if install_externals: externals_list = ci_utilities.get_package_dependencies( ''.join(plugin_package)) if len(externals_list) > 0: ci_utilities.install_externals_from_list(externals_list, get_externals_directory()) python_script = 'irods_consortium_continuous_integration_test_hook.py' output_directory = '/irods_test_env/{0}/{1}/{2}'.format( plugin_name, irods_python_ci_utilities.get_irods_platform_string(), database_type) time.sleep(10) cmd = [ 'python', python_script, '--output_root_directory', output_directory, '--built_packages_root_directory', plugin_build_dir ] + passthru_args print(cmd) return irods_python_ci_utilities.subprocess_get_output( cmd, cwd=git_checkout_dir, check_rc=True)
def create_test_configuration_json(irodsbuild_proxy_copy, irodsbuild_distinguished_name): config = { 'client_user_proxy': irodsbuild_proxy_copy, 'client_user_DN': irodsbuild_distinguished_name } config_file = '/tmp/gsi_test_cfg.json' with open(config_file, 'w') as f: json.dump(config, f) irods_python_ci_utilities.subprocess_get_output( ['sudo', 'chmod', '777', config_file], check_rc=True)
def install_os_specific_dependencies_apt(): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages([ 'fakeroot', 'help2man', 'libbz2-dev', 'libcurl4-gnutls-dev', 'libkrb5-dev', 'libpam0g-dev', 'libssl-dev', 'make', 'python-dev', 'unixodbc', 'unixodbc-dev', 'zlib1g-dev', ])
def main(output_root_directory, irods_packages_root_directory): irods_python_ci_utilities.install_irods_core_dev_repository() install_cmake_and_add_to_front_of_path() install_build_prerequisites() if irods_packages_root_directory: install_irods_dev_and_runtime_packages(irods_packages_root_directory) build_directory = tempfile.mkdtemp(prefix='irods_gsi_plugin_build_directory') irods_python_ci_utilities.subprocess_get_output(['cmake', os.path.dirname(os.path.realpath(__file__))], check_rc=True, cwd=build_directory) irods_python_ci_utilities.subprocess_get_output(['make', '-j', str(multiprocessing.cpu_count()), 'package'], check_rc=True, cwd=build_directory) if output_root_directory: copy_output_packages(build_directory, output_root_directory)
def restart_irods(): if irods_python_ci_utilities.get_irods_version() >= (4, 2): irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', '/var/lib/irods/irodsctl restart' ], check_rc=True) else: irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', '/var/lib/irods/iRODS/irodsctl restart' ], check_rc=True)
def add_domain_to_krb5_conf(): with tempfile.NamedTemporaryFile() as conf_copy: with open('/etc/krb5.conf', 'r') as conf: for l in conf: conf_copy.write(l) if '[domain_realm]' in l: conf_copy.write(' .example.org = EXAMPLE.ORG\n') conf_copy.write(' example.org = EXAMPLE.ORG\n') conf_copy.flush() irods_python_ci_utilities.subprocess_get_output( ['sudo', 'cp', conf_copy.name, '/etc/krb5.conf'], check_rc=True)
def do_globus_config(): irods_python_ci_utilities.subprocess_get_output( ['sudo', 'chmod', 'o+rx', '/home/irodsbuild'], check_rc=True ) # so user simpleca can read ~irodsbuild/.globus/usercert_request.pem irodsbuild_password = create_irodsbuild_certificate() create_irods_certificate() generate_proxy('irodsbuild', irodsbuild_password) generate_proxy('irods', None) irodsbuild_proxy_copy = make_irods_readable_copy_of_irodsbuild_proxy() irodsbuild_distinguished_name = get_irodsbuild_distinguished_name() create_test_configuration_json(irodsbuild_proxy_copy, irodsbuild_distinguished_name)
def generate_proxy(username, password): if password: irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', username, '-c' 'echo {0} | grid-proxy-init -pwstdin'.format(password) ], check_rc=True) else: irods_python_ci_utilities.subprocess_get_output( ['sudo', 'su', '-', username, '-c' 'grid-proxy-init'], check_rc=True)
def main(output_root_directory): build_directory = os.getcwd() irods_python_ci_utilities.subprocess_get_output( ['python', 'install_prerequisites.py'], check_rc=True, cwd=os.path.dirname(os.path.realpath(__file__))) irods_python_ci_utilities.subprocess_get_output( ['make'], check_rc=True, cwd=os.path.dirname(os.path.realpath(__file__))) if output_root_directory: copy_output_packages(build_directory, output_root_directory)
def install_testing_dependencies(): irods_python_ci_utilities.subprocess_get_output( ['sudo', '-EH', 'pip', 'install', 'unittest-xml-reporting==1.14.0']) globus_toolkit_package_name = get_test_prerequisites() irods_python_ci_utilities.subprocess_get_output([ 'wget', 'http://toolkit.globus.org/ftppub/gt6/installers/repo/{0}'. format(globus_toolkit_package_name) ], check_rc=True) irods_python_ci_utilities.install_os_packages_from_files( [globus_toolkit_package_name]) irods_python_ci_utilities.install_os_packages(['globus-gsi'])
def enable_admin_privileges_yum(): irods_python_ci_utilities.subprocess_get_output( ['sudo', 'chmod', '644', '/var/kerberos/krb5kdc/kadm5.acl'], check_rc=True) with tempfile.NamedTemporaryFile() as kadm5_copy: with open('/var/kerberos/krb5kdc/kadm5.acl', 'r') as kadm5_file: for l in kadm5_file: kadm5_copy.write(l) kadm5_copy.write('*/[email protected] *\n') kadm5_copy.flush() irods_python_ci_utilities.subprocess_get_output( ['sudo', 'cp', kadm5_copy.name, '/var/kerberos/krb5kdc/kadm5.acl'], check_rc=True)
def add_shortname_to_etc_hosts(): fullname = socket.gethostname() shortname = fullname.partition('.')[0] with tempfile.NamedTemporaryFile() as hosts_copy: with open('/etc/hosts', 'r') as hosts_file: for l in hosts_file: if fullname in l: hosts_copy.write(l.strip() + ' ' + shortname + '\n') else: hosts_copy.write(l) hosts_copy.flush() irods_python_ci_utilities.subprocess_get_output( ['sudo', 'cp', hosts_copy.name, '/etc/hosts'], check_rc=True)
def create_irods_certificate(): irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'grid-cert-request -nopw -force -cn irods_service'], check_rc=True) temporary_certificate_location = '/tmp/gsicert' irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-s', '/bin/bash', '-c', 'grid-ca-sign -in /var/lib/irods/.globus/usercert_request.pem -out {0}'.format(temporary_certificate_location), 'simpleca'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'cp', temporary_certificate_location, '/var/lib/irods/.globus/usercert.pem'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'rm', temporary_certificate_location], check_rc=True)
def install_build_prerequisites_apt(): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12 if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.install_os_packages(['python-software-properties']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) irods_python_ci_utilities.install_os_packages(['libstdc++6']) irods_python_ci_utilities.install_os_packages(get_build_prerequisites()) if irods_python_ci_utilities.get_distribution() == 'Ubuntu': if irods_python_ci_utilities.get_distribution_version_major() == '12': irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/libglobus_callout.so.0', '/usr/lib/libglobus_callout.so') irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/libglobus_gss_assist.so.3', '/usr/lib/libglobus_gss_assist.so') elif irods_python_ci_utilities.get_distribution_version_major() == '14': irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_callout.so.0', '/usr/lib/x86_64-linux-gnu/libglobus_callout.so') irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so.3', '/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so') elif irods_python_ci_utilities.get_distribution_version_major() == '16': irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_callout.so.0', '/usr/lib/x86_64-linux-gnu/libglobus_callout.so') irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so.3', '/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so')
def build_irods(debug_build): irods_source_dir = os.path.dirname(os.path.realpath(__file__)) irods_build_dir = tempfile.mkdtemp(prefix='irods_build_dir') logging.getLogger(__name__).info('Using iRODS build directory: %s', irods_build_dir) if debug_build: cmake_build_type = 'Debug' else: cmake_build_type = 'Release' irods_python_ci_utilities.subprocess_get_output('cmake {0} -DCMAKE_BUILD_TYPE={1} > cmake.output'.format(irods_source_dir, cmake_build_type), shell=True, cwd=irods_build_dir, check_rc=True) irods_python_ci_utilities.subprocess_get_output('make -j{0} > make.output'.format(str(multiprocessing.cpu_count())), shell=True, cwd=irods_build_dir, check_rc=True) irods_python_ci_utilities.subprocess_get_output('fakeroot make package >> make.output', shell=True, cwd=irods_build_dir, check_rc=True) return irods_build_dir
def build_icommands(icommands_git_repository, icommands_git_commitish, debug_build): icommands_source_dir = irods_python_ci_utilities.git_clone(icommands_git_repository, icommands_git_commitish) logging.getLogger(__name__).info('Using icommands source directory: %s', icommands_source_dir) icommands_build_dir = tempfile.mkdtemp(prefix='icommands_build_dir') logging.getLogger(__name__).info('Using icommands build directory: %s', icommands_build_dir) if debug_build: cmake_build_type = 'Debug' else: cmake_build_type = 'Release' irods_python_ci_utilities.subprocess_get_output('cmake {0} -DCMAKE_BUILD_TYPE={1} > cmake.output'.format(icommands_source_dir, cmake_build_type), shell=True, cwd=icommands_build_dir, check_rc=True) irods_python_ci_utilities.subprocess_get_output('make -j{0} > make.output'.format(str(multiprocessing.cpu_count())), shell=True, cwd=icommands_build_dir, check_rc=True) irods_python_ci_utilities.subprocess_get_output('fakeroot make package >> make.output', shell=True, cwd=icommands_build_dir, check_rc=True) return icommands_build_dir
def main(): parser = optparse.OptionParser() parser.add_option('--output_root_directory') parser.add_option('--built_packages_root_directory') options, _ = parser.parse_args() output_root_directory = options.output_root_directory built_packages_root_directory = options.built_packages_root_directory plugin_dir = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory) irods_python_ci_utilities.install_os_packages_from_files([os.path.join(plugin_dir, entry) for entry in os.listdir(plugin_dir)]) irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/setup_python_rule_engine_as_only_rule_engine.py'], check_rc=True) test_output_file = 'log/test_output.log' try: irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_python_suite > {0} 2>&1'.format(test_output_file)], check_rc=True) finally: if output_root_directory: irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'chmod o+r -R {0}'.format(' '.join(['test-reports', 'log', test_output_file]))]) shutil.copytree(os.path.join(os.path.expanduser('~irods'), 'test-reports'), os.path.join(output_root_directory, 'test-reports')) shutil.copytree(os.path.join(os.path.expanduser('~irods'), 'log'), os.path.join(output_root_directory, 'log')) shutil.copy(os.path.join(os.path.expanduser('~irods'), test_output_file), output_root_directory)
def build_irods(irods_git_dir, irods_build_dir, irods_install_dir): ci.subprocess_get_output('cmake {0} -DCMAKE_INSTALL_PREFIX={1} > cmake_irods.output'.format(irods_git_dir, irods_install_dir), cwd=irods_build_dir, shell=True, check_rc=True) ci.subprocess_get_output('make -j4 non-package-install-postgres > make_irods.output', cwd=irods_build_dir, shell=True, check_rc=True)
def install_building_dependencies_debian(): ci.install_os_packages(['git', 'g++', 'make', 'python-dev', 'help2man', 'unixodbc', 'libfuse-dev', 'libcurl4-gnutls-dev', 'libbz2-dev', 'zlib1g-dev', 'libpam0g-dev', 'libssl-dev', 'libxml2-dev', 'libkrb5-dev', 'unixodbc-dev', 'libjson-perl', 'python-psutil', 'fakeroot']) if ci.get_distribution_version_major() == '12': ci.install_os_packages(['python-software-properties']) ci.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True) ci.install_os_packages(['libstdc++6'])
def install_building_dependencies_suse(): ci.subprocess_get_output(['sudo', 'zypper', 'addrepo', 'http://download.opensuse.org/repositories/devel:tools/openSUSE_13.2/devel:tools.repo'], check_rc=True) ci.subprocess_get_output(['sudo', 'zypper', '--gpg-auto-import-keys', 'refresh'], check_rc=True) ci.install_os_packages(['git', 'python-devel', 'help2man', 'unixODBC', 'fuse-devel', 'libcurl-devel', 'libbz2-devel', 'libopenssl-devel', 'libxml2-devel', 'krb5-devel', 'perl-JSON', 'unixODBC-devel', 'python-psutil', 'fakeroot'])
def run_irods_tests(irods_install_dir): ci.subprocess_get_output('python run_tests.py --xml_output --run_python_suite > test_output.txt 2>&1', shell=True, cwd=os.path.join(irods_install_dir, 'var', 'lib', 'irods', 'scripts'))
def copy_output(irods_install_dir, output_directory): ci.subprocess_get_output(['mkdir', '-p', output_directory], check_rc=True) ci.subprocess_get_output('cp test_output.txt {0}'.format(output_directory), cwd=os.path.join(irods_install_dir, 'var', 'lib', 'irods', 'scripts'), shell=True, check_rc=True) ci.subprocess_get_output('cp * {0}'.format(output_directory), cwd=os.path.join(irods_install_dir, 'var', 'lib', 'irods', 'scripts', 'test-reports'), shell=True, check_rc=True) ci.subprocess_get_output('cp rodsLog* {0}'.format(output_directory), cwd=os.path.join(irods_install_dir, 'var', 'lib', 'irods', 'log'), shell=True, check_rc=True)
def setup_irods(irods_install_dir): ci.subprocess_get_output('python scripts/setup_irods.py < packaging/localhost_setup_postgres.input', cwd=os.path.join(irods_install_dir, 'var', 'lib', 'irods'), shell=True, check_rc=True)
def install_testing_dependencies(): ci.subprocess_get_output(['sudo', '-E', 'pip2', 'install', '--upgrade', 'unittest-xml-reporting==1.14.0'], check_rc=True) if not (ci.get_distribution() == 'Ubuntu' and ci.get_distribution_version_major() == '12'): ci.install_os_packages(['python-jsonschema'])
def create_irodsbuild_certificate(): irods_python_ci_utilities.subprocess_get_output(['grid-cert-request', '-nopw', '-force', '-cn', 'gsi_client_user'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', 'u+w', '/home/irodsbuild/.globus/userkey.pem'], check_rc=True) private_key_password = '******' irods_python_ci_utilities.subprocess_get_output(['openssl', 'rsa', '-in', '/home/irodsbuild/.globus/userkey.pem', '-out', '/home/irodsbuild/.globus/userkey.pem', '-des3', '-passout', 'pass:{0}'.format(private_key_password)], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', '400', '/home/irodsbuild/.globus/userkey.pem'], check_rc=True) temporary_certificate_location = '/tmp/gsicert' irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-s', '/bin/bash', '-c', 'grid-ca-sign -in /home/irodsbuild/.globus/usercert_request.pem -out {0}'.format(temporary_certificate_location), 'simpleca'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'cp', temporary_certificate_location, '/home/irodsbuild/.globus/usercert.pem'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'rm', temporary_certificate_location], check_rc=True) return private_key_password
def configure_hostname(): ci.subprocess_get_output(['sudo', 'hostname', 'icat.example.org'], check_rc=True) ci.subprocess_get_output(['sudo', 'su', '-', '-c', 'echo -e "127.0.0.1 icat.example.org\n$(cat /etc/hosts)" > /etc/hosts'], check_rc=True)
def build_icommands(icommands_git_dir, icommands_build_dir, irods_install_dir): IRODS_DIR = os.path.join(irods_install_dir, 'usr/lib/irods/cmake') ci.subprocess_get_output('cmake {0} -DCMAKE_INSTALL_PREFIX={1} -DIRODS_DIR={2} > cmake_icommands.output'.format(icommands_git_dir, irods_install_dir, IRODS_DIR), cwd=icommands_build_dir, shell=True, check_rc=True) ci.subprocess_get_output('make -j4 install > make_icommands.output', cwd=icommands_build_dir, shell=True, check_rc=True)
def run_tests(): # prerequisites install_testing_dependencies() irods_python_ci_utilities.subprocess_get_output(['sudo', 'cp', '-r', '/var/lib/irods/.irods', '/home/irodsbuild/'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'chown', '-R', 'irodsbuild', '/home/irodsbuild/.irods'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', '-R', '777', '/etc/irods'], check_rc=True) # test run test_env = os.environ.copy() test_env['IRODS_CI_TEST_RUN'] = '1' irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'run', 'irods/test/runner.py'], env=test_env) # reports irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'report'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'xml'], check_rc=True)
def generate_proxy(username, password): if password: irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', username, '-c' 'echo {0} | grid-proxy-init -pwstdin'.format(password)], check_rc=True) else: irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', username, '-c' 'grid-proxy-init'], check_rc=True)
def install_testing_dependencies(): irods_python_ci_utilities.subprocess_get_output(['sudo', '-EH', 'pip', 'install', 'unittest-xml-reporting==1.14.0']) globus_toolkit_package_name = get_test_prerequisites() irods_python_ci_utilities.subprocess_get_output(['wget', 'http://toolkit.globus.org/ftppub/gt6/installers/repo/{0}'.format(globus_toolkit_package_name)], check_rc=True) irods_python_ci_utilities.install_os_packages_from_files([globus_toolkit_package_name]) irods_python_ci_utilities.install_os_packages(['globus-gsi'])
def get_irodsbuild_distinguished_name(): _, name, _ = irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irodsbuild', '-c', 'grid-cert-info -subject'], check_rc=True) return name.strip()
def install_testing_dependencies_apt(): irods_python_ci_utilities.install_os_packages(['git', 'python-prettytable', 'python-coverage', 'python-dev', 'libkrb5-dev']) irods_python_ci_utilities.subprocess_get_output(['sudo', '-H', 'pip', 'install', 'gssapi'], check_rc=True)
def run_tests(): install_testing_dependencies() irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', '-R', '777', '/etc/irods'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'run', 'irods/test/runner.py']) irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'report'], check_rc=True) irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'xml'], check_rc=True)
def configure_database(): ci.subprocess_get_output(['sudo', 'su', '-', 'postgres', '-c', 'createuser -s irods'], check_rc=True) ci.subprocess_get_output(['sudo', 'su', '-', 'postgres', '-c', '''psql -c "alter role irods with password 'testpassword'"'''], check_rc=True) ci.subprocess_get_output(['sudo', 'su', '-', 'postgres', '-c', "createdb 'ICAT'"], check_rc=True)
def checkout_git_repo_and_run_build_hook(git_repository, git_commitish, python_script, passthrough_arguments): git_checkout_dir = irods_python_ci_utilities.git_clone(git_repository, git_commitish) return irods_python_ci_utilities.subprocess_get_output(['python', python_script] + passthrough_arguments, cwd=git_checkout_dir, check_rc=True)