def build_baton(baton_git_repository, baton_git_commitish, output_root_directory): baton_source_dir = irods_python_ci_utilities.git_clone( baton_git_repository, baton_git_commitish) logging.getLogger(__name__).info('Using baton source directory: %s', baton_source_dir) irods_python_ci_utilities.subprocess_get_output(['autoreconf', '-i'], cwd=baton_source_dir, check_rc=True) try: irods_python_ci_utilities.subprocess_get_output( [os.path.join(baton_source_dir, 'configure'), '--with-irods'], cwd=baton_source_dir, check_rc=True) finally: if output_root_directory: irods_python_ci_utilities.mkdir_p(output_root_directory) irods_python_ci_utilities.copy_file_if_exists( os.path.join(baton_source_dir, 'config.log'), output_root_directory) irods_python_ci_utilities.subprocess_get_output( ['make', '-j', str(multiprocessing.cpu_count())], cwd=baton_source_dir, check_rc=True) return baton_source_dir
def build_icommands(icommands_git_repository, icommands_git_commitish, debug_build): icommands_source_dir = irods_python_ci_utilities.git_clone( icommands_git_repository, icommands_git_commitish) logging.getLogger(__name__).info('Using icommands source directory: %s', icommands_source_dir) icommands_build_dir = tempfile.mkdtemp(prefix='icommands_build_dir') logging.getLogger(__name__).info('Using icommands build directory: %s', icommands_build_dir) if debug_build: cmake_build_type = 'Debug' else: cmake_build_type = 'Release' irods_python_ci_utilities.subprocess_get_output( 'cmake {0} -DCMAKE_BUILD_TYPE={1} > cmake.output'.format( icommands_source_dir, cmake_build_type), shell=True, cwd=icommands_build_dir, check_rc=True) irods_python_ci_utilities.subprocess_get_output( 'make -j{0} > make.output'.format(str(multiprocessing.cpu_count())), shell=True, cwd=icommands_build_dir, check_rc=True) irods_python_ci_utilities.subprocess_get_output( 'fakeroot make package >> make.output', shell=True, cwd=icommands_build_dir, check_rc=True) return icommands_build_dir
def build_and_test_jargon(jargon_git_repository): install_testing_dependencies() source_directory = os.path.dirname(os.path.realpath(__file__)) irods_python_ci_utilities.subprocess_get_output( ['chmod', '-R', 'a+rx', '{0}'.format(source_directory)]) edit_core_re_file = os.path.join(source_directory, 'edit_core_re_for_jargon.py') irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'rsync -r {0} /var/lib/irods/scripts/'.format(edit_core_re_file) ]) prepare_irods = os.path.join(source_directory, 'prepare-irods.sh') irods_python_ci_utilities.subprocess_get_output( ['sudo', 'su', '-', 'irods', '-c', '{0}'.format(prepare_irods)]) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', 'python2 scripts/edit_core_re_for_jargon.py' ]) local_jargon_git_dir = irods_python_ci_utilities.git_clone( jargon_git_repository, 'master') jargon_commit = irods_python_ci_utilities.subprocess_get_output( ['git', 'rev-parse', 'HEAD'], cwd=local_jargon_git_dir)[1].strip() maven_output_file = os.path.expanduser('~/maven_output.log') irods_python_ci_utilities.subprocess_get_output( "mvn install -fn --settings '{0}/maven-settings.xml' > {1}".format( source_directory, maven_output_file), cwd=local_jargon_git_dir, shell=True) return local_jargon_git_dir
def checkout_git_repo_and_run_build_hook(git_repository, git_commitish, python_script, passthrough_arguments): git_checkout_dir = irods_python_ci_utilities.git_clone( git_repository, git_commitish) return irods_python_ci_utilities.subprocess_get_output( ['python', python_script] + passthrough_arguments, cwd=git_checkout_dir, check_rc=True)
def checkout_git_repo_and_run_test_hook(git_repo, git_commitish, passthrough_arguments, install_externals, database_type): if irods_python_ci_utilities.get_distribution() == 'Ubuntu': irods_python_ci_utilities.subprocess_get_output(['apt-get', 'update'], check_rc=True) _git_repo = git_repo.split('/') plugin_name = _git_repo[len(_git_repo) - 1] git_sha = ci_utilities.get_sha_from_commitish(git_repo, git_commitish) git_checkout_dir = irods_python_ci_utilities.git_clone(git_repo, git_sha) plugin_build_dir = '/plugin_mount_dir/{0}'.format(plugin_name) passthru_args = [] if passthrough_arguments is not None: for args in passthrough_arguments.split(','): arg1 = args.split(' ') passthru_args = passthru_args + arg1 if 'kerberos' in plugin_name: plugin_name = plugin_name.replace('kerberos', 'krb') if 'capability_storage_tiering' in plugin_name: if len(passthru_args) > 0: plugin_name = 'irods_rule_engine_plugin_unified_storage_tiering' passthru_args.extend([ '--munge_path', 'export PATH={}:$PATH'.format(get_mungefs_directory()) ]) plugin_basename = plugin_name.replace('_', '-') + '*' plugin_package = glob.glob( os.path.join(plugin_build_dir, irods_python_ci_utilities.get_irods_platform_string(), plugin_basename)) if install_externals: externals_list = ci_utilities.get_package_dependencies( ''.join(plugin_package)) if len(externals_list) > 0: ci_utilities.install_externals_from_list(externals_list, get_externals_directory()) python_script = 'irods_consortium_continuous_integration_test_hook.py' output_directory = '/irods_test_env/{0}/{1}/{2}'.format( plugin_name, irods_python_ci_utilities.get_irods_platform_string(), database_type) time.sleep(10) cmd = [ 'python', python_script, '--output_root_directory', output_directory, '--built_packages_root_directory', plugin_build_dir ] + passthru_args print(cmd) return irods_python_ci_utilities.subprocess_get_output( cmd, cwd=git_checkout_dir, check_rc=True)
def install_qpid_proton(): local_qpid_proton_dir = tempfile.mkdtemp(prefix='qpid_proton_dir') irods_python_ci_utilities.git_clone('https://github.com/apache/qpid-proton.git', '0.17.0', local_qpid_proton_dir) if irods_python_ci_utilities.get_distribution() == 'Ubuntu': irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'gcc', 'cmake', 'cmake-curses-gui', 'uuid-dev', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'libssl-dev', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'libsasl2-2','libsasl2-dev', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'swig', 'python-dev', 'ruby-dev', 'libperl-dev', '-y']) else: irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'gcc', 'make', 'cmake', 'libuuid-devel', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'openssl-devel', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'cyrus-sasl-devel', '-y']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'swig', 'python-devel', 'ruby-devel', 'rubygem-minitest', 'php-devel', 'perl-devel', '-y']) qpid_proton_build_dir = local_qpid_proton_dir + '/build' if not os.path.exists(qpid_proton_build_dir): os.makedirs(qpid_proton_build_dir) os.chdir(qpid_proton_build_dir) irods_python_ci_utilities.subprocess_get_output(['sudo', 'cmake', '..', '-DCMAKE_INSTALL_PREFIX=/usr', '-DSYSINSTALL_BINDINGS=ON']) irods_python_ci_utilities.subprocess_get_output(['sudo', 'make', 'install'])
def build_icommands(icommands_git_repository, icommands_git_commitish, debug_build): icommands_source_dir = irods_python_ci_utilities.git_clone(icommands_git_repository, icommands_git_commitish) logging.getLogger(__name__).info('Using icommands source directory: %s', icommands_source_dir) icommands_build_dir = tempfile.mkdtemp(prefix='icommands_build_dir') logging.getLogger(__name__).info('Using icommands build directory: %s', icommands_build_dir) if debug_build: cmake_build_type = 'Debug' else: cmake_build_type = 'Release' irods_python_ci_utilities.subprocess_get_output('cmake {0} -DCMAKE_BUILD_TYPE={1} > cmake.output'.format(icommands_source_dir, cmake_build_type), shell=True, cwd=icommands_build_dir, check_rc=True) irods_python_ci_utilities.subprocess_get_output('make -j{0} > make.output'.format(str(multiprocessing.cpu_count())), shell=True, cwd=icommands_build_dir, check_rc=True) irods_python_ci_utilities.subprocess_get_output('fakeroot make package >> make.output', shell=True, cwd=icommands_build_dir, check_rc=True) return icommands_build_dir
def do(irods_repository, irods_commitish, icommands_repository, icommands_commitish, output_directory): install_building_dependencies() irods_git_dir = ci.git_clone(irods_repository, irods_commitish) install_irods_externals_dependencies(irods_git_dir) irods_build_dir = tempfile.mkdtemp(prefix='irods_build_dir') irods_install_dir = tempfile.mkdtemp(prefix='irods_install_dir') build_irods(irods_git_dir, irods_build_dir, irods_install_dir) icommands_git_dir = ci.git_clone(icommands_repository, icommands_commitish) icommands_build_dir = tempfile.mkdtemp(prefix='icommands_build_dir') build_icommands(icommands_git_dir, icommands_build_dir, irods_install_dir) ci.install_database('postgres') configure_database() install_database_plugin_dependencies() configure_hostname() set_non_package_install_environment_variables(irods_install_dir) setup_irods(irods_install_dir) install_testing_dependencies() run_irods_tests(irods_install_dir) if output_directory: copy_output(irods_install_dir, output_directory)
def build_and_test_jargon(jargon_git_repository): install_testing_dependencies() local_jargon_git_dir = irods_python_ci_utilities.git_clone( jargon_git_repository, 'master') jargon_commit = irods_python_ci_utilities.subprocess_get_output( ['git', 'rev-parse', 'HEAD'], cwd=local_jargon_git_dir)[1].strip() source_directory = os.path.dirname(os.path.realpath(__file__)) irods_python_ci_utilities.subprocess_get_output([ 'sudo', 'su', '-', 'irods', '-c', os.path.join(source_directory, 'prepare-irods.sh') ]) maven_output_file = os.path.expanduser('~/maven_output.log') irods_python_ci_utilities.subprocess_get_output( "mvn install -fn --settings '{0}/maven-settings.xml' > {1}".format( source_directory, maven_output_file), cwd=local_jargon_git_dir, shell=True) return local_jargon_git_dir
def checkout_git_repo_and_run_build_hook(git_repository, git_commitish, python_script, passthrough_arguments): git_checkout_dir = irods_python_ci_utilities.git_clone(git_repository, git_commitish) return irods_python_ci_utilities.subprocess_get_output(['python', python_script] + passthrough_arguments, cwd=git_checkout_dir, check_rc=True)