def build_baton(baton_git_repository, baton_git_commitish,
output_root_directory):
baton_source_dir = irods_python_ci_utilities.git_clone(
baton_git_repository, baton_git_commitish)
logging.getLogger(__name__).info('Using baton source directory: %s',
baton_source_dir)
irods_python_ci_utilities.subprocess_get_output(['autoreconf', '-i'],
cwd=baton_source_dir,
check_rc=True)
try:
irods_python_ci_utilities.subprocess_get_output(
[os.path.join(baton_source_dir, 'configure'), '--with-irods'],
cwd=baton_source_dir,
check_rc=True)
finally:
if output_root_directory:
irods_python_ci_utilities.mkdir_p(output_root_directory)
irods_python_ci_utilities.copy_file_if_exists(
os.path.join(baton_source_dir, 'config.log'),
output_root_directory)
irods_python_ci_utilities.subprocess_get_output(
['make', '-j', str(multiprocessing.cpu_count())],
cwd=baton_source_dir,
check_rc=True)
return baton_source_dir
def build_icommands(icommands_git_repository, icommands_git_commitish,
debug_build):
icommands_source_dir = irods_python_ci_utilities.git_clone(
icommands_git_repository, icommands_git_commitish)
logging.getLogger(__name__).info('Using icommands source directory: %s',
icommands_source_dir)
icommands_build_dir = tempfile.mkdtemp(prefix='icommands_build_dir')
logging.getLogger(__name__).info('Using icommands build directory: %s',
icommands_build_dir)
if debug_build:
cmake_build_type = 'Debug'
else:
cmake_build_type = 'Release'
irods_python_ci_utilities.subprocess_get_output(
'cmake {0} -DCMAKE_BUILD_TYPE={1} > cmake.output'.format(
icommands_source_dir, cmake_build_type),
shell=True,
cwd=icommands_build_dir,
check_rc=True)
irods_python_ci_utilities.subprocess_get_output(
'make -j{0} > make.output'.format(str(multiprocessing.cpu_count())),
shell=True,
cwd=icommands_build_dir,
check_rc=True)
irods_python_ci_utilities.subprocess_get_output(
'fakeroot make package >> make.output',
shell=True,
cwd=icommands_build_dir,
check_rc=True)
return icommands_build_dir
def build_and_test_jargon(jargon_git_repository):
install_testing_dependencies()
source_directory = os.path.dirname(os.path.realpath(__file__))
irods_python_ci_utilities.subprocess_get_output(
['chmod', '-R', 'a+rx', '{0}'.format(source_directory)])
edit_core_re_file = os.path.join(source_directory,
'edit_core_re_for_jargon.py')
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'rsync -r {0} /var/lib/irods/scripts/'.format(edit_core_re_file)
])
prepare_irods = os.path.join(source_directory, 'prepare-irods.sh')
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'su', '-', 'irods', '-c', '{0}'.format(prepare_irods)])
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'python2 scripts/edit_core_re_for_jargon.py'
])
local_jargon_git_dir = irods_python_ci_utilities.git_clone(
jargon_git_repository, 'master')
jargon_commit = irods_python_ci_utilities.subprocess_get_output(
['git', 'rev-parse', 'HEAD'], cwd=local_jargon_git_dir)[1].strip()
maven_output_file = os.path.expanduser('~/maven_output.log')
irods_python_ci_utilities.subprocess_get_output(
"mvn install -fn --settings '{0}/maven-settings.xml' > {1}".format(
source_directory, maven_output_file),
cwd=local_jargon_git_dir,
shell=True)
return local_jargon_git_dir
def checkout_git_repo_and_run_build_hook(git_repository, git_commitish,
python_script, passthrough_arguments):
git_checkout_dir = irods_python_ci_utilities.git_clone(
git_repository, git_commitish)
return irods_python_ci_utilities.subprocess_get_output(
['python', python_script] + passthrough_arguments,
cwd=git_checkout_dir,
check_rc=True)
def checkout_git_repo_and_run_test_hook(git_repo, git_commitish,
passthrough_arguments,
install_externals, database_type):
if irods_python_ci_utilities.get_distribution() == 'Ubuntu':
irods_python_ci_utilities.subprocess_get_output(['apt-get', 'update'],
check_rc=True)
_git_repo = git_repo.split('/')
plugin_name = _git_repo[len(_git_repo) - 1]
git_sha = ci_utilities.get_sha_from_commitish(git_repo, git_commitish)
git_checkout_dir = irods_python_ci_utilities.git_clone(git_repo, git_sha)
plugin_build_dir = '/plugin_mount_dir/{0}'.format(plugin_name)
passthru_args = []
if passthrough_arguments is not None:
for args in passthrough_arguments.split(','):
arg1 = args.split(' ')
passthru_args = passthru_args + arg1
if 'kerberos' in plugin_name:
plugin_name = plugin_name.replace('kerberos', 'krb')
if 'capability_storage_tiering' in plugin_name:
if len(passthru_args) > 0:
plugin_name = 'irods_rule_engine_plugin_unified_storage_tiering'
passthru_args.extend([
'--munge_path',
'export PATH={}:$PATH'.format(get_mungefs_directory())
])
plugin_basename = plugin_name.replace('_', '-') + '*'
plugin_package = glob.glob(
os.path.join(plugin_build_dir,
irods_python_ci_utilities.get_irods_platform_string(),
plugin_basename))
if install_externals:
externals_list = ci_utilities.get_package_dependencies(
''.join(plugin_package))
if len(externals_list) > 0:
ci_utilities.install_externals_from_list(externals_list,
get_externals_directory())
python_script = 'irods_consortium_continuous_integration_test_hook.py'
output_directory = '/irods_test_env/{0}/{1}/{2}'.format(
plugin_name, irods_python_ci_utilities.get_irods_platform_string(),
database_type)
time.sleep(10)
cmd = [
'python', python_script, '--output_root_directory', output_directory,
'--built_packages_root_directory', plugin_build_dir
] + passthru_args
print(cmd)
return irods_python_ci_utilities.subprocess_get_output(
cmd, cwd=git_checkout_dir, check_rc=True)
def install_qpid_proton():
local_qpid_proton_dir = tempfile.mkdtemp(prefix='qpid_proton_dir')
irods_python_ci_utilities.git_clone('https://github.com/apache/qpid-proton.git', '0.17.0', local_qpid_proton_dir)
if irods_python_ci_utilities.get_distribution() == 'Ubuntu':
irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'gcc', 'cmake', 'cmake-curses-gui', 'uuid-dev', '-y'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'libssl-dev', '-y'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'libsasl2-2','libsasl2-dev', '-y'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'install', 'swig', 'python-dev', 'ruby-dev', 'libperl-dev', '-y'])
else:
irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'gcc', 'make', 'cmake', 'libuuid-devel', '-y'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'openssl-devel', '-y'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'cyrus-sasl-devel', '-y'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'yum', 'install', 'swig', 'python-devel', 'ruby-devel', 'rubygem-minitest', 'php-devel', 'perl-devel', '-y'])
qpid_proton_build_dir = local_qpid_proton_dir + '/build'
if not os.path.exists(qpid_proton_build_dir):
os.makedirs(qpid_proton_build_dir)
os.chdir(qpid_proton_build_dir)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'cmake', '..', '-DCMAKE_INSTALL_PREFIX=/usr', '-DSYSINSTALL_BINDINGS=ON'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'make', 'install'])
def build_icommands(icommands_git_repository, icommands_git_commitish, debug_build):
icommands_source_dir = irods_python_ci_utilities.git_clone(icommands_git_repository, icommands_git_commitish)
logging.getLogger(__name__).info('Using icommands source directory: %s', icommands_source_dir)
icommands_build_dir = tempfile.mkdtemp(prefix='icommands_build_dir')
logging.getLogger(__name__).info('Using icommands build directory: %s', icommands_build_dir)
if debug_build:
cmake_build_type = 'Debug'
else:
cmake_build_type = 'Release'
irods_python_ci_utilities.subprocess_get_output('cmake {0} -DCMAKE_BUILD_TYPE={1} > cmake.output'.format(icommands_source_dir, cmake_build_type), shell=True, cwd=icommands_build_dir, check_rc=True)
irods_python_ci_utilities.subprocess_get_output('make -j{0} > make.output'.format(str(multiprocessing.cpu_count())), shell=True, cwd=icommands_build_dir, check_rc=True)
irods_python_ci_utilities.subprocess_get_output('fakeroot make package >> make.output', shell=True, cwd=icommands_build_dir, check_rc=True)
return icommands_build_dir
def do(irods_repository, irods_commitish, icommands_repository, icommands_commitish, output_directory):
install_building_dependencies()
irods_git_dir = ci.git_clone(irods_repository, irods_commitish)
install_irods_externals_dependencies(irods_git_dir)
irods_build_dir = tempfile.mkdtemp(prefix='irods_build_dir')
irods_install_dir = tempfile.mkdtemp(prefix='irods_install_dir')
build_irods(irods_git_dir, irods_build_dir, irods_install_dir)
icommands_git_dir = ci.git_clone(icommands_repository, icommands_commitish)
icommands_build_dir = tempfile.mkdtemp(prefix='icommands_build_dir')
build_icommands(icommands_git_dir, icommands_build_dir, irods_install_dir)
ci.install_database('postgres')
configure_database()
install_database_plugin_dependencies()
configure_hostname()
set_non_package_install_environment_variables(irods_install_dir)
setup_irods(irods_install_dir)
install_testing_dependencies()
run_irods_tests(irods_install_dir)
if output_directory:
copy_output(irods_install_dir, output_directory)
def build_and_test_jargon(jargon_git_repository):
install_testing_dependencies()
local_jargon_git_dir = irods_python_ci_utilities.git_clone(
jargon_git_repository, 'master')
jargon_commit = irods_python_ci_utilities.subprocess_get_output(
['git', 'rev-parse', 'HEAD'], cwd=local_jargon_git_dir)[1].strip()
source_directory = os.path.dirname(os.path.realpath(__file__))
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
os.path.join(source_directory, 'prepare-irods.sh')
])
maven_output_file = os.path.expanduser('~/maven_output.log')
irods_python_ci_utilities.subprocess_get_output(
"mvn install -fn --settings '{0}/maven-settings.xml' > {1}".format(
source_directory, maven_output_file),
cwd=local_jargon_git_dir,
shell=True)
return local_jargon_git_dir
def checkout_git_repo_and_run_build_hook(git_repository, git_commitish, python_script, passthrough_arguments):
git_checkout_dir = irods_python_ci_utilities.git_clone(git_repository, git_commitish)
return irods_python_ci_utilities.subprocess_get_output(['python', python_script] + passthrough_arguments, cwd=git_checkout_dir, check_rc=True)