def make_irods_readable_copy_of_irodsbuild_proxy():
uid = pwd.getpwnam('irodsbuild').pw_uid
proxy_file = '/tmp/x509up_u' + str(uid)
irods_copy_of_proxy = '/tmp/irods_copy_of_irodsbuild_gsi_proxy'
irods_python_ci_utilities.subprocess_get_output(['sudo', 'cp', proxy_file, irods_copy_of_proxy], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'chown', 'irods:irods', irods_copy_of_proxy], check_rc=True)
return irods_copy_of_proxy
def main():
parser = optparse.OptionParser()
parser.add_option('--output_root_directory')
parser.add_option('--built_packages_root_directory')
options, _ = parser.parse_args()
output_root_directory = options.output_root_directory
built_packages_root_directory = options.built_packages_root_directory
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory)
irods_python_ci_utilities.install_os_packages_from_files(glob.glob(os.path.join(os_specific_directory, 'irods-resource-plugin-s3*.{0}'.format(package_suffix))))
install_build_prerequisites()
minio_processes = download_and_start_minio_server()
time.sleep(10)
try:
test_output_file = 'log/test_output.log'
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s test_irods_resource_plugin_s3_minio 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True)
minio_processes[0].terminate()
minio_processes[1].terminate()
finally:
if output_root_directory:
irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_root_directory, lambda x: True)
shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory)
shutil.copytree('/var/lib/irods/test-reports', os.path.join(output_root_directory, 'test-reports'))
def install_build_prerequisites():
global Java_Home
irods_python_ci_utilities.install_os_packages(get_build_prerequisites())
java_version_check = re.compile('openjdk version[^\d]*1\.8\.',
re.MULTILINE)
java_version_text = '\n'.join(
irods_python_ci_utilities.subprocess_get_output(
['/usr/bin/java', '-version'])[1:3])
java_real_bin = os.path.realpath('/usr/bin/java')
Java_Home = os.path.sep.join((java_real_bin.split(os.path.sep))[:-2])
if not java_version_check.search(java_version_text):
raise WrongJavaAsDefault
if irods_python_ci_utilities.get_distribution(
) == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12
if irods_python_ci_utilities.get_distribution_version_major() == '12':
irods_python_ci_utilities.install_os_packages(
['python-software-properties'])
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'add-apt-repository', '-y',
'ppa:ubuntu-toolchain-r/test'
],
check_rc=True)
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'update-java-alternatives', '--set',
'java-1.8.0-openjdk-amd64'
])
irods_python_ci_utilities.install_os_packages(['libstdc++6'])
def main():
parser = optparse.OptionParser()
parser.add_option('--output_root_directory')
parser.add_option('--built_packages_root_directory')
options, _ = parser.parse_args()
output_root_directory = options.output_root_directory
built_packages_root_directory = options.built_packages_root_directory
package_suffix = irods_python_ci_utilities.get_package_suffix()
os_specific_directory = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory)
irods_python_ci_utilities.install_os_packages_from_files(glob.glob(os.path.join(os_specific_directory, 'irods-resource-plugin-s3*.{0}'.format(package_suffix))))
install_build_prerequisites()
time.sleep(10)
try:
test_output_file = 'log/test_output.log'
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_s test_irods_resource_plugin_s3 2>&1 | tee {0}; exit $PIPESTATUS'.format(test_output_file)], check_rc=True)
finally:
if output_root_directory:
irods_python_ci_utilities.gather_files_satisfying_predicate('/var/lib/irods/log', output_root_directory, lambda x: True)
shutil.copy('/var/lib/irods/log/test_output.log', output_root_directory)
shutil.copytree('/var/lib/irods/test-reports', os.path.join(output_root_directory, 'test-reports'))
def create_test_configuration_json(irodsbuild_proxy_copy, irodsbuild_distinguished_name):
config = {'client_user_proxy': irodsbuild_proxy_copy,
'client_user_DN': irodsbuild_distinguished_name}
config_file = '/tmp/gsi_test_cfg.json'
with open(config_file, 'w') as f:
json.dump(config, f)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', '777', config_file], check_rc=True)
def install_build_prerequisites_apt():
if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12
if irods_python_ci_utilities.get_distribution_version_major() == '12':
irods_python_ci_utilities.install_os_packages(['python-software-properties'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True)
irods_python_ci_utilities.install_os_packages(['libstdc++6'])
irods_python_ci_utilities.install_os_packages(get_build_prerequisites())
def install_kerberos_packages_yum():
irods_python_ci_utilities.install_os_packages(
['krb5-server', 'krb5-libs', 'krb5-workstation'])
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'systemctl', 'enable', 'krb5kdc.service'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'systemctl', 'enable', 'kadmin.service'], check_rc=True)
def install_r_packages():
install_testing_dependencies()
irods_python_ci_utilities.install_os_packages(['r-base'])
# copy source into tmp dir, build it, install it as irods user
build_dir = os.getcwd()
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'chmod', '-R', '777', build_dir], check_rc=True)
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'export R_LIBS={0} && cd {0} && R CMD BATCH install-Rcpp.R'.format(
build_dir)
],
check_rc=True)
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'export R_LIBS={0} && cd {0} && make r_cmd_pkg'.format(build_dir)
],
check_rc=True)
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'export R_LIBS={0} && cd {0} && R CMD BATCH install-packages.R'.format(
build_dir)
],
check_rc=True)
# sync test files
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'rsync -r {0}/tests/ /var/lib/irods/tests/pydevtest/'.format(build_dir)
],
check_rc=True)
def install_build_prerequisites():
if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12
if irods_python_ci_utilities.get_distribution_version_major() == '12':
irods_python_ci_utilities.install_os_packages(['python-software-properties'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True)
irods_python_ci_utilities.install_os_packages(['libstdc++6'])
irods_python_ci_utilities.install_os_packages(get_build_prerequisites())
def install_testing_dependencies_apt():
irods_python_ci_utilities.install_os_packages([
'git', 'python-prettytable', 'python-coverage', 'python-dev',
'libkrb5-dev'
])
irods_python_ci_utilities.subprocess_get_output(
['sudo', '-H', 'pip', 'install', 'gssapi'], check_rc=True)
def install_os_specific_dependencies_apt():
if irods_python_ci_utilities.get_distribution(
) == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12
if irods_python_ci_utilities.get_distribution_version_major() == '12':
irods_python_ci_utilities.install_os_packages(
['python-software-properties'])
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'add-apt-repository', '-y',
'ppa:ubuntu-toolchain-r/test'
],
check_rc=True)
irods_python_ci_utilities.install_os_packages(['libstdc++6'])
irods_python_ci_utilities.install_os_packages([
'fakeroot',
'help2man',
'libbz2-dev',
'libcurl4-gnutls-dev',
'libkrb5-dev',
'libpam0g-dev',
'libssl-dev',
'make',
'python-dev',
'unixodbc',
'unixodbc-dev',
'zlib1g-dev',
])
def run_baton_tests(baton_source_dir):
env = os.environ.copy()
env['CK_DEFAULT_TIMEOUT'] = '20' # the 'check' unit test framework has a default timeout that is too short for three of the baton tests. If you see 'Test timeout expired', may need to increase this
irods_python_ci_utilities.subprocess_get_output(['make', 'check'],
env=env,
cwd=baton_source_dir,
check_rc=True)
def run_tests():
# install_testing_dependencies()
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'export R_LIBS={0} && cd /var/lib/irods/tests/pydevtest && python run_tests.py --xml_output --run_specific_test test_client_rirods'
.format(os.getcwd())
],
check_rc=False)
def create_privileged_principal():
stdin = '''addprinc root/admin
krbtest
krbtest
'''
irods_python_ci_utilities.subprocess_get_output(['sudo', 'kadmin.local'],
data=stdin,
check_rc=True)
def install_os_specific_dependencies_apt():
if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12
irods_python_ci_utilities.subprocess_get_output(['sudo', 'apt-get', 'update'], check_rc=True)
if irods_python_ci_utilities.get_distribution_version_major() == '12':
irods_python_ci_utilities.install_os_packages(['python-software-properties'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True)
irods_python_ci_utilities.install_os_packages(['libstdc++6'])
irods_python_ci_utilities.install_os_packages(['make', 'libssl-dev', 'libxml2-dev', 'libcurl4-gnutls-dev', 'gcc'])
def gather_logs(output_root_directory):
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'chmod', '-R', '777', '/var/lib/irods'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output([
'rsync', '-r', '/var/lib/irods/iRODS/server/log/',
output_root_directory + '/'
],
check_rc=True)
def do_globus_config():
irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', 'o+rx', '/home/irodsbuild'], check_rc=True) # so user simpleca can read ~irodsbuild/.globus/usercert_request.pem
irodsbuild_password = create_irodsbuild_certificate()
create_irods_certificate()
generate_proxy('irodsbuild', irodsbuild_password)
generate_proxy('irods', None)
irodsbuild_proxy_copy = make_irods_readable_copy_of_irodsbuild_proxy()
irodsbuild_distinguished_name = get_irodsbuild_distinguished_name()
create_test_configuration_json(irodsbuild_proxy_copy, irodsbuild_distinguished_name)
def change_server_hash_scheme_to_md5():
server_config_path = '/etc/irods/server_config.json'
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'chmod', 'o+rw', server_config_path], check_rc=True)
with open(server_config_path) as f:
d = json.load(f)
d['default_hash_scheme'] = 'MD5'
with open(server_config_path, 'w') as f:
json.dump(d, f, ensure_ascii=True, sort_keys=True, indent=4)
def main(output_root_directory, irods_packages_root_directory, externals_directory):
install_building_dependencies(externals_directory)
if irods_packages_root_directory:
install_irods_dev_and_runtime_packages(irods_packages_root_directory)
build_directory = tempfile.mkdtemp(prefix='irods_gsi_plugin_build_directory')
irods_python_ci_utilities.subprocess_get_output(['cmake', os.path.dirname(os.path.realpath(__file__))], check_rc=True, cwd=build_directory)
irods_python_ci_utilities.subprocess_get_output(['make', '-j', str(multiprocessing.cpu_count()), 'package'], check_rc=True, cwd=build_directory)
if output_root_directory:
copy_output_packages(build_directory, output_root_directory)
def make_irods_readable_copy_of_irodsbuild_proxy():
uid = pwd.getpwnam('irodsbuild').pw_uid
proxy_file = '/tmp/x509up_u' + str(uid)
irods_copy_of_proxy = '/tmp/irods_copy_of_irodsbuild_gsi_proxy'
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'cp', proxy_file, irods_copy_of_proxy], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'chown', 'irods:irods', irods_copy_of_proxy], check_rc=True)
return irods_copy_of_proxy
def create_keytab():
stdin = '''krbtest
ktadd -k /var/lib/irods/irods.keytab irods/[email protected]
'''
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'kadmin', '-p', 'root/admin'], data=stdin, check_rc=True)
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'chown', 'irods:irods', '/var/lib/irods/irods.keytab'],
check_rc=True)
def create_unprivileged_principal(principal):
stdin = '''{0}
addprinc {1}
{2}
{2}
'''.format(kdc_database_master_key(), principal,
unprivileged_principal_password())
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'kadmin', '-p', 'root/admin'], data=stdin, check_rc=True)
def install_irods_repository_apt():
irods_python_ci_utilities.subprocess_get_output(
'wget -qO - https://core-dev.irods.org/irods-core-dev-signing-key.asc | sudo apt-key add -',
shell=True,
check_rc=True)
irods_python_ci_utilities.subprocess_get_output(
'echo "deb [arch=amd64] https://core-dev.irods.org/apt/ $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/renci-irods-core-dev.list',
shell=True,
check_rc=True)
subprocess.check_call('apt-get clean && apt-get update', shell=True)
def install_irods_repository_zypper():
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'rpm', '--import',
'https://core-dev.irods.org/irods-core-dev-signing-key.asc'
],
check_rc=True)
irods_python_ci_utilities.subprocess_get_output(
'wget -qO - https://core-dev.irods.org/renci-irods-core-dev.zypp.repo | sudo tee /etc/zypp/repos.d/renci-irods-core-dev.zypp.repo',
shell=True,
check_rc=True)
def checkout_git_repo_and_run_test_hook(git_repo, git_commitish,
passthrough_arguments,
install_externals, database_type):
if irods_python_ci_utilities.get_distribution() == 'Ubuntu':
irods_python_ci_utilities.subprocess_get_output(['apt-get', 'update'],
check_rc=True)
_git_repo = git_repo.split('/')
plugin_name = _git_repo[len(_git_repo) - 1]
git_sha = ci_utilities.get_sha_from_commitish(git_repo, git_commitish)
git_checkout_dir = irods_python_ci_utilities.git_clone(git_repo, git_sha)
plugin_build_dir = '/plugin_mount_dir/{0}'.format(plugin_name)
passthru_args = []
if passthrough_arguments is not None:
for args in passthrough_arguments.split(','):
arg1 = args.split(' ')
passthru_args = passthru_args + arg1
if 'kerberos' in plugin_name:
plugin_name = plugin_name.replace('kerberos', 'krb')
if 'capability_storage_tiering' in plugin_name:
if len(passthru_args) > 0:
plugin_name = 'irods_rule_engine_plugin_unified_storage_tiering'
passthru_args.extend([
'--munge_path',
'export PATH={}:$PATH'.format(get_mungefs_directory())
])
plugin_basename = plugin_name.replace('_', '-') + '*'
plugin_package = glob.glob(
os.path.join(plugin_build_dir,
irods_python_ci_utilities.get_irods_platform_string(),
plugin_basename))
if install_externals:
externals_list = ci_utilities.get_package_dependencies(
''.join(plugin_package))
if len(externals_list) > 0:
ci_utilities.install_externals_from_list(externals_list,
get_externals_directory())
python_script = 'irods_consortium_continuous_integration_test_hook.py'
output_directory = '/irods_test_env/{0}/{1}/{2}'.format(
plugin_name, irods_python_ci_utilities.get_irods_platform_string(),
database_type)
time.sleep(10)
cmd = [
'python', python_script, '--output_root_directory', output_directory,
'--built_packages_root_directory', plugin_build_dir
] + passthru_args
print(cmd)
return irods_python_ci_utilities.subprocess_get_output(
cmd, cwd=git_checkout_dir, check_rc=True)
def create_test_configuration_json(irodsbuild_proxy_copy,
irodsbuild_distinguished_name):
config = {
'client_user_proxy': irodsbuild_proxy_copy,
'client_user_DN': irodsbuild_distinguished_name
}
config_file = '/tmp/gsi_test_cfg.json'
with open(config_file, 'w') as f:
json.dump(config, f)
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'chmod', '777', config_file], check_rc=True)
def install_os_specific_dependencies_apt():
if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12
if irods_python_ci_utilities.get_distribution_version_major() == '12':
irods_python_ci_utilities.install_os_packages(['python-software-properties'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True)
irods_python_ci_utilities.install_os_packages(['libstdc++6'])
irods_python_ci_utilities.install_os_packages([
'fakeroot', 'help2man', 'libbz2-dev', 'libcurl4-gnutls-dev', 'libkrb5-dev', 'libpam0g-dev',
'libssl-dev', 'make', 'python-dev', 'unixodbc', 'unixodbc-dev', 'zlib1g-dev',
])
def main(output_root_directory, irods_packages_root_directory):
irods_python_ci_utilities.install_irods_core_dev_repository()
install_cmake_and_add_to_front_of_path()
install_build_prerequisites()
if irods_packages_root_directory:
install_irods_dev_and_runtime_packages(irods_packages_root_directory)
build_directory = tempfile.mkdtemp(prefix='irods_gsi_plugin_build_directory')
irods_python_ci_utilities.subprocess_get_output(['cmake', os.path.dirname(os.path.realpath(__file__))], check_rc=True, cwd=build_directory)
irods_python_ci_utilities.subprocess_get_output(['make', '-j', str(multiprocessing.cpu_count()), 'package'], check_rc=True, cwd=build_directory)
if output_root_directory:
copy_output_packages(build_directory, output_root_directory)
def restart_irods():
if irods_python_ci_utilities.get_irods_version() >= (4, 2):
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c', '/var/lib/irods/irodsctl restart'
],
check_rc=True)
else:
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', 'irods', '-c',
'/var/lib/irods/iRODS/irodsctl restart'
],
check_rc=True)
def add_domain_to_krb5_conf():
with tempfile.NamedTemporaryFile() as conf_copy:
with open('/etc/krb5.conf', 'r') as conf:
for l in conf:
conf_copy.write(l)
if '[domain_realm]' in l:
conf_copy.write(' .example.org = EXAMPLE.ORG\n')
conf_copy.write(' example.org = EXAMPLE.ORG\n')
conf_copy.flush()
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'cp', conf_copy.name, '/etc/krb5.conf'],
check_rc=True)
def do_globus_config():
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'chmod', 'o+rx', '/home/irodsbuild'], check_rc=True
) # so user simpleca can read ~irodsbuild/.globus/usercert_request.pem
irodsbuild_password = create_irodsbuild_certificate()
create_irods_certificate()
generate_proxy('irodsbuild', irodsbuild_password)
generate_proxy('irods', None)
irodsbuild_proxy_copy = make_irods_readable_copy_of_irodsbuild_proxy()
irodsbuild_distinguished_name = get_irodsbuild_distinguished_name()
create_test_configuration_json(irodsbuild_proxy_copy,
irodsbuild_distinguished_name)
def generate_proxy(username, password):
if password:
irods_python_ci_utilities.subprocess_get_output([
'sudo', 'su', '-', username, '-c'
'echo {0} | grid-proxy-init -pwstdin'.format(password)
],
check_rc=True)
else:
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'su', '-', username, '-c'
'grid-proxy-init'],
check_rc=True)
def main(output_root_directory):
build_directory = os.getcwd()
irods_python_ci_utilities.subprocess_get_output(
['python', 'install_prerequisites.py'],
check_rc=True,
cwd=os.path.dirname(os.path.realpath(__file__)))
irods_python_ci_utilities.subprocess_get_output(
['make'],
check_rc=True,
cwd=os.path.dirname(os.path.realpath(__file__)))
if output_root_directory:
copy_output_packages(build_directory, output_root_directory)
def install_testing_dependencies():
irods_python_ci_utilities.subprocess_get_output(
['sudo', '-EH', 'pip', 'install', 'unittest-xml-reporting==1.14.0'])
globus_toolkit_package_name = get_test_prerequisites()
irods_python_ci_utilities.subprocess_get_output([
'wget', 'http://toolkit.globus.org/ftppub/gt6/installers/repo/{0}'.
format(globus_toolkit_package_name)
],
check_rc=True)
irods_python_ci_utilities.install_os_packages_from_files(
[globus_toolkit_package_name])
irods_python_ci_utilities.install_os_packages(['globus-gsi'])
def enable_admin_privileges_yum():
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'chmod', '644', '/var/kerberos/krb5kdc/kadm5.acl'],
check_rc=True)
with tempfile.NamedTemporaryFile() as kadm5_copy:
with open('/var/kerberos/krb5kdc/kadm5.acl', 'r') as kadm5_file:
for l in kadm5_file:
kadm5_copy.write(l)
kadm5_copy.write('*/[email protected] *\n')
kadm5_copy.flush()
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'cp', kadm5_copy.name, '/var/kerberos/krb5kdc/kadm5.acl'],
check_rc=True)
def add_shortname_to_etc_hosts():
fullname = socket.gethostname()
shortname = fullname.partition('.')[0]
with tempfile.NamedTemporaryFile() as hosts_copy:
with open('/etc/hosts', 'r') as hosts_file:
for l in hosts_file:
if fullname in l:
hosts_copy.write(l.strip() + ' ' + shortname + '\n')
else:
hosts_copy.write(l)
hosts_copy.flush()
irods_python_ci_utilities.subprocess_get_output(
['sudo', 'cp', hosts_copy.name, '/etc/hosts'], check_rc=True)
def create_irods_certificate():
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'grid-cert-request -nopw -force -cn irods_service'], check_rc=True)
temporary_certificate_location = '/tmp/gsicert'
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-s', '/bin/bash', '-c', 'grid-ca-sign -in /var/lib/irods/.globus/usercert_request.pem -out {0}'.format(temporary_certificate_location), 'simpleca'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'cp', temporary_certificate_location, '/var/lib/irods/.globus/usercert.pem'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'rm', temporary_certificate_location], check_rc=True)
def install_build_prerequisites_apt():
if irods_python_ci_utilities.get_distribution() == 'Ubuntu': # cmake from externals requires newer libstdc++ on ub12
if irods_python_ci_utilities.get_distribution_version_major() == '12':
irods_python_ci_utilities.install_os_packages(['python-software-properties'])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True)
irods_python_ci_utilities.install_os_packages(['libstdc++6'])
irods_python_ci_utilities.install_os_packages(get_build_prerequisites())
if irods_python_ci_utilities.get_distribution() == 'Ubuntu':
if irods_python_ci_utilities.get_distribution_version_major() == '12':
irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/libglobus_callout.so.0', '/usr/lib/libglobus_callout.so')
irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/libglobus_gss_assist.so.3', '/usr/lib/libglobus_gss_assist.so')
elif irods_python_ci_utilities.get_distribution_version_major() == '14':
irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_callout.so.0', '/usr/lib/x86_64-linux-gnu/libglobus_callout.so')
irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so.3', '/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so')
elif irods_python_ci_utilities.get_distribution_version_major() == '16':
irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_callout.so.0', '/usr/lib/x86_64-linux-gnu/libglobus_callout.so')
irods_python_ci_utilities.make_symbolic_link_as_root('/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so.3', '/usr/lib/x86_64-linux-gnu/libglobus_gss_assist.so')
def build_irods(debug_build):
irods_source_dir = os.path.dirname(os.path.realpath(__file__))
irods_build_dir = tempfile.mkdtemp(prefix='irods_build_dir')
logging.getLogger(__name__).info('Using iRODS build directory: %s', irods_build_dir)
if debug_build:
cmake_build_type = 'Debug'
else:
cmake_build_type = 'Release'
irods_python_ci_utilities.subprocess_get_output('cmake {0} -DCMAKE_BUILD_TYPE={1} > cmake.output'.format(irods_source_dir, cmake_build_type), shell=True, cwd=irods_build_dir, check_rc=True)
irods_python_ci_utilities.subprocess_get_output('make -j{0} > make.output'.format(str(multiprocessing.cpu_count())), shell=True, cwd=irods_build_dir, check_rc=True)
irods_python_ci_utilities.subprocess_get_output('fakeroot make package >> make.output', shell=True, cwd=irods_build_dir, check_rc=True)
return irods_build_dir
def build_icommands(icommands_git_repository, icommands_git_commitish, debug_build):
icommands_source_dir = irods_python_ci_utilities.git_clone(icommands_git_repository, icommands_git_commitish)
logging.getLogger(__name__).info('Using icommands source directory: %s', icommands_source_dir)
icommands_build_dir = tempfile.mkdtemp(prefix='icommands_build_dir')
logging.getLogger(__name__).info('Using icommands build directory: %s', icommands_build_dir)
if debug_build:
cmake_build_type = 'Debug'
else:
cmake_build_type = 'Release'
irods_python_ci_utilities.subprocess_get_output('cmake {0} -DCMAKE_BUILD_TYPE={1} > cmake.output'.format(icommands_source_dir, cmake_build_type), shell=True, cwd=icommands_build_dir, check_rc=True)
irods_python_ci_utilities.subprocess_get_output('make -j{0} > make.output'.format(str(multiprocessing.cpu_count())), shell=True, cwd=icommands_build_dir, check_rc=True)
irods_python_ci_utilities.subprocess_get_output('fakeroot make package >> make.output', shell=True, cwd=icommands_build_dir, check_rc=True)
return icommands_build_dir
def main():
parser = optparse.OptionParser()
parser.add_option('--output_root_directory')
parser.add_option('--built_packages_root_directory')
options, _ = parser.parse_args()
output_root_directory = options.output_root_directory
built_packages_root_directory = options.built_packages_root_directory
plugin_dir = irods_python_ci_utilities.append_os_specific_directory(built_packages_root_directory)
irods_python_ci_utilities.install_os_packages_from_files([os.path.join(plugin_dir, entry) for entry in os.listdir(plugin_dir)])
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/setup_python_rule_engine_as_only_rule_engine.py'], check_rc=True)
test_output_file = 'log/test_output.log'
try:
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'python2 scripts/run_tests.py --xml_output --run_python_suite > {0} 2>&1'.format(test_output_file)], check_rc=True)
finally:
if output_root_directory:
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irods', '-c', 'chmod o+r -R {0}'.format(' '.join(['test-reports', 'log', test_output_file]))])
shutil.copytree(os.path.join(os.path.expanduser('~irods'), 'test-reports'), os.path.join(output_root_directory, 'test-reports'))
shutil.copytree(os.path.join(os.path.expanduser('~irods'), 'log'), os.path.join(output_root_directory, 'log'))
shutil.copy(os.path.join(os.path.expanduser('~irods'), test_output_file), output_root_directory)
def build_irods(irods_git_dir, irods_build_dir, irods_install_dir):
ci.subprocess_get_output('cmake {0} -DCMAKE_INSTALL_PREFIX={1} > cmake_irods.output'.format(irods_git_dir, irods_install_dir), cwd=irods_build_dir, shell=True, check_rc=True)
ci.subprocess_get_output('make -j4 non-package-install-postgres > make_irods.output', cwd=irods_build_dir, shell=True, check_rc=True)
def install_building_dependencies_debian():
ci.install_os_packages(['git', 'g++', 'make', 'python-dev', 'help2man', 'unixodbc', 'libfuse-dev', 'libcurl4-gnutls-dev', 'libbz2-dev', 'zlib1g-dev', 'libpam0g-dev', 'libssl-dev', 'libxml2-dev', 'libkrb5-dev', 'unixodbc-dev', 'libjson-perl', 'python-psutil', 'fakeroot'])
if ci.get_distribution_version_major() == '12':
ci.install_os_packages(['python-software-properties'])
ci.subprocess_get_output(['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test'], check_rc=True)
ci.install_os_packages(['libstdc++6'])
def install_building_dependencies_suse():
ci.subprocess_get_output(['sudo', 'zypper', 'addrepo', 'http://download.opensuse.org/repositories/devel:tools/openSUSE_13.2/devel:tools.repo'], check_rc=True)
ci.subprocess_get_output(['sudo', 'zypper', '--gpg-auto-import-keys', 'refresh'], check_rc=True)
ci.install_os_packages(['git', 'python-devel', 'help2man', 'unixODBC', 'fuse-devel', 'libcurl-devel', 'libbz2-devel', 'libopenssl-devel', 'libxml2-devel', 'krb5-devel', 'perl-JSON', 'unixODBC-devel', 'python-psutil', 'fakeroot'])
def run_irods_tests(irods_install_dir):
ci.subprocess_get_output('python run_tests.py --xml_output --run_python_suite > test_output.txt 2>&1', shell=True, cwd=os.path.join(irods_install_dir, 'var', 'lib', 'irods', 'scripts'))
def copy_output(irods_install_dir, output_directory):
ci.subprocess_get_output(['mkdir', '-p', output_directory], check_rc=True)
ci.subprocess_get_output('cp test_output.txt {0}'.format(output_directory), cwd=os.path.join(irods_install_dir, 'var', 'lib', 'irods', 'scripts'), shell=True, check_rc=True)
ci.subprocess_get_output('cp * {0}'.format(output_directory), cwd=os.path.join(irods_install_dir, 'var', 'lib', 'irods', 'scripts', 'test-reports'), shell=True, check_rc=True)
ci.subprocess_get_output('cp rodsLog* {0}'.format(output_directory), cwd=os.path.join(irods_install_dir, 'var', 'lib', 'irods', 'log'), shell=True, check_rc=True)
def setup_irods(irods_install_dir):
ci.subprocess_get_output('python scripts/setup_irods.py < packaging/localhost_setup_postgres.input', cwd=os.path.join(irods_install_dir, 'var', 'lib', 'irods'), shell=True, check_rc=True)
def install_testing_dependencies():
ci.subprocess_get_output(['sudo', '-E', 'pip2', 'install', '--upgrade', 'unittest-xml-reporting==1.14.0'], check_rc=True)
if not (ci.get_distribution() == 'Ubuntu' and ci.get_distribution_version_major() == '12'):
ci.install_os_packages(['python-jsonschema'])
def create_irodsbuild_certificate():
irods_python_ci_utilities.subprocess_get_output(['grid-cert-request', '-nopw', '-force', '-cn', 'gsi_client_user'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', 'u+w', '/home/irodsbuild/.globus/userkey.pem'], check_rc=True)
private_key_password = '******'
irods_python_ci_utilities.subprocess_get_output(['openssl', 'rsa', '-in', '/home/irodsbuild/.globus/userkey.pem', '-out', '/home/irodsbuild/.globus/userkey.pem', '-des3', '-passout', 'pass:{0}'.format(private_key_password)], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', '400', '/home/irodsbuild/.globus/userkey.pem'], check_rc=True)
temporary_certificate_location = '/tmp/gsicert'
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-s', '/bin/bash', '-c', 'grid-ca-sign -in /home/irodsbuild/.globus/usercert_request.pem -out {0}'.format(temporary_certificate_location), 'simpleca'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'cp', temporary_certificate_location, '/home/irodsbuild/.globus/usercert.pem'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'rm', temporary_certificate_location], check_rc=True)
return private_key_password
def configure_hostname():
ci.subprocess_get_output(['sudo', 'hostname', 'icat.example.org'], check_rc=True)
ci.subprocess_get_output(['sudo', 'su', '-', '-c', 'echo -e "127.0.0.1 icat.example.org\n$(cat /etc/hosts)" > /etc/hosts'], check_rc=True)
def build_icommands(icommands_git_dir, icommands_build_dir, irods_install_dir):
IRODS_DIR = os.path.join(irods_install_dir, 'usr/lib/irods/cmake')
ci.subprocess_get_output('cmake {0} -DCMAKE_INSTALL_PREFIX={1} -DIRODS_DIR={2} > cmake_icommands.output'.format(icommands_git_dir, irods_install_dir, IRODS_DIR), cwd=icommands_build_dir, shell=True, check_rc=True)
ci.subprocess_get_output('make -j4 install > make_icommands.output', cwd=icommands_build_dir, shell=True, check_rc=True)
def run_tests():
# prerequisites
install_testing_dependencies()
irods_python_ci_utilities.subprocess_get_output(['sudo', 'cp', '-r', '/var/lib/irods/.irods', '/home/irodsbuild/'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'chown', '-R', 'irodsbuild', '/home/irodsbuild/.irods'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', '-R', '777', '/etc/irods'], check_rc=True)
# test run
test_env = os.environ.copy()
test_env['IRODS_CI_TEST_RUN'] = '1'
irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'run', 'irods/test/runner.py'], env=test_env)
# reports
irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'report'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'xml'], check_rc=True)
def generate_proxy(username, password):
if password:
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', username, '-c' 'echo {0} | grid-proxy-init -pwstdin'.format(password)], check_rc=True)
else:
irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', username, '-c' 'grid-proxy-init'], check_rc=True)
def install_testing_dependencies():
irods_python_ci_utilities.subprocess_get_output(['sudo', '-EH', 'pip', 'install', 'unittest-xml-reporting==1.14.0'])
globus_toolkit_package_name = get_test_prerequisites()
irods_python_ci_utilities.subprocess_get_output(['wget', 'http://toolkit.globus.org/ftppub/gt6/installers/repo/{0}'.format(globus_toolkit_package_name)], check_rc=True)
irods_python_ci_utilities.install_os_packages_from_files([globus_toolkit_package_name])
irods_python_ci_utilities.install_os_packages(['globus-gsi'])
def get_irodsbuild_distinguished_name():
_, name, _ = irods_python_ci_utilities.subprocess_get_output(['sudo', 'su', '-', 'irodsbuild', '-c', 'grid-cert-info -subject'], check_rc=True)
return name.strip()
def install_testing_dependencies_apt():
irods_python_ci_utilities.install_os_packages(['git', 'python-prettytable', 'python-coverage', 'python-dev', 'libkrb5-dev'])
irods_python_ci_utilities.subprocess_get_output(['sudo', '-H', 'pip', 'install', 'gssapi'], check_rc=True)
def run_tests():
install_testing_dependencies()
irods_python_ci_utilities.subprocess_get_output(['sudo', 'chmod', '-R', '777', '/etc/irods'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'run', 'irods/test/runner.py'])
irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'report'], check_rc=True)
irods_python_ci_utilities.subprocess_get_output(['python-coverage', 'xml'], check_rc=True)
def configure_database():
ci.subprocess_get_output(['sudo', 'su', '-', 'postgres', '-c', 'createuser -s irods'], check_rc=True)
ci.subprocess_get_output(['sudo', 'su', '-', 'postgres', '-c', '''psql -c "alter role irods with password 'testpassword'"'''], check_rc=True)
ci.subprocess_get_output(['sudo', 'su', '-', 'postgres', '-c', "createdb 'ICAT'"], check_rc=True)
def checkout_git_repo_and_run_build_hook(git_repository, git_commitish, python_script, passthrough_arguments):
git_checkout_dir = irods_python_ci_utilities.git_clone(git_repository, git_commitish)
return irods_python_ci_utilities.subprocess_get_output(['python', python_script] + passthrough_arguments, cwd=git_checkout_dir, check_rc=True)
