def main():
parser = argparse.ArgumentParser(
description='Build irods in base os-containers')
parser.add_argument('-p', '--platform_target', type=str, required=True)
parser.add_argument('--image_tag',
type=str,
required=True,
help='Tag id or name for the base image')
parser.add_argument('-b', '--build_id', type=str, required=True)
parser.add_argument('--irods_repo', type=str, required=True)
parser.add_argument('--irods_commitish', type=str, required=True)
parser.add_argument('--icommands_repo', type=str, required=True)
parser.add_argument('--icommands_commitish', type=str, required=True)
parser.add_argument('--externals_packages_directory',
type=str,
default=None)
parser.add_argument('-o', '--output_directory', type=str, required=True)
args = parser.parse_args()
print('irods_repo:' + args.irods_repo)
print('irods_commitish:' + args.irods_commitish)
print('icommands_repo:' + args.icommands_repo)
print('icommands_commitish:' + args.icommands_commitish)
print('externals_packages_directory:' + args.externals_packages_directory)
irods_sha = ci_utilities.get_sha_from_commitish(args.irods_repo,
args.irods_commitish)
icommands_sha = ci_utilities.get_sha_from_commitish(
args.icommands_repo, args.icommands_commitish)
build_irods_in_containers(args.platform_target, args.image_tag,
args.build_id, args.irods_repo, irods_sha,
args.icommands_repo, icommands_sha,
args.output_directory,
args.externals_packages_directory)
def checkout_git_repo_and_run_test_hook(git_repo, git_commitish,
passthrough_arguments,
install_externals, database_type):
if irods_python_ci_utilities.get_distribution() == 'Ubuntu':
irods_python_ci_utilities.subprocess_get_output(['apt-get', 'update'],
check_rc=True)
_git_repo = git_repo.split('/')
plugin_name = _git_repo[len(_git_repo) - 1]
git_sha = ci_utilities.get_sha_from_commitish(git_repo, git_commitish)
git_checkout_dir = irods_python_ci_utilities.git_clone(git_repo, git_sha)
plugin_build_dir = '/plugin_mount_dir/{0}'.format(plugin_name)
passthru_args = []
if passthrough_arguments is not None:
for args in passthrough_arguments.split(','):
arg1 = args.split(' ')
passthru_args = passthru_args + arg1
if 'kerberos' in plugin_name:
plugin_name = plugin_name.replace('kerberos', 'krb')
if 'capability_storage_tiering' in plugin_name:
if len(passthru_args) > 0:
plugin_name = 'irods_rule_engine_plugin_unified_storage_tiering'
passthru_args.extend([
'--munge_path',
'export PATH={}:$PATH'.format(get_mungefs_directory())
])
plugin_basename = plugin_name.replace('_', '-') + '*'
plugin_package = glob.glob(
os.path.join(plugin_build_dir,
irods_python_ci_utilities.get_irods_platform_string(),
plugin_basename))
if install_externals:
externals_list = ci_utilities.get_package_dependencies(
''.join(plugin_package))
if len(externals_list) > 0:
ci_utilities.install_externals_from_list(externals_list,
get_externals_directory())
python_script = 'irods_consortium_continuous_integration_test_hook.py'
output_directory = '/irods_test_env/{0}/{1}/{2}'.format(
plugin_name, irods_python_ci_utilities.get_irods_platform_string(),
database_type)
time.sleep(10)
cmd = [
'python', python_script, '--output_root_directory', output_directory,
'--built_packages_root_directory', plugin_build_dir
] + passthru_args
print(cmd)
return irods_python_ci_utilities.subprocess_get_output(
cmd, cwd=git_checkout_dir, check_rc=True)
def main():
parser = argparse.ArgumentParser(description='Build irods in base os-containers')
parser.add_argument('-p', '--platform_target', type=str, required=True)
parser.add_argument('-b', '--build_id', type=str, required=True)
parser.add_argument('--externals_repo', type=str, required=True)
parser.add_argument('--externals_commitish', type=str, required=True)
parser.add_argument('-o', '--output_directory', type=str, required=True)
args = parser.parse_args()
machine_name = args.platform_target + '-externals-' + args.build_id
externals_sha = ci_utilities.get_sha_from_commitish(args.externals_repo, args.externals_commitish)
build_externals_in_containers(args.platform_target, args.build_id, args.externals_repo, externals_sha, args.output_directory, machine_name)
def main():
parser = argparse.ArgumentParser(description='Run tests in os-containers')
parser.add_argument('-p', '--platform_target', type=str, required=True)
parser.add_argument('-b', '--build_id', type=str, required=True)
parser.add_argument('--test_name_prefix', type=str, required=True)
parser.add_argument('--irods_build_dir', type=str, required=True)
parser.add_argument('--upgrade_packages_dir', type=str, required=True)
parser.add_argument('--irods_repo', type=str, required=True)
parser.add_argument('--irods_commitish', type=str, required=True)
parser.add_argument('--externals_dir',
type=str,
help='externals build directory')
parser.add_argument('--database_type',
default='postgres',
help='database type',
required=True)
parser.add_argument('--test_parallelism',
default='4',
help='The number of tests to run in parallel',
required=False)
parser.add_argument('-o', '--output_directory', type=str, required=True)
args = parser.parse_args()
build_tag = None
base_image = ci_utilities.get_base_image(args.platform_target,
args.build_id)
build_tag = ci_utilities.get_build_tag(args.platform_target,
'irods-install-upgrade',
args.build_id)
install_irods(build_tag, base_image, args.database_type)
test_name_prefix = args.platform_target + '-' + args.test_name_prefix
print(args.externals_dir)
irods_sha = ci_utilities.get_sha_from_commitish(args.irods_repo,
args.irods_commitish)
run_tests(build_tag, args.irods_repo, irods_sha, args.irods_build_dir,
args.upgrade_packages_dir, args.output_directory,
args.database_type, args.test_parallelism, test_name_prefix,
args.externals_dir)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-i',
'--image_name',
default='ubuntu_16:latest',
help='base image name',
required=True)
parser.add_argument('-j',
'--jenkins_output',
default='/jenkins_output',
help='jenkins output directory on the host machine',
required=True)
parser.add_argument('-t', '--test_name_prefix', help='test name prefix')
parser.add_argument('-b',
'--build_dir',
help='irods build directory',
required=True)
parser.add_argument('--externals_dir',
help='externals build directory',
default=None)
parser.add_argument('-d',
'--database_type',
default='postgres',
help='database type',
required=True)
parser.add_argument('--upgrade_packages_dir', required=False, default=None)
parser.add_argument('--irods_repo', type=str, required=True)
parser.add_argument('--irods_commitish', type=str, required=True)
parser.add_argument('--test_parallelism',
type=str,
default='4',
required=True)
parser.add_argument('--is_unit_test', action='store_true', default=False)
parser.add_argument('--run_timing_tests',
action='store_true',
default=False)
args = parser.parse_args()
# Add unit-test commands to the list.
docker_cmds_list = []
irods_sha = ci_utilities.get_sha_from_commitish(args.irods_repo,
args.irods_commitish)
if args.run_timing_tests:
test_list = ['timing_tests']
else:
if args.is_unit_test:
test_list = download_list_of_tests(
args.irods_repo, irods_sha, 'unit_tests/unit_tests_list.json')
docker_cmds_list.extend(
to_docker_commands(test_list, args, args.is_unit_test))
# Add core-test commands to the list.
test_list = download_list_of_tests(args.irods_repo, irods_sha,
'scripts/core_tests_list.json')
docker_cmds_list.extend(to_docker_commands(test_list, args))
print(docker_cmds_list)
run_pool = Pool(processes=int(args.test_parallelism))
containers = [{
'test_name':
docker_cmd['test_name'],
'proc':
run_pool.apply_async(docker_cmds_utilities.run_command_in_container, (
docker_cmd['run_cmd'],
docker_cmd['exec_cmd'],
docker_cmd['stop_cmd'],
docker_cmd['container_name'],
docker_cmd['alias_name'],
docker_cmd['database_container'],
docker_cmd['database_type'],
docker_cmd['network_name'],
), {
'test_type': docker_cmd['test_type'],
})
} for docker_cmd in docker_cmds_list]
container_error_codes = [{
'test_name': c['test_name'],
'error_code': c['proc'].get()
} for c in containers]
print(container_error_codes)
failures = []
for ec in container_error_codes:
if ec['error_code'] != 0:
failures.append(ec['test_name'])
if len(failures) > 0:
print('Failing Tests:')
for test_name in failures:
print('\t{0}'.format(test_name))
sys.exit(1)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-i',
'--image_name',
default='ubuntu_16:latest',
help='base image name',
required=True)
parser.add_argument('-j',
'--jenkins_output',
default='/jenkins_output',
help='jenkins output directory on the host machine',
required=True)
parser.add_argument('-t', '--test_name_prefix', help='test name prefix')
parser.add_argument('-b',
'--build_dir',
help='irods build directory',
required=True)
parser.add_argument('--externals_dir',
help='externals build directory',
default=None)
parser.add_argument('-d',
'--database_type',
default='postgres',
help='database type',
required=True)
parser.add_argument('--upgrade_packages_dir', required=False, default=None)
parser.add_argument('--irods_repo', type=str, required=True)
parser.add_argument('--irods_commitish', type=str, required=True)
parser.add_argument('--test_parallelism',
type=str,
default='4',
required=True)
parser.add_argument('--is_unit_test', action='store_true', default=False)
parser.add_argument('--run_timing_tests',
action='store_true',
default=False)
args = parser.parse_args()
# Add unit-test commands to the list.
docker_cmds_list = []
irods_sha = ci_utilities.get_sha_from_commitish(args.irods_repo,
args.irods_commitish)
if args.run_timing_tests:
test_list = ['timing_tests']
docker_cmds_list.extend(to_docker_commands(test_list, args))
elif args.is_unit_test:
test_list = download_list_of_tests(args.irods_repo, irods_sha,
'unit_tests/unit_tests_list.json')
docker_cmds_list.extend(
to_docker_commands(test_list, args, args.is_unit_test))
else:
# Add core-test commands to the list.
test_list = download_list_of_tests(args.irods_repo, irods_sha,
'scripts/core_tests_list.json')
docker_cmds_list.extend(to_docker_commands(test_list, args))
run_pool = Pool(processes=int(args.test_parallelism))
job_output_dir = generate_job_output_directory_path(
args.jenkins_output, args.image_name)
try:
os.makedirs(job_output_dir)
except:
pass
print('Launching tests ...')
containers = [
{
'test_name':
docker_cmd['test_name'],
'proc':
run_pool.apply_async(
# The operation to run asynchronously.
docker_cmds_utilities.run_command_in_container,
# The arguments to the operation.
(
docker_cmd['run_cmd'],
docker_cmd['exec_cmd'],
docker_cmd['stop_cmd'],
docker_cmd['container_name'],
docker_cmd['alias_name'],
docker_cmd['database_container'],
docker_cmd['database_type'],
docker_cmd['network_name'],
),
# The dictionary that maps to **kwargs within the operation. This will be appended
# to the end of the arguments to the operation (e.g. operation(*tuple_args, **kwargs)).
{
'test_type':
docker_cmd['test_type'],
# The path of the file that will hold the execution results of docker commands and other
# information relating to the test. For example, the file will contain things such as the
# output of the iRODS setup script.
'log_path':
generate_log_path(docker_cmd['test_name'],
docker_cmd['is_unit_test'],
args.image_name, job_output_dir)
})
} for docker_cmd in docker_cmds_list
]
print('Waiting for test results ...')
container_results = [{
'test_name': c['test_name'],
'error_code': c['proc'].get()
} for c in containers]
failures = []
for r in container_results:
if r['error_code'] != 0:
failures.append(r['test_name'])
if len(failures) > 0:
print('\nFAILING TESTS:')
for f in failures:
print('\t' + f)
print('\nSee {0}/job_<test_name>.log for details.'.format(
job_output_dir))
sys.exit(1)
else:
print('\nALL TESTS PASSED!')
def main():
parser = argparse.ArgumentParser(description='Run tests in os-containers')
parser.add_argument('-p', '--platform_target', type=str, required=True)
parser.add_argument('--image_tag',
type=str,
required=True,
help='Tag id or name for the base image')
parser.add_argument('-b', '--build_id', type=str, required=True)
parser.add_argument('--irods_repo', type=str, required=False)
parser.add_argument('--irods_commitish', type=str, required=False)
parser.add_argument('--test_name_prefix', type=str, required=True)
parser.add_argument('--irods_build_dir', type=str, required=True)
parser.add_argument('--test_plugin', action='store_true', default=False)
parser.add_argument('--externals_dir',
type=str,
help='externals build directory')
parser.add_argument('--plugin_build_dir',
type=str,
help='plugin build directory')
parser.add_argument('--plugin_repo', help='plugin git repo')
parser.add_argument('--plugin_commitish', help='plugin git commit sha')
parser.add_argument('--database_type',
default='postgres',
help='database type',
required=True)
parser.add_argument('--test_parallelism',
default='4',
help='The number of tests to run in parallel',
required=False)
parser.add_argument('-o', '--output_directory', type=str, required=True)
parser.add_argument('--passthrough_arguments', type=str)
parser.add_argument('--run_unit_tests', action='store_true', default=False)
parser.add_argument('--run_timing_tests',
action='store_true',
default=False)
args = parser.parse_args()
build_tag = None
base_image = ci_utilities.get_base_image(args.platform_target,
args.image_tag)
if not args.test_plugin:
build_tag = ci_utilities.get_build_tag(args.platform_target,
'irods-install',
args.database_type,
args.build_id)
else:
build_tag = ci_utilities.get_build_tag(args.platform_target,
'plugin-install',
args.database_type,
args.build_id)
docker_cmds_utilities.build_irods_zone(build_tag, base_image,
args.database_type,
'Dockerfile.install_and_test', True)
test_name_prefix = args.platform_target + '_' + args.test_name_prefix.replace(
'-', '_')
if not args.test_plugin:
irods_sha = ci_utilities.get_sha_from_commitish(
args.irods_repo, args.irods_commitish)
run_tests(build_tag, irods_sha, test_name_prefix, args)
else:
plugin_repo = args.plugin_repo
plugin_repo_split = plugin_repo.split('/')
plugin = plugin_repo_split[len(plugin_repo_split) - 1]
plugin_name = plugin.split('.git')[0]
if 'audit' in plugin_name:
if '--message_broker' in args.passthrough_arguments:
message_broker = args.passthrough_arguments.split(' ')[1]
machine_name = args.platform_target + '-' + plugin_name + '-' + message_broker + '-' + args.database_type + '-' + args.build_id
elif 'storage' in plugin_name and args.passthrough_arguments is not None and 'unified' in args.passthrough_arguments:
plugin_name = plugin_name + '-unified'
machine_name = args.platform_target + '-' + plugin_name + '-' + args.database_type + '-' + args.build_id
else:
machine_name = args.platform_target + '-' + plugin_name + '-' + args.database_type + '-' + args.build_id
plugin_sha = ci_utilities.get_sha_from_commitish(
args.plugin_repo, args.plugin_commitish)
run_plugin_tests(build_tag, plugin_sha, machine_name, plugin_name,
test_name_prefix, args)
