def _prepare_build_container(registry, image, tag):
if tag is not None:
if utils.local_image_exist(image, tag):
image_name = image + ':' + tag
utils.logger.info("Using build container: %(image_name)s",
dict(image_name=image_name))
return image_name
if utils.remote_image_exist(registry, image, tag):
fqdn_image = utils.generate_fqdn_image(registry, image, tag)
utils.logger.info("Using build container: %(fqdn_image)s",
dict(fqdn_image=fqdn_image))
return fqdn_image
raise click.exceptions.ClickException(
"Couldn't find build image %(image)s with tag %(tag)s" %
dict(image=image, tag=tag))
utils.logger.info(
"No build container tag was provided. Building from scratch...")
dockerfile = utils.image_to_dockerfile(image)
command = ['docker', 'build', '-t', image, '-f', dockerfile, '.']
runner.run(command)
return image
def _push(ctx, force, image, image_name, namespace, tag):
fqdn_image = utils.generate_fqdn_image(ctx.obj['registry'], namespace,
image, tag)
utils.logger.debug("Adding tag %(tag)s", dict(tag=fqdn_image))
command = ['docker', 'tag', image_name, fqdn_image]
ret = runner.run(command)
if ret != 0:
utils.logger.error('Failed to tag image: %(tag)s as fqdn',
dict(tag=image_name, fqdn=fqdn_image))
sys.exit(ret)
repo_name = utils.generate_fqdn_image(None, namespace, image, tag=None)
images_info = utils.get_remote_images_info([repo_name],
ctx.obj['registry'])
tags = [info[-1] for info in images_info]
if tag in tags:
if not force:
utils.logger.info(
"Image %(image)s is already in registry %(registry)s, not pushing",
dict(image=fqdn_image, registry=ctx.obj['registry']))
else:
utils.logger.warning(
"Image %(image)s is already in registry %(registry)s, pushing anyway",
dict(image=fqdn_image, registry=ctx.obj['registry']))
_push_to_registry(ctx.obj['registry'], fqdn_image)
else:
_push_to_registry(ctx.obj['registry'], fqdn_image)
utils.logger.debug("Removing tag %(tag)s", dict(tag=fqdn_image))
command = ['docker', 'rmi', fqdn_image]
ret = runner.run(command)
if ret != 0:
utils.logger.warning('Failed to remove image tag: %(tag)s',
dict(tag=fqdn_image))
return ret
def test_run_simple_command_nested_network_not_exist(
self, check_output_mock, popen_mock, grp_getgrnam_mock,
os_getuid_mock):
check_output_mock.side_effect = ['', 'new-net-hash', '']
popen_mock.return_value.stdout.readline.side_effect = [
'aaa', 'bbb', 'ccc', ''
]
popen_mock.return_value.poll.return_value = -1
grp_getgrnam_mock.return_value.gr_gid = 978
os_getuid_mock.return_value = USER_ID
command = ['pwd']
runner.run(command, FQDN_IMAGE)
expected_nested_command = [
'docker', 'run', '-t', '-e', 'KEEP_CONTAINERS=True',
'--privileged', '--net', 'host', '-e', 'SKIPPER_USERNAME=testuser',
'-e',
'SKIPPER_UID=%(user_uid)s' % dict(user_uid=USER_ID), '-e',
'HOME=%(homedir)s' % dict(homedir=HOME_DIR), '-e',
'SKIPPER_DOCKER_GID=978', '-v',
'%(workdir)s:%(workdir)s:rw,Z' % dict(workdir=WORKDIR), '-v',
'%(homedir)s/.netrc:%(homedir)s/.netrc:ro' %
dict(homedir=HOME_DIR), '-v',
'%(homedir)s/.gitconfig:%(homedir)s/.gitconfig:ro' %
dict(homedir=HOME_DIR), '-v',
'/var/lib/osmosis:/var/lib/osmosis:rw,Z', '-v',
'/var/run/docker.sock:/var/run/docker.sock:Z', '-v',
'/opt/skipper/skipper-entrypoint.sh:/opt/skipper/skipper-entrypoint.sh:Z',
'-w', PROJECT_DIR, '--entrypoint',
'/opt/skipper/skipper-entrypoint.sh', FQDN_IMAGE, command[0]
]
popen_mock.assert_called_once_with(expected_nested_command)
def test_run_complex_command_nested_with_env(self, resource_filename_mock,
check_output_mock, popen_mock,
os_getuid_mock):
resource_filename_mock.return_value = "entrypoint.sh"
check_output_mock.side_effect = [self.NET_LS, '']
popen_mock.return_value.stdout.readline.side_effect = [
'aaa', 'bbb', 'ccc', ''
]
popen_mock.return_value.poll.return_value = -1
os_getuid_mock.return_value = USER_ID
command = ['ls', '-l']
runner.run(command, FQDN_IMAGE, ENV)
expected_nested_command = [
self.runtime, 'run', '-t', '-e', 'KEEP_CONTAINERS=True',
'--privileged', '--net', 'host', '-e', 'KEY1=VAL1', '-e',
'KEY2=VAL2', '-e', 'SKIPPER_USERNAME=testuser', '-e',
'SKIPPER_UID=%(user_uid)s' % dict(user_uid=USER_ID), '-e',
'HOME=%(homedir)s' % dict(homedir=HOME_DIR), '-v',
'%(workdir)s:%(workdir)s:rw,Z' % dict(workdir=WORKDIR), '-v',
'%(homedir)s/.netrc:%(homedir)s/.netrc:ro' %
dict(homedir=HOME_DIR), '-v',
'%(homedir)s/.gitconfig:%(homedir)s/.gitconfig:ro' %
dict(homedir=HOME_DIR), '-v',
'/var/lib/osmosis:/var/lib/osmosis:rw,Z', '-v',
'/var/run/docker.sock:/var/run/docker.sock:Z', '-v',
'entrypoint.sh:/opt/skipper/skipper-entrypoint.sh:Z', '-w',
PROJECT_DIR, '--entrypoint', '/opt/skipper/skipper-entrypoint.sh',
FQDN_IMAGE, ' '.join(command)
]
popen_mock.assert_called_once_with(expected_nested_command)
def test_run_complex_command_not_nested(self, popen_mock):
popen_mock.return_value.stdout.readline.side_effect = [
'aaa', 'bbb', 'ccc', ''
]
popen_mock.return_value.poll.return_value = -1
command = ['ls -l']
runner.run(command)
popen_mock.assert_called_once_with(command)
def _push_to_registry(registry, fqdn_image):
utils.logger.debug("Pushing to registry %(registry)s", dict(registry=registry))
command = ['docker', 'push', fqdn_image]
ret = runner.run(command)
if ret != 0:
utils.logger.error('Failed to push image: %(tag)s', dict(tag=fqdn_image))
sys.exit(ret)
def build(images_to_build):
'''
Build a container
'''
utils.logger.debug("Executing build command")
images_to_build = images_to_build or utils.get_images_from_dockerfiles()
tag = git.get_hash()
for image in images_to_build:
utils.logger.info('Building image: %(image)s', dict(image=image))
dockerfile = utils.image_to_dockerfile(image)
if not os.path.exists(dockerfile):
utils.logger.warning(
'Image %(image)s is not valid for this project! Skipping...',
dict(image=image))
continue
fqdn_image = image + ':' + tag
command = ['docker', 'build', '-f', dockerfile, '-t', fqdn_image, '.']
ret = runner.run(command)
if ret != 0:
utils.logger.error('Failed to build image: %(image)s',
dict(image=image))
return ret
return 0
def _prepare_build_container(registry, image, tag, git_revision, container_context, use_cache):
if tag is not None:
tagged_image_name = image + ':' + tag
if utils.local_image_exist(image, tag):
utils.logger.info("Using build container: %(image_name)s", dict(image_name=tagged_image_name))
return tagged_image_name
if utils.remote_image_exist(registry, image, tag):
fqdn_image = utils.generate_fqdn_image(registry, None, image, tag)
utils.logger.info("Using build container: %(fqdn_image)s", dict(fqdn_image=fqdn_image))
return fqdn_image
if not git_revision:
raise click.exceptions.ClickException(
"Couldn't find build image %(image)s with tag %(tag)s" % dict(image=image, tag=tag))
else:
tagged_image_name = image
utils.logger.info("No build container tag was provided")
docker_file = utils.image_to_dockerfile(image)
utils.logger.info("Building image using docker file: %(docker_file)s", dict(docker_file=docker_file))
if container_context is not None:
build_context = container_context
else:
build_context = '.'
command = ['docker', 'build', '--network=host', '-t', tagged_image_name, '-f', docker_file, build_context]
if use_cache:
cache_image = utils.generate_fqdn_image(registry, namespace=None, image=image, tag=DOCKER_TAG_FOR_CACHE)
runner.run(['docker', 'pull', cache_image])
command.extend(['--cache-from', cache_image])
ret = runner.run(command)
if ret != 0:
exit('Failed to build image: %(image)s' % dict(image=image))
if git_revision and not git.uncommitted_changes():
utils.logger.info("Tagging image with git revision: %(tag)s", dict(tag=tag))
runner.run(['docker', 'tag', image, tagged_image_name])
if use_cache:
cache_image = utils.generate_fqdn_image(registry, namespace=None, image=image, tag=DOCKER_TAG_FOR_CACHE)
runner.run(['docker', 'tag', image, cache_image])
runner.run(['docker', 'push', cache_image])
return image
def run(ctx, interactive, env, command):
'''
Run arbitrary commands
'''
utils.logger.debug("Executing run command")
_validate_global_params(ctx, 'build_container_image')
build_container = _prepare_build_container(
ctx.obj['registry'], ctx.obj['build_container_image'],
ctx.obj['build_container_tag'])
return runner.run(list(command),
fqdn_image=build_container,
environment=_expend_env(ctx, env),
interactive=interactive)
def shell(ctx, env):
'''
Start a shell
'''
utils.logger.debug("Starting a shell")
_validate_global_params(ctx, 'build_container_image')
build_container = _prepare_build_container(
ctx.obj['registry'], ctx.obj['build_container_image'],
ctx.obj['build_container_tag'])
return runner.run(['bash'],
fqdn_image=build_container,
environment=_expend_env(ctx, env),
interactive=True)
def test_run_simple_command_nested(self, popen_mock, grp_getgrnam_mock,
os_getuid_mock, *args):
popen_mock.return_value.stdout.readline.side_effect = [
'aaa', 'bbb', 'ccc', ''
]
popen_mock.return_value.poll.return_value = -1
grp_getgrnam_mock.return_value.gr_gid = 978
os_getuid_mock.return_value = USER_ID
command = ['pwd']
runner.run(command, FQDN_IMAGE)
expected_nested_command = [
'docker', 'run', '-t', '--rm', '--net', 'host', '-e',
'SKIPPER_USERNAME=testuser', '-e',
'SKIPPER_UID=%(user_uid)s' % dict(user_uid=USER_ID), '-e',
'SKIPPER_DOCKER_GID=978', '-v',
'%(workdir)s:%(workdir)s:rw,Z' % dict(workdir=WORKDIR), '-v',
'/var/lib/osmosis:/var/lib/osmosis:rw,Z', '-v',
'/var/run/docker.sock:/var/run/docker.sock:Z', '-v',
'/opt/skipper/skipper-entrypoint.sh:/opt/skipper/skipper-entrypoint.sh:Z',
'-w', PROJECT_DIR, '--entrypoint',
'/opt/skipper/skipper-entrypoint.sh', FQDN_IMAGE, command[0]
]
popen_mock.assert_called_once_with(expected_nested_command)
def make(ctx, interactive, env, makefile, target):
'''
Execute makefile target
'''
utils.logger.debug("Executing make command")
_validate_global_params(ctx, 'build_container_image')
build_container = _prepare_build_container(
ctx.obj['registry'], ctx.obj['build_container_image'],
ctx.obj['build_container_tag'])
command = ['make', '-f', makefile, target]
return runner.run(command,
fqdn_image=build_container,
environment=_expend_env(ctx, env),
interactive=interactive)
def build(ctx, images_to_build, container_context, cache):
"""
Build a container
"""
utils.logger.debug("Executing build command")
valid_images = ctx.obj.get('containers') or utils.get_images_from_dockerfiles()
valid_images = {image: os.path.abspath(dockerfile) for image, dockerfile in six.iteritems(valid_images)}
valid_images_to_build = {}
if not images_to_build:
valid_images_to_build = valid_images
else:
for image in images_to_build:
if image not in valid_images:
utils.logger.warning('Image %(image)s is not valid for this project! Skipping...', dict(image=image))
continue
valid_images_to_build[image] = valid_images[image]
tag = git.get_hash()
for image, dockerfile in six.iteritems(valid_images_to_build):
utils.logger.info('Building image: %(image)s', dict(image=image))
if not os.path.exists(dockerfile):
utils.logger.warning('Dockerfile %(dockerfile)s does not exist! Skipping...', dict(dockerfile=dockerfile))
continue
fqdn_image = image + ':' + tag
if container_context is not None:
build_context = container_context
elif ctx.obj['container_context']:
build_context = ctx.obj['container_context']
else:
build_context = os.path.dirname(dockerfile)
command = ['docker', 'build', '--network=host', '--build-arg', 'TAG={}'.format(tag),
'-f', dockerfile, '-t', fqdn_image, build_context]
if cache:
cache_image = utils.generate_fqdn_image(ctx.obj['registry'], namespace=None, image=image, tag=DOCKER_TAG_FOR_CACHE)
runner.run(['docker', 'pull', cache_image])
command.extend(['--cache-from', cache_image])
ret = runner.run(command)
if ret != 0:
utils.logger.error('Failed to build image: %(image)s', dict(image=image))
return ret
if cache:
cache_image = utils.generate_fqdn_image(ctx.obj['registry'], namespace=None, image=image, tag=DOCKER_TAG_FOR_CACHE)
runner.run(['docker', 'tag', fqdn_image, cache_image])
runner.run(['docker', 'push', cache_image])
return 0
def run(ctx, interactive, name, env, cache, command):
"""
Run arbitrary commands
"""
utils.logger.debug("Executing run command")
_validate_global_params(ctx, 'build_container_image')
build_container = _prepare_build_container(
ctx.obj['registry'], ctx.obj['build_container_image'],
ctx.obj['build_container_tag'], ctx.obj['git_revision'],
ctx.obj['container_context'], cache)
return runner.run(list(command),
fqdn_image=build_container,
environment=_expend_env(ctx, env),
interactive=interactive,
name=name,
net=ctx.obj['build_container_net'],
volumes=ctx.obj.get('volumes'),
workdir=ctx.obj.get('workdir'),
use_cache=cache)
def shell(ctx, env, name, cache):
"""
Start a shell
"""
utils.logger.debug("Starting a shell")
_validate_global_params(ctx, 'build_container_image')
build_container = _prepare_build_container(
ctx.obj['registry'], ctx.obj['build_container_image'],
ctx.obj['build_container_tag'], ctx.obj['git_revision'],
ctx.obj['container_context'], cache)
return runner.run(['bash'],
fqdn_image=build_container,
environment=_expend_env(ctx, env),
interactive=True,
name=name,
net=ctx.obj['build_container_net'],
volumes=ctx.obj.get('volumes'),
workdir=ctx.obj.get('workdir'),
use_cache=cache)
def make(ctx, interactive, name, env, makefile, cache, make_params):
"""
Execute makefile target(s)
"""
utils.logger.debug("Executing make command")
_validate_global_params(ctx, 'build_container_image')
build_container = _prepare_build_container(
ctx.obj['registry'], ctx.obj['build_container_image'],
ctx.obj['build_container_tag'], ctx.obj['git_revision'],
ctx.obj['container_context'], cache)
command = ['make', '-f', makefile] + list(make_params)
return runner.run(command,
fqdn_image=build_container,
environment=_expend_env(ctx, env),
interactive=interactive,
name=name,
net=ctx.obj['build_container_net'],
volumes=ctx.obj.get('volumes'),
workdir=ctx.obj.get('workdir'),
use_cache=cache)
def push(ctx, image):
'''
Push a container
'''
utils.logger.debug("Executing push command")
_validate_global_params(ctx, 'registry')
tag = git.get_hash()
image_name = image + ':' + tag
fqdn_image = utils.generate_fqdn_image(ctx.obj['registry'], image, tag)
utils.logger.debug("Adding tag %(tag)s", dict(tag=fqdn_image))
command = ['docker', 'tag', image_name, fqdn_image]
runner.run(command)
utils.logger.debug("Pushing to registry %(registry)s",
dict(registry=ctx.obj['registry']))
command = ['docker', 'push', fqdn_image]
runner.run(command)
utils.logger.debug("Removing tag %(tag)s", dict(tag=fqdn_image))
command = ['docker', 'rmi', fqdn_image]
runner.run(command)
