def _prepare_build_container(registry, image, tag): if tag is not None: if utils.local_image_exist(image, tag): image_name = image + ':' + tag utils.logger.info("Using build container: %(image_name)s", dict(image_name=image_name)) return image_name if utils.remote_image_exist(registry, image, tag): fqdn_image = utils.generate_fqdn_image(registry, image, tag) utils.logger.info("Using build container: %(fqdn_image)s", dict(fqdn_image=fqdn_image)) return fqdn_image raise click.exceptions.ClickException( "Couldn't find build image %(image)s with tag %(tag)s" % dict(image=image, tag=tag)) utils.logger.info( "No build container tag was provided. Building from scratch...") dockerfile = utils.image_to_dockerfile(image) command = ['docker', 'build', '-t', image, '-f', dockerfile, '.'] runner.run(command) return image
def _push(ctx, force, image, image_name, namespace, tag): fqdn_image = utils.generate_fqdn_image(ctx.obj['registry'], namespace, image, tag) utils.logger.debug("Adding tag %(tag)s", dict(tag=fqdn_image)) command = ['docker', 'tag', image_name, fqdn_image] ret = runner.run(command) if ret != 0: utils.logger.error('Failed to tag image: %(tag)s as fqdn', dict(tag=image_name, fqdn=fqdn_image)) sys.exit(ret) repo_name = utils.generate_fqdn_image(None, namespace, image, tag=None) images_info = utils.get_remote_images_info([repo_name], ctx.obj['registry']) tags = [info[-1] for info in images_info] if tag in tags: if not force: utils.logger.info( "Image %(image)s is already in registry %(registry)s, not pushing", dict(image=fqdn_image, registry=ctx.obj['registry'])) else: utils.logger.warning( "Image %(image)s is already in registry %(registry)s, pushing anyway", dict(image=fqdn_image, registry=ctx.obj['registry'])) _push_to_registry(ctx.obj['registry'], fqdn_image) else: _push_to_registry(ctx.obj['registry'], fqdn_image) utils.logger.debug("Removing tag %(tag)s", dict(tag=fqdn_image)) command = ['docker', 'rmi', fqdn_image] ret = runner.run(command) if ret != 0: utils.logger.warning('Failed to remove image tag: %(tag)s', dict(tag=fqdn_image)) return ret
def test_run_simple_command_nested_network_not_exist( self, check_output_mock, popen_mock, grp_getgrnam_mock, os_getuid_mock): check_output_mock.side_effect = ['', 'new-net-hash', ''] popen_mock.return_value.stdout.readline.side_effect = [ 'aaa', 'bbb', 'ccc', '' ] popen_mock.return_value.poll.return_value = -1 grp_getgrnam_mock.return_value.gr_gid = 978 os_getuid_mock.return_value = USER_ID command = ['pwd'] runner.run(command, FQDN_IMAGE) expected_nested_command = [ 'docker', 'run', '-t', '-e', 'KEEP_CONTAINERS=True', '--privileged', '--net', 'host', '-e', 'SKIPPER_USERNAME=testuser', '-e', 'SKIPPER_UID=%(user_uid)s' % dict(user_uid=USER_ID), '-e', 'HOME=%(homedir)s' % dict(homedir=HOME_DIR), '-e', 'SKIPPER_DOCKER_GID=978', '-v', '%(workdir)s:%(workdir)s:rw,Z' % dict(workdir=WORKDIR), '-v', '%(homedir)s/.netrc:%(homedir)s/.netrc:ro' % dict(homedir=HOME_DIR), '-v', '%(homedir)s/.gitconfig:%(homedir)s/.gitconfig:ro' % dict(homedir=HOME_DIR), '-v', '/var/lib/osmosis:/var/lib/osmosis:rw,Z', '-v', '/var/run/docker.sock:/var/run/docker.sock:Z', '-v', '/opt/skipper/skipper-entrypoint.sh:/opt/skipper/skipper-entrypoint.sh:Z', '-w', PROJECT_DIR, '--entrypoint', '/opt/skipper/skipper-entrypoint.sh', FQDN_IMAGE, command[0] ] popen_mock.assert_called_once_with(expected_nested_command)
def test_run_complex_command_nested_with_env(self, resource_filename_mock, check_output_mock, popen_mock, os_getuid_mock): resource_filename_mock.return_value = "entrypoint.sh" check_output_mock.side_effect = [self.NET_LS, ''] popen_mock.return_value.stdout.readline.side_effect = [ 'aaa', 'bbb', 'ccc', '' ] popen_mock.return_value.poll.return_value = -1 os_getuid_mock.return_value = USER_ID command = ['ls', '-l'] runner.run(command, FQDN_IMAGE, ENV) expected_nested_command = [ self.runtime, 'run', '-t', '-e', 'KEEP_CONTAINERS=True', '--privileged', '--net', 'host', '-e', 'KEY1=VAL1', '-e', 'KEY2=VAL2', '-e', 'SKIPPER_USERNAME=testuser', '-e', 'SKIPPER_UID=%(user_uid)s' % dict(user_uid=USER_ID), '-e', 'HOME=%(homedir)s' % dict(homedir=HOME_DIR), '-v', '%(workdir)s:%(workdir)s:rw,Z' % dict(workdir=WORKDIR), '-v', '%(homedir)s/.netrc:%(homedir)s/.netrc:ro' % dict(homedir=HOME_DIR), '-v', '%(homedir)s/.gitconfig:%(homedir)s/.gitconfig:ro' % dict(homedir=HOME_DIR), '-v', '/var/lib/osmosis:/var/lib/osmosis:rw,Z', '-v', '/var/run/docker.sock:/var/run/docker.sock:Z', '-v', 'entrypoint.sh:/opt/skipper/skipper-entrypoint.sh:Z', '-w', PROJECT_DIR, '--entrypoint', '/opt/skipper/skipper-entrypoint.sh', FQDN_IMAGE, ' '.join(command) ] popen_mock.assert_called_once_with(expected_nested_command)
def test_run_complex_command_not_nested(self, popen_mock): popen_mock.return_value.stdout.readline.side_effect = [ 'aaa', 'bbb', 'ccc', '' ] popen_mock.return_value.poll.return_value = -1 command = ['ls -l'] runner.run(command) popen_mock.assert_called_once_with(command)
def _push_to_registry(registry, fqdn_image): utils.logger.debug("Pushing to registry %(registry)s", dict(registry=registry)) command = ['docker', 'push', fqdn_image] ret = runner.run(command) if ret != 0: utils.logger.error('Failed to push image: %(tag)s', dict(tag=fqdn_image)) sys.exit(ret)
def build(images_to_build): ''' Build a container ''' utils.logger.debug("Executing build command") images_to_build = images_to_build or utils.get_images_from_dockerfiles() tag = git.get_hash() for image in images_to_build: utils.logger.info('Building image: %(image)s', dict(image=image)) dockerfile = utils.image_to_dockerfile(image) if not os.path.exists(dockerfile): utils.logger.warning( 'Image %(image)s is not valid for this project! Skipping...', dict(image=image)) continue fqdn_image = image + ':' + tag command = ['docker', 'build', '-f', dockerfile, '-t', fqdn_image, '.'] ret = runner.run(command) if ret != 0: utils.logger.error('Failed to build image: %(image)s', dict(image=image)) return ret return 0
def _prepare_build_container(registry, image, tag, git_revision, container_context, use_cache): if tag is not None: tagged_image_name = image + ':' + tag if utils.local_image_exist(image, tag): utils.logger.info("Using build container: %(image_name)s", dict(image_name=tagged_image_name)) return tagged_image_name if utils.remote_image_exist(registry, image, tag): fqdn_image = utils.generate_fqdn_image(registry, None, image, tag) utils.logger.info("Using build container: %(fqdn_image)s", dict(fqdn_image=fqdn_image)) return fqdn_image if not git_revision: raise click.exceptions.ClickException( "Couldn't find build image %(image)s with tag %(tag)s" % dict(image=image, tag=tag)) else: tagged_image_name = image utils.logger.info("No build container tag was provided") docker_file = utils.image_to_dockerfile(image) utils.logger.info("Building image using docker file: %(docker_file)s", dict(docker_file=docker_file)) if container_context is not None: build_context = container_context else: build_context = '.' command = ['docker', 'build', '--network=host', '-t', tagged_image_name, '-f', docker_file, build_context] if use_cache: cache_image = utils.generate_fqdn_image(registry, namespace=None, image=image, tag=DOCKER_TAG_FOR_CACHE) runner.run(['docker', 'pull', cache_image]) command.extend(['--cache-from', cache_image]) ret = runner.run(command) if ret != 0: exit('Failed to build image: %(image)s' % dict(image=image)) if git_revision and not git.uncommitted_changes(): utils.logger.info("Tagging image with git revision: %(tag)s", dict(tag=tag)) runner.run(['docker', 'tag', image, tagged_image_name]) if use_cache: cache_image = utils.generate_fqdn_image(registry, namespace=None, image=image, tag=DOCKER_TAG_FOR_CACHE) runner.run(['docker', 'tag', image, cache_image]) runner.run(['docker', 'push', cache_image]) return image
def run(ctx, interactive, env, command): ''' Run arbitrary commands ''' utils.logger.debug("Executing run command") _validate_global_params(ctx, 'build_container_image') build_container = _prepare_build_container( ctx.obj['registry'], ctx.obj['build_container_image'], ctx.obj['build_container_tag']) return runner.run(list(command), fqdn_image=build_container, environment=_expend_env(ctx, env), interactive=interactive)
def shell(ctx, env): ''' Start a shell ''' utils.logger.debug("Starting a shell") _validate_global_params(ctx, 'build_container_image') build_container = _prepare_build_container( ctx.obj['registry'], ctx.obj['build_container_image'], ctx.obj['build_container_tag']) return runner.run(['bash'], fqdn_image=build_container, environment=_expend_env(ctx, env), interactive=True)
def test_run_simple_command_nested(self, popen_mock, grp_getgrnam_mock, os_getuid_mock, *args): popen_mock.return_value.stdout.readline.side_effect = [ 'aaa', 'bbb', 'ccc', '' ] popen_mock.return_value.poll.return_value = -1 grp_getgrnam_mock.return_value.gr_gid = 978 os_getuid_mock.return_value = USER_ID command = ['pwd'] runner.run(command, FQDN_IMAGE) expected_nested_command = [ 'docker', 'run', '-t', '--rm', '--net', 'host', '-e', 'SKIPPER_USERNAME=testuser', '-e', 'SKIPPER_UID=%(user_uid)s' % dict(user_uid=USER_ID), '-e', 'SKIPPER_DOCKER_GID=978', '-v', '%(workdir)s:%(workdir)s:rw,Z' % dict(workdir=WORKDIR), '-v', '/var/lib/osmosis:/var/lib/osmosis:rw,Z', '-v', '/var/run/docker.sock:/var/run/docker.sock:Z', '-v', '/opt/skipper/skipper-entrypoint.sh:/opt/skipper/skipper-entrypoint.sh:Z', '-w', PROJECT_DIR, '--entrypoint', '/opt/skipper/skipper-entrypoint.sh', FQDN_IMAGE, command[0] ] popen_mock.assert_called_once_with(expected_nested_command)
def make(ctx, interactive, env, makefile, target): ''' Execute makefile target ''' utils.logger.debug("Executing make command") _validate_global_params(ctx, 'build_container_image') build_container = _prepare_build_container( ctx.obj['registry'], ctx.obj['build_container_image'], ctx.obj['build_container_tag']) command = ['make', '-f', makefile, target] return runner.run(command, fqdn_image=build_container, environment=_expend_env(ctx, env), interactive=interactive)
def build(ctx, images_to_build, container_context, cache): """ Build a container """ utils.logger.debug("Executing build command") valid_images = ctx.obj.get('containers') or utils.get_images_from_dockerfiles() valid_images = {image: os.path.abspath(dockerfile) for image, dockerfile in six.iteritems(valid_images)} valid_images_to_build = {} if not images_to_build: valid_images_to_build = valid_images else: for image in images_to_build: if image not in valid_images: utils.logger.warning('Image %(image)s is not valid for this project! Skipping...', dict(image=image)) continue valid_images_to_build[image] = valid_images[image] tag = git.get_hash() for image, dockerfile in six.iteritems(valid_images_to_build): utils.logger.info('Building image: %(image)s', dict(image=image)) if not os.path.exists(dockerfile): utils.logger.warning('Dockerfile %(dockerfile)s does not exist! Skipping...', dict(dockerfile=dockerfile)) continue fqdn_image = image + ':' + tag if container_context is not None: build_context = container_context elif ctx.obj['container_context']: build_context = ctx.obj['container_context'] else: build_context = os.path.dirname(dockerfile) command = ['docker', 'build', '--network=host', '--build-arg', 'TAG={}'.format(tag), '-f', dockerfile, '-t', fqdn_image, build_context] if cache: cache_image = utils.generate_fqdn_image(ctx.obj['registry'], namespace=None, image=image, tag=DOCKER_TAG_FOR_CACHE) runner.run(['docker', 'pull', cache_image]) command.extend(['--cache-from', cache_image]) ret = runner.run(command) if ret != 0: utils.logger.error('Failed to build image: %(image)s', dict(image=image)) return ret if cache: cache_image = utils.generate_fqdn_image(ctx.obj['registry'], namespace=None, image=image, tag=DOCKER_TAG_FOR_CACHE) runner.run(['docker', 'tag', fqdn_image, cache_image]) runner.run(['docker', 'push', cache_image]) return 0
def run(ctx, interactive, name, env, cache, command): """ Run arbitrary commands """ utils.logger.debug("Executing run command") _validate_global_params(ctx, 'build_container_image') build_container = _prepare_build_container( ctx.obj['registry'], ctx.obj['build_container_image'], ctx.obj['build_container_tag'], ctx.obj['git_revision'], ctx.obj['container_context'], cache) return runner.run(list(command), fqdn_image=build_container, environment=_expend_env(ctx, env), interactive=interactive, name=name, net=ctx.obj['build_container_net'], volumes=ctx.obj.get('volumes'), workdir=ctx.obj.get('workdir'), use_cache=cache)
def shell(ctx, env, name, cache): """ Start a shell """ utils.logger.debug("Starting a shell") _validate_global_params(ctx, 'build_container_image') build_container = _prepare_build_container( ctx.obj['registry'], ctx.obj['build_container_image'], ctx.obj['build_container_tag'], ctx.obj['git_revision'], ctx.obj['container_context'], cache) return runner.run(['bash'], fqdn_image=build_container, environment=_expend_env(ctx, env), interactive=True, name=name, net=ctx.obj['build_container_net'], volumes=ctx.obj.get('volumes'), workdir=ctx.obj.get('workdir'), use_cache=cache)
def make(ctx, interactive, name, env, makefile, cache, make_params): """ Execute makefile target(s) """ utils.logger.debug("Executing make command") _validate_global_params(ctx, 'build_container_image') build_container = _prepare_build_container( ctx.obj['registry'], ctx.obj['build_container_image'], ctx.obj['build_container_tag'], ctx.obj['git_revision'], ctx.obj['container_context'], cache) command = ['make', '-f', makefile] + list(make_params) return runner.run(command, fqdn_image=build_container, environment=_expend_env(ctx, env), interactive=interactive, name=name, net=ctx.obj['build_container_net'], volumes=ctx.obj.get('volumes'), workdir=ctx.obj.get('workdir'), use_cache=cache)
def push(ctx, image): ''' Push a container ''' utils.logger.debug("Executing push command") _validate_global_params(ctx, 'registry') tag = git.get_hash() image_name = image + ':' + tag fqdn_image = utils.generate_fqdn_image(ctx.obj['registry'], image, tag) utils.logger.debug("Adding tag %(tag)s", dict(tag=fqdn_image)) command = ['docker', 'tag', image_name, fqdn_image] runner.run(command) utils.logger.debug("Pushing to registry %(registry)s", dict(registry=ctx.obj['registry'])) command = ['docker', 'push', fqdn_image] runner.run(command) utils.logger.debug("Removing tag %(tag)s", dict(tag=fqdn_image)) command = ['docker', 'rmi', fqdn_image] runner.run(command)