def _revert_images(self, digests):
images = self.__get_images()
for service, image in images.items():
digest = digests[image]
command = 'docker service update --image {digest} {service}'
command = command.format(digest=digest, service=service)
fabricio.run(command)
def rotate_sentinel_images(self, rollback=False):
backup_tag = self.backup_settings_tag
current_tag = self.current_settings_tag
if rollback:
backup_tag, current_tag = current_tag, backup_tag
backup_images = [backup_tag]
try:
backup_images.append(Image(backup_tag).info['Parent'])
except ImageNotFoundError:
pass
try:
# TODO make separate call for each docker command
fabricio.run(
(
'docker rmi {backup_images}'
'; docker tag {current_tag} {backup_tag}'
'; docker rmi {current_tag}'
).format(
backup_images=' '.join(backup_images),
current_tag=current_tag,
backup_tag=backup_tag,
),
)
except fabricio.host_errors:
pass
def delete(self, force=False, ignore_errors=False):
command = 'docker rm {force}{container}'
force = force and '--force ' or ''
fabricio.run(
command.format(container=self, force=force),
ignore_errors=ignore_errors,
)
def _destroy(
self,
options, # type: utils.Options
):
self.images # get list of images before configuration remove
options.setdefault('filename', os.path.basename(self.config))
fabricio.run('kubectl delete {options}'.format(options=options))
def _create_service(self, image):
command = 'docker service create {options} {image} {cmd}'
fabricio.run(command.format(
options=utils.Options(self.options, name=self, mode=self.mode),
image=image,
cmd=self.cmd,
))
def delete(
self,
delete_image=False,
delete_dangling_volumes=None, # deprecated
**options
):
options = utils.Options(options)
if delete_dangling_volumes is not None: # pragma: no cover
warnings.warn(
'`delete_dangling_volumes` parameter is deprecated '
'and will be removed in v0.6, use `volumes` instead',
RuntimeWarning, stacklevel=2,
)
options.setdefault('volumes', delete_dangling_volumes)
delete_image_callback = delete_image and self.image.get_delete_callback()
options.setdefault('volumes', True) # default option
fabricio.run('docker rm {options} {container}'.format(
container=self,
options=options,
))
if delete_image_callback:
delete_image_callback()
def save_new_settings(self, configuration, image):
self.rotate_sentinel_images()
labels = [(self.configuration_label, b64encode(configuration).decode())]
try:
digests = self._get_digests(self.images)
digests_bucket = json.dumps(digests, sort_keys=True)
digests_bucket = b64encode(digests_bucket.encode()).decode()
labels.append((self.digests_label, digests_bucket))
except fabricio.host_errors:
pass
dockerfile = (
'FROM {image}\n'
'LABEL {labels}\n'
).format(
image=image or 'scratch',
labels=' '.join(itertools.starmap('{0}={1}'.format, labels)),
)
build_command = 'echo {dockerfile} | docker build --tag {tag} -'.format(
dockerfile=shlex_quote(dockerfile),
tag=self.current_settings_tag,
)
try:
fabricio.run(build_command)
except fabricio.host_errors as error:
fabricio.log(
'WARNING: {error}'.format(error=error),
output=sys.stderr,
color=colors.red,
)
def delete(
self,
delete_image=False,
delete_dangling_volumes=None, # deprecated
**options):
options = utils.Options(options)
if delete_dangling_volumes is not None: # pragma: no cover
warnings.warn(
'`delete_dangling_volumes` parameter is deprecated '
'and will be removed in v0.6, use `volumes` instead',
RuntimeWarning,
stacklevel=2,
)
options.setdefault('volumes', delete_dangling_volumes)
delete_image_callback = delete_image and self.image.get_delete_callback(
)
options.setdefault('volumes', True) # default option
fabricio.run('docker rm {options} {container}'.format(
container=self,
options=options,
))
if delete_image_callback:
delete_image_callback()
def save_new_settings(self, settings, image):
self.rotate_sentinel_images()
labels = [(self.compose_label, settings)]
with contextlib.suppress(host_errors):
images_info = self.get_images_info()
if images_info:
labels.append((self.images_info_label, images_info))
dockerfile = ('FROM {image}\n'
'LABEL {labels}\n').format(
image=image or 'scratch',
labels=' '.join(
itertools.starmap('{0}={1}'.format, labels)),
)
build_command = 'echo {dockerfile} | docker build --tag {tag} -'.format(
dockerfile=shlex_quote(dockerfile),
tag=self.current_settings_tag,
)
try:
fabricio.run(build_command)
except host_errors as error:
fabricio.log(
'WARNING: {error}'.format(error=error),
output=sys.stderr,
color=colors.red,
)
def _destroy(
self,
options, # type: utils.Options
):
self.images # get list of images before configuration remove
options.setdefault('filename', os.path.basename(self.config))
fabricio.run('kubectl delete {options}'.format(options=options))
def _revert_images(self, digests):
images = self.__get_images()
for service, image in images.items():
digest = digests[image]
command = 'docker service update --image {digest} {service}'
command = command.format(digest=digest, service=service)
fabricio.run(command)
def rotate_sentinel_images(self, rollback=False):
backup_tag = self.backup_settings_tag
current_tag = self.current_settings_tag
if rollback:
backup_tag, current_tag = current_tag, backup_tag
backup_images = [backup_tag]
try:
backup_images.append(Image(backup_tag).info['Parent'])
except ImageNotFoundError:
pass
try:
# TODO make separate call for each docker command
fabricio.run(
(
'docker rmi {backup_images}'
'; docker tag {current_tag} {backup_tag}'
'; docker rmi {current_tag}'
).format(
backup_images=' '.join(backup_images),
current_tag=current_tag,
backup_tag=backup_tag,
),
)
except fabricio.host_errors:
pass
def save_new_settings(self, configuration, image):
self.rotate_sentinel_images()
labels = [(self.configuration_label, b64encode(configuration).decode())]
try:
digests = self._get_digests(self.images)
digests_bucket = json.dumps(digests, sort_keys=True)
digests_bucket = b64encode(digests_bucket.encode()).decode()
labels.append((self.digests_label, digests_bucket))
except fabricio.host_errors:
pass
dockerfile = (
'FROM {image}\n'
'LABEL {labels}\n'
).format(
image=image or 'scratch',
labels=' '.join(itertools.starmap('{0}={1}'.format, labels)),
)
build_command = 'echo {dockerfile} | docker build --tag {tag} -'.format(
dockerfile=shlex_quote(dockerfile),
tag=self.current_settings_tag,
)
try:
fabricio.run(build_command)
except fabricio.host_errors as error:
fabricio.log(
'WARNING: {error}'.format(error=error),
output=sys.stderr,
color=colors.red,
)
def delete(self, force=False, ignore_errors=False):
command = 'docker rmi {force}{image}'
force = force and '--force ' or ''
fabricio.run(
command.format(image=self.id, force=force),
ignore_errors=ignore_errors,
)
def pull(self, tag=None):
"""
pull[:tag=None] - pull Docker image from registry
"""
fabricio.run('docker pull {image}'.format(
image=self.image[self.registry:tag],
))
def pull(self, tag=None):
"""
pull Docker image from registry
"""
fabricio.run(
'docker pull {image}'.format(image=self.image[self.registry:tag]),
quiet=False,
)
def _revert_images(self, digests):
spec = self.__get_images_spec()
for kind, images in spec.items():
image_updates = ' '.join('{0}={1}'.format(name, digests[image])
for name, image in images.items())
command = 'kubectl set image {kind} {images}'
command = command.format(kind=kind, images=image_updates)
fabricio.run(command)
def _destroy(
self,
options, # type: utils.Options
):
self.images # get list of images before stack remove
fabricio.run('docker stack rm {options} {name}'.format(
options=options,
name=self.name,
))
def _destroy(
self,
options, # type: utils.Options
):
self.images # get list of images before stack remove
fabricio.run('docker stack rm {options} {name}'.format(
options=options,
name=self.name,
))
def remove_sentinel_images(self):
images = [self.current_settings_tag, self.backup_settings_tag]
with contextlib.suppress(ImageNotFoundError):
images.append(Image(self.current_settings_tag).info['Parent'])
images.append(Image(self.backup_settings_tag).info['Parent'])
fabricio.run(
'docker rmi {images}'.format(images=' '.join(images)),
ignore_errors=True,
)
def _revert_images(self, digests):
spec = self.__get_images_spec()
for kind, images in spec.items():
image_updates = ' '.join(
'{0}={1}'.format(name, digests[image])
for name, image in images.items()
)
command = 'kubectl set image {kind} {images}'
command = command.format(kind=kind, images=image_updates)
fabricio.run(command)
def restore(self, src=''):
return # TODO restore arguments should be provided to __init__
self.stop()
try:
command = 'gzip --decompress < {src} | tar --extract --directory {dst}'
fabricio.run(
command.format(src=src, dst=self.data),
sudo=True,
)
finally:
self.start()
def pull(self, tag=None):
"""
pull[:tag=None] - pull Docker image from registry
"""
with fab.remote_tunnel(
remote_port=self.registry.port,
local_port=self.local_registry.port,
local_host=self.local_registry.host,
):
fabricio.run('docker pull {image}'.format(
image=self.image[self.registry:tag]),
)
def _destroy(self, **options):
configuration, _ = self.current_settings
if not configuration:
raise docker.ServiceError('current configuration not found')
with fab.cd(self.temp_dir):
self.upload_configuration(configuration)
options = utils.Options(options)
options.setdefault('filename', self.config)
fabricio.run('kubectl delete {options}'.format(options=options))
def _get_digests(images, pull=False):
images = list(images)
if not images:
return {}
if pull:
for image in images:
command = 'docker pull %s' % image
fabricio.run(command, ignore_errors=True, quiet=False)
command = 'docker inspect --type image --format {{.RepoDigests}} %s'
command %= ' '.join(images)
digests = fabricio.run(command, ignore_errors=True, use_cache=True)
return dict(zip_longest(images, filter(None, digests.splitlines())))
def _remove_images(self):
images = [self.current_settings_tag, self.backup_settings_tag]
try:
images.append(Image(self.current_settings_tag).info['Parent'])
images.append(Image(self.backup_settings_tag).info['Parent'])
except ImageNotFoundError:
pass
images.extend(self.images)
fabricio.run(
'docker rmi {images}'.format(images=' '.join(images)),
ignore_errors=True,
)
def rotate_sentinel_images(self, rollback=False):
backup_tag = self.backup_settings_tag
current_tag = self.current_settings_tag
if rollback:
backup_tag, current_tag = current_tag, backup_tag
with contextlib.suppress(host_errors):
fabricio.run(('docker rmi {backup_tag}'
'; docker tag {current_tag} {backup_tag}'
'; docker rmi {current_tag}').format(
backup_tag=backup_tag,
current_tag=current_tag,
), )
def _remove_images(self):
images = [self.current_settings_tag, self.backup_settings_tag]
try:
images.append(Image(self.current_settings_tag).info['Parent'])
images.append(Image(self.backup_settings_tag).info['Parent'])
except ImageNotFoundError:
pass
images.extend(self.images)
fabricio.run(
'docker rmi {images}'.format(images=' '.join(images)),
ignore_errors=True,
)
def _update(self, new_configuration, force=False):
if not force:
configuration, digests = self.current_settings
if configuration == new_configuration and digests is not None:
new_digests = self._get_digests(digests)
if digests == new_digests:
return False
options = utils.Options(self.options)
command = self.get_update_command(options=options, name=self.name)
fabricio.run(command)
return True
def update_config(content, path):
old_file = six.StringIO()
fab.get(remote_path=path, local_path=old_file, use_sudo=True)
old_content = old_file.getvalue()
fabricio.run(
'mv {path_from} {path_to}'.format(
path_from=path,
path_to=path + '.backup',
),
sudo=True,
)
fab.put(six.StringIO(content), path, use_sudo=True, mode='0644')
return content != old_content
def _update(self, new_configuration, force=False):
if not force:
configuration, digests = self.current_settings
if configuration == new_configuration and digests is not None:
new_digests = self._get_digests(digests)
if digests == new_digests:
return False
options = utils.Options(self.options)
command = self.get_update_command(options=options, name=self.name)
fabricio.run(command)
return True
def info(self):
command = 'docker service inspect {service}'
info = fabricio.run(
command.format(service=self),
abort_exception=ServiceNotFoundError,
)
return json.loads(info)[0]
def init():
if not init.join_command:
fabricio.run(
'docker swarm init --advertise-addr {0}'.format(fab.env.host),
ignore_errors=True,
quiet=False,
)
join_token = fabricio.run(
'docker swarm join-token --quiet manager',
ignore_errors=True,
)
init.join_command = (
'docker swarm join --token {join_token} {host}:2377').format(
join_token=join_token, host=fab.env.host)
else:
fabricio.run(init.join_command, ignore_errors=True, quiet=False)
def info(self):
command = 'docker inspect --type image {image}'
info = fabricio.run(
command.format(image=self),
abort_exception=ImageNotFoundError,
)
return json.loads(info)[0]
def is_manager(self):
try:
if self.pull_errors.get(fab.env.host, False):
return False
is_manager = fabricio.run(
"docker info 2>&1 | grep 'Is Manager:'",
use_cache=True,
).endswith('true')
if is_manager:
self.manager_found.set()
return is_manager
except host_errors as error:
fabricio.log(
'WARNING: {error}'.format(error=error),
output=sys.stderr,
color=colors.red,
)
return False
finally:
with self.is_manager_call_count.get_lock():
self.is_manager_call_count.value += 1
if self.is_manager_call_count.value >= len(fab.env.all_hosts):
if not self.manager_found.is_set():
msg = 'Service manager with pulled image was not found'
raise ServiceError(msg)
self.manager_found.clear()
self.is_manager_call_count.value = 0
def run(
self,
cmd=None,
temporary=True,
quiet=True,
name=None,
options=(),
**kwargs
):
if kwargs:
warnings.warn(
'Container options must be provided in `options` arg, '
'kwargs behavior will be removed in v0.4',
category=RuntimeWarning, stacklevel=2,
)
options = dict(options, **kwargs)
command = 'docker run {options} {image} {cmd}'
return fabricio.run(
command.format(
image=self,
cmd=cmd or '',
options=self.make_container_options(
temporary=temporary,
name=name,
options=options,
),
),
quiet=quiet,
)
def __get_images_spec(self):
template = ( # noqa
'{{define "images"}}'
'{{$kind := .kind}}'
'{{$name := .metadata.name}}'
'{{with .spec.template.spec.containers}}'
'{{range .}}'
r'{{$kind}}/{{$name}} {{.name}} {{.image}}{{"\n"}}'
'{{end}}'
'{{end}}'
'{{end}}'
'{{if eq .kind "List"}}'
'{{range .items}}{{template "images" .}}{{end}}'
'{{else}}'
'{{template "images" .}}'
'{{end}}')
command = ('kubectl get'
' --output go-template'
' --filename {filename}'
' --template {template}'
''.format(
template=shlex_quote(template),
filename=shlex_quote(self.config),
))
result = dict()
for line in filter(None, fabricio.run(command).splitlines()):
kind, image_spec = line.split(None, 1)
name, image = image_spec.rsplit(None, 1)
result.setdefault(kind, dict())[name] = image
return result
def info(self):
command = 'docker service inspect {service}'
info = fabricio.run(
command.format(service=self),
abort_exception=ServiceNotFoundError,
)
return json.loads(info)[0]
def info(self):
command = 'docker inspect --type image {image}'
try:
info = fabricio.run(command.format(image=self))
except RuntimeError:
raise RuntimeError("Image '{image}' not found".format(image=self))
return json.loads(str(info))[0]
def info(self):
command = 'docker inspect --type container {container}'
info = fabricio.run(
command.format(container=self),
abort_exception=ContainerNotFoundError,
)
return json.loads(info)[0]
def _update(self, compose_file, new_settings, force=False):
if not force:
settings, digests = self.current_settings
digests = digests and json.loads(b64decode(digests).decode())
if settings == new_settings and digests is not None:
new_digests = self._get_digests(digests, pull=True)
if digests == new_digests:
return False
with fab.cd(self.temp_dir):
fab.put(six.BytesIO(compose_file), self.actual_compose_file)
fabricio.run('docker stack deploy {options} {name}'.format(
options=utils.Options(self.options),
name=self.name,
))
self.stack_updated.set()
return True
def __get_images_spec(self):
template = ( # noqa
'{{define "images"}}'
'{{$kind := .kind}}'
'{{$name := .metadata.name}}'
'{{with .spec.template.spec.containers}}'
'{{range .}}'
r'{{$kind}}/{{$name}} {{.name}} {{.image}}{{"\n"}}'
'{{end}}'
'{{end}}'
'{{end}}'
'{{if eq .kind "List"}}'
'{{range .items}}{{template "images" .}}{{end}}'
'{{else}}'
'{{template "images" .}}'
'{{end}}')
command = ('kubectl get {options}'.format(options=utils.Options([
('output', 'go-template'),
('filename', os.path.basename(self.config)),
('template', template),
])))
result = dict()
for line in filter(None, fabricio.run(command).splitlines()):
kind, image_spec = line.split(None, 1)
name, image = image_spec.rsplit(None, 1)
result.setdefault(kind, dict())[name] = image
return result
def info(self):
command = 'docker inspect --type image {image}'
info = fabricio.run(
command.format(image=self),
abort_exception=ImageNotFoundError,
)
return json.loads(info)[0]
def info(self):
command = 'docker inspect --type container {container}'
info = fabricio.run(
command.format(container=self),
abort_exception=ContainerNotFoundError,
)
return json.loads(info)[0]
def execute(self, cmd, ignore_errors=False, quiet=True):
command = 'docker exec --tty --interactive {container} {cmd}'
return fabricio.run(
command.format(container=self, cmd=cmd),
ignore_errors=ignore_errors,
quiet=quiet,
)
def init():
if not init.join_command:
fabricio.run(
'docker swarm init --advertise-addr {0}'.format(fab.env.host),
ignore_errors=True,
quiet=False,
)
join_token = fabricio.run(
'docker swarm join-token --quiet manager',
ignore_errors=True,
)
init.join_command = (
'docker swarm join --token {join_token} {host}:2377'
).format(join_token=join_token, host=fab.env.host)
else:
fabricio.run(init.join_command, ignore_errors=True, quiet=False)
def __get_images_spec(self):
template = ( # noqa
'{{define "images"}}'
'{{$kind := .kind}}'
'{{$name := .metadata.name}}'
'{{with .spec.template.spec.containers}}'
'{{range .}}'
r'{{$kind}}/{{$name}} {{.name}} {{.image}}{{"\n"}}'
'{{end}}'
'{{end}}'
'{{end}}'
'{{if eq .kind "List"}}'
'{{range .items}}{{template "images" .}}{{end}}'
'{{else}}'
'{{template "images" .}}'
'{{end}}'
)
command = (
'kubectl get {options}'.format(options=utils.Options([
('output', 'go-template'),
('filename', os.path.basename(self.config)),
('template', template),
]))
)
result = dict()
for line in filter(None, fabricio.run(command).splitlines()):
kind, image_spec = line.split(None, 1)
name, image = image_spec.rsplit(None, 1)
result.setdefault(kind, dict())[name] = image
return result
def info(self):
command = 'docker inspect --type container {container}'
try:
info = fabricio.run(command.format(container=self))
except RuntimeError:
raise RuntimeError("Container '{container}' not found".format(
container=self, ))
return json.loads(info)[0]
def backup(self, dst='', username='******'):
return # TODO backup arguments should be provided to __init__
self.execute('psql --username {username} --command "{command};"'.format(
username=username,
command="SELECT pg_start_backup('backup')",
))
try:
command = 'tar --create --exclude postmaster.pid {src} | gzip > {dst}'
fabricio.run(
command.format(src=self.data, dst=dst),
sudo=True,
)
finally:
self.execute('psql --username {username} --command "{command};"'.format(
username=username,
command="SELECT pg_stop_backup()",
))
def execute(self, cmd, ignore_errors=False, quiet=True, use_cache=False):
command = 'docker exec --tty --interactive {container} {cmd}'
return fabricio.run(
command.format(container=self, cmd=cmd),
ignore_errors=ignore_errors,
quiet=quiet,
use_cache=use_cache,
)
def create(self, command=None, name=None, options=()):
run_command = 'docker create {options} {image}{command}'
return fabricio.run(
run_command.format(
image=self,
command=command and ' {0}'.format(command) or '',
options=self.make_container_options(name=name,
options=options),
), )
def _swarm_init():
if swarm_init.worker_join_command is None:
fabricio.run(
'docker swarm init --advertise-addr {0}'.format(fab.env.host),
ignore_errors=True,
)
join_token = fabricio.run(
'docker swarm join-token --quiet manager',
ignore_errors=True,
)
swarm_init.worker_join_command = (
'docker swarm join --token {join_token} {host}:2377'
).format(join_token=join_token, host=fab.env.host)
else:
fabricio.run(
swarm_init.worker_join_command,
ignore_errors=True,
)
def _swarm_init():
if swarm_init.worker_join_command is None:
fabricio.run(
'docker swarm init --advertise-addr {0}'.format(fab.env.host),
ignore_errors=True,
)
join_token = fabricio.run(
'docker swarm join-token --quiet manager',
ignore_errors=True,
)
swarm_init.worker_join_command = (
'docker swarm join --token {join_token} {host}:2377').format(
join_token=join_token, host=fab.env.host)
else:
fabricio.run(
swarm_init.worker_join_command,
ignore_errors=True,
)
def revert(self):
main_conf = os.path.join(self.data, 'postgresql.conf')
hba_conf = os.path.join(self.data, 'pg_hba.conf')
fabricio.run(
'mv {path_from} {path_to}'.format(
path_from=main_conf + '.backup',
path_to=main_conf,
),
ignore_errors=True,
sudo=True,
)
fabricio.run(
'mv {path_from} {path_to}'.format(
path_from=hba_conf + '.backup',
path_to=hba_conf,
),
ignore_errors=True,
sudo=True,
)
super(PostgresqlContainer, self).revert()
def _get_digests(images):
if not images:
return {}
for image in images:
Image(image).pull(use_cache=True, ignore_errors=True)
command = (
'docker inspect --type image --format "{{index .RepoDigests 0}}" %s'
) % ' '.join(images)
digests = fabricio.run(command, ignore_errors=True, use_cache=True)
return dict(zip_longest(images, filter(None, digests.splitlines())))
def create(self, command=None, name=None, options=()): # pragma: no cover
warnings.warn('Image.create() is deprecated', DeprecationWarning)
warnings.warn(
'Image.create() is deprecated',
RuntimeWarning, stacklevel=2,
)
run_command = 'docker create {options} {image} {command}'.rstrip()
return fabricio.run(
run_command.format(
image=self,
command=command or '',
options=self.make_container_options(name=name, options=options),
),
)
def run(
self,
command=None,
name=None,
temporary=True,
options=(),
quiet=True,
):
run_command = 'docker run {options} {image} {command}'
return fabricio.run(
run_command.format(
image=self,
command=command or '',
options=self.make_container_options(
temporary=temporary,
name=name,
options=options,
),
),
quiet=quiet,
)
def run(
self,
cmd=None,
temporary=True,
quiet=True,
name=None,
user=None,
ports=None,
env=None,
volumes=None,
links=None,
hosts=None,
network=None,
restart_policy=None,
stop_signal=None,
options=None,
):
command = 'docker run {options} {image} {cmd}'
return fabricio.run(
command.format(
image=self,
cmd=cmd or '',
options=self.make_container_options(
temporary=temporary,
name=name,
user=user,
ports=ports,
env=env,
volumes=volumes,
links=links,
hosts=hosts,
network=network,
restart_policy=restart_policy,
stop_signal=stop_signal,
options=options,
),
),
quiet=quiet,
)
def execute(
self,
command=None,
quiet=True,
use_cache=False,
options=(),
):
if not command:
raise ValueError('Must provide command to execute')
options = utils.Options(options)
options.setdefault('tty', True)
options.setdefault('interactive', True)
exec_command = 'docker exec {options} {container} {command}'
return fabricio.run(
exec_command.format(
container=self,
command=command,
options=options,
),
quiet=quiet,
use_cache=use_cache,
)
