def test_create_kwargs_with_host_config(self):
cfg_name = 'app_server'
cfg = self.sample_map.get_existing(cfg_name)
c_name = 'main.app_server'
hc_kwargs = dict(binds={'/new_h': {'bind': '/new_c', 'ro': False}})
kwargs = BasePolicy.get_create_kwargs(self.sample_map, cfg_name, cfg, '__default__',
self.sample_client_config, c_name, 'instance1',
include_host_config=True, kwargs=dict(host_config=hc_kwargs))
self.assertDictEqual(kwargs, dict(
name=c_name,
image='registry.example.com/app',
environment=[],
volumes=[
'/var/lib/app/config',
'/var/lib/app/data'
],
user='******',
hostname='main.app_server',
domainname=None,
ports=[8880],
host_config=create_host_config(
links={},
binds={
'/var/lib/site/config/app1': {'bind': '/var/lib/app/config', 'ro': True},
'/var/lib/site/data/app1': {'bind': '/var/lib/app/data', 'ro': False},
'/new_h': {'bind': '/new_c', 'ro': False},
},
volumes_from=['main.app_log', 'main.app_server_socket'],
port_bindings={},
),
))
def test_attached_preparation_create_kwargs(self):
cfg_name = 'app_server'
cfg = self.sample_map.get_existing(cfg_name)
c_name = 'temp'
alias = 'app_server_socket'
v_name = 'main.app_server_socket'
kwargs = BasePolicy.get_attached_preparation_create_kwargs(
self.sample_map,
cfg_name,
cfg,
'__default__',
self.sample_client_config,
c_name,
alias,
v_name,
include_host_config=True)
self.assertDictEqual(
kwargs,
dict(
image=BasePolicy.core_image,
command=
'chown -R 2000:2000 /var/lib/app/socket && chmod -R u=rwX,g=rX,o= /var/lib/app/socket',
user='******',
host_config=create_host_config(
volumes_from=[v_name],
version=self.client_version,
),
network_disabled=True,
))
def test_create_kwargs_with_host_config(self):
cfg_name = "app_server"
cfg = self.sample_map.get_existing(cfg_name)
c_name = "main.app_server"
self.sample_client_config.use_host_config = True
config = ActionConfig(
"main", self.sample_map, cfg_name, cfg, "__default__", self.sample_client_config, None, "instance1"
)
hc_kwargs = dict(binds=["/new_h:/new_c:rw"])
kwargs = self.runner.get_create_kwargs(config, c_name, kwargs=dict(host_config=hc_kwargs))
self.assertDictEqual(
kwargs,
dict(
name=c_name,
image="registry.example.com/app:custom",
volumes=["/var/lib/app/config", "/var/lib/app/data"],
user="******",
hostname="main.app_server",
domainname=None,
ports=[8880],
host_config=create_host_config(
links={},
binds=[
"/var/lib/site/config/app1:/var/lib/app/config:ro",
"/var/lib/site/data/app1:/var/lib/app/data:rw",
"/new_h:/new_c:rw",
],
volumes_from=["main.app_log", "main.app_server_socket"],
port_bindings={},
version=self.client_version,
),
),
)
def get_attached_preparation_create_kwargs(cls, container_map, config_name, container_config, client_name,
client_config, container_name, alias, volume_container,
include_host_config=True, kwargs=None):
"""
Generates keyword arguments for the Docker client to prepare an attached container (i.e. adjust user and
permissions).
:param container_map: Container map object.
:type container_map: dockermap.map.container.ContainerMap
:param config_name: Container configuration name.
:type config_name: unicode
:param container_config: Container configuration object.
:type container_config: dockermap.map.config.ContainerConfiguration
:param client_name: Client configuration name.
:type client_name: unicode
:param client_config: Client configuration object.
:type client_config: dockermap.map.config.ClientConfiguration
:param container_name: Container name.
:type container_name: unicode
:param alias: Alias name of the container volume.
:type alias: unicode
:param volume_container: Name of the container that shares the volume.
:type volume_container: unicode
:param include_host_config: Whether to generate and include the HostConfig.
:type include_host_config: Set to ``False``, if calling :meth:`get_start_kwargs:` later.
:param kwargs: Additional keyword arguments to complement or override the configuration-based values.
:type kwargs: dict | NoneType
:return: Resulting keyword arguments.
:rtype: dict
"""
def _get_cmd():
user = resolve_value(container_config.user)
if user:
yield 'chown -R {0} {1}'.format(get_user_group(user), str_arg(path))
permissions = container_config.permissions
if permissions:
yield 'chmod -R {0} {1}'.format(permissions, str_arg(path))
path = resolve_value(container_map.volumes[alias])
c_kwargs = dict(
image=cls.core_image,
command=' && '.join(_get_cmd()),
user='******',
network_disabled=True,
)
hc_extra_kwargs = kwargs.pop('host_config', None) if kwargs else None
if include_host_config:
hc_kwargs = cls.get_attached_preparation_host_config_kwargs(container_map, config_name, container_config,
client_name, client_config, None, alias,
volume_container, kwargs=hc_extra_kwargs)
if hc_kwargs:
c_kwargs['host_config'] = create_host_config(**hc_kwargs)
update_kwargs(c_kwargs, kwargs)
return c_kwargs
def get_create_kwargs(cls, container_map, config_name, container_config, client_name, client_config, container_name,
instance, include_host_config=True, kwargs=None):
"""
Generates keyword arguments for the Docker client to create a container.
:param config_name:
:param container_map: Container map object.
:type container_map: dockermap.map.container.ContainerMap
:param config_name: Container configuration name.
:type config_name: unicode
:param container_config: Container configuration object.
:type container_config: dockermap.map.config.ContainerConfiguration
:param client_name: Client configuration name.
:type client_name: unicode
:param client_config: Client configuration object.
:type client_config: dockermap.map.config.ClientConfiguration
:param container_name: Container name.
:type container_name: unicode
:param instance: Instance name.
:type instance: unicode | NoneType
:param include_host_config: Whether to generate and include the HostConfig.
:type include_host_config: Set to ``False``, if calling :meth:`get_start_kwargs:` later.
:param kwargs: Additional keyword arguments to complement or override the configuration-based values.
:type kwargs: dict | NoneType
:return: Resulting keyword arguments.
:rtype: dict
"""
c_kwargs = dict(
name=container_name,
image=cls.iname(container_map, container_config.image or config_name),
volumes=get_volumes(container_map, container_config),
user=extract_user(container_config.user),
ports=[resolve_value(port_binding.exposed_port)
for port_binding in container_config.exposes if port_binding.exposed_port],
hostname=cls.get_hostname(client_name, container_name) if container_map.set_hostname else None,
domainname=cls.get_domainname(container_map, container_config, client_config),
environment=get_environment(container_map, container_config)
)
if container_config.network == 'disabled':
c_kwargs['network_disabled'] = True
hc_extra_kwargs = kwargs.pop('host_config', None) if kwargs else None
if include_host_config:
hc_kwargs = cls.get_host_config_kwargs(container_map, config_name, container_config, client_name,
client_config, None, instance, kwargs=hc_extra_kwargs)
if hc_kwargs:
c_kwargs['host_config'] = create_host_config(**hc_kwargs)
update_kwargs(c_kwargs, init_options(container_config.create_options), kwargs)
return c_kwargs
def test_create_kwargs_with_host_config(self):
cfg_name = 'app_server'
cfg = self.sample_map.get_existing(cfg_name)
c_name = 'main.app_server'
hc_kwargs = dict(binds={'/new_h': {'bind': '/new_c', 'ro': False}})
kwargs = BasePolicy.get_create_kwargs(
self.sample_map,
cfg_name,
cfg,
'__default__',
self.sample_client_config,
c_name,
'instance1',
include_host_config=True,
kwargs=dict(host_config=hc_kwargs))
self.assertDictEqual(
kwargs,
dict(
name=c_name,
image='registry.example.com/app:custom',
volumes=['/var/lib/app/config', '/var/lib/app/data'],
user='******',
hostname='main.app_server',
domainname=None,
ports=[8880],
host_config=create_host_config(
links={},
binds={
'/var/lib/site/config/app1': {
'bind': '/var/lib/app/config',
'ro': True
},
'/var/lib/site/data/app1': {
'bind': '/var/lib/app/data',
'ro': False
},
'/new_h': {
'bind': '/new_c',
'ro': False
},
},
volumes_from=['main.app_log', 'main.app_server_socket'],
port_bindings={},
version=self.client_version,
),
))
def test_attached_preparation_create_kwargs(self):
cfg_name = 'app_server'
cfg = self.sample_map.get_existing(cfg_name)
c_name = 'temp'
alias = 'app_server_socket'
v_name = 'main.app_server_socket'
kwargs = BasePolicy.get_attached_preparation_create_kwargs(self.sample_map, cfg_name, cfg, '__default__',
self.sample_client_config, c_name, alias,
v_name, include_host_config=True)
self.assertDictEqual(kwargs, dict(
image=BasePolicy.core_image,
command='chown -R 2000:2000 /var/lib/app/socket && chmod -R u=rwX,g=rX,o= /var/lib/app/socket',
user='******',
host_config=create_host_config(
volumes_from=[v_name],
),
network_disabled=True,
))
def _host_config(self, service_name):
"""Returns a host configuration object for use with docker commands.
This is mostly the network mode and the binding to the folders on the file system.
"""
res = self.ctx.fs.info_deploy(service_name)
binds = {
res['log']: {
'bind': '/data/log',
'ro': False,
},
res['config']: {
'bind': '/data/config',
'ro': True
}
}
return create_host_config(binds=binds, network_mode='host')
def get_attached_create_kwargs(cls, container_map, config_name, container_config, client_name, client_config,
container_name, alias, include_host_config=True, kwargs=None):
"""
Generates keyword arguments for the Docker client to create an attached container.
:param container_map: Container map object.
:type container_map: dockermap.map.container.ContainerMap
:param config_name: Container configuration name.
:type config_name: unicode
:param container_config: Container configuration object.
:type container_config: dockermap.map.config.ContainerConfiguration
:param client_name: Client configuration name.
:type client_name: unicode
:param client_config: Client configuration object.
:type client_config: dockermap.map.config.ClientConfiguration
:param container_name: Container name.
:type container_name: unicode
:param alias: Alias name of the container volume.
:type alias: unicode
:param include_host_config: Whether to generate and include the HostConfig.
:type include_host_config: Set to ``False``, if calling :meth:`get_start_kwargs:` later.
:param kwargs: Additional keyword arguments to complement or override the configuration-based values.
:type kwargs: dict | NoneType
:return: Resulting keyword arguments.
:rtype: dict
"""
path = resolve_value(container_map.volumes[alias])
user = extract_user(container_config.user)
c_kwargs = dict(
name=container_name,
image=cls.base_image,
volumes=[path],
user=user,
network_disabled=True,
)
hc_extra_kwargs = kwargs.pop('host_config', None) if kwargs else None
if include_host_config:
hc_kwargs = cls.get_attached_host_config_kwargs(container_map, config_name, container_config, client_name,
client_config, None, alias, kwargs=hc_extra_kwargs)
if hc_kwargs:
c_kwargs['host_config'] = create_host_config(**hc_kwargs)
update_kwargs(c_kwargs, kwargs)
return c_kwargs
def _host_config(self, service_name):
"""Returns a host configuration object for use with docker commands.
This is mostly the network mode and the binding to the folders on the file system.
"""
res = self.ctx.fs.info_deploy(service_name)
binds = {
res['log']: {
'bind': '/data/log',
'ro': False,
},
res['config']: {
'bind': '/data/config',
'ro': True
}
}
return create_host_config(binds=binds, network_mode='host')
def test_attached_preparation_create_kwargs(self):
cfg_name = "app_server"
cfg = self.sample_map.get_existing(cfg_name)
alias = "app_server_socket"
v_name = "main.app_server_socket"
self.sample_client_config.use_host_config = True
config = ActionConfig(
"main", self.sample_map, cfg_name, cfg, "__default__", self.sample_client_config, None, alias
)
kwargs = self.runner.get_attached_preparation_create_kwargs(config, v_name)
self.assertDictEqual(
kwargs,
dict(
image=BasePolicy.core_image,
command="chown -R 2000:2000 /var/lib/app/socket && chmod -R u=rwX,g=rX,o= /var/lib/app/socket",
user="******",
host_config=create_host_config(volumes_from=[v_name], version=self.client_version),
network_disabled=True,
),
)
def create_container(self, conf, detach, tty):
"""Create a single container"""
name = conf.name
image_name = conf.image_name
container_name = conf.container_name
env = dict(e.pair for e in conf.env)
ports = [port.host_port for port in conf.ports]
links = [link.pair for link in conf.links]
binds = conf.volumes.binds
command = conf.formatted_command
volume_names = conf.volumes.volume_names
volumes_from = list(conf.volumes.share_with_names)
port_bindings = dict([port.pair for port in conf.ports])
uncreated = []
for name in binds:
if not os.path.exists(name):
log.info("Making volume for mounting\tvolume=%s", name)
try:
os.makedirs(name)
except OSError as error:
uncreated.append((name, error))
if uncreated:
raise BadOption("Failed to create some volumes on the host",
uncreated=uncreated)
log.info(
"Creating container from %s\timage=%s\tcontainer_name=%s\ttty=%s",
image_name, name, container_name, tty)
if binds:
log.info("\tUsing volumes\tvolumes=%s", volume_names)
if env:
log.info("\tUsing environment\tenv=%s", sorted(env.keys()))
if links:
log.info("\tLinks: %s", links)
if ports:
log.info("\tUsing ports\tports=%s", ports)
if port_bindings:
log.info("\tPort bindings: %s", ports)
if volumes_from:
log.info("\tVolumes from: %s", volumes_from)
host_config = utils.create_host_config(
links=links,
binds=binds,
volumes_from=volumes_from,
port_bindings=port_bindings,
devices=conf.devices,
lxc_conf=conf.lxc_conf,
privileged=conf.privileged,
restart_policy=conf.restart_policy,
dns=conf.network.dns,
dns_search=conf.network.dns_search,
extra_hosts=conf.network.extra_hosts,
network_mode=conf.network.network_mode,
publish_all_ports=conf.network.publish_all_ports,
cap_add=conf.cpu.cap_add,
cap_drop=conf.cpu.cap_drop,
mem_limit=conf.cpu.mem_limit,
memswap_limit=conf.cpu.memswap_limit,
ulimits=conf.ulimits,
read_only=conf.read_only_rootfs,
log_config=conf.log_config,
security_opt=conf.security_opt,
**conf.other_options.host_config)
container_id = conf.harpoon.docker_context.create_container(
image_name,
name=container_name,
detach=detach,
command=command,
volumes=volume_names,
environment=env,
tty=tty,
user=conf.user,
ports=ports,
stdin_open=tty,
dns=conf.network.dns,
hostname=conf.network.hostname,
domainname=conf.network.domainname,
network_disabled=conf.network.disabled,
cpuset=conf.cpu.cpuset,
cpu_shares=conf.cpu.cpu_shares,
host_config=host_config,
**conf.other_options.create)
if isinstance(container_id, dict):
if "errorDetail" in container_id:
raise BadImage("Failed to create container",
image=name,
error=container_id["errorDetail"])
container_id = container_id["Id"]
return container_id
def create_container(self, conf, detach, tty):
"""Create a single container"""
name = conf.name
image_name = conf.image_name
container_name = conf.container_name
env = dict(e.pair for e in conf.env)
ports = [port.host_port for port in conf.ports]
links = [link.pair for link in conf.links]
binds = conf.volumes.binds
command = conf.formatted_command
volume_names = conf.volumes.volume_names
volumes_from = list(conf.volumes.share_with_names)
port_bindings = dict([port.pair for port in conf.ports])
uncreated = []
for name in binds:
if not os.path.exists(name):
log.info("Making volume for mounting\tvolume=%s", name)
try:
os.makedirs(name)
except OSError as error:
uncreated.append((name, error))
if uncreated:
raise BadOption("Failed to create some volumes on the host", uncreated=uncreated)
log.info("Creating container from %s\timage=%s\tcontainer_name=%s\ttty=%s", image_name, name, container_name, tty)
if binds:
log.info("\tUsing volumes\tvolumes=%s", volume_names)
if env:
log.info("\tUsing environment\tenv=%s", sorted(env.keys()))
if links:
log.info("\tLinks: %s", links)
if ports:
log.info("\tUsing ports\tports=%s", ports)
if port_bindings:
log.info("\tPort bindings: %s", ports)
if volumes_from:
log.info("\tVolumes from: %s", volumes_from)
host_config = utils.create_host_config(
links = links
, binds = binds
, volumes_from = volumes_from
, port_bindings = port_bindings
, devices = conf.devices
, lxc_conf = conf.lxc_conf
, privileged = conf.privileged
, restart_policy = conf.restart_policy
, dns = conf.network.dns
, dns_search = conf.network.dns_search
, extra_hosts = conf.network.extra_hosts
, network_mode = conf.network.network_mode
, publish_all_ports = conf.network.publish_all_ports
, cap_add = conf.cpu.cap_add
, cap_drop = conf.cpu.cap_drop
, mem_limit = conf.cpu.mem_limit
, memswap_limit = conf.cpu.memswap_limit
, ulimits = conf.ulimits
, read_only = conf.read_only_rootfs
, log_config = conf.log_config
, security_opt = conf.security_opt
, **conf.other_options.host_config
)
container_id = conf.harpoon.docker_context.create_container(image_name
, name=container_name
, detach=detach
, command=command
, volumes=volume_names
, environment=env
, tty = tty
, user = conf.user
, ports = ports
, stdin_open = tty
, dns = conf.network.dns
, hostname = conf.network.hostname
, domainname = conf.network.domainname
, network_disabled = conf.network.disabled
, cpuset = conf.cpu.cpuset
, cpu_shares = conf.cpu.cpu_shares
, host_config = host_config
, **conf.other_options.create
)
if isinstance(container_id, dict):
if "errorDetail" in container_id:
raise BadImage("Failed to create container", image=name, error=container_id["errorDetail"])
container_id = container_id["Id"]
return container_id