def test_create_host_config_with_mem_reservation(self):
config = create_host_config(version='1.21', mem_reservation=67108864)
self.assertEqual(config.get('MemoryReservation'), 67108864)
self.assertRaises(
InvalidVersion,
lambda: create_host_config(version='1.20',
mem_reservation=67108864))
def test_create_host_config_pid_mode(self):
with pytest.raises(ValueError):
create_host_config(version='1.23', pid_mode='baccab125')
config = create_host_config(version='1.23', pid_mode='host')
assert config.get('PidMode') == 'host'
config = create_host_config(version='1.24', pid_mode='baccab125')
assert config.get('PidMode') == 'baccab125'
def test_create_host_config_with_pids_limit(self):
config = create_host_config(version='1.23', pids_limit=1024)
self.assertEqual(config.get('PidsLimit'), 1024)
with pytest.raises(InvalidVersion):
create_host_config(version='1.22', pids_limit=1024)
with pytest.raises(TypeError):
create_host_config(version='1.23', pids_limit='1024')
def test_create_host_config_with_pids_limit(self):
config = create_host_config(version='1.23', pids_limit=1024)
self.assertEqual(config.get('PidsLimit'), 1024)
with pytest.raises(InvalidVersion):
create_host_config(version='1.22', pids_limit=1024)
with pytest.raises(TypeError):
create_host_config(version='1.23', pids_limit='1024')
def test_create_host_config_with_isolation(self):
config = create_host_config(version='1.24', isolation='hyperv')
self.assertEqual(config.get('Isolation'), 'hyperv')
with pytest.raises(InvalidVersion):
create_host_config(version='1.23', isolation='hyperv')
with pytest.raises(TypeError):
create_host_config(version='1.24',
isolation={'isolation': 'hyperv'})
def test_create_host_config_with_oom_score_adj(self):
config = create_host_config(version='1.22', oom_score_adj=100)
self.assertEqual(config.get('OomScoreAdj'), 100)
self.assertRaises(
InvalidVersion,
lambda: create_host_config(version='1.21', oom_score_adj=100))
self.assertRaises(
TypeError,
lambda: create_host_config(version='1.22', oom_score_adj='100'))
def test_create_host_config_with_userns_mode(self):
config = create_host_config(version='1.23', userns_mode='host')
self.assertEqual(config.get('UsernsMode'), 'host')
self.assertRaises(
InvalidVersion, lambda: create_host_config(version='1.22',
userns_mode='host'))
self.assertRaises(
ValueError, lambda: create_host_config(version='1.23',
userns_mode='host12'))
def test_create_host_config_with_userns_mode(self):
config = create_host_config(version='1.23', userns_mode='host')
self.assertEqual(config.get('UsernsMode'), 'host')
self.assertRaises(
InvalidVersion,
lambda: create_host_config(version='1.22', userns_mode='host'))
self.assertRaises(
ValueError,
lambda: create_host_config(version='1.23', userns_mode='host12'))
def test_create_host_config_with_oom_score_adj(self):
config = create_host_config(version='1.22', oom_score_adj=100)
self.assertEqual(config.get('OomScoreAdj'), 100)
self.assertRaises(
InvalidVersion, lambda: create_host_config(version='1.21',
oom_score_adj=100))
self.assertRaises(
TypeError, lambda: create_host_config(version='1.22',
oom_score_adj='100'))
def test_create_host_config_with_isolation(self):
config = create_host_config(version='1.24', isolation='hyperv')
self.assertEqual(config.get('Isolation'), 'hyperv')
with pytest.raises(InvalidVersion):
create_host_config(version='1.23', isolation='hyperv')
with pytest.raises(TypeError):
create_host_config(
version='1.24', isolation={'isolation': 'hyperv'}
)
def test_create_host_config_with_dns_opt(self):
tested_opts = ['use-vc', 'no-tld-query']
config = create_host_config(version='1.21', dns_opt=tested_opts)
dns_opts = config.get('DnsOptions')
self.assertTrue('use-vc' in dns_opts)
self.assertTrue('no-tld-query' in dns_opts)
self.assertRaises(
InvalidVersion, lambda: create_host_config(version='1.20',
dns_opt=tested_opts))
def test_create_host_config_with_dns_opt(self):
tested_opts = ['use-vc', 'no-tld-query']
config = create_host_config(version='1.21', dns_opt=tested_opts)
dns_opts = config.get('DnsOptions')
self.assertTrue('use-vc' in dns_opts)
self.assertTrue('no-tld-query' in dns_opts)
self.assertRaises(
InvalidVersion,
lambda: create_host_config(version='1.20', dns_opt=tested_opts))
def build_container(self):
if self.port_mappings:
self._container = self.docker_client.create_container(
image=self.full_image_name,
environment=self.environment,
ports=list(self.port_mappings.keys()),
host_config=create_host_config(
port_bindings=self.port_mappings,
network_mode=self.network_mode))
else:
self._container = self.docker_client.create_container(
image=self.full_image_name,
environment=self.environment,
host_config=create_host_config(network_mode=self.network_mode))
def _make_host_config(self):
return create_host_config(
binds=self.volume_binds,
port_bindings=self.port_bindings,
network_mode=self.network_mode,
extra_hosts=self.extra_hosts,
# TODO: Support all of these.
lxc_conf=None,
publish_all_ports=False,
links=None,
privileged=False,
dns=None,
dns_search=None,
volumes_from=None,
restart_policy=None,
cap_add=None,
cap_drop=None,
devices=None,
read_only=None,
pid_mode=None,
ipc_mode=None,
security_opt=None,
ulimits=None,
log_config=None,
)
def web_command(command,
ro=None,
rw=None,
links=None,
image='datacats/web',
volumes_from=None,
commit=False,
clean_up=False,
stream_output=None,
entrypoint=None):
"""
Run a single command in a web image optionally preloaded with the ckan
source and virtual envrionment.
:param command: command to execute
:param ro: {localdir: binddir} dict for read-only volumes
:param rw: {localdir: binddir} dict for read-write volumes
:param links: links passed to start
:param image: docker image name to use
:param volumes_from:
:param commit: True to create a new image based on result
:param clean_up: True to remove container even on error
:param stream_output: file to write stderr+stdout from command
:param entrypoint: override entrypoint (script that runs command)
:returns: image id if commit=True
"""
binds = ro_rw_to_binds(ro, rw)
c = _get_docker().create_container(
image=image,
command=command,
volumes=binds_to_volumes(binds),
detach=False,
host_config=create_host_config(binds=binds),
entrypoint=entrypoint)
_get_docker().start(container=c['Id'],
links=links,
binds=binds,
volumes_from=volumes_from)
if stream_output:
for output in _get_docker().attach(c['Id'],
stdout=True,
stderr=True,
stream=True):
stream_output.write(output)
if _get_docker().wait(c['Id']):
# Before the (potential) cleanup, grab the logs!
logs = _get_docker().logs(c['Id'])
if clean_up:
remove_container(c['Id'])
raise WebCommandError(command, c['Id'][:12], logs)
if commit:
rval = _get_docker().commit(c['Id'])
if not remove_container(c['Id']):
# circle ci doesn't let us remove containers, quiet the warnings
if not environ.get('CIRCLECI', False):
warn('failed to remove container: {0}'.format(c['Id']))
if commit:
return rval['Id']
def get_attached_create_kwargs(self, config, container_name, kwargs=None):
"""
Generates keyword arguments for the Docker client to create an attached container.
:param config: Configuration.
:type config: ActionConfig
:param container_name: Container name.
:type container_name: unicode | str
:param kwargs: Additional keyword arguments to complement or override the configuration-based values.
:type kwargs: dict | NoneType
:return: Resulting keyword arguments.
:rtype: dict
"""
client_config = config.client_config
path = resolve_value(config.container_map.volumes[config.instance_name])
user = extract_user(config.container_config.user)
c_kwargs = dict(
name=container_name,
image=self._policy.base_image,
volumes=[path],
user=user,
network_disabled=True,
)
hc_extra_kwargs = kwargs.pop('host_config', None) if kwargs else None
use_host_config = client_config.get('use_host_config')
if use_host_config:
hc_kwargs = self.get_attached_host_config_kwargs(config, None, kwargs=hc_extra_kwargs)
if hc_kwargs:
if use_host_config == USE_HC_MERGE:
c_kwargs.update(hc_kwargs)
else:
c_kwargs['host_config'] = create_host_config(version=client_config.version, **hc_kwargs)
update_kwargs(c_kwargs, kwargs)
return c_kwargs
def get_container_host_config(self):
"""
Create the ``host_config`` settings for the container.
It mainly generates the proper path bindings between the Docker
container and the Host by mounting them with the proper permissions.
Besides, it mounts the ``GLOBAL_PIP_CACHE`` if it's set and we are under
``DEBUG``.
The object returned is passed to Docker function
``client.create_container``.
"""
binds = {
SPHINX_TEMPLATE_DIR: {
'bind': SPHINX_TEMPLATE_DIR,
'mode': 'ro',
},
MKDOCS_TEMPLATE_DIR: {
'bind': MKDOCS_TEMPLATE_DIR,
'mode': 'ro',
},
self.project.doc_path: {
'bind': self.project.doc_path,
'mode': 'rw',
},
}
if getattr(settings, 'GLOBAL_PIP_CACHE', False) and settings.DEBUG:
binds.update({
self.project.pip_cache_path: {
'bind': self.project.pip_cache_path,
'mode': 'rw',
}
})
return create_host_config(binds=binds)
def create_container(self):
'''Create docker container'''
client = self.get_client()
try:
self.container = client.create_container(
image=self.container_image,
command=('/bin/sh -c "sleep {time}; exit {exit}"'
.format(time=self.container_time_limit,
exit=DOCKER_TIMEOUT_EXIT_CODE)),
name=self.container_id,
hostname=self.container_id,
host_config=create_host_config(binds={
self.project.doc_path: {
'bind': self.project.doc_path,
'mode': 'rw'
}
}),
detach=True,
mem_limit=self.container_mem_limit,
)
client.start(container=self.container_id)
except DockerAPIError as e:
log.error(LOG_TEMPLATE
.format(
project=self.project.slug,
version=self.version.slug,
msg=e.explanation),
exc_info=True)
raise BuildEnvironmentError('Build environment creation failed')
def _create():
binds = {
volume.node_path.path: {
'bind': volume.container_path.path,
'ro': False,
}
for volume in volumes
}
port_bindings = {p.internal_port: p.external_port for p in ports}
host_config = create_host_config(
binds=binds,
port_bindings=port_bindings,
restart_policy=restart_policy_dict,
)
# We're likely to get e.g. pvector, so make sure we're passing
# in something JSON serializable:
command_line_values = command_line
if command_line_values is not None:
command_line_values = list(command_line_values)
self._client.create_container(
name=container_name,
image=image_name,
command=command_line_values,
environment=environment,
ports=[p.internal_port for p in ports],
mem_limit=mem_limit,
cpu_shares=cpu_shares,
host_config=host_config,
)
def test_native_fields(socat_containers, client, pull_images):
docker_client = get_docker_client(host(client))
name = 'native-%s' % random_str()
host_config = create_host_config(privileged=True,
publish_all_ports=True,
dns=['1.2.3.4'],
dns_search=['search.dns.com'],
cap_add=['SYSLOG'],
cap_drop=['KILL', 'LEASE'],
restart_policy={
'MaximumRetryCount': 5,
'Name': 'on-failure'
},
devices=['/dev/null:/dev/xnull:rw'])
docker_container = docker_client.create_container(
NATIVE_TEST_IMAGE,
name=name,
hostname='hostname1',
domainname='domainname1',
user='******',
mem_limit='16MB',
memswap_limit='32MB',
cpu_shares=1024,
cpuset='0',
tty=True,
stdin_open=True,
working_dir='/root',
environment={'FOO': 'BA'},
command=['-c', 'sleep 3'],
entrypoint=['/bin/sh'],
host_config=host_config)
rancher_container, _ = start_and_wait(client, docker_container,
docker_client, name)
assert rancher_container.hostname == 'hostname1'
assert rancher_container.domainName == 'domainname1'
assert rancher_container.user == 'root'
assert rancher_container.memory == 16777216
assert rancher_container.cpuShares == 1024
assert rancher_container.cpuSet == '0'
assert rancher_container.tty is True
assert rancher_container.stdinOpen is True
assert rancher_container.imageUuid == 'docker:' + NATIVE_TEST_IMAGE
assert rancher_container.workingDir == '/root'
assert rancher_container.environment['FOO'] == 'BA'
assert rancher_container.command == ['-c', 'sleep 3']
assert rancher_container.entryPoint == ['/bin/sh']
assert rancher_container.privileged is True
assert rancher_container.publishAllPorts is True
assert rancher_container.dns == ['1.2.3.4']
assert rancher_container.dnsSearch == ['search.dns.com']
assert rancher_container.capAdd == ['SYSLOG']
assert rancher_container.capDrop == ['KILL', 'LEASE']
assert rancher_container.restartPolicy == {
'name': 'on-failure',
'maximumRetryCount': 5
}
assert rancher_container.devices == ['/dev/null:/dev/xnull:rw']
def _create():
binds = {
volume.node_path.path: {
'bind': volume.container_path.path,
'ro': False,
}
for volume in volumes
}
port_bindings = {
p.internal_port: p.external_port
for p in ports
}
host_config = create_host_config(
binds=binds,
port_bindings=port_bindings,
restart_policy=restart_policy_dict,
)
# We're likely to get e.g. pvector, so make sure we're passing
# in something JSON serializable:
command_line_values = command_line
if command_line_values is not None:
command_line_values = list(command_line_values)
self._client.create_container(
name=container_name,
image=image_name,
command=command_line_values,
environment=environment,
ports=[p.internal_port for p in ports],
mem_limit=mem_limit,
cpu_shares=cpu_shares,
host_config=host_config,
)
def _test_create_container(self, cluster):
"""
Create a container running a simple HTTP server that writes to
its volume on POST and reads the same data back from the volume
on GET.
"""
data = random_name(self).encode("utf-8")
node = cluster.nodes[0]
http_port = 8080
host_port = find_free_port()[1]
volume_name = random_name(self)
self.run_python_container(
cluster,
node.public_address,
{
"host_config":
create_host_config(binds=["{}:/data".format(volume_name)],
port_bindings={http_port: host_port},
restart_policy={"Name": "always"}),
"ports": [http_port]
},
SCRIPTS.child(b"datahttp.py"),
# This tells the script where it should store its data,
# and we want it to specifically use the volume:
[u"/data"])
d = post_http_server(self, node.public_address, host_port,
{"data": data})
d.addCallback(lambda _: assert_http_server(
self, node.public_address, host_port, expected_response=data))
return d
def _start_db_container(self):
"""Start a new db container if it hasn't been started yet."""
try:
return self.docker.inspect_container(settings.SQL_CONTAINER_NAME)
except NotFound:
pass
# TODO: start if stopped
# TODO: handle docker connection error
logger.info("Starting new db container")
try:
self.docker.inspect_image(settings.SQL_CONTAINER_IMAGE)
except NotFound:
logger.info("Docker image for db container wasn't found, pulling...")
self.docker.pull(settings.SQL_CONTAINER_IMAGE)
logger.info("Docker image for db container has been pulled successfully")
port_bindings = {settings.SQL_CONTAINER_PORT: (settings.SQL_BIND_HOST,
settings.SQL_BIND_PORT)}
environment = {'MYSQL_ROOT_PASSWORD': settings.SQL_DB_ROOT_PASS}
host_config = create_host_config(binds=['{}:{}'.format(settings.SQL_CONTAINER_NAME,
settings.SQL_CONTAINER_VOLUME)])
try:
ctr = self.docker.create_container(settings.SQL_CONTAINER_IMAGE,
command=settings.SQL_DB_CONF_OPTIONS,
name=settings.SQL_CONTAINER_NAME,
environment=environment,
host_config=host_config)
self.docker.start(ctr, port_bindings=port_bindings,
restart_policy={'Name': 'always'})
except (APIError, DockerException):
logger.exception("Failed to start db container '{0}'"
.format(settings.SQL_CONTAINER_NAME))
raise PluginError("Failed to start db container")
self._wait_db_container_started(ctr)
return ctr
def copy(self, src, dst):
def parse(path):
if ':' in path:
box_name, path = path.split(':', 1)
return BoxLocation(self.box(box_name), path)
else:
return LocalLocation(self.client, path)
src = parse(src)
dst = parse(dst)
container = self.client.create_container(
image=self.BASE_IMAGE,
host_config=create_host_config(binds={
src.get_mountpoint(): {
'bind': '/mnt/src',
'ro': True,
},
dst.get_mountpoint(): {
'bind': '/mnt/dst',
'ro': False,
}
}),
command=[
'cp', '-afv',
os.path.join('/mnt/src', src.get_path()),
os.path.join('/mnt/dst', dst.get_path()),
],
)
self.client.start(container=container['Id'])
return self.client.logs(container=container['Id'], stream=True)
def test_run_container_with_volume(self, cluster):
"""
Docker can run a container with a volume provisioned by Flocker.
"""
data = random_name(self).encode("utf-8")
node = cluster.nodes[0]
http_port = 8080
host_port = find_free_port()[1]
volume_name = random_name(self)
self.run_python_container(
cluster,
node.public_address,
{
"host_config":
create_host_config(binds=["{}:/data".format(volume_name)],
port_bindings={http_port: host_port},
restart_policy={"Name": "always"}),
"ports": [http_port]
},
SCRIPTS.child(b"datahttp.py"),
# This tells the script where it should store its data,
# and we want it to specifically use the volume:
[u"/data"])
d = post_http_server(self, node.public_address, host_port,
{"data": data})
d.addCallback(lambda _: assert_http_server(
self, node.public_address, host_port, expected_response=data))
return d
def _test_create_container(self, cluster):
"""
Create a container running a simple HTTP server that writes to
its volume on POST and reads the same data back from the volume
on GET.
"""
data = random_name(self).encode("utf-8")
node = cluster.nodes[0]
http_port = 8080
host_port = find_free_port()[1]
volume_name = random_name(self)
self.run_python_container(
cluster,
node.public_address,
{
"host_config": create_host_config(
binds=["{}:/data".format(volume_name)],
port_bindings={http_port: host_port},
restart_policy={"Name": "always"},
),
"ports": [http_port],
},
SCRIPTS.child(b"datahttp.py"),
# This tells the script where it should store its data,
# and we want it to specifically use the volume:
[u"/data"],
)
d = post_http_server(self, node.public_address, host_port, {"data": data})
d.addCallback(lambda _: assert_http_server(self, node.public_address, host_port, expected_response=data))
return d
def test_run_container_with_volume(self, cluster):
"""
Docker can run a container with a volume provisioned by Flocker.
"""
data = random_name(self).encode("utf-8")
node = cluster.nodes[0]
http_port = 8080
host_port = find_free_port()[1]
volume_name = random_name(self)
self.run_python_container(
cluster, node.public_address,
{"host_config": create_host_config(
binds=["{}:/data".format(volume_name)],
port_bindings={http_port: host_port},
restart_policy={"Name": "always"}),
"ports": [http_port]},
SCRIPTS.child(b"datahttp.py"),
# This tells the script where it should store its data,
# and we want it to specifically use the volume:
[u"/data"])
d = post_http_server(self, node.public_address, host_port,
{"data": data})
d.addCallback(lambda _: assert_http_server(
self, node.public_address, host_port, expected_response=data))
return d
def add_mock_ec2_instance(ec2_reservation):
# TODO: take into account AMIs
instances = ec2_reservation.instances
instance = instances[0] # Assume only 1
try:
image = client.inspect_image(SSHD_IMAGE_NAME)
except docker.errors.NotFound:
image = None
if image is None:
with tempfile.NamedTemporaryFile() as f:
with tempfile.NamedTemporaryFile() as ssh_f:
ssh_f.write(SETUP_SSH_USER)
ssh_f.flush()
ssh_f.seek(0)
f.write(SSHD_DOCKERFILE.format(run_file_path=ssh_f.name))
f.flush()
f.seek(0)
for line in client.build(fileobj=f, rm=True, pull=True, tag=SSHD_IMAGE_NAME):
print line
print
env = {'SSH_DEFAULT_USER': '******', 'SSH_PUB_KEY': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDHQcmuqr2AhogRkzbEdAN3kP3debHEbhXiR8DGyZGptMWstrSOP7RUAG1NsFUtb6+M1rjUKFrNVaR2vTladsF8JlWAfra4DTv23NuncRUll/8ETwgOts2UnIzhVuJ5QzWa2HtPqSSAgvj037yg9vIwZwSYV/FWJv+nKY16z2Hh6q4RAr21Gv+FRXAw1zwDv1jBlargjRvTYzoyD33DcQvrLw6YSjm7scaLBCU02hjelSYvrRuQw6lAJracb70ROSCNngXkj8GujBoT1JXHFOn+qjtxkslCViSJHk2o6hOKpfLWNZ+0Z7GZ6Wy3oZEQJhx5ALl5k74yDQVD5cImTCz88ErLW2tw4en8/qRou5AjzLyy5vAtn7IojDnnHMr4ZLJfnHzvsGjUYa0fv7BnPsJLIuprBncVXJc7NHYrlrtdruUCw7ALo+2EuskVFfaDNQ12GGwahWJJJWZPvr3w7FLNqpGJZdv9l1tJEf9kji7zjPpAZWmKXdt/cpukrWKBCLra5Nt0cTyDf/qmk6Rh34L9BVLkG4yUo6nMiFZilh+6tDmDDWQJTKIMK0LnM2yJx2jdWUBsOPf7M3kyhIN0dZMYyUp4cMznDqq/yWObJ0gaQLYsN9Hh9fi9Q9dXJ9Km/lM2QAAsud3RJErGViba2xE0yKaEwmqJWTZN51E+krvKiQ== [email protected]'}
host_config = create_host_config(port_bindings={22: None})
container = client.create_container(SSHD_IMAGE_NAME,
ports=[22],
host_config=host_config,
environment=env,
name=instance.id)
container_id = container.get('Id')
client.start(container_id)
raw_input('foozle')
def _create():
binds = {
volume.node_path.path: {
'bind': volume.container_path.path,
'ro': False,
}
for volume in volumes
}
port_bindings = {
p.internal_port: p.external_port
for p in ports
}
host_config = create_host_config(
binds=binds,
port_bindings=port_bindings,
restart_policy=restart_policy_dict,
)
self._client.create_container(
name=container_name,
image=image_name,
command=None,
environment=environment,
ports=[p.internal_port for p in ports],
mem_limit=mem_limit,
cpu_shares=cpu_shares,
host_config=host_config,
)
def test_create_host_config_dict_logconfig(self):
dct = {'type': LogConfig.types.SYSLOG, 'config': {'key1': 'val1'}}
config = create_host_config(version=DEFAULT_DOCKER_API_VERSION,
log_config=dct)
self.assertIn('LogConfig', config)
self.assertTrue(isinstance(config['LogConfig'], LogConfig))
self.assertEqual(dct['type'], config['LogConfig'].type)
def add_mock_db_instance(database):
# TODO: Support other DB engines
PG_PORT = 5432
pg_user = database.master_username
pg_password = database.master_user_password
host_config = create_host_config(port_bindings={5432: None})
container_name = 'rds2-{}-{}'.format(database.engine, database.db_instance_identifier)
env = {'POSTGRES_USER': pg_user, 'POSTGRES_PASSWORD': pg_password}
pg_version = database.engine_version
parsed_version = StrictVersion(pg_version)
desired_container_version = '.'.join(map(str, parsed_version.version[:2]))
client.pull('postgres', desired_container_version)
client.get_image('postgres:{}'.format(desired_container_version))
container = client.create_container('postgres',
ports=[PG_PORT],
host_config=host_config,
environment=env,
name=container_name)
container_id = container.get('Id')
client.start(container_id)
# Update access information
container_details = client.inspect_container(container_id)
network_details = container_details['NetworkSettings']
port_details = network_details['Ports']['{}/tcp'.format(PG_PORT)][0]
port = port_details['HostPort']
database.port = port
database.address = docker_ip
def test_create_host_config_obj_logconfig(self):
obj = LogConfig(type=LogConfig.types.SYSLOG, config={'key1': 'val1'})
config = create_host_config(version=DEFAULT_DOCKER_API_VERSION,
log_config=obj)
self.assertIn('LogConfig', config)
self.assertTrue(isinstance(config['LogConfig'], LogConfig))
self.assertEqual(obj, config['LogConfig'])
def _create():
binds = list(
# The "Z" mode tells Docker to "relabel file objects" on the
# volume. This makes things work when SELinux is enabled, at
# least in the default configuration on CentOS 7. See
# <https://docs.docker.com/reference/commandline/run/>, in the
# `--volumes-from` section (or just search for SELinux).
u"{}:{}:Z".format(volume.node_path.path,
volume.container_path.path)
for volume in volumes)
port_bindings = {p.internal_port: p.external_port for p in ports}
host_config = create_host_config(
binds=binds,
port_bindings=port_bindings,
restart_policy=restart_policy_dict,
)
# We're likely to get e.g. pvector, so make sure we're passing
# in something JSON serializable:
command_line_values = command_line
if command_line_values is not None:
command_line_values = list(command_line_values)
self._client.create_container(
name=container_name,
image=image_name,
command=command_line_values,
environment=environment,
ports=[p.internal_port for p in ports],
mem_limit=mem_limit,
cpu_shares=cpu_shares,
host_config=host_config,
)
def create_container(self):
'''Create docker container'''
client = self.get_client()
try:
self.container = client.create_container(
image=self.container_image,
command=('/bin/sh -c "sleep {time}; exit {exit}"'
.format(time=self.container_time_limit,
exit=DOCKER_TIMEOUT_EXIT_CODE)),
name=self.container_id,
hostname=self.container_id,
host_config=create_host_config(binds={
self.project.doc_path: {
'bind': self.project.doc_path,
'mode': 'rw'
}
}),
detach=True,
mem_limit=self.container_mem_limit,
)
client.start(container=self.container_id)
except DockerAPIError as e:
log.error(LOG_TEMPLATE
.format(
project=self.project.slug,
version=self.version.slug,
msg=e.explanation),
exc_info=True)
raise BuildEnvironmentError('Build environment creation failed')
def run(self, build, *tests):
"""
Run all the test suites passed in as parameters on the given build
This method will start a container of the build, run the tests and stop it
"""
from docker.utils import create_host_config
print("Running tests on {}".format(build.name))
ports = self.config['environment']['ports']
host = self.config['global'].get('docker_host', os.getenv('DOCKER_HOST').split('/')[-1].split(':')[0])
container = self.docker.create_container(
image=build.docker_tag,
command='/bin/bash -c "nc -l 8080"',
ports=ports,
host_config=create_host_config(port_bindings=dict(zip(ports, [None] * len(ports))))
).get('Id')
self.docker.start(container)
info = self.docker.inspect_container(container)
port_bindings = {port: bind[0]['HostPort'] for port, bind in info['NetworkSettings']['Ports'].items()}
for test in tests:
test.run(host, port_bindings, build.context)
self.docker.stop(container, timeout=self.STOP_TIMEOUT)
log_file_path = os.path.join(self.config['global'].get('logs_dir', '/tmp'), '{}.log'.format(build.name))
with open(log_file_path, 'wb') as logs:
logs.write(self.docker.logs(container, stdout=True, stderr=True, stream=False))
print("Container logs wrote to {}".format(log_file_path))
def create_one_container(host, version, entrypoint, env='prod', cores=None):
if cores is None:
cores = []
client = get_docker_client(host.addr)
local_images = {r['RepoTags'][0] for r in client.images()}
appconfig = version.appconfig
appname = appconfig.appname
entry = appconfig.entrypoints[entrypoint]
envconfig = version.get_resource_config(env)
image = '{0}/{1}:{2}'.format(settings.DOCKER_REGISTRY, appname, version.short_sha)
if image not in local_images:
repo = '{0}/{1}'.format(settings.DOCKER_REGISTRY, appname)
for line in client.pull(repo, tag=version.short_sha, stream=True,
insecure_registry=settings.DOCKER_REGISTRY_INSECURE):
print line
env_dict = {
'APP_NAME': appname,
'ERU_RUNENV': env.upper(),
'ERU_POD': host.pod.name,
'ERU_HOST': host.name,
}
env_dict.update(envconfig.to_env_dict())
# TODO use settings!!!
# This modification for applying sysctl params
volumes = ['/writable-proc/sys']
binds = {'/proc/sys': {'bind': '/writable-proc/sys', 'ro': False}}
if settings.ERU_CONTAINER_PERMDIR:
permdir = settings.ERU_CONTAINER_PERMDIR % appname
env_dict['ERU_PERMDIR'] = permdir
volumes.append(permdir)
binds[settings.ERU_HOST_PERMDIR % appname] = {'bind': permdir, 'ro': False}
# container name: {appname}_{entrypoint}_{ident_id}
container_name = '_'.join([appname, entrypoint, random_string(6)])
# cpuset: '0,1,2,3'
cpuset = ','.join([c.label for c in cores])
# host_config, include log_config
host_config = create_host_config(log_config=LogConfig(type=settings.DOCKER_LOG_DRIVER))
container = client.create_container(
image=image,
command=entry['cmd'],
environment=env_dict,
entrypoint='launch',
name=container_name,
cpuset=cpuset,
working_dir='/%s' % appname,
network_disabled=settings.DOCKER_NETWORK_DISABLED,
volumes=volumes,
host_config=host_config,
)
container_id = container['Id']
client.start(container=container_id, network_mode=settings.DOCKER_NETWORK_MODE, binds=binds)
return container_id, container_name
def test_create_host_config_obj_ulimit(self):
ulimit_dct = Ulimit(name="nofile", soft=8096)
config = create_host_config(ulimits=[ulimit_dct], version=DEFAULT_DOCKER_API_VERSION)
self.assertIn("Ulimits", config)
self.assertEqual(len(config["Ulimits"]), 1)
ulimit_obj = config["Ulimits"][0]
self.assertTrue(isinstance(ulimit_obj, Ulimit))
self.assertEqual(ulimit_obj, ulimit_dct)
def test_create_host_config_obj_ulimit(self):
ulimit_dct = Ulimit(name='nofile', soft=8096)
config = create_host_config(ulimits=[ulimit_dct])
self.assertIn('Ulimits', config)
self.assertEqual(len(config['Ulimits']), 1)
ulimit_obj = config['Ulimits'][0]
self.assertTrue(isinstance(ulimit_obj, Ulimit))
self.assertEqual(ulimit_obj, ulimit_dct)
def test_create_host_config_obj_logconfig(self):
obj = LogConfig(type=LogConfig.types.SYSLOG, config={'key1': 'val1'})
config = create_host_config(
version=DEFAULT_DOCKER_API_VERSION, log_config=obj
)
self.assertIn('LogConfig', config)
self.assertTrue(isinstance(config['LogConfig'], LogConfig))
self.assertEqual(obj, config['LogConfig'])
def start_container(self):
host_config = create_host_config(publish_all_ports=True,restart_policy={'Name':'always'})
app_name = self._app_prefix+str(1)
cli = options.docker_client
try:
cli.remove_container(container=app_name,force=True)
except APIError,e:
logging.info("can't find container")
def test_create_host_config_obj_ulimit(self):
ulimit_dct = Ulimit(name='nofile', soft=8096)
config = create_host_config(ulimits=[ulimit_dct])
self.assertIn('Ulimits', config)
self.assertEqual(len(config['Ulimits']), 1)
ulimit_obj = config['Ulimits'][0]
self.assertTrue(isinstance(ulimit_obj, Ulimit))
self.assertEqual(ulimit_obj, ulimit_dct)
def test_create_host_config_dict_logconfig(self):
dct = {'type': LogConfig.types.SYSLOG, 'config': {'key1': 'val1'}}
config = create_host_config(
version=DEFAULT_DOCKER_API_VERSION, log_config=dct
)
self.assertIn('LogConfig', config)
self.assertTrue(isinstance(config['LogConfig'], LogConfig))
self.assertEqual(dct['type'], config['LogConfig'].type)
def create_container(self, service_definition):
"""Creates a new container based on a service definition object."""
container = self.client.create_container(
image=service_definition['image'],
command=service_definition['command'],
ports=service_definition['ports'],
environment=service_definition['environment'],
host_config=utils.create_host_config(publish_all_ports=True))
self.client.start(container.get('Id'))
def test_native_net_host(socat_containers, client, native_name, pull_images):
docker_client = get_docker_client(host(client))
host_config = create_host_config(network_mode='host')
docker_container = docker_client.create_container(NATIVE_TEST_IMAGE,
name=native_name,
host_config=host_config)
rancher_container, docker_container = start_and_wait(
client, docker_container, docker_client, native_name)
common_network_asserts(rancher_container, docker_container, 'host')
def _do_instance_activate(self, instance, host, progress):
if is_no_op(instance):
return
client = self._get_docker_client(host)
image_tag = self._get_image_tag(instance)
name = instance.uuid
create_config = {"name": name, "detach": True}
start_config = {
"publish_all_ports": False,
"privileged": self._is_true(instance, "privileged"),
"read_only": self._is_true(instance, "readOnly"),
}
# These _setup_simple_config_fields calls should happen before all
# other config because they stomp over config fields that other
# setup methods might append to. Example: the environment field
self._setup_simple_config_fields(create_config, instance, CREATE_CONFIG_FIELDS)
self._setup_simple_config_fields(start_config, instance, START_CONFIG_FIELDS)
add_label(create_config, {"io.rancher.container.uuid": instance.uuid})
self._setup_logging(start_config, instance)
self._setup_hostname(create_config, instance)
self._setup_command(create_config, instance)
self._setup_ports(create_config, instance)
self._setup_volumes(create_config, instance, start_config, client)
self._setup_links(start_config, instance)
self._setup_networking(instance, host, create_config, start_config)
self._flag_system_container(instance, create_config)
self._setup_proxy(instance, create_config)
setup_cattle_config_url(instance, create_config)
create_config["host_config"] = create_host_config(**start_config)
container = self._create_container(client, create_config, image_tag, instance, name, progress)
container_id = container["Id"]
log.info("Starting docker container [%s] docker id [%s] %s", name, container_id, start_config)
client.start(container_id)
self._record_state(client, instance, docker_id=container["Id"])
def create_container(self, service_definition):
"""Creates a new container based on a service definition object."""
container = self.client.create_container(
image=service_definition['image'],
command=service_definition['command'],
ports=service_definition['ports'],
environment=service_definition['environment'],
host_config=utils.create_host_config(publish_all_ports=True))
self.client.start(container.get('Id'))
def create_container(self):
"""Create docker container."""
client = self.get_client()
image = self.container_image
if self.project.container_image:
image = self.project.container_image
try:
self.container = client.create_container(
image=image,
command=('/bin/sh -c "sleep {time}; exit {exit}"'.format(
time=self.container_time_limit,
exit=DOCKER_TIMEOUT_EXIT_CODE)),
name=self.container_id,
hostname=self.container_id,
host_config=create_host_config(
binds={
SPHINX_TEMPLATE_DIR: {
'bind': SPHINX_TEMPLATE_DIR,
'mode': 'ro'
},
MKDOCS_TEMPLATE_DIR: {
'bind': MKDOCS_TEMPLATE_DIR,
'mode': 'ro'
},
self.project.doc_path: {
'bind': self.project.doc_path,
'mode': 'rw'
},
}),
detach=True,
environment=self.environment,
mem_limit=self.container_mem_limit,
)
client.start(container=self.container_id)
except ConnectionError as e:
log.exception(
LOG_TEMPLATE.format(
project=self.project.slug,
version=self.version.slug,
msg=('Could not connect to the Docker API, '
'make sure Docker is running'),
), )
# We don't raise an error here mentioning Docker, that is a
# technical detail that the user can't resolve on their own.
# Instead, give the user a generic failure
raise BuildEnvironmentError(
BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(
build_id=self.build['id'], ))
except DockerAPIError as e:
log.exception(
LOG_TEMPLATE.format(
project=self.project.slug,
version=self.version.slug,
msg=e.explanation,
), )
raise BuildEnvironmentCreationFailed
def build_image_dockerhost(self, build_image, json_args_path):
"""
Build docker image inside privileged container using docker from host
(mount docker socket inside container).
There are possible races here. Use wisely.
This operation is asynchronous and you should wait for container to finish.
:param build_image: str, name of image where build is performed
:param json_args_path: str, this dir is mounted inside build container and used
as a way to transport data between host and buildroot; there
has to be a file inside this dir with name atomic_reactor.BUILD_JSON which
is used to feed build
:return: str, container id
"""
logger.info("building image '%s' in container using docker from host",
build_image)
self._check_build_input(build_image, json_args_path)
self._obtain_source_from_path_if_needed(json_args_path,
CONTAINER_SHARE_PATH)
if not os.path.exists(DOCKER_SOCKET_PATH):
logger.error(
"looks like docker is not running because there is no socket at: %s",
DOCKER_SOCKET_PATH)
raise RuntimeError("docker socket not found: %s" %
DOCKER_SOCKET_PATH)
volume_bindings = {
DOCKER_SOCKET_PATH: {
'bind': DOCKER_SOCKET_PATH,
},
json_args_path: {
'bind': CONTAINER_SHARE_PATH,
},
}
if self._volume_bind_understands_mode():
volume_bindings[DOCKER_SOCKET_PATH]['mode'] = 'ro'
volume_bindings[json_args_path]['mode'] = 'rw,Z'
else:
volume_bindings[DOCKER_SOCKET_PATH]['ro'] = True
volume_bindings[json_args_path]['rw'] = True
logger.debug('build json mounted in container: %s',
open(os.path.join(json_args_path, BUILD_JSON)).read())
container_id = self.tasker.run(
ImageName.parse(build_image),
create_kwargs={
'volumes': [DOCKER_SOCKET_PATH, json_args_path],
'host_config':
create_host_config(binds=volume_bindings, privileged=True)
})
return container_id
def test_native_fields(socat_containers, client, pull_images):
docker_client = get_docker_client(host(client))
name = 'native-%s' % random_str()
host_config = create_host_config(
privileged=True,
publish_all_ports=True,
dns=['1.2.3.4'], dns_search=['search.dns.com'],
cap_add=['SYSLOG'], cap_drop=['KILL', 'LEASE'],
restart_policy={'MaximumRetryCount': 5,
'Name': 'on-failure'},
devices=['/dev/null:/dev/xnull:rw'])
docker_container = docker_client.create_container(NATIVE_TEST_IMAGE,
name=name,
hostname='hostname1',
domainname='domainname1',
user='******',
mem_limit='16MB',
memswap_limit='32MB',
cpu_shares=1024,
cpuset='0',
tty=True,
stdin_open=True,
working_dir='/root',
environment={
'FOO': 'BA'},
command=['-c',
'sleep 3'],
entrypoint=['/bin/sh'],
host_config=host_config)
rancher_container, _ = start_and_wait(client, docker_container,
docker_client, name)
assert rancher_container.hostname == 'hostname1'
assert rancher_container.domainName == 'domainname1'
assert rancher_container.user == 'root'
assert rancher_container.memory == 16777216
assert rancher_container.cpuShares == 1024
assert rancher_container.cpuSet == '0'
assert rancher_container.tty is True
assert rancher_container.stdinOpen is True
assert rancher_container.imageUuid == 'docker:' + NATIVE_TEST_IMAGE
assert rancher_container.workingDir == '/root'
assert rancher_container.environment['FOO'] == 'BA'
assert rancher_container.command == ['-c', 'sleep 3']
assert rancher_container.entryPoint == ['/bin/sh']
assert rancher_container.privileged is True
assert rancher_container.publishAllPorts is True
assert rancher_container.dns == ['1.2.3.4']
assert rancher_container.dnsSearch == ['search.dns.com']
assert rancher_container.capAdd == ['SYSLOG']
assert rancher_container.capDrop == ['KILL', 'LEASE']
assert rancher_container.restartPolicy["name"] == u"on-failure"
assert rancher_container.restartPolicy["maximumRetryCount"] == 5
assert rancher_container.devices == ['/dev/null:/dev/xnull:rw']
def build_container(self):
if self.port_mappings:
self._container = self.docker_client.create_container(
image=self.full_image_name,
environment=self.environment,
ports=list(self.port_mappings.keys()),
host_config=create_host_config(
port_bindings=self.port_mappings,
network_mode=self.network_mode
)
)
else:
self._container = self.docker_client.create_container(
image=self.full_image_name,
environment=self.environment,
host_config=create_host_config(
network_mode=self.network_mode
)
)
def test_create_host_config_dict_ulimit(self):
ulimit_dct = {'name': 'nofile', 'soft': 8096}
config = create_host_config(ulimits=[ulimit_dct])
self.assertIn('Ulimits', config)
self.assertEqual(len(config['Ulimits']), 1)
ulimit_obj = config['Ulimits'][0]
self.assertTrue(isinstance(ulimit_obj, Ulimit))
self.assertEqual(ulimit_obj.name, ulimit_dct['name'])
self.assertEqual(ulimit_obj.soft, ulimit_dct['soft'])
self.assertEqual(ulimit_obj['Soft'], ulimit_obj.soft)
def test_create_host_config_dict_ulimit(self):
ulimit_dct = {"name": "nofile", "soft": 8096}
config = create_host_config(ulimits=[ulimit_dct])
self.assertIn("Ulimits", config)
self.assertEqual(len(config["Ulimits"]), 1)
ulimit_obj = config["Ulimits"][0]
self.assertTrue(isinstance(ulimit_obj, Ulimit))
self.assertEqual(ulimit_obj.name, ulimit_dct["name"])
self.assertEqual(ulimit_obj.soft, ulimit_dct["soft"])
self.assertEqual(ulimit_obj["Soft"], ulimit_obj.soft)
def test_create_host_config_invalid_cpu_cfs_types(self):
with pytest.raises(TypeError):
create_host_config(version='1.20', cpu_quota='0')
with pytest.raises(TypeError):
create_host_config(version='1.20', cpu_period='0')
with pytest.raises(TypeError):
create_host_config(version='1.20', cpu_quota=23.11)
with pytest.raises(TypeError):
create_host_config(version='1.20', cpu_period=1999.0)
def test_create_host_config_invalid_cpu_cfs_types(self):
with pytest.raises(TypeError):
create_host_config(version='1.20', cpu_quota='0')
with pytest.raises(TypeError):
create_host_config(version='1.20', cpu_period='0')
with pytest.raises(TypeError):
create_host_config(version='1.20', cpu_quota=23.11)
with pytest.raises(TypeError):
create_host_config(version='1.20', cpu_period=1999.0)
def test_create_host_config_dict_ulimit(self):
ulimit_dct = {'name': 'nofile', 'soft': 8096}
config = create_host_config(ulimits=[ulimit_dct])
self.assertIn('Ulimits', config)
self.assertEqual(len(config['Ulimits']), 1)
ulimit_obj = config['Ulimits'][0]
self.assertTrue(isinstance(ulimit_obj, Ulimit))
self.assertEqual(ulimit_obj.name, ulimit_dct['name'])
self.assertEqual(ulimit_obj.soft, ulimit_dct['soft'])
self.assertEqual(ulimit_obj['Soft'], ulimit_obj.soft)
def run_container(name,
image,
command=None,
environment=None,
ro=None,
rw=None,
links=None,
detach=True,
volumes_from=None,
port_bindings=None,
log_syslog=False):
"""
Wrapper for docker create_container, start calls
:param log_syslog: bool flag to redirect container's logs to host's syslog
:returns: container info dict or None if container couldn't be created
Raises PortAllocatedError if container couldn't start on the
requested port.
"""
binds = ro_rw_to_binds(ro, rw)
log_config = LogConfig(type=LogConfig.types.JSON)
if log_syslog:
log_config = LogConfig(type=LogConfig.types.SYSLOG,
config={'syslog-tag': name})
host_config = create_host_config(binds=binds, log_config=log_config)
c = _get_docker().create_container(
name=name,
image=image,
command=command,
environment=environment,
volumes=binds_to_volumes(binds),
detach=detach,
stdin_open=False,
tty=False,
ports=list(port_bindings) if port_bindings else None,
host_config=host_config)
try:
_get_docker().start(container=c['Id'],
links=links,
binds=binds,
volumes_from=volumes_from,
port_bindings=port_bindings)
except APIError as e:
if 'address already in use' in e.explanation:
try:
_get_docker().remove_container(name, force=True)
except APIError:
pass
raise PortAllocatedError()
raise
return c
def _test_move(self, cluster, origin_node, destination_node):
"""
Assert that Docker can run a container with a volume provisioned by
Flocker, shut down the container and then start a new container
with the same volume on the specified node.
:param cluster: The ``Cluster`` to talk to.
:param Node origin_node: Original node to start container on.
:param Node destination_node: Original node to start container on.
:return: ``Deferred`` that fires on assertion success, or failure.
"""
data = "hello world"
http_port = 8080
host_port = find_free_port()[1]
volume_name = random_name(self)
container_args = {
"host_config":
create_host_config(binds=["{}:/data".format(volume_name)],
port_bindings={http_port: host_port}),
"ports": [http_port]
}
cid = self.run_python_container(
cluster,
origin_node.public_address,
container_args,
SCRIPTS.child(b"datahttp.py"),
# This tells the script where it should store its data,
# and we want it to specifically use the volume:
[u"/data"],
cleanup=False)
# Post to container on origin node:
d = post_http_server(self, origin_node.public_address, host_port,
{"data": data})
def posted(_):
# Shutdown original container:
client = get_docker_client(cluster, origin_node.public_address)
client.remove_container(cid, force=True)
# Start container on destination node with same volume:
self.run_python_container(cluster, destination_node.public_address,
container_args,
SCRIPTS.child(b"datahttp.py"),
[u"/data"])
d.addCallback(posted)
d.addCallback(
lambda _: assert_http_server(self,
destination_node.public_address,
host_port,
expected_response=data))
return d
def run_image(image_name, port_host, port_container):
print time.strftime("%a, %d %b %Y %X +0000", time.gmtime())
if has_image(image_name) == True:
config = create_host_config(port_bindings={port_container: port_host})
container = client.create_container(image=image_name,
ports=[port_container],
host_config=config)
client.start(container=container.get('Id'))
print time.strftime("%a, %d %b %Y %X +0000", time.gmtime())
return is_image_running(image_name)
print time.strftime("%a, %d %b %Y %X +0000", time.gmtime())
return False
def test_create_host_config_dict_ulimit_capitals(self):
ulimit_dct = {'Name': 'nofile', 'Soft': 8096, 'Hard': 8096 * 4}
config = create_host_config(ulimits=[ulimit_dct],
version=DEFAULT_DOCKER_API_VERSION)
self.assertIn('Ulimits', config)
self.assertEqual(len(config['Ulimits']), 1)
ulimit_obj = config['Ulimits'][0]
self.assertTrue(isinstance(ulimit_obj, Ulimit))
self.assertEqual(ulimit_obj.name, ulimit_dct['Name'])
self.assertEqual(ulimit_obj.soft, ulimit_dct['Soft'])
self.assertEqual(ulimit_obj.hard, ulimit_dct['Hard'])
self.assertEqual(ulimit_obj['Soft'], ulimit_obj.soft)