def web_command(command,
ro=None,
rw=None,
links=None,
image='datacats/web',
volumes_from=None,
commit=False,
clean_up=False,
stream_output=None,
entrypoint=None):
"""
Run a single command in a web image optionally preloaded with the ckan
source and virtual envrionment.
:param command: command to execute
:param ro: {localdir: binddir} dict for read-only volumes
:param rw: {localdir: binddir} dict for read-write volumes
:param links: links passed to start
:param image: docker image name to use
:param volumes_from:
:param commit: True to create a new image based on result
:param clean_up: True to remove container even on error
:param stream_output: file to write stderr+stdout from command
:param entrypoint: override entrypoint (script that runs command)
:returns: image id if commit=True
"""
binds = ro_rw_to_binds(ro, rw)
c = _get_docker().create_container(image=image,
command=command,
volumes=binds_to_volumes(binds),
detach=False,
host_config=HostConfig(
version=MINIMUM_API_VERSION,
binds=binds,
links=links,
volumes_from=volumes_from),
entrypoint=entrypoint)
_get_docker().start(container=c['Id'])
if stream_output:
for output in _get_docker().attach(c['Id'],
stdout=True,
stderr=True,
stream=True):
stream_output.write(output)
if _get_docker().wait(c['Id']):
# Before the (potential) cleanup, grab the logs!
logs = _get_docker().logs(c['Id'])
if clean_up:
remove_container(c['Id'])
raise WebCommandError(command, c['Id'][:12], logs)
if commit:
rval = _get_docker().commit(c['Id'])
if not remove_container(c['Id']):
# circle ci doesn't let us remove containers, quiet the warnings
if not environ.get('CIRCLECI', False):
warn('failed to remove container: {0}'.format(c['Id']))
if commit:
return rval['Id']
def run_config_init(self):
workdir = os.getcwd()
image = f"gluufederation/config-init:{self.settings['CONFIG_INIT_VERSION']}"
volumes = [
f"{workdir}/{CONFIG_DIR}:/app/db/",
f"{workdir}/vault_role_id.txt:/etc/certs/vault_role_id",
f"{workdir}/vault_secret_id.txt:/etc/certs/vault_secret_id",
]
gen_file = f"{workdir}/generate.json"
if os.path.isfile(gen_file):
volumes.append(f"{gen_file}:/app/db/generate.json")
with self.top_level_cmd() as tlc:
retry = 0
while retry < 3:
try:
if not tlc.project.client.images(name=image):
print(
f"{self.settings['CONFIG_INIT_VERSION']}: Pulling from gluufederation/config-init"
)
tlc.project.client.pull(image)
break
except (requests.exceptions.Timeout,
docker.errors.APIError) as exc:
print(f"[W] Unable to get {image}; reason={exc}; "
"retrying in 10 seconds")
time.sleep(10)
retry += 1
cid = None
try:
cid = tlc.project.client.create_container(
image=
f"gluufederation/config-init:{self.settings['CONFIG_INIT_VERSION']}",
name="config-init",
command="load",
environment={
"GLUU_CONFIG_CONSUL_HOST": "consul",
"GLUU_SECRET_VAULT_HOST": "vault",
},
host_config=HostConfig(
version="1.25",
network_mode=self.network_name,
binds=volumes,
),
).get("Id")
tlc.project.client.start(cid)
for log in tlc.project.client.logs(cid, stream=True):
print(log.decode().strip())
except Exception:
raise
finally:
if cid:
tlc.project.client.remove_container(cid, force=True)
def run_container(name,
image,
command=None,
environment=None,
ro=None,
rw=None,
links=None,
detach=True,
volumes_from=None,
port_bindings=None,
log_syslog=False):
"""
Wrapper for docker create_container, start calls
:param log_syslog: bool flag to redirect container's logs to host's syslog
:returns: container info dict or None if container couldn't be created
Raises PortAllocatedError if container couldn't start on the
requested port.
"""
binds = ro_rw_to_binds(ro, rw)
log_config = LogConfig(type=LogConfig.types.JSON)
if log_syslog:
log_config = LogConfig(type=LogConfig.types.SYSLOG,
config={'syslog-tag': name})
host_config = HostConfig(version=MINIMUM_API_VERSION,
binds=binds,
log_config=log_config,
links=links,
volumes_from=volumes_from,
port_bindings=port_bindings)
c = _get_docker().create_container(
name=name,
image=image,
command=command,
environment=environment,
volumes=binds_to_volumes(binds),
detach=detach,
stdin_open=False,
tty=False,
ports=list(port_bindings) if port_bindings else None,
host_config=host_config)
try:
_get_docker().start(container=c['Id'])
except APIError as e:
if 'address already in use' in e.explanation:
try:
_get_docker().remove_container(name, force=True)
except APIError:
pass
raise PortAllocatedError()
raise
return c
def create_host_config(docker_api, resource_limits, binds, port_bindings):
resource_limits = resource_limits or {}
privileged = get_ship_config().get('privileged', 'false').lower() == 'true'
params = {
'privileged': privileged,
'publish_all_ports': True,
'binds': binds,
'port_bindings': port_bindings,
'mem_limit': resource_limits.get('memory'),
'memswap_limit': resource_limits.get('memory_swap'),
'cgroup_parent': resource_limits.get('cgroup_parent'),
'cpu_shares': resource_limits.get('cpu_shares'),
}
return HostConfig(DOCKER_API_VERSION, **params)
def _get_container_host_config(self,
override_options,
one_off=False,
intermediate_container=None):
options = dict(self.options, **override_options)
port_bindings = build_port_bindings(options.get('ports') or [])
volume_bindings = dict(
build_volume_binding(parse_volume_spec(volume))
for volume in options.get('volumes') or [] if ':' in volume)
privileged = options.get('privileged', False)
cap_add = options.get('cap_add', None)
cap_drop = options.get('cap_drop', None)
pid = options.get('pid', None)
dns = options.get('dns', None)
if isinstance(dns, six.string_types):
dns = [dns]
dns_search = options.get('dns_search', None)
if isinstance(dns_search, six.string_types):
dns_search = [dns_search]
restart = parse_restart_spec(options.get('restart', None))
extra_hosts = build_extra_hosts(options.get('extra_hosts', None))
return HostConfig(
version='1.21',
links=self._get_links(link_to_self=one_off),
port_bindings=port_bindings,
binds=volume_bindings,
volumes_from=self._get_volumes_from(intermediate_container),
privileged=privileged,
network_mode=self._get_net(),
dns=dns,
dns_search=dns_search,
restart_policy=restart,
cap_add=cap_add,
cap_drop=cap_drop,
extra_hosts=extra_hosts,
pid_mode=pid)
def recreate_container(self, container, **override_options):
"""Recreate a container. An intermediate container is created so that
the new container has the same name, while still supporting
`volumes-from` the original container.
"""
try:
container.stop()
except APIError as e:
if (e.response.status_code == 500
and e.explanation
and 'no such process' in str(e.explanation)):
pass
else:
raise
intermediate_container = Container.create(
self.client,
image=container.image,
entrypoint=['/bin/echo'],
command=[],
detach=True,
host_config=HostConfig(volumes_from=[container.id]),
)
intermediate_container.start()
intermediate_container.wait()
container.remove()
options = dict(override_options)
new_container = self.create_container(
do_build=False,
intermediate_container=intermediate_container,
**options
)
self.start_container(new_container)
intermediate_container.remove()
return (intermediate_container, new_container)
def create_host_config(*args, **kwargs):
return HostConfig(*args, **kwargs)
