def installer(self, config):
"""Returns an initialized partially mocked SwarmInstaller."""
return SwarmInstaller(
config,
"docker-swarm",
docker_client=DockerClient()
)
async def postgres(loop):
tag = 'latest'
image = 'postgres'
host = '127.0.0.1'
timeout = 60
unused_tcp_port = get_free_port()
client = DockerClient(version='auto', **kwargs_from_env())
client.images.pull(image, tag=tag)
print('Stating %s:%s on %s:%s' % (image, tag, host, unused_tcp_port))
cont = client.containers.run('%s:%s' % (image, tag), detach=True,
ports={'5432/tcp': ('0.0.0.0',
unused_tcp_port)})
try:
start_time = time.time()
conn = None
while conn is None:
if start_time + timeout < time.time():
raise Exception("Initialization timeout, failed to "
"initialize postgresql container")
try:
conn = await asyncpg.connect(
'postgresql://postgres@%s:%s/postgres'
'' % (host, unused_tcp_port),
loop=loop)
except Exception as e:
time.sleep(.1)
await conn.close()
yield (host, unused_tcp_port)
finally:
cont.kill()
cont.remove()
def docker_context():
"""Make a docker context"""
host = os.environ.get('DOCKER_HOST')
cert_path = os.environ.get('DOCKER_CERT_PATH')
tls_verify = os.environ.get('DOCKER_TLS_VERIFY')
options = {"timeout": 60}
if host:
options['base_url'] = (host.replace('tcp://', 'https://')
if tls_verify else host)
if tls_verify and cert_path:
options['tls'] = docker.tls.TLSConfig(
verify=True,
ca_cert=os.path.join(cert_path, 'ca.pem'),
client_cert=(os.path.join(cert_path, 'cert.pem'),
os.path.join(cert_path, 'key.pem')),
ssl_version=ssl.PROTOCOL_TLSv1,
assert_hostname=False)
client = DockerClient(**options)
try:
info = client.info()
log.info("Connected to docker daemon\tdriver=%s\tkernel=%s",
info["Driver"], info["KernelVersion"])
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout) as error:
raise BadDockerConnection(base_url=options['base_url'], error=error)
return client
def docker_client(version='auto', base_url=None, tls=False, **kwargs):
kwargs = kwargs_from_env(**kwargs)
kwargs['version'] = version
kwargs['base_url'] = base_url
if tls:
cert, _ = AUTH_CONFIG
kwargs['tls'] = TLSConfig(client_cert=cert, verify=False)
return DockerClient(**kwargs)
def __init__(self):
"""Initialize Docker base wrapper."""
self.docker: DockerClient = DockerClient(
base_url=f"unix:/{str(SOCKET_DOCKER)}", version="auto", timeout=900
)
self.network: DockerNetwork = DockerNetwork(self.docker)
self._info: DockerInfo = DockerInfo.new(self.docker.info())
self.config: DockerConfig = DockerConfig()
def __init__(self, docker='unix:///var/run/docker.sock', domain='docker'):
from docker.client import DockerClient
self._docker = DockerClient(docker, version='auto')
self._domain = domain
self._storage = Storage()
self._lock = threading.Lock()
threading.Thread(group=None, target=self.listen).start()
def deploy_instance(docker_api_endpoint, service_name, env_vars, revision, memory, cpu, ports=None):
c = DockerClient(docker_api_endpoint)
res = c.create_container(
image="docker-service-provisioner/%s:v%s" % (service_name, revision),
environment=env_vars,
# TODO: Implement memory and CPU
#mem_limit=memory,
)
container_id = res['Id']
c.start(container_id, port_bindings={"%s/tcp" % p: [{'HostIp': '', 'HostPort': ''}] for p in ports})
# Use inspect_container, as c.ports() doesn't seem to work for some reason
container = c.inspect_container(container_id)
return container['ID'], {p: container['NetworkSettings']['Ports']["%s/tcp" % p][0]['HostPort'] for p in ports}
def docker_context():
"""Make a docker context"""
options = kwargs_from_env(assert_hostname=False)
options["version"] = "auto"
options["timeout"] = int(os.environ.get("DOCKER_CLIENT_TIMEOUT", 180))
client = DockerClient(**options)
try:
info = client.info()
log.info("Connected to docker daemon\tdriver=%s\tkernel=%s",
info["Driver"], info["KernelVersion"])
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout) as error:
raise BadDockerConnection(base_url=options.get('base_url'),
error=error)
return client
def client(self) -> DockerClient:
"""Get DockerClient"""
try:
client = None
if self.local:
client = DockerClient.from_env()
else:
client = DockerClient(
base_url=self.url,
tls=DockerInlineTLS(
verification_kp=self.tls_verification,
authentication_kp=self.tls_authentication,
).write(),
)
client.containers.list()
except DockerException as exc:
LOGGER.warning(exc)
raise ServiceConnectionInvalid from exc
return client
def handle(self, *args, **kwargs):
hosts = Host.objects.filter(use_for_building_images=True).order_by('?')
if hosts:
c = DockerClient(base_url=hosts[0].docker_api_endpoint)
for plugin_dict in pool.get_all_plugin_dicts():
result, log = c.build(
tag="docker-service-provisioner/%s:v%s" %
(plugin_dict['service'], plugin_dict['version']),
path=urljoin(
settings.DOCKER_PROVISION_URL, "dockerfile/%s/%s/" %
(plugin_dict['service'], plugin_dict['version'])))
if result:
print "Converted", plugin_dict['service'], plugin_dict[
'version'], 'to', result
else:
print "Failed converting", plugin_dict[
'service'], plugin_dict['version'], 'to', result
else:
raise Exception("No hosts available for building images!")
def _create_docker_pg_db(
image,
version="latest",
user="******",
dbname="postgres",
host="127.0.0.1",
init_timeout=10,
):
# searching free port
sock = socket.socket()
sock.bind(("", 0))
port = sock.getsockname()[1]
sock.close()
client = DockerClient(version="auto", **kwargs_from_env())
cont = client.containers.run(
f"{image}:{version}",
detach=True,
ports={"5432/tcp": (host, port)},
environment={"POSTGRES_HOST_AUTH_METHOD": "trust"},
)
try:
start_time = time.time()
conn = None
while conn is None:
if start_time + init_timeout < time.time():
raise Exception(
"Initialization timeout, failed to "
"initialize postgresql container"
)
try:
conn = psycopg2.connect(
f"dbname={dbname} user={user} "
f"host={host} port={port}"
)
except psycopg2.OperationalError:
time.sleep(0.10)
conn.close()
yield (host, port), cont
finally:
cont.kill()
cont.remove()
def testInstallInDockerWithoutGUI(base_fixture):
""" Start an installation with the installer running without the updater ui. """
client = DockerClient()
docker_base_cmd = f"docker build {str(base_fixture.base_path)} -t {WAQD_IMAGE} -f ./test/testdata/auto_updater/dockerfile_install"
if platform.system() == "Linux":
docker_base_cmd = docker_base_cmd + " | tee install.log"
ret = os.system(docker_base_cmd)
assert ret == 0
cont = client.containers.create(WAQD_IMAGE,
name="waqd-install-test",
stdin_open=True,
auto_remove=True)
cont.start()
# check if pipx installed
res = cont.exec_run("python3 -m pipx --version", user="******")
assert res.exit_code == 0
# check if pyqt-5 is installed
res = cont.exec_run("qtchooser -l", user="******")
assert b"qt5" in res.output
# check if waqd is installed
res = cont.exec_run("/home/pi/.local/bin/waqd.{VERSION} --version")
assert VERSION in res.output.decode("utf-8")
# get waqd-start executable
# check if it was set for autostart
# check system setup
# autostart
#arch = cont.get_archive("/home/pi/.config/lxsession/LXDE-pi/autostart")
# ... TODO
#cont.attach()
cont.stop()
client.images.prune()
def _create_postgres_like_container(request):
host = "127.0.0.1"
timeout = 600
# getting free port
sock = socket.socket()
sock.bind(("", 0))
port = sock.getsockname()[1]
sock.close()
image, tag, user, dbname = request.param
client = DockerClient(version="auto", **kwargs_from_env())
client.images.pull(image, tag=tag)
cont = client.containers.run(
f"{image}:{tag}",
detach=True,
ports={"5432/tcp": (host, port)},
environment={"POSTGRES_HOST_AUTH_METHOD": "trust"},
)
try:
start_time = time.time()
conn = None
while conn is None:
if start_time + timeout < time.time():
raise Exception(f"Initialization timeout, failed to initialize"
f" {image} container")
try:
conn = psycopg2.connect(f"dbname={dbname} user={user} "
f"host={host} port={port}")
except psycopg2.OperationalError:
time.sleep(0.10)
conn.close()
yield host, port, user, dbname, image, tag
finally:
cont.kill()
cont.remove()
async def rabbit(loop, rabbit_override_addr):
if rabbit_override_addr:
yield rabbit_override_addr.split(':')
return
tag = '3.7.1'
image = 'rabbitmq:{}'.format(tag)
host = '0.0.0.0'
timeout = 60
unused_tcp_port = get_free_port()
client = DockerClient(version='auto', **kwargs_from_env())
print('Stating rabbitmq %s on %s:%s' % (image, host, unused_tcp_port))
cont = client.containers.run(image, detach=True,
ports={'5672/tcp': ('0.0.0.0',
unused_tcp_port)})
try:
start_time = time.time()
conn = transport = None
while conn is None:
if start_time + timeout < time.time():
raise Exception("Initialization timeout, failed t o "
"initialize rabbitmq container")
try:
transport, conn = await aioamqp.connect(host, unused_tcp_port,
loop=loop)
except Exception:
time.sleep(.1)
await conn.close()
transport.close()
yield (host, unused_tcp_port)
finally:
cont.kill()
cont.remove()
def test_init(self, config):
SwarmInstaller(
config,
"docker-swarm",
docker_client=DockerClient()
)
def docker_client(self):
return DockerClient()
def delete_instance(docker_api_endpoint, service_name, container_id):
c = DockerClient(docker_api_endpoint)
c.stop(container_id)
c.remove_container(container_id)
import yaml
from docker.models.services import ServiceCollection
from docker.models.nodes import NodeCollection
from docker.client import DockerClient
client = DockerClient()
sc = ServiceCollection(client)
nc = NodeCollection(client)
class InfoGettingError(Exception):
pass
def get_task_instance_config(task_instance):
config_file = task_instance.dc_config.file
return yaml.safe_load(config_file)
def get_services2ports(task_instance):
services2ports = {}
config = get_task_instance_config(task_instance)
for service_name, service in config['services'].items():
if 'ports' in service:
services2ports[service_name] = [
int(e.split(':')[0]) for e in service['ports']
]
return services2ports
def get_services_name_prefix(task_instance):
def _docker(self):
kw = kwargs_from_env()
kw['tls'].verify = True
return DockerClient(**kw)
def get_docker_client():
"""
Try to fire up boot2docker and set any environmental variables
"""
# For Mac
try:
# Get boot2docker info (will fail if not Mac)
process = ['boot2docker', 'info']
p = subprocess.Popen(process, stdout=PIPE)
boot2docker_info = json.loads(p.communicate()[0])
# Defaults
docker_host = ''
docker_cert_path = ''
docker_tls_verify = ''
# Start the boot2docker VM if it is not already running
if boot2docker_info['State'] != "running":
print('Starting Boot2Docker VM:')
# Start up the Docker VM
process = ['boot2docker', 'start']
subprocess.call(process)
if ('DOCKER_HOST' not in os.environ) or (
'DOCKER_CERT_PATH' not in os.environ) or ('DOCKER_TLS_VERIFY'
not in os.environ):
# Get environmental variable values
process = ['boot2docker', 'shellinit']
p = subprocess.Popen(process, stdout=PIPE)
boot2docker_envs = p.communicate()[0].split()
for env in boot2docker_envs:
if 'DOCKER_HOST' in env:
docker_host = env.split('=')[1]
elif 'DOCKER_CERT_PATH' in env:
docker_cert_path = env.split('=')[1]
elif 'DOCKER_TLS_VERIFY' in env:
docker_tls_verify = env.split('=')[1]
# Set environmental variables
os.environ['DOCKER_TLS_VERIFY'] = docker_tls_verify
os.environ['DOCKER_HOST'] = docker_host
os.environ['DOCKER_CERT_PATH'] = docker_cert_path
else:
# Handle case when boot2docker is already running
docker_host = os.environ['DOCKER_HOST'].split('=')[1]
# Get the arguments form the environment
client_kwargs = kwargs_from_env(assert_hostname=False)
client_kwargs['version'] = MINIMUM_API_VERSION
# Find the right version of the API by creating a DockerClient with the minimum working version
# Then test to see if the Docker is running a later version than the minimum
# See: https://github.com/docker/docker-py/issues/439
version_client = DockerClient(**client_kwargs)
client_kwargs['version'] = get_api_version(
MAX_CLIENT_DOCKER_API_VERSION,
version_client.version()['ApiVersion'])
# Create Real Docker client
docker_client = DockerClient(**client_kwargs)
# Derive the host address only from string formatted: "tcp://<host>:<port>"
docker_client.host = docker_host.split(':')[1].strip('//')
return docker_client
# For Linux
except OSError:
# Find the right version of the API by creating a DockerClient with the minimum working version
# Then test to see if the Docker is running a later version than the minimum
# See: https://github.com/docker/docker-py/issues/439
version_client = DockerClient(base_url='unix://var/run/docker.sock',
version=MINIMUM_API_VERSION)
version = get_api_version(MAX_CLIENT_DOCKER_API_VERSION,
version_client.version()['ApiVersion'])
docker_client = DockerClient(base_url='unix://var/run/docker.sock',
version=version)
docker_client.host = DEFAULT_DOCKER_HOST
return docker_client
except:
raise
gateway_host = os.getenv("KIP_GATEWAY_HOST", "http://localhost:8888")
num_pullers = int(os.getenv("KIP_NUM_PULLERS", "2"))
num_retries = int(os.getenv("KIP_NUM_RETRIES", "3"))
interval = int(os.getenv("KIP_INTERVAL", "300"))
log_level = os.getenv("KIP_LOG_LEVEL", "INFO")
POLICY_IF_NOT_PRESENT = "IfNotPresent"
POLICY_ALYWAYS = "Always"
policies = (POLICY_IF_NOT_PRESENT, POLICY_ALYWAYS)
policy = os.getenv("KIP_PULL_POLICY", POLICY_IF_NOT_PRESENT)
#docker_client = DockerClient.from_env()
docker_client = DockerClient(base_url='unix://var/run/docker.sock')
logging.basicConfig(format='[%(levelname)1.1s %(asctime)s %(name)s.%(threadName)s] %(message)s')
def get_kernelspecs():
"""Fetches the set of kernelspecs from the gateway, returning a dict of configured kernel specs"""
end_point = '{}/api/kernelspecs'.format(gateway_host)
logger.info("Fetching kernelspecs from '{}' ...".format(end_point))
resp = requests.get(end_point)
if not resp.ok:
raise requests.exceptions.HTTPError('Gateway server response: {}'.format(resp.status_code))
return resp.json()
def fetch_image_names():
def get_docker_socket():
return DockerClient(
base_url='unix://var/run/docker.sock',
version='auto'
)