def test_container_lifecycles():
docker = Docker()
containers = yield from docker.containers.list(all=True)
for container in containers:
yield from container.delete(force=True)
yield from docker.pull("debian:jessie")
config = {
"Cmd": ["dmesg"],
"Image": "debian:jessie",
"AttachStdin": True,
"AttachStdout": True,
"AttachStderr": True,
"Tty": False,
"OpenStdin": False,
"StdinOnce": False,
}
for i in range(3):
container = yield from docker.containers.create(config=config)
assert container
containers = yield from docker.containers.list(all=True)
assert len(containers) == 3
f_container = containers[0]
yield from f_container.start(config)
info = yield from f_container.show()
for container in containers:
yield from container.delete(force=True)
def test_events():
docker = Docker()
queue = yield from docker.events.query()
yield from docker.pull("debian:jessie")
config = {
"Cmd": ["sh"],
"Image": "debian:jessie",
}
container = yield from docker.containers.create_or_replace(config=config,
name='testing')
#print("put archive response:", result)
yield from container.start(config)
i = yield from queue.__aiter__()
while True:
try:
event = yield from asyncio.wait_for(i.__anext__(), 2)
except StopAsyncIteration:
assert False, "Event queue should not terminate"
else:
if event.get('status', None) == 'start':
if event['id'] == container._id:
yield from container.stop()
break
yield from container.delete(force=True)
def test_wait_timeout():
docker = Docker()
yield from docker.pull("debian:jessie")
config = {
"Cmd": ["sh"],
"Image": "debian:jessie",
"AttachStdin": True,
"AttachStdout": True,
"AttachStderr": True,
"Tty": False,
"OpenStdin": True,
"StdinOnce": True,
}
container = yield from docker.containers.create_or_replace(config=config, name="testing")
yield from container.start()
print("waiting for container to stop")
try:
yield from container.wait(timeout=1)
except TimeoutError:
pass
else:
assert "TimeoutError should have occured"
print("removing container")
yield from container.delete(force=True)
def test_stdio_stdin():
docker = Docker()
yield from docker.pull("debian:jessie")
config = {
"Cmd": ["sh"],
"Image": "debian:jessie",
"AttachStdin": True,
"AttachStdout": True,
"AttachStderr": True,
"Tty": False,
"OpenStdin": True,
"StdinOnce": True,
}
container = yield from docker.containers.create_or_replace(config=config,
name='testing')
yield from container.start(config)
ws = yield from container.websocket(stdin=True, stdout=True, stream=True)
ws.send_str('echo hello world\n')
resp = yield from ws.receive()
assert resp.data == "hello world\n"
ws.close()
output = yield from container.log(stdout=True)
print("log output:", output)
assert output == "hello world\n"
#TODO ensure container stopped on its own because stdin was closed
print("removing container")
yield from container.delete(force=True)
def test_container_lifecycles():
docker = Docker()
containers = yield from docker.containers.list(all=True)
for container in containers:
yield from container.delete(force=True)
yield from docker.pull("debian:jessie")
config = {
"Cmd": ["dmesg"],
"Image": "debian:jessie",
"AttachStdin": True,
"AttachStdout": True,
"AttachStderr": True,
"Tty": False,
"OpenStdin": False,
"StdinOnce": False,
}
for i in range(3):
container = yield from docker.containers.create(config=config)
assert container
containers = yield from docker.containers.list(all=True)
assert len(containers) == 3
f_container = containers[0]
yield from f_container.start()
info = yield from f_container.show()
for container in containers:
yield from container.delete(force=True)
def test_stdio_stdin():
docker = Docker()
yield from docker.pull("debian:jessie")
config = {
"Cmd": ["sh"],
"Image": "debian:jessie",
"AttachStdin": True,
"AttachStdout": True,
"AttachStderr": True,
"Tty": False,
"OpenStdin": True,
"StdinOnce": True,
}
container = yield from docker.containers.create_or_replace(config=config, name="testing")
yield from container.start()
ws = yield from container.websocket(stdin=True, stdout=True, stream=True)
ws.send_str("echo hello world\n")
resp = yield from ws.receive()
assert resp.data == "hello world\n"
ws.close()
output = yield from container.log(stdout=True)
print("log output:", output)
assert output == "hello world\n"
# TODO ensure container stopped on its own because stdin was closed
print("removing container")
yield from container.delete(force=True)
def test_events():
docker = Docker()
queue = yield from docker.events.query()
yield from docker.pull("debian:jessie")
config = {"Cmd": ["sh"], "Image": "debian:jessie"}
container = yield from docker.containers.create_or_replace(config=config, name="testing")
# print("put archive response:", result)
yield from container.start()
i = yield from queue.__aiter__()
while True:
try:
event = yield from asyncio.wait_for(i.__anext__(), 2)
except StopAsyncIteration:
assert False, "Event queue should not terminate"
else:
if event.get("status", None) == "start":
if event["id"] == container._id:
yield from container.stop()
break
yield from container.delete(force=True)
def test_wait_timeout():
docker = Docker()
yield from docker.pull("debian:jessie")
config = {
"Cmd": ["sh"],
"Image": "debian:jessie",
"AttachStdin": True,
"AttachStdout": True,
"AttachStderr": True,
"Tty": False,
"OpenStdin": True,
"StdinOnce": True,
}
container = yield from docker.containers.create_or_replace(config=config,
name='testing')
yield from container.start(config)
print("waiting for container to stop")
try:
yield from container.wait(timeout=1)
except TimeoutError:
pass
else:
assert "TimeoutError should have occured"
print("removing container")
yield from container.delete(force=True)
async def cleanup_service(msg):
global UPTIME_SNAPSHOTS
data = msg.data.decode()
docker = Docker()
for container in (await docker.containers.list()):
if f"/{SERVICE_NAME}-{data}" in container._container['Names']:
print(f"Remove container {SERVICE_NAME}-{data}")
await container.delete(force=True)
await docker.close()
# wait container is remove
remove = 0
while remove > 10:
try:
await asyncio.sleep(1)
snapshot = zfs.ZFSVolume(
f"{ZPOOL_NAME}/from-snapshot-{SERVICE_NAME}-{data}")
await snapshot.destroy()
remove = 11
except Exception as e:
remove = remove + 1
print(f"Cleanup snapshot {snapshot.name}")
await nc.publish(f"{SERVICE_NAME}-cleanup-status", bytes(msg, "utf-8"))
del UPTIME_SNAPSHOTS[f"{ZPOOL_NAME}/from-snapshot-{SERVICE_NAME}-{data}"]
await store_services()
return
async def pull_image(message=False):
global SERVICE_IMAGE
if message:
print(f'First pull image {SERVICE_IMAGE}')
docker = Docker()
await docker.pull(SERVICE_IMAGE)
await docker.close()
async def test_connect_envvar(monkeypatch):
monkeypatch.setenv("DOCKER_HOST", "unix:///var/run/does-not-exist-docker.sock")
docker = Docker()
assert isinstance(docker.connector, aiohttp.connector.UnixConnector)
assert docker.docker_host == "unix://localhost"
with pytest.raises(aiohttp.ClientOSError):
await docker.containers.list()
await docker.close()
monkeypatch.setenv("DOCKER_HOST", "http://localhost:9999")
docker = Docker()
assert isinstance(docker.connector, aiohttp.TCPConnector)
assert docker.docker_host == "http://localhost:9999"
with pytest.raises(aiohttp.ClientOSError):
await docker.containers.list()
await docker.close()
def docker(event_loop):
docker = Docker()
yield docker
async def _finalize():
await docker.close()
event_loop.run_until_complete(_finalize())
def test_put_archive():
docker = Docker()
yield from docker.pull("debian:jessie")
config = {
"Cmd": ["cat", "/tmp/bar/foo.txt"],
#"Cmd":["ls", "-l", "/tmp"],
"Image": "debian:jessie",
"AttachStdin": True,
"AttachStdout": True,
"AttachStderr": True,
"Tty": False,
"OpenStdin": False
}
file_data = b"hello world"
file_like_object = io.BytesIO()
tar = tarfile.open(fileobj=file_like_object, mode='w')
info = tarfile.TarInfo(name='bar')
info.type = tarfile.DIRTYPE
info.mode = 493 #755 oct -> dec
info.mtime = time.time()
tar.addfile(tarinfo=info)
tarinfo = tarfile.TarInfo(name="bar/foo.txt")
tarinfo.size = len(file_data)
tar.addfile(tarinfo, io.BytesIO(file_data))
tar.list()
tar.close()
container = yield from docker.containers.create_or_replace(config=config,
name='testing')
result = yield from container.put_archive(path='/tmp',
data=file_like_object.getvalue())
#print("put archive response:", result)
yield from container.start(config)
yield from container.wait(timeout=1)
output = yield from container.log(stdout=True, stderr=True)
print("log output:", output)
assert output == "hello world\n"
print("removing container")
yield from container.delete(force=True)
def test_closed_unix_socket():
docker = Docker('unix://var/run/does-not-exist-docker.sock')
try:
info = yield from docker.containers.list()
except aiohttp.errors.ClientOSError as error:
assert "No such file or directory" in str(error)
else:
assert False, "docker commands should not work with invalid hosts"
async def test_ssl_context(monkeypatch):
cert_dir = pathlib.Path(__file__).parent / "certs"
monkeypatch.setenv("DOCKER_HOST", "tcp://127.0.0.1:3456")
monkeypatch.setenv("DOCKER_TLS_VERIFY", "1")
monkeypatch.setenv("DOCKER_CERT_PATH", str(cert_dir))
docker = Docker()
assert docker.connector._ssl
await docker.close()
def test_put_archive():
docker = Docker()
yield from docker.pull("debian:jessie")
config = {
"Cmd": ["cat", "/tmp/bar/foo.txt"],
# "Cmd":["ls", "-l", "/tmp"],
"Image": "debian:jessie",
"AttachStdin": True,
"AttachStdout": True,
"AttachStderr": True,
"Tty": False,
"OpenStdin": False,
}
file_data = b"hello world"
file_like_object = io.BytesIO()
tar = tarfile.open(fileobj=file_like_object, mode="w")
info = tarfile.TarInfo(name="bar")
info.type = tarfile.DIRTYPE
info.mode = 493 # 755 oct -> dec
info.mtime = time.time()
tar.addfile(tarinfo=info)
tarinfo = tarfile.TarInfo(name="bar/foo.txt")
tarinfo.size = len(file_data)
tar.addfile(tarinfo, io.BytesIO(file_data))
tar.list()
tar.close()
container = yield from docker.containers.create_or_replace(config=config, name="testing")
result = yield from container.put_archive(path="/tmp", data=file_like_object.getvalue())
# print("put archive response:", result)
yield from container.start()
yield from container.wait(timeout=1)
output = yield from container.log(stdout=True, stderr=True)
print("log output:", output)
assert output == "hello world\n"
print("removing container")
yield from container.delete(force=True)
def test_port():
docker = Docker()
yield from docker.pull("redis:latest")
config = {"Image": "redis:latest", "PublishAllPorts": True}
container = yield from docker.containers.create_or_replace(config=config, name="testing")
# print("put archive response:", result)
yield from container.start()
port = yield from container.port(6379)
print(container._container.get("NetworkSettings"))
assert port
yield from container.delete(force=True)
async def _pull():
docker = Docker()
required_images = ["python:latest", "python:3.6.1", "python:3.7.4"]
for img in required_images:
try:
await docker.images.inspect(img)
except DockerError as e:
assert e.status == 404
print('Pulling "{img}" for the testing session...'.format(img=img))
await docker.pull(img)
await docker.close()
def docker(event_loop, testing_images):
kwargs = {}
if "DOCKER_VERSION" in ENV:
kwargs["api_version"] = _api_versions[ENV["DOCKER_VERSION"]]
docker = Docker(**kwargs)
yield docker
async def _finalize():
await docker.close()
event_loop.run_until_complete(_finalize())
async def _clean():
docker = Docker()
images = await docker.images.list()
for img in images:
if img['RepoTags'] is None:
continue
try:
if img['RepoTags'][0].startswith('aiodocker-'):
print('Deleting image id: {0}'.format(img['Id']))
await docker.images.delete(img['Id'], force=True)
except DockerError as e:
traceback.print_exc()
await docker.close()
def test_port():
docker = Docker()
yield from docker.pull("redis:latest")
config = {
"Image": "redis:latest",
"PublishAllPorts": True,
}
container = yield from docker.containers.create_or_replace(config=config,
name='testing')
#print("put archive response:", result)
yield from container.start(config)
port = yield from container.port(6379)
print(container._container.get("NetworkSettings"))
assert port
yield from container.delete(force=True)
async def _pull():
docker = Docker()
required_images = [image_name]
if image_name != "python:latest":
required_images.append("python:latest")
for img in required_images:
try:
await docker.images.inspect(img)
except DockerError as e:
assert e.status == 404
print('Pulling "{img}" for the testing session...'.format(img=img))
await docker.pull(img)
await docker.close()
async def _clean():
docker = Docker()
images = await docker.images.list()
for img in images:
if img["RepoTags"] is None:
continue
try:
if img["RepoTags"][0].startswith("aiodocker-"):
print("Deleting image id: {0}".format(img["Id"]))
await docker.images.delete(img["Id"], force=True)
except DockerError:
traceback.print_exc()
await docker.close()
async def test_autodetect_host(monkeypatch):
docker = Docker()
if 'DOCKER_HOST' in os.environ:
if (os.environ['DOCKER_HOST'].startswith('http://')
or os.environ['DOCKER_HOST'].startswith('https://')
or os.environ['DOCKER_HOST'].startswith('tcp://')):
assert docker.docker_host == os.environ['DOCKER_HOST']
else:
assert docker.docker_host == 'unix://localhost'
else:
# assuming that docker daemon is installed locally.
assert docker.docker_host is not None
await docker.close()
async def initialize():
nonlocal docker, agent, server, events
nonlocal hb_task, stats_task, timer_task
nonlocal monitor_handle_task, monitor_fetch_task
args.inst_id = await utils.get_instance_id()
args.inst_type = await utils.get_instance_type()
if not args.agent_ip:
args.agent_ip = await utils.get_instance_ip()
log.info(f'myself: {args.inst_id} ({args.inst_type}), '
f'ip: {args.agent_ip}')
log.info('using gateway event server at ' f'tcp://{args.event_addr}')
# Connect to the events server.
event_addr = f'tcp://{args.event_addr}'
try:
with timeout(5.0):
events = await aiozmq.rpc.connect_rpc(connect=event_addr)
events.transport.setsockopt(zmq.LINGER, 50)
await events.call.dispatch('instance_started', args.inst_id)
except asyncio.TimeoutError:
events.close()
await events.wait_closed()
log.critical('cannot connect to the manager.')
raise SystemExit(1)
# Initialize Docker
docker = Docker()
docker_version = await docker.version()
log.info('running with Docker {0} with API {1}'.format(
docker_version['Version'], docker_version['ApiVersion']))
# Start RPC server.
agent_addr = 'tcp://*:{}'.format(args.agent_port)
agent = AgentRPCServer(docker,
args,
events,
ddagent,
sentry,
loop=loop)
await agent.init()
server = await aiozmq.rpc.serve_rpc(agent, bind=agent_addr)
server.transport.setsockopt(zmq.LINGER, 200)
log.info('serving at {0}'.format(agent_addr))
# Send the first heartbeat.
hb_task = loop.create_task(heartbeat_timer(agent))
stats_task = loop.create_task(stats_timer(agent))
timer_task = loop.create_task(cleanup_timer(agent))
monitor_fetch_task = loop.create_task(agent.fetch_docker_events())
monitor_handle_task = loop.create_task(agent.monitor())
await asyncio.sleep(0.01)
async def _pull():
docker = Docker()
required_images = [
'alpine:latest', 'redis:latest', 'python:3.6.1-alpine'
]
for img in required_images:
try:
await docker.images.get(img)
except DockerError as e:
assert e.status == 404
print('Pulling "{img}" for the testing session...'
.format(img=img))
await docker.pull(img)
await docker.close()
async def test_autodetect_host(monkeypatch):
docker = Docker()
if "DOCKER_HOST" in os.environ:
if (os.environ["DOCKER_HOST"].startswith("http://")
or os.environ["DOCKER_HOST"].startswith("https://")
or os.environ["DOCKER_HOST"].startswith("tcp://")):
assert docker.docker_host == os.environ["DOCKER_HOST"]
else:
assert docker.docker_host in [
"unix://localhost", "npipe://localhost"
]
else:
# assuming that docker daemon is installed locally.
assert docker.docker_host is not None
await docker.close()
async def remove_sleep_docker(snapshot):
docker = Docker()
for container in (await docker.containers.list()):
tmp_container_name = re.sub(rf"^({ZPOOL_NAME}/from-snapshot-)", "/",
snapshot.name)
if tmp_container_name in container._container['Names']:
print(f"Drop container {tmp_container_name} ({container._id})")
await container.delete(force=True)
await docker.close()
print(f"Remove {snapshot.name}")
try:
await snapshot.destroy()
except Exception as e:
print(f"Can't remove {snapshot.name}")
return
async def _create_base_image(labels, tag):
dockerfile = """
FROM alpine
CMD while true; do sleep 10; done
"""
f = BytesIO(dockerfile.encode("utf-8"))
tar_obj = utils.mktar_from_dockerfile(f)
# build docker base image
docker = Docker()
base_docker_image = await docker.images.build(fileobj=tar_obj,
encoding="gzip",
rm=True,
labels=labels,
tag=tag)
await docker.close()
return base_docker_image[0]
async def connect_scheduler():
scheduler = AsyncIOScheduler(timezone="UTC")
scheduler.add_job(check_create_data_snapshot, 'interval', seconds=20)
scheduler.add_job(destroy_data_snapshot, 'interval', seconds=20)
scheduler.add_job(store_services, 'interval', seconds=1)
scheduler.add_job(test_data_snaphost, 'interval', seconds=5)
scheduler.add_job(remove_sleep, 'interval', seconds=60)
scheduler.add_job(pull_image,
'interval',
seconds=int(os.getenv('PULL_IMAGE', 600)))
scheduler.start()
# add docker to japronto app
docker = Docker()
app.extend_request(lambda x: docker, name='docker', property=True)
print(f"=> created and started container {container._id[:12]}")
while True:
event = await subscriber.get()
if event is None:
break
print(f"event: {event!r}")
# Demonstrate simple event-driven container mgmt.
if event['Actor']['ID'] == container._id:
if event['Action'] == 'start':
await container.stop()
print(f"=> killed {container._id[:12]}")
elif event['Action'] == 'stop':
await container.delete(force=True)
print(f"=> deleted {container._id[:12]}")
elif event['Action'] == 'destroy':
print('=> done with this container!')
break
if __name__ == '__main__':
loop = asyncio.get_event_loop()
docker = Docker()
try:
# do our stuffs.
loop.run_until_complete(demo(docker))
finally:
loop.run_until_complete(docker.close())
loop.close()
async def _make_docker():
return Docker(**kwargs)
async def test_connect_with_connector(monkeypatch):
connector = aiohttp.BaseConnector()
docker = Docker(connector=connector)
assert docker.connector == connector
await docker.close()
async def test_connect_invalid_unix_socket():
docker = Docker('unix:///var/run/does-not-exist-docker.sock')
assert isinstance(docker.connector, aiohttp.connector.UnixConnector)
with pytest.raises(aiohttp.ClientOSError):
await docker.containers.list()
await docker.close()
def test_image_push():
docker = Docker()
yield from docker.pull("debian")
yield from docker.images.tag("debian", repo=os.environ["DOCKER_REGISTRY"] + "/debian")
import aiopg.sa
import asyncio
import aiohttp
from aiodocker.docker import Docker
import json
import humanize.time
from sqlalchemy import select, join, desc
from moxie.server import MoxieApp
from moxie.models import Job, Maintainer, Run
from moxie.core import DATABASE_URL
from aiocore import Service
app = MoxieApp()
docker = Docker()
@asyncio.coroutine
def get_logs(conn, job, limit=10):
runs = yield from conn.execute(
select([Run.__table__]).where(Run.job_id == job.id).order_by(
desc(Run.start_time)).limit(limit))
return runs
@asyncio.coroutine
def get_jobs(conn, jobs, limit=10):
ret = []
for job in jobs:
ret.append((job, (yield from get_logs(conn, job, limit=limit))))
async def datanode_first_up():
docker = Docker()
const_find_service_dirs = find_service_dirs()
for container in (await docker.containers.list()):
await container_by_data(container, const_find_service_dirs)
await docker.close()
while True:
event = await subscriber.get()
if event is None:
break
for key, value in event.items():
print(key,':', value)
# Demonstrate simple event-driven container mgmt.
if event['Actor']['ID'] == container._id:
if event['Action'] == 'start':
await container.stop()
print("=> killed {}".format(container._id[:12]))
elif event['Action'] == 'stop':
await container.delete(force=True)
print("=> deleted {}".format(container._id[:12]))
elif event['Action'] == 'destroy':
print('=> done with this container!')
break
if __name__ == '__main__':
loop = asyncio.get_event_loop()
docker = Docker()
try:
# do our stuffs.
loop.run_until_complete(demo(docker))
finally:
loop.run_until_complete(docker.close())
loop.close()
