def test_container_lifecycles(): docker = Docker() containers = yield from docker.containers.list(all=True) for container in containers: yield from container.delete(force=True) yield from docker.pull("debian:jessie") config = { "Cmd": ["dmesg"], "Image": "debian:jessie", "AttachStdin": True, "AttachStdout": True, "AttachStderr": True, "Tty": False, "OpenStdin": False, "StdinOnce": False, } for i in range(3): container = yield from docker.containers.create(config=config) assert container containers = yield from docker.containers.list(all=True) assert len(containers) == 3 f_container = containers[0] yield from f_container.start(config) info = yield from f_container.show() for container in containers: yield from container.delete(force=True)
def test_events(): docker = Docker() queue = yield from docker.events.query() yield from docker.pull("debian:jessie") config = { "Cmd": ["sh"], "Image": "debian:jessie", } container = yield from docker.containers.create_or_replace(config=config, name='testing') #print("put archive response:", result) yield from container.start(config) i = yield from queue.__aiter__() while True: try: event = yield from asyncio.wait_for(i.__anext__(), 2) except StopAsyncIteration: assert False, "Event queue should not terminate" else: if event.get('status', None) == 'start': if event['id'] == container._id: yield from container.stop() break yield from container.delete(force=True)
def test_wait_timeout(): docker = Docker() yield from docker.pull("debian:jessie") config = { "Cmd": ["sh"], "Image": "debian:jessie", "AttachStdin": True, "AttachStdout": True, "AttachStderr": True, "Tty": False, "OpenStdin": True, "StdinOnce": True, } container = yield from docker.containers.create_or_replace(config=config, name="testing") yield from container.start() print("waiting for container to stop") try: yield from container.wait(timeout=1) except TimeoutError: pass else: assert "TimeoutError should have occured" print("removing container") yield from container.delete(force=True)
def test_stdio_stdin(): docker = Docker() yield from docker.pull("debian:jessie") config = { "Cmd": ["sh"], "Image": "debian:jessie", "AttachStdin": True, "AttachStdout": True, "AttachStderr": True, "Tty": False, "OpenStdin": True, "StdinOnce": True, } container = yield from docker.containers.create_or_replace(config=config, name='testing') yield from container.start(config) ws = yield from container.websocket(stdin=True, stdout=True, stream=True) ws.send_str('echo hello world\n') resp = yield from ws.receive() assert resp.data == "hello world\n" ws.close() output = yield from container.log(stdout=True) print("log output:", output) assert output == "hello world\n" #TODO ensure container stopped on its own because stdin was closed print("removing container") yield from container.delete(force=True)
def test_container_lifecycles(): docker = Docker() containers = yield from docker.containers.list(all=True) for container in containers: yield from container.delete(force=True) yield from docker.pull("debian:jessie") config = { "Cmd": ["dmesg"], "Image": "debian:jessie", "AttachStdin": True, "AttachStdout": True, "AttachStderr": True, "Tty": False, "OpenStdin": False, "StdinOnce": False, } for i in range(3): container = yield from docker.containers.create(config=config) assert container containers = yield from docker.containers.list(all=True) assert len(containers) == 3 f_container = containers[0] yield from f_container.start() info = yield from f_container.show() for container in containers: yield from container.delete(force=True)
def test_stdio_stdin(): docker = Docker() yield from docker.pull("debian:jessie") config = { "Cmd": ["sh"], "Image": "debian:jessie", "AttachStdin": True, "AttachStdout": True, "AttachStderr": True, "Tty": False, "OpenStdin": True, "StdinOnce": True, } container = yield from docker.containers.create_or_replace(config=config, name="testing") yield from container.start() ws = yield from container.websocket(stdin=True, stdout=True, stream=True) ws.send_str("echo hello world\n") resp = yield from ws.receive() assert resp.data == "hello world\n" ws.close() output = yield from container.log(stdout=True) print("log output:", output) assert output == "hello world\n" # TODO ensure container stopped on its own because stdin was closed print("removing container") yield from container.delete(force=True)
def test_events(): docker = Docker() queue = yield from docker.events.query() yield from docker.pull("debian:jessie") config = {"Cmd": ["sh"], "Image": "debian:jessie"} container = yield from docker.containers.create_or_replace(config=config, name="testing") # print("put archive response:", result) yield from container.start() i = yield from queue.__aiter__() while True: try: event = yield from asyncio.wait_for(i.__anext__(), 2) except StopAsyncIteration: assert False, "Event queue should not terminate" else: if event.get("status", None) == "start": if event["id"] == container._id: yield from container.stop() break yield from container.delete(force=True)
def test_wait_timeout(): docker = Docker() yield from docker.pull("debian:jessie") config = { "Cmd": ["sh"], "Image": "debian:jessie", "AttachStdin": True, "AttachStdout": True, "AttachStderr": True, "Tty": False, "OpenStdin": True, "StdinOnce": True, } container = yield from docker.containers.create_or_replace(config=config, name='testing') yield from container.start(config) print("waiting for container to stop") try: yield from container.wait(timeout=1) except TimeoutError: pass else: assert "TimeoutError should have occured" print("removing container") yield from container.delete(force=True)
async def cleanup_service(msg): global UPTIME_SNAPSHOTS data = msg.data.decode() docker = Docker() for container in (await docker.containers.list()): if f"/{SERVICE_NAME}-{data}" in container._container['Names']: print(f"Remove container {SERVICE_NAME}-{data}") await container.delete(force=True) await docker.close() # wait container is remove remove = 0 while remove > 10: try: await asyncio.sleep(1) snapshot = zfs.ZFSVolume( f"{ZPOOL_NAME}/from-snapshot-{SERVICE_NAME}-{data}") await snapshot.destroy() remove = 11 except Exception as e: remove = remove + 1 print(f"Cleanup snapshot {snapshot.name}") await nc.publish(f"{SERVICE_NAME}-cleanup-status", bytes(msg, "utf-8")) del UPTIME_SNAPSHOTS[f"{ZPOOL_NAME}/from-snapshot-{SERVICE_NAME}-{data}"] await store_services() return
async def pull_image(message=False): global SERVICE_IMAGE if message: print(f'First pull image {SERVICE_IMAGE}') docker = Docker() await docker.pull(SERVICE_IMAGE) await docker.close()
async def test_connect_envvar(monkeypatch): monkeypatch.setenv("DOCKER_HOST", "unix:///var/run/does-not-exist-docker.sock") docker = Docker() assert isinstance(docker.connector, aiohttp.connector.UnixConnector) assert docker.docker_host == "unix://localhost" with pytest.raises(aiohttp.ClientOSError): await docker.containers.list() await docker.close() monkeypatch.setenv("DOCKER_HOST", "http://localhost:9999") docker = Docker() assert isinstance(docker.connector, aiohttp.TCPConnector) assert docker.docker_host == "http://localhost:9999" with pytest.raises(aiohttp.ClientOSError): await docker.containers.list() await docker.close()
def docker(event_loop): docker = Docker() yield docker async def _finalize(): await docker.close() event_loop.run_until_complete(_finalize())
def test_put_archive(): docker = Docker() yield from docker.pull("debian:jessie") config = { "Cmd": ["cat", "/tmp/bar/foo.txt"], #"Cmd":["ls", "-l", "/tmp"], "Image": "debian:jessie", "AttachStdin": True, "AttachStdout": True, "AttachStderr": True, "Tty": False, "OpenStdin": False } file_data = b"hello world" file_like_object = io.BytesIO() tar = tarfile.open(fileobj=file_like_object, mode='w') info = tarfile.TarInfo(name='bar') info.type = tarfile.DIRTYPE info.mode = 493 #755 oct -> dec info.mtime = time.time() tar.addfile(tarinfo=info) tarinfo = tarfile.TarInfo(name="bar/foo.txt") tarinfo.size = len(file_data) tar.addfile(tarinfo, io.BytesIO(file_data)) tar.list() tar.close() container = yield from docker.containers.create_or_replace(config=config, name='testing') result = yield from container.put_archive(path='/tmp', data=file_like_object.getvalue()) #print("put archive response:", result) yield from container.start(config) yield from container.wait(timeout=1) output = yield from container.log(stdout=True, stderr=True) print("log output:", output) assert output == "hello world\n" print("removing container") yield from container.delete(force=True)
def test_closed_unix_socket(): docker = Docker('unix://var/run/does-not-exist-docker.sock') try: info = yield from docker.containers.list() except aiohttp.errors.ClientOSError as error: assert "No such file or directory" in str(error) else: assert False, "docker commands should not work with invalid hosts"
async def test_ssl_context(monkeypatch): cert_dir = pathlib.Path(__file__).parent / "certs" monkeypatch.setenv("DOCKER_HOST", "tcp://127.0.0.1:3456") monkeypatch.setenv("DOCKER_TLS_VERIFY", "1") monkeypatch.setenv("DOCKER_CERT_PATH", str(cert_dir)) docker = Docker() assert docker.connector._ssl await docker.close()
def test_put_archive(): docker = Docker() yield from docker.pull("debian:jessie") config = { "Cmd": ["cat", "/tmp/bar/foo.txt"], # "Cmd":["ls", "-l", "/tmp"], "Image": "debian:jessie", "AttachStdin": True, "AttachStdout": True, "AttachStderr": True, "Tty": False, "OpenStdin": False, } file_data = b"hello world" file_like_object = io.BytesIO() tar = tarfile.open(fileobj=file_like_object, mode="w") info = tarfile.TarInfo(name="bar") info.type = tarfile.DIRTYPE info.mode = 493 # 755 oct -> dec info.mtime = time.time() tar.addfile(tarinfo=info) tarinfo = tarfile.TarInfo(name="bar/foo.txt") tarinfo.size = len(file_data) tar.addfile(tarinfo, io.BytesIO(file_data)) tar.list() tar.close() container = yield from docker.containers.create_or_replace(config=config, name="testing") result = yield from container.put_archive(path="/tmp", data=file_like_object.getvalue()) # print("put archive response:", result) yield from container.start() yield from container.wait(timeout=1) output = yield from container.log(stdout=True, stderr=True) print("log output:", output) assert output == "hello world\n" print("removing container") yield from container.delete(force=True)
def test_port(): docker = Docker() yield from docker.pull("redis:latest") config = {"Image": "redis:latest", "PublishAllPorts": True} container = yield from docker.containers.create_or_replace(config=config, name="testing") # print("put archive response:", result) yield from container.start() port = yield from container.port(6379) print(container._container.get("NetworkSettings")) assert port yield from container.delete(force=True)
async def _pull(): docker = Docker() required_images = ["python:latest", "python:3.6.1", "python:3.7.4"] for img in required_images: try: await docker.images.inspect(img) except DockerError as e: assert e.status == 404 print('Pulling "{img}" for the testing session...'.format(img=img)) await docker.pull(img) await docker.close()
def docker(event_loop, testing_images): kwargs = {} if "DOCKER_VERSION" in ENV: kwargs["api_version"] = _api_versions[ENV["DOCKER_VERSION"]] docker = Docker(**kwargs) yield docker async def _finalize(): await docker.close() event_loop.run_until_complete(_finalize())
async def _clean(): docker = Docker() images = await docker.images.list() for img in images: if img['RepoTags'] is None: continue try: if img['RepoTags'][0].startswith('aiodocker-'): print('Deleting image id: {0}'.format(img['Id'])) await docker.images.delete(img['Id'], force=True) except DockerError as e: traceback.print_exc() await docker.close()
def test_port(): docker = Docker() yield from docker.pull("redis:latest") config = { "Image": "redis:latest", "PublishAllPorts": True, } container = yield from docker.containers.create_or_replace(config=config, name='testing') #print("put archive response:", result) yield from container.start(config) port = yield from container.port(6379) print(container._container.get("NetworkSettings")) assert port yield from container.delete(force=True)
async def _pull(): docker = Docker() required_images = [image_name] if image_name != "python:latest": required_images.append("python:latest") for img in required_images: try: await docker.images.inspect(img) except DockerError as e: assert e.status == 404 print('Pulling "{img}" for the testing session...'.format(img=img)) await docker.pull(img) await docker.close()
async def _clean(): docker = Docker() images = await docker.images.list() for img in images: if img["RepoTags"] is None: continue try: if img["RepoTags"][0].startswith("aiodocker-"): print("Deleting image id: {0}".format(img["Id"])) await docker.images.delete(img["Id"], force=True) except DockerError: traceback.print_exc() await docker.close()
async def test_autodetect_host(monkeypatch): docker = Docker() if 'DOCKER_HOST' in os.environ: if (os.environ['DOCKER_HOST'].startswith('http://') or os.environ['DOCKER_HOST'].startswith('https://') or os.environ['DOCKER_HOST'].startswith('tcp://')): assert docker.docker_host == os.environ['DOCKER_HOST'] else: assert docker.docker_host == 'unix://localhost' else: # assuming that docker daemon is installed locally. assert docker.docker_host is not None await docker.close()
async def initialize(): nonlocal docker, agent, server, events nonlocal hb_task, stats_task, timer_task nonlocal monitor_handle_task, monitor_fetch_task args.inst_id = await utils.get_instance_id() args.inst_type = await utils.get_instance_type() if not args.agent_ip: args.agent_ip = await utils.get_instance_ip() log.info(f'myself: {args.inst_id} ({args.inst_type}), ' f'ip: {args.agent_ip}') log.info('using gateway event server at ' f'tcp://{args.event_addr}') # Connect to the events server. event_addr = f'tcp://{args.event_addr}' try: with timeout(5.0): events = await aiozmq.rpc.connect_rpc(connect=event_addr) events.transport.setsockopt(zmq.LINGER, 50) await events.call.dispatch('instance_started', args.inst_id) except asyncio.TimeoutError: events.close() await events.wait_closed() log.critical('cannot connect to the manager.') raise SystemExit(1) # Initialize Docker docker = Docker() docker_version = await docker.version() log.info('running with Docker {0} with API {1}'.format( docker_version['Version'], docker_version['ApiVersion'])) # Start RPC server. agent_addr = 'tcp://*:{}'.format(args.agent_port) agent = AgentRPCServer(docker, args, events, ddagent, sentry, loop=loop) await agent.init() server = await aiozmq.rpc.serve_rpc(agent, bind=agent_addr) server.transport.setsockopt(zmq.LINGER, 200) log.info('serving at {0}'.format(agent_addr)) # Send the first heartbeat. hb_task = loop.create_task(heartbeat_timer(agent)) stats_task = loop.create_task(stats_timer(agent)) timer_task = loop.create_task(cleanup_timer(agent)) monitor_fetch_task = loop.create_task(agent.fetch_docker_events()) monitor_handle_task = loop.create_task(agent.monitor()) await asyncio.sleep(0.01)
async def _pull(): docker = Docker() required_images = [ 'alpine:latest', 'redis:latest', 'python:3.6.1-alpine' ] for img in required_images: try: await docker.images.get(img) except DockerError as e: assert e.status == 404 print('Pulling "{img}" for the testing session...' .format(img=img)) await docker.pull(img) await docker.close()
async def test_autodetect_host(monkeypatch): docker = Docker() if "DOCKER_HOST" in os.environ: if (os.environ["DOCKER_HOST"].startswith("http://") or os.environ["DOCKER_HOST"].startswith("https://") or os.environ["DOCKER_HOST"].startswith("tcp://")): assert docker.docker_host == os.environ["DOCKER_HOST"] else: assert docker.docker_host in [ "unix://localhost", "npipe://localhost" ] else: # assuming that docker daemon is installed locally. assert docker.docker_host is not None await docker.close()
async def remove_sleep_docker(snapshot): docker = Docker() for container in (await docker.containers.list()): tmp_container_name = re.sub(rf"^({ZPOOL_NAME}/from-snapshot-)", "/", snapshot.name) if tmp_container_name in container._container['Names']: print(f"Drop container {tmp_container_name} ({container._id})") await container.delete(force=True) await docker.close() print(f"Remove {snapshot.name}") try: await snapshot.destroy() except Exception as e: print(f"Can't remove {snapshot.name}") return
async def _create_base_image(labels, tag): dockerfile = """ FROM alpine CMD while true; do sleep 10; done """ f = BytesIO(dockerfile.encode("utf-8")) tar_obj = utils.mktar_from_dockerfile(f) # build docker base image docker = Docker() base_docker_image = await docker.images.build(fileobj=tar_obj, encoding="gzip", rm=True, labels=labels, tag=tag) await docker.close() return base_docker_image[0]
async def connect_scheduler(): scheduler = AsyncIOScheduler(timezone="UTC") scheduler.add_job(check_create_data_snapshot, 'interval', seconds=20) scheduler.add_job(destroy_data_snapshot, 'interval', seconds=20) scheduler.add_job(store_services, 'interval', seconds=1) scheduler.add_job(test_data_snaphost, 'interval', seconds=5) scheduler.add_job(remove_sleep, 'interval', seconds=60) scheduler.add_job(pull_image, 'interval', seconds=int(os.getenv('PULL_IMAGE', 600))) scheduler.start() # add docker to japronto app docker = Docker() app.extend_request(lambda x: docker, name='docker', property=True)
print(f"=> created and started container {container._id[:12]}") while True: event = await subscriber.get() if event is None: break print(f"event: {event!r}") # Demonstrate simple event-driven container mgmt. if event['Actor']['ID'] == container._id: if event['Action'] == 'start': await container.stop() print(f"=> killed {container._id[:12]}") elif event['Action'] == 'stop': await container.delete(force=True) print(f"=> deleted {container._id[:12]}") elif event['Action'] == 'destroy': print('=> done with this container!') break if __name__ == '__main__': loop = asyncio.get_event_loop() docker = Docker() try: # do our stuffs. loop.run_until_complete(demo(docker)) finally: loop.run_until_complete(docker.close()) loop.close()
async def _make_docker(): return Docker(**kwargs)
async def test_connect_with_connector(monkeypatch): connector = aiohttp.BaseConnector() docker = Docker(connector=connector) assert docker.connector == connector await docker.close()
async def test_connect_invalid_unix_socket(): docker = Docker('unix:///var/run/does-not-exist-docker.sock') assert isinstance(docker.connector, aiohttp.connector.UnixConnector) with pytest.raises(aiohttp.ClientOSError): await docker.containers.list() await docker.close()
def test_image_push(): docker = Docker() yield from docker.pull("debian") yield from docker.images.tag("debian", repo=os.environ["DOCKER_REGISTRY"] + "/debian")
import aiopg.sa import asyncio import aiohttp from aiodocker.docker import Docker import json import humanize.time from sqlalchemy import select, join, desc from moxie.server import MoxieApp from moxie.models import Job, Maintainer, Run from moxie.core import DATABASE_URL from aiocore import Service app = MoxieApp() docker = Docker() @asyncio.coroutine def get_logs(conn, job, limit=10): runs = yield from conn.execute( select([Run.__table__]).where(Run.job_id == job.id).order_by( desc(Run.start_time)).limit(limit)) return runs @asyncio.coroutine def get_jobs(conn, jobs, limit=10): ret = [] for job in jobs: ret.append((job, (yield from get_logs(conn, job, limit=limit))))
async def datanode_first_up(): docker = Docker() const_find_service_dirs = find_service_dirs() for container in (await docker.containers.list()): await container_by_data(container, const_find_service_dirs) await docker.close()
while True: event = await subscriber.get() if event is None: break for key, value in event.items(): print(key,':', value) # Demonstrate simple event-driven container mgmt. if event['Actor']['ID'] == container._id: if event['Action'] == 'start': await container.stop() print("=> killed {}".format(container._id[:12])) elif event['Action'] == 'stop': await container.delete(force=True) print("=> deleted {}".format(container._id[:12])) elif event['Action'] == 'destroy': print('=> done with this container!') break if __name__ == '__main__': loop = asyncio.get_event_loop() docker = Docker() try: # do our stuffs. loop.run_until_complete(demo(docker)) finally: loop.run_until_complete(docker.close()) loop.close()