class PlaygroundDockerClient(object):
_envoy_label = "envoy.playground"
def __init__(self, api):
self.api = api
self.docker = aiodocker.Docker()
self.images = PlaygroundDockerImages(self)
self.volumes = PlaygroundDockerVolumes(self)
self.proxies = PlaygroundDockerProxies(self)
self.services = PlaygroundDockerServices(self)
self.networks = PlaygroundDockerNetworks(self)
self.events = PlaygroundDockerEvents(self)
self._subscribers = []
@method_decorator(cmd)
async def clear(self) -> None:
await self.networks.clear()
await self.proxies.clear()
await self.services.clear()
@method_decorator(cmd(sync=True))
async def dump_resources(self) -> list:
proxies = {proxy['name']: proxy for proxy in await self.proxies.list()}
services = {
service['name']: service
for service in await self.services.list()
}
networks = {
network['name']: network
for network in await self.networks.list()
}
return dict(networks=networks, proxies=proxies, services=services)
async def emit_error(self, message: str):
for _subscriber in self._subscribers:
await _subscriber(message)
async def get_container(self, id: str):
return await self.docker.containers.get(id)
async def get_network(self, id: str):
return await self.docker.networks.get(id)
def subscribe(self, handler, debug):
errors = handler.pop("errors", None)
if errors:
self._subscribers.append(errors)
self.events.subscribe(handler, debug)
class PlaygroundDockerServices(PlaygroundDockerResources):
_docker_resource = 'containers'
name = 'service'
@method_decorator(cmd(attribs=ServiceCreateCommandAttribs))
async def create(self, command: PlaygroundCommand) -> None:
logger.debug(f'Creating service: {command.data.name}')
if not command.data.image:
# todo: add build logic
return
if not await self.connector.images.exists(command.data.image):
await self.connector.events.publish('image_pull', 'service',
command.data.name,
command.data.image)
if not await self.connector.images.pull(command.data.image):
logger.debug(f'Failed pulling image ({command.data.image}) '
f'for {command.data.name}')
return
_environment = ["%s=%s" % (k, v) for k, v in command.data.env.items()]
await self._start_container(
self._get_service_config(
command.data.service_type, command.data.image,
command.data.name, _environment, await
self._get_service_mounts(command.data)), command.data.name)
async def _get_service_mounts(
self, data: ServiceCreateCommandAttribs) -> OrderedDict:
mounts = OrderedDict()
if data.configuration and data.config_path:
config = base64.b64encode(
data.configuration.encode('utf-8')).decode()
mounts[os.path.dirname(
data.config_path)] = (await self.connector.volumes.populate(
'service', f'{data.service_type}:{data.name}', 'config',
{os.path.basename(data.config_path): config}))
return mounts
def _get_service_config(self, service_type: str, image: str, name: str,
environment: list, mounts: dict) -> dict:
labels = {
"envoy.playground.service": name,
"envoy.playground.service.type": service_type,
}
return {
'Image': image,
"AttachStdin": False,
"AttachStdout": False,
"AttachStderr": False,
"Tty": False,
"OpenStdin": False,
"Env": environment,
"Labels": labels,
"HostConfig": {
"Binds": ['%s:%s' % (v, k) for k, v in mounts.items()]
}
}
async def _mangle_resource(self, resource, _resource):
_resource['image'] = resource['Image']
_resource['type'] = 'service'
_resource["service_type"] = resource["Labels"][
"envoy.playground.service.type"]
class PlaygroundDockerNetworks(PlaygroundDockerResources):
_docker_resource = 'networks'
name = 'network'
@method_decorator(cmd(attribs=NetworkAddAttribs))
async def create(
self,
command: PlaygroundCommand) -> None:
logger.debug(
f'Creating network: {command.data}')
try:
await self._create_network(command)
except DockerError as e:
logger.error(
f'Failed creating network: {command.data} {e}')
@method_decorator(cmd(attribs=NetworkDeleteAttribs))
async def delete(
self,
command: PlaygroundCommand) -> None:
logger.debug(
f'Deleting network: {command.data}')
try:
await self._delete_network(command)
except DockerError as e:
logger.error(
f'Failed deleting network: {command.data} {e}')
@method_decorator(cmd(attribs=NetworkEditAttribs))
async def edit(
self,
command: PlaygroundCommand) -> None:
logger.debug(
f'Updating network: {command.data}')
try:
await self._edit_network(command)
except DockerError as e:
logger.error(
f'Failed updating network: {command.data} {e}')
async def _connect(self, network, name, containers):
for container in containers:
logger.debug(
f'Connecting {container["type"]} {container["name"]} '
f'to network: {name}')
try:
await network.connect(self._get_container_config(container))
except DockerError as e:
logger.debug(
f'Failed connecting {container["type"]} '
f'{container["name"]}'
f'to network: {name} {e}')
continue
if container['type'] == 'proxy':
try:
await self.docker.containers.container(
container['id']).kill(signal='SIGHUP')
except DockerError as e:
logger.error(
f'Failed restarting proxy on connect: '
f'{container["name"]} {e}')
async def _create_network(
self,
command: PlaygroundCommand) -> None:
logger.debug(
f'Creating network: {command.data}')
try:
network = await self.docker.networks.create(
dict(name="__playground_%s" % command.data.name,
labels={"envoy.playground.network": command.data.name}))
except DockerError as e:
logger.error(
f'Failed creating network: {command.data} {e}')
return
await self._connect(
network,
command.data.name,
(container
for container in await self.connector.proxies.list()
if container['name'] in command.data.proxies))
await self._connect(
network,
command.data.name,
(container
for container in await self.connector.services.list()
if container['name'] in command.data.services))
async def _delete_network(
self,
command: PlaygroundCommand) -> None:
logger.debug(
f'Deleting network: {command.data}')
try:
network = await self.docker.networks.get(command.data.id)
info = await network.show()
except DockerError as e:
logger.error(
f'Failed getting network for delete: {command.data} {e}')
return
if "envoy.playground.network" not in info["Labels"]:
logger.warng(
f'Received spurious network: {command.data}')
return
for container in info["Containers"].keys():
logger.debug(
f'Disconnecting ({container}) from network: '
f'{command.data}')
try:
await network.disconnect({"Container": container})
except DockerError as e:
logger.error(
f'Failed disconnecting container: '
f'{command.data} {container} {e}')
try:
await network.delete()
except DockerError as e:
logger.error(
f'Failed removing network: '
f'{command.data} {e}')
async def _disconnect(self, network, name, containers):
for container in containers:
logger.debug(
f'Disconnecting {container["type"]} {container["name"]} '
f'from network: {name}')
try:
await network.disconnect({"Container": container["id"]})
except DockerError as e:
logger.error(
f'Failed disconnecting container: '
f'{name} {container["type"]} {container["name"]} {e}')
async def _edit_network(
self,
command: PlaygroundCommand) -> None:
network = await self.docker.networks.get(command.data.id)
info = await network.show()
containers = {
container['Name'].replace(
'envoy__playground__service__', '').replace(
'envoy__playground__proxy__', '')
for container
in info["Containers"].values()}
expected = (
set(command.data.proxies or [])
| set(command.data.services or []))
connect = expected - containers
disconnect = containers - expected
_containers = (
await self.connector.proxies.list()
+ await self.connector.services.list())
await self._connect(
network,
info['Labels']['envoy.playground.network'],
(container
for container in _containers
if container['name'] in connect))
await self._disconnect(
network,
info['Labels']['envoy.playground.network'],
(container
for container in _containers
if container['name'] in disconnect))
def _get_container_config(self, container):
return {
"Container": container["id"],
"EndpointConfig": {
"Aliases": [container['name']]}}
async def _mangle_resource(self, resource, _resource):
try:
_actual_network = await self.docker.networks.get(resource["Id"])
info = await _actual_network.show()
except DockerError as e:
logger.error(
f'Failed getting info for network: {resource} {e}')
else:
if info["Containers"]:
_resource["containers"] = [
container[:10]
for container
in info["Containers"].keys()]
class PlaygroundDockerProxies(PlaygroundDockerResources):
_docker_resource = 'containers'
name = 'proxy'
@method_decorator(cmd(attribs=ProxyCreateCommandAttribs))
async def create(self, command: PlaygroundCommand) -> None:
logger.debug(f'Creating proxy: {command.data.name}')
use_dev = (not command.data.version
or command.data.version.startswith('envoy-dev'))
image = ('envoyproxy/envoy-dev' if use_dev else 'envoyproxy/envoy')
tag = (command.data.version.split(':')[1]
if ':' in command.data.version else 'latest')
base_image = f'{image}:{tag}'
envoy_image = (f"envoyproxy/{base_image.split('/')[1].split(':')[0]}"
f"-playground:{base_image.split(':')[1]}")
should_pull = (command.data.pull_latest
or not await self.connector.images.exists(envoy_image))
if should_pull:
await self.connector.events.publish('image_pull', 'proxy',
command.data.name, base_image)
if await self.connector.images.pull(base_image):
await self.connector.events.publish('image_build',
command.data.name,
base_image, envoy_image)
built = await self.connector.images.build(
base_image, envoy_image)
if not built:
# todo: publish failure ?
logger.error(f'Failed building proxy image for {base_image}')
return
_mappings = [[m['mapping_from'], m['mapping_to']]
for m in command.data.port_mappings]
config = self._get_proxy_config(envoy_image, command.data.name,
command.data.logging, await
self._get_mounts(command.data),
_mappings)
await self._start_container(config, command.data.name)
async def _get_mounts(self, data: ProxyCreateCommandAttribs) -> dict:
return {
"/etc/envoy":
await self.connector.volumes.populate(
'proxy', data.name, 'envoy', {
'envoy.yaml':
base64.b64encode(
data.configuration.encode('utf-8')).decode()
}),
"/certs":
await self.connector.volumes.populate('proxy', data.name, 'certs',
data.certs),
'/binary':
await self.connector.volumes.populate('proxy', data.name, 'binary',
data.binaries),
'/logs':
await self.connector.volumes.create('proxy', data.name, 'logs')
}
def _get_port_bindings(self, port_mappings: list) -> OrderedDict:
# todo: handle udp etc
port_bindings: OrderedDict = OrderedDict()
for host, internal in port_mappings:
port_bindings[f"{internal}/tcp"] = port_bindings.get(
f"{internal}/tcp", [])
port_bindings[f"{internal}/tcp"].append({"HostPort": f"{host}"})
return port_bindings
def _get_proxy_config(self, image: str, name: str, logging: dict,
mounts: dict, port_mappings: list) -> dict:
environment = ([] if logging.get("default", "info") in ['', "info"]
else [f"ENVOY_LOG_LEVEL={logging['default']}"])
exposed = {
f"{internal}/tcp": {}
for external, internal in port_mappings
}
return {
'Image': image,
'Cmd': ["python", "/hot-restarter.py", "/start_envoy.sh"],
"AttachStdin": False,
"AttachStdout": False,
"AttachStderr": False,
"Tty": False,
"OpenStdin": False,
"Labels": {
"envoy.playground.proxy": name,
},
"Env": environment,
"ExposedPorts": exposed,
"HostConfig": {
"PortBindings": self._get_port_bindings(port_mappings),
"Binds": ['%s:%s' % (v, k) for k, v in mounts.items()]
}
}
async def _mangle_resource(self, resource, _resource):
_resource['image'] = self._get_image_name(resource['Image'])
_resource['port_mappings'] = [{
'mapping_from': m.get('PublicPort'),
'mapping_to': m.get('PrivatePort')
} for m in resource['Ports'] if m.get('PublicPort')]
if not _resource['port_mappings']:
del _resource['port_mappings']
class PlaygroundDockerResources(PlaygroundDockerContext):
async def clear(self):
for resource in await self.list():
await self.delete(dict(id=resource['id']))
async def get(self, uuid):
types = {
f'envoy.playground.{container_type}'
for container_type in ['service', 'proxy']
}
try:
container = await self.docker.containers.get(uuid)
except DockerError:
logger.error(f'Failed getting container {uuid}')
return
return (container if types.intersection(
container['Config']['Labels'].keys()) else None)
@method_decorator(cmd(attribs=ContainerDeleteAttribs))
async def delete(self, command: PlaygroundCommand) -> None:
logger.debug(f'Deleting {self.name}: {command.data.id}')
container = await self.get(command.data.id)
if not container:
logger.warning(
f'Unable to find container to delete: {command.data.id}')
return
await self._delete_container(container)
async def _delete_container(self, container):
try:
await container.stop()
await container.wait()
await container.delete(v=True, force=True)
volumes = [
v['Name'] for v in container.__dict__['_container']['Mounts']
]
await self.connector.volumes.delete(volumes)
return True
except DockerError as e:
logger.warning(f'Failed deleting {container}: {e}')
if e.args[0] == 409:
raise PlaytimeError(e.args[1]['message'])
return False
def _get_image_name(self, playground_image):
envoy_image = playground_image.split('/')[1].split(':')[0].replace(
'-playground', '')
tag = playground_image.split(':')[1]
return f"envoyproxy/{envoy_image}:{tag}"
async def list(self) -> list:
if not self._docker_resource or not self.name:
return []
return await self._list_resources(
getattr(self.docker, self._docker_resource), self.name)
async def _list_resources(self, resources: Union[DockerContainers,
DockerNetworks],
name: str) -> list:
_resources = []
label = "%s.%s" % (self.connector._envoy_label, name)
try:
docker_resources = await resources.list()
except DockerError:
logger.error(f'Failed listing containers: {name}')
return []
for resource in docker_resources:
if label not in resource["Labels"]:
continue
_resource = dict(name=resource["Labels"][label],
id=resource["Id"][:10],
type=name)
await self._mangle_resource(resource, _resource)
_resources.append(_resource)
return _resources
async def _remove_container(self, container):
volumes = [v['Name'] for v in container['Mounts']]
try:
await self._delete_container(container)
except PlaytimeError as e:
logger.error(f'Failed removing container {e}')
await self.connector.emit_error(e.args[0])
return
if volumes:
await self._remove_volumes(volumes)
async def _remove_volumes(self, volumes):
_volumes = await self.docker.volumes.list()
for volume in _volumes['Volumes']:
volume_name = volume['Name']
if volume_name not in volumes:
continue
volume_delete = self.docker._query(f"volumes/{volume_name}",
method="DELETE")
try:
async with volume_delete:
pass
except DockerError as e:
logger.error(f'Failed removing volume {volume_name} {e}')
async def _start_container(self, config, name):
logger.debug(f'Starting {self.name}: {name}')
try:
container = await self.docker.containers.create_or_replace(
config=config, name=f"envoy__playground__{self.name}__{name}")
await container.start()
except DockerError as e:
logger.error(f'Failed starting container {name} {config} {e}')