def __init__(self, context, backend_addr, friendly_name, concurrency, tasks_fs: FileSystemProvider, address_host=None, external_ports=None, tmp_dir="./agent_tmp"):
"""
:param context: ZeroMQ context for this process
:param backend_addr: address of the backend (for example, "tcp://127.0.0.1:2222")
:param friendly_name: a string containing a friendly name to identify agent
:param concurrency: number of simultaneous jobs that can be run by this agent
:param tasks_fs: FileSystemProvider for the course / tasks
:param address_host: hostname/ip/... to which external client should connect to access to the docker
:param external_ports: iterable containing ports to which the docker instance can bind internal ports
:param tmp_dir: temp dir that is used by the agent to start new containers
"""
super(DockerAgent, self).__init__(context, backend_addr, friendly_name, concurrency, tasks_fs)
self._logger = logging.getLogger("inginious.agent.docker")
self._max_memory_per_slot = int(psutil.virtual_memory().total / concurrency / 1024 / 1024)
self.tasks_fs = tasks_fs
# Temp dir
self._tmp_dir = tmp_dir
# SSH remote debug
self._address_host = address_host
self._external_ports = set(external_ports) if external_ports is not None else set()
# Async proxy to os
self._aos = AsyncProxy(os)
self._ashutil = AsyncProxy(shutil)
def __init__(self, context, backend_addr, friendly_name, concurrency, tasks_fs: FileSystemProvider, address_host=None, external_ports=None, tmp_dir="./agent_tmp", runtimes=None, ssh_allowed=False):
"""
:param context: ZeroMQ context for this process
:param backend_addr: address of the backend (for example, "tcp://127.0.0.1:2222")
:param friendly_name: a string containing a friendly name to identify agent
:param concurrency: number of simultaneous jobs that can be run by this agent
:param tasks_fs: FileSystemProvider for the course / tasks
:param address_host: hostname/ip/... to which external client should connect to access to the docker
:param external_ports: iterable containing ports to which the docker instance can bind internal ports
:param tmp_dir: temp dir that is used by the agent to start new containers
:param type: type of the container ("docker" or "kata")
:param runtime: runtime used by docker (the defaults are "runc" with docker or "kata-runtime" with kata)
:param ssh_allowed: boolean to make this agent accept tasks with ssh or not
"""
super(DockerAgent, self).__init__(context, backend_addr, friendly_name, concurrency, tasks_fs, ssh_allowed=ssh_allowed)
self._runtimes = {x.envtype: x for x in runtimes} if runtimes is not None else None
self._logger = logging.getLogger("inginious.agent.docker")
self._max_memory_per_slot = int(psutil.virtual_memory().total / concurrency / 1024 / 1024)
# Temp dir
self._tmp_dir = tmp_dir
# SSH remote debug
self._address_host = address_host
self._external_ports = set(external_ports) if external_ports is not None else set()
# Async proxy to os
self._aos = AsyncProxy(os)
self._ashutil = AsyncProxy(shutil)
async def _init_clean(self):
""" Must be called when the agent is starting """
# Data about running containers
self._containers_running: Dict[str, DockerRunningJob] = {
} # container_id -> info
self._container_for_job: Dict[str, str] = {} # job id -> container_id
self._student_containers_running: Dict[
str, DockerRunningStudentContainer] = {}
self._containers_killed = dict()
# Delete tmp_dir, and recreate-it again
try:
await self._ashutil.rmtree(self._tmp_dir)
except OSError:
pass
try:
await self._aos.mkdir(self._tmp_dir)
except OSError:
pass
# Docker
self._docker = AsyncProxy(DockerInterface())
if self._runtimes is None:
self._runtimes = self._detect_runtimes()
# Auto discover containers
self._logger.info("Discovering containers")
self._containers = await self._docker.get_containers(
self._runtimes.values())
if self._address_host is None and len(self._containers) != 0:
self._logger.info("Guessing external host IP")
available_bare_container_images = [
image for envtype_containers in self._containers.values()
for image in envtype_containers.values()
]
if len(available_bare_container_images) != 0:
self._address_host = await self._docker.get_host_ip(
available_bare_container_images[0]["id"])
else:
self._logger.error(
"Cannot find the external IP without at least an installed container."
)
if self._address_host is None:
self._logger.warning(
"Cannot find external host IP. Please indicate it in the configuration. "
"Remote SSH debug has been deactivated.")
self._external_ports = None
else:
self._logger.info("External address for SSH remote debug is %s",
self._address_host)
# Watchers
self._timeout_watcher = TimeoutWatcher(self._docker)
async def _init_clean(self):
""" Must be called when the agent is starting """
# Data about running containers
self._containers_running = {}
self._container_for_job = {}
self._student_containers_running = {}
self._student_containers_for_job = {}
self._containers_killed = dict()
# Delete tmp_dir, and recreate-it again
try:
await self._ashutil.rmtree(self._tmp_dir)
except OSError:
pass
try:
await self._aos.mkdir(self._tmp_dir)
except OSError:
pass
# Docker
self._docker = AsyncProxy(DockerInterface())
# Auto discover containers
self._logger.info("Discovering containers")
self._containers = await self._docker.get_containers()
for idx in self._containers:
self._containers[idx][
"type"] = "docker" # type is not given by self._docker.get_containers()
self._assigned_external_ports = {} # container_id : [external_ports]
if self._address_host is None and len(self._containers) != 0:
self._logger.info("Guessing external host IP")
self._address_host = await self._docker.get_host_ip(
next(iter(self._containers.values()))["id"])
if self._address_host is None:
self._logger.warning(
"Cannot find external host IP. Please indicate it in the configuration. Remote SSH debug has been deactivated."
)
self._external_ports = None
else:
self._logger.info("External address for SSH remote debug is %s",
self._address_host)
# Watchers
self._timeout_watcher = TimeoutWatcher(self._docker)
