def spawn_container(self,
add_envs=None,
add_labels=None,
entrypoint_func=None):
add_envs = sly.take_with_default(add_envs, {})
add_labels = sly.take_with_default(add_labels, {})
if entrypoint_func is None:
entrypoint_func = self.get_spawn_entrypoint
self._container_lock.acquire()
volumes = self._get_task_volumes()
try:
self._container = self._docker_api.containers.run(
self.docker_image_name,
runtime=self.docker_runtime,
entrypoint=entrypoint_func(),
detach=True,
name='sly_task_{}_{}'.format(self.info['task_id'],
constants.TASKS_DOCKER_LABEL()),
remove=False,
volumes=volumes,
environment={
'LOG_LEVEL': 'DEBUG',
'LANG': 'C.UTF-8',
'PYTHONUNBUFFERED': '1',
constants._HTTP_PROXY: constants.HTTP_PROXY(),
constants._HTTPS_PROXY: constants.HTTPS_PROXY(),
'HOST_TASK_DIR': self.dir_task_host,
constants._NO_PROXY: constants.NO_PROXY(),
constants._HTTP_PROXY.lower(): constants.HTTP_PROXY(),
constants._HTTPS_PROXY.lower(): constants.HTTPS_PROXY(),
constants._NO_PROXY.lower(): constants.NO_PROXY(),
**add_envs
},
labels={
'ecosystem': 'supervisely',
'ecosystem_token': constants.TASKS_DOCKER_LABEL(),
'task_id': str(self.info['task_id']),
**add_labels
},
shm_size="1G",
stdin_open=False,
tty=False,
cpu_period=constants.CPU_PERIOD(),
cpu_quota=constants.CPU_QUOTA(),
mem_limit=constants.MEM_LIMIT(),
memswap_limit=constants.MEM_LIMIT())
self._container.reload()
self.logger.debug('After spawning. Container status: {}'.format(
str(self._container.status)))
self.logger.info('Docker container is spawned',
extra={
'container_id': self._container.id,
'container_name': self._container.name
})
finally:
self._container_lock.release()
def spawn_container(self, add_envs=None):
if add_envs is None:
add_envs = {}
self._container_lock.acquire()
volumes = self._get_task_volumes()
try:
self._container = self._docker_api.containers.run(
self.docker_image_name,
runtime=self.docker_runtime,
entrypoint=[
"sh", "-c", "python -u {}".format(self.entrypoint)
],
detach=True,
name='sly_task_{}_{}'.format(self.info['task_id'],
constants.TASKS_DOCKER_LABEL()),
remove=False,
volumes=volumes,
environment={
'LOG_LEVEL': 'DEBUG',
'LANG': 'C.UTF-8',
constants._HTTP_PROXY: constants.HTTP_PROXY(),
constants._HTTPS_PROXY: constants.HTTPS_PROXY(),
**add_envs
},
labels={
'ecosystem': 'supervisely',
'ecosystem_token': constants.TASKS_DOCKER_LABEL(),
'task_id': str(self.info['task_id'])
},
shm_size="1G",
stdin_open=False,
tty=False,
cpu_period=constants.CPU_PERIOD(),
cpu_quota=constants.CPU_QUOTA(),
mem_limit=constants.MEM_LIMIT(),
memswap_limit=constants.MEM_LIMIT())
self._container.reload()
self.logger.debug('After spawning. Container status: {}'.format(
str(self._container.status)))
self.logger.info('Docker container is spawned',
extra={
'container_id': self._container.id,
'container_name': self._container.name
})
finally:
self._container_lock.release()
def spawn_container(self, add_envs=None):
if add_envs is None:
add_envs = {}
self._container_lock.acquire()
try:
self._container = self._docker_api.containers.run(
self.docker_image_name,
runtime=self.docker_runtime,
entrypoint=[
"sh", "-c", "python -u {}".format(self.entrypoint)
],
detach=True,
name='sly_task_{}_{}'.format(self.info['task_id'],
constants.TASKS_DOCKER_LABEL()),
remove=False,
volumes={
self.dir_task_host: {
'bind': '/sly_task_data',
'mode': 'rw'
}
},
environment={
'LOG_LEVEL': 'DEBUG',
'LANG': 'C.UTF-8',
**add_envs
},
labels={
'ecosystem': 'supervisely',
'ecosystem_token': constants.TASKS_DOCKER_LABEL(),
'task_id': str(self.info['task_id'])
},
shm_size="1G",
stdin_open=False,
tty=False)
self._container.reload()
self.logger.debug('After spawning. Container status: {}'.format(
str(self._container.status)))
self.logger.info('Docker container is spawned',
extra={
'container_id': self._container.id,
'container_name': self._container.name
})
finally:
self._container_lock.release()
def __init__(self):
self.logger = sly.get_task_logger('agent')
sly.change_formatters_default_values(self.logger, 'service_type',
sly.ServiceType.AGENT)
sly.change_formatters_default_values(self.logger, 'event_type',
sly.EventType.LOGJ)
self.log_queue = LogQueue()
add_task_handler(self.logger, self.log_queue)
sly.add_default_logging_into_file(self.logger,
constants.AGENT_LOG_DIR())
self._stop_log_event = threading.Event()
self.executor_log = ThreadPoolExecutor(max_workers=1)
self.future_log = None
self.logger.info('Agent comes back...')
self.task_pool_lock = threading.Lock()
self.task_pool = {} # task_id -> task_manager (process_id)
self.thread_pool = ThreadPoolExecutor(max_workers=10)
self.thread_list = []
self.daemons_list = []
self._remove_old_agent()
self._validate_duplicated_agents()
sly.fs.clean_dir(constants.AGENT_TMP_DIR())
self._stop_missed_containers(constants.TASKS_DOCKER_LABEL())
# for compatibility with old plugins
self._stop_missed_containers(constants.TASKS_DOCKER_LABEL_LEGACY())
self.docker_api = docker.from_env(
version='auto', timeout=constants.DOCKER_API_CALL_TIMEOUT())
self._docker_login()
self.logger.info('Agent is ready to get tasks.')
self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(),
self.logger, constants.TIMEOUT_CONFIG_PATH())
self.agent_connect_initially()
self.logger.info('Agent connected to server.')
def _stop_missed_containers(self):
self.logger.info('Searching for missed containers...')
label_filter = {
'label':
'ecosystem_token={}'.format(constants.TASKS_DOCKER_LABEL())
}
stopped_list = Agent._remove_containers(label_filter=label_filter)
if len(stopped_list) == 0:
self.logger.info('There are no missed containers.')
for cont in stopped_list:
self.logger.info('Container stopped',
extra={
'cont_id': cont.id,
'labels': cont.labels
})
self.logger.info('TASK_MISSED',
extra={
'service_type': sly.ServiceType.TASK,
'event_type': sly.EventType.MISSED_TASK_FOUND,
'task_id': int(cont.labels['task_id'])
})