def spawn_container(self, add_envs=None, add_labels=None, entrypoint_func=None): add_envs = sly.take_with_default(add_envs, {}) add_labels = sly.take_with_default(add_labels, {}) if entrypoint_func is None: entrypoint_func = self.get_spawn_entrypoint self._container_lock.acquire() volumes = self._get_task_volumes() try: self._container = self._docker_api.containers.run( self.docker_image_name, runtime=self.docker_runtime, entrypoint=entrypoint_func(), detach=True, name='sly_task_{}_{}'.format(self.info['task_id'], constants.TASKS_DOCKER_LABEL()), remove=False, volumes=volumes, environment={ 'LOG_LEVEL': 'DEBUG', 'LANG': 'C.UTF-8', 'PYTHONUNBUFFERED': '1', constants._HTTP_PROXY: constants.HTTP_PROXY(), constants._HTTPS_PROXY: constants.HTTPS_PROXY(), 'HOST_TASK_DIR': self.dir_task_host, constants._NO_PROXY: constants.NO_PROXY(), constants._HTTP_PROXY.lower(): constants.HTTP_PROXY(), constants._HTTPS_PROXY.lower(): constants.HTTPS_PROXY(), constants._NO_PROXY.lower(): constants.NO_PROXY(), **add_envs }, labels={ 'ecosystem': 'supervisely', 'ecosystem_token': constants.TASKS_DOCKER_LABEL(), 'task_id': str(self.info['task_id']), **add_labels }, shm_size="1G", stdin_open=False, tty=False, cpu_period=constants.CPU_PERIOD(), cpu_quota=constants.CPU_QUOTA(), mem_limit=constants.MEM_LIMIT(), memswap_limit=constants.MEM_LIMIT()) self._container.reload() self.logger.debug('After spawning. Container status: {}'.format( str(self._container.status))) self.logger.info('Docker container is spawned', extra={ 'container_id': self._container.id, 'container_name': self._container.name }) finally: self._container_lock.release()
def spawn_container(self, add_envs=None): if add_envs is None: add_envs = {} self._container_lock.acquire() volumes = self._get_task_volumes() try: self._container = self._docker_api.containers.run( self.docker_image_name, runtime=self.docker_runtime, entrypoint=[ "sh", "-c", "python -u {}".format(self.entrypoint) ], detach=True, name='sly_task_{}_{}'.format(self.info['task_id'], constants.TASKS_DOCKER_LABEL()), remove=False, volumes=volumes, environment={ 'LOG_LEVEL': 'DEBUG', 'LANG': 'C.UTF-8', constants._HTTP_PROXY: constants.HTTP_PROXY(), constants._HTTPS_PROXY: constants.HTTPS_PROXY(), **add_envs }, labels={ 'ecosystem': 'supervisely', 'ecosystem_token': constants.TASKS_DOCKER_LABEL(), 'task_id': str(self.info['task_id']) }, shm_size="1G", stdin_open=False, tty=False, cpu_period=constants.CPU_PERIOD(), cpu_quota=constants.CPU_QUOTA(), mem_limit=constants.MEM_LIMIT(), memswap_limit=constants.MEM_LIMIT()) self._container.reload() self.logger.debug('After spawning. Container status: {}'.format( str(self._container.status))) self.logger.info('Docker container is spawned', extra={ 'container_id': self._container.id, 'container_name': self._container.name }) finally: self._container_lock.release()
def spawn_container(self, add_envs=None): if add_envs is None: add_envs = {} self._container_lock.acquire() try: self._container = self._docker_api.containers.run( self.docker_image_name, runtime=self.docker_runtime, entrypoint=[ "sh", "-c", "python -u {}".format(self.entrypoint) ], detach=True, name='sly_task_{}_{}'.format(self.info['task_id'], constants.TASKS_DOCKER_LABEL()), remove=False, volumes={ self.dir_task_host: { 'bind': '/sly_task_data', 'mode': 'rw' } }, environment={ 'LOG_LEVEL': 'DEBUG', 'LANG': 'C.UTF-8', **add_envs }, labels={ 'ecosystem': 'supervisely', 'ecosystem_token': constants.TASKS_DOCKER_LABEL(), 'task_id': str(self.info['task_id']) }, shm_size="1G", stdin_open=False, tty=False) self._container.reload() self.logger.debug('After spawning. Container status: {}'.format( str(self._container.status))) self.logger.info('Docker container is spawned', extra={ 'container_id': self._container.id, 'container_name': self._container.name }) finally: self._container_lock.release()
def __init__(self): self.logger = sly.get_task_logger('agent') sly.change_formatters_default_values(self.logger, 'service_type', sly.ServiceType.AGENT) sly.change_formatters_default_values(self.logger, 'event_type', sly.EventType.LOGJ) self.log_queue = LogQueue() add_task_handler(self.logger, self.log_queue) sly.add_default_logging_into_file(self.logger, constants.AGENT_LOG_DIR()) self._stop_log_event = threading.Event() self.executor_log = ThreadPoolExecutor(max_workers=1) self.future_log = None self.logger.info('Agent comes back...') self.task_pool_lock = threading.Lock() self.task_pool = {} # task_id -> task_manager (process_id) self.thread_pool = ThreadPoolExecutor(max_workers=10) self.thread_list = [] self.daemons_list = [] self._remove_old_agent() self._validate_duplicated_agents() sly.fs.clean_dir(constants.AGENT_TMP_DIR()) self._stop_missed_containers(constants.TASKS_DOCKER_LABEL()) # for compatibility with old plugins self._stop_missed_containers(constants.TASKS_DOCKER_LABEL_LEGACY()) self.docker_api = docker.from_env( version='auto', timeout=constants.DOCKER_API_CALL_TIMEOUT()) self._docker_login() self.logger.info('Agent is ready to get tasks.') self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(), self.logger, constants.TIMEOUT_CONFIG_PATH()) self.agent_connect_initially() self.logger.info('Agent connected to server.')
def _stop_missed_containers(self): self.logger.info('Searching for missed containers...') label_filter = { 'label': 'ecosystem_token={}'.format(constants.TASKS_DOCKER_LABEL()) } stopped_list = Agent._remove_containers(label_filter=label_filter) if len(stopped_list) == 0: self.logger.info('There are no missed containers.') for cont in stopped_list: self.logger.info('Container stopped', extra={ 'cont_id': cont.id, 'labels': cont.labels }) self.logger.info('TASK_MISSED', extra={ 'service_type': sly.ServiceType.TASK, 'event_type': sly.EventType.MISSED_TASK_FOUND, 'task_id': int(cont.labels['task_id']) })