def task_main_func(self): if constants.TOKEN() != self.info['config']['access_token']: raise RuntimeError('Current token != new token') docker_inspect_cmd = "curl -s --unix-socket /var/run/docker.sock http:/containers/$(hostname)/json" docker_img_info = subprocess.Popen( [docker_inspect_cmd], shell=True, executable="/bin/bash", stdout=subprocess.PIPE).communicate()[0].decode("utf-8") docker_img_info = json.loads(docker_img_info) #docker_image = docker_img_info["Config"]["Image"] #cur_version = docker_img_info["Config"]["Labels"]["VERSION"] cur_container_id = docker_img_info["Config"]["Hostname"] #cur_container_name = docker_img_info["Name"].split("/")[1] cur_volumes = docker_img_info["HostConfig"]["Binds"] cur_envs = docker_img_info["Config"]["Env"] if docker_img_info["Config"]["Labels"].get( "com.docker.compose.project", None) == "supervisely": raise RuntimeError( 'Docker container was started from docker-compose. Please, use docker-compose to upgrade.' ) return self._docker_pull(self.info['docker_image']) new_volumes = {} for vol in cur_volumes: parts = vol.split(":") src = parts[0] dst = parts[1] new_volumes[src] = {'bind': dst, 'mode': 'rw'} cur_envs.append("REMOVE_OLD_AGENT={}".format(cur_container_id)) container = self._docker_api.containers.run( self.info['docker_image'], runtime=self.info['config']['docker_runtime'], detach=True, name='supervisely-agent-{}-{}'.format(constants.TOKEN(), sly.rand_str(5)), remove=False, restart_policy={"Name": "unless-stopped"}, volumes=new_volumes, environment=cur_envs, stdin_open=False, tty=False) container.reload() self.logger.debug('After spawning. Container status: {}'.format( str(container.status))) self.logger.info('Docker container is spawned', extra={ 'container_id': container.id, 'container_name': container.name })
def download_step(self): if self.info.get('nn_model', None) is None: self.logger.critical('TASK_NN_EMPTY') raise ValueError('TASK_NN_EMPTY') self.data_mgr.download_nn(self.info['nn_model']['title'], self.dir_model) #@TODO: only for compatibility with old models shutil.move(self.dir_model, self.dir_model + '_delme') shutil.move(os.path.join(self.dir_model + '_delme', self.info['nn_model']['title']), self.dir_model) sly.fs.remove_dir(self.dir_model + '_delme') out_cfg = { **self.info['task_settings'], # settings from server 'connection': { 'server_address': constants.SERVER_ADDRESS(), 'token': constants.TOKEN(), 'task_id': str(self.info['task_id']), }, 'model_settings': self.info['task_settings'] } json.dump(out_cfg, open(self.config_path1, 'w')) # Deprecated 'task_settings.json' json.dump(out_cfg, open(self.config_path2, 'w')) # New style task_config.json self.report_step_done(TaskStep.DOWNLOAD)
def __init__(self): self.logger = sly.get_task_logger('agent') sly.change_formatters_default_values(self.logger, 'service_type', sly.ServiceType.AGENT) sly.change_formatters_default_values(self.logger, 'event_type', sly.EventType.LOGJ) self.log_queue = LogQueue() add_task_handler(self.logger, self.log_queue) sly.add_default_logging_into_file(self.logger, constants.AGENT_LOG_DIR()) self.logger.info('Agent comes back...') self.task_pool_lock = threading.Lock() self.task_pool = {} # task_id -> task_manager (process_id) self.thread_pool = ThreadPoolExecutor(max_workers=10) self.thread_list = [] self.daemons_list = [] sly.fs.clean_dir(constants.AGENT_TMP_DIR()) self._stop_missed_containers() self.docker_api = docker.from_env(version='auto') self._docker_login() self.logger.info('Agent is ready to get tasks.') self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(), self.logger, constants.TIMEOUT_CONFIG_PATH()) self.agent_connect_initially() self.logger.info('Agent connected to server.')
def init_api(self): self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(), self.logger, constants.TIMEOUT_CONFIG_PATH()) if 'user_api_key' in self.info: self.public_api = sly.Api(constants.SERVER_ADDRESS(), self.info['user_api_key']) self.public_api.add_additional_field('taskId', self.info['task_id']) self.public_api_context = self.public_api.task.get_context(self.info['task_id'])
def _validate_duplicated_agents(self): dc = docker.from_env() agent_same_token = [] for cont in dc.containers.list(): if constants.TOKEN() in cont.name: agent_same_token.append(cont) if len(agent_same_token) > 1: raise RuntimeError("Agent with the same token already exists.")
def init_api(self): self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(), self.logger, constants.TIMEOUT_CONFIG_PATH()) if self._user_api_key is not None: self.public_api = sly.Api(constants.SERVER_ADDRESS(), self._user_api_key, external_logger=self.logger, retry_count=constants.PUBLIC_API_RETRY_LIMIT()) task_id = self.info['task_id'] self.public_api.add_additional_field('taskId', task_id) self.public_api.add_header('x-task-id', str(task_id)) self.public_api_context = self.public_api.task.get_context(task_id)
def _remove_old_agent(self): container_id = os.getenv('REMOVE_OLD_AGENT', None) if container_id is None: return dc = docker.from_env() olg_agent = dc.containers.get(container_id) olg_agent.remove(force=True) agent_same_token = [] for cont in dc.containers.list(): if constants.TOKEN() in cont.name: agent_same_token.append(cont) if len(agent_same_token) > 1: raise RuntimeError( "Several agents with the same token are running. Please, kill them or contact support." ) agent_same_token[0].rename('supervisely-agent-{}'.format( constants.TOKEN()))
def spawn_container(self, add_envs=None): if add_envs is None: add_envs = {} self._container_lock.acquire() try: #@TODO: DEBUG_COPY_IMAGES only for compatibility with old plugins self._container = self._docker_api.containers.run( self.docker_image_name, runtime=self.docker_runtime, entrypoint=[ "sh", "-c", "python -u {}".format(self.entrypoint) ], detach=True, name='sly_task_{}_{}'.format(constants.TOKEN(), self.info['task_id']), remove=False, volumes={ self.dir_task_host: { 'bind': '/sly_task_data', 'mode': 'rw' } }, environment={ 'LOG_LEVEL': 'DEBUG', 'LANG': 'C.UTF-8', 'DEBUG_COPY_IMAGES': 1, **add_envs }, labels={ 'ecosystem': 'supervisely', 'ecosystem_token': constants.TASKS_DOCKER_LABEL(), 'task_id': str(self.info['task_id']) }, shm_size="1G", stdin_open=False, tty=False) self._container.reload() self.logger.debug('After spawning. Container status: {}'.format( str(self._container.status))) self.logger.info('Docker container spawned', extra={ 'container_id': self._container.id, 'container_name': self._container.name }) finally: self._container_lock.release()
def _exec_command(self, command, add_envs=None, container_id=None): add_envs = sly.take_with_default(add_envs, {}) self._exec_id = self._docker_api.api.exec_create(self._container.id if container_id is None else container_id, cmd=command, environment={ 'LOG_LEVEL': 'DEBUG', 'LANG': 'C.UTF-8', 'PYTHONUNBUFFERED': '1', constants._HTTP_PROXY: constants.HTTP_PROXY(), constants._HTTPS_PROXY: constants.HTTPS_PROXY(), 'HOST_TASK_DIR': self.dir_task_host, 'TASK_ID': self.info['task_id'], 'SERVER_ADDRESS': self.info['server_address'], 'API_TOKEN': self.info['api_token'], 'AGENT_TOKEN': constants.TOKEN(), **add_envs }) self._logs_output = self._docker_api.api.exec_start(self._exec_id, stream=True, demux=False)
def __init__(self): self.logger = sly.get_task_logger('agent') sly.change_formatters_default_values(self.logger, 'service_type', sly.ServiceType.AGENT) sly.change_formatters_default_values(self.logger, 'event_type', sly.EventType.LOGJ) self.log_queue = LogQueue() add_task_handler(self.logger, self.log_queue) sly.add_default_logging_into_file(self.logger, constants.AGENT_LOG_DIR()) self._stop_log_event = threading.Event() self.executor_log = ThreadPoolExecutor(max_workers=1) self.future_log = None self.logger.info('Agent comes back...') self.task_pool_lock = threading.Lock() self.task_pool = {} # task_id -> task_manager (process_id) self.thread_pool = ThreadPoolExecutor(max_workers=10) self.thread_list = [] self.daemons_list = [] self._remove_old_agent() self._validate_duplicated_agents() sly.fs.clean_dir(constants.AGENT_TMP_DIR()) self._stop_missed_containers(constants.TASKS_DOCKER_LABEL()) # for compatibility with old plugins self._stop_missed_containers(constants.TASKS_DOCKER_LABEL_LEGACY()) self.docker_api = docker.from_env( version='auto', timeout=constants.DOCKER_API_CALL_TIMEOUT()) self._docker_login() self.logger.info('Agent is ready to get tasks.') self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(), self.logger, constants.TIMEOUT_CONFIG_PATH()) self.agent_connect_initially() self.logger.info('Agent connected to server.')