def task_main_func(self):
if constants.TOKEN() != self.info['config']['access_token']:
raise RuntimeError('Current token != new token')
docker_inspect_cmd = "curl -s --unix-socket /var/run/docker.sock http:/containers/$(hostname)/json"
docker_img_info = subprocess.Popen(
[docker_inspect_cmd],
shell=True,
executable="/bin/bash",
stdout=subprocess.PIPE).communicate()[0].decode("utf-8")
docker_img_info = json.loads(docker_img_info)
#docker_image = docker_img_info["Config"]["Image"]
#cur_version = docker_img_info["Config"]["Labels"]["VERSION"]
cur_container_id = docker_img_info["Config"]["Hostname"]
#cur_container_name = docker_img_info["Name"].split("/")[1]
cur_volumes = docker_img_info["HostConfig"]["Binds"]
cur_envs = docker_img_info["Config"]["Env"]
if docker_img_info["Config"]["Labels"].get(
"com.docker.compose.project", None) == "supervisely":
raise RuntimeError(
'Docker container was started from docker-compose. Please, use docker-compose to upgrade.'
)
return
self._docker_pull(self.info['docker_image'])
new_volumes = {}
for vol in cur_volumes:
parts = vol.split(":")
src = parts[0]
dst = parts[1]
new_volumes[src] = {'bind': dst, 'mode': 'rw'}
cur_envs.append("REMOVE_OLD_AGENT={}".format(cur_container_id))
container = self._docker_api.containers.run(
self.info['docker_image'],
runtime=self.info['config']['docker_runtime'],
detach=True,
name='supervisely-agent-{}-{}'.format(constants.TOKEN(),
sly.rand_str(5)),
remove=False,
restart_policy={"Name": "unless-stopped"},
volumes=new_volumes,
environment=cur_envs,
stdin_open=False,
tty=False)
container.reload()
self.logger.debug('After spawning. Container status: {}'.format(
str(container.status)))
self.logger.info('Docker container is spawned',
extra={
'container_id': container.id,
'container_name': container.name
})
def download_step(self):
if self.info.get('nn_model', None) is None:
self.logger.critical('TASK_NN_EMPTY')
raise ValueError('TASK_NN_EMPTY')
self.data_mgr.download_nn(self.info['nn_model']['title'], self.dir_model)
#@TODO: only for compatibility with old models
shutil.move(self.dir_model, self.dir_model + '_delme')
shutil.move(os.path.join(self.dir_model + '_delme', self.info['nn_model']['title']), self.dir_model)
sly.fs.remove_dir(self.dir_model + '_delme')
out_cfg = {
**self.info['task_settings'], # settings from server
'connection': {
'server_address': constants.SERVER_ADDRESS(),
'token': constants.TOKEN(),
'task_id': str(self.info['task_id']),
},
'model_settings': self.info['task_settings']
}
json.dump(out_cfg, open(self.config_path1, 'w')) # Deprecated 'task_settings.json'
json.dump(out_cfg, open(self.config_path2, 'w')) # New style task_config.json
self.report_step_done(TaskStep.DOWNLOAD)
def __init__(self):
self.logger = sly.get_task_logger('agent')
sly.change_formatters_default_values(self.logger, 'service_type',
sly.ServiceType.AGENT)
sly.change_formatters_default_values(self.logger, 'event_type',
sly.EventType.LOGJ)
self.log_queue = LogQueue()
add_task_handler(self.logger, self.log_queue)
sly.add_default_logging_into_file(self.logger,
constants.AGENT_LOG_DIR())
self.logger.info('Agent comes back...')
self.task_pool_lock = threading.Lock()
self.task_pool = {} # task_id -> task_manager (process_id)
self.thread_pool = ThreadPoolExecutor(max_workers=10)
self.thread_list = []
self.daemons_list = []
sly.fs.clean_dir(constants.AGENT_TMP_DIR())
self._stop_missed_containers()
self.docker_api = docker.from_env(version='auto')
self._docker_login()
self.logger.info('Agent is ready to get tasks.')
self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(),
self.logger, constants.TIMEOUT_CONFIG_PATH())
self.agent_connect_initially()
self.logger.info('Agent connected to server.')
def init_api(self):
self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(), self.logger, constants.TIMEOUT_CONFIG_PATH())
if 'user_api_key' in self.info:
self.public_api = sly.Api(constants.SERVER_ADDRESS(), self.info['user_api_key'])
self.public_api.add_additional_field('taskId', self.info['task_id'])
self.public_api_context = self.public_api.task.get_context(self.info['task_id'])
def _validate_duplicated_agents(self):
dc = docker.from_env()
agent_same_token = []
for cont in dc.containers.list():
if constants.TOKEN() in cont.name:
agent_same_token.append(cont)
if len(agent_same_token) > 1:
raise RuntimeError("Agent with the same token already exists.")
def init_api(self):
self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(), self.logger, constants.TIMEOUT_CONFIG_PATH())
if self._user_api_key is not None:
self.public_api = sly.Api(constants.SERVER_ADDRESS(), self._user_api_key, external_logger=self.logger,
retry_count=constants.PUBLIC_API_RETRY_LIMIT())
task_id = self.info['task_id']
self.public_api.add_additional_field('taskId', task_id)
self.public_api.add_header('x-task-id', str(task_id))
self.public_api_context = self.public_api.task.get_context(task_id)
def _remove_old_agent(self):
container_id = os.getenv('REMOVE_OLD_AGENT', None)
if container_id is None:
return
dc = docker.from_env()
olg_agent = dc.containers.get(container_id)
olg_agent.remove(force=True)
agent_same_token = []
for cont in dc.containers.list():
if constants.TOKEN() in cont.name:
agent_same_token.append(cont)
if len(agent_same_token) > 1:
raise RuntimeError(
"Several agents with the same token are running. Please, kill them or contact support."
)
agent_same_token[0].rename('supervisely-agent-{}'.format(
constants.TOKEN()))
def spawn_container(self, add_envs=None):
if add_envs is None:
add_envs = {}
self._container_lock.acquire()
try:
#@TODO: DEBUG_COPY_IMAGES only for compatibility with old plugins
self._container = self._docker_api.containers.run(
self.docker_image_name,
runtime=self.docker_runtime,
entrypoint=[
"sh", "-c", "python -u {}".format(self.entrypoint)
],
detach=True,
name='sly_task_{}_{}'.format(constants.TOKEN(),
self.info['task_id']),
remove=False,
volumes={
self.dir_task_host: {
'bind': '/sly_task_data',
'mode': 'rw'
}
},
environment={
'LOG_LEVEL': 'DEBUG',
'LANG': 'C.UTF-8',
'DEBUG_COPY_IMAGES': 1,
**add_envs
},
labels={
'ecosystem': 'supervisely',
'ecosystem_token': constants.TASKS_DOCKER_LABEL(),
'task_id': str(self.info['task_id'])
},
shm_size="1G",
stdin_open=False,
tty=False)
self._container.reload()
self.logger.debug('After spawning. Container status: {}'.format(
str(self._container.status)))
self.logger.info('Docker container spawned',
extra={
'container_id': self._container.id,
'container_name': self._container.name
})
finally:
self._container_lock.release()
def _exec_command(self, command, add_envs=None, container_id=None):
add_envs = sly.take_with_default(add_envs, {})
self._exec_id = self._docker_api.api.exec_create(self._container.id if container_id is None else container_id,
cmd=command,
environment={
'LOG_LEVEL': 'DEBUG',
'LANG': 'C.UTF-8',
'PYTHONUNBUFFERED': '1',
constants._HTTP_PROXY: constants.HTTP_PROXY(),
constants._HTTPS_PROXY: constants.HTTPS_PROXY(),
'HOST_TASK_DIR': self.dir_task_host,
'TASK_ID': self.info['task_id'],
'SERVER_ADDRESS': self.info['server_address'],
'API_TOKEN': self.info['api_token'],
'AGENT_TOKEN': constants.TOKEN(),
**add_envs
})
self._logs_output = self._docker_api.api.exec_start(self._exec_id, stream=True, demux=False)
def __init__(self):
self.logger = sly.get_task_logger('agent')
sly.change_formatters_default_values(self.logger, 'service_type',
sly.ServiceType.AGENT)
sly.change_formatters_default_values(self.logger, 'event_type',
sly.EventType.LOGJ)
self.log_queue = LogQueue()
add_task_handler(self.logger, self.log_queue)
sly.add_default_logging_into_file(self.logger,
constants.AGENT_LOG_DIR())
self._stop_log_event = threading.Event()
self.executor_log = ThreadPoolExecutor(max_workers=1)
self.future_log = None
self.logger.info('Agent comes back...')
self.task_pool_lock = threading.Lock()
self.task_pool = {} # task_id -> task_manager (process_id)
self.thread_pool = ThreadPoolExecutor(max_workers=10)
self.thread_list = []
self.daemons_list = []
self._remove_old_agent()
self._validate_duplicated_agents()
sly.fs.clean_dir(constants.AGENT_TMP_DIR())
self._stop_missed_containers(constants.TASKS_DOCKER_LABEL())
# for compatibility with old plugins
self._stop_missed_containers(constants.TASKS_DOCKER_LABEL_LEGACY())
self.docker_api = docker.from_env(
version='auto', timeout=constants.DOCKER_API_CALL_TIMEOUT())
self._docker_login()
self.logger.info('Agent is ready to get tasks.')
self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(),
self.logger, constants.TIMEOUT_CONFIG_PATH())
self.agent_connect_initially()
self.logger.info('Agent connected to server.')
