def __init__(self):
self.logger = sly.get_task_logger('agent')
sly.change_formatters_default_values(self.logger, 'service_type',
sly.ServiceType.AGENT)
sly.change_formatters_default_values(self.logger, 'event_type',
sly.EventType.LOGJ)
self.log_queue = LogQueue()
add_task_handler(self.logger, self.log_queue)
sly.add_default_logging_into_file(self.logger,
constants.AGENT_LOG_DIR())
self.logger.info('Agent comes back...')
self.task_pool_lock = threading.Lock()
self.task_pool = {} # task_id -> task_manager (process_id)
self.thread_pool = ThreadPoolExecutor(max_workers=10)
self.thread_list = []
self.daemons_list = []
sly.fs.clean_dir(constants.AGENT_TMP_DIR())
self._stop_missed_containers()
self.docker_api = docker.from_env(version='auto')
self._docker_login()
self.logger.info('Agent is ready to get tasks.')
self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(),
self.logger, constants.TIMEOUT_CONFIG_PATH())
self.agent_connect_initially()
self.logger.info('Agent connected to server.')
def upload_archive(self, task_id, dir_to_archive, archive_name):
self.logger.info("PACK_TO_ARCHIVE ...")
archive_name = archive_name if len(archive_name) > 0 else sly.rand_str(30)
local_tar_path = os.path.join(constants.AGENT_TMP_DIR(), archive_name + '.tar')
sly.fs.archive_directory(dir_to_archive, local_tar_path)
try:
self.upload_tar_file(task_id, local_tar_path)
finally:
sly.fs.silent_remove(local_tar_path)
self.logger.info('ARCHIVE_UPLOADED', extra={'archive_name': archive_name})
def upload_archive(self, task_id, dir_to_archive, archive_name):
self.logger.info("PACK_TO_ARCHIVE ...")
archive_name = archive_name if len(archive_name) > 0 else sly.rand_str(30)
local_tar_path = os.path.join(constants.AGENT_TMP_DIR(), archive_name + '.tar')
sly.fs.archive_directory(dir_to_archive, local_tar_path)
size_mb = sly.fs.get_file_size(local_tar_path) / 1024.0 / 1024
progress = sly.Progress("Upload archive", size_mb, ext_logger=self.logger)
try:
self.public_api.task.upload_dtl_archive(task_id, local_tar_path, progress.set_current_value)
finally:
sly.fs.silent_remove(local_tar_path)
self.logger.info('ARCHIVE_UPLOADED', extra={'archive_name': archive_name})
def upload_nn(self, nn_id, nn_hash):
local_service_log = {'nn_id': nn_id, 'nn_hash': nn_hash}
storage_nn_dir = self.storage.nns.check_storage_object(nn_hash)
if storage_nn_dir is None:
self.logger.critical('NN_NOT_FOUND', extra=local_service_log)
local_tar_path = os.path.join(constants.AGENT_TMP_DIR(), sly.rand_str(30) + '.tar')
sly.fs.archive_directory(storage_nn_dir, local_tar_path)
size_mb = sly.fs.get_file_size(local_tar_path) / 1024.0 / 1024
progress = sly.Progress("Upload NN weights", size_mb, ext_logger=self.logger)
try:
self.public_api.model.upload(nn_hash, local_tar_path, progress.set_current_value)
finally:
sly.fs.silent_remove(local_tar_path)
self.logger.info('ARCHIVE_UPLOADED')
self.logger.info('NN_UPLOADED', extra=local_service_log)
def __init__(self):
self.logger = sly.get_task_logger('agent')
sly.change_formatters_default_values(self.logger, 'service_type',
sly.ServiceType.AGENT)
sly.change_formatters_default_values(self.logger, 'event_type',
sly.EventType.LOGJ)
self.log_queue = LogQueue()
add_task_handler(self.logger, self.log_queue)
sly.add_default_logging_into_file(self.logger,
constants.AGENT_LOG_DIR())
self._stop_log_event = threading.Event()
self.executor_log = ThreadPoolExecutor(max_workers=1)
self.future_log = None
self.logger.info('Agent comes back...')
self.task_pool_lock = threading.Lock()
self.task_pool = {} # task_id -> task_manager (process_id)
self.thread_pool = ThreadPoolExecutor(max_workers=10)
self.thread_list = []
self.daemons_list = []
self._remove_old_agent()
self._validate_duplicated_agents()
sly.fs.clean_dir(constants.AGENT_TMP_DIR())
self._stop_missed_containers(constants.TASKS_DOCKER_LABEL())
# for compatibility with old plugins
self._stop_missed_containers(constants.TASKS_DOCKER_LABEL_LEGACY())
self.docker_api = docker.from_env(
version='auto', timeout=constants.DOCKER_API_CALL_TIMEOUT())
self._docker_login()
self.logger.info('Agent is ready to get tasks.')
self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(),
self.logger, constants.TIMEOUT_CONFIG_PATH())
self.agent_connect_initially()
self.logger.info('Agent connected to server.')