def __init__(self):
self.logger = get_task_logger('agent')
sly.change_formatters_default_values(self.logger, 'service_type',
ServiceType.AGENT)
sly.change_formatters_default_values(self.logger, 'event_type',
EventType.LOGJ)
self.log_queue = LogQueue()
add_task_handler(self.logger, self.log_queue)
sly.add_default_logging_into_file(self.logger, constants.AGENT_LOG_DIR)
self.task_pool_lock = threading.Lock()
self.task_pool = {} # task_id -> task_manager (process_id)
self.thread_pool = ThreadPoolExecutor(max_workers=10)
self.thread_list = []
self.api = AgentAPI(constants.TOKEN, constants.SERVER_ADDRESS,
self.logger)
self.agent_connect_initially()
sly.clean_dir(constants.AGENT_TMP_DIR)
self._stop_missed_containers()
self.docker_api = docker.from_env(version='auto')
self._docker_login()
self.logger.info('Agent is ready to get tasks.',
extra={'event_type': EventType.AGENT_READY_FOR_TASKS})
def __init__(self):
self.logger = sly.get_task_logger('agent')
sly.change_formatters_default_values(self.logger, 'service_type',
sly.ServiceType.AGENT)
sly.change_formatters_default_values(self.logger, 'event_type',
sly.EventType.LOGJ)
self.log_queue = LogQueue()
add_task_handler(self.logger, self.log_queue)
sly.add_default_logging_into_file(self.logger,
constants.AGENT_LOG_DIR())
self.logger.info('Agent comes back...')
self.task_pool_lock = threading.Lock()
self.task_pool = {} # task_id -> task_manager (process_id)
self.thread_pool = ThreadPoolExecutor(max_workers=10)
self.thread_list = []
self.daemons_list = []
sly.fs.clean_dir(constants.AGENT_TMP_DIR())
self._stop_missed_containers()
self.docker_api = docker.from_env(version='auto')
self._docker_login()
self.logger.info('Agent is ready to get tasks.')
self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(),
self.logger, constants.TIMEOUT_CONFIG_PATH())
self.agent_connect_initially()
self.logger.info('Agent connected to server.')
def init_logger(self, loglevel=None):
self.logger = sly.get_task_logger(self.info['task_id'], loglevel=loglevel)
sly.change_formatters_default_values(self.logger, 'service_type', sly.ServiceType.AGENT)
sly.change_formatters_default_values(self.logger, 'event_type', sly.EventType.LOGJ)
self.log_queue = LogQueue()
add_task_handler(self.logger, self.log_queue)
sly.add_default_logging_into_file(self.logger, self.dir_logs)
self.executor_log = concurrent.futures.ThreadPoolExecutor(max_workers=1)
def __init__(self):
self.logger = sly.get_task_logger('agent')
sly.change_formatters_default_values(self.logger, 'service_type',
sly.ServiceType.AGENT)
sly.change_formatters_default_values(self.logger, 'event_type',
sly.EventType.LOGJ)
self.log_queue = LogQueue()
add_task_handler(self.logger, self.log_queue)
sly.add_default_logging_into_file(self.logger,
constants.AGENT_LOG_DIR())
self._stop_log_event = threading.Event()
self.executor_log = ThreadPoolExecutor(max_workers=1)
self.future_log = None
self.logger.info('Agent comes back...')
self.task_pool_lock = threading.Lock()
self.task_pool = {} # task_id -> task_manager (process_id)
self.thread_pool = ThreadPoolExecutor(max_workers=10)
self.thread_list = []
self.daemons_list = []
self._remove_old_agent()
self._validate_duplicated_agents()
sly.fs.clean_dir(constants.AGENT_TMP_DIR())
self._stop_missed_containers(constants.TASKS_DOCKER_LABEL())
# for compatibility with old plugins
self._stop_missed_containers(constants.TASKS_DOCKER_LABEL_LEGACY())
self.docker_api = docker.from_env(
version='auto', timeout=constants.DOCKER_API_CALL_TIMEOUT())
self._docker_login()
self.logger.info('Agent is ready to get tasks.')
self.api = sly.AgentAPI(constants.TOKEN(), constants.SERVER_ADDRESS(),
self.logger, constants.TIMEOUT_CONFIG_PATH())
self.agent_connect_initially()
self.logger.info('Agent connected to server.')
init_fn = slim.assign_from_checkpoint_fn(
join(self.helper.paths.model_dir, 'model_weights',
'model.ckpt'),
variables_to_restore,
ignore_missing_vars=ignore_missing_vars)
init_fn(sess)
input_shape_hw = (self.input_size_wh[1], self.input_size_wh[0])
train(data_dicts=self.tf_data_dicts,
class_num=len(self.out_classes),
input_size=input_shape_hw,
lr=self.config['lr'],
n_epochs=self.config['epochs'],
num_clones=len(device_ids),
iters_cnt=self.iters_cnt,
val_every=self.config['val_every'],
model_init_fn=init_model_fn,
save_cback=dump_model)
def main():
cv2.setNumThreads(0)
x = DeepLabTrainer()
x.train()
if __name__ == '__main__':
if os.getenv('DEBUG_LOG_TO_FILE', None):
sly.add_default_logging_into_file(logger, sly.TaskPaths().debug_dir)
sly.main_wrapper('DEEPLAB_TRAIN', main)
'AGENT_HOST_DIR',
'SERVER_ADDRESS',
'ACCESS_TOKEN',
'DOCKER_LOGIN',
'DOCKER_PASSWORD',
'DOCKER_REGISTRY',
]}
args_opt = {x: os.getenv(x, def_val) for x, def_val in [
('WITH_LOCAL_STORAGE', 'true'),
('UPLOAD_RESULT_IMAGES', 'false'),
('PULL_ALWAYS', 'true'),
('DEFAULT_TIMEOUTS', 'true'),
('DELETE_TASK_DIR_ON_FINISH', 'true'),
('DELETE_TASK_DIR_ON_FAILURE', 'false'),
('CHECK_VERSION_COMPATIBILITY', 'false')
]}
args = {**args_opt, **args_req}
return args
def main(args):
sly.logger.info('ENVS', extra={**args, 'DOCKER_PASSWORD': '******'})
agent = Agent()
agent.inf_loop()
agent.wait_all()
if __name__ == '__main__':
sly.add_default_logging_into_file(sly.logger, constants.AGENT_LOG_DIR())
sly.main_wrapper('agent', main, parse_envs())
import os
import supervisely_lib as sly
from supervisely_lib.metric.precision_recall_metric import PrecisionRecallMetric
from supervisely_lib.metric.iou_metric import IOU
from supervisely_lib.metric.common import check_class_mapping, CLASSES_MAPPING
from supervisely_lib.io.json import load_json_file
def main():
settings = load_json_file(sly.TaskPaths.SETTINGS_PATH)
sly.logger.info('Input settings:', extra={'config': settings})
if IOU not in settings:
raise RuntimeError(
'"{}" field is missing. Please set Intersection over Union threshold'
.format(IOU))
metric = PrecisionRecallMetric(settings[CLASSES_MAPPING], settings[IOU])
applier = sly.MetricProjectsApplier(metric, settings)
check_class_mapping(applier.project_gt, applier.project_pred,
settings[CLASSES_MAPPING])
applier.run_evaluation()
metric.log_total_metrics()
if __name__ == '__main__':
if os.getenv('DEBUG_LOG_TO_FILE', None):
sly.add_default_logging_into_file(sly.logger, sly.TaskPaths.DEBUG_DIR)
sly.main_wrapper('METRIC_EVALUATION', main)
