def __init__(self, task_info):
super().__init__()
self.daemon = True
self.info = deepcopy(task_info)
# Move API key out of the task info message so that it does not get into
# logs.
self._user_api_key = self.info.pop('user_api_key', None)
self.dir_task = osp.join(constants.AGENT_TASKS_DIR(),
str(self.info['task_id']))
self.dir_logs = osp.join(self.dir_task, 'logs')
sly.fs.mkdir(self.dir_task)
sly.fs.mkdir(self.dir_logs)
self.dir_task_host = osp.join(constants.AGENT_TASKS_DIR_HOST(),
str(self.info['task_id']))
self._stop_log_event = threading.Event()
self._stop_event = multiprocessing.Event()
# pre-init for static analysis
self.logger = None
self.log_queue = None
self.executor_log = None
self.future_log = None
self.api = None
self.data_mgr = None
self.public_api = None
self.public_api_context = None
def get_telemetry_str(self):
htop_str = 'echo q | htop -C | ' \
'aha --line-fix | html2text -width 999 | grep -v "F1Help" | grep -v "xml version="'
htop_output = self._get_subprocess_out_if_possible('htop', [htop_str])
nvsmi_output = self.NO_OUTPUT
try:
nvsmi_output = self._get_subprocess_out_if_possible_no_shell(['nvidia-smi'], timeout=5)
except OSError:
pass
docker_inspect_cmd = "curl -s --unix-socket /var/run/docker.sock http:/containers/$(hostname)/json"
docker_inspect_out = self._get_subprocess_out_if_possible('curl', [docker_inspect_cmd]).decode("utf-8")
docker_image = json.loads(docker_inspect_out)["Config"]["Image"]
img_sizeb = get_directory_size_bytes(self.data_mgr.storage.images.storage_root_path)
nn_sizeb = get_directory_size_bytes(self.data_mgr.storage.nns.storage_root_path)
tasks_sizeb = get_directory_size_bytes(constants.AGENT_TASKS_DIR())
node_storage = [
{'Images': bytes_to_human(img_sizeb)},
{'NN weights': bytes_to_human(nn_sizeb)},
{'Tasks': bytes_to_human(tasks_sizeb)},
{'Total': bytes_to_human(img_sizeb + nn_sizeb + tasks_sizeb)},
]
server_info = {
'htop': htop_output.decode("utf-8"),
'nvsmi': nvsmi_output.decode("utf-8"),
'node_storage': node_storage,
'docker_image': docker_image
}
info_str = json.dumps(server_info)
return info_str
def clean_tasks_dir(self):
self.logger.info('Will remove temporary tasks data.')
task_dir = constants.AGENT_TASKS_DIR()
task_names = os.listdir(task_dir)
for subdir_n in task_names:
dir_task = osp.join(task_dir, subdir_n)
TaskDirCleaner(dir_task).clean_forced()
self.logger.info('Temporary tasks data has been removed.')
def __init__(self, task_info):
super().__init__()
self.daemon = True
self.info = task_info
self.dir_task = osp.join(constants.AGENT_TASKS_DIR(), str(self.info['task_id']))
self.dir_logs = osp.join(self.dir_task, 'logs')
sly.fs.mkdir(self.dir_task)
sly.fs.mkdir(self.dir_logs)
self.dir_task_host = osp.join(constants.AGENT_TASKS_DIR_HOST(), str(self.info['task_id']))
self._stop_log_event = threading.Event()
self._stop_event = multiprocessing.Event()
# pre-init for static analysis
self.logger = None
self.log_queue = None
self.executor_log = None
self.future_log = None
self.api = None
self.data_mgr = None
self.public_api = None
self.public_api_context = None
def init_task_dir(self):
self.dir_task = osp.join(constants.AGENT_TASKS_DIR(),
str(self.info['task_id']))
self.dir_task_host = osp.join(constants.AGENT_TASKS_DIR_HOST(),
str(self.info['task_id']))