def _update_frame_task_preview(self):
sent_color = (0, 255, 0)
failed_color = (255, 0, 0)
for sub in self.subtasks_given.values():
if SubtaskStatus.is_computed(sub['status']):
for frame in sub['frames']:
self.__mark_sub_frame(sub, frame, sent_color)
if sub['status'] in [
SubtaskStatus.failure, SubtaskStatus.restarted
]:
for frame in sub['frames']:
self.__mark_sub_frame(sub, frame, failed_color)
def restart_subtask(self, subtask_id):
subtask_info = self.subtasks_given[subtask_id]
was_failure_before = subtask_info['status'] in [
SubtaskStatus.failure, SubtaskStatus.resent
]
if SubtaskStatus.is_computed(subtask_info['status']):
self._mark_subtask_failed(subtask_id)
elif subtask_info['status'] == SubtaskStatus.finished:
self._mark_subtask_failed(subtask_id)
tasks = subtask_info['end_task'] - subtask_info['start_task'] + 1
self.num_tasks_received -= tasks
if not was_failure_before:
subtask_info['status'] = SubtaskStatus.restarted
def _update_task_preview(self):
sent_color = (0, 255, 0)
failed_color = (255, 0, 0)
self.preview_task_file_path = "{}".format(
os.path.join(self.tmp_dir, "current_task_preview"))
img_task = self._open_preview()
for sub in self.subtasks_given.values():
if SubtaskStatus.is_computed(sub['status']):
self._mark_task_area(sub, img_task, sent_color)
if sub['status'] in [
SubtaskStatus.failure, SubtaskStatus.restarted
]:
self._mark_task_area(sub, img_task, failed_color)
img_task.save(self.preview_task_file_path, "BMP")
self._update_preview_task_file_path(self.preview_task_file_path)
def computed_task_received(self, subtask_id, result, result_type):
task_id = self.subtask2task_mapping[subtask_id]
subtask_state = self.tasks_states[task_id].subtask_states[subtask_id]
subtask_status = subtask_state.subtask_status
if not SubtaskStatus.is_computed(subtask_status):
logger.warning(
"Result for subtask {} when subtask state is {}".format(
subtask_id, subtask_status))
self.notice_task_updated(task_id)
return False
self.tasks[task_id].computation_finished(subtask_id, result,
result_type)
ss = self.tasks_states[task_id].subtask_states[subtask_id]
ss.subtask_progress = 1.0
ss.subtask_rem_time = 0.0
ss.subtask_status = SubtaskStatus.finished
ss.stdout = self.tasks[task_id].get_stdout(subtask_id)
ss.stderr = self.tasks[task_id].get_stderr(subtask_id)
ss.results = self.tasks[task_id].get_results(subtask_id)
if not self.tasks[task_id].verify_subtask(subtask_id):
logger.debug("Subtask {} not accepted\n".format(subtask_id))
ss.subtask_status = SubtaskStatus.failure
self.notice_task_updated(task_id)
return False
if self.tasks_states[task_id].status in self.activeStatus:
if not self.tasks[task_id].finished_computation():
self.tasks_states[task_id].status = TaskStatus.computing
else:
if self.tasks[task_id].verify_task():
logger.debug("Task {} accepted".format(task_id))
self.tasks_states[task_id].status = TaskStatus.finished
else:
logger.debug("Task {} not accepted".format(task_id))
self.notice_task_updated(task_id)
return True
def task_computation_failure(self, subtask_id, err):
task_id = self.subtask2task_mapping[subtask_id]
subtask_state = self.tasks_states[task_id].subtask_states[subtask_id]
subtask_status = subtask_state.subtask_status
if not SubtaskStatus.is_computed(subtask_status):
logger.warning(
"Result for subtask {} when subtask state is {}".format(
subtask_id, subtask_status))
self.notice_task_updated(task_id)
return False
self.tasks[task_id].computation_failed(subtask_id)
ss = self.tasks_states[task_id].subtask_states[subtask_id]
ss.subtask_progress = 1.0
ss.subtask_rem_time = 0.0
ss.subtask_status = SubtaskStatus.failure
ss.stderr = str(err)
self.notice_task_updated(task_id)
return True
def check_timeouts(self):
nodes_with_timeouts = []
for t in self.tasks.values():
th = t.header
if self.tasks_states[th.task_id].status not in self.activeStatus:
continue
cur_time = get_timestamp_utc()
if cur_time > th.deadline:
logger.info("Task {} dies".format(th.task_id))
t.task_stats = TaskStatus.timeout
self.tasks_states[th.task_id].status = TaskStatus.timeout
self.notice_task_updated(th.task_id)
ts = self.tasks_states[th.task_id]
for s in ts.subtask_states.values():
if SubtaskStatus.is_computed(s.subtask_status):
if cur_time > s.deadline:
logger.info("Subtask {} dies".format(s.subtask_id))
s.subtask_status = SubtaskStatus.failure
nodes_with_timeouts.append(s.computer.node_id)
t.computation_failed(s.subtask_id)
s.stderr = "[GOLEM] Timeout"
self.notice_task_updated(th.task_id)
return nodes_with_timeouts
def should_accept(self, subtask_id):
status = self.subtasks_given[subtask_id]['status']
return SubtaskStatus.is_computed(status)