def test_is_correct(self): tk = TaskHeaderKeeper(EnvironmentsManager(), 10) th = get_dict_task_header() correct, err = tk.is_correct(th) assert correct assert err is None th['deadline'] = datetime.now() correct, err = tk.is_correct(th) assert not correct assert err == "Deadline is not a timestamp" th['deadline'] = get_timestamp_utc() - 10 correct, err = tk.is_correct(th) assert not correct assert err == "Deadline already passed" th['deadline'] = get_timestamp_utc() + 20 correct, err = tk.is_correct(th) assert correct assert err is None th['subtask_timeout'] = "abc" correct, err = tk.is_correct(th) assert not correct assert err == "Subtask timeout is not a number" th['subtask_timeout'] = -131 correct, err = tk.is_correct(th) assert not correct assert err == "Subtask timeout is less than 0"
def test_change_timeouts(self, mock_addr): mock_addr.return_value = self.addr_return t = self._get_task_mock(timeout=20, subtask_timeout=40) sync_wait(self.tm.add_new_task(t)) assert get_timestamp_utc() + 15 <= t.header.deadline assert t.header.deadline <= get_timestamp_utc() + 20 assert t.header.subtask_timeout == 40 self.tm.change_timeouts("xyz", 60, 10) assert get_timestamp_utc() + 55 <= t.header.deadline assert t.header.deadline <= get_timestamp_utc() + 60 assert t.header.subtask_timeout == 10
def test_check_comp_task_def(self): ctk = CompTaskKeeper(self.new_path) header = get_task_header() task_id = header.task_id ctk.add_request(header, 40003) ctk.active_tasks[task_id].requests = 0 subtask_id = idgenerator.generate_new_id_from_id(task_id) comp_task_def = { 'task_id': task_id, 'subtask_id': subtask_id, 'deadline': get_timestamp_utc() + 100, } with self.assertLogs(logger, level="INFO") as logs: assert not ctk.check_comp_task_def(comp_task_def) assert 'Cannot accept subtask %s for task %s. ' \ 'Request for this task was not send.' % (subtask_id, task_id)\ in logs.output[0] ctk.active_tasks[task_id].requests = 1 comp_task_def['deadline'] = 0 with self.assertLogs(logger, level="INFO") as logs: assert not ctk.check_comp_task_def(comp_task_def) assert 'Cannot accept subtask %s for task %s. ' \ 'Request for this task has wrong deadline 0' % (subtask_id, task_id) \ in logs.output[0] comp_task_def['deadline'] = get_timestamp_utc() + 240 with self.assertLogs(logger, level="INFO"): assert not ctk.check_comp_task_def(comp_task_def) comp_task_def['deadline'] = get_timestamp_utc() + 100 assert ctk.check_comp_task_def(comp_task_def) ctk.active_tasks[task_id].subtasks[subtask_id] = comp_task_def with self.assertLogs(logger, level="INFO") as logs: assert not ctk.check_comp_task_def(comp_task_def) assert 'Cannot accept subtask %s for task %s. ' \ 'Definition of this subtask was already received.' % (subtask_id, task_id) \ in logs.output[0] del ctk.active_tasks[task_id].subtasks[subtask_id] assert ctk.check_comp_task_def(comp_task_def) comp_task_def['subtask_id'] = "abc" with self.assertLogs(logger, level="INFO") as log_: assert not ctk.check_comp_task_def(comp_task_def) assert "Cannot accept subtask abc for task %s. " \ "Subtask id was not generated from requestor's " \ "key." % (task_id) in log_.output[0]
def check_timeouts(self): nodes_with_timeouts = [] for t in list(self.tasks.values()): th = t.header if self.tasks_states[th.task_id].status not in self.activeStatus: continue cur_time = get_timestamp_utc() # Check subtask timeout ts = self.tasks_states[th.task_id] for s in list(ts.subtask_states.values()): if s.subtask_status.is_computed(): if cur_time > s.deadline: logger.info("Subtask %r dies with status %r", s.subtask_id, s.subtask_status.value) s.subtask_status = SubtaskStatus.failure nodes_with_timeouts.append(s.node_id) t.computation_failed(s.subtask_id) s.stderr = "[GOLEM] Timeout" self.notice_task_updated(th.task_id, subtask_id=s.subtask_id, op=SubtaskOp.TIMEOUT) # Check task timeout if cur_time > th.deadline: logger.info("Task %r dies", th.task_id) self.tasks_states[th.task_id].status = TaskStatus.timeout # TODO: t.tell_it_has_timeout()? self.notice_task_updated(th.task_id, op=TaskOp.TIMEOUT) return nodes_with_timeouts
def test_validate_deadline_passed(self): self.th_dict_repr['fixed_header']['deadline'] = get_timestamp_utc() - 10 with self.assertRaisesRegex( exceptions.TaskHeaderError, "Deadline already passed" ): TaskHeader.validate(self.th_dict_repr)
def __init__(self, mask: Optional[Mask] = None, timestamp: Optional[float] = None, signature: Optional[bytes] = None, *args, **kwargs) -> None: self.fixed_header = TaskFixedHeader(*args, **kwargs) self.mask = mask or Mask() self.timestamp = timestamp or get_timestamp_utc() self.signature = signature
def remove_old_tasks(self): for t in self.task_headers.values(): cur_time = get_timestamp_utc() if cur_time > t.deadline: logger.warning("Task {} dies".format(t.task_id)) self.remove_task_header(t.task_id) for task_id, remove_time in self.removed_tasks.items(): cur_time = time.time() if cur_time - remove_time > self.removed_task_timeout: del self.removed_tasks[task_id]
def remove_old_tasks(self): for t in list(self.task_headers.values()): cur_time = common.get_timestamp_utc() if cur_time > t.deadline: logger.warning("Task owned by %s dies, task_id: %s", t.task_owner.key, t.task_id) self.remove_task_header(t.task_id) for task_id, remove_time in list(self.removed_tasks.items()): cur_time = time.time() if cur_time - remove_time > self.removed_task_timeout: del self.removed_tasks[task_id]
def test_to_dictionary(): ss = SubtaskState() ss.subtask_definition = "My long task definition" ss.subtask_id = "ABCDEF" ss.subtask_progress = 0.92 ss.time_started = get_timestamp_utc() ss.deadline = timeout_to_deadline(ss.time_started + 5) ss.extra_data = {"param1": 1323, "param2": "myparam"} ss.subtask_rem_time = deadline_to_timeout( ss.deadline) - ss.time_started ss.subtask_status = SubtaskStatus.starting ss.value = 138 ss.stdout = "path/to/file" ss.stderr = "path/to/file2" ss.results = ["path/to/file3", "path/to/file4"] ss.computation_time = 130 ss.node_id = "NODE1" ss_dict = ss.to_dictionary() assert ss_dict['description'] == "My long task definition" assert ss_dict['subtask_id'] == "ABCDEF" assert ss_dict['progress'] == 0.92 assert ss_dict['time_started'] == get_timestamp_utc() assert ss_dict.get('deadline') is None assert ss_dict.get('extra_data') is None assert ss_dict['time_remaining'] == 5 assert ss_dict['status'] == SubtaskStatus.starting.value assert ss_dict.get('value') is None assert ss_dict['stdout'] == "path/to/file" assert ss_dict['stderr'] == "path/to/file2" assert ss_dict['results'] == ["path/to/file3", "path/to/file4"] assert ss_dict.get('computation_time') is None assert ss_dict['node_id'] == "NODE1"
def check_deadline(self, deadline: float) -> bool: """ Checks if subtask deadline defined in newly received ComputeTaskDef is properly set, ie. it's set to future date, but not much further than it was declared in subtask timeout. :param float deadline: subtask deadline :return bool: """ now_ = common.get_timestamp_utc() expected_deadline = now_ + self.header.subtask_timeout if now_ < deadline < expected_deadline + MTD.seconds: return True logger.debug( 'check_deadline failed: (now: %r, deadline: %r, ' 'timeout: %r)', now_, deadline, self.header.subtask_timeout) return False
def remove_old_tasks(self): for task_id in frozenset(self.active_tasks): deadline = self.active_tasks[task_id].keeping_deadline delta = deadline - common.get_timestamp_utc() if delta > 0: continue logger.info("Removing comp_task after deadline: %s", task_id) for subtask_id in self.active_tasks[task_id].subtasks: del self.subtask_to_task[subtask_id] del self.active_tasks[task_id] if task_id in self.task_package_paths: del self.task_package_paths[task_id] self.dump()
def is_correct(th_dict_repr): """Checks if task header dict representation has correctly defined parameters :param dict th_dict_repr: task header dictionary representation :return (bool, error): First element is True if task is properly defined (the second element is then None). Otheriwse first element is False and the second is the string describing wrong element """ if not isinstance(th_dict_repr['deadline'], (int, long, float)): return False, "Deadline is not a timestamp" if th_dict_repr['deadline'] < get_timestamp_utc(): return False, "Deadline already passed" if not isinstance(th_dict_repr['subtask_timeout'], int): return False, "Subtask timeout is not a number" if th_dict_repr['subtask_timeout'] < 0: return False, "Subtask timeout is less than 0" return True, None
def check_timeouts(self): nodes_with_timeouts = [] for t in self.tasks.values(): th = t.header if self.tasks_states[th.task_id].status not in self.activeStatus: continue cur_time = get_timestamp_utc() if cur_time > th.deadline: logger.info("Task {} dies".format(th.task_id)) t.task_stats = TaskStatus.timeout self.tasks_states[th.task_id].status = TaskStatus.timeout self.notice_task_updated(th.task_id) ts = self.tasks_states[th.task_id] for s in ts.subtask_states.values(): if SubtaskStatus.is_computed(s.subtask_status): if cur_time > s.deadline: logger.info("Subtask {} dies".format(s.subtask_id)) s.subtask_status = SubtaskStatus.failure nodes_with_timeouts.append(s.computer.node_id) t.computation_failed(s.subtask_id) s.stderr = "[GOLEM] Timeout" self.notice_task_updated(th.task_id) return nodes_with_timeouts
def do_maintenance(self): """Updates information on unsupported task reasons and other related task statistics by consuming tasks and support statuses scheduled for processing by add_task() and add_support_status() functions. Optimizes internal structures and, if needed, writes the entire structure to a file. """ input_tasks, self._input_tasks = self._input_tasks, [] input_statuses, self._input_statuses = self._input_statuses, [] with self._archive_lock: ntasks_to_take = self._max_tasks - len(self._archive.tasks) if ntasks_to_take < len(input_tasks): log.warning("Maximum number of current tasks exceeded.") input_tasks = input_tasks[:ntasks_to_take] for tsk in input_tasks: self._archive.tasks[tsk.uuid] = tsk for (uuid, status) in input_statuses: if uuid in self._archive.tasks: if UnsupportReason.REQUESTOR_TRUST in status.desc: self._archive.tasks[uuid].requesting_trust = \ status.desc[UnsupportReason.REQUESTOR_TRUST] self._archive.tasks[uuid].unsupport_reasons = \ list(status.desc.keys()) cur_time = get_timestamp_utc() for tsk in list(self._archive.tasks.values()): if cur_time > tsk.deadline: self._merge_to_interval(tsk) del self._archive.tasks[tsk.uuid] self._purge_old_intervals() if self._dump_file: request = golem_async.AsyncRequest(self._dump_archive) golem_async.async_run( request, None, lambda e: log.info("Dumping archive failed: %s", e), )
def test_datetime_to_timestamp(self): ts = get_timestamp_utc() assert ts dt = timestamp_to_datetime(ts) assert round(datetime_to_timestamp(dt), 5) == round(ts, 5)
def test_datetime_to_timestamp(self): ts = get_timestamp_utc() assert ts dt = timestamp_to_datetime(ts) assert datetime_to_timestamp(dt) == ts
def validate(th_dict_repr: dict) -> None: """Checks if task header dict representation has correctly defined parameters :param dict th_dict_repr: task header dictionary representation """ task_id = th_dict_repr.get('task_id') task_owner = th_dict_repr.get('task_owner') try: node_name = task_owner.get('node_name') # type: ignore except AttributeError: raise exceptions.TaskHeaderError( 'Task owner missing', task_id=task_id, task_owner=task_owner, ) if not isinstance(task_id, str): raise exceptions.TaskHeaderError( 'Task ID missing', task_id=task_id, node_name=node_name, ) if not isinstance(node_name, str): raise exceptions.TaskHeaderError( 'Task owner node name missing', task_id=task_id, node_name=node_name, ) if not isinstance(th_dict_repr['deadline'], (int, float)): raise exceptions.TaskHeaderError( "Deadline is not a timestamp", task_id=task_id, node_name=node_name, deadline=th_dict_repr['deadline'], ) now = common.get_timestamp_utc() if th_dict_repr['deadline'] < now: raise exceptions.TaskHeaderError( "Deadline already passed", task_id=task_id, node_name=node_name, deadline=th_dict_repr['deadline'], now=now, ) if not isinstance(th_dict_repr['subtask_timeout'], int): raise exceptions.TaskHeaderError( "Subtask timeout is not a number", task_id=task_id, node_name=node_name, subtask_timeout=th_dict_repr['subtask_timeout'], ) if th_dict_repr['subtask_timeout'] < 0: raise exceptions.TaskHeaderError( "Subtask timeout is less than 0", task_id=task_id, node_name=node_name, subtask_timeout=th_dict_repr['subtask_timeout'], ) try: if th_dict_repr['subtasks_count'] < 1: raise exceptions.TaskHeaderError( "Subtasks count is less than 1", task_id=task_id, node_name=node_name, subtasks_count=th_dict_repr['subtasks_count'], ) except (KeyError, TypeError): raise exceptions.TaskHeaderError( "Subtasks count is missing", task_id=task_id, node_name=node_name, subtask_count=th_dict_repr.get('subtask_count'), )