def test_is_correct(self):
tk = TaskHeaderKeeper(EnvironmentsManager(), 10)
th = get_dict_task_header()
correct, err = tk.is_correct(th)
assert correct
assert err is None
th['deadline'] = datetime.now()
correct, err = tk.is_correct(th)
assert not correct
assert err == "Deadline is not a timestamp"
th['deadline'] = get_timestamp_utc() - 10
correct, err = tk.is_correct(th)
assert not correct
assert err == "Deadline already passed"
th['deadline'] = get_timestamp_utc() + 20
correct, err = tk.is_correct(th)
assert correct
assert err is None
th['subtask_timeout'] = "abc"
correct, err = tk.is_correct(th)
assert not correct
assert err == "Subtask timeout is not a number"
th['subtask_timeout'] = -131
correct, err = tk.is_correct(th)
assert not correct
assert err == "Subtask timeout is less than 0"
def test_change_timeouts(self, mock_addr):
mock_addr.return_value = self.addr_return
t = self._get_task_mock(timeout=20, subtask_timeout=40)
sync_wait(self.tm.add_new_task(t))
assert get_timestamp_utc() + 15 <= t.header.deadline
assert t.header.deadline <= get_timestamp_utc() + 20
assert t.header.subtask_timeout == 40
self.tm.change_timeouts("xyz", 60, 10)
assert get_timestamp_utc() + 55 <= t.header.deadline
assert t.header.deadline <= get_timestamp_utc() + 60
assert t.header.subtask_timeout == 10
def test_check_comp_task_def(self):
ctk = CompTaskKeeper(self.new_path)
header = get_task_header()
task_id = header.task_id
ctk.add_request(header, 40003)
ctk.active_tasks[task_id].requests = 0
subtask_id = idgenerator.generate_new_id_from_id(task_id)
comp_task_def = {
'task_id': task_id,
'subtask_id': subtask_id,
'deadline': get_timestamp_utc() + 100,
}
with self.assertLogs(logger, level="INFO") as logs:
assert not ctk.check_comp_task_def(comp_task_def)
assert 'Cannot accept subtask %s for task %s. ' \
'Request for this task was not send.' % (subtask_id, task_id)\
in logs.output[0]
ctk.active_tasks[task_id].requests = 1
comp_task_def['deadline'] = 0
with self.assertLogs(logger, level="INFO") as logs:
assert not ctk.check_comp_task_def(comp_task_def)
assert 'Cannot accept subtask %s for task %s. ' \
'Request for this task has wrong deadline 0' % (subtask_id,
task_id) \
in logs.output[0]
comp_task_def['deadline'] = get_timestamp_utc() + 240
with self.assertLogs(logger, level="INFO"):
assert not ctk.check_comp_task_def(comp_task_def)
comp_task_def['deadline'] = get_timestamp_utc() + 100
assert ctk.check_comp_task_def(comp_task_def)
ctk.active_tasks[task_id].subtasks[subtask_id] = comp_task_def
with self.assertLogs(logger, level="INFO") as logs:
assert not ctk.check_comp_task_def(comp_task_def)
assert 'Cannot accept subtask %s for task %s. ' \
'Definition of this subtask was already received.' % (subtask_id,
task_id) \
in logs.output[0]
del ctk.active_tasks[task_id].subtasks[subtask_id]
assert ctk.check_comp_task_def(comp_task_def)
comp_task_def['subtask_id'] = "abc"
with self.assertLogs(logger, level="INFO") as log_:
assert not ctk.check_comp_task_def(comp_task_def)
assert "Cannot accept subtask abc for task %s. " \
"Subtask id was not generated from requestor's " \
"key." % (task_id) in log_.output[0]
def check_timeouts(self):
nodes_with_timeouts = []
for t in list(self.tasks.values()):
th = t.header
if self.tasks_states[th.task_id].status not in self.activeStatus:
continue
cur_time = get_timestamp_utc()
# Check subtask timeout
ts = self.tasks_states[th.task_id]
for s in list(ts.subtask_states.values()):
if s.subtask_status.is_computed():
if cur_time > s.deadline:
logger.info("Subtask %r dies with status %r",
s.subtask_id, s.subtask_status.value)
s.subtask_status = SubtaskStatus.failure
nodes_with_timeouts.append(s.node_id)
t.computation_failed(s.subtask_id)
s.stderr = "[GOLEM] Timeout"
self.notice_task_updated(th.task_id,
subtask_id=s.subtask_id,
op=SubtaskOp.TIMEOUT)
# Check task timeout
if cur_time > th.deadline:
logger.info("Task %r dies", th.task_id)
self.tasks_states[th.task_id].status = TaskStatus.timeout
# TODO: t.tell_it_has_timeout()?
self.notice_task_updated(th.task_id, op=TaskOp.TIMEOUT)
return nodes_with_timeouts
def test_validate_deadline_passed(self):
self.th_dict_repr['fixed_header']['deadline'] = get_timestamp_utc() - 10
with self.assertRaisesRegex(
exceptions.TaskHeaderError,
"Deadline already passed"
):
TaskHeader.validate(self.th_dict_repr)
def __init__(self,
mask: Optional[Mask] = None,
timestamp: Optional[float] = None,
signature: Optional[bytes] = None,
*args,
**kwargs) -> None:
self.fixed_header = TaskFixedHeader(*args, **kwargs)
self.mask = mask or Mask()
self.timestamp = timestamp or get_timestamp_utc()
self.signature = signature
def remove_old_tasks(self):
for t in self.task_headers.values():
cur_time = get_timestamp_utc()
if cur_time > t.deadline:
logger.warning("Task {} dies".format(t.task_id))
self.remove_task_header(t.task_id)
for task_id, remove_time in self.removed_tasks.items():
cur_time = time.time()
if cur_time - remove_time > self.removed_task_timeout:
del self.removed_tasks[task_id]
def remove_old_tasks(self):
for t in list(self.task_headers.values()):
cur_time = common.get_timestamp_utc()
if cur_time > t.deadline:
logger.warning("Task owned by %s dies, task_id: %s",
t.task_owner.key, t.task_id)
self.remove_task_header(t.task_id)
for task_id, remove_time in list(self.removed_tasks.items()):
cur_time = time.time()
if cur_time - remove_time > self.removed_task_timeout:
del self.removed_tasks[task_id]
def test_to_dictionary():
ss = SubtaskState()
ss.subtask_definition = "My long task definition"
ss.subtask_id = "ABCDEF"
ss.subtask_progress = 0.92
ss.time_started = get_timestamp_utc()
ss.deadline = timeout_to_deadline(ss.time_started + 5)
ss.extra_data = {"param1": 1323, "param2": "myparam"}
ss.subtask_rem_time = deadline_to_timeout(
ss.deadline) - ss.time_started
ss.subtask_status = SubtaskStatus.starting
ss.value = 138
ss.stdout = "path/to/file"
ss.stderr = "path/to/file2"
ss.results = ["path/to/file3", "path/to/file4"]
ss.computation_time = 130
ss.node_id = "NODE1"
ss_dict = ss.to_dictionary()
assert ss_dict['description'] == "My long task definition"
assert ss_dict['subtask_id'] == "ABCDEF"
assert ss_dict['progress'] == 0.92
assert ss_dict['time_started'] == get_timestamp_utc()
assert ss_dict.get('deadline') is None
assert ss_dict.get('extra_data') is None
assert ss_dict['time_remaining'] == 5
assert ss_dict['status'] == SubtaskStatus.starting.value
assert ss_dict.get('value') is None
assert ss_dict['stdout'] == "path/to/file"
assert ss_dict['stderr'] == "path/to/file2"
assert ss_dict['results'] == ["path/to/file3", "path/to/file4"]
assert ss_dict.get('computation_time') is None
assert ss_dict['node_id'] == "NODE1"
def check_deadline(self, deadline: float) -> bool:
"""
Checks if subtask deadline defined in newly received ComputeTaskDef
is properly set, ie. it's set to future date, but not much further than
it was declared in subtask timeout.
:param float deadline: subtask deadline
:return bool:
"""
now_ = common.get_timestamp_utc()
expected_deadline = now_ + self.header.subtask_timeout
if now_ < deadline < expected_deadline + MTD.seconds:
return True
logger.debug(
'check_deadline failed: (now: %r, deadline: %r, '
'timeout: %r)', now_, deadline, self.header.subtask_timeout)
return False
def remove_old_tasks(self):
for task_id in frozenset(self.active_tasks):
deadline = self.active_tasks[task_id].keeping_deadline
delta = deadline - common.get_timestamp_utc()
if delta > 0:
continue
logger.info("Removing comp_task after deadline: %s", task_id)
for subtask_id in self.active_tasks[task_id].subtasks:
del self.subtask_to_task[subtask_id]
del self.active_tasks[task_id]
if task_id in self.task_package_paths:
del self.task_package_paths[task_id]
self.dump()
def is_correct(th_dict_repr):
"""Checks if task header dict representation has correctly
defined parameters
:param dict th_dict_repr: task header dictionary representation
:return (bool, error): First element is True if task is properly
defined (the second element is then None).
Otheriwse first element is False and the second is the string
describing wrong element
"""
if not isinstance(th_dict_repr['deadline'], (int, long, float)):
return False, "Deadline is not a timestamp"
if th_dict_repr['deadline'] < get_timestamp_utc():
return False, "Deadline already passed"
if not isinstance(th_dict_repr['subtask_timeout'], int):
return False, "Subtask timeout is not a number"
if th_dict_repr['subtask_timeout'] < 0:
return False, "Subtask timeout is less than 0"
return True, None
def check_timeouts(self):
nodes_with_timeouts = []
for t in self.tasks.values():
th = t.header
if self.tasks_states[th.task_id].status not in self.activeStatus:
continue
cur_time = get_timestamp_utc()
if cur_time > th.deadline:
logger.info("Task {} dies".format(th.task_id))
t.task_stats = TaskStatus.timeout
self.tasks_states[th.task_id].status = TaskStatus.timeout
self.notice_task_updated(th.task_id)
ts = self.tasks_states[th.task_id]
for s in ts.subtask_states.values():
if SubtaskStatus.is_computed(s.subtask_status):
if cur_time > s.deadline:
logger.info("Subtask {} dies".format(s.subtask_id))
s.subtask_status = SubtaskStatus.failure
nodes_with_timeouts.append(s.computer.node_id)
t.computation_failed(s.subtask_id)
s.stderr = "[GOLEM] Timeout"
self.notice_task_updated(th.task_id)
return nodes_with_timeouts
def do_maintenance(self):
"""Updates information on unsupported task reasons and
other related task statistics by consuming tasks and support statuses
scheduled for processing by add_task() and add_support_status()
functions. Optimizes internal structures and, if needed, writes the
entire structure to a file.
"""
input_tasks, self._input_tasks = self._input_tasks, []
input_statuses, self._input_statuses = self._input_statuses, []
with self._archive_lock:
ntasks_to_take = self._max_tasks - len(self._archive.tasks)
if ntasks_to_take < len(input_tasks):
log.warning("Maximum number of current tasks exceeded.")
input_tasks = input_tasks[:ntasks_to_take]
for tsk in input_tasks:
self._archive.tasks[tsk.uuid] = tsk
for (uuid, status) in input_statuses:
if uuid in self._archive.tasks:
if UnsupportReason.REQUESTOR_TRUST in status.desc:
self._archive.tasks[uuid].requesting_trust = \
status.desc[UnsupportReason.REQUESTOR_TRUST]
self._archive.tasks[uuid].unsupport_reasons = \
list(status.desc.keys())
cur_time = get_timestamp_utc()
for tsk in list(self._archive.tasks.values()):
if cur_time > tsk.deadline:
self._merge_to_interval(tsk)
del self._archive.tasks[tsk.uuid]
self._purge_old_intervals()
if self._dump_file:
request = golem_async.AsyncRequest(self._dump_archive)
golem_async.async_run(
request,
None,
lambda e: log.info("Dumping archive failed: %s", e),
)
def test_datetime_to_timestamp(self):
ts = get_timestamp_utc()
assert ts
dt = timestamp_to_datetime(ts)
assert round(datetime_to_timestamp(dt), 5) == round(ts, 5)
def test_datetime_to_timestamp(self):
ts = get_timestamp_utc()
assert ts
dt = timestamp_to_datetime(ts)
assert datetime_to_timestamp(dt) == ts
def validate(th_dict_repr: dict) -> None:
"""Checks if task header dict representation has correctly
defined parameters
:param dict th_dict_repr: task header dictionary representation
"""
task_id = th_dict_repr.get('task_id')
task_owner = th_dict_repr.get('task_owner')
try:
node_name = task_owner.get('node_name') # type: ignore
except AttributeError:
raise exceptions.TaskHeaderError(
'Task owner missing',
task_id=task_id,
task_owner=task_owner,
)
if not isinstance(task_id, str):
raise exceptions.TaskHeaderError(
'Task ID missing',
task_id=task_id,
node_name=node_name,
)
if not isinstance(node_name, str):
raise exceptions.TaskHeaderError(
'Task owner node name missing',
task_id=task_id,
node_name=node_name,
)
if not isinstance(th_dict_repr['deadline'], (int, float)):
raise exceptions.TaskHeaderError(
"Deadline is not a timestamp",
task_id=task_id,
node_name=node_name,
deadline=th_dict_repr['deadline'],
)
now = common.get_timestamp_utc()
if th_dict_repr['deadline'] < now:
raise exceptions.TaskHeaderError(
"Deadline already passed",
task_id=task_id,
node_name=node_name,
deadline=th_dict_repr['deadline'],
now=now,
)
if not isinstance(th_dict_repr['subtask_timeout'], int):
raise exceptions.TaskHeaderError(
"Subtask timeout is not a number",
task_id=task_id,
node_name=node_name,
subtask_timeout=th_dict_repr['subtask_timeout'],
)
if th_dict_repr['subtask_timeout'] < 0:
raise exceptions.TaskHeaderError(
"Subtask timeout is less than 0",
task_id=task_id,
node_name=node_name,
subtask_timeout=th_dict_repr['subtask_timeout'],
)
try:
if th_dict_repr['subtasks_count'] < 1:
raise exceptions.TaskHeaderError(
"Subtasks count is less than 1",
task_id=task_id,
node_name=node_name,
subtasks_count=th_dict_repr['subtasks_count'],
)
except (KeyError, TypeError):
raise exceptions.TaskHeaderError(
"Subtasks count is missing",
task_id=task_id,
node_name=node_name,
subtask_count=th_dict_repr.get('subtask_count'),
)
