def test__skips_those_that_have_not_expired(self):
self.useFixture(SignalsDisabled("power"))
current_time = datetime.now()
self.patch(status_monitor, "now").return_value = current_time
expired_time = current_time + timedelta(minutes=1)
nodes = [
factory.make_Node(status=status, status_expires=expired_time)
for status in NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys()
]
mark_nodes_failed_after_expiring()
failed_statuses = [reload_object(node).status for node in nodes]
self.assertItemsEqual(NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys(),
failed_statuses)
def test_skips_those_that_have_not_expired(self):
maaslog = self.patch(status_monitor.maaslog, "info")
self.useFixture(SignalsDisabled("power"))
current_time = now()
expired_time = current_time + timedelta(minutes=1)
nodes = [
factory.make_Node(status=status, status_expires=expired_time)
for status in NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys()
]
mark_nodes_failed_after_expiring(current_time, 20)
failed_statuses = [reload_object(node).status for node in nodes]
self.assertItemsEqual(NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys(),
failed_statuses)
self.assertThat(maaslog, MockNotCalled())
def test__marks_all_possible_failed_status_as_failed(self):
maaslog = self.patch(status_monitor.maaslog, 'info')
self.useFixture(SignalsDisabled("power"))
current_time = datetime.now()
self.patch(status_monitor, "now").return_value = current_time
expired_time = current_time - timedelta(minutes=1)
nodes = [
factory.make_Node(status=status, status_expires=expired_time)
for status in NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys()
]
mark_nodes_failed_after_expiring()
failed_statuses = [reload_object(node).status for node in nodes]
self.assertItemsEqual(
NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.values(),
failed_statuses)
# MAAS logs in the status_monitor that the timeout was detected. It
# then logs the transisition in the node signal handler.
self.assertEquals(len(NODE_FAILURE_MONITORED_STATUS_TRANSITIONS),
len(maaslog.call_args_list) / 2)
def mark_nodes_failed_after_expiring():
"""Mark all nodes in that database as failed where the status did not
transition in time. `status_expires` is checked on the node to see if the
current time is newer than the expired time.
"""
current_db_time = now()
expired_nodes = Node.objects.filter(
status__in=NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys(),
status_expires__isnull=False,
status_expires__lte=current_db_time)
for node in expired_nodes:
comment = "Node operation '%s' timed out after %s minutes." % (
NODE_STATUS_CHOICES_DICT[node.status],
NODE_FAILURE_MONITORED_STATUS_TIMEOUTS[node.status],
)
node.mark_failed(comment=comment,
script_result_status=SCRIPT_STATUS.ABORTED)