def test__skips_those_that_have_not_expired(self): self.useFixture(SignalsDisabled("power")) current_time = datetime.now() self.patch(status_monitor, "now").return_value = current_time expired_time = current_time + timedelta(minutes=1) nodes = [ factory.make_Node(status=status, status_expires=expired_time) for status in NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys() ] mark_nodes_failed_after_expiring() failed_statuses = [reload_object(node).status for node in nodes] self.assertItemsEqual(NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys(), failed_statuses)
def test_skips_those_that_have_not_expired(self): maaslog = self.patch(status_monitor.maaslog, "info") self.useFixture(SignalsDisabled("power")) current_time = now() expired_time = current_time + timedelta(minutes=1) nodes = [ factory.make_Node(status=status, status_expires=expired_time) for status in NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys() ] mark_nodes_failed_after_expiring(current_time, 20) failed_statuses = [reload_object(node).status for node in nodes] self.assertItemsEqual(NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys(), failed_statuses) self.assertThat(maaslog, MockNotCalled())
def test__marks_all_possible_failed_status_as_failed(self): maaslog = self.patch(status_monitor.maaslog, 'info') self.useFixture(SignalsDisabled("power")) current_time = datetime.now() self.patch(status_monitor, "now").return_value = current_time expired_time = current_time - timedelta(minutes=1) nodes = [ factory.make_Node(status=status, status_expires=expired_time) for status in NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys() ] mark_nodes_failed_after_expiring() failed_statuses = [reload_object(node).status for node in nodes] self.assertItemsEqual( NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.values(), failed_statuses) # MAAS logs in the status_monitor that the timeout was detected. It # then logs the transisition in the node signal handler. self.assertEquals(len(NODE_FAILURE_MONITORED_STATUS_TRANSITIONS), len(maaslog.call_args_list) / 2)
def mark_nodes_failed_after_expiring(): """Mark all nodes in that database as failed where the status did not transition in time. `status_expires` is checked on the node to see if the current time is newer than the expired time. """ current_db_time = now() expired_nodes = Node.objects.filter( status__in=NODE_FAILURE_MONITORED_STATUS_TRANSITIONS.keys(), status_expires__isnull=False, status_expires__lte=current_db_time) for node in expired_nodes: comment = "Node operation '%s' timed out after %s minutes." % ( NODE_STATUS_CHOICES_DICT[node.status], NODE_FAILURE_MONITORED_STATUS_TIMEOUTS[node.status], ) node.mark_failed(comment=comment, script_result_status=SCRIPT_STATUS.ABORTED)