def test_get_power_state_queries_node(self):
system_id = factory.make_name("system_id")
hostname = factory.make_name("hostname")
power_driver = random.choice(
[driver for _, driver in PowerDriverRegistry if driver.queryable]
)
power_state = random.choice(["on", "off"])
context = {
factory.make_name("context-key"): factory.make_name("context-val")
}
self.patch(power, "is_driver_available").return_value = True
_, markNodeBroken, io = self.patch_rpc_methods()
mock_perform_power_driver_query = self.patch(
power, "perform_power_driver_query"
)
mock_perform_power_driver_query.return_value = power_state
d = power.get_power_state(
system_id, hostname, power_driver.name, context
)
# This blocks until the deferred is complete.
io.flush()
self.assertEqual(power_state, extract_result(d))
self.assertThat(
power_driver.detect_missing_packages, MockCalledOnceWith()
)
self.assertThat(
mock_perform_power_driver_query,
MockCallsMatch(
call(system_id, hostname, power_driver.name, context)
),
)
def test_logs_other_errors(self):
service = ImageDownloadService(sentinel.rpc, sentinel.tftp_root,
Clock())
maybe_start_download = self.patch(service, "maybe_start_download")
maybe_start_download.return_value = defer.fail(
ZeroDivisionError("Such a shame I can't divide by zero"))
with FakeLogger("maas") as maaslog, TwistedLoggerFixture() as logger:
d = service.try_download()
self.assertEqual(None, extract_result(d))
self.assertDocTestMatches(
"Failed to download images: "
"Such a shame I can't divide by zero",
maaslog.output,
)
self.assertDocTestMatches(
"""\
Downloading images failed.
Traceback (most recent call last):
Failure: builtins.ZeroDivisionError: Such a shame ...
""",
logger.output,
)
def test_query_nodes_calls_query_all_nodes(self):
service = self.make_monitor_service()
service.max_nodes_at_once = sentinel.max_nodes_at_once
example_power_parameters = {
"system_id": factory.make_UUID(),
"hostname": factory.make_hostname(),
"power_state": factory.make_name("power_state"),
"power_type": factory.make_name("power_type"),
"context": {},
}
rpc_fixture = self.useFixture(MockClusterToRegionRPCFixture())
proto_region, io = rpc_fixture.makeEventLoop(
region.ListNodePowerParameters)
proto_region.ListNodePowerParameters.side_effect = [
succeed({"nodes": [example_power_parameters]}),
succeed({"nodes": []}),
]
query_all_nodes = self.patch(npms, "query_all_nodes")
d = service.query_nodes(getRegionClient())
io.flush()
self.assertEqual(None, extract_result(d))
self.assertThat(
query_all_nodes,
MockCalledOnceWith(
[example_power_parameters],
max_concurrency=sentinel.max_nodes_at_once,
clock=service.clock,
),
)
def test_maybe_make_stats_request_does_not_error(self):
service = stats.PrometheusService()
deferToDatabase = self.patch(stats, "deferToDatabase")
exception_type = factory.make_exception_type()
deferToDatabase.return_value = fail(exception_type())
d = service.maybe_push_prometheus_stats()
self.assertIsNone(extract_result(d))
def test__returns_output_from_RequestNodeInfoByMACAddress(self):
client = self.patch(windows_module, "getRegionClient").return_value
client.side_effect = always_succeed_with(sentinel.node_info)
d = windows_module.request_node_info_by_mac_address(sentinel.mac)
self.assertThat(extract_result(d), Is(sentinel.node_info))
self.assertThat(client, MockCalledOnceWith(
RequestNodeInfoByMACAddress, mac_address=sentinel.mac))
def test__returns_None_when_MAC_is_None(self):
logger = self.useFixture(FakeLogger("maas", logging.DEBUG))
d = windows_module.request_node_info_by_mac_address(None)
self.assertThat(extract_result(d), Is(None))
self.assertDocTestMatches(
"Cannot determine node; MAC address is unknown.",
logger.output)
def test__fire_calls_hooks(self):
dhooks = DeferredHooks()
ds = Deferred(), Deferred()
for d in ds:
dhooks.add(d)
dhooks.fire()
for d in ds:
self.assertIsNone(extract_result(d))
def test__reset_suppresses_CancelledError(self):
logger = self.useFixture(TwistedLoggerFixture())
dhooks = DeferredHooks()
d = Deferred()
dhooks.add(d)
dhooks.reset()
self.assertThat(dhooks.hooks, HasLength(0))
self.assertThat(extract_result(d), Is(None))
self.assertEqual("", logger.output)
def test__returns_None_when_node_not_found(self):
logger = self.useFixture(FakeLogger("maas", logging.DEBUG))
client = self.patch(windows_module, "getRegionClient").return_value
client.side_effect = always_fail_with(NoSuchNode())
mac = factory.make_mac_address()
d = windows_module.request_node_info_by_mac_address(mac)
self.assertThat(extract_result(d), Is(None))
self.assertDocTestMatches(
"Node doesn't exist for MAC address: %s" % mac,
logger.output)
def test_power_state_update_calls_UpdateNodePowerState(self):
system_id = factory.make_name('system_id')
state = random.choice(['on', 'off'])
protocol, io = self.patch_rpc_methods()
d = power.power_state_update(system_id, state)
# This blocks until the deferred is complete
io.flush()
self.expectThat(extract_result(d), Equals({}))
self.assertThat(
protocol.UpdateNodePowerState,
MockCalledOnceWith(ANY, system_id=system_id, power_state=state))
def test_authenticateCluster_accepts_matching_digests(self):
server = self.make_running_server()
def calculate_digest(_, message):
# Use the region's own authentication responder.
return Region().authenticate(message)
callRemote = self.patch_autospec(server, "callRemote")
callRemote.side_effect = calculate_digest
d = server.authenticateCluster()
self.assertTrue(extract_result(d))
def test_query_nodes_copes_with_losing_connection_to_region(self):
service = self.make_monitor_service()
client = Mock(return_value=fail(
ConnectionDone("Connection was closed cleanly.")))
with FakeLogger("maas") as maaslog:
d = service.query_nodes(client)
d.addErrback(service.query_nodes_failed, sentinel.ident)
self.assertEqual(None, extract_result(d))
self.assertDocTestMatches("Lost connection to region controller.",
maaslog.output)
def test_makes_calls_to_each_client_given(self):
rack_controllers = [factory.make_RackController() for _ in range(3)]
clients = self.patch_clients(rack_controllers)
tag_name = factory.make_name("tag")
tag_definition = factory.make_name("definition")
tag_nsmap_prefix = factory.make_name("prefix")
tag_nsmap_uri = factory.make_name("uri")
tag_nsmap = [{"prefix": tag_nsmap_prefix, "uri": tag_nsmap_uri}]
work = []
rack_creds = []
rack_nodes = []
for rack, client in zip(rack_controllers, clients):
creds = factory.make_name("creds")
rack_creds.append(creds)
nodes = [{
"system_id": factory.make_Node().system_id
} for _ in range(3)]
rack_nodes.append(nodes)
work.append({
"system_id": rack.system_id,
"hostname": rack.hostname,
"client": client,
"tag_name": tag_name,
"tag_definition": tag_definition,
"tag_nsmap": tag_nsmap,
"credentials": creds,
"nodes": nodes,
})
[d] = _do_populate_tags(work)
self.assertIsNone(extract_result(d))
for rack, client, creds, nodes in zip(rack_controllers, clients,
rack_creds, rack_nodes):
self.expectThat(
client,
MockCallsMatch(
call(
EvaluateTag,
tag_name=tag_name,
tag_definition=tag_definition,
system_id=rack.system_id,
tag_nsmap=tag_nsmap,
credentials=creds,
nodes=nodes,
)),
)
def test_power_change_starting_emits_event(self):
system_id = factory.make_name('system_id')
hostname = factory.make_name('hostname')
power_change = 'on'
protocol, io = self.patch_rpc_methods()
d = power.power_change_starting(system_id, hostname, power_change)
io.flush()
self.assertThat(
protocol.SendEvent,
MockCalledOnceWith(ANY,
type_name=EVENT_TYPES.NODE_POWER_ON_STARTING,
system_id=system_id,
description=''))
self.assertIsNone(extract_result(d))
def test_power_query_failure_emits_event(self):
system_id = factory.make_name('system_id')
hostname = factory.make_name('hostname')
message = factory.make_name('message')
SendEvent, _, io = self.patch_rpc_methods()
d = power.power_query_failure(
system_id, hostname, Failure(Exception(message)))
# This blocks until the deferred is complete.
io.flush()
self.assertIsNone(extract_result(d))
self.assertThat(
SendEvent,
MockCalledOnceWith(
ANY, type_name=EVENT_TYPES.NODE_POWER_QUERY_FAILED,
system_id=system_id, description=message))
def test_authenticateCluster_rejects_non_matching_digests(self):
server = self.make_running_server()
def calculate_digest(_, message):
# Return some nonsense.
response = {
"digest": factory.make_bytes(),
"salt": factory.make_bytes(),
}
return succeed(response)
callRemote = self.patch_autospec(server, "callRemote")
callRemote.side_effect = calculate_digest
d = server.authenticateCluster()
self.assertFalse(extract_result(d))
def test_power_query_success_emits_event(self):
system_id = factory.make_name('system_id')
hostname = factory.make_name('hostname')
state = factory.make_name('state')
message = "Power state queried: %s" % state
SendEvent, _, io = self.patch_rpc_methods()
d = power.power_query_success(system_id, hostname, state)
# This blocks until the deferred is complete.
io.flush()
self.assertIsNone(extract_result(d))
self.assertThat(
SendEvent,
MockCalledOnceWith(ANY,
type_name=EVENT_TYPES.NODE_POWER_QUERIED_DEBUG,
system_id=system_id,
description=message))
def test_report_power_state_changes_power_state_if_unknown(self):
system_id = factory.make_name('system_id')
hostname = factory.make_name('hostname')
power_state = "unknown"
_, _, io = self.patch_rpc_methods()
self.patch_autospec(power, 'power_state_update')
# Simulate a success when querying state.
query = succeed(power_state)
report = power.report_power_state(query, system_id, hostname)
# This blocks until the deferred is complete.
io.flush()
self.assertEqual(power_state, extract_result(report))
self.assertThat(power.power_state_update,
MockCalledOnceWith(system_id, power_state))
def test_query_nodes_calls_the_region(self):
service = self.make_monitor_service()
rpc_fixture = self.useFixture(MockClusterToRegionRPCFixture())
proto_region, io = rpc_fixture.makeEventLoop(
region.ListNodePowerParameters)
proto_region.ListNodePowerParameters.return_value = succeed(
{"nodes": []})
client = getRegionClient()
d = service.query_nodes(client)
io.flush()
self.assertEqual(None, extract_result(d))
self.assertThat(
proto_region.ListNodePowerParameters,
MockCalledOnceWith(ANY, uuid=client.localIdent))
def test_try_query_nodes_logs_other_errors(self):
service = self.make_monitor_service()
self.patch(npms, "getRegionClient").return_value = sentinel.client
sentinel.client.localIdent = factory.make_UUID()
query_nodes = self.patch(service, "query_nodes")
query_nodes.return_value = fail(
ZeroDivisionError("Such a shame I can't divide by zero"))
with FakeLogger("maas") as maaslog, TwistedLoggerFixture():
d = service.try_query_nodes()
self.assertEqual(None, extract_result(d))
self.assertDocTestMatches(
"Failed to query nodes' power status: "
"Such a shame I can't divide by zero",
maaslog.output)
def test_query_nodes_copes_with_NoSuchCluster(self):
service = self.make_monitor_service()
rpc_fixture = self.useFixture(MockClusterToRegionRPCFixture())
proto_region, io = rpc_fixture.makeEventLoop(
region.ListNodePowerParameters)
client = getRegionClient()
proto_region.ListNodePowerParameters.return_value = fail(
exceptions.NoSuchCluster.from_uuid(client.localIdent))
d = service.query_nodes(client)
d.addErrback(service.query_nodes_failed, client.localIdent)
with FakeLogger("maas") as maaslog:
io.flush()
self.assertEqual(None, extract_result(d))
self.assertDocTestMatches("Rack controller '...' is not recognised.",
maaslog.output)
def test_regionRefresh_refreshes_a_region(self):
region = factory.make_RegionController()
self.patch(region, "refresh").return_value = Deferred()
d = start_up.refreshRegion(region)
self.assertThat(d, IsInstance(Deferred))
exception = factory.make_exception_type()
with TwistedLoggerFixture() as logger:
d.errback(exception("boom"))
# The exception is suppressed ...
self.assertThat(extract_result(d), Is(None))
# ... but it has been logged.
self.assertThat(
logger.output,
DocTestMatches("""
Failure when refreshing region.
Traceback (most recent call last):...
Failure: maastesting.factory.TestException#...: boom
"""))
def test_report_power_state_changes_power_state_if_success(self):
system_id = factory.make_name("system_id")
hostname = factory.make_name("hostname")
power_state = random.choice(["on", "off"])
_, _, io = self.patch_rpc_methods()
self.patch_autospec(power, "power_state_update")
# Simulate a success when querying state.
query = succeed(power_state)
report = power.report_power_state(query, system_id, hostname)
# This blocks until the deferred is complete.
io.flush()
self.assertEqual(power_state, extract_result(report))
self.assertThat(
power.power_state_update,
MockCalledOnceWith(system_id, power_state),
)
def test_power_change_failure_emits_event(self):
system_id = factory.make_name("system_id")
hostname = factory.make_name("hostname")
message = factory.make_name("message")
power_change = "on"
protocol, io = self.patch_rpc_methods()
d = power.power_change_failure(system_id, hostname, power_change,
message)
io.flush()
self.assertThat(
protocol.SendEvent,
MockCalledOnceWith(
ANY,
type_name=EVENT_TYPES.NODE_POWER_ON_FAILED,
system_id=system_id,
description=message,
),
)
self.assertIsNone(extract_result(d))
def test_logs_failures(self):
rack_controllers = [factory.make_RackController()]
clients = self.patch_clients(rack_controllers)
clients[0].side_effect = always_fail_with(
ZeroDivisionError("splendid day for a spot of cricket"))
tag_name = factory.make_name("tag")
tag_definition = factory.make_name("definition")
tag_nsmap = {}
work = []
for rack, client in zip(rack_controllers, clients):
work.append({
"system_id":
rack.system_id,
"hostname":
rack.hostname,
"client":
client,
"tag_name":
tag_name,
"tag_definition":
tag_definition,
"tag_nsmap":
tag_nsmap,
"credentials":
factory.make_name("creds"),
"nodes": [{
"system_id": factory.make_Node().system_id
} for _ in range(3)],
})
with FakeLogger("maas") as log:
[d] = _do_populate_tags(work)
self.assertIsNone(extract_result(d))
self.assertDocTestMatches(
"Tag tag-... (definition-...) could not be evaluated ... (...): "
"splendid day for a spot of cricket",
log.output,
)
def test_unhandled_errors_logged_and_do_not_cause_disconnection(self):
protocol = common.RPCProtocol()
protocol.makeConnection(StringTransport())
# Poke a request into the dispatcher that will always fail.
d = Deferred().addCallback(lambda _: 0 / 0)
protocol._outstandingRequests[self.seq] = d
# Push a box in response to the request.
with TwistedLoggerFixture() as logger:
protocol.ampBoxReceived(self.box)
# The Deferred does not have a dangling error.
self.assertThat(extract_result(d), Is(None))
# The transport is still connected.
self.assertThat(protocol.transport.disconnecting, Is(False))
# The error has been logged.
self.assertDocTestMatches(
"""\
Unhandled failure during AMP request. This is probably a bug.
Please ensure that this error is handled within application code.
Traceback (most recent call last):
...
""", logger.output)
def test_logs_successes(self):
rack_controllers = [factory.make_RackController()]
clients = self.patch_clients(rack_controllers)
tag_name = factory.make_name("tag")
tag_definition = factory.make_name("definition")
tag_nsmap = {}
work = []
for rack, client in zip(rack_controllers, clients):
work.append({
"system_id":
rack.system_id,
"hostname":
rack.hostname,
"client":
client,
"tag_name":
tag_name,
"tag_definition":
tag_definition,
"tag_nsmap":
tag_nsmap,
"credentials":
factory.make_name("creds"),
"nodes": [{
"system_id": factory.make_Node().system_id
} for _ in range(3)],
})
with FakeLogger("maas") as log:
[d] = _do_populate_tags(work)
self.assertIsNone(extract_result(d))
self.assertDocTestMatches(
"Tag tag-... (definition-...) evaluated on rack "
"controller ... (...)",
log.output,
)
def test_power_change_success_emits_event(self):
system_id = factory.make_name("system_id")
hostname = factory.make_name("hostname")
power_change = "on"
protocol, io = self.patch_rpc_methods()
d = power.power_change_success(system_id, hostname, power_change)
io.flush()
self.assertThat(
protocol.UpdateNodePowerState,
MockCalledOnceWith(
ANY, system_id=system_id, power_state=power_change
),
)
self.assertThat(
protocol.SendEvent,
MockCalledOnceWith(
ANY,
type_name=EVENT_TYPES.NODE_POWERED_ON,
system_id=system_id,
description="",
),
)
self.assertIsNone(extract_result(d))
def test__returns_None_when_MAC_is_None(self):
d = windows_module.request_node_info_by_mac_address(None)
self.assertThat(extract_result(d), Is(None))
def test__returns_None_when_node_not_found(self):
client = self.patch(windows_module, "getRegionClient").return_value
client.side_effect = always_fail_with(NoSuchNode())
mac = factory.make_mac_address()
d = windows_module.request_node_info_by_mac_address(mac)
self.assertThat(extract_result(d), Is(None))