def test_fallback_to_ListBootImages_on_old_clusters(self):
rack_1 = factory.make_RackController()
rack_2 = factory.make_RackController()
rack_3 = factory.make_RackController()
images = [make_rpc_boot_image() for _ in range(3)]
# Limit the region's event loop to only the "rpc" service.
self.useFixture(RegionEventLoopFixture("rpc"))
# Now start the region's event loop.
self.useFixture(RunningEventLoopFixture())
# This fixture allows us to simulate mock clusters.
rpc = self.useFixture(MockLiveRegionToClusterRPCFixture())
# This simulates an older cluster, one without ListBootImagesV2.
rack_1 = rpc.makeCluster(rack_1, ListBootImages)
rack_1.ListBootImages.return_value = succeed({"images": images})
# This simulates a newer cluster, one with ListBootImagesV2.
rack_2 = rpc.makeCluster(rack_2, ListBootImagesV2)
rack_2.ListBootImagesV2.return_value = succeed({"images": images})
# This simulates a broken cluster.
rack_3 = rpc.makeCluster(rack_3, ListBootImagesV2)
rack_3.ListBootImagesV2.side_effect = ZeroDivisionError
self.assertItemsEqual(images, self.get())
def prepare_rack_rpc(self):
rack_controller = factory.make_RackController()
self.useFixture(RegionEventLoopFixture('rpc'))
self.useFixture(RunningEventLoopFixture())
fixture = self.useFixture(MockLiveRegionToClusterRPCFixture())
protocol = fixture.makeCluster(rack_controller)
return protocol
def setUp(self):
super().setUp()
# Limit the region's event loop to only the "rpc" service.
self.useFixture(RegionEventLoopFixture("rpc"))
# Now start the region's event loop.
self.useFixture(RunningEventLoopFixture())
# This fixture allows us to simulate mock clusters.
self.rpc = self.useFixture(MockLiveRegionToClusterRPCFixture())
def test_rpc_info_from_running_ipc_master(self):
# Run the IPC master, IPC worker, and RPC service so the endpoints
# are updated in the database.
region = factory.make_RegionController()
self.useFixture(MAASIDFixture(region.system_id))
region.owner = factory.make_admin()
region.save()
# `workers` is only included so ipc-master will not actually get the
# workers service because this test runs in all-in-one mode.
self.useFixture(
RegionEventLoopFixture(
"ipc-master", "ipc-worker", "rpc", "workers"
)
)
eventloop.start(master=True, all_in_one=True).wait(5)
self.addCleanup(lambda: eventloop.reset().wait(5))
getServiceNamed = eventloop.services.getServiceNamed
ipcMaster = getServiceNamed("ipc-master")
@wait_for(5)
@inlineCallbacks
def wait_for_startup():
# Wait for the service to complete startup.
yield ipcMaster.starting
yield getServiceNamed("ipc-worker").starting
yield getServiceNamed("rpc").starting
# Force an update, because it's very hard to track when the
# first iteration of the ipc-master service has completed.
yield ipcMaster.update()
wait_for_startup()
response = self.client.get(reverse("rpc-info"))
self.assertEqual("application/json", response["Content-Type"])
info = json.loads(response.content.decode("unicode_escape"))
self.assertThat(info, KeysEqual("eventloops"))
self.assertThat(
info["eventloops"],
MatchesDict(
{
# Each entry in the endpoints dict is a mapping from an
# event loop to a list of (host, port) tuples. Each tuple is
# a potential endpoint for connecting into that event loop.
eventloop.loop.name: MatchesSetwise(
*(
MatchesListwise((Equals(addr), is_valid_port))
for addr, _ in ipcMaster._getListenAddresses(5240)
)
)
}
),
)
def prepare_rack_rpc(self):
rack_controller = factory.make_RackController()
self.useFixture(RegionEventLoopFixture('rpc'))
self.useFixture(RunningEventLoopFixture())
fixture = self.useFixture(MockLiveRegionToClusterRPCFixture())
protocol = fixture.makeCluster(rack_controller, DescribePowerTypes)
self.power_types = PowerDriverRegistry.get_schema()
protocol.DescribePowerTypes.side_effect = always_succeed_with(
{'power_types': self.power_types})
return protocol
def setUp(self):
super(PreseedRPCMixin, self).setUp()
self.useFixture(RegionEventLoopFixture("rpc"))
self.useFixture(RunningEventLoopFixture())
self.rpc_fixture = MockLiveRegionToClusterRPCFixture()
self.useFixture(self.rpc_fixture)
# Create a cluster that's connected by RPC that responds to the
# GetPreseedData call with a simple NotImplementedError.
self.rpc_rack_controller = factory.make_RackController()
self.rpc_cluster = self.rpc_fixture.makeCluster(
self.rpc_rack_controller, GetPreseedData)
self.rpc_cluster.GetPreseedData.side_effect = (NotImplementedError())
def test_rpc_info_when_rpc_advertise_startup_failed(self):
self.useFixture(RegionEventLoopFixture("rpc", "rpc-advertise"))
# Simulate a crash when the rpc-advertise service starts.
self.simulateExceptionInAdvertiseService(factory.make_exception())
eventloop.start().wait(2.0)
self.addCleanup(lambda: eventloop.reset().wait(5))
response = self.client.get(reverse('rpc-info'))
self.assertEqual("application/json", response["Content-Type"])
info = json.loads(response.content.decode("unicode_escape"))
self.assertEqual({"eventloops": None}, info)
def test_rpc_info_when_rpc_advertise_not_fully_started(self):
self.useFixture(RegionEventLoopFixture("rpc", "rpc-advertise"))
# Simulate a time-out when getting the advertising instance.
self.simulateExceptionInAdvertiseService(CancelledError())
eventloop.start().wait(2.0)
self.addCleanup(lambda: eventloop.reset().wait(5))
response = self.client.get(reverse('rpc-info'))
self.assertEqual("application/json", response["Content-Type"])
info = json.loads(response.content.decode("unicode_escape"))
self.assertEqual({"eventloops": None}, info)
def test_reset(self):
# Replace the factories in RegionEventLoop with non-functional
# dummies to avoid bringing up real services here, and ensure
# that the services list is empty.
self.useFixture(RegionEventLoopFixture())
# Patch prepare so it's not actually run.
self.patch(eventloop.loop,
"prepare").return_value = (defer.succeed(None))
eventloop.loop.start().wait(5)
eventloop.loop.reset().wait(5)
# After stopping the loop, the services list is also emptied.
self.assertFalse(eventloop.loop.services.running)
self.assertFalse(eventloop.loop.running)
self.assertEqual(set(eventloop.loop.services), set())
# The hook has also been cleared.
self.assertIsNone(eventloop.loop.handle)
def test_start_and_stop(self):
# Replace the factories in RegionEventLoop with non-functional
# dummies to avoid bringing up real services here, and ensure
# that the services list is empty.
self.useFixture(RegionEventLoopFixture())
# At the outset, the eventloop's services are dorment.
self.assertFalse(eventloop.loop.services.running)
# RegionEventLoop.running is an alias for .services.running.
self.assertFalse(eventloop.loop.running)
self.assertEqual(set(eventloop.loop.services), set())
# Patch prepare so it's not actually run.
self.patch(eventloop.loop,
"prepare").return_value = defer.succeed(None)
# After starting the loop, the services list is populated, and
# the services are started too.
eventloop.loop.start().wait(5)
self.addCleanup(lambda: eventloop.loop.reset().wait(5))
self.assertTrue(eventloop.loop.services.running)
self.assertTrue(eventloop.loop.running)
self.assertEqual(
{service.name
for service in eventloop.loop.services},
{name
for name in eventloop.loop.factories.keys()},
)
# A shutdown hook is registered with the reactor.
stopService = eventloop.loop.services.stopService
self.assertEqual(
("shutdown", ("before", stopService, (), {})),
eventloop.loop.handle,
)
# After stopping the loop, the services list remains populated,
# but the services are all stopped.
eventloop.loop.stop().wait(5)
self.assertFalse(eventloop.loop.services.running)
self.assertFalse(eventloop.loop.running)
self.assertEqual(
{service.name
for service in eventloop.loop.services},
{name
for name in eventloop.loop.factories.keys()},
)
# The hook has been cleared.
self.assertIsNone(eventloop.loop.handle)
def test_rpc_info_when_rpc_advertise_running(self):
region = factory.make_RegionController()
self.useFixture(MAASIDFixture(region.system_id))
region.owner = factory.make_admin()
region.save()
self.useFixture(RegionEventLoopFixture("rpc", "rpc-advertise"))
eventloop.start().wait(5)
self.addCleanup(lambda: eventloop.reset().wait(5))
getServiceNamed = eventloop.services.getServiceNamed
@wait_for(5)
@inlineCallbacks
def wait_for_startup():
# Wait for the rpc and the rpc-advertise services to start.
yield getServiceNamed("rpc").starting
yield getServiceNamed("rpc-advertise").starting
# Force an update, because it's very hard to track when the
# first iteration of the rpc-advertise service has completed.
yield getServiceNamed("rpc-advertise")._tryUpdate()
wait_for_startup()
response = self.client.get(reverse('rpc-info'))
self.assertEqual("application/json", response["Content-Type"])
info = json.loads(response.content.decode("unicode_escape"))
self.assertThat(info, KeysEqual("eventloops"))
self.assertThat(
info["eventloops"],
MatchesDict({
# Each entry in the endpoints dict is a mapping from an
# event loop to a list of (host, port) tuples. Each tuple is
# a potential endpoint for connecting into that event loop.
eventloop.loop.name:
MatchesSetwise(*(MatchesListwise((Equals(addr), is_valid_port))
for addr in get_all_interface_addresses()
if not IPAddress(addr).is_link_local()
and not IPAddress(addr).is_loopback())),
}))
def setUp(self):
super().setUp()
self.useFixture(RegionEventLoopFixture("rpc"))
self.useFixture(RunningEventLoopFixture())
self.useFixture(MockRegionToClusterRPCFixture())
def setUp(self):
super(TestStartUp, self).setUp()
self.useFixture(RegionEventLoopFixture())
def setUp(self):
super(TestRegionProtocol_SendEventMACAddress, self).setUp()
self.useFixture(RegionEventLoopFixture("database-tasks"))
def setUp(self):
super(TestRegionProtocol_UpdateLease, self).setUp()
self.useFixture(RegionEventLoopFixture("database-tasks"))
def setUp(self):
super().setUp()
self.useFixture(RegionEventLoopFixture())
def setUp(self):
super(RunningClusterRPCFixture, self).setUp()
self.useFixture(RegionEventLoopFixture("rpc"))
self.useFixture(RunningEventLoopFixture())
self.useFixture(ClusterRPCFixture())
def setUp(self):
super().setUp()
self.useFixture(RegionEventLoopFixture("database-tasks"))
def setUp(self):
super().setUp()
self.useFixture(RegionEventLoopFixture())
self.patch(ipaddr, "get_ip_addr").return_value = {}
def prepare_live_rpc(self):
self.useFixture(RegionEventLoopFixture("rpc"))
self.useFixture(RunningEventLoopFixture())
return self.useFixture(MockLiveRegionToClusterRPCFixture())
def get_rack_rpc_protocol(self, rack_controller, *commands):
self.useFixture(RegionEventLoopFixture("rpc"))
self.useFixture(RunningEventLoopFixture())
fixture = self.useFixture(MockLiveRegionToClusterRPCFixture())
return fixture.makeCluster(rack_controller, *commands)
