def getExpectedState(self):
@transactional
def db_getExpectedState():
if (Config.objects.get_config("enable_http_proxy")
and Config.objects.get_config("http_proxy")
and not Config.objects.get_config("use_peer_proxy")):
return (SERVICE_STATE.OFF,
"disabled, alternate proxy is configured in settings.")
elif config.is_config_present() is False:
return (SERVICE_STATE.OFF, "no configuration file present.")
else:
return (SERVICE_STATE.ON, None)
return deferToDatabase(db_getExpectedState)
def test_get_refresh_details_running(self):
region = yield deferToDatabase(factory.make_RegionController)
region.owner = yield deferToDatabase(factory.make_admin)
yield deferToDatabase(region.save)
# Declare this region controller as the one running here.
self.useFixture(MAASIDFixture(region.system_id))
service = RegionNetworksMonitoringService(
reactor,
enable_beaconing=False,
)
deferred = self.wrap_deferred(service, "do_action")
yield service.startService()
yield deferred
details = yield service.getRefreshDetails()
region_token = yield deferToDatabase(region._get_token_for_controller)
region_credentials = {
"consumer_key": region_token.consumer.key,
"token_key": region_token.key,
"token_secret": region_token.secret,
}
self.assertEqual((None, region.system_id, region_credentials), details)
def test_calls_handler_on_notification_with_delayed_registration(self):
listener = PostgresListenerService()
dv = DeferredValue()
yield listener.startService()
try:
# Register after the service has been started. The handler should
# still be called.
listener.register("machine", lambda *args: dv.set(args))
yield listener.channelRegistrarDone
yield deferToDatabase(self.send_notification, "machine_create", 1)
yield dv.get(timeout=2)
self.assertEqual(("create", "1"), dv.value)
finally:
yield listener.stopService()
def test_send_event_stores_event_with_timestamp_received(self):
# Use a random time in the recent past and coerce the responder to use
# it as the time-stamp for the event. We'll check this later on.
timestamp = datetime.now() - timedelta(seconds=randint(99, 99999))
self.patch(regionservice, "datetime").now.return_value = timestamp
event_type = factory.make_name('type_name')
yield deferToDatabase(self.create_event_type, event_type, "", 0)
system_id = yield deferToDatabase(self.create_node)
yield eventloop.start()
try:
yield call_responder(
Region(), SendEvent, {
'system_id': system_id,
'type_name': event_type,
'description': factory.make_name('description'),
})
finally:
yield eventloop.reset()
event = yield deferToDatabase(self.get_event, system_id, event_type)
self.expectThat(event.created, Equals(timestamp))
def test_authenticate_calls_loseConnection_if_csrftoken_is_missing(self):
user, session_id = yield deferToDatabase(self.get_user_and_session_id)
uri = self.make_ws_uri(csrftoken=None)
protocol, factory = self.make_protocol(patch_authenticate=False,
transport_uri=uri)
mock_loseConnection = self.patch_autospec(protocol, "loseConnection")
other_csrftoken = maas_factory.make_name("csrftoken")
yield protocol.authenticate(session_id, other_csrftoken)
self.expectThat(protocol.user, Equals(None))
self.expectThat(
mock_loseConnection,
MockCalledOnceWith(STATUSES.PROTOCOL_ERROR, "Invalid CSRF token."))
def test_logs_other_errors(self):
node = yield deferToDatabase(transactional(factory.make_Node))
mock_power_query = self.patch(Node, "power_query")
mock_power_query.side_effect = factory.make_exception("Error")
mock_log_err = self.patch(power.log, "err")
yield power.update_power_state_of_node(node.system_id)
self.assertThat(
mock_log_err,
MockCalledOnceWith(
ANY,
"Failed to update power state of machine after state "
"transition.",
),
)
def test__updates_rdns_entry(self):
hostname = factory.make_hostname()
hostname2 = factory.make_hostname()
self.set_fake_twisted_dns_reply([hostname])
service = ReverseDNSService()
yield service.startService()
ip = factory.make_ip_address(ipv6=False)
yield service.consumeNeighbourEvent("create", "%s/32" % ip)
self.set_fake_twisted_dns_reply([hostname2])
yield service.consumeNeighbourEvent("update", "%s/32" % ip)
service.stopService()
result = yield deferToDatabase(RDNS.objects.first)
self.assertThat(result.ip, Equals(ip))
self.assertThat(result.hostname, Equals(hostname2))
def test_update_creates_process_when_removed(self):
yield deferToDatabase(load_builtin_scripts)
master = self.make_IPCMasterService()
yield master.startService()
pids = {random.randint(1, 512) for _ in range(3)}
for pid in pids:
yield master.registerWorker(pid, MagicMock())
def delete_all():
region = RegionController.objects.get_running_controller()
region.processes.all().delete()
yield deferToDatabase(delete_all)
yield master.update()
for pid, data in master.connections.items():
process = yield deferToDatabase(
RegionControllerProcess.objects.get, id=data["process_id"])
self.assertEquals(pid, process.pid)
yield master.stopService()
def test_register_acks_version(self):
yield self.installFakeRegion()
rack_controller = yield deferToDatabase(factory.make_RackController)
protocol = self.make_Region()
protocol.transport = MagicMock()
response = yield call_responder(
protocol, RegisterRackController, {
"system_id": rack_controller.system_id,
"hostname": rack_controller.hostname,
"interfaces": {},
"version": "2.3",
})
self.assertThat(
response['version'], Equals(get_maas_version()))
def test_handles_missing_system_handler_on_notification(self):
listener = PostgresListenerServiceSpy()
yield listener.startService()
# Use a randomised channel name even though LISTEN/NOTIFY is
# per-database and _not_ per-cluster.
channel = factory.make_name("sys_test", sep="_").lower()
self.assertTrue(listener.isSystemChannel(channel))
payload = factory.make_name("payload")
yield deferToDatabase(listener.registerChannel, channel)
listener.listeners[channel] = []
try:
deferred = listener.wait_notification(channel)
# Notify our channel with a payload and wait for it to come
yield deferToDatabase(self.send_notification, channel, payload)
notice = yield deferred
self.assertEqual(notice.payload, payload)
# Our channel has been deleted from the listeners map.
self.assertNotIn(channel, listener.listeners)
finally:
yield listener.stopService()
def test_register_acks_beacon_support(self):
yield self.installFakeRegion()
rack_controller = yield deferToDatabase(factory.make_RackController)
protocol = self.make_Region()
protocol.transport = MagicMock()
response = yield call_responder(
protocol, RegisterRackController, {
"system_id": rack_controller.system_id,
"hostname": rack_controller.hostname,
"interfaces": {},
"beacon_support": True,
})
self.assertThat(
response['beacon_support'], Is(True))
def test_with_use_peer_proxy_with_http_proxy(self):
self.patch(settings, "PROXY_CONNECT", True)
yield deferToDatabase(
transactional(Config.objects.set_config), "enable_http_proxy", True
)
yield deferToDatabase(
transactional(Config.objects.set_config), "use_peer_proxy", True
)
yield deferToDatabase(
transactional(Config.objects.set_config),
"http_proxy",
"http://example.com:8000/",
)
yield deferToDatabase(self.make_subnet, allow_proxy=False)
yield deferToDatabase(self.make_subnet)
yield proxyconfig.proxy_update_config(reload_proxy=False)
cache_peer_line = (
"cache_peer example.com parent 8000 0 no-query default"
)
with self.proxy_path.open() as proxy_file:
lines = [line.strip() for line in proxy_file.readlines()]
self.assertIn("never_direct allow all", lines)
self.assertIn(cache_peer_line, lines)
def test_queueMessages_processes_top_level_status_messages_instantly(self):
for name in EVENT_STATUS_MESSAGES.keys():
worker = StatusWorkerService(sentinel.dbtasks)
mock_processMessage = self.patch(worker, "_processMessage")
message = self.make_message()
message['event_type'] = 'start'
message['name'] = name
nodes_with_tokens = yield deferToDatabase(
self.make_nodes_with_tokens)
node, token = nodes_with_tokens[0]
yield worker.queueMessage(token.key, message)
self.assertThat(
mock_processMessage,
MockCalledOnceWith(node, message))
def update_interfaces(self, system_id, interfaces, topology_hints=None):
"""update_interfaces()
Implementation of
:py:class:`~provisioningserver.rpc.region.UpdateInterfaces`.
"""
d = deferToDatabase(
rackcontrollers.update_interfaces,
system_id,
interfaces,
topology_hints=topology_hints,
)
d.addCallback(lambda args: {})
return d
def consumeNeighbourEvent(self, action: str = None, cidr: str = None):
"""Given an event from the postgresListener, resolve RDNS for an IP.
This method is called when an observed neighbour is changed.
:param action: one of {'create', 'update', 'delete'}
:param cidr: the 'ip' field in the neighbour table, after PostgreSQL
casts it to a string. It will end up looking like "x.x.x.x/32"
or "yyyy:yyyy::yyyy/128".
"""
ip = cidr.split("/")[0] # Strip off the "/<prefixlen>".
if action in ("create", "update"):
# XXX mpontillo 2016-10-19: We might consider throttling this on
# a per-IP-address basis, both because multiple racks can observe
# the same IP address, and because an IP address might repeatedly
# go back-and-forth between two MACs in the case of a duplicate IP
# address.
results = yield reverseResolve(ip).addErrback(suppress,
defer.TimeoutError,
instead=None)
if results is not None:
if len(results) > 0:
yield deferToDatabase(self.set_rdns_entry, ip, results)
else:
yield deferToDatabase(self.delete_rdns_entry, ip)
else:
# A return of 'None' indicates a timeout or other possibly-
# temporary failure, so take no action.
pass
elif action == "delete":
yield deferToDatabase(self.delete_rdns_entry, ip)
else:
log.msg(
"Unsupported event from listener: action=%r, cidr=%r" %
(action, cidr),
system="reverse-dns",
)
def start_up(master=False):
"""Perform start-up tasks for this MAAS server.
This is used to:
- make sure the singletons required by the application are created
- sync the configuration of the external systems driven by MAAS
The method will be executed multiple times if multiple processes are used
but this method uses database locking to ensure that the methods it calls
internally are not run concurrently.
"""
while True:
try:
# Get the shared secret from Tidmouth sheds which was generated
# when Sir Topham Hatt graduated Sodor Academy. (Ensure we have a
# shared-secret so that a cluster on the same host as this region
# can authenticate.)
yield security.get_shared_secret()
# Execute other start-up tasks that must not run concurrently with
# other invocations of themselves, across the whole of this MAAS
# installation.
yield deferToDatabase(inner_start_up, master=master)
except SystemExit:
raise
except KeyboardInterrupt:
raise
except DatabaseError as e:
psycopg2_exception = get_psycopg2_exception(e)
if psycopg2_exception is None:
maaslog.warning("Database error during start-up; "
"pausing for 3 seconds.")
elif psycopg2_exception.pgcode is None:
maaslog.warning(
"Database error during start-up (PostgreSQL error "
"not reported); pausing for 3 seconds.")
else:
maaslog.warning(
"Database error during start-up (PostgreSQL error %s); "
"pausing for 3 seconds.",
psycopg2_exception.pgcode,
)
logger.error("Database error during start-up", exc_info=True)
yield pause(3.0) # Wait 3 seconds before having another go.
except Exception:
maaslog.warning("Error during start-up; pausing for 3 seconds.")
logger.error("Error during start-up.", exc_info=True)
yield pause(3.0) # Wait 3 seconds before having another go.
else:
break
def test__calls_system_handler_on_notification(self):
listener = PostgresListenerService()
# Change notifications to a frozenset. This makes sure that
# the system message does not go into the queue. Instead if should
# call the handler directly in `doRead`.
listener.notifications = frozenset()
dv = DeferredValue()
listener.register("sys_test", lambda *args: dv.set(args))
yield listener.startService()
try:
yield deferToDatabase(self.send_notification, "sys_test", 1)
yield dv.get(timeout=2)
self.assertEqual(("sys_test", "1"), dv.value)
finally:
yield listener.stopService()
def test__processMessages_calls_processMessage_and_updateLastPing(self):
worker = StatusWorkerService(sentinel.dbtasks)
mock_processMessage = self.patch(worker, "_processMessage")
mock_updateLastPing = self.patch(worker, "_updateLastPing")
yield deferToDatabase(
worker._processMessages, sentinel.node,
[sentinel.message1, sentinel.message2])
self.assertThat(
mock_processMessage,
MockCallsMatch(
call(sentinel.node, sentinel.message1),
call(sentinel.node, sentinel.message2)))
self.assertThat(
mock_updateLastPing,
MockCalledOnceWith(sentinel.node, sentinel.message2))
def test_register_event_type_creates_object(self):
name = factory.make_name('name')
description = factory.make_name('description')
level = random.randint(0, 100)
response = yield call_responder(
Region(), RegisterEventType,
{'name': name, 'description': description, 'level': level})
self.assertEqual({}, response)
event_type = yield deferToDatabase(self.get_event_type, name)
self.assertThat(
event_type,
MatchesStructure.byEquality(
name=name, description=description, level=level)
)
def test_update_removes_old_processes(self):
yield deferToDatabase(load_builtin_scripts)
master = self.make_IPCMasterService()
yield master.startService()
def make_old_processes():
old_time = now() - timedelta(seconds=90)
region = RegionController.objects.get_running_controller()
other_region = factory.make_RegionController()
old_region_process = RegionControllerProcess.objects.create(
region=region,
pid=random.randint(1, 1000),
created=old_time,
updated=old_time,
)
old_other_region_process = RegionControllerProcess.objects.create(
region=other_region,
pid=random.randint(1000, 2000),
created=old_time,
updated=old_time,
)
return old_region_process, old_other_region_process
old_region_process, old_other_region_process = yield deferToDatabase(
make_old_processes)
yield master.update()
old_region_process = yield deferToDatabase(reload_object,
old_region_process)
old_other_region_process = yield deferToDatabase(
reload_object, old_other_region_process)
self.assertIsNone(old_region_process)
self.assertIsNone(old_other_region_process)
yield master.stopService()
def test_update_sets_regiond_running_with_all_workers(self):
yield deferToDatabase(load_builtin_scripts)
master = self.make_IPCMasterService()
yield master.startService()
pids = set()
while len(pids) < workers.MAX_WORKERS_COUNT:
pids.add(random.randint(1, 512))
for pid in pids:
yield master.registerWorker(pid, MagicMock())
# The service status for the region should now be running.
regiond_service = yield deferToDatabase(self.getRegiondService)
self.assertThat(
regiond_service,
MatchesStructure.byEquality(status=SERVICE_STATUS.RUNNING,
status_info=""),
)
# Delete all the processes and an update should re-create them
# and the service status should still be running.
def delete_all():
region = RegionController.objects.get_running_controller()
region.processes.all().delete()
yield deferToDatabase(delete_all)
yield master.update()
regiond_service = yield deferToDatabase(self.getRegiondService)
self.assertThat(
regiond_service,
MatchesStructure.byEquality(status=SERVICE_STATUS.RUNNING,
status_info=""),
)
yield master.stopService()
def async_compose_machine(
result, power_type, power_paramaters, **kwargs
):
client, result = result
requested_machine = yield deferToDatabase(
self.get_requested_machine, result
)
result = yield compose_machine(
client,
power_type,
power_paramaters,
requested_machine,
**kwargs,
)
return requested_machine, result
def test_processMessages_calls_processMessage(self):
worker = StatusWorkerService(sentinel.dbtasks)
mock_processMessage = self.patch(worker, "_processMessage")
yield deferToDatabase(
worker._processMessages,
sentinel.node,
[sentinel.message1, sentinel.message2],
)
self.assertThat(
mock_processMessage,
MockCallsMatch(
call(sentinel.node, sentinel.message1),
call(sentinel.node, sentinel.message2),
),
)
def test_queueMessages_processes_top_level_message_instantly(self):
worker = StatusWorkerService(sentinel.dbtasks)
mock_processMessage = self.patch(worker, "_processMessage")
mock_updateLastPing = self.patch(worker, "_updateLastPing")
message = self.make_message()
message['event_type'] = 'finish'
nodes_with_tokens = yield deferToDatabase(self.make_nodes_with_tokens)
node, token = nodes_with_tokens[0]
yield worker.queueMessage(token.key, message)
self.assertThat(
mock_processMessage,
MockCalledOnceWith(node, message))
self.assertThat(
mock_updateLastPing,
MockCalledOnceWith(node, message))
def test_register_creates_new_rack(self):
yield self.installFakeRegion()
protocol = self.make_Region()
protocol.transport = MagicMock()
hostname = factory.make_hostname()
yield call_responder(
protocol,
RegisterRackController,
{
"system_id": None,
"hostname": hostname,
"interfaces": {}
},
)
yield deferToDatabase(RackController.objects.get, hostname=hostname)
def test_update_calls_mark_dead_on_regions_without_processes(self):
master = self.make_IPCMasterService()
yield master.startService()
mock_mark_dead = self.patch(Service.objects, "mark_dead")
other_region = yield deferToDatabase(factory.make_RegionController)
pid = random.randint(1, 512)
yield master.registerWorker(pid, MagicMock())
yield master.update()
self.assertThat(mock_mark_dead,
MockCalledOnceWith(other_region, dead_region=True))
yield master.stopService()
def test_register_sets_ident(self):
yield self.installFakeRegion()
rack_controller = yield deferToDatabase(factory.make_RackController)
protocol = self.make_Region()
protocol.transport = MagicMock()
yield call_responder(
protocol,
RegisterRackController,
{
"system_id": rack_controller.system_id,
"hostname": rack_controller.hostname,
"interfaces": {},
},
)
self.assertEquals(rack_controller.system_id, protocol.ident)
def test__tryUpdate_updates_ntp_server(self):
service = ntp.RegionNetworkTimeProtocolService(reactor)
refs, peers = yield deferToDatabase(self.make_example_configuration)
configure_region = self.patch_autospec(ntp, "configure_region")
restartService = self.patch_autospec(service_monitor, "restartService")
yield service._tryUpdate()
self.assertThat(
configure_region,
MockCalledOnceWith(refs, Matches(AllMatch(ContainedBy(peers)))))
self.assertThat(restartService, MockCalledOnceWith("ntp_region"))
# If the configuration has not changed then a second call to
# `_tryUpdate` does not result in another call to `configure_region`.
yield service._tryUpdate()
self.assertThat(configure_region, MockCalledOnce())
self.assertThat(restartService, MockCalledOnceWith("ntp_region"))
def get_syslog_configuration(self, system_id):
"""Get settings to use for configuring syslog.
Implementation of
:py:class:`~provisioningserver.rpc.region.GetSyslogConfiguration`.
"""
# For consistency `system_id` is passed, but at the moment it is not
# used to customise the syslog configuration.
@transactional
def get_from_db():
port = Config.objects.get_config("maas_syslog_port")
return {"port": port}
return deferToDatabase(get_from_db)
def test_registerWorker_sets_regiond_degraded_with_less_than_workers(self):
master = self.make_IPCMasterService()
yield master.startService()
pid = random.randint(1, 512)
yield master.registerWorker(pid, MagicMock())
regiond_service = yield deferToDatabase(self.getRegiondService)
self.assertThat(regiond_service, MatchesStructure.byEquality(
status=SERVICE_STATUS.DEGRADED,
status_info="1 process running but %s were expected." % (
workers.MAX_WORKERS_COUNT)))
yield master.stopService()
