def test_getExpectedState_returns_off_for_proxy_on_and_set(self):
# Disable boot source cache signals.
self.addCleanup(bootsources.signals.enable)
bootsources.signals.disable()
service = self.make_proxy_service()
yield deferToDatabase(transactional(Config.objects.set_config),
"enable_http_proxy", True)
yield deferToDatabase(transactional(Config.objects.set_config),
"http_proxy", factory.make_url())
self.patch(config, "is_config_present").return_value = True
expected_state = yield maybeDeferred(service.getExpectedState)
self.assertEqual(
(SERVICE_STATE.OFF,
'disabled, alternate proxy is configured in settings.'),
expected_state)
def execute(self, method_name, params):
"""Execute the given method on the handler.
Checks to make sure the method is valid and allowed perform executing
the method.
"""
if method_name in self._meta.allowed_methods:
try:
method = getattr(self, method_name)
except AttributeError:
raise HandlerNoSuchMethodError(method_name)
else:
# Handler methods are predominantly transactional and thus
# blocking/synchronous. Genuinely non-blocking/asynchronous
# methods must out themselves explicitly.
if IAsynchronous.providedBy(method):
# The @asynchronous decorator will DTRT.
return method(params)
else:
# This is going to block and hold a database connection so
# we limit its concurrency.
return concurrency.webapp.run(deferToDatabase,
transactional(method),
params)
else:
raise HandlerNoSuchMethodError(method_name)
def test_get_boot_config_returns_expected_result(self):
rack_controller = yield deferToDatabase(
transactional(factory.make_RackController))
yield deferToDatabase(make_usable_architecture, self)
local_ip = factory.make_ip_address()
remote_ip = factory.make_ip_address()
response = yield call_responder(
Region(), GetBootConfig, {
"system_id": rack_controller.system_id,
"local_ip": local_ip,
"remote_ip": remote_ip,
})
self.assertThat(
response,
ContainsAll([
"arch",
"subarch",
"osystem",
"release",
"kernel",
"initrd",
"boot_dtb",
"purpose",
"hostname",
"domain",
"preseed_url",
"fs_host",
"log_host",
"extra_opts",
]))
def test_getExpectedState_returns_on_for_proxy_off_and_unset(self):
# Disable boot source cache signals.
self.addCleanup(bootsources.signals.enable)
bootsources.signals.disable()
service = self.make_proxy_service()
yield deferToDatabase(
transactional(Config.objects.set_config),
"enable_http_proxy",
False,
)
yield deferToDatabase(transactional(Config.objects.set_config),
"http_proxy", "")
self.patch(config, "is_config_present").return_value = True
expected_state = yield maybeDeferred(service.getExpectedState)
self.assertEqual((SERVICE_STATE.ON, None), expected_state)
def process(self):
"""Process the DNS and/or proxy update."""
defers = []
if self.needsDNSUpdate:
self.needsDNSUpdate = False
d = deferToDatabase(transactional(dns_update_all_zones))
d.addCallback(
lambda _: log.msg(
"Successfully configured DNS."))
d.addErrback(
log.err,
"Failed configuring DNS.")
defers.append(d)
if self.needsProxyUpdate:
self.needsProxyUpdate = False
d = proxy_update_config(reload_proxy=True)
d.addCallback(
lambda _: log.msg(
"Successfully configured proxy."))
d.addErrback(
log.err,
"Failed configuring proxy.")
defers.append(d)
if len(defers) == 0:
# Nothing more to do.
self.processing.stop()
self.processingDefer = None
else:
return DeferredList(defers)
def delete(self, params):
"""Delete the object."""
assert self.user.is_superuser, "Permission denied."
d = deferToDatabase(transactional(self.get_object), params)
d.addCallback(lambda pod: pod.async_delete())
return d
def maybe_push_prometheus_stats(self):
def determine_stats_request():
config = Config.objects.get_configs(
[
"maas_name",
"prometheus_enabled",
"prometheus_push_gateway",
"prometheus_push_interval",
]
)
# Update interval
self._update_interval(
timedelta(minutes=config["prometheus_push_interval"])
)
# Determine if we can run the actual update.
if (
not PROMETHEUS_SUPPORTED
or not config["prometheus_enabled"]
or config["prometheus_push_gateway"] is None
):
return
# Run updates.
push_stats_to_prometheus(
config["maas_name"], config["prometheus_push_gateway"]
)
d = deferToDatabase(transactional(determine_stats_request))
d.addErrback(log.err, "Failure pushing stats to prometheus gateway")
return d
def refresh(self, params):
"""Refresh a specific Pod.
Performs pod discovery and updates all discovered information and
discovered machines.
"""
@transactional
def get_form(obj, params):
# Clear rbac cache before check (this is in its own thread).
rbac.clear()
obj = self.get_object(params)
if not self.user.has_perm(self._meta.edit_permission, obj):
raise HandlerPermissionError()
request = HttpRequest()
request.user = self.user
return PodForm(
instance=obj, data=self.preprocess_form("refresh", params),
request=request)
@transactional
def render_obj(obj):
return self.full_dehydrate(obj)
d = deferToDatabase(transactional(self.get_object), params)
d.addCallback(partial(deferToDatabase, get_form), params)
d.addCallback(lambda form: form.discover_and_sync_pod())
d.addCallback(partial(deferToDatabase, render_obj))
return d
def refresh(self, params):
"""Refresh a specific Pod.
Performs pod discovery and updates all discovered information and
discovered machines.
"""
assert self.user.is_superuser, "Permission denied."
@transactional
def get_form(obj, params):
request = HttpRequest()
request.user = self.user
return PodForm(instance=obj,
data=self.preprocess_form("refresh", params),
request=request)
@transactional
def render_obj(obj):
return self.full_dehydrate(obj)
d = deferToDatabase(transactional(self.get_object), params)
d.addCallback(partial(deferToDatabase, get_form), params)
d.addCallback(lambda form: form.discover_and_sync_pod())
d.addCallback(partial(deferToDatabase, render_obj))
return d
def process(self):
"""Process the DNS and/or proxy update."""
def _onFailureRetry(failure, attr):
"""Retry update on failure.
Doesn't mask the failure, the failure is still raised.
"""
if self.retryOnFailure:
setattr(self, attr, True)
return failure
def _rbacInit(result):
"""Mark initialization took place."""
if result is not None:
# A sync occurred.
self.rbacInit = True
return result
def _rbacFailure(failure, delay):
log.err(failure, "Failed syncing resources to RBAC.")
if delay:
return pause(delay)
defers = []
if self.needsDNSUpdate:
self.needsDNSUpdate = False
d = deferToDatabase(transactional(dns_update_all_zones))
d.addCallback(self._checkSerial)
d.addCallback(self._logDNSReload)
# Order here matters, first needsDNSUpdate is set then pass the
# failure onto `_onDNSReloadFailure` to do the correct thing
# with the DNS server.
d.addErrback(_onFailureRetry, "needsDNSUpdate")
d.addErrback(self._onDNSReloadFailure)
d.addErrback(log.err, "Failed configuring DNS.")
defers.append(d)
if self.needsProxyUpdate:
self.needsProxyUpdate = False
d = proxy_update_config(reload_proxy=True)
d.addCallback(lambda _: log.msg("Successfully configured proxy."))
d.addErrback(_onFailureRetry, "needsProxyUpdate")
d.addErrback(log.err, "Failed configuring proxy.")
defers.append(d)
if self.needsRBACUpdate:
self.needsRBACUpdate = False
d = deferToDatabase(self._rbacSync)
d.addCallback(_rbacInit)
d.addCallback(self._logRBACSync)
d.addErrback(_onFailureRetry, "needsRBACUpdate")
d.addErrback(
_rbacFailure,
self.rbacRetryOnFailureDelay if self.retryOnFailure else None,
)
defers.append(d)
if len(defers) == 0:
# Nothing more to do.
self.processing.stop()
self.processingDefer = None
else:
return DeferredList(defers)
def check_power(self, params):
"""Check the power state of the node."""
def eb_unknown(failure):
failure.trap(UnknownPowerType, NotImplementedError)
return POWER_STATE.UNKNOWN
def eb_error(failure):
log.err(failure, "Failed to update power state of machine.")
return POWER_STATE.ERROR
@transactional
def update_state(state):
if state in [POWER_STATE.ERROR, POWER_STATE.UNKNOWN]:
# Update the power state only if it was an error or unknown as
# that could have come from the previous errbacks.
obj = self.get_object(params)
obj.update_power_state(state)
return state
d = deferToDatabase(transactional(self.get_object), params)
d.addCallback(lambda node: node.power_query())
d.addErrback(eb_unknown)
d.addErrback(eb_error)
d.addCallback(partial(deferToDatabase, update_state))
return d
def maybe_check_release_notifications(self):
def check_config():
return Config.objects.get_config("release_notifications")
d = deferToDatabase(transactional(check_config))
d.addCallback(self.check_notifications)
d.addErrback(log.err, "Failure checking release notifications.")
return d
def test__with_prefer_v4_proxy_True(self):
self.patch(settings, "PROXY_CONNECT", True)
yield deferToDatabase(transactional(Config.objects.set_config),
"prefer_v4_proxy", True)
yield proxyconfig.proxy_update_config(reload_proxy=False)
with self.proxy_path.open() as proxy_file:
lines = [line.strip() for line in proxy_file.readlines()]
self.assertIn("dns_v4_first on", lines)
def test__with_use_peer_proxy_with_http_proxy(self):
self.patch(settings, "PROXY_CONNECT", True)
yield deferToDatabase(transactional(Config.objects.set_config),
"enable_http_proxy", True)
yield deferToDatabase(transactional(Config.objects.set_config),
"use_peer_proxy", True)
yield deferToDatabase(transactional(Config.objects.set_config),
"http_proxy", "http://example.com:8000/")
yield deferToDatabase(self.make_subnet, allow_proxy=False)
yield deferToDatabase(self.make_subnet)
yield proxyconfig.proxy_update_config(reload_proxy=False)
cache_peer_line = (
"cache_peer example.com parent 8000 0 no-query default")
with self.proxy_path.open() as proxy_file:
lines = [line.strip() for line in proxy_file.readlines()]
self.assertIn('never_direct allow all', lines)
self.assertIn(cache_peer_line, lines)
def maybe_make_stats_request(self):
def determine_stats_request():
if Config.objects.get_config("enable_analytics"):
make_maas_user_agent_request()
d = deferToDatabase(transactional(determine_stats_request))
d.addErrback(log.err, "Failure performing user agent request.")
return d
def test_processMessageNow_fails_when_in_transaction(self):
worker = StatusWorkerService(sentinel.dbtasks)
with ExpectedException(TransactionManagementError):
yield deferToDatabase(
transactional(worker._processMessageNow),
sentinel.node,
sentinel.message,
)
def execute(self, method_name, params):
"""Execute the given method on the handler.
Checks to make sure the method is valid and allowed perform executing
the method.
"""
if method_name in self._meta.allowed_methods:
try:
method = getattr(self, method_name)
except AttributeError:
raise HandlerNoSuchMethodError(method_name)
else:
# Handler methods are predominantly transactional and thus
# blocking/synchronous. Genuinely non-blocking/asynchronous
# methods must out themselves explicitly.
if IAsynchronous.providedBy(
method
) or asyncio.iscoroutinefunction(method):
# Running in the io thread so clear RBAC now.
rbac.clear()
# Reload the user from the database.
d = concurrency.webapp.run(
deferToDatabase,
transactional(self.user.refresh_from_db),
)
d.addCallback(lambda _: ensureDeferred(method(params)))
return d
else:
@wraps(method)
@transactional
def prep_user_execute(params):
# Clear RBAC and reload the user to ensure that
# its up to date. `rbac.clear` must be done inside
# the thread because it uses thread locals internally.
rbac.clear()
self.user.refresh_from_db()
# Perform the work in the database.
return self._call_method_track_queries(
method_name, method, params
)
# Force the name of the function to include the handler
# name so the debug logging is useful.
prep_user_execute.__name__ = "%s.%s" % (
self.__class__.__name__,
method_name,
)
# This is going to block and hold a database connection so
# we limit its concurrency.
return concurrency.webapp.run(
deferToDatabase, prep_user_execute, params
)
else:
raise HandlerNoSuchMethodError(method_name)
def test_updateRackController_calls_onNotify_for_controller_update(self):
user = yield deferToDatabase(transactional(maas_factory.make_User))
controller = yield deferToDatabase(
transactional(maas_factory.make_RackController))
protocol, factory = self.make_protocol_with_factory(user=user)
mock_onNotify = self.patch(factory, "onNotify")
controller_handler = MagicMock()
factory.handlers["controller"] = controller_handler
yield factory.updateRackController(controller.system_id)
self.assertThat(
mock_onNotify,
MockCalledOnceWith(
controller_handler,
"controller",
"update",
controller.system_id,
),
)
def test__with_use_peer_proxy_without_http_proxy(self):
self.patch(settings, "PROXY_CONNECT", True)
yield deferToDatabase(
transactional(Config.objects.set_config),
"enable_http_proxy", True)
yield deferToDatabase(
transactional(Config.objects.set_config),
"use_peer_proxy", True)
yield deferToDatabase(
transactional(Config.objects.set_config),
"http_proxy", "")
yield deferToDatabase(self.make_subnet, allow_proxy=False)
yield deferToDatabase(self.make_subnet)
yield proxyconfig.proxy_update_config(reload_proxy=False)
with self.proxy_path.open() as proxy_file:
lines = [line.strip() for line in proxy_file.readlines()]
self.assertNotIn('never_direct allow all', lines)
self.assertNotIn('cache_peer', lines)
def test_processDHCP_calls_configure_dhcp(self):
rack = yield deferToDatabase(transactional(
factory.make_RackController))
service = RackControllerService(sentinel.ipcWorker, sentinel.listener)
mock_configure_dhcp = self.patch(rack_controller.dhcp,
"configure_dhcp")
mock_configure_dhcp.return_value = succeed(None)
yield service.processDHCP(rack.id)
self.assertThat(mock_configure_dhcp, MockCalledOnceWith(rack))
def save(self, *args, **kwargs):
"""Persist the pod into the database."""
def check_for_duplicate(power_type, power_parameters):
# When the Pod is new try to get a BMC of the same type and
# parameters to convert the BMC to a new Pod. When the Pod is not
# new the form will use the already existing pod instance to update
# those fields. If updating the fields causes a duplicate BMC then
# a validation erorr will be raised from the model level.
if self.is_new:
bmc = BMC.objects.filter(
power_type=power_type,
power_parameters=power_parameters).first()
if bmc is not None:
if bmc.bmc_type == BMC_TYPE.BMC:
# Convert the BMC to a Pod and set as the instance for
# the PodForm.
bmc.bmc_type = BMC_TYPE.POD
bmc.default_pool = (
ResourcePool.objects.get_default_resource_pool())
return bmc.as_pod()
else:
# Pod already exists with the same power_type and
# parameters.
raise ValidationError("Pod %s with type and "
"parameters already exist." %
bmc.name)
def update_obj(existing_obj):
if existing_obj is not None:
self.instance = existing_obj
self.instance = super(PodForm, self).save(commit=False)
self.instance.power_type = power_type
self.instance.power_parameters = power_parameters
return self.instance
power_type = self.cleaned_data['type']
# Set power_parameters to the generated param_fields.
power_parameters = {
param_name: self.cleaned_data[param_name]
for param_name in self.param_fields.keys()
if param_name in self.cleaned_data
}
if isInIOThread():
# Running in twisted reactor, do the work inside the reactor.
d = deferToDatabase(transactional(check_for_duplicate), power_type,
power_parameters)
d.addCallback(update_obj)
d.addCallback(lambda _: self.discover_and_sync_pod())
return d
else:
# Perform the actions inside the executing thread.
existing_obj = check_for_duplicate(power_type, power_parameters)
if existing_obj is not None:
self.instance = existing_obj
self.instance = update_obj(self.instance)
return self.discover_and_sync_pod()
def test__with_new_maas_proxy_port_changes_port(self):
self.patch(settings, "PROXY_CONNECT", True)
port = random.randint(1, 65535)
yield deferToDatabase(transactional(Config.objects.set_config),
"maas_proxy_port", port)
yield proxyconfig.proxy_update_config(reload_proxy=False)
with self.proxy_path.open() as proxy_file:
lines = [line.strip() for line in proxy_file.readlines()]
self.assertIn("http_port %s" % port, lines)
def processDHCP(self, rack_id):
"""Process DHCP for the rack controller."""
log.debug("[pid:{pid()}] pushing DHCP to rack: {rack_id}",
pid=os.getpid,
rack_id=rack_id)
d = deferToDatabase(transactional(RackController.objects.get),
id=rack_id)
d.addCallback(dhcp.configure_dhcp)
return d
def request(self, **request):
# Make sure that requests are done within a transaction. Some kinds of
# tests will already have a transaction in progress, in which case
# this will act like a sub-transaction, but that's fine.
upcall = transactional(super().request)
# If we're outside of a transaction right now then the transactional()
# wrapper above will ensure that post-commit hooks are run or reset on
# return from the request. However, we want to ensure that post-commit
# hooks are fired in any case, hence the belt-n-braces context.
with post_commit_hooks:
return upcall(**request)
def test__logs_other_errors(self):
node = yield deferToDatabase(transactional(factory.make_Node))
mock_power_query = self.patch(Node, "power_query")
mock_power_query.side_effect = factory.make_exception('Error')
mock_log_err = self.patch(power.log, "err")
yield power.update_power_state_of_node(node.system_id)
self.assertThat(
mock_log_err,
MockCalledOnceWith(
ANY, "Failed to update power state of machine after state "
"transition."))
def test_get_archive_mirrors_with_ports_archive_set(self):
ports_archive = yield deferToDatabase(
lambda: PackageRepository.get_ports_archive())
url = factory.make_parsed_url(scheme='http')
ports_archive.url = url.geturl()
yield deferToDatabase(transactional(ports_archive.save))
response = yield call_responder(Region(), GetArchiveMirrors, {})
self.assertEqual(
{"main": urlparse("http://archive.ubuntu.com/ubuntu"),
"ports": url},
response)
def save_other(self, params):
"""Update `BootSourceSelection`'s to only include the selected
images."""
# Must be administrator.
assert self.user.is_superuser, "Permission denied."
@transactional
def update_selections(params):
# Remove all selections that are not Ubuntu.
BootSourceSelection.objects.exclude(
Q(os="ubuntu") | Q(os="ubuntu-core")).delete()
# Break down the images into os/release with multiple arches.
selections = defaultdict(list)
for image in params["images"]:
os, arch, _, release = image.split("/", 4)
name = "%s/%s" % (os, release)
selections[name].append(arch)
# Create each selection for the source.
for name, arches in selections.items():
os, release = name.split("/")
cache = BootSourceCache.objects.filter(
os=os, arch=arch, release=release).first()
if cache is None:
# It is possible the cache changed while waiting for the
# user to perform an action. Ignore the selection as its
# no longer available.
continue
# Create the selection for the source.
BootSourceSelection.objects.create(
boot_source=cache.boot_source,
os=os,
release=release,
arches=arches,
subarches=["*"],
labels=["*"],
)
notify = Deferred()
d = stop_import_resources()
d.addCallback(lambda _: deferToDatabase(update_selections, params))
d.addCallback(callOut, import_resources, notify=notify)
d.addCallback(lambda _: notify)
d.addCallback(lambda _: deferToDatabase(transactional(self.poll), {}))
d.addErrback(
log.err,
"Failed to start the image import. Unable to save the non-Ubuntu "
"image(s) source information",
)
return d
def wrap_compose_machine(client_idents, pod_type, parameters,
request, pod_id, name):
"""Wrapper to get the client."""
d = getClientFromIdentifiers(client_idents)
d.addCallback(
partial(deferToDatabase,
transactional(check_over_commit_ratios)))
d.addCallback(compose_machine,
pod_type,
parameters,
request,
pod_id=pod_id,
name=name)
return d
def check_notifications(self, notifications_enabled):
if not notifications_enabled:
maaslog.debug("Release notifications are disabled")
# Notifications are disabled, we can delete any that currently exist.
yield deferToDatabase(transactional(self.cleanup_notification))
return
if not notification_available(self.release_notification.maas_version):
maaslog.debug("No new release notifications available")
return
maaslog.debug("Notification to display")
yield deferToDatabase(ensure_notification_exists,
self.release_notification.message)
def save_ubuntu(self, params):
"""Called to save the Ubuntu section of the websocket."""
# Must be administrator.
assert self.user.is_superuser, "Permission denied."
@transactional
def update_source(params):
os = "ubuntu"
releases = params["releases"]
arches = params["arches"]
boot_source = self.get_bootsource(params, from_db=True)
# Remove all selections, that are not of release.
BootSourceSelection.objects.filter(
boot_source=boot_source,
os=os).exclude(release__in=releases).delete()
if len(releases) > 0:
# Create or update the selections.
for release in releases:
selection, _ = BootSourceSelection.objects.get_or_create(
boot_source=boot_source, os=os, release=release)
selection.arches = arches
selection.subarches = ["*"]
selection.labels = ["*"]
selection.save()
else:
# Create a selection that will cause nothing to be downloaded,
# since no releases are selected.
selection, _ = BootSourceSelection.objects.get_or_create(
boot_source=boot_source, os=os, release="")
selection.arches = arches
selection.subarches = ["*"]
selection.labels = ["*"]
selection.save()
notify = Deferred()
d = stop_import_resources()
d.addCallback(lambda _: deferToDatabase(update_source, params))
d.addCallback(callOut, import_resources, notify=notify)
d.addCallback(lambda _: notify)
d.addCallback(lambda _: deferToDatabase(transactional(self.poll), {}))
d.addErrback(
log.err,
"Failed to start the image import. Unable to save the Ubuntu "
"image(s) source information.",
)
return d
