def test_getExpectedState_returns_off_for_proxy_on_and_set(self): # Disable boot source cache signals. self.addCleanup(bootsources.signals.enable) bootsources.signals.disable() service = self.make_proxy_service() yield deferToDatabase(transactional(Config.objects.set_config), "enable_http_proxy", True) yield deferToDatabase(transactional(Config.objects.set_config), "http_proxy", factory.make_url()) self.patch(config, "is_config_present").return_value = True expected_state = yield maybeDeferred(service.getExpectedState) self.assertEqual( (SERVICE_STATE.OFF, 'disabled, alternate proxy is configured in settings.'), expected_state)
def execute(self, method_name, params): """Execute the given method on the handler. Checks to make sure the method is valid and allowed perform executing the method. """ if method_name in self._meta.allowed_methods: try: method = getattr(self, method_name) except AttributeError: raise HandlerNoSuchMethodError(method_name) else: # Handler methods are predominantly transactional and thus # blocking/synchronous. Genuinely non-blocking/asynchronous # methods must out themselves explicitly. if IAsynchronous.providedBy(method): # The @asynchronous decorator will DTRT. return method(params) else: # This is going to block and hold a database connection so # we limit its concurrency. return concurrency.webapp.run(deferToDatabase, transactional(method), params) else: raise HandlerNoSuchMethodError(method_name)
def test_get_boot_config_returns_expected_result(self): rack_controller = yield deferToDatabase( transactional(factory.make_RackController)) yield deferToDatabase(make_usable_architecture, self) local_ip = factory.make_ip_address() remote_ip = factory.make_ip_address() response = yield call_responder( Region(), GetBootConfig, { "system_id": rack_controller.system_id, "local_ip": local_ip, "remote_ip": remote_ip, }) self.assertThat( response, ContainsAll([ "arch", "subarch", "osystem", "release", "kernel", "initrd", "boot_dtb", "purpose", "hostname", "domain", "preseed_url", "fs_host", "log_host", "extra_opts", ]))
def test_getExpectedState_returns_on_for_proxy_off_and_unset(self): # Disable boot source cache signals. self.addCleanup(bootsources.signals.enable) bootsources.signals.disable() service = self.make_proxy_service() yield deferToDatabase( transactional(Config.objects.set_config), "enable_http_proxy", False, ) yield deferToDatabase(transactional(Config.objects.set_config), "http_proxy", "") self.patch(config, "is_config_present").return_value = True expected_state = yield maybeDeferred(service.getExpectedState) self.assertEqual((SERVICE_STATE.ON, None), expected_state)
def process(self): """Process the DNS and/or proxy update.""" defers = [] if self.needsDNSUpdate: self.needsDNSUpdate = False d = deferToDatabase(transactional(dns_update_all_zones)) d.addCallback( lambda _: log.msg( "Successfully configured DNS.")) d.addErrback( log.err, "Failed configuring DNS.") defers.append(d) if self.needsProxyUpdate: self.needsProxyUpdate = False d = proxy_update_config(reload_proxy=True) d.addCallback( lambda _: log.msg( "Successfully configured proxy.")) d.addErrback( log.err, "Failed configuring proxy.") defers.append(d) if len(defers) == 0: # Nothing more to do. self.processing.stop() self.processingDefer = None else: return DeferredList(defers)
def delete(self, params): """Delete the object.""" assert self.user.is_superuser, "Permission denied." d = deferToDatabase(transactional(self.get_object), params) d.addCallback(lambda pod: pod.async_delete()) return d
def maybe_push_prometheus_stats(self): def determine_stats_request(): config = Config.objects.get_configs( [ "maas_name", "prometheus_enabled", "prometheus_push_gateway", "prometheus_push_interval", ] ) # Update interval self._update_interval( timedelta(minutes=config["prometheus_push_interval"]) ) # Determine if we can run the actual update. if ( not PROMETHEUS_SUPPORTED or not config["prometheus_enabled"] or config["prometheus_push_gateway"] is None ): return # Run updates. push_stats_to_prometheus( config["maas_name"], config["prometheus_push_gateway"] ) d = deferToDatabase(transactional(determine_stats_request)) d.addErrback(log.err, "Failure pushing stats to prometheus gateway") return d
def refresh(self, params): """Refresh a specific Pod. Performs pod discovery and updates all discovered information and discovered machines. """ @transactional def get_form(obj, params): # Clear rbac cache before check (this is in its own thread). rbac.clear() obj = self.get_object(params) if not self.user.has_perm(self._meta.edit_permission, obj): raise HandlerPermissionError() request = HttpRequest() request.user = self.user return PodForm( instance=obj, data=self.preprocess_form("refresh", params), request=request) @transactional def render_obj(obj): return self.full_dehydrate(obj) d = deferToDatabase(transactional(self.get_object), params) d.addCallback(partial(deferToDatabase, get_form), params) d.addCallback(lambda form: form.discover_and_sync_pod()) d.addCallback(partial(deferToDatabase, render_obj)) return d
def refresh(self, params): """Refresh a specific Pod. Performs pod discovery and updates all discovered information and discovered machines. """ assert self.user.is_superuser, "Permission denied." @transactional def get_form(obj, params): request = HttpRequest() request.user = self.user return PodForm(instance=obj, data=self.preprocess_form("refresh", params), request=request) @transactional def render_obj(obj): return self.full_dehydrate(obj) d = deferToDatabase(transactional(self.get_object), params) d.addCallback(partial(deferToDatabase, get_form), params) d.addCallback(lambda form: form.discover_and_sync_pod()) d.addCallback(partial(deferToDatabase, render_obj)) return d
def process(self): """Process the DNS and/or proxy update.""" def _onFailureRetry(failure, attr): """Retry update on failure. Doesn't mask the failure, the failure is still raised. """ if self.retryOnFailure: setattr(self, attr, True) return failure def _rbacInit(result): """Mark initialization took place.""" if result is not None: # A sync occurred. self.rbacInit = True return result def _rbacFailure(failure, delay): log.err(failure, "Failed syncing resources to RBAC.") if delay: return pause(delay) defers = [] if self.needsDNSUpdate: self.needsDNSUpdate = False d = deferToDatabase(transactional(dns_update_all_zones)) d.addCallback(self._checkSerial) d.addCallback(self._logDNSReload) # Order here matters, first needsDNSUpdate is set then pass the # failure onto `_onDNSReloadFailure` to do the correct thing # with the DNS server. d.addErrback(_onFailureRetry, "needsDNSUpdate") d.addErrback(self._onDNSReloadFailure) d.addErrback(log.err, "Failed configuring DNS.") defers.append(d) if self.needsProxyUpdate: self.needsProxyUpdate = False d = proxy_update_config(reload_proxy=True) d.addCallback(lambda _: log.msg("Successfully configured proxy.")) d.addErrback(_onFailureRetry, "needsProxyUpdate") d.addErrback(log.err, "Failed configuring proxy.") defers.append(d) if self.needsRBACUpdate: self.needsRBACUpdate = False d = deferToDatabase(self._rbacSync) d.addCallback(_rbacInit) d.addCallback(self._logRBACSync) d.addErrback(_onFailureRetry, "needsRBACUpdate") d.addErrback( _rbacFailure, self.rbacRetryOnFailureDelay if self.retryOnFailure else None, ) defers.append(d) if len(defers) == 0: # Nothing more to do. self.processing.stop() self.processingDefer = None else: return DeferredList(defers)
def check_power(self, params): """Check the power state of the node.""" def eb_unknown(failure): failure.trap(UnknownPowerType, NotImplementedError) return POWER_STATE.UNKNOWN def eb_error(failure): log.err(failure, "Failed to update power state of machine.") return POWER_STATE.ERROR @transactional def update_state(state): if state in [POWER_STATE.ERROR, POWER_STATE.UNKNOWN]: # Update the power state only if it was an error or unknown as # that could have come from the previous errbacks. obj = self.get_object(params) obj.update_power_state(state) return state d = deferToDatabase(transactional(self.get_object), params) d.addCallback(lambda node: node.power_query()) d.addErrback(eb_unknown) d.addErrback(eb_error) d.addCallback(partial(deferToDatabase, update_state)) return d
def maybe_check_release_notifications(self): def check_config(): return Config.objects.get_config("release_notifications") d = deferToDatabase(transactional(check_config)) d.addCallback(self.check_notifications) d.addErrback(log.err, "Failure checking release notifications.") return d
def test__with_prefer_v4_proxy_True(self): self.patch(settings, "PROXY_CONNECT", True) yield deferToDatabase(transactional(Config.objects.set_config), "prefer_v4_proxy", True) yield proxyconfig.proxy_update_config(reload_proxy=False) with self.proxy_path.open() as proxy_file: lines = [line.strip() for line in proxy_file.readlines()] self.assertIn("dns_v4_first on", lines)
def test__with_use_peer_proxy_with_http_proxy(self): self.patch(settings, "PROXY_CONNECT", True) yield deferToDatabase(transactional(Config.objects.set_config), "enable_http_proxy", True) yield deferToDatabase(transactional(Config.objects.set_config), "use_peer_proxy", True) yield deferToDatabase(transactional(Config.objects.set_config), "http_proxy", "http://example.com:8000/") yield deferToDatabase(self.make_subnet, allow_proxy=False) yield deferToDatabase(self.make_subnet) yield proxyconfig.proxy_update_config(reload_proxy=False) cache_peer_line = ( "cache_peer example.com parent 8000 0 no-query default") with self.proxy_path.open() as proxy_file: lines = [line.strip() for line in proxy_file.readlines()] self.assertIn('never_direct allow all', lines) self.assertIn(cache_peer_line, lines)
def maybe_make_stats_request(self): def determine_stats_request(): if Config.objects.get_config("enable_analytics"): make_maas_user_agent_request() d = deferToDatabase(transactional(determine_stats_request)) d.addErrback(log.err, "Failure performing user agent request.") return d
def test_processMessageNow_fails_when_in_transaction(self): worker = StatusWorkerService(sentinel.dbtasks) with ExpectedException(TransactionManagementError): yield deferToDatabase( transactional(worker._processMessageNow), sentinel.node, sentinel.message, )
def execute(self, method_name, params): """Execute the given method on the handler. Checks to make sure the method is valid and allowed perform executing the method. """ if method_name in self._meta.allowed_methods: try: method = getattr(self, method_name) except AttributeError: raise HandlerNoSuchMethodError(method_name) else: # Handler methods are predominantly transactional and thus # blocking/synchronous. Genuinely non-blocking/asynchronous # methods must out themselves explicitly. if IAsynchronous.providedBy( method ) or asyncio.iscoroutinefunction(method): # Running in the io thread so clear RBAC now. rbac.clear() # Reload the user from the database. d = concurrency.webapp.run( deferToDatabase, transactional(self.user.refresh_from_db), ) d.addCallback(lambda _: ensureDeferred(method(params))) return d else: @wraps(method) @transactional def prep_user_execute(params): # Clear RBAC and reload the user to ensure that # its up to date. `rbac.clear` must be done inside # the thread because it uses thread locals internally. rbac.clear() self.user.refresh_from_db() # Perform the work in the database. return self._call_method_track_queries( method_name, method, params ) # Force the name of the function to include the handler # name so the debug logging is useful. prep_user_execute.__name__ = "%s.%s" % ( self.__class__.__name__, method_name, ) # This is going to block and hold a database connection so # we limit its concurrency. return concurrency.webapp.run( deferToDatabase, prep_user_execute, params ) else: raise HandlerNoSuchMethodError(method_name)
def test_updateRackController_calls_onNotify_for_controller_update(self): user = yield deferToDatabase(transactional(maas_factory.make_User)) controller = yield deferToDatabase( transactional(maas_factory.make_RackController)) protocol, factory = self.make_protocol_with_factory(user=user) mock_onNotify = self.patch(factory, "onNotify") controller_handler = MagicMock() factory.handlers["controller"] = controller_handler yield factory.updateRackController(controller.system_id) self.assertThat( mock_onNotify, MockCalledOnceWith( controller_handler, "controller", "update", controller.system_id, ), )
def test__with_use_peer_proxy_without_http_proxy(self): self.patch(settings, "PROXY_CONNECT", True) yield deferToDatabase( transactional(Config.objects.set_config), "enable_http_proxy", True) yield deferToDatabase( transactional(Config.objects.set_config), "use_peer_proxy", True) yield deferToDatabase( transactional(Config.objects.set_config), "http_proxy", "") yield deferToDatabase(self.make_subnet, allow_proxy=False) yield deferToDatabase(self.make_subnet) yield proxyconfig.proxy_update_config(reload_proxy=False) with self.proxy_path.open() as proxy_file: lines = [line.strip() for line in proxy_file.readlines()] self.assertNotIn('never_direct allow all', lines) self.assertNotIn('cache_peer', lines)
def test_processDHCP_calls_configure_dhcp(self): rack = yield deferToDatabase(transactional( factory.make_RackController)) service = RackControllerService(sentinel.ipcWorker, sentinel.listener) mock_configure_dhcp = self.patch(rack_controller.dhcp, "configure_dhcp") mock_configure_dhcp.return_value = succeed(None) yield service.processDHCP(rack.id) self.assertThat(mock_configure_dhcp, MockCalledOnceWith(rack))
def save(self, *args, **kwargs): """Persist the pod into the database.""" def check_for_duplicate(power_type, power_parameters): # When the Pod is new try to get a BMC of the same type and # parameters to convert the BMC to a new Pod. When the Pod is not # new the form will use the already existing pod instance to update # those fields. If updating the fields causes a duplicate BMC then # a validation erorr will be raised from the model level. if self.is_new: bmc = BMC.objects.filter( power_type=power_type, power_parameters=power_parameters).first() if bmc is not None: if bmc.bmc_type == BMC_TYPE.BMC: # Convert the BMC to a Pod and set as the instance for # the PodForm. bmc.bmc_type = BMC_TYPE.POD bmc.default_pool = ( ResourcePool.objects.get_default_resource_pool()) return bmc.as_pod() else: # Pod already exists with the same power_type and # parameters. raise ValidationError("Pod %s with type and " "parameters already exist." % bmc.name) def update_obj(existing_obj): if existing_obj is not None: self.instance = existing_obj self.instance = super(PodForm, self).save(commit=False) self.instance.power_type = power_type self.instance.power_parameters = power_parameters return self.instance power_type = self.cleaned_data['type'] # Set power_parameters to the generated param_fields. power_parameters = { param_name: self.cleaned_data[param_name] for param_name in self.param_fields.keys() if param_name in self.cleaned_data } if isInIOThread(): # Running in twisted reactor, do the work inside the reactor. d = deferToDatabase(transactional(check_for_duplicate), power_type, power_parameters) d.addCallback(update_obj) d.addCallback(lambda _: self.discover_and_sync_pod()) return d else: # Perform the actions inside the executing thread. existing_obj = check_for_duplicate(power_type, power_parameters) if existing_obj is not None: self.instance = existing_obj self.instance = update_obj(self.instance) return self.discover_and_sync_pod()
def test__with_new_maas_proxy_port_changes_port(self): self.patch(settings, "PROXY_CONNECT", True) port = random.randint(1, 65535) yield deferToDatabase(transactional(Config.objects.set_config), "maas_proxy_port", port) yield proxyconfig.proxy_update_config(reload_proxy=False) with self.proxy_path.open() as proxy_file: lines = [line.strip() for line in proxy_file.readlines()] self.assertIn("http_port %s" % port, lines)
def processDHCP(self, rack_id): """Process DHCP for the rack controller.""" log.debug("[pid:{pid()}] pushing DHCP to rack: {rack_id}", pid=os.getpid, rack_id=rack_id) d = deferToDatabase(transactional(RackController.objects.get), id=rack_id) d.addCallback(dhcp.configure_dhcp) return d
def request(self, **request): # Make sure that requests are done within a transaction. Some kinds of # tests will already have a transaction in progress, in which case # this will act like a sub-transaction, but that's fine. upcall = transactional(super().request) # If we're outside of a transaction right now then the transactional() # wrapper above will ensure that post-commit hooks are run or reset on # return from the request. However, we want to ensure that post-commit # hooks are fired in any case, hence the belt-n-braces context. with post_commit_hooks: return upcall(**request)
def test__logs_other_errors(self): node = yield deferToDatabase(transactional(factory.make_Node)) mock_power_query = self.patch(Node, "power_query") mock_power_query.side_effect = factory.make_exception('Error') mock_log_err = self.patch(power.log, "err") yield power.update_power_state_of_node(node.system_id) self.assertThat( mock_log_err, MockCalledOnceWith( ANY, "Failed to update power state of machine after state " "transition."))
def test_get_archive_mirrors_with_ports_archive_set(self): ports_archive = yield deferToDatabase( lambda: PackageRepository.get_ports_archive()) url = factory.make_parsed_url(scheme='http') ports_archive.url = url.geturl() yield deferToDatabase(transactional(ports_archive.save)) response = yield call_responder(Region(), GetArchiveMirrors, {}) self.assertEqual( {"main": urlparse("http://archive.ubuntu.com/ubuntu"), "ports": url}, response)
def save_other(self, params): """Update `BootSourceSelection`'s to only include the selected images.""" # Must be administrator. assert self.user.is_superuser, "Permission denied." @transactional def update_selections(params): # Remove all selections that are not Ubuntu. BootSourceSelection.objects.exclude( Q(os="ubuntu") | Q(os="ubuntu-core")).delete() # Break down the images into os/release with multiple arches. selections = defaultdict(list) for image in params["images"]: os, arch, _, release = image.split("/", 4) name = "%s/%s" % (os, release) selections[name].append(arch) # Create each selection for the source. for name, arches in selections.items(): os, release = name.split("/") cache = BootSourceCache.objects.filter( os=os, arch=arch, release=release).first() if cache is None: # It is possible the cache changed while waiting for the # user to perform an action. Ignore the selection as its # no longer available. continue # Create the selection for the source. BootSourceSelection.objects.create( boot_source=cache.boot_source, os=os, release=release, arches=arches, subarches=["*"], labels=["*"], ) notify = Deferred() d = stop_import_resources() d.addCallback(lambda _: deferToDatabase(update_selections, params)) d.addCallback(callOut, import_resources, notify=notify) d.addCallback(lambda _: notify) d.addCallback(lambda _: deferToDatabase(transactional(self.poll), {})) d.addErrback( log.err, "Failed to start the image import. Unable to save the non-Ubuntu " "image(s) source information", ) return d
def wrap_compose_machine(client_idents, pod_type, parameters, request, pod_id, name): """Wrapper to get the client.""" d = getClientFromIdentifiers(client_idents) d.addCallback( partial(deferToDatabase, transactional(check_over_commit_ratios))) d.addCallback(compose_machine, pod_type, parameters, request, pod_id=pod_id, name=name) return d
def check_notifications(self, notifications_enabled): if not notifications_enabled: maaslog.debug("Release notifications are disabled") # Notifications are disabled, we can delete any that currently exist. yield deferToDatabase(transactional(self.cleanup_notification)) return if not notification_available(self.release_notification.maas_version): maaslog.debug("No new release notifications available") return maaslog.debug("Notification to display") yield deferToDatabase(ensure_notification_exists, self.release_notification.message)
def save_ubuntu(self, params): """Called to save the Ubuntu section of the websocket.""" # Must be administrator. assert self.user.is_superuser, "Permission denied." @transactional def update_source(params): os = "ubuntu" releases = params["releases"] arches = params["arches"] boot_source = self.get_bootsource(params, from_db=True) # Remove all selections, that are not of release. BootSourceSelection.objects.filter( boot_source=boot_source, os=os).exclude(release__in=releases).delete() if len(releases) > 0: # Create or update the selections. for release in releases: selection, _ = BootSourceSelection.objects.get_or_create( boot_source=boot_source, os=os, release=release) selection.arches = arches selection.subarches = ["*"] selection.labels = ["*"] selection.save() else: # Create a selection that will cause nothing to be downloaded, # since no releases are selected. selection, _ = BootSourceSelection.objects.get_or_create( boot_source=boot_source, os=os, release="") selection.arches = arches selection.subarches = ["*"] selection.labels = ["*"] selection.save() notify = Deferred() d = stop_import_resources() d.addCallback(lambda _: deferToDatabase(update_source, params)) d.addCallback(callOut, import_resources, notify=notify) d.addCallback(lambda _: notify) d.addCallback(lambda _: deferToDatabase(transactional(self.poll), {})) d.addErrback( log.err, "Failed to start the image import. Unable to save the Ubuntu " "image(s) source information.", ) return d