def test_gatherCallResults_yields_call_result_tuples(self): values = [ self.getUniqueInteger(), self.getUniqueString(), ] calls = [self.wrap(lambda v=value: v) for value in values] results = list(asynchronous.gatherCallResults(calls)) expected_results = [(calls[0], values[0]), (calls[1], values[1])] self.assertItemsEqual(expected_results, results)
def test_gatherCallResults_returns_use_once_iterator(self): calls = [] results = asynchronous.gatherCallResults(calls) self.assertIsInstance(results, asynchronous.UseOnceIterator)
def call_clusters(command, *, kwargs=None, timeout=10, controllers=None, ignore_errors=True, available_callback=_none, unavailable_callback=_none, success_callback=_none, failed_callback=_none, failure_callback=_none, timeout_callback=_none): """Make an RPC call to all rack controllers in parallel. Includes optional callbacks to report the status of the call for each controller. If the call was a success, the `success_callback` will be called immediately before the response is yielded (so that the caller can determine which controller was contacted successfully). All optional callbacks are called with a single argument: the `RackController` model object that corresponds to the RPC call. :param controllers: The :class:`RackController`s on which to make the RPC call. If None, defaults to all :class:`RackController`s. :param timeout: The maximum number of seconds to wait for responses from all controllers. :param command: An :class:`amp.Command` to call on the clusters. :param ignore_errors: If True, errors encountered whilst calling `command` on the clusters won't raise an exception. :param available_callback: Optional callback; called with the controller when an RPC connection to the controller was established. :param unavailable_callback: Optional callback; called with the controller when an RPC connection to the controller failed to be established. :param success_callback: Optional callback; called with the controller when the RPC call was a success and this method is about to yield the result. :param failed_callback: Optional callback; called with the controller if the RPC call fails. :param failure_callback: Optional callback; called with the `Failure` object if the RPC call fail with a well-known exception. :param timeout_callback: Optional callback; called if the RPC call fails with a timeout. :param kwargs: Optional keyword arguments to pass to the command :return: A generator of results, i.e. the dicts returned by the RPC call. :raises: :py:class:`ClusterUnavailable` when a cluster is not connected or there's an error during the call, and errors are not being ignored. """ # Get the name of the RPC function for logging purposes. Each RPC function # is enacapsulated in a `class`, so should have a corresponding `__name__`. # However, we don't want to crash if that isn't the case. if kwargs is None: kwargs = {} command_name = (command.commandName.decode("ascii") if hasattr( command, 'commandName') else "<unknown>") calls = {} if controllers is None: controllers = RackController.objects.all() for controller in controllers: try: client = getClientFor(controller.system_id) except NoConnectionsAvailable: logger.error( "Error while calling %s: Unable to get RPC connection for " "rack controller '%s' (%s).", command_name, controller.hostname, controller.system_id) unavailable_callback(controller) if not ignore_errors: raise ClusterUnavailable( "Unable to get RPC connection for rack controller " "'%s' (%s)" % (controller.hostname, controller.system_id)) else: # The call to partial() requires a `callable`, but `getClientFor()` # might return a `Deferred` if it runs in the reactor. assert callable(client), ( "call_clusters() must not be called in the reactor thread. " "You probably want to use deferToDatabase().") available_callback(controller) call = partial(client, command, **kwargs) calls[call] = controller for call, response in asynchronous.gatherCallResults(calls, timeout=timeout): # When a call returns results, figure out which controller it came from # and remove it from the list, so we can report which controllers # timed out. controller = calls[call] del calls[call] if isinstance(response, Failure): # Create a nice message for logging purposes. We can rely on # the 'type' ivar being filled in with the Exception type in a # Failure object, so use that to get a nice version of the name. exception_class = response.type.__name__ error = str(response.value).strip() if len(error) > 0: error = ": " + error human_readable_error = ( "Exception during %s() on rack controller '%s' (%s): %s%s" % (command_name, controller.hostname, controller.system_id, exception_class, error)) logger.warning(human_readable_error) # For failures, there are two callbacks: one for the controller # that failed, the second for the specific failure that occurred. failed_callback(controller) failure_callback(response) if not ignore_errors: raise ClusterUnavailable(human_readable_error) else: success_callback(controller) yield response # Each remaining controller [value] in the `calls` dict has timed out. for controller in calls.values(): timeout_callback(controller) logger.error( "Error while calling %s: RPC connection timed out to rack " "controller '%s' (%s).", command_name, controller.hostname, controller.system_id)