def test_retry_sequence_fails_if_mismatch_sequence(self): """ Fail if the wrong number of performers are given. """ r = Retry( effect=Effect(1), should_retry=ShouldDelayAndRetry( can_retry=retry_times(5), next_interval=repeating_interval(10))) seq = [ retry_sequence(r, [lambda _: raise_(Exception()), lambda _: raise_(Exception())]) ] self.assertRaises(AssertionError, perform_sequence, seq, Effect(r))
def test_set_metadata_item(self): """ :obj:`SetMetadataItemOnServer.as_effect` produces a request for setting a metadata item on a particular server. It succeeds if successful, but does not fail for any errors. """ server_id = u'abc123' meta = SetMetadataItemOnServer(server_id=server_id, key='metadata_key', value='teapot') eff = meta.as_effect() seq = [ (eff.intent, lambda i: (StubResponse(202, {}), {})), (Log(ANY, ANY), lambda _: None) ] self.assertEqual( perform_sequence(seq, eff), (StepResult.SUCCESS, [])) exceptions = (NoSuchServerError("msg", server_id=server_id), ServerMetadataOverLimitError("msg", server_id=server_id), NovaRateLimitError("msg"), APIError(code=500, body="", headers={})) for exception in exceptions: self.assertRaises( type(exception), perform_sequence, [(eff.intent, lambda i: raise_(exception))], eff)
def test_retry_sequence_retries_without_delays(self): """ Perform the wrapped effect with the performers given, without any delay even if the original intent had a delay. """ r = Retry( effect=Effect(1), should_retry=ShouldDelayAndRetry( can_retry=retry_times(5), next_interval=repeating_interval(10))) seq = [ retry_sequence(r, [lambda _: raise_(Exception()), lambda _: raise_(Exception()), lambda _: "yay done"]) ] self.assertEqual(perform_sequence(seq, Effect(r)), "yay done")
def test_ignores_errors(self): """ Errors are not logged and are propogated """ seq = [("internal", lambda i: raise_(ValueError("oops")))] self.assertRaises(ValueError, perform_sequence, seq, msg_with_time("mt", Effect("internal")), self.disp) self.assertFalse(self.log.msg.called)
def test_list_servers_details_all_propagates_errors(self): """ :func:`list_servers_details_all` propagates exceptions from making the individual requests (from :func:`list_servers_details_page`). """ eff = list_servers_details_all({'marker': ['1']}) seq = [(self._list_server_details_intent({'marker': ['1']}), lambda _: raise_(NovaComputeFaultError('error')))] self.assertRaises(NovaComputeFaultError, perform_sequence, seq, eff)
def test_quit_game(): for exc in (KeyboardInterrupt(), EOFError()): expected_effects = [ (Display(render(initial_state)), noop), (Prompt("> "), lambda i: raise_(exc)), (Display("\nThanks for playing!"), noop), ] eff = step(initial_state) with raises(SystemExit): perform_sequence(expected_effects, eff)
def test_group_deleted(self): """ Does nothing if group has been deleted """ seq = [(GetScalingGroupInfo(tenant_id="tid", group_id="gid"), lambda i: raise_(NoSuchScalingGroupError("tid", "gid"))), (Log("selfheal-group-deleted", dict(tenant_id="tid", scaling_group_id="gid")), noop)] self.assertIsNone( perform_sequence(seq, sh.check_and_trigger("tid", "gid")))
def test_list_servers_details_all_propagates_errors(self): """ :func:`list_servers_details_all` propagates exceptions from making the individual requests (from :func:`list_servers_details_page`). """ eff = list_servers_details_all({'marker': ['1']}) seq = [ (self._list_server_details_intent({'marker': ['1']}), lambda _: raise_(NovaComputeFaultError('error'))) ] self.assertRaises(NovaComputeFaultError, perform_sequence, seq, eff)
def test_change_clb_node_terminal_errors(self): """Some errors during :obj:`ChangeCLBNode` make convergence fail.""" eff = self._change_node_eff() terminal = (NoSuchCLBNodeError(lb_id=u'abc123', node_id=u'node1'), CLBNotFoundError(lb_id=u'abc123'), APIError(code=400, body="", headers={})) for exception in terminal: self.assertEqual( perform_sequence([(eff.intent, lambda i: raise_(exception))], eff), (StepResult.FAILURE, [ANY]))
def test_change_clb_node_nonterminal_errors(self): """Some errors during :obj:`ChangeCLBNode` make convergence retry.""" eff = self._change_node_eff() nonterminal = (APIError(code=500, body="", headers={}), CLBNotActiveError(lb_id=u'abc123'), CLBRateLimitError(lb_id=u'abc123')) for exception in nonterminal: self.assertEqual( perform_sequence([(eff.intent, lambda i: raise_(exception))], eff), (StepResult.RETRY, ANY))
def test_mainloop(): expected_effects = [(Display(render(initial_state)), noop), (Prompt("> "), lambda i: "move east"), (Display("Okay."), noop), (SaveGame(state=in_street), noop), (Display(render(in_street)), noop), (Prompt("> "), lambda i: raise_(KeyboardInterrupt())), (Display("\nThanks for playing!"), noop)] eff = mainloop(initial_state) with raises(SystemExit): perform_sequence(expected_effects, eff)
def test_can_have_a_different_should_retry_function(self): """ The ``should_retry`` function does not have to be a :obj:`ShouldDelayAndRetry`. """ expected = Retry(effect=Effect(1), should_retry=ANY) actual = Retry(effect=Effect(1), should_retry=lambda _: False) seq = [ retry_sequence(expected, [lambda _: raise_(Exception())]) ] self.assertRaises(Exception, perform_sequence, seq, Effect(actual))
def test_group_deleted(self): """ Does nothing if group has been deleted """ seq = [ (GetScalingGroupInfo(tenant_id="tid", group_id="gid"), lambda i: raise_(NoSuchScalingGroupError("tid", "gid"))), (Log("selfheal-group-deleted", dict(tenant_id="tid", scaling_group_id="gid")), noop) ] self.assertIsNone( perform_sequence(seq, sh.check_and_trigger("tid", "gid")))
def remove_clb_nodes(lb_id, node_ids): """ Remove multiple nodes from a load balancer. :param str lb_id: A load balancer ID. :param node_ids: iterable of node IDs. :return: Effect of None. Succeeds on 202. This function will handle the case where *some* of the nodes are valid and some aren't, by retrying deleting only the valid ones. """ node_ids = list(node_ids) partial = None if len(node_ids) > CLB_BATCH_DELETE_LIMIT: not_removing = node_ids[CLB_BATCH_DELETE_LIMIT:] node_ids = node_ids[:CLB_BATCH_DELETE_LIMIT] partial = CLBPartialNodesRemoved(six.text_type(lb_id), map(six.text_type, not_removing), map(six.text_type, node_ids)) eff = service_request( ServiceType.CLOUD_LOAD_BALANCERS, 'DELETE', append_segments('loadbalancers', lb_id, 'nodes'), params={'id': map(str, node_ids)}, success_pred=has_code(202)) def check_invalid_nodes(exc_info): code = exc_info[1].code body = exc_info[1].body if code == 400: message = try_json_with_keys( body, ["validationErrors", "messages", 0]) if message is not None: match = _CLB_NODE_REMOVED_PATTERN.match(message) if match: removed = concat([group.split(',') for group in match.groups()]) return remove_clb_nodes(lb_id, set(node_ids) - set(removed)) six.reraise(*exc_info) return eff.on( error=catch(APIError, check_invalid_nodes) ).on( error=_only_json_api_errors( lambda c, b: _process_clb_api_error(c, b, lb_id)) ).on(success=lambda _: None if partial is None else raise_(partial))
def test_error_validating_observer(self): """ The observer returned replaces event with error if it fails to type check """ wrapper = SpecificationObserverWrapper( self.observer, lambda e: raise_(ValueError('hm'))) wrapper({'message': ("something-bad",), 'a': 'b'}) self.assertEqual( self.e, [{'original_event': {'message': ("something-bad",), 'a': 'b'}, 'isError': True, 'failure': CheckFailureValue(ValueError('hm')), 'why': 'Error validating event', 'message': ()}])
def test_other_errors(self): """ Any error other than `BulkErrors` results in RETRY """ non_terminals = (ValueError("internal"), APIError(code=500, body="why?"), APIError(code=503, body="bad service")) eff = self.step.as_effect() for exc in non_terminals: seq = [(("bd", self.pairs), lambda i: raise_(exc))] self.assertEqual( perform_sequence(seq, eff), (StepResult.RETRY, [ ErrorReason.Exception((type(exc), exc, ANY))]) )
def test_failure(self): """ Returns FAILURE if rcv3.bulk_delete raises BulkErrors """ terminals = (rcv3.BulkErrors([rcv3.LBInactive("l1")]), APIError(code=403, body="You're out of luck."), APIError(code=422, body="Oh look another 422.")) eff = self.step.as_effect() for exc in terminals: seq = [(("bd", self.pairs), lambda i: raise_(exc))] self.assertEqual( perform_sequence(seq, eff), (StepResult.FAILURE, [ ErrorReason.Exception((type(exc), exc, ANY))]) )
def test_do_not_have_to_expect_an_exact_can_retry(self): """ The expected retry intent does not actually have to specify the exact ``can_retry`` function, since it might just be a lambda, which is hard to compare or hash. """ expected = Retry(effect=Effect(1), should_retry=ANY) actual = Retry(effect=Effect(1), should_retry=ShouldDelayAndRetry( can_retry=lambda _: False, next_interval=repeating_interval(10))) seq = [ retry_sequence(expected, [lambda _: raise_(Exception())]) ] self.assertRaises(Exception, perform_sequence, seq, Effect(actual))
def test_remove_nodes_from_clb_success_failures(self): """ :obj:`AddNodesToCLB` succeeds if the CLB is not in existence (has been deleted or is not found). """ successes = [CLBNotFoundError(lb_id=u'12345'), CLBDeletedError(lb_id=u'12345'), NoSuchCLBError(lb_id=u'12345')] eff = RemoveNodesFromCLB(lb_id='12345', node_ids=pset(['1', '2'])).as_effect() for exc in successes: seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))]) with seq.consume(): self.assertEquals(sync_perform(seq, eff), (StepResult.SUCCESS, []))
def test_remove_nodes_from_clb_retry(self): """ :obj:`RemoveNodesFromCLB`, on receiving a 400, parses out the nodes that are no longer on the load balancer, and retries the bulk delete with those nodes removed. TODO: this has been left in as a regression test - this can probably be removed the next time it's touched, as this functionality happens in cloud_client now and there is a similar test there. """ lb_id = "12345" node_ids = [str(i) for i in range(5)] error_body = { "validationErrors": { "messages": [ "Node ids 1,2,3 are not a part of your loadbalancer" ] }, "message": "Validation Failure", "code": 400, "details": "The object is not valid" } expected_req = service_request( ServiceType.CLOUD_LOAD_BALANCERS, 'DELETE', 'loadbalancers/12345/nodes', params={'id': transform_eq(sorted, node_ids)}, success_pred=ANY, json_response=True).intent expected_req2 = service_request( ServiceType.CLOUD_LOAD_BALANCERS, 'DELETE', 'loadbalancers/12345/nodes', params={'id': transform_eq(sorted, ['0', '4'])}, success_pred=ANY, json_response=True).intent step = RemoveNodesFromCLB(lb_id=lb_id, node_ids=pset(node_ids)) seq = [ (expected_req, lambda i: raise_(APIError(400, json.dumps(error_body)))), (expected_req2, lambda i: stub_pure_response('', 202)), ] r = perform_sequence(seq, step.as_effect()) self.assertEqual(r, (StepResult.SUCCESS, []))
def test_remove_nodes_from_clb_terminal_failures(self): """ :obj:`AddNodesToCLB` fails if there are any 4xx errors, then the error is propagated up and the result is a failure. """ terminals = (APIError(code=403, body="You're out of luck."), APIError(code=422, body="Oh look another 422.")) eff = RemoveNodesFromCLB(lb_id='12345', node_ids=pset(['1', '2'])).as_effect() for exc in terminals: seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))]) with seq.consume(): self.assertEquals( sync_perform(seq, eff), (StepResult.FAILURE, [ErrorReason.Exception( matches(ContainsAll([type(exc), exc])))]))
def remove_clb_nodes(lb_id, node_ids): """ Remove multiple nodes from a load balancer. :param str lb_id: A load balancer ID. :param node_ids: iterable of node IDs. :return: Effect of None. Succeeds on 202. This function will handle the case where *some* of the nodes are valid and some aren't, by retrying deleting only the valid ones. """ node_ids = list(node_ids) partial = None if len(node_ids) > CLB_BATCH_DELETE_LIMIT: not_removing = node_ids[CLB_BATCH_DELETE_LIMIT:] node_ids = node_ids[:CLB_BATCH_DELETE_LIMIT] partial = CLBPartialNodesRemoved(six.text_type(lb_id), map(six.text_type, not_removing), map(six.text_type, node_ids)) eff = service_request(ServiceType.CLOUD_LOAD_BALANCERS, 'DELETE', append_segments('loadbalancers', lb_id, 'nodes'), params={'id': map(str, node_ids)}, success_pred=has_code(202)) def check_invalid_nodes(exc_info): code = exc_info[1].code body = exc_info[1].body if code == 400: message = try_json_with_keys(body, ["validationErrors", "messages", 0]) if message is not None: match = _CLB_NODE_REMOVED_PATTERN.match(message) if match: removed = concat( [group.split(',') for group in match.groups()]) return remove_clb_nodes(lb_id, set(node_ids) - set(removed)) six.reraise(*exc_info) return eff.on(error=catch(APIError, check_invalid_nodes)).on( error=only_json_api_errors( lambda c, b: _process_clb_api_error(c, b, lb_id))).on( success=lambda _: None if partial is None else raise_(partial))
def test_error_validating_observer(self): """ The observer returned replaces event with error if it fails to type check """ wrapper = SpecificationObserverWrapper( self.observer, lambda e: raise_(ValueError('hm'))) wrapper({'message': ("something-bad", ), 'a': 'b'}) self.assertEqual(self.e, [{ 'original_event': { 'message': ("something-bad", ), 'a': 'b' }, 'isError': True, 'failure': CheckFailureValue(ValueError('hm')), 'why': 'Error validating event', 'message': () }])
def test_remove_nodes_from_clb_non_terminal_failures_to_retry(self): """ :obj:`RemoveNodesFromCLB` retries if the CLB is temporarily locked, or if the request was rate-limited, or if there was an API error and the error is unknown but not a 4xx. """ non_terminals = (CLBImmutableError(lb_id=u"12345"), CLBRateLimitError(lb_id=u"12345"), APIError(code=500, body="oops!"), TypeError("You did something wrong in your code.")) eff = RemoveNodesFromCLB(lb_id='12345', node_ids=pset(['1', '2'])).as_effect() for exc in non_terminals: seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))]) with seq.consume(): self.assertEquals( sync_perform(seq, eff), (StepResult.RETRY, [ErrorReason.Exception( matches(ContainsAll([type(exc), exc])))]))
def test_add_nodes_to_clb_terminal_failures(self): """ :obj:`AddNodesToCLB` fails if the CLB is not found or deleted, or if there is any other 4xx error, then the error is propagated up and the result is a failure. """ terminals = (CLBNotFoundError(lb_id=u"12345"), CLBDeletedError(lb_id=u"12345"), NoSuchCLBError(lb_id=u"12345"), CLBNodeLimitError(lb_id=u"12345", node_limit=25), APIError(code=403, body="You're out of luck."), APIError(code=422, body="Oh look another 422.")) eff = self._add_one_node_to_clb() for exc in terminals: seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))]) with seq.consume(): self.assertEquals( sync_perform(seq, eff), (StepResult.FAILURE, [ErrorReason.Exception( matches(ContainsAll([type(exc), exc])))]))
def test_add_event_only_retries_5_times_on_non_4xx_api_errors(self): """ Attempting to add an event is only retried up to a maximum of 5 times, and only if it's not an 4XX APIError. """ responses = [ lambda _: raise_(Exception("oh noes!")), lambda _: raise_(ResponseFailed(Failure(Exception(":(")))), lambda _: raise_(APIError(code=100, body="<some xml>")), lambda _: raise_(APIError(code=202, body="<some xml>")), lambda _: raise_(APIError(code=301, body="<some xml>")), lambda _: raise_(APIError(code=501, body="<some xml>")), ] with self.assertRaises(APIError) as cm: self._perform_add_event(responses) self.assertEqual(cm.exception.code, 501)
def test_add_event_bails_on_4xx_api_errors(self): """ If CF returns a 4xx error, adding an event is not retried. """ response = [lambda _: raise_(APIError(code=409, body="<some xml>"))] self.assertRaises(APIError, self._perform_add_event, response)
def ba_raiser(self, *errors): return lambda i: raise_(rcv3.BulkErrors(errors))
def conste(e): """ Like ``const`` but takes and exception and returns function that raises the exception """ return lambda i: raise_(e)