def test_retry_sequence_fails_if_mismatch_sequence(self): """ Fail if the wrong number of performers are given. """ r = Retry( effect=Effect(1), should_retry=ShouldDelayAndRetry( can_retry=retry_times(5), next_interval=repeating_interval(10))) seq = [ retry_sequence(r, [lambda _: raise_(Exception()), lambda _: raise_(Exception())]) ] self.assertRaises(AssertionError, perform_sequence, seq, Effect(r))
def test_set_metadata_item(self): """ :obj:`SetMetadataItemOnServer.as_effect` produces a request for setting a metadata item on a particular server. It succeeds if successful, but does not fail for any errors. """ server_id = u'abc123' meta = SetMetadataItemOnServer(server_id=server_id, key='metadata_key', value='teapot') eff = meta.as_effect() seq = [ (eff.intent, lambda i: (StubResponse(202, {}), {})), (Log(ANY, ANY), lambda _: None) ] self.assertEqual( perform_sequence(seq, eff), (StepResult.SUCCESS, [])) exceptions = (NoSuchServerError("msg", server_id=server_id), ServerMetadataOverLimitError("msg", server_id=server_id), NovaRateLimitError("msg"), APIError(code=500, body="", headers={})) for exception in exceptions: self.assertRaises( type(exception), perform_sequence, [(eff.intent, lambda i: raise_(exception))], eff)
def test_retry_sequence_retries_without_delays(self): """ Perform the wrapped effect with the performers given, without any delay even if the original intent had a delay. """ r = Retry( effect=Effect(1), should_retry=ShouldDelayAndRetry( can_retry=retry_times(5), next_interval=repeating_interval(10))) seq = [ retry_sequence(r, [lambda _: raise_(Exception()), lambda _: raise_(Exception()), lambda _: "yay done"]) ] self.assertEqual(perform_sequence(seq, Effect(r)), "yay done")
def test_buckets_acquired_errors(self): """ Errors raised from performing the converge_all_groups effect are logged, and None is the ultimate result. """ def converge_all_groups(currently_converging, recent, _my_buckets, all_buckets, divergent_flags, build_timeout, interval): return Effect('converge-all') bound_sequence = [ (GetChildren(CONVERGENCE_DIRTY_DIR), lambda i: ['flag1', 'flag2']), ('converge-all', lambda i: raise_(RuntimeError('foo'))), (LogErr( CheckFailureValue(RuntimeError('foo')), 'converge-all-groups-error', {}), noop) ] sequence = self._log_sequence(bound_sequence) # relying on the side-effect of setting up self.fake_partitioner self._converger(converge_all_groups, dispatcher=sequence) with sequence.consume(): result, = self.fake_partitioner.got_buckets([0]) self.assertEqual(self.successResultOf(result), None)
def test_ignores_errors(self): """ Errors are not logged and are propogated """ seq = [("internal", lambda i: raise_(ValueError("oops")))] self.assertRaises(ValueError, perform_sequence, seq, msg_with_time("mt", Effect("internal")), self.disp) self.assertFalse(self.log.msg.called)
def test_ignores_errors(self): """ Errors are not logged and are propogated """ seq = [("internal", lambda i: raise_(ValueError("oops")))] self.assertRaises( ValueError, perform_sequence, seq, msg_with_time("mt", Effect("internal")), self.disp) self.assertFalse(self.log.msg.called)
def test_change_clb_node_terminal_errors(self): """Some errors during :obj:`ChangeCLBNode` make convergence fail.""" eff = self._change_node_eff() terminal = (NoSuchCLBNodeError(lb_id=u'abc123', node_id=u'node1'), CLBNotFoundError(lb_id=u'abc123'), APIError(code=400, body="", headers={})) for exception in terminal: self.assertEqual( perform_sequence([(eff.intent, lambda i: raise_(exception))], eff), (StepResult.FAILURE, [ANY]))
def test_change_clb_node_nonterminal_errors(self): """Some errors during :obj:`ChangeCLBNode` make convergence retry.""" eff = self._change_node_eff() nonterminal = (APIError(code=500, body="", headers={}), CLBNotActiveError(lb_id=u'abc123'), CLBRateLimitError(lb_id=u'abc123')) for exception in nonterminal: self.assertEqual( perform_sequence([(eff.intent, lambda i: raise_(exception))], eff), (StepResult.RETRY, ANY))
def test_failure(self): """ If setting divergent flag errors, then error is logged and raised """ seq = [ (CreateOrSet(path="/groups/divergent/t_g", content="dirty"), lambda i: raise_(ValueError("oops"))), (LogErr(CheckFailureValue(ValueError("oops")), "mark-dirty-failure", {}), noop) ] self.assertRaises( ValueError, perform_sequence, seq, trigger_convergence("t", "g"))
def test_can_have_a_different_should_retry_function(self): """ The ``should_retry`` function does not have to be a :obj:`ShouldDelayAndRetry`. """ expected = Retry(effect=Effect(1), should_retry=ANY) actual = Retry(effect=Effect(1), should_retry=lambda _: False) seq = [ retry_sequence(expected, [lambda _: raise_(Exception())]) ] self.assertRaises(Exception, perform_sequence, seq, Effect(actual))
def test_delete_node_other_error(self): """When marking clean raises arbitrary errors, an error is logged.""" sequence = [ (('ec', self.tenant_id, self.group_id, 3600), lambda i: (StepResult.SUCCESS, ScalingGroupStatus.ACTIVE)), (DeleteNode(path='/groups/divergent/tenant-id_g1', version=self.version), lambda i: raise_(ZeroDivisionError())), (LogErr(CheckFailureValue(ZeroDivisionError()), 'mark-clean-failure', dict(path='/groups/divergent/tenant-id_g1', dirty_version=self.version)), noop) ] self._verify_sequence(sequence)
def test_error_validating_observer(self): """ The observer returned replaces event with error if it fails to type check """ wrapper = SpecificationObserverWrapper( self.observer, lambda e: raise_(ValueError('hm'))) wrapper({'message': ("something-bad",), 'a': 'b'}) self.assertEqual( self.e, [{'original_event': {'message': ("something-bad",), 'a': 'b'}, 'isError': True, 'failure': CheckFailureValue(ValueError('hm')), 'why': 'Error validating event', 'message': ()}])
def test_other_errors(self): """ Any error other than `BulkErrors` results in RETRY """ non_terminals = (ValueError("internal"), APIError(code=500, body="why?"), APIError(code=503, body="bad service")) eff = self.step.as_effect() for exc in non_terminals: seq = [(("bd", self.pairs), lambda i: raise_(exc))] self.assertEqual( perform_sequence(seq, eff), (StepResult.RETRY, [ ErrorReason.Exception((type(exc), exc, ANY))]) )
def test_failure(self): """ Returns FAILURE if rcv3.bulk_delete raises BulkErrors """ terminals = (rcv3.BulkErrors([rcv3.LBInactive("l1")]), APIError(code=403, body="You're out of luck."), APIError(code=422, body="Oh look another 422.")) eff = self.step.as_effect() for exc in terminals: seq = [(("bd", self.pairs), lambda i: raise_(exc))] self.assertEqual( perform_sequence(seq, eff), (StepResult.FAILURE, [ ErrorReason.Exception((type(exc), exc, ANY))]) )
def test_delete_node_not_found(self): """ When DeleteNode raises a NoNodeError, a message is logged and nothing else is cleaned up. """ sequence = [ (('ec', self.tenant_id, self.group_id, 3600), lambda i: (StepResult.SUCCESS, ScalingGroupStatus.ACTIVE)), (DeleteNode(path='/groups/divergent/tenant-id_g1', version=self.version), lambda i: raise_(NoNodeError())), (Log('mark-clean-not-found', dict(path='/groups/divergent/tenant-id_g1', dirty_version=self.version)), noop) ] self._verify_sequence(sequence)
def test_remove_nodes_from_clb_success_failures(self): """ :obj:`AddNodesToCLB` succeeds if the CLB is not in existence (has been deleted or is not found). """ successes = [CLBNotFoundError(lb_id=u'12345'), CLBDeletedError(lb_id=u'12345'), NoSuchCLBError(lb_id=u'12345')] eff = RemoveNodesFromCLB(lb_id='12345', node_ids=pset(['1', '2'])).as_effect() for exc in successes: seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))]) with seq.consume(): self.assertEquals(sync_perform(seq, eff), (StepResult.SUCCESS, []))
def test_do_not_have_to_expect_an_exact_can_retry(self): """ The expected retry intent does not actually have to specify the exact ``can_retry`` function, since it might just be a lambda, which is hard to compare or hash. """ expected = Retry(effect=Effect(1), should_retry=ANY) actual = Retry(effect=Effect(1), should_retry=ShouldDelayAndRetry( can_retry=lambda _: False, next_interval=repeating_interval(10))) seq = [ retry_sequence(expected, [lambda _: raise_(Exception())]) ] self.assertRaises(Exception, perform_sequence, seq, Effect(actual))
def test_remove_nodes_from_clb_retry(self): """ :obj:`RemoveNodesFromCLB`, on receiving a 400, parses out the nodes that are no longer on the load balancer, and retries the bulk delete with those nodes removed. TODO: this has been left in as a regression test - this can probably be removed the next time it's touched, as this functionality happens in cloud_client now and there is a similar test there. """ lb_id = "12345" node_ids = [str(i) for i in range(5)] error_body = { "validationErrors": { "messages": [ "Node ids 1,2,3 are not a part of your loadbalancer" ] }, "message": "Validation Failure", "code": 400, "details": "The object is not valid" } expected_req = service_request( ServiceType.CLOUD_LOAD_BALANCERS, 'DELETE', 'loadbalancers/12345/nodes', params={'id': transform_eq(sorted, node_ids)}, success_pred=ANY, json_response=True).intent expected_req2 = service_request( ServiceType.CLOUD_LOAD_BALANCERS, 'DELETE', 'loadbalancers/12345/nodes', params={'id': transform_eq(sorted, ['0', '4'])}, success_pred=ANY, json_response=True).intent step = RemoveNodesFromCLB(lb_id=lb_id, node_ids=pset(node_ids)) seq = [ (expected_req, lambda i: raise_(APIError(400, json.dumps(error_body)))), (expected_req2, lambda i: stub_pure_response('', 202)), ] r = perform_sequence(seq, step.as_effect()) self.assertEqual(r, (StepResult.SUCCESS, []))
def test_delete_node_version_mismatch(self): """ When the version of the dirty flag changes during a call to converge_one_group, and DeleteNode raises a BadVersionError, the error is logged and nothing else is cleaned up. """ sequence = [ (('ec', self.tenant_id, self.group_id, 3600), lambda i: (StepResult.SUCCESS, ScalingGroupStatus.ACTIVE)), (DeleteNode(path='/groups/divergent/tenant-id_g1', version=self.version), lambda i: raise_(BadVersionError())), (Log('mark-clean-skipped', dict(path='/groups/divergent/tenant-id_g1', dirty_version=self.version)), noop) ] self._verify_sequence(sequence)
def test_no_scaling_group(self): """ When the scaling group disappears, a fatal error is logged and the dirty flag is cleaned up. """ expected_error = NoSuchScalingGroupError(self.tenant_id, self.group_id) sequence = [ (('ec', self.tenant_id, self.group_id, 3600), lambda i: raise_(expected_error)), (LogErr(CheckFailureValue(expected_error), 'converge-fatal-error', {}), noop), (DeleteNode(path='/groups/divergent/tenant-id_g1', version=self.version), noop), (Log('mark-clean-success', {}), noop) ] self._verify_sequence(sequence)
def test_error_writing(self): """ Logs and ignores error writing to the file """ seq = [ (GetAllGroups(), const(self.groups)), (ReadFileLines("file"), const(["2", "0.0"])), (Func(datetime.utcnow), const(datetime(1970, 1, 2))), (WriteFileLines("file", [7, 86400.0]), lambda i: raise_(IOError("bad"))), (LogErr(mock.ANY, "error updating number of tenants", {}), noop) ] r = perform_sequence(seq, get_todays_scaling_groups(["t1"], "file")) self.assertEqual( r, keyfilter(lambda k: k in ["t{}".format(i) for i in range(1, 9)], self.groups))
def test_remove_nodes_from_clb_terminal_failures(self): """ :obj:`AddNodesToCLB` fails if there are any 4xx errors, then the error is propagated up and the result is a failure. """ terminals = (APIError(code=403, body="You're out of luck."), APIError(code=422, body="Oh look another 422.")) eff = RemoveNodesFromCLB(lb_id='12345', node_ids=pset(['1', '2'])).as_effect() for exc in terminals: seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))]) with seq.consume(): self.assertEquals( sync_perform(seq, eff), (StepResult.FAILURE, [ErrorReason.Exception( matches(ContainsAll([type(exc), exc])))]))
def test_no_last_info(self): """ Returns first 5 non-convergence tenants if could not fetch last info from file """ seq = [ (GetAllGroups(), const(self.groups)), (ReadFileLines("file"), lambda i: raise_(IOError("e"))), (LogErr(mock.ANY, "error reading previous number of tenants", {}), noop), (Func(datetime.utcnow), const(datetime(1970, 1, 2))), (WriteFileLines("file", [5, 86400.0]), noop) ] r = perform_sequence(seq, get_todays_scaling_groups(["t1"], "file")) self.assertEqual( r, keyfilter(lambda k: k in ["t{}".format(i) for i in range(1, 7)], self.groups))
def test_error_validating_observer(self): """ The observer returned replaces event with error if it fails to type check """ wrapper = SpecificationObserverWrapper(self.observer, lambda e: raise_(ValueError("hm"))) wrapper({"message": ("something-bad",), "a": "b"}) self.assertEqual( self.e, [ { "original_event": {"message": ("something-bad",), "a": "b"}, "isError": True, "failure": CheckFailureValue(ValueError("hm")), "why": "Error validating event", "message": (), } ], )
def test_error_validating_observer(self): """ The observer returned replaces event with error if it fails to type check """ wrapper = SpecificationObserverWrapper( self.observer, lambda e: raise_(ValueError('hm'))) wrapper({'message': ("something-bad", ), 'a': 'b'}) self.assertEqual(self.e, [{ 'original_event': { 'message': ("something-bad", ), 'a': 'b' }, 'isError': True, 'failure': CheckFailureValue(ValueError('hm')), 'why': 'Error validating event', 'message': () }])
def test_remove_nodes_from_clb_non_terminal_failures_to_retry(self): """ :obj:`RemoveNodesFromCLB` retries if the CLB is temporarily locked, or if the request was rate-limited, or if there was an API error and the error is unknown but not a 4xx. """ non_terminals = (CLBImmutableError(lb_id=u"12345"), CLBRateLimitError(lb_id=u"12345"), APIError(code=500, body="oops!"), TypeError("You did something wrong in your code.")) eff = RemoveNodesFromCLB(lb_id='12345', node_ids=pset(['1', '2'])).as_effect() for exc in non_terminals: seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))]) with seq.consume(): self.assertEquals( sync_perform(seq, eff), (StepResult.RETRY, [ErrorReason.Exception( matches(ContainsAll([type(exc), exc])))]))
def test_add_nodes_to_clb_terminal_failures(self): """ :obj:`AddNodesToCLB` fails if the CLB is not found or deleted, or if there is any other 4xx error, then the error is propagated up and the result is a failure. """ terminals = (CLBNotFoundError(lb_id=u"12345"), CLBDeletedError(lb_id=u"12345"), NoSuchCLBError(lb_id=u"12345"), CLBNodeLimitError(lb_id=u"12345", node_limit=25), APIError(code=403, body="You're out of luck."), APIError(code=422, body="Oh look another 422.")) eff = self._add_one_node_to_clb() for exc in terminals: seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))]) with seq.consume(): self.assertEquals( sync_perform(seq, eff), (StepResult.FAILURE, [ErrorReason.Exception( matches(ContainsAll([type(exc), exc])))]))
def test_add_event_only_retries_5_times_on_non_4xx_api_errors(self): """ Attempting to add an event is only retried up to a maximum of 5 times, and only if it's not an 4XX APIError. """ responses = [ lambda _: raise_(Exception("oh noes!")), lambda _: raise_(ResponseFailed(Failure(Exception(":(")))), lambda _: raise_(APIError(code=100, body="<some xml>")), lambda _: raise_(APIError(code=202, body="<some xml>")), lambda _: raise_(APIError(code=301, body="<some xml>")), lambda _: raise_(APIError(code=501, body="<some xml>")), ] with self.assertRaises(APIError) as cm: self._perform_add_event(responses) self.assertEqual(cm.exception.code, 501)
def test_first_error_extraction(self): """ If the GetScalingGroupInfo effect fails, its exception is raised directly, without the FirstError wrapper. """ # Perform the GetScalingGroupInfo by raising an exception sequence = [ (Log("begin-convergence", {}), noop), (Func(datetime.utcnow), lambda i: self.now), (MsgWithTime("gather-convergence-data", mock.ANY), nested_sequence([ parallel_sequence([ [(self.gsgi, lambda i: raise_(RuntimeError('foo')))], [("anything", noop)] ]) ])) ] # And make sure that exception isn't wrapped in FirstError. e = self.assertRaises( RuntimeError, perform_sequence, sequence, self._invoke(), test_dispatcher()) self.assertEqual(str(e), 'foo')
def test_unexpected_errors(self): """ Unexpected exceptions log a non-fatal error and don't clean up the dirty flag. """ converging = Reference(pset()) recent = Reference(pmap()) expected_error = RuntimeError('oh no!') sequence = [ (ReadReference(converging), lambda i: pset()), add_to_currently(converging, self.group_id), (('ec', self.tenant_id, self.group_id, 3600), lambda i: raise_(expected_error)), (Func(time.time), lambda i: 100), add_to_recently(recent, self.group_id, 100), (ModifyReference(converging, match_func(pset([self.group_id]), pset())), noop), (LogErr(CheckFailureValue(expected_error), 'converge-non-fatal-error', {}), noop), ] self._verify_sequence(sequence, converging=converging, recent=recent, allow_refs=False)
def ba_raiser(self, *errors): return lambda i: raise_(rcv3.BulkErrors(errors))
def test_add_event_bails_on_4xx_api_errors(self): """ If CF returns a 4xx error, adding an event is not retried. """ response = [lambda _: raise_(APIError(code=409, body="<some xml>"))] self.assertRaises(APIError, self._perform_add_event, response)