def test_fail_max_thread_safety(self): """CircuitBreaker: it should not allow more failed calls than 'fail_max' setting. Note that with Redis, where we have separate systems incrementing the counter, we can get concurrent updates such that the counter is greater than the 'fail_max' by the number of systems. To prevent this, we'd need to take out a lock amongst all systems before trying the call. """ breaker = CircuitBreaker() @breaker def err(): raise DummyException() def trigger_error(): for i in range(2000): try: err() except DummyException: pass except CircuitBreakerError: pass class SleepListener(CircuitBreakerListener): def before_call(self, breaker, func, *args, **kwargs): sleep(0.00005) breaker.add_listener(SleepListener()) num_threads = 3 self._start_threads(trigger_error, num_threads) self.assertTrue(breaker.fail_counter < breaker.fail_max + num_threads)
def test_fail_thread_safety(self): """It should compute a failed call atomically to avoid race conditions.""" breaker = CircuitBreaker(fail_max=3000, timeout_duration=timedelta(seconds=1), state_storage=self.storage) @breaker def err(): raise DummyException() def trigger_error(): for n in range(500): try: err() except DummyException: pass def _inc_counter(): sleep(0.00005) breaker._state_storage.increment_counter() breaker._inc_counter = _inc_counter self._start_threads(trigger_error, 3) self.assertEqual(1500, breaker.fail_counter)
def test_failed_call_after_timeout(storage, delta): """It should half-open the circuit after timeout and close immediately on fail.""" breaker = CircuitBreaker(fail_max=3, timeout_duration=delta, state_storage=storage) with raises(DummyException): breaker.call(func_exception) with raises(DummyException): breaker.call(func_exception) assert CircuitBreakerState.CLOSED == breaker.current_state # Circuit should open with raises(CircuitBreakerError): breaker.call(func_exception) assert 3 == breaker.fail_counter sleep(delta.total_seconds() * 2) # Circuit should open again with raises(CircuitBreakerError): breaker.call(func_exception) assert 4 == breaker.fail_counter assert CircuitBreakerState.OPEN == breaker.current_state
def test_successful_call(storage): """It should keep the circuit closed after a successful call.""" breaker = CircuitBreaker(state_storage=storage) assert breaker.call(func_succeed) assert 0 == breaker.fail_counter assert CircuitBreakerState.CLOSED == breaker.current_state
def test_excluded_exceptions(): """CircuitBreaker: it should ignore specific exceptions. """ breaker = CircuitBreaker(exclude=[LookupError]) def err_1(): raise DummyException() def err_2(): raise LookupError() def err_3(): raise KeyError() with raises(DummyException): breaker.call(err_1) assert 1 == breaker.fail_counter # LookupError is not considered a system error with raises(LookupError): breaker.call(err_2) assert 0 == breaker.fail_counter with raises(DummyException): breaker.call(err_1) assert 1 == breaker.fail_counter # Should consider subclasses as well (KeyError is a subclass of # LookupError) with raises(KeyError): breaker.call(err_3) assert 0 == breaker.fail_counter
def test_add_listeners(): """ It should allow the user to add listeners at a later time.""" breaker = CircuitBreaker() first, second = CircuitBreakerListener(), CircuitBreakerListener() breaker.add_listeners(first, second) assert (first, second) == breaker.listeners
def test_call_events(storage): """It should call the appropriate functions on every successful/failed call. """ class Listener(CircuitBreakerListener): def __init__(self): self.out = [] def before_call(self, breaker, func, *args, **kwargs): assert breaker self.out.append("CALL") def success(self, breaker): assert breaker self.out.append("SUCCESS") def failure(self, breaker, exception): assert breaker assert isinstance(exception, DummyException) self.out.append("FAILURE") listener = Listener() breaker = CircuitBreaker(listeners=(listener, ), state_storage=storage) assert breaker.call(func_succeed) with raises(DummyException): breaker.call(func_exception) assert ["CALL", "SUCCESS", "CALL", "FAILURE"] == listener.out
def test_success_thread_safety(self): """It should compute a successful call atomically to avoid race conditions.""" breaker = CircuitBreaker(fail_max=3000, timeout_duration=timedelta(seconds=1), state_storage=self.storage) @breaker def suc(): return True def trigger_success(): for n in range(500): suc() class SuccessListener(CircuitBreakerListener): def success(self, breaker): c = 0 if hasattr(breaker, '_success_counter'): c = breaker._success_counter sleep(0.00005) breaker._success_counter = c + 1 breaker.add_listener(SuccessListener()) self._start_threads(trigger_success, 3) self.assertEqual(1500, breaker._success_counter)
async def setUp(self): self.monitor = Monitor("monitor@localhost", "passw0rd", "data0") self.engine = create_engine("sqlite:///database.sql") self.breaker = CircuitBreaker() self.cache_to_save = 10 self.connection = Connection(self.engine, self.cache_to_save, 100000, self.breaker) self.analyser_predictor = HelperAnalyserPredictor(self.connection) self.sink = HelperSink(self.breaker) self.sink.external.reset_mock() self.analyser = Analyser("analyser@localhost", "passw0rd", self.analyser_predictor, self.connection, ["monitor@localhost"]) self.executor = Executor("executor@localhost", "passw0rd", self.sink) self.planner_predictor = HelperPlannerPredictor(self.connection) self.planner = Planner("planner@localhost", "passw0rd", self.planner_predictor, ["executor@localhost"]) self.monitor.start() self.executor.start() await asyncio.sleep(2) self.analyser.start() self.planner.start() await asyncio.sleep(2)
def test_call_with_args(): """ It should be able to invoke functions with args.""" def func(arg1, arg2): return arg1, arg2 breaker = CircuitBreaker() assert (42, 'abc') == breaker.call(func, 42, 'abc')
def test_fail_max_setter(): """CircuitBreaker: it should allow the user to set a new value for 'fail_max'. """ breaker = CircuitBreaker() assert 5 == breaker.fail_max breaker.fail_max = 10 assert 10 == breaker.fail_max
def test_name(): """It should allow an optional name to be set and retrieved.""" name = "test_breaker" breaker = CircuitBreaker(name=name) assert breaker.name == name name = "breaker_test" breaker.name = name assert breaker.name == name
def test_reset_timeout_setter(): """CircuitBreaker: it should allow the user to set a new value for 'reset_timeout'. """ breaker = CircuitBreaker() assert timedelta(seconds=60) == breaker.timeout_duration breaker.timeout_duration = timedelta(seconds=30) assert timedelta(seconds=30) == breaker.timeout_duration
def test_call_with_kwargs(): """ It should be able to invoke functions with kwargs.""" def func(**kwargs): return kwargs breaker = CircuitBreaker() kwargs = {'a': 1, 'b': 2} assert kwargs == breaker.call(func, **kwargs)
def test_one_failed_call(storage): """It should keep the circuit closed after a few failures.""" breaker = CircuitBreaker(state_storage=storage) with raises(DummyException): breaker.call(func_exception) assert 1 == breaker.fail_counter assert CircuitBreakerState.CLOSED == breaker.current_state
def test_one_successful_call_after_failed_call(storage): """It should keep the circuit closed after few mixed outcomes.""" breaker = CircuitBreaker(state_storage=storage) with raises(DummyException): breaker.call(func_exception) assert 1 == breaker.fail_counter assert breaker.call(func_succeed) assert 0 == breaker.fail_counter assert CircuitBreakerState.CLOSED == breaker.current_state
async def setUp(self): self.monitor = Monitor("monitor@localhost", "passw0rd", "data0") self.engine = create_engine("sqlite:///database.sql") self.breaker = CircuitBreaker() self.connection = Connection(self.engine, 10, 10000, self.breaker) self.predictor = HelperAnalyserPredictor(self.connection) self.analyser = Analyser("analyser@localhost", "passw0rd", self.predictor, self.connection, ["monitor@localhost"]) self.monitor.start() await asyncio.sleep(2) self.analyser.start() await asyncio.sleep(1)
def test_successful_call_when_half_open(storage): """It should close the circuit when a call succeeds in half-open state.""" breaker = CircuitBreaker(state_storage=storage) breaker.half_open() assert 0 == breaker.fail_counter assert CircuitBreakerState.HALF_OPEN == breaker.current_state # Circuit should open assert breaker.call(func_succeed) assert 0 == breaker.fail_counter assert CircuitBreakerState.CLOSED == breaker.current_state
async def setUp(self): if not path.exists("test/resources/database.dump"): raise IOError("database not exists") self.engine = create_engine("sqlite:///test/resources/database.dump") self.breaker = CircuitBreaker() self.connection = Connection(self.engine, 10, 10000, self.breaker) self.sink = HelperSink(self.breaker) self.sink.external.reset_mock() self.executor = Executor("executor@localhost", "passw0rd", self.sink) self.planner_predictor = HelperPlannerPredictor(self.connection) self.cache_number = 3 self.planner = Planner("planner@localhost", "passw0rd", self.planner_predictor, ["executor@localhost"], self.cache_number)
async def test_failed_call_when_half_open(storage): """It should open the circuit when a call fails in half-open state.""" breaker = CircuitBreaker(state_storage=storage) breaker.half_open() assert 0 == breaker.fail_counter assert CircuitBreakerState.HALF_OPEN == breaker.current_state with raises(CircuitBreakerError): await breaker.call_async(func_exception_async) assert 1 == breaker.fail_counter assert CircuitBreakerState.OPEN == breaker.current_state
def test_transition_events(storage): """It should call the appropriate functions on every state transition.""" class Listener(CircuitBreakerListener): def __init__(self): self.out = [] def state_change(self, breaker, old, new): assert breaker self.out.append((old.state, new.state)) listener = Listener() breaker = CircuitBreaker(listeners=(listener, ), state_storage=storage) assert CircuitBreakerState.CLOSED == breaker.current_state breaker.open() assert CircuitBreakerState.OPEN == breaker.current_state breaker.half_open() assert CircuitBreakerState.HALF_OPEN == breaker.current_state breaker.close() assert CircuitBreakerState.CLOSED == breaker.current_state assert [(CircuitBreakerState.CLOSED, CircuitBreakerState.OPEN), (CircuitBreakerState.OPEN, CircuitBreakerState.HALF_OPEN), (CircuitBreakerState.HALF_OPEN, CircuitBreakerState.CLOSED) ] == listener.out
def test_default_state(): """It should get initial state from state_storage.""" for state in CircuitBreakerState: storage = CircuitMemoryStorage(state) breaker = CircuitBreaker(state_storage=storage) assert isinstance(breaker.state, state.value) assert breaker.state.state == state
def __init__(self, db_engine: Engine, bulk_size: int, bulk_time: Union[int, float], circuit_breaker: CircuitBreaker = CircuitBreaker()): """Connects with SQLAlchemy Engine to store and query data for concept drift datection. :param db_engine: SQLAlchemy Engine to use as backend :type db_engine: Engine :param bulk_size: Quantity of data that connection will wait to make bulk insert :type bulk_size: int :param bulk_time: Time in seconds between last insert and now. If bulk_size is not reached in bulk_time interval, then an insert was done :type bulk_time: Union[int, float] :param circuit_breaker: Circuit Breaker configuration to connect with Database, defaults to CircuitBreaker() :type circuit_breaker: CircuitBreaker, optional """ self._jid = None self._conn = db_engine self._bulk_size = bulk_size self._bulk_time = bulk_time self._last_insert_time = datetime.utcnow() self._bulk_df = pd.DataFrame() self.get_between = circuit_breaker(self.get_between) self._insert = circuit_breaker(self._insert) self._logger = getLogger("connection")
async def test_successful_after_timeout(storage, delta): """It should close the circuit when a call succeeds after timeout.""" breaker = CircuitBreaker(fail_max=3, timeout_duration=delta, state_storage=storage) func_succeed_async = func_succeed_counted_async() with raises(DummyException): await breaker.call_async(func_exception_async) with raises(DummyException): await breaker.call_async(func_exception_async) assert CircuitBreakerState.CLOSED == breaker.current_state # Circuit should open with raises(CircuitBreakerError): await breaker.call_async(func_exception_async) assert CircuitBreakerState.OPEN == breaker.current_state with raises(CircuitBreakerError): await breaker.call_async(func_succeed_async) assert 3 == breaker.fail_counter # Wait for timeout, at least a second since redis rounds to a second await sleep(delta.total_seconds() * 2) # Circuit should close again assert await breaker.call_async(func_succeed_async) assert 0 == breaker.fail_counter assert CircuitBreakerState.CLOSED == breaker.current_state assert 1 == func_succeed_async.call_count
def test_double_count(): """It should not trigger twice if you call CircuitBreaker#call on a decorated function.""" breaker = CircuitBreaker() @breaker def err(): """Docstring""" raise DummyException() assert 0 == breaker.fail_counter with raises(DummyException): breaker.call(err) assert 1 == breaker.fail_counter
async def test_decorator_async(): """ It should also be an async decorator. """ breaker = CircuitBreaker() @breaker async def suc(): """Docstring""" pass @breaker async def err(): """Docstring""" raise DummyException() assert 'Docstring' == suc.__doc__ assert 'Docstring' == err.__doc__ assert 'suc' == suc.__name__ assert 'err' == err.__name__ assert 0 == breaker.fail_counter with raises(DummyException): await err() assert 1 == breaker.fail_counter await suc() assert 0 == breaker.fail_counter
def test_generator(storage): """It should support generator values.""" breaker = CircuitBreaker(state_storage=storage) @breaker def func_yield_succeed(): """Docstring""" yield True @breaker def func_yield_exception(): """Docstring""" x = yield True raise DummyException(x) s = func_yield_succeed() e = func_yield_exception() next(e) with raises(DummyException): e.send(True) assert 1 == breaker.fail_counter assert next(s) with raises(StopIteration): next(s) assert 0 == breaker.fail_counter
class TestMAPEIntegration(TestCase): async def setUp(self): self.monitor = Monitor("monitor@localhost", "passw0rd", "data0") self.engine = create_engine("sqlite:///database.sql") self.breaker = CircuitBreaker() self.cache_to_save = 10 self.connection = Connection(self.engine, self.cache_to_save, 100000, self.breaker) self.analyser_predictor = HelperAnalyserPredictor(self.connection) self.sink = HelperSink(self.breaker) self.sink.external.reset_mock() self.analyser = Analyser("analyser@localhost", "passw0rd", self.analyser_predictor, self.connection, ["monitor@localhost"]) self.executor = Executor("executor@localhost", "passw0rd", self.sink) self.planner_predictor = HelperPlannerPredictor(self.connection) self.planner = Planner("planner@localhost", "passw0rd", self.planner_predictor, ["executor@localhost"]) self.monitor.start() self.executor.start() await asyncio.sleep(2) self.analyser.start() self.planner.start() await asyncio.sleep(2) def tearDown(self): self.monitor.stop() self.analyser.stop() self.planner.stop() self.executor.stop() self.breaker.close() os.unlink("database.sql") @patch("driftage.planner.behaviour.predict.datetime") async def test_should_execute_monitored_data(self, mock_dt): now = datetime.utcnow() mock_dt.utcnow.return_value = now for i in range(self.cache_to_save): self.monitor({"my data": i}) await asyncio.sleep(1) self.sink.external.assert_called_once_with({ 'timestamp': now.timestamp(), 'identifier': 'data0', 'predicted': True })
def test_new_with_custom_fail_max(): """It should support a custom maximum number of failures.""" breaker = CircuitBreaker(fail_max=10) assert 0 == breaker.fail_counter assert timedelta(seconds=60) == breaker.timeout_duration assert 10 == breaker.fail_max assert () == breaker.excluded_exceptions assert () == breaker.listeners assert 'memory' == breaker._state_storage.name
def test_state_opened_at_not_reset_during_creation(): for state in CircuitBreakerState: storage = CircuitMemoryStorage(state) now = datetime.now() storage.opened_at = now breaker = CircuitBreaker(state_storage=storage) assert breaker.state.state == state assert storage.opened_at == now