def test_fail_max_thread_safety(self):
"""CircuitBreaker: it should not allow more failed calls than 'fail_max'
setting. Note that with Redis, where we have separate systems
incrementing the counter, we can get concurrent updates such that the
counter is greater than the 'fail_max' by the number of systems. To
prevent this, we'd need to take out a lock amongst all systems before
trying the call.
"""
breaker = CircuitBreaker()
@breaker
def err():
raise DummyException()
def trigger_error():
for i in range(2000):
try:
err()
except DummyException:
pass
except CircuitBreakerError:
pass
class SleepListener(CircuitBreakerListener):
def before_call(self, breaker, func, *args, **kwargs):
sleep(0.00005)
breaker.add_listener(SleepListener())
num_threads = 3
self._start_threads(trigger_error, num_threads)
self.assertTrue(breaker.fail_counter < breaker.fail_max + num_threads)
def test_fail_thread_safety(self):
"""It should compute a failed call atomically to avoid race conditions."""
breaker = CircuitBreaker(fail_max=3000,
timeout_duration=timedelta(seconds=1),
state_storage=self.storage)
@breaker
def err():
raise DummyException()
def trigger_error():
for n in range(500):
try:
err()
except DummyException:
pass
def _inc_counter():
sleep(0.00005)
breaker._state_storage.increment_counter()
breaker._inc_counter = _inc_counter
self._start_threads(trigger_error, 3)
self.assertEqual(1500, breaker.fail_counter)
def test_failed_call_after_timeout(storage, delta):
"""It should half-open the circuit after timeout and close immediately on fail."""
breaker = CircuitBreaker(fail_max=3,
timeout_duration=delta,
state_storage=storage)
with raises(DummyException):
breaker.call(func_exception)
with raises(DummyException):
breaker.call(func_exception)
assert CircuitBreakerState.CLOSED == breaker.current_state
# Circuit should open
with raises(CircuitBreakerError):
breaker.call(func_exception)
assert 3 == breaker.fail_counter
sleep(delta.total_seconds() * 2)
# Circuit should open again
with raises(CircuitBreakerError):
breaker.call(func_exception)
assert 4 == breaker.fail_counter
assert CircuitBreakerState.OPEN == breaker.current_state
def test_successful_call(storage):
"""It should keep the circuit closed after a successful call."""
breaker = CircuitBreaker(state_storage=storage)
assert breaker.call(func_succeed)
assert 0 == breaker.fail_counter
assert CircuitBreakerState.CLOSED == breaker.current_state
def test_excluded_exceptions():
"""CircuitBreaker: it should ignore specific exceptions.
"""
breaker = CircuitBreaker(exclude=[LookupError])
def err_1():
raise DummyException()
def err_2():
raise LookupError()
def err_3():
raise KeyError()
with raises(DummyException):
breaker.call(err_1)
assert 1 == breaker.fail_counter
# LookupError is not considered a system error
with raises(LookupError):
breaker.call(err_2)
assert 0 == breaker.fail_counter
with raises(DummyException):
breaker.call(err_1)
assert 1 == breaker.fail_counter
# Should consider subclasses as well (KeyError is a subclass of
# LookupError)
with raises(KeyError):
breaker.call(err_3)
assert 0 == breaker.fail_counter
def test_add_listeners():
""" It should allow the user to add listeners at a later time."""
breaker = CircuitBreaker()
first, second = CircuitBreakerListener(), CircuitBreakerListener()
breaker.add_listeners(first, second)
assert (first, second) == breaker.listeners
def test_call_events(storage):
"""It should call the appropriate functions on every successful/failed call.
"""
class Listener(CircuitBreakerListener):
def __init__(self):
self.out = []
def before_call(self, breaker, func, *args, **kwargs):
assert breaker
self.out.append("CALL")
def success(self, breaker):
assert breaker
self.out.append("SUCCESS")
def failure(self, breaker, exception):
assert breaker
assert isinstance(exception, DummyException)
self.out.append("FAILURE")
listener = Listener()
breaker = CircuitBreaker(listeners=(listener, ), state_storage=storage)
assert breaker.call(func_succeed)
with raises(DummyException):
breaker.call(func_exception)
assert ["CALL", "SUCCESS", "CALL", "FAILURE"] == listener.out
def test_success_thread_safety(self):
"""It should compute a successful call atomically to avoid race conditions."""
breaker = CircuitBreaker(fail_max=3000,
timeout_duration=timedelta(seconds=1),
state_storage=self.storage)
@breaker
def suc():
return True
def trigger_success():
for n in range(500):
suc()
class SuccessListener(CircuitBreakerListener):
def success(self, breaker):
c = 0
if hasattr(breaker, '_success_counter'):
c = breaker._success_counter
sleep(0.00005)
breaker._success_counter = c + 1
breaker.add_listener(SuccessListener())
self._start_threads(trigger_success, 3)
self.assertEqual(1500, breaker._success_counter)
async def setUp(self):
self.monitor = Monitor("monitor@localhost", "passw0rd", "data0")
self.engine = create_engine("sqlite:///database.sql")
self.breaker = CircuitBreaker()
self.cache_to_save = 10
self.connection = Connection(self.engine, self.cache_to_save, 100000,
self.breaker)
self.analyser_predictor = HelperAnalyserPredictor(self.connection)
self.sink = HelperSink(self.breaker)
self.sink.external.reset_mock()
self.analyser = Analyser("analyser@localhost", "passw0rd",
self.analyser_predictor, self.connection,
["monitor@localhost"])
self.executor = Executor("executor@localhost", "passw0rd", self.sink)
self.planner_predictor = HelperPlannerPredictor(self.connection)
self.planner = Planner("planner@localhost", "passw0rd",
self.planner_predictor, ["executor@localhost"])
self.monitor.start()
self.executor.start()
await asyncio.sleep(2)
self.analyser.start()
self.planner.start()
await asyncio.sleep(2)
def test_call_with_args():
""" It should be able to invoke functions with args."""
def func(arg1, arg2):
return arg1, arg2
breaker = CircuitBreaker()
assert (42, 'abc') == breaker.call(func, 42, 'abc')
def test_fail_max_setter():
"""CircuitBreaker: it should allow the user to set a new value for
'fail_max'.
"""
breaker = CircuitBreaker()
assert 5 == breaker.fail_max
breaker.fail_max = 10
assert 10 == breaker.fail_max
def test_name():
"""It should allow an optional name to be set and retrieved."""
name = "test_breaker"
breaker = CircuitBreaker(name=name)
assert breaker.name == name
name = "breaker_test"
breaker.name = name
assert breaker.name == name
def test_reset_timeout_setter():
"""CircuitBreaker: it should allow the user to set a new value for
'reset_timeout'.
"""
breaker = CircuitBreaker()
assert timedelta(seconds=60) == breaker.timeout_duration
breaker.timeout_duration = timedelta(seconds=30)
assert timedelta(seconds=30) == breaker.timeout_duration
def test_call_with_kwargs():
""" It should be able to invoke functions with kwargs."""
def func(**kwargs):
return kwargs
breaker = CircuitBreaker()
kwargs = {'a': 1, 'b': 2}
assert kwargs == breaker.call(func, **kwargs)
def test_one_failed_call(storage):
"""It should keep the circuit closed after a few failures."""
breaker = CircuitBreaker(state_storage=storage)
with raises(DummyException):
breaker.call(func_exception)
assert 1 == breaker.fail_counter
assert CircuitBreakerState.CLOSED == breaker.current_state
def test_one_successful_call_after_failed_call(storage):
"""It should keep the circuit closed after few mixed outcomes."""
breaker = CircuitBreaker(state_storage=storage)
with raises(DummyException):
breaker.call(func_exception)
assert 1 == breaker.fail_counter
assert breaker.call(func_succeed)
assert 0 == breaker.fail_counter
assert CircuitBreakerState.CLOSED == breaker.current_state
async def setUp(self):
self.monitor = Monitor("monitor@localhost", "passw0rd", "data0")
self.engine = create_engine("sqlite:///database.sql")
self.breaker = CircuitBreaker()
self.connection = Connection(self.engine, 10, 10000, self.breaker)
self.predictor = HelperAnalyserPredictor(self.connection)
self.analyser = Analyser("analyser@localhost", "passw0rd",
self.predictor, self.connection,
["monitor@localhost"])
self.monitor.start()
await asyncio.sleep(2)
self.analyser.start()
await asyncio.sleep(1)
def test_successful_call_when_half_open(storage):
"""It should close the circuit when a call succeeds in half-open state."""
breaker = CircuitBreaker(state_storage=storage)
breaker.half_open()
assert 0 == breaker.fail_counter
assert CircuitBreakerState.HALF_OPEN == breaker.current_state
# Circuit should open
assert breaker.call(func_succeed)
assert 0 == breaker.fail_counter
assert CircuitBreakerState.CLOSED == breaker.current_state
async def setUp(self):
if not path.exists("test/resources/database.dump"):
raise IOError("database not exists")
self.engine = create_engine("sqlite:///test/resources/database.dump")
self.breaker = CircuitBreaker()
self.connection = Connection(self.engine, 10, 10000, self.breaker)
self.sink = HelperSink(self.breaker)
self.sink.external.reset_mock()
self.executor = Executor("executor@localhost", "passw0rd", self.sink)
self.planner_predictor = HelperPlannerPredictor(self.connection)
self.cache_number = 3
self.planner = Planner("planner@localhost", "passw0rd",
self.planner_predictor, ["executor@localhost"],
self.cache_number)
async def test_failed_call_when_half_open(storage):
"""It should open the circuit when a call fails in half-open state."""
breaker = CircuitBreaker(state_storage=storage)
breaker.half_open()
assert 0 == breaker.fail_counter
assert CircuitBreakerState.HALF_OPEN == breaker.current_state
with raises(CircuitBreakerError):
await breaker.call_async(func_exception_async)
assert 1 == breaker.fail_counter
assert CircuitBreakerState.OPEN == breaker.current_state
def test_transition_events(storage):
"""It should call the appropriate functions on every state transition."""
class Listener(CircuitBreakerListener):
def __init__(self):
self.out = []
def state_change(self, breaker, old, new):
assert breaker
self.out.append((old.state, new.state))
listener = Listener()
breaker = CircuitBreaker(listeners=(listener, ), state_storage=storage)
assert CircuitBreakerState.CLOSED == breaker.current_state
breaker.open()
assert CircuitBreakerState.OPEN == breaker.current_state
breaker.half_open()
assert CircuitBreakerState.HALF_OPEN == breaker.current_state
breaker.close()
assert CircuitBreakerState.CLOSED == breaker.current_state
assert [(CircuitBreakerState.CLOSED, CircuitBreakerState.OPEN),
(CircuitBreakerState.OPEN, CircuitBreakerState.HALF_OPEN),
(CircuitBreakerState.HALF_OPEN, CircuitBreakerState.CLOSED)
] == listener.out
def test_default_state():
"""It should get initial state from state_storage."""
for state in CircuitBreakerState:
storage = CircuitMemoryStorage(state)
breaker = CircuitBreaker(state_storage=storage)
assert isinstance(breaker.state, state.value)
assert breaker.state.state == state
def __init__(self,
db_engine: Engine,
bulk_size: int,
bulk_time: Union[int, float],
circuit_breaker: CircuitBreaker = CircuitBreaker()):
"""Connects with SQLAlchemy Engine to store and query
data for concept drift datection.
:param db_engine: SQLAlchemy Engine to use as backend
:type db_engine: Engine
:param bulk_size: Quantity of data that connection will
wait to make bulk insert
:type bulk_size: int
:param bulk_time: Time in seconds between last insert and now.
If bulk_size is not reached in bulk_time interval,
then an insert was done
:type bulk_time: Union[int, float]
:param circuit_breaker: Circuit Breaker configuration to
connect with Database, defaults to CircuitBreaker()
:type circuit_breaker: CircuitBreaker, optional
"""
self._jid = None
self._conn = db_engine
self._bulk_size = bulk_size
self._bulk_time = bulk_time
self._last_insert_time = datetime.utcnow()
self._bulk_df = pd.DataFrame()
self.get_between = circuit_breaker(self.get_between)
self._insert = circuit_breaker(self._insert)
self._logger = getLogger("connection")
async def test_successful_after_timeout(storage, delta):
"""It should close the circuit when a call succeeds after timeout."""
breaker = CircuitBreaker(fail_max=3,
timeout_duration=delta,
state_storage=storage)
func_succeed_async = func_succeed_counted_async()
with raises(DummyException):
await breaker.call_async(func_exception_async)
with raises(DummyException):
await breaker.call_async(func_exception_async)
assert CircuitBreakerState.CLOSED == breaker.current_state
# Circuit should open
with raises(CircuitBreakerError):
await breaker.call_async(func_exception_async)
assert CircuitBreakerState.OPEN == breaker.current_state
with raises(CircuitBreakerError):
await breaker.call_async(func_succeed_async)
assert 3 == breaker.fail_counter
# Wait for timeout, at least a second since redis rounds to a second
await sleep(delta.total_seconds() * 2)
# Circuit should close again
assert await breaker.call_async(func_succeed_async)
assert 0 == breaker.fail_counter
assert CircuitBreakerState.CLOSED == breaker.current_state
assert 1 == func_succeed_async.call_count
def test_double_count():
"""It should not trigger twice if you call CircuitBreaker#call on a decorated function."""
breaker = CircuitBreaker()
@breaker
def err():
"""Docstring"""
raise DummyException()
assert 0 == breaker.fail_counter
with raises(DummyException):
breaker.call(err)
assert 1 == breaker.fail_counter
async def test_decorator_async():
"""
It should also be an async decorator.
"""
breaker = CircuitBreaker()
@breaker
async def suc():
"""Docstring"""
pass
@breaker
async def err():
"""Docstring"""
raise DummyException()
assert 'Docstring' == suc.__doc__
assert 'Docstring' == err.__doc__
assert 'suc' == suc.__name__
assert 'err' == err.__name__
assert 0 == breaker.fail_counter
with raises(DummyException):
await err()
assert 1 == breaker.fail_counter
await suc()
assert 0 == breaker.fail_counter
def test_generator(storage):
"""It should support generator values."""
breaker = CircuitBreaker(state_storage=storage)
@breaker
def func_yield_succeed():
"""Docstring"""
yield True
@breaker
def func_yield_exception():
"""Docstring"""
x = yield True
raise DummyException(x)
s = func_yield_succeed()
e = func_yield_exception()
next(e)
with raises(DummyException):
e.send(True)
assert 1 == breaker.fail_counter
assert next(s)
with raises(StopIteration):
next(s)
assert 0 == breaker.fail_counter
class TestMAPEIntegration(TestCase):
async def setUp(self):
self.monitor = Monitor("monitor@localhost", "passw0rd", "data0")
self.engine = create_engine("sqlite:///database.sql")
self.breaker = CircuitBreaker()
self.cache_to_save = 10
self.connection = Connection(self.engine, self.cache_to_save, 100000,
self.breaker)
self.analyser_predictor = HelperAnalyserPredictor(self.connection)
self.sink = HelperSink(self.breaker)
self.sink.external.reset_mock()
self.analyser = Analyser("analyser@localhost", "passw0rd",
self.analyser_predictor, self.connection,
["monitor@localhost"])
self.executor = Executor("executor@localhost", "passw0rd", self.sink)
self.planner_predictor = HelperPlannerPredictor(self.connection)
self.planner = Planner("planner@localhost", "passw0rd",
self.planner_predictor, ["executor@localhost"])
self.monitor.start()
self.executor.start()
await asyncio.sleep(2)
self.analyser.start()
self.planner.start()
await asyncio.sleep(2)
def tearDown(self):
self.monitor.stop()
self.analyser.stop()
self.planner.stop()
self.executor.stop()
self.breaker.close()
os.unlink("database.sql")
@patch("driftage.planner.behaviour.predict.datetime")
async def test_should_execute_monitored_data(self, mock_dt):
now = datetime.utcnow()
mock_dt.utcnow.return_value = now
for i in range(self.cache_to_save):
self.monitor({"my data": i})
await asyncio.sleep(1)
self.sink.external.assert_called_once_with({
'timestamp': now.timestamp(),
'identifier': 'data0',
'predicted': True
})
def test_new_with_custom_fail_max():
"""It should support a custom maximum number of failures."""
breaker = CircuitBreaker(fail_max=10)
assert 0 == breaker.fail_counter
assert timedelta(seconds=60) == breaker.timeout_duration
assert 10 == breaker.fail_max
assert () == breaker.excluded_exceptions
assert () == breaker.listeners
assert 'memory' == breaker._state_storage.name
def test_state_opened_at_not_reset_during_creation():
for state in CircuitBreakerState:
storage = CircuitMemoryStorage(state)
now = datetime.now()
storage.opened_at = now
breaker = CircuitBreaker(state_storage=storage)
assert breaker.state.state == state
assert storage.opened_at == now
