def tor_mock(
self,
side_effect: Optional[Exception] = None,
read_data: Sequence[str] = ["1.2.3.4", "5.6.7.8"],
) -> Iterator[mock.Mock]:
# We need to reset the circuitbreaker before starting. We
# patch the .opened property to be false, then call the
# function, so it resets to closed.
with mock.patch("builtins.open", mock.mock_open(read_data=orjson.dumps(["1.2.3.4"]))):
with mock.patch(
"circuitbreaker.CircuitBreaker.opened", new_callable=mock.PropertyMock
) as mock_opened:
mock_opened.return_value = False
decorator.get_tor_ips()
# Having closed it, it's now cached. Clear the cache.
assert CircuitBreakerMonitor.get("get_tor_ips").closed
cache_delete("tor_ip_addresses:")
builtin_open = open
if side_effect:
tor_open = mock.MagicMock(side_effect=side_effect)
else:
tor_open = mock.mock_open(read_data=orjson.dumps(read_data))
def selective_mock_open(*args: Any, **kwargs: Any) -> IO[Any]:
if args[0] == settings.TOR_EXIT_NODE_FILE_PATH:
return tor_open(*args, **kwargs)
return builtin_open(*args, **kwargs)
with mock.patch("builtins.open", selective_mock_open):
yield tor_open
def test_circuitbreaker_monitor():
assert CircuitBreakerMonitor.all_closed() is True
assert len(list(CircuitBreakerMonitor.get_circuits())) == 5
assert len(list(CircuitBreakerMonitor.get_closed())) == 5
assert len(list(CircuitBreakerMonitor.get_open())) == 0
with raises(IOError):
circuit_failure()
assert CircuitBreakerMonitor.all_closed() is False
assert len(list(CircuitBreakerMonitor.get_circuits())) == 5
assert len(list(CircuitBreakerMonitor.get_closed())) == 4
assert len(list(CircuitBreakerMonitor.get_open())) == 1
def test_circuitbreaker_monitor():
assert CircuitBreakerMonitor.all_closed() is True
assert len(list(CircuitBreakerMonitor.get_circuits())) == 5
assert len(list(CircuitBreakerMonitor.get_closed())) == 5
assert len(list(CircuitBreakerMonitor.get_open())) == 0
with raises(ConnectionError):
circuit_failure()
assert CircuitBreakerMonitor.all_closed() is False
assert len(list(CircuitBreakerMonitor.get_circuits())) == 5
assert len(list(CircuitBreakerMonitor.get_closed())) == 4
assert len(list(CircuitBreakerMonitor.get_open())) == 1
def getCBStats(request):
all_closed = CircuitBreakerMonitor.all_closed()
html = "<h1> Circuit breaker Status: "
if all_closed is False:
html += "Open"
else:
html += "Closed"
html += "</h1>"
return HttpResponse(html)
def test_threshold_hit_prevents_consequent_calls(mock_remote: Mock):
mock_remote.side_effect = ConnectionError('Connection refused')
circuitbreaker = CircuitBreakerMonitor.get('threshold_1')
assert circuitbreaker.closed
with raises(ConnectionError):
circuit_threshold_1()
assert circuitbreaker.opened
with raises(CircuitBreakerError):
circuit_threshold_1()
mock_remote.assert_called_once()
def test_circuitbreaker_reopens_after_successful_calls(mock_remote):
# type: (Mock) -> None
circuitbreaker = CircuitBreakerMonitor.get('threshold_2')
assert str(circuitbreaker) == 'threshold_2'
# initial state: closed
assert circuitbreaker.closed
assert circuitbreaker.state == STATE_CLOSED
assert circuitbreaker.failure_count == 0
# successful call -> no exception
assert circuit_threshold_2_timeout_1()
# from now all subsequent calls will fail
mock_remote.side_effect = IOError('Connection refused')
# 1. failed call -> original exception
with raises(IOError):
circuit_threshold_2_timeout_1()
assert circuitbreaker.closed
assert circuitbreaker.failure_count == 1
# 2. failed call -> original exception
with raises(IOError):
circuit_threshold_2_timeout_1()
# Circuit breaker opens, threshold has been reached
assert circuitbreaker.opened
assert circuitbreaker.state == STATE_OPEN
assert circuitbreaker.failure_count == 2
assert 0 < circuitbreaker.open_remaining <= 1
# 4. failed call -> not passed to function -> CircuitBreakerError
with raises(CircuitBreakerError):
circuit_threshold_2_timeout_1()
assert circuitbreaker.opened
assert circuitbreaker.failure_count == 2
assert 0 < circuitbreaker.open_remaining <= 1
# from now all subsequent calls will succeed
mock_remote.side_effect = None
# but recover timeout has not been reached -> still open
# 5. failed call -> not passed to function -> CircuitBreakerError
with raises(CircuitBreakerError):
circuit_threshold_2_timeout_1()
assert circuitbreaker.opened
assert circuitbreaker.failure_count == 2
assert 0 < circuitbreaker.open_remaining <= 1
# wait for 1 second (recover timeout)
sleep(1)
# circuit half-open -> next call will be passed through
assert not circuitbreaker.closed
assert circuitbreaker.failure_count == 2
assert circuitbreaker.open_remaining < 0
assert circuitbreaker.state == STATE_HALF_OPEN
# successful call
assert circuit_threshold_2_timeout_1()
# circuit closed and reset'ed
assert circuitbreaker.closed
assert circuitbreaker.state == STATE_CLOSED
assert circuitbreaker.failure_count == 0
# some another successful calls
assert circuit_threshold_2_timeout_1()
assert circuit_threshold_2_timeout_1()
assert circuit_threshold_2_timeout_1()
def update_stats():
""" Update current mertics """
intent_gauge.set(len(app().get_intents()))
thread_count.set(active_count())
open_circuit_breakers.set(sum(1 for _ in CircuitBreakerMonitor.get_open()))
from circuitbreaker import CircuitBreakerMonitor
from datetime import datetime
import random
from time import sleep
if __name__ == "__main__":
while True:
print(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), end='')
print(" Registered Circuits = {}".format(
CircuitBreakerMonitor.get_circuits()))
sleep(1)
def print_summary():
for x in CircuitBreakerMonitor.get_circuits():
msg = "{} circuit state: {}. Time till open: {}"
print(msg.format(x.name, x.state, x.open_remaining))
def circutinfo():
return render_template("ciruit_monitor.html",
all_circuits=CircuitBreakerMonitor.get_circuits(),
closed_circuits=CircuitBreakerMonitor.get_closed())
def request(self,
request,
allow_control_chars=None,
operation_name=None,
api_reference_link=None):
self.logger.info(utc_now() + "Request: %s %s" %
(str(request.method), request.url))
initial_circuit_breaker_state = None
if self.circuit_breaker_name:
initial_circuit_breaker_state = CircuitBreakerMonitor.get(
self.circuit_breaker_name).state
if initial_circuit_breaker_state != circuitbreaker.STATE_CLOSED:
self.logger.debug("Circuit Breaker State is {}!".format(
initial_circuit_breaker_state))
signer = self.signer
if not request.enforce_content_headers:
signer = signer.without_content_headers
stream = False
if request.response_type == STREAM_RESPONSE_TYPE:
stream = True
try:
start = timer()
response = self.session.request(request.method,
request.url,
auth=signer,
params=request.query_params,
headers=request.header_params,
data=request.body,
stream=stream,
timeout=self.timeout)
end = timer()
if request.header_params[constants.HEADER_REQUEST_ID]:
self.logger.debug(
utc_now() + 'time elapsed for request {}: {}'.format(
request.header_params[constants.HEADER_REQUEST_ID],
str(end - start)))
if response and hasattr(response, 'elapsed'):
self.logger.debug(utc_now() + "time elapsed in response: " +
str(response.elapsed))
except requests.exceptions.ConnectTimeout as e:
if not e.args:
e.args = ('', )
e.args = e.args + (
"Request Endpoint: " + request.method + " " + request.url +
" See {} for help troubleshooting this error, or contact support and provide this full error message."
.format(TROUBLESHOOT_URL), )
raise exceptions.ConnectTimeout(e)
except requests.exceptions.RequestException as e:
if not e.args:
e.args = ('', )
e.args = e.args + (
"Request Endpoint: " + request.method + " " + request.url +
" See {} for help troubleshooting this error, or contact support and provide this full error message."
.format(TROUBLESHOOT_URL), )
raise exceptions.RequestException(e)
response_type = request.response_type
self.logger.debug(utc_now() +
"Response status: %s" % str(response.status_code))
# Raise Service Error or Transient Service Error
if not 200 <= response.status_code <= 299:
target_service = self.service
request_endpoint = request.method + " " + request.url
client_version = USER_INFO
timestamp = datetime.now().isoformat()
service_code, message = self.get_deserialized_service_code_and_message(
response, allow_control_chars)
if isinstance(
self.circuit_breaker_strategy, CircuitBreakerStrategy
) and self.circuit_breaker_strategy.is_transient_error(
response.status_code, service_code):
new_circuit_breaker_state = CircuitBreakerMonitor.get(
self.circuit_breaker_name).state
if initial_circuit_breaker_state != new_circuit_breaker_state:
self.logger.warning(
"Circuit Breaker state changed from {} to {}".format(
initial_circuit_breaker_state,
new_circuit_breaker_state))
self.raise_transient_service_error(
request, response, service_code, message, operation_name,
api_reference_link, target_service, request_endpoint,
client_version, timestamp)
else:
self.raise_service_error(request, response, service_code,
message, operation_name,
api_reference_link, target_service,
request_endpoint, client_version,
timestamp)
if stream:
# Don't unpack a streaming response body
deserialized_data = response
elif response_type == BYTES_RESPONSE_TYPE:
# Don't deserialize data responses.
deserialized_data = response.content
elif response_type:
deserialized_data = self.deserialize_response_data(
response.content, response_type, allow_control_chars)
else:
deserialized_data = None
resp = Response(response.status_code, response.headers,
deserialized_data, request)
self.logger.debug(utc_now() + "Response returned")
return resp
def test_circuitbreaker_reopens_after_successful_calls(mock_remote: Mock):
circuitbreaker = CircuitBreakerMonitor.get('threshold_2')
assert str(circuitbreaker) == 'threshold_2'
# initial state: closed
assert circuitbreaker.closed
assert circuitbreaker.state == STATE_CLOSED
assert circuitbreaker.failure_count == 0
# successful call -> no exception
assert circuit_threshold_2_timeout_1()
# from now all subsequent calls will fail
mock_remote.side_effect = ConnectionError('Connection refused')
# 1. failed call -> original exception
with raises(ConnectionError):
circuit_threshold_2_timeout_1()
assert circuitbreaker.closed
assert circuitbreaker.failure_count == 1
# 2. failed call -> original exception
with raises(ConnectionError):
circuit_threshold_2_timeout_1()
# Circuit breaker opens, threshold has been reached
assert circuitbreaker.opened
assert circuitbreaker.state == STATE_OPEN
assert circuitbreaker.failure_count == 2
assert 0 < circuitbreaker.open_remaining <= 1
# 4. failed call -> not passed to function -> CircuitBreakerError
with raises(CircuitBreakerError):
circuit_threshold_2_timeout_1()
assert circuitbreaker.opened
assert circuitbreaker.failure_count == 2
assert 0 < circuitbreaker.open_remaining <= 1
# from now all subsequent calls will succeed
mock_remote.side_effect = None
# but recover timeout has not been reached -> still open
# 5. failed call -> not passed to function -> CircuitBreakerError
with raises(CircuitBreakerError):
circuit_threshold_2_timeout_1()
assert circuitbreaker.opened
assert circuitbreaker.failure_count == 2
assert 0 < circuitbreaker.open_remaining <= 1
# wait for 1 second (recover timeout)
sleep(1)
# circuit half-open -> next call will be passed through
assert not circuitbreaker.closed
assert circuitbreaker.failure_count == 2
assert circuitbreaker.open_remaining < 0
assert circuitbreaker.state == STATE_HALF_OPEN
# successful call
assert circuit_threshold_2_timeout_1()
# circuit closed and reset'ed
assert circuitbreaker.closed
assert circuitbreaker.state == STATE_CLOSED
assert circuitbreaker.failure_count == 0
# some another successful calls
assert circuit_threshold_2_timeout_1()
assert circuit_threshold_2_timeout_1()
assert circuit_threshold_2_timeout_1()
def request(self, request):
self.logger.info(utc_now() + "Request: %s %s" %
(str(request.method), request.url))
initial_circuit_breaker_state = None
if self.circuit_breaker_name:
initial_circuit_breaker_state = CircuitBreakerMonitor.get(
self.circuit_breaker_name).state
if initial_circuit_breaker_state != circuitbreaker.STATE_CLOSED:
self.logger.debug("Circuit Breaker State is {}!".format(
initial_circuit_breaker_state))
signer = self.signer
if not request.enforce_content_headers:
signer = signer.without_content_headers
stream = False
if request.response_type == STREAM_RESPONSE_TYPE:
stream = True
try:
start = timer()
response = self.session.request(request.method,
request.url,
auth=signer,
params=request.query_params,
headers=request.header_params,
data=request.body,
stream=stream,
timeout=self.timeout)
end = timer()
if request.header_params[constants.HEADER_REQUEST_ID]:
self.logger.debug(
utc_now() + 'time elapsed for request {}: {}'.format(
request.header_params[constants.HEADER_REQUEST_ID],
str(end - start)))
if response and hasattr(response, 'elapsed'):
self.logger.debug(utc_now() + "time elapsed in response: " +
str(response.elapsed))
except requests.exceptions.ConnectTimeout as e:
raise exceptions.ConnectTimeout(e)
except requests.exceptions.RequestException as e:
raise exceptions.RequestException(e)
response_type = request.response_type
self.logger.debug(utc_now() +
"Response status: %s" % str(response.status_code))
# Raise Service Error or Transient Service Error
if not 200 <= response.status_code <= 299:
service_code, message = self.get_deserialized_service_code_and_message(
response)
if isinstance(
self.circuit_breaker_strategy, CircuitBreakerStrategy
) and self.circuit_breaker_strategy.is_transient_error(
response.status_code, service_code):
new_circuit_breaker_state = CircuitBreakerMonitor.get(
self.circuit_breaker_name).state
if initial_circuit_breaker_state != new_circuit_breaker_state:
self.logger.warning(
"Circuit Breaker state changed from {} to {}".format(
initial_circuit_breaker_state,
new_circuit_breaker_state))
self.raise_transient_service_error(request, response,
service_code, message)
else:
self.raise_service_error(request, response, service_code,
message)
if stream:
# Don't unpack a streaming response body
deserialized_data = response
elif response_type == BYTES_RESPONSE_TYPE:
# Don't deserialize data responses.
deserialized_data = response.content
elif response_type:
deserialized_data = self.deserialize_response_data(
response.content, response_type)
else:
deserialized_data = None
resp = Response(response.status_code, response.headers,
deserialized_data, request)
self.logger.debug(utc_now() + "Response returned")
return resp
def __init__(self, service, config, signer, type_mapping, **kwargs):
validate_config(config, signer=signer)
self.signer = signer
# Default to true (is a regional client) if there is nothing explicitly set. Regional
# clients allow us to call set_region and that'll also set the endpoint. For non-regional
# clients we require an endpoint
self.regional_client = kwargs.get('regional_client', True)
self._endpoint = None
self._base_path = kwargs.get('base_path')
self.service_endpoint_template = kwargs.get(
'service_endpoint_template')
self.endpoint_service_name = kwargs.get('endpoint_service_name')
if self.regional_client:
if kwargs.get('service_endpoint'):
self.endpoint = kwargs.get('service_endpoint')
else:
region_to_use = None
if 'region' in config and config['region']:
region_to_use = config.get('region')
elif hasattr(signer, 'region'):
region_to_use = signer.region
self.endpoint = regions.endpoint_for(
service,
service_endpoint_template=self.service_endpoint_template,
region=region_to_use,
endpoint=config.get('endpoint'),
endpoint_service_name=self.endpoint_service_name)
else:
if not kwargs.get('service_endpoint'):
raise exceptions.MissingEndpointForNonRegionalServiceClientError(
'An endpoint must be provided for a non-regional service client'
)
self.endpoint = kwargs.get('service_endpoint')
self.service = service
self.complex_type_mappings = type_mapping
self.type_mappings = merge_type_mappings(self.primitive_type_map,
type_mapping)
self.session = requests.Session()
# If the user doesn't specify timeout explicitly we would use default timeout.
self.timeout = kwargs.get('timeout') if 'timeout' in kwargs else (
DEFAULT_CONNECTION_TIMEOUT, DEFAULT_READ_TIMEOUT)
self.user_agent = build_user_agent(
get_config_value_or_default(config, "additional_user_agent"))
self.logger = logging.getLogger("{}.{}".format(__name__, id(self)))
self.logger.addHandler(logging.NullHandler())
if get_config_value_or_default(config, "log_requests"):
self.logger.disabled = False
self.logger.setLevel(logging.DEBUG)
is_http_log_enabled(True)
else:
self.logger.disabled = True
is_http_log_enabled(False)
self.skip_deserialization = kwargs.get('skip_deserialization')
# Circuit Breaker at client level
self.circuit_breaker_strategy = kwargs.get('circuit_breaker_strategy')
self.circuit_breaker_name = None
# Log if Circuit Breaker Strategy is not enabled or if using Default Circuit breaker Strategy
if self.circuit_breaker_strategy is None or isinstance(
self.circuit_breaker_strategy, NoCircuitBreakerStrategy):
self.logger.debug('No circuit breaker strategy enabled!')
else:
# Enable Circuit breaker if a valid circuit breaker strategy is available
if not isinstance(self.circuit_breaker_strategy,
CircuitBreakerStrategy):
raise TypeError('Invalid Circuit Breaker Strategy!')
self.circuit_breaker_name = str(
uuid.uuid4()
) if self.circuit_breaker_strategy.name is None else self.circuit_breaker_strategy.name
# Re-use Circuit breaker if sharing a Circuit Breaker Strategy.
circuit_breaker = CircuitBreakerMonitor.get(
self.circuit_breaker_name)
if circuit_breaker is None:
circuit_breaker = CircuitBreaker(
failure_threshold=self.circuit_breaker_strategy.
failure_threshold,
recovery_timeout=self.circuit_breaker_strategy.
recovery_timeout,
expected_exception=self.circuit_breaker_strategy.
expected_exception,
name=self.circuit_breaker_name)
# Equivalent to decorating the request function with Circuit Breaker
self.request = circuit_breaker(self.request)
self.logger.debug('Endpoint: {}'.format(self._endpoint))
def test_circuitbreaker_recover_half_open(mock_remote: Mock):
circuitbreaker = CircuitBreakerMonitor.get('threshold_3')
# initial state: closed
assert circuitbreaker.closed
assert circuitbreaker.state == STATE_CLOSED
# no exception -> success
assert circuit_threshold_3_timeout_1()
# from now all subsequent calls will fail
mock_remote.side_effect = ConnectionError('Connection refused')
# 1. failed call -> original exception
with raises(ConnectionError):
circuit_threshold_3_timeout_1()
assert circuitbreaker.closed
assert circuitbreaker.failure_count == 1
# 2. failed call -> original exception
with raises(ConnectionError):
circuit_threshold_3_timeout_1()
assert circuitbreaker.closed
assert circuitbreaker.failure_count == 2
# 3. failed call -> original exception
with raises(ConnectionError):
circuit_threshold_3_timeout_1()
# Circuit breaker opens, threshold has been reached
assert circuitbreaker.opened
assert circuitbreaker.state == STATE_OPEN
assert circuitbreaker.failure_count == 3
assert 0 < circuitbreaker.open_remaining <= 1
# 4. failed call -> not passed to function -> CircuitBreakerError
with raises(CircuitBreakerError):
circuit_threshold_3_timeout_1()
assert circuitbreaker.opened
assert circuitbreaker.failure_count == 3
assert 0 < circuitbreaker.open_remaining <= 1
# 5. failed call -> not passed to function -> CircuitBreakerError
with raises(CircuitBreakerError):
circuit_threshold_3_timeout_1()
assert circuitbreaker.opened
assert circuitbreaker.failure_count == 3
assert 0 < circuitbreaker.open_remaining <= 1
# wait for 1 second (recover timeout)
sleep(1)
# circuit half-open -> next call will be passed through
assert not circuitbreaker.closed
assert circuitbreaker.open_remaining < 0
assert circuitbreaker.state == STATE_HALF_OPEN
# State half-open -> function is executed -> original exception
with raises(ConnectionError):
circuit_threshold_3_timeout_1()
assert circuitbreaker.opened
assert circuitbreaker.failure_count == 4
assert 0 < circuitbreaker.open_remaining <= 1
# State open > not passed to function -> CircuitBreakerError
with raises(CircuitBreakerError):
circuit_threshold_3_timeout_1()
def test_circuitbreaker_recover_half_open(mock_remote):
# type: (Mock) -> None
circuitbreaker = CircuitBreakerMonitor.get('threshold_3')
# initial state: closed
assert circuitbreaker.closed
assert circuitbreaker.state == STATE_CLOSED
# no exception -> success
assert circuit_threshold_3_timeout_1()
# from now all subsequent calls will fail
mock_remote.side_effect = IOError('Connection refused')
# 1. failed call -> original exception
with raises(IOError):
circuit_threshold_3_timeout_1()
assert circuitbreaker.closed
assert circuitbreaker.failure_count == 1
# 2. failed call -> original exception
with raises(IOError):
circuit_threshold_3_timeout_1()
assert circuitbreaker.closed
assert circuitbreaker.failure_count == 2
# 3. failed call -> original exception
with raises(IOError):
circuit_threshold_3_timeout_1()
# Circuit breaker opens, threshold has been reached
assert circuitbreaker.opened
assert circuitbreaker.state == STATE_OPEN
assert circuitbreaker.failure_count == 3
assert 0 < circuitbreaker.open_remaining <= 1
# 4. failed call -> not passed to function -> CircuitBreakerError
with raises(CircuitBreakerError):
circuit_threshold_3_timeout_1()
assert circuitbreaker.opened
assert circuitbreaker.failure_count == 3
assert 0 < circuitbreaker.open_remaining <= 1
# 5. failed call -> not passed to function -> CircuitBreakerError
with raises(CircuitBreakerError):
circuit_threshold_3_timeout_1()
assert circuitbreaker.opened
assert circuitbreaker.failure_count == 3
assert 0 < circuitbreaker.open_remaining <= 1
# wait for 1 second (recover timeout)
sleep(1)
# circuit half-open -> next call will be passed through
assert not circuitbreaker.closed
assert circuitbreaker.open_remaining < 0
assert circuitbreaker.state == STATE_HALF_OPEN
# State half-open -> function is executed -> original exception
with raises(IOError):
circuit_threshold_3_timeout_1()
assert circuitbreaker.opened
assert circuitbreaker.failure_count == 4
assert 0 < circuitbreaker.open_remaining <= 1
# State open > not passed to function -> CircuitBreakerError
with raises(CircuitBreakerError):
circuit_threshold_3_timeout_1()
def consumers_health_status():
c = consul.Consul()
health = [
c.health.service(s)[1][0]["Checks"][1]["Status"]
for s in c.agent.services()
]
unhealthy = [u for u in health if u != "passing"]
if len(unhealthy) > 0:
raise Exception("Consumers services are down !")
print("::: Health status = passed", end="")
return True
if __name__ == "__main__":
while True:
print(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), end='')
cb = CircuitBreakerMonitor.get('my_circuit_breaker')
try:
consumers_health_status()
except Exception as e:
print(
" Monitor state: {} opened: {} closed: {} failure_count: {} open_remaining: {}"
.format(cb.state, cb.opened, cb.closed, cb.failure_count,
cb.open_remaining))
print(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), end='')
print(' / {} {}'.format(type(e), e), end='')
finally:
print('')
sleep(1)
