def tor_mock( self, side_effect: Optional[Exception] = None, read_data: Sequence[str] = ["1.2.3.4", "5.6.7.8"], ) -> Iterator[mock.Mock]: # We need to reset the circuitbreaker before starting. We # patch the .opened property to be false, then call the # function, so it resets to closed. with mock.patch("builtins.open", mock.mock_open(read_data=orjson.dumps(["1.2.3.4"]))): with mock.patch( "circuitbreaker.CircuitBreaker.opened", new_callable=mock.PropertyMock ) as mock_opened: mock_opened.return_value = False decorator.get_tor_ips() # Having closed it, it's now cached. Clear the cache. assert CircuitBreakerMonitor.get("get_tor_ips").closed cache_delete("tor_ip_addresses:") builtin_open = open if side_effect: tor_open = mock.MagicMock(side_effect=side_effect) else: tor_open = mock.mock_open(read_data=orjson.dumps(read_data)) def selective_mock_open(*args: Any, **kwargs: Any) -> IO[Any]: if args[0] == settings.TOR_EXIT_NODE_FILE_PATH: return tor_open(*args, **kwargs) return builtin_open(*args, **kwargs) with mock.patch("builtins.open", selective_mock_open): yield tor_open
def test_circuitbreaker_monitor(): assert CircuitBreakerMonitor.all_closed() is True assert len(list(CircuitBreakerMonitor.get_circuits())) == 5 assert len(list(CircuitBreakerMonitor.get_closed())) == 5 assert len(list(CircuitBreakerMonitor.get_open())) == 0 with raises(IOError): circuit_failure() assert CircuitBreakerMonitor.all_closed() is False assert len(list(CircuitBreakerMonitor.get_circuits())) == 5 assert len(list(CircuitBreakerMonitor.get_closed())) == 4 assert len(list(CircuitBreakerMonitor.get_open())) == 1
def test_circuitbreaker_monitor(): assert CircuitBreakerMonitor.all_closed() is True assert len(list(CircuitBreakerMonitor.get_circuits())) == 5 assert len(list(CircuitBreakerMonitor.get_closed())) == 5 assert len(list(CircuitBreakerMonitor.get_open())) == 0 with raises(ConnectionError): circuit_failure() assert CircuitBreakerMonitor.all_closed() is False assert len(list(CircuitBreakerMonitor.get_circuits())) == 5 assert len(list(CircuitBreakerMonitor.get_closed())) == 4 assert len(list(CircuitBreakerMonitor.get_open())) == 1
def getCBStats(request): all_closed = CircuitBreakerMonitor.all_closed() html = "<h1> Circuit breaker Status: " if all_closed is False: html += "Open" else: html += "Closed" html += "</h1>" return HttpResponse(html)
def test_threshold_hit_prevents_consequent_calls(mock_remote: Mock): mock_remote.side_effect = ConnectionError('Connection refused') circuitbreaker = CircuitBreakerMonitor.get('threshold_1') assert circuitbreaker.closed with raises(ConnectionError): circuit_threshold_1() assert circuitbreaker.opened with raises(CircuitBreakerError): circuit_threshold_1() mock_remote.assert_called_once()
def test_circuitbreaker_reopens_after_successful_calls(mock_remote): # type: (Mock) -> None circuitbreaker = CircuitBreakerMonitor.get('threshold_2') assert str(circuitbreaker) == 'threshold_2' # initial state: closed assert circuitbreaker.closed assert circuitbreaker.state == STATE_CLOSED assert circuitbreaker.failure_count == 0 # successful call -> no exception assert circuit_threshold_2_timeout_1() # from now all subsequent calls will fail mock_remote.side_effect = IOError('Connection refused') # 1. failed call -> original exception with raises(IOError): circuit_threshold_2_timeout_1() assert circuitbreaker.closed assert circuitbreaker.failure_count == 1 # 2. failed call -> original exception with raises(IOError): circuit_threshold_2_timeout_1() # Circuit breaker opens, threshold has been reached assert circuitbreaker.opened assert circuitbreaker.state == STATE_OPEN assert circuitbreaker.failure_count == 2 assert 0 < circuitbreaker.open_remaining <= 1 # 4. failed call -> not passed to function -> CircuitBreakerError with raises(CircuitBreakerError): circuit_threshold_2_timeout_1() assert circuitbreaker.opened assert circuitbreaker.failure_count == 2 assert 0 < circuitbreaker.open_remaining <= 1 # from now all subsequent calls will succeed mock_remote.side_effect = None # but recover timeout has not been reached -> still open # 5. failed call -> not passed to function -> CircuitBreakerError with raises(CircuitBreakerError): circuit_threshold_2_timeout_1() assert circuitbreaker.opened assert circuitbreaker.failure_count == 2 assert 0 < circuitbreaker.open_remaining <= 1 # wait for 1 second (recover timeout) sleep(1) # circuit half-open -> next call will be passed through assert not circuitbreaker.closed assert circuitbreaker.failure_count == 2 assert circuitbreaker.open_remaining < 0 assert circuitbreaker.state == STATE_HALF_OPEN # successful call assert circuit_threshold_2_timeout_1() # circuit closed and reset'ed assert circuitbreaker.closed assert circuitbreaker.state == STATE_CLOSED assert circuitbreaker.failure_count == 0 # some another successful calls assert circuit_threshold_2_timeout_1() assert circuit_threshold_2_timeout_1() assert circuit_threshold_2_timeout_1()
def update_stats(): """ Update current mertics """ intent_gauge.set(len(app().get_intents())) thread_count.set(active_count()) open_circuit_breakers.set(sum(1 for _ in CircuitBreakerMonitor.get_open()))
from circuitbreaker import CircuitBreakerMonitor from datetime import datetime import random from time import sleep if __name__ == "__main__": while True: print(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), end='') print(" Registered Circuits = {}".format( CircuitBreakerMonitor.get_circuits())) sleep(1)
def print_summary(): for x in CircuitBreakerMonitor.get_circuits(): msg = "{} circuit state: {}. Time till open: {}" print(msg.format(x.name, x.state, x.open_remaining))
def circutinfo(): return render_template("ciruit_monitor.html", all_circuits=CircuitBreakerMonitor.get_circuits(), closed_circuits=CircuitBreakerMonitor.get_closed())
def request(self, request, allow_control_chars=None, operation_name=None, api_reference_link=None): self.logger.info(utc_now() + "Request: %s %s" % (str(request.method), request.url)) initial_circuit_breaker_state = None if self.circuit_breaker_name: initial_circuit_breaker_state = CircuitBreakerMonitor.get( self.circuit_breaker_name).state if initial_circuit_breaker_state != circuitbreaker.STATE_CLOSED: self.logger.debug("Circuit Breaker State is {}!".format( initial_circuit_breaker_state)) signer = self.signer if not request.enforce_content_headers: signer = signer.without_content_headers stream = False if request.response_type == STREAM_RESPONSE_TYPE: stream = True try: start = timer() response = self.session.request(request.method, request.url, auth=signer, params=request.query_params, headers=request.header_params, data=request.body, stream=stream, timeout=self.timeout) end = timer() if request.header_params[constants.HEADER_REQUEST_ID]: self.logger.debug( utc_now() + 'time elapsed for request {}: {}'.format( request.header_params[constants.HEADER_REQUEST_ID], str(end - start))) if response and hasattr(response, 'elapsed'): self.logger.debug(utc_now() + "time elapsed in response: " + str(response.elapsed)) except requests.exceptions.ConnectTimeout as e: if not e.args: e.args = ('', ) e.args = e.args + ( "Request Endpoint: " + request.method + " " + request.url + " See {} for help troubleshooting this error, or contact support and provide this full error message." .format(TROUBLESHOOT_URL), ) raise exceptions.ConnectTimeout(e) except requests.exceptions.RequestException as e: if not e.args: e.args = ('', ) e.args = e.args + ( "Request Endpoint: " + request.method + " " + request.url + " See {} for help troubleshooting this error, or contact support and provide this full error message." .format(TROUBLESHOOT_URL), ) raise exceptions.RequestException(e) response_type = request.response_type self.logger.debug(utc_now() + "Response status: %s" % str(response.status_code)) # Raise Service Error or Transient Service Error if not 200 <= response.status_code <= 299: target_service = self.service request_endpoint = request.method + " " + request.url client_version = USER_INFO timestamp = datetime.now().isoformat() service_code, message = self.get_deserialized_service_code_and_message( response, allow_control_chars) if isinstance( self.circuit_breaker_strategy, CircuitBreakerStrategy ) and self.circuit_breaker_strategy.is_transient_error( response.status_code, service_code): new_circuit_breaker_state = CircuitBreakerMonitor.get( self.circuit_breaker_name).state if initial_circuit_breaker_state != new_circuit_breaker_state: self.logger.warning( "Circuit Breaker state changed from {} to {}".format( initial_circuit_breaker_state, new_circuit_breaker_state)) self.raise_transient_service_error( request, response, service_code, message, operation_name, api_reference_link, target_service, request_endpoint, client_version, timestamp) else: self.raise_service_error(request, response, service_code, message, operation_name, api_reference_link, target_service, request_endpoint, client_version, timestamp) if stream: # Don't unpack a streaming response body deserialized_data = response elif response_type == BYTES_RESPONSE_TYPE: # Don't deserialize data responses. deserialized_data = response.content elif response_type: deserialized_data = self.deserialize_response_data( response.content, response_type, allow_control_chars) else: deserialized_data = None resp = Response(response.status_code, response.headers, deserialized_data, request) self.logger.debug(utc_now() + "Response returned") return resp
def test_circuitbreaker_reopens_after_successful_calls(mock_remote: Mock): circuitbreaker = CircuitBreakerMonitor.get('threshold_2') assert str(circuitbreaker) == 'threshold_2' # initial state: closed assert circuitbreaker.closed assert circuitbreaker.state == STATE_CLOSED assert circuitbreaker.failure_count == 0 # successful call -> no exception assert circuit_threshold_2_timeout_1() # from now all subsequent calls will fail mock_remote.side_effect = ConnectionError('Connection refused') # 1. failed call -> original exception with raises(ConnectionError): circuit_threshold_2_timeout_1() assert circuitbreaker.closed assert circuitbreaker.failure_count == 1 # 2. failed call -> original exception with raises(ConnectionError): circuit_threshold_2_timeout_1() # Circuit breaker opens, threshold has been reached assert circuitbreaker.opened assert circuitbreaker.state == STATE_OPEN assert circuitbreaker.failure_count == 2 assert 0 < circuitbreaker.open_remaining <= 1 # 4. failed call -> not passed to function -> CircuitBreakerError with raises(CircuitBreakerError): circuit_threshold_2_timeout_1() assert circuitbreaker.opened assert circuitbreaker.failure_count == 2 assert 0 < circuitbreaker.open_remaining <= 1 # from now all subsequent calls will succeed mock_remote.side_effect = None # but recover timeout has not been reached -> still open # 5. failed call -> not passed to function -> CircuitBreakerError with raises(CircuitBreakerError): circuit_threshold_2_timeout_1() assert circuitbreaker.opened assert circuitbreaker.failure_count == 2 assert 0 < circuitbreaker.open_remaining <= 1 # wait for 1 second (recover timeout) sleep(1) # circuit half-open -> next call will be passed through assert not circuitbreaker.closed assert circuitbreaker.failure_count == 2 assert circuitbreaker.open_remaining < 0 assert circuitbreaker.state == STATE_HALF_OPEN # successful call assert circuit_threshold_2_timeout_1() # circuit closed and reset'ed assert circuitbreaker.closed assert circuitbreaker.state == STATE_CLOSED assert circuitbreaker.failure_count == 0 # some another successful calls assert circuit_threshold_2_timeout_1() assert circuit_threshold_2_timeout_1() assert circuit_threshold_2_timeout_1()
def request(self, request): self.logger.info(utc_now() + "Request: %s %s" % (str(request.method), request.url)) initial_circuit_breaker_state = None if self.circuit_breaker_name: initial_circuit_breaker_state = CircuitBreakerMonitor.get( self.circuit_breaker_name).state if initial_circuit_breaker_state != circuitbreaker.STATE_CLOSED: self.logger.debug("Circuit Breaker State is {}!".format( initial_circuit_breaker_state)) signer = self.signer if not request.enforce_content_headers: signer = signer.without_content_headers stream = False if request.response_type == STREAM_RESPONSE_TYPE: stream = True try: start = timer() response = self.session.request(request.method, request.url, auth=signer, params=request.query_params, headers=request.header_params, data=request.body, stream=stream, timeout=self.timeout) end = timer() if request.header_params[constants.HEADER_REQUEST_ID]: self.logger.debug( utc_now() + 'time elapsed for request {}: {}'.format( request.header_params[constants.HEADER_REQUEST_ID], str(end - start))) if response and hasattr(response, 'elapsed'): self.logger.debug(utc_now() + "time elapsed in response: " + str(response.elapsed)) except requests.exceptions.ConnectTimeout as e: raise exceptions.ConnectTimeout(e) except requests.exceptions.RequestException as e: raise exceptions.RequestException(e) response_type = request.response_type self.logger.debug(utc_now() + "Response status: %s" % str(response.status_code)) # Raise Service Error or Transient Service Error if not 200 <= response.status_code <= 299: service_code, message = self.get_deserialized_service_code_and_message( response) if isinstance( self.circuit_breaker_strategy, CircuitBreakerStrategy ) and self.circuit_breaker_strategy.is_transient_error( response.status_code, service_code): new_circuit_breaker_state = CircuitBreakerMonitor.get( self.circuit_breaker_name).state if initial_circuit_breaker_state != new_circuit_breaker_state: self.logger.warning( "Circuit Breaker state changed from {} to {}".format( initial_circuit_breaker_state, new_circuit_breaker_state)) self.raise_transient_service_error(request, response, service_code, message) else: self.raise_service_error(request, response, service_code, message) if stream: # Don't unpack a streaming response body deserialized_data = response elif response_type == BYTES_RESPONSE_TYPE: # Don't deserialize data responses. deserialized_data = response.content elif response_type: deserialized_data = self.deserialize_response_data( response.content, response_type) else: deserialized_data = None resp = Response(response.status_code, response.headers, deserialized_data, request) self.logger.debug(utc_now() + "Response returned") return resp
def __init__(self, service, config, signer, type_mapping, **kwargs): validate_config(config, signer=signer) self.signer = signer # Default to true (is a regional client) if there is nothing explicitly set. Regional # clients allow us to call set_region and that'll also set the endpoint. For non-regional # clients we require an endpoint self.regional_client = kwargs.get('regional_client', True) self._endpoint = None self._base_path = kwargs.get('base_path') self.service_endpoint_template = kwargs.get( 'service_endpoint_template') self.endpoint_service_name = kwargs.get('endpoint_service_name') if self.regional_client: if kwargs.get('service_endpoint'): self.endpoint = kwargs.get('service_endpoint') else: region_to_use = None if 'region' in config and config['region']: region_to_use = config.get('region') elif hasattr(signer, 'region'): region_to_use = signer.region self.endpoint = regions.endpoint_for( service, service_endpoint_template=self.service_endpoint_template, region=region_to_use, endpoint=config.get('endpoint'), endpoint_service_name=self.endpoint_service_name) else: if not kwargs.get('service_endpoint'): raise exceptions.MissingEndpointForNonRegionalServiceClientError( 'An endpoint must be provided for a non-regional service client' ) self.endpoint = kwargs.get('service_endpoint') self.service = service self.complex_type_mappings = type_mapping self.type_mappings = merge_type_mappings(self.primitive_type_map, type_mapping) self.session = requests.Session() # If the user doesn't specify timeout explicitly we would use default timeout. self.timeout = kwargs.get('timeout') if 'timeout' in kwargs else ( DEFAULT_CONNECTION_TIMEOUT, DEFAULT_READ_TIMEOUT) self.user_agent = build_user_agent( get_config_value_or_default(config, "additional_user_agent")) self.logger = logging.getLogger("{}.{}".format(__name__, id(self))) self.logger.addHandler(logging.NullHandler()) if get_config_value_or_default(config, "log_requests"): self.logger.disabled = False self.logger.setLevel(logging.DEBUG) is_http_log_enabled(True) else: self.logger.disabled = True is_http_log_enabled(False) self.skip_deserialization = kwargs.get('skip_deserialization') # Circuit Breaker at client level self.circuit_breaker_strategy = kwargs.get('circuit_breaker_strategy') self.circuit_breaker_name = None # Log if Circuit Breaker Strategy is not enabled or if using Default Circuit breaker Strategy if self.circuit_breaker_strategy is None or isinstance( self.circuit_breaker_strategy, NoCircuitBreakerStrategy): self.logger.debug('No circuit breaker strategy enabled!') else: # Enable Circuit breaker if a valid circuit breaker strategy is available if not isinstance(self.circuit_breaker_strategy, CircuitBreakerStrategy): raise TypeError('Invalid Circuit Breaker Strategy!') self.circuit_breaker_name = str( uuid.uuid4() ) if self.circuit_breaker_strategy.name is None else self.circuit_breaker_strategy.name # Re-use Circuit breaker if sharing a Circuit Breaker Strategy. circuit_breaker = CircuitBreakerMonitor.get( self.circuit_breaker_name) if circuit_breaker is None: circuit_breaker = CircuitBreaker( failure_threshold=self.circuit_breaker_strategy. failure_threshold, recovery_timeout=self.circuit_breaker_strategy. recovery_timeout, expected_exception=self.circuit_breaker_strategy. expected_exception, name=self.circuit_breaker_name) # Equivalent to decorating the request function with Circuit Breaker self.request = circuit_breaker(self.request) self.logger.debug('Endpoint: {}'.format(self._endpoint))
def test_circuitbreaker_recover_half_open(mock_remote: Mock): circuitbreaker = CircuitBreakerMonitor.get('threshold_3') # initial state: closed assert circuitbreaker.closed assert circuitbreaker.state == STATE_CLOSED # no exception -> success assert circuit_threshold_3_timeout_1() # from now all subsequent calls will fail mock_remote.side_effect = ConnectionError('Connection refused') # 1. failed call -> original exception with raises(ConnectionError): circuit_threshold_3_timeout_1() assert circuitbreaker.closed assert circuitbreaker.failure_count == 1 # 2. failed call -> original exception with raises(ConnectionError): circuit_threshold_3_timeout_1() assert circuitbreaker.closed assert circuitbreaker.failure_count == 2 # 3. failed call -> original exception with raises(ConnectionError): circuit_threshold_3_timeout_1() # Circuit breaker opens, threshold has been reached assert circuitbreaker.opened assert circuitbreaker.state == STATE_OPEN assert circuitbreaker.failure_count == 3 assert 0 < circuitbreaker.open_remaining <= 1 # 4. failed call -> not passed to function -> CircuitBreakerError with raises(CircuitBreakerError): circuit_threshold_3_timeout_1() assert circuitbreaker.opened assert circuitbreaker.failure_count == 3 assert 0 < circuitbreaker.open_remaining <= 1 # 5. failed call -> not passed to function -> CircuitBreakerError with raises(CircuitBreakerError): circuit_threshold_3_timeout_1() assert circuitbreaker.opened assert circuitbreaker.failure_count == 3 assert 0 < circuitbreaker.open_remaining <= 1 # wait for 1 second (recover timeout) sleep(1) # circuit half-open -> next call will be passed through assert not circuitbreaker.closed assert circuitbreaker.open_remaining < 0 assert circuitbreaker.state == STATE_HALF_OPEN # State half-open -> function is executed -> original exception with raises(ConnectionError): circuit_threshold_3_timeout_1() assert circuitbreaker.opened assert circuitbreaker.failure_count == 4 assert 0 < circuitbreaker.open_remaining <= 1 # State open > not passed to function -> CircuitBreakerError with raises(CircuitBreakerError): circuit_threshold_3_timeout_1()
def test_circuitbreaker_recover_half_open(mock_remote): # type: (Mock) -> None circuitbreaker = CircuitBreakerMonitor.get('threshold_3') # initial state: closed assert circuitbreaker.closed assert circuitbreaker.state == STATE_CLOSED # no exception -> success assert circuit_threshold_3_timeout_1() # from now all subsequent calls will fail mock_remote.side_effect = IOError('Connection refused') # 1. failed call -> original exception with raises(IOError): circuit_threshold_3_timeout_1() assert circuitbreaker.closed assert circuitbreaker.failure_count == 1 # 2. failed call -> original exception with raises(IOError): circuit_threshold_3_timeout_1() assert circuitbreaker.closed assert circuitbreaker.failure_count == 2 # 3. failed call -> original exception with raises(IOError): circuit_threshold_3_timeout_1() # Circuit breaker opens, threshold has been reached assert circuitbreaker.opened assert circuitbreaker.state == STATE_OPEN assert circuitbreaker.failure_count == 3 assert 0 < circuitbreaker.open_remaining <= 1 # 4. failed call -> not passed to function -> CircuitBreakerError with raises(CircuitBreakerError): circuit_threshold_3_timeout_1() assert circuitbreaker.opened assert circuitbreaker.failure_count == 3 assert 0 < circuitbreaker.open_remaining <= 1 # 5. failed call -> not passed to function -> CircuitBreakerError with raises(CircuitBreakerError): circuit_threshold_3_timeout_1() assert circuitbreaker.opened assert circuitbreaker.failure_count == 3 assert 0 < circuitbreaker.open_remaining <= 1 # wait for 1 second (recover timeout) sleep(1) # circuit half-open -> next call will be passed through assert not circuitbreaker.closed assert circuitbreaker.open_remaining < 0 assert circuitbreaker.state == STATE_HALF_OPEN # State half-open -> function is executed -> original exception with raises(IOError): circuit_threshold_3_timeout_1() assert circuitbreaker.opened assert circuitbreaker.failure_count == 4 assert 0 < circuitbreaker.open_remaining <= 1 # State open > not passed to function -> CircuitBreakerError with raises(CircuitBreakerError): circuit_threshold_3_timeout_1()
def consumers_health_status(): c = consul.Consul() health = [ c.health.service(s)[1][0]["Checks"][1]["Status"] for s in c.agent.services() ] unhealthy = [u for u in health if u != "passing"] if len(unhealthy) > 0: raise Exception("Consumers services are down !") print("::: Health status = passed", end="") return True if __name__ == "__main__": while True: print(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), end='') cb = CircuitBreakerMonitor.get('my_circuit_breaker') try: consumers_health_status() except Exception as e: print( " Monitor state: {} opened: {} closed: {} failure_count: {} open_remaining: {}" .format(cb.state, cb.opened, cb.closed, cb.failure_count, cb.open_remaining)) print(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), end='') print(' / {} {}'.format(type(e), e), end='') finally: print('') sleep(1)