def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # TODO allow user to choose it's own user agent storefront_header = "storefront ({commit_hash};{environment})".format( commit_hash=os.getenv("COMMIT_ID", "commit_id"), environment=os.getenv("ENVIRONMENT", "devel"), ) headers = {"User-Agent": storefront_header} self.headers.update(headers) self.api_breaker = CircuitBreaker(fail_max=5, reset_timeout=60)
class BaseSession: """A base session interface to implement common functionality Create an interface to manage exceptions and return API exceptions """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # TODO allow user to choose it's own user agent storefront_header = "storefront ({commit_hash};{environment})".format( commit_hash=os.getenv("COMMIT_ID", "commit_id"), environment=os.getenv("ENVIRONMENT", "devel"), ) headers = {"User-Agent": storefront_header} self.headers.update(headers) self.api_breaker = CircuitBreaker(fail_max=5, reset_timeout=60) def request(self, method, url, **kwargs): try: request = self.api_breaker.call(super().request, method=method, url=url, **kwargs) except requests.exceptions.Timeout: raise ApiTimeoutError( "The request to {} took too long".format(url)) except requests.exceptions.ConnectionError: raise ApiConnectionError( "Failed to establish connection to {}.".format(url)) except CircuitBreakerError: raise ApiCircuitBreaker( "Requests are closed because of too many failures".format(url)) return request
class BaseSession: """A base session interface to implement common functionality Create an interface to manage exceptions and return API exceptions """ def __init__(self, timeout=(0.5, 3), *args, **kwargs): super().__init__(*args, **kwargs) self.mount("http://", TimeoutHTTPAdapter(timeout=timeout)) self.mount("https://", TimeoutHTTPAdapter(timeout=timeout)) # TODO allow user to choose it's own user agent storefront_header = "storefront ({commit_hash};{environment})".format( commit_hash=os.getenv("COMMIT_ID", "commit_id"), environment=os.getenv("ENVIRONMENT", "devel"), ) headers = {"User-Agent": storefront_header} self.headers.update(headers) self.api_breaker = CircuitBreaker(fail_max=5, reset_timeout=60) def request(self, method, url, **kwargs): domain = urlparse(url).netloc try: request = self.api_breaker.call( super().request, method=method, url=url, **kwargs ) except requests.exceptions.Timeout: timeout_counter.labels(domain=domain).inc() raise ApiTimeoutError( "The request to {} took too long".format(url) ) except requests.exceptions.ConnectionError: connection_failed_counter.labels(domain=domain).inc() raise ApiConnectionError( "Failed to establish connection to {}.".format(url) ) except CircuitBreakerError: raise ApiCircuitBreaker( "Requests are closed because of too many failures".format(url) ) latency_histogram.labels( domain=domain, code=request.status_code ).observe(request.elapsed.total_seconds()) return request
def __init__(self, timeout=(0.5, 3), *args, **kwargs): super().__init__(*args, **kwargs) self.mount("http://", TimeoutHTTPAdapter(timeout=timeout)) self.mount("https://", TimeoutHTTPAdapter(timeout=timeout)) # TODO allow user to choose it's own user agent storefront_header = "storefront ({commit_hash};{environment})".format( commit_hash=os.getenv("COMMIT_ID", "commit_id"), environment=os.getenv("ENVIRONMENT", "devel"), ) headers = {"User-Agent": storefront_header} self.headers.update(headers) self.api_breaker = CircuitBreaker(fail_max=5, reset_timeout=60)
class BaseSession: """A base session interface to implement common functionality Create an interface to manage exceptions and return API exceptions """ def __init__(self, timeout=(0.5, 3), *args, **kwargs): super().__init__(*args, **kwargs) self.mount("http://", TimeoutHTTPAdapter(timeout=timeout)) self.mount("https://", TimeoutHTTPAdapter(timeout=timeout)) # TODO allow user to choose it's own user agent storefront_header = "storefront ({commit_hash};{environment})".format( commit_hash=os.getenv("COMMIT_ID", "commit_id"), environment=os.getenv("ENVIRONMENT", "devel"), ) headers = {"User-Agent": storefront_header} self.headers.update(headers) self.api_breaker = CircuitBreaker(fail_max=5, reset_timeout=60) def request(self, method, url, **kwargs): domain = urlparse(url).netloc try: request = self.api_breaker.call(super().request, method=method, url=url, **kwargs) except requests.exceptions.Timeout: timeout_counter.labels(domain=domain).inc() raise ApiTimeoutError( "The request to {} took too long".format(url)) except requests.exceptions.ConnectionError: connection_failed_counter.labels(domain=domain).inc() raise ApiConnectionError( "Failed to establish connection to {}.".format(url)) except CircuitBreakerError: raise ApiCircuitBreaker( "Requests are closed because of too many failures".format(url)) latency_histogram.labels(domain=domain, code=request.status_code).observe( request.elapsed.total_seconds()) return request
class CircuitBreakerThreadsTestCase(unittest.TestCase): """ Tests to reproduce common synchronization errors on CircuitBreaker class. """ def setUp(self): self.breaker = CircuitBreaker(fail_max=3000, reset_timeout=600) def _start_threads(self, target, n): """ Starts `n` threads that calls `target` and waits for them to finish. """ threads = [threading.Thread(target=target) for i in range(n)] [t.start() for t in threads] [t.join() for t in threads] def _mock_function(self, obj, func): """ Replaces a bounded function in `self.breaker` by another. """ setattr(obj, func.__name__, MethodType(func, self.breaker)) def test_fail_thread_safety_context_manager(self): """CircuitBreaker: it should compute a failed call atomically to avoid race conditions, even when using context manager """ def trigger_error(): for n in range(500): try: with self.breaker.context(): raise Exception() except: pass def _inc_counter(self): c = self._fail_counter sleep(0.00005) self._fail_counter = c + 1 self._mock_function(self.breaker, _inc_counter) self._start_threads(trigger_error, 3) self.assertEqual(1500, self.breaker.fail_counter) def test_fail_thread_safety(self): """CircuitBreaker: it should compute a failed call atomically to avoid race conditions. """ @self.breaker def err(): raise Exception() def trigger_error(): for n in range(500): try: err() except: pass def _inc_counter(self): c = self._fail_counter sleep(0.00005) self._fail_counter = c + 1 self._mock_function(self.breaker, _inc_counter) self._start_threads(trigger_error, 3) self.assertEqual(1500, self.breaker.fail_counter) def test_success_thread_safety(self): """CircuitBreaker: it should compute a successful call atomically to avoid race conditions. """ @self.breaker def suc(): return True def trigger_success(): for n in range(500): suc() class SuccessListener(CircuitBreakerListener): def success(self, cb): c = 0 if hasattr(cb, '_success_counter'): c = cb._success_counter sleep(0.00005) cb._success_counter = c + 1 self.breaker.listeners.append(SuccessListener()) self._start_threads(trigger_success, 3) self.assertEqual(1500, self.breaker._success_counter) def test_half_open_thread_safety(self): """CircuitBreaker: it should allow only one trial call when the circuit is half-open. """ self.breaker.half_open() @self.breaker def err(): raise Exception() def trigger_failure(): try: err() except: pass class StateListener(CircuitBreakerListener): def __init__(self): self._count = 0 def before_call(self, cb, fun, *args, **kwargs): sleep(0.00005) def state_change(self, cb, old_state, new_state): if new_state == 'half_open': self._count += 1 state_listener = StateListener() self.breaker.listeners.append(state_listener) self._start_threads(trigger_failure, 5) self.assertEqual(1, state_listener._count) def test_fail_max_thread_safety(self): """CircuitBreaker: it should not allow more failed calls than 'fail_max' setting. """ @self.breaker def err(): raise Exception() def trigger_error(): for i in range(2000): try: err() except: pass class SleepListener(CircuitBreakerListener): def before_call(self, cb, func, *args, **kwargs): sleep(0.00005) self.breaker.listeners.append(SleepListener()) self._start_threads(trigger_error, 3) self.assertEqual(self.breaker.fail_max, self.breaker.fail_counter)
def setUp(self): self.breaker = CircuitBreaker(fail_max=3000, reset_timeout=600)
#!/usr/bin/env python # -*-coding:utf-8-*- from pybreaker import CircuitBreaker from webbreaker.webinspect.authentication import WebInspectAuth from webbreaker.webinspect.common.helper import WebInspectAPIHelper from webbreaker.webinspect.webinspect_config import WebInspectConfig from webbreaker.common.webbreakerlogger import Logger from webbreaker.common.circuitbreakerhelper import APIListener, LogListener # Experimental api_breaker = CircuitBreaker(listeners=[APIListener(), LogListener()]) class WebInspectListScans: def __init__(self, scan_name, server, username, password): self.list_scans(scan_name, server, username, password) @staticmethod def list_scans(scan_name, server, username, password): if server: # if any cli servers were passed. servers = [] for s in server: servers.append(s) else: servers = [(e[0]) for e in WebInspectConfig().endpoints] auth_config = WebInspectAuth() username, password = auth_config.authenticate(username, password) for server in servers:
from app.services import data_handler from app.errors import tag_not_found, item_not_found from app.jaeger import Jaeger from flask import jsonify, request, Blueprint from datetime import datetime from pybreaker import CircuitBreaker hotels_blueprint = Blueprint("hotels_blueprint", __name__) info_breaker = CircuitBreaker(fail_max=5, reset_timeout=30) id_breaker = CircuitBreaker(fail_max=5, reset_timeout=30) breaker = CircuitBreaker(fail_max=5, reset_timeout=30) context = Jaeger() def string_to_array(string): return string.split(",") def get_query_param(key, query_data, func): if key in query_data.keys(): if query_data[key] == "NaN": return None return func(query_data[key]) return None @hotels_blueprint.route("/info/<tag>", methods=["GET"]) def filter_list(tag): """