def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# TODO allow user to choose it's own user agent
storefront_header = "storefront ({commit_hash};{environment})".format(
commit_hash=os.getenv("COMMIT_ID", "commit_id"),
environment=os.getenv("ENVIRONMENT", "devel"),
)
headers = {"User-Agent": storefront_header}
self.headers.update(headers)
self.api_breaker = CircuitBreaker(fail_max=5, reset_timeout=60)
class BaseSession:
"""A base session interface to implement common functionality
Create an interface to manage exceptions and return API exceptions
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# TODO allow user to choose it's own user agent
storefront_header = "storefront ({commit_hash};{environment})".format(
commit_hash=os.getenv("COMMIT_ID", "commit_id"),
environment=os.getenv("ENVIRONMENT", "devel"),
)
headers = {"User-Agent": storefront_header}
self.headers.update(headers)
self.api_breaker = CircuitBreaker(fail_max=5, reset_timeout=60)
def request(self, method, url, **kwargs):
try:
request = self.api_breaker.call(super().request,
method=method,
url=url,
**kwargs)
except requests.exceptions.Timeout:
raise ApiTimeoutError(
"The request to {} took too long".format(url))
except requests.exceptions.ConnectionError:
raise ApiConnectionError(
"Failed to establish connection to {}.".format(url))
except CircuitBreakerError:
raise ApiCircuitBreaker(
"Requests are closed because of too many failures".format(url))
return request
class BaseSession:
"""A base session interface to implement common functionality
Create an interface to manage exceptions and return API exceptions
"""
def __init__(self, timeout=(0.5, 3), *args, **kwargs):
super().__init__(*args, **kwargs)
self.mount("http://", TimeoutHTTPAdapter(timeout=timeout))
self.mount("https://", TimeoutHTTPAdapter(timeout=timeout))
# TODO allow user to choose it's own user agent
storefront_header = "storefront ({commit_hash};{environment})".format(
commit_hash=os.getenv("COMMIT_ID", "commit_id"),
environment=os.getenv("ENVIRONMENT", "devel"),
)
headers = {"User-Agent": storefront_header}
self.headers.update(headers)
self.api_breaker = CircuitBreaker(fail_max=5, reset_timeout=60)
def request(self, method, url, **kwargs):
domain = urlparse(url).netloc
try:
request = self.api_breaker.call(
super().request, method=method, url=url, **kwargs
)
except requests.exceptions.Timeout:
timeout_counter.labels(domain=domain).inc()
raise ApiTimeoutError(
"The request to {} took too long".format(url)
)
except requests.exceptions.ConnectionError:
connection_failed_counter.labels(domain=domain).inc()
raise ApiConnectionError(
"Failed to establish connection to {}.".format(url)
)
except CircuitBreakerError:
raise ApiCircuitBreaker(
"Requests are closed because of too many failures".format(url)
)
latency_histogram.labels(
domain=domain, code=request.status_code
).observe(request.elapsed.total_seconds())
return request
def __init__(self, timeout=(0.5, 3), *args, **kwargs):
super().__init__(*args, **kwargs)
self.mount("http://", TimeoutHTTPAdapter(timeout=timeout))
self.mount("https://", TimeoutHTTPAdapter(timeout=timeout))
# TODO allow user to choose it's own user agent
storefront_header = "storefront ({commit_hash};{environment})".format(
commit_hash=os.getenv("COMMIT_ID", "commit_id"),
environment=os.getenv("ENVIRONMENT", "devel"),
)
headers = {"User-Agent": storefront_header}
self.headers.update(headers)
self.api_breaker = CircuitBreaker(fail_max=5, reset_timeout=60)
class BaseSession:
"""A base session interface to implement common functionality
Create an interface to manage exceptions and return API exceptions
"""
def __init__(self, timeout=(0.5, 3), *args, **kwargs):
super().__init__(*args, **kwargs)
self.mount("http://", TimeoutHTTPAdapter(timeout=timeout))
self.mount("https://", TimeoutHTTPAdapter(timeout=timeout))
# TODO allow user to choose it's own user agent
storefront_header = "storefront ({commit_hash};{environment})".format(
commit_hash=os.getenv("COMMIT_ID", "commit_id"),
environment=os.getenv("ENVIRONMENT", "devel"),
)
headers = {"User-Agent": storefront_header}
self.headers.update(headers)
self.api_breaker = CircuitBreaker(fail_max=5, reset_timeout=60)
def request(self, method, url, **kwargs):
domain = urlparse(url).netloc
try:
request = self.api_breaker.call(super().request,
method=method,
url=url,
**kwargs)
except requests.exceptions.Timeout:
timeout_counter.labels(domain=domain).inc()
raise ApiTimeoutError(
"The request to {} took too long".format(url))
except requests.exceptions.ConnectionError:
connection_failed_counter.labels(domain=domain).inc()
raise ApiConnectionError(
"Failed to establish connection to {}.".format(url))
except CircuitBreakerError:
raise ApiCircuitBreaker(
"Requests are closed because of too many failures".format(url))
latency_histogram.labels(domain=domain,
code=request.status_code).observe(
request.elapsed.total_seconds())
return request
class CircuitBreakerThreadsTestCase(unittest.TestCase):
"""
Tests to reproduce common synchronization errors on CircuitBreaker class.
"""
def setUp(self):
self.breaker = CircuitBreaker(fail_max=3000, reset_timeout=600)
def _start_threads(self, target, n):
"""
Starts `n` threads that calls `target` and waits for them to finish.
"""
threads = [threading.Thread(target=target) for i in range(n)]
[t.start() for t in threads]
[t.join() for t in threads]
def _mock_function(self, obj, func):
"""
Replaces a bounded function in `self.breaker` by another.
"""
setattr(obj, func.__name__, MethodType(func, self.breaker))
def test_fail_thread_safety_context_manager(self):
"""CircuitBreaker: it should compute a failed call atomically to avoid race conditions, even when using context manager
"""
def trigger_error():
for n in range(500):
try:
with self.breaker.context():
raise Exception()
except: pass
def _inc_counter(self):
c = self._fail_counter
sleep(0.00005)
self._fail_counter = c + 1
self._mock_function(self.breaker, _inc_counter)
self._start_threads(trigger_error, 3)
self.assertEqual(1500, self.breaker.fail_counter)
def test_fail_thread_safety(self):
"""CircuitBreaker: it should compute a failed call atomically to avoid race conditions.
"""
@self.breaker
def err(): raise Exception()
def trigger_error():
for n in range(500):
try: err()
except: pass
def _inc_counter(self):
c = self._fail_counter
sleep(0.00005)
self._fail_counter = c + 1
self._mock_function(self.breaker, _inc_counter)
self._start_threads(trigger_error, 3)
self.assertEqual(1500, self.breaker.fail_counter)
def test_success_thread_safety(self):
"""CircuitBreaker: it should compute a successful call atomically to avoid race conditions.
"""
@self.breaker
def suc(): return True
def trigger_success():
for n in range(500):
suc()
class SuccessListener(CircuitBreakerListener):
def success(self, cb):
c = 0
if hasattr(cb, '_success_counter'):
c = cb._success_counter
sleep(0.00005)
cb._success_counter = c + 1
self.breaker.listeners.append(SuccessListener())
self._start_threads(trigger_success, 3)
self.assertEqual(1500, self.breaker._success_counter)
def test_half_open_thread_safety(self):
"""CircuitBreaker: it should allow only one trial call when the circuit is half-open.
"""
self.breaker.half_open()
@self.breaker
def err(): raise Exception()
def trigger_failure():
try: err()
except: pass
class StateListener(CircuitBreakerListener):
def __init__(self):
self._count = 0
def before_call(self, cb, fun, *args, **kwargs):
sleep(0.00005)
def state_change(self, cb, old_state, new_state):
if new_state == 'half_open':
self._count += 1
state_listener = StateListener()
self.breaker.listeners.append(state_listener)
self._start_threads(trigger_failure, 5)
self.assertEqual(1, state_listener._count)
def test_fail_max_thread_safety(self):
"""CircuitBreaker: it should not allow more failed calls than 'fail_max' setting.
"""
@self.breaker
def err(): raise Exception()
def trigger_error():
for i in range(2000):
try: err()
except: pass
class SleepListener(CircuitBreakerListener):
def before_call(self, cb, func, *args, **kwargs):
sleep(0.00005)
self.breaker.listeners.append(SleepListener())
self._start_threads(trigger_error, 3)
self.assertEqual(self.breaker.fail_max, self.breaker.fail_counter)
def setUp(self):
self.breaker = CircuitBreaker(fail_max=3000, reset_timeout=600)
#!/usr/bin/env python
# -*-coding:utf-8-*-
from pybreaker import CircuitBreaker
from webbreaker.webinspect.authentication import WebInspectAuth
from webbreaker.webinspect.common.helper import WebInspectAPIHelper
from webbreaker.webinspect.webinspect_config import WebInspectConfig
from webbreaker.common.webbreakerlogger import Logger
from webbreaker.common.circuitbreakerhelper import APIListener, LogListener
# Experimental
api_breaker = CircuitBreaker(listeners=[APIListener(), LogListener()])
class WebInspectListScans:
def __init__(self, scan_name, server, username, password):
self.list_scans(scan_name, server, username, password)
@staticmethod
def list_scans(scan_name, server, username, password):
if server: # if any cli servers were passed.
servers = []
for s in server:
servers.append(s)
else:
servers = [(e[0]) for e in WebInspectConfig().endpoints]
auth_config = WebInspectAuth()
username, password = auth_config.authenticate(username, password)
for server in servers:
from app.services import data_handler
from app.errors import tag_not_found, item_not_found
from app.jaeger import Jaeger
from flask import jsonify, request, Blueprint
from datetime import datetime
from pybreaker import CircuitBreaker
hotels_blueprint = Blueprint("hotels_blueprint", __name__)
info_breaker = CircuitBreaker(fail_max=5, reset_timeout=30)
id_breaker = CircuitBreaker(fail_max=5, reset_timeout=30)
breaker = CircuitBreaker(fail_max=5, reset_timeout=30)
context = Jaeger()
def string_to_array(string):
return string.split(",")
def get_query_param(key, query_data, func):
if key in query_data.keys():
if query_data[key] == "NaN":
return None
return func(query_data[key])
return None
@hotels_blueprint.route("/info/<tag>", methods=["GET"])
def filter_list(tag):
"""
