def __init__(self,
servers,
partition_key=None,
timeout=3,
allow_time_travel=False,
checkpoint=0,
checkpoint_time=0,
max_drift=600,
mark_bad_timeout=60,
max_backend_tries=5):
self._lock = Lock()
self.timeout = timeout
self.backends = Backends(servers, partition_key)
self.checkpoint = checkpoint
self.checkpoint = checkpoint_time
self.allow_time_travel = allow_time_travel
self.max_drift = max_drift
self.mark_bad_timeout = mark_bad_timeout
self.max_backend_tries = max_backend_tries
self.users = Users(self, "users")
self.groups = Groups(self, "groups")
self.permissions = Permissions(self, "permissions")
def test_markdead():
""" Test marking the active server dead."""
backend = Backends([
"localhost:8881",
"localhost:8882",
"localhost:8883",
"localhost:8884",
"localhost:8885",
], "pkey")
assert backend.server == ("localhost", 8884)
backend.mark_dead(backend.server, .2)
assert backend.server == ("localhost", 8885)
time.sleep(.3)
assert backend.server == ("localhost", 8884)
def __init__(self,
servers,
partition_key=None,
timeout=3,
allow_time_travel=False,
checkpoint=0,
checkpoint_time=0,
max_drift=600,
mark_bad_timeout=60,
max_backend_tries=5):
"""
The grouper client.
Args:
servers (list of clowncar.server.Server): available API servers
partition_key (str): key to use for picking a server, None defaults
to hostname
timeout (int): connection and request sent to tornado's HTTPClient
allow_time_travel (bool): allow checkpoint[_time] to go backwards
in subsequent queries
checkpoint (int): starting checkpoint
checkpoint_time (float): starting checkpoint unix epoch time
max_drift (int): how much time in seconds before we consider data
from server to be stale and raise BackendMaxDriftError
mark_bad_timeout (int): time in seconds to not use servers that
have been marked as dead
max_backend_tries (int): number of backend servers to try before
giving up and raising a BackendConnectionError
"""
self._lock = Lock()
self.timeout = timeout
self.backends = Backends(servers, partition_key)
self.checkpoint = Checkpoint(checkpoint, checkpoint_time)
self.allow_time_travel = allow_time_travel
self.max_drift = max_drift
self.mark_bad_timeout = mark_bad_timeout
self.max_backend_tries = max_backend_tries
self.users = Users(self, "users")
self.groups = Groups(self, "groups")
self.permissions = Permissions(self, "permissions")
self.service_accounts = ServiceAccounts(self, "service_accounts")
def test_nobackends():
""" Test when all backends have been marked dead."""
backend = Backends([
"localhost:8881",
"localhost:8882",
], "pkey")
assert backend.server == ("localhost", 8882)
backend.mark_dead(backend.server, .2)
assert backend.server == ("localhost", 8881)
backend.mark_dead(backend.server, .2)
with pytest.raises(NoAvailableBackends):
backend.server
time.sleep(.3)
assert backend.server == ("localhost", 8882)
def __init__(
self,
servers, # type: List[Server]
partition_key=None, # type: Optional[str]
timeout=3, # type: int
allow_time_travel=False, # type: bool
checkpoint=0, # type: int
checkpoint_time=0, # type: float
mark_bad_timeout=60, # type: int
max_backend_tries=5, # type: int
):
# type: (...) -> None
"""
The grouper client.
Args:
servers (list of clowncar.server.Server): available API servers
partition_key (str): key to use for picking a server, None defaults
to hostname
timeout (int): connection and request sent to tornado's HTTPClient
allow_time_travel (bool): allow checkpoint[_time] to go backwards
in subsequent queries
checkpoint (int): starting checkpoint
checkpoint_time (float): starting checkpoint unix epoch time
mark_bad_timeout (int): time in seconds to not use servers that
have been marked as dead
max_backend_tries (int): number of backend servers to try before
giving up and raising a BackendConnectionError
"""
self._lock = Lock()
self.timeout = timeout
self.backends = Backends(servers, partition_key)
self.checkpoint = Checkpoint(checkpoint, checkpoint_time)
self.allow_time_travel = allow_time_travel
self.mark_bad_timeout = mark_bad_timeout
self.max_backend_tries = max_backend_tries
self.users = Users(self, "users")
self.groups = Groups(self, "groups")
self.permissions = Permissions(self, "permissions")
self.service_accounts = ServiceAccounts(self, "service_accounts")
def __init__(self, servers, partition_key=None, timeout=3,
allow_time_travel=False, checkpoint=0, checkpoint_time=0,
max_drift=600, mark_bad_timeout=60, max_backend_tries=5):
self._lock = Lock()
self.timeout = timeout
self.backends = Backends(servers, partition_key)
self.checkpoint = checkpoint
self.checkpoint = checkpoint_time
self.allow_time_travel = allow_time_travel
self.max_drift = max_drift
self.mark_bad_timeout = mark_bad_timeout
self.max_backend_tries = max_backend_tries
self.users = Users(self, "users")
self.groups = Groups(self, "groups")
self.permissions = Permissions(self, "permissions")
class Groupy(object):
def __init__(
self,
servers, # type: List[Server]
partition_key=None, # type: Optional[str]
timeout=3, # type: int
allow_time_travel=False, # type: bool
checkpoint=0, # type: int
checkpoint_time=0, # type: float
mark_bad_timeout=60, # type: int
max_backend_tries=5, # type: int
):
# type: (...) -> None
"""
The grouper client.
Args:
servers (list of clowncar.server.Server): available API servers
partition_key (str): key to use for picking a server, None defaults
to hostname
timeout (int): connection and request sent to tornado's HTTPClient
allow_time_travel (bool): allow checkpoint[_time] to go backwards
in subsequent queries
checkpoint (int): starting checkpoint
checkpoint_time (float): starting checkpoint unix epoch time
mark_bad_timeout (int): time in seconds to not use servers that
have been marked as dead
max_backend_tries (int): number of backend servers to try before
giving up and raising a BackendConnectionError
"""
self._lock = Lock()
self.timeout = timeout
self.backends = Backends(servers, partition_key)
self.checkpoint = Checkpoint(checkpoint, checkpoint_time)
self.allow_time_travel = allow_time_travel
self.mark_bad_timeout = mark_bad_timeout
self.max_backend_tries = max_backend_tries
self.users = Users(self, "users")
self.groups = Groups(self, "groups")
self.permissions = Permissions(self, "permissions")
self.service_accounts = ServiceAccounts(self, "service_accounts")
def _try_fetch(self, path, **kwargs):
# type: (str, **Any) -> Dict[str, Any]
last_failed_server = None
for idx in range(self.max_backend_tries):
try:
return self._fetch(path, **kwargs)
except exc.BackendConnectionError as err:
logging.warning("Marking server {} as dead.".format(err.server.hostname))
self.backends.mark_dead(err.server, self.mark_bad_timeout)
last_failed_server = err.server
raise exc.BackendConnectionError(
"Tried {} servers, all failed.".format(self.max_backend_tries), last_failed_server
)
def _fetch(self, path, **kwargs):
# type: (str, **Any) -> Dict[str, Any]
http_client = HTTPClient()
server = self.backends.server
url = HTTPRequest(
"http://{}:{}{}".format(server.hostname, server.port, path),
connect_timeout=self.timeout,
request_timeout=self.timeout,
**kwargs
)
try:
out = json.loads(http_client.fetch(url).body)
except HTTPError as err:
message = err.message or ""
if err.code == 599:
raise exc.BackendConnectionError(message, server)
if err.response:
try:
out = json.loads(err.response.body)
if "status" not in out:
raise exc.BackendIntegrityError(message, server)
except (ValueError, TypeError):
raise exc.BackendIntegrityError(message, server)
else:
raise exc.BackendIntegrityError(message, server)
except socket.error as err:
if err.errno == errno.ECONNREFUSED:
raise exc.BackendConnectionError("socket error (Connection Refused)", server)
raise
with self._lock:
new_checkpoint = Checkpoint(out["checkpoint"], out["checkpoint_time"])
old_checkpoint = self.checkpoint
if (
not _checkpoint_is_greater(new_checkpoint, old_checkpoint)
and not self.allow_time_travel
):
raise exc.TimeTravelNotAllowed(
"Received checkpoint of {} when previously {}".format(
new_checkpoint, old_checkpoint
),
server,
)
self.checkpoint = new_checkpoint
return out
def authenticate(self, token):
# type: (str) -> Dict[str, Any]
return self._try_fetch("/token/validate", method="POST", body=urlencode({"token": token}))
class Groupy(object):
def __init__(self,
servers,
partition_key=None,
timeout=3,
allow_time_travel=False,
checkpoint=0,
checkpoint_time=0,
max_drift=600,
mark_bad_timeout=60,
max_backend_tries=5):
self._lock = Lock()
self.timeout = timeout
self.backends = Backends(servers, partition_key)
self.checkpoint = checkpoint
self.checkpoint = checkpoint_time
self.allow_time_travel = allow_time_travel
self.max_drift = max_drift
self.mark_bad_timeout = mark_bad_timeout
self.max_backend_tries = max_backend_tries
self.users = Users(self, "users")
self.groups = Groups(self, "groups")
self.permissions = Permissions(self, "permissions")
def _try_get(self, path):
for idx in range(self.max_backend_tries):
try:
return self._get(path)
except exc.BackendConnectionError as err:
self.backends.mark_dead(err.server, self.mark_bad_timeout)
raise exc.BackendConnectionError(
"Tried {} servers, all failed.".format(self.max_backend_tries),
err.server)
def _get(self, path):
http_client = HTTPClient()
server = self.backends.server
url = "http://{}:{}{}".format(server.hostname, server.port, path)
try:
out = json.loads(
http_client.fetch(
url, **{
"connect_timeout": self.timeout,
"request_timeout": self.timeout,
}).body)
except HTTPError as err:
if err.code == 599:
raise exc.BackendConnectionError(err.message, server)
try:
out = json.loads(err.response.body)
if "status" not in out:
raise exc.BackendIntegrityError(err.message, server)
except (ValueError, TypeError):
raise exc.BackendIntegrityError(err.message, server)
now = time.time()
drift = now - out["checkpoint_time"]
if self.max_drift is not None and self.max_drift > abs(drift):
raise exc.BackendMaxDriftError(
"Backend last checkpoint stale by {} seconds.".format(drift),
server)
with self._lock:
new_checkpoint = out["checkpoint"]
old_checkpoint = self.checkpoint
if new_checkpoint < old_checkpoint and not self.allow_time_travel:
raise exc.TimeTravelNotAllowed(
"Received checkpoint of {} when previously {}".format(
new_checkpoint, old_checkpoint), server)
self.checkpoint = new_checkpoint
return out
class Groupy(object):
def __init__(self,
servers,
partition_key=None,
timeout=3,
allow_time_travel=False,
checkpoint=0,
checkpoint_time=0,
mark_bad_timeout=60,
max_backend_tries=5):
"""
The grouper client.
Args:
servers (list of clowncar.server.Server): available API servers
partition_key (str): key to use for picking a server, None defaults
to hostname
timeout (int): connection and request sent to tornado's HTTPClient
allow_time_travel (bool): allow checkpoint[_time] to go backwards
in subsequent queries
checkpoint (int): starting checkpoint
checkpoint_time (float): starting checkpoint unix epoch time
mark_bad_timeout (int): time in seconds to not use servers that
have been marked as dead
max_backend_tries (int): number of backend servers to try before
giving up and raising a BackendConnectionError
"""
self._lock = Lock()
self.timeout = timeout
self.backends = Backends(servers, partition_key)
self.checkpoint = Checkpoint(checkpoint, checkpoint_time)
self.allow_time_travel = allow_time_travel
self.mark_bad_timeout = mark_bad_timeout
self.max_backend_tries = max_backend_tries
self.users = Users(self, "users")
self.groups = Groups(self, "groups")
self.permissions = Permissions(self, "permissions")
self.service_accounts = ServiceAccounts(self, "service_accounts")
def _try_fetch(self, path, **kwargs):
for idx in range(self.max_backend_tries):
try:
return self._fetch(path, **kwargs)
except exc.BackendConnectionError as err:
logging.warning("Marking server {} as dead.".format(
err.server.hostname))
self.backends.mark_dead(err.server, self.mark_bad_timeout)
raise exc.BackendConnectionError(
"Tried {} servers, all failed.".format(self.max_backend_tries),
err.server)
def _fetch(self, path, **kwargs):
http_client = HTTPClient()
server = self.backends.server
url = HTTPRequest("http://{}:{}{}".format(server.hostname, server.port,
path),
connect_timeout=self.timeout,
request_timeout=self.timeout,
**kwargs)
try:
out = json.loads(http_client.fetch(url).body)
except HTTPError as err:
if err.code == 599:
raise exc.BackendConnectionError(err.message, server)
try:
out = json.loads(err.response.body)
if "status" not in out:
raise exc.BackendIntegrityError(err.message, server)
except (ValueError, TypeError):
raise exc.BackendIntegrityError(err.message, server)
with self._lock:
new_checkpoint = Checkpoint(out["checkpoint"],
out["checkpoint_time"])
old_checkpoint = self.checkpoint
if not _checkpoint_is_greater(new_checkpoint, old_checkpoint) and \
not self.allow_time_travel:
raise exc.TimeTravelNotAllowed(
"Received checkpoint of {} when previously {}".format(
new_checkpoint, old_checkpoint), server)
self.checkpoint = new_checkpoint
return out
def authenticate(self, token):
return self._try_fetch('/token/validate',
method='POST',
body=urllib.urlencode({
"token": token,
}))
class Groupy(object):
def __init__(self, servers, partition_key=None, timeout=3,
allow_time_travel=False, checkpoint=0, checkpoint_time=0,
max_drift=600, mark_bad_timeout=60, max_backend_tries=5):
self._lock = Lock()
self.timeout = timeout
self.backends = Backends(servers, partition_key)
self.checkpoint = checkpoint
self.checkpoint = checkpoint_time
self.allow_time_travel = allow_time_travel
self.max_drift = max_drift
self.mark_bad_timeout = mark_bad_timeout
self.max_backend_tries = max_backend_tries
self.users = Users(self, "users")
self.groups = Groups(self, "groups")
self.permissions = Permissions(self, "permissions")
def _try_get(self, path):
for idx in range(self.max_backend_tries):
try:
return self._get(path)
except exc.BackendConnectionError as err:
self.backends.mark_dead(err.server, self.mark_bad_timeout)
raise exc.BackendConnectionError(
"Tried {} servers, all failed.".format(self.max_backend_tries),
err.server
)
def _get(self, path):
http_client = HTTPClient()
server = self.backends.server
url = "http://{}:{}{}".format(server.hostname, server.port, path)
try:
out = json.loads(http_client.fetch(url, **{
"connect_timeout": self.timeout,
"request_timeout": self.timeout,
}).body)
except HTTPError as err:
if err.code == 599:
raise exc.BackendConnectionError(err.message, server)
try:
out = json.loads(err.response.body)
if "status" not in out:
raise exc.BackendIntegrityError(err.message, server)
except (ValueError, TypeError):
raise exc.BackendIntegrityError(err.message, server)
now = time.time()
drift = now - out["checkpoint_time"]
if self.max_drift is not None and self.max_drift > abs(drift):
raise exc.BackendMaxDriftError(
"Backend last checkpoint stale by {} seconds.".format(drift),
server
)
with self._lock:
new_checkpoint = out["checkpoint"]
old_checkpoint = self.checkpoint
if new_checkpoint < old_checkpoint and not self.allow_time_travel:
raise exc.TimeTravelNotAllowed(
"Received checkpoint of {} when previously {}".format(
new_checkpoint, old_checkpoint
), server
)
self.checkpoint = new_checkpoint
return out
class Groupy(object):
def __init__(self,
servers,
partition_key=None,
timeout=3,
allow_time_travel=False,
checkpoint=0,
checkpoint_time=0,
max_drift=600,
mark_bad_timeout=60,
max_backend_tries=5):
"""
The grouper client.
Args:
servers (list of clowncar.server.Server): avaialble API servers
partition_key (str): key to use for picking a server, None defaults
to hostname
timeout (int): connection and request sent to tornado's HTTPClient
allow_time_travel (bool): allow checkpoint[_time] to go backwards
in subsequent queries
checkpoint (int): starting checkpoint
checkpoint_time (float): starting checkpoint unix epoch time
max_drift (int): how much time in seconds before we consider data
from server to be stale and raise BackendMaxDriftError
mark_bad_timeout (int): time in seconds to not use servers that
have been marked as dead
max_backend_tries (int): number of backend servers to try before
giving up and raising a BackendConnectionError
"""
self._lock = Lock()
self.timeout = timeout
self.backends = Backends(servers, partition_key)
self.checkpoint = Checkpoint(checkpoint, checkpoint_time)
self.allow_time_travel = allow_time_travel
self.max_drift = max_drift
self.mark_bad_timeout = mark_bad_timeout
self.max_backend_tries = max_backend_tries
self.users = Users(self, "users")
self.groups = Groups(self, "groups")
self.permissions = Permissions(self, "permissions")
def _checkpoint_is_greater(self, a, b):
"""Ensure elements of checkpoint 'a' are all greater than those in
checkpoint 'b'."""
return all((x > y) for x, y in zip(a, b))
def _try_get(self, path):
for idx in range(self.max_backend_tries):
try:
return self._get(path)
except exc.BackendConnectionError as err:
self.backends.mark_dead(err.server, self.mark_bad_timeout)
raise exc.BackendConnectionError(
"Tried {} servers, all failed.".format(self.max_backend_tries),
err.server)
def _get(self, path):
http_client = HTTPClient()
server = self.backends.server
url = "http://{}:{}{}".format(server.hostname, server.port, path)
try:
out = json.loads(
http_client.fetch(
url, **{
"connect_timeout": self.timeout,
"request_timeout": self.timeout,
}).body)
except HTTPError as err:
if err.code == 599:
raise exc.BackendConnectionError(err.message, server)
try:
out = json.loads(err.response.body)
if "status" not in out:
raise exc.BackendIntegrityError(err.message, server)
except (ValueError, TypeError):
raise exc.BackendIntegrityError(err.message, server)
now = time.time()
drift = now - out["checkpoint_time"]
if self.max_drift is not None and self.max_drift > abs(drift):
raise exc.BackendMaxDriftError(
"Backend last checkpoint stale by {} seconds.".format(drift),
server)
with self._lock:
new_checkpoint = Checkpoint(out["checkpoint"],
out["checkpoint_time"])
old_checkpoint = self.checkpoint
if not self._checkpoint_is_greater(
new_checkpoint,
old_checkpoint) and not self.allow_time_travel:
raise exc.TimeTravelNotAllowed(
"Received checkpoint of {} when previously {}".format(
new_checkpoint, old_checkpoint), server)
self.checkpoint = new_checkpoint
return out