def api_execute(self, path, method, params=None, **kw):
""" Executes the query. """
if not path.startswith('/'):
raise ValueError('Path does not start with /')
url = self._base_uri + path
if (method == self._MGET) or (method == self._MDELETE):
return self._client.request(
method,
url,
params=params,
auth=self._get_auth(),
allow_redirects=self.allow_redirect,
**kw
)
elif (method == self._MPUT) or (method == self._MPOST):
return self._client.request(
method,
url,
data=params,
auth=self._get_auth(),
allow_redirects=self.allow_redirect,
**kw
)
else:
raise etcd.EtcdException(
'HTTP method {} not supported'.format(method))
async def _result_from_response(self, response):
""" Creates an EtcdResult from json dictionary """
raw_response = await response.read()
try:
res = await response.json()
except (TypeError, ValueError, UnicodeError) as e:
raise etcd.EtcdException('Server response was not valid JSON: %r',
raw_response)
_log.debug("result: %s", res)
try:
r = etcd.EtcdResult(**res)
if response.status == 201:
r.newKey = True
r.parse_headers(response)
return r
except Exception as e:
raise etcd.EtcdException('Unable to decode server response') from e
async def _stats(self, what='self'):
""" Internal method to access the stats endpoints"""
data = await self.api_execute(self.version_prefix + '/stats/' + what, self._MGET)
data = await data.read()
data = data.decode('utf-8')
try:
return json.loads(data)
except (TypeError,ValueError) as e:
raise etcd.EtcdException("Cannot parse json data in the response") from e
async def leader(self, **kw):
"""
Returns:
dict. the leader of the cluster.
>>> print (loop.run_until_complete(client.leader()))
{"id":"ce2a822cea30bfca","name":"default","peerURLs":["http://localhost:2380","http://localhost:7001"],"clientURLs":["http://127.0.0.1:4001"]}
"""
try:
response = await self.api_execute(self.version_prefix + '/stats/self', self._MGET, **kw)
data = await response.read()
leader = json.loads(data.decode('utf-8'))
return (await self.members())[leader['leaderInfo']['leader']]
except Exception as exc:
raise etcd.EtcdException("Cannot get leader data") from exc
async def members(self, **kw):
"""
A more structured view of peers in the cluster.
"""
# Empty the members list
self._members = {}
try:
response = await self.api_execute(self.version_prefix + '/members', self._MGET, **kw)
data = await response.read()
res = json.loads(data.decode('utf-8'))
for member in res['members']:
self._members[member['id']] = member
return self._members
except Exception as e:
raise etcd.EtcdException("Could not get the members list, maybe the cluster has gone away?") from e
async def delete(self):
try:
await self.client.api_execute(self.uri, self.client._MDELETE)
except etcd.EtcdInsufficientPermissions as e:
_log.error(
"Any action on the authorization requires the root role")
raise
except etcd.EtcdKeyNotFound:
_log.info("%s '%s' not found", self.entity, self.name)
raise
except Exception as e:
_log.error("Failed to delete %s in %s%s: %r", self.entity,
self._base_uri, self.version_prefix, e)
raise etcd.EtcdException("Could not delete {} '{}'".format(
self.entity, self.name)) from e
async def read(self):
try:
response = await self.client.api_execute(self.uri,
self.client._MGET)
except etcd.EtcdInsufficientPermissions as e:
_log.error(
"Any action on the authorization requires the root role")
raise
except etcd.EtcdKeyNotFound:
_log.info("%s '%s' not found", self.entity, self.name)
raise
except Exception as e:
_log.error("Failed to fetch %s in %s%s: %r", self.entity,
self.client._base_uri, self.client.version_prefix, e)
raise etcd.EtcdException("Could not fetch {} '{}'".format(
self.entity, self.name)) from e
self._from_net((await response.read()))
async def machines(self):
"""
Members of the cluster.
Returns:
list. str with all the nodes in the cluster.
>>> print client.machines
['http://127.0.0.1:4001', 'http://127.0.0.1:4002']
"""
# We can't use api_execute here, or it causes a logical loop
retries = 0
while retries < 5:
for m in [self._base_uri] + self._machines_cache:
try:
uri = m + self.version_prefix + '/machines'
response = await self._client.request(
self._MGET,
uri,
allow_redirects=self.allow_redirect,
)
response = await self._handle_server_response(response)
response = await response.read()
if response != b"":
machines = [
node.strip()
for node in response.decode('utf-8').split(',')
]
_log.debug("Retrieved list of machines: %s", machines)
return machines
except (HTTPException, socket.error, DisconnectedError,
ClientConnectionError) as e:
# We can't get the list of machines, if one server is in the
# machines cache, try on it
_log.error("Failed to get list of machines from %s%s: %r",
self._base_uri, self.version_prefix, e)
retries += 1
await asyncio.sleep(retries / 10, loop=self._loop)
raise etcd.EtcdException("Could not get the list of servers, "
"maybe you provided the wrong "
"host(s) to connect to?")
async def members(self):
"""
A more structured view of peers in the cluster.
Note that while we have an internal DS called _members, accessing the public property will call etcd.
"""
# Empty the members list
self._members = {}
try:
response = await self.api_execute(self.version_prefix + '/members',
self._MGET)
data = await response.read()
res = json.loads(data.decode('utf-8'))
for member in res['members']:
self._members[member['id']] = member
return self._members
except Exception as e:
raise etcd.EtcdException(
"Could not get the members list, maybe the cluster has gone away?"
) from e
async def write(self):
try:
r = self.__class__(self.client, self.name)
await r.read()
except etcd.EtcdKeyNotFound:
r = None
try:
for payload in self._to_net(r):
response = await self.client.api_execute_json(
self.uri, self.client._MPUT, params=payload)
# This will fail if the response is an error
self._from_net(await response.read())
except etcd.EtcdInsufficientPermissions as e:
_log.error(
"Any action on the authorization requires the root role")
raise
except Exception as e:
_log.error("Failed to write %s '%s'", self.entity, self.name)
# TODO: fine-grained exception handling
raise
raise etcd.EtcdException("Could not write {} '{}': {}".format(
self.entity, self.name, e)) from e
def __init__(
self,
host='127.0.0.1',
port=2379,
srv_domain=None,
version_prefix='/v2',
allow_redirect=True,
protocol='http',
cert=None,
ca_cert=None,
username=None,
password=None,
allow_reconnect=False,
use_proxies=False,
expected_cluster_id=None,
per_host_pool_size=10,
ssl_verify=ssl.CERT_REQUIRED,
loop=None,
lock_prefix="/_locks"
):
"""
Initialize the client.
Args:
host (mixed):
If a string, IP to connect to.
If a tuple ((host, port), (host, port), ...)
port (int): Port used to connect to etcd.
srv_domain (str): Domain to search the SRV record for cluster autodiscovery.
version_prefix (str): Url or version prefix in etcd url (default=/v2).
allow_redirect (bool): allow the client to connect to other nodes.
protocol (str): Protocol used to connect to etcd.
cert (mixed): If a string, the whole ssl client certificate;
if a tuple, the cert and key file names.
ca_cert (str): The ca certificate. If pressent it will enable
validation.
username (str): username for etcd authentication.
password (str): password for etcd authentication.
allow_reconnect (bool): allow the client to reconnect to another
etcd server in the cluster in the case the
default one does not respond.
use_proxies (bool): we are using a list of proxies to which we connect,
and don't want to connect to the original etcd cluster.
expected_cluster_id (str): If a string, recorded as the expected
UUID of the cluster (rather than
learning it from the first request),
reads will raise EtcdClusterIdChanged
if they receive a response with a
different cluster ID.
per_host_pool_size (int): specifies maximum number of connections to pool
by host. By default this will use up to 10
connections.
lock_prefix (str): Set the key prefix at etcd when client to lock object.
By default this will be use /_locks.
"""
# If a DNS record is provided, use it to get the hosts list
if srv_domain is not None:
try:
host = self._discover(srv_domain)
except Exception as e:
_log.error("Could not discover the etcd hosts from %s: %s",
srv_domain, e)
self._protocol = protocol
self._loop = loop if loop is not None else asyncio.get_event_loop()
def uri(protocol, host, port):
return '%s://%s:%d' % (protocol, host, port)
if not isinstance(host, tuple):
self._machines_cache = []
self._base_uri = uri(self._protocol, host, port)
else:
if not allow_reconnect:
_log.error("List of hosts incompatible with allow_reconnect.")
raise etcd.EtcdException("A list of hosts to connect to was given, but reconnection not allowed?")
self._machines_cache = [uri(self._protocol, *conn) for conn in host]
self._base_uri = self._machines_cache.pop(0)
self.expected_cluster_id = expected_cluster_id
self.version_prefix = version_prefix
self._allow_redirect = allow_redirect
self._use_proxies = use_proxies
self._allow_reconnect = allow_reconnect
self._lock_prefix = lock_prefix
# SSL Client certificate support
ssl_ctx = ssl.create_default_context()
if protocol == 'https':
# If we don't allow TLSv1, clients using older version of OpenSSL
# (<1.0) won't be able to connect.
_log.debug("HTTPS enabled.")
#kw['ssl_version'] = ssl.PROTOCOL_TLSv1
if ssl_verify == ssl.CERT_NONE:
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl_verify
if cert:
if isinstance(cert, tuple):
# Key and cert are separate
ssl_ctx.load_cert_chain(*cert)
else:
ssl_ctx.load_cert_chain(cert)
if ca_cert:
ssl_ctx.load_verify_locations(ca_cert)
self.username = None
self.password = None
if username and password:
self.username = username
self.password = password
elif username:
_log.warning('Username provided without password, both are required for authentication')
elif password:
_log.warning('Password provided without username, both are required for authentication')
if self._allow_reconnect:
# we need the set of servers in the cluster in order to try
# reconnecting upon error. The cluster members will be
# added to the hosts list you provided. If you are using
# proxies, set all
#
# Beware though: if you input '127.0.0.1' as your host and
# etcd advertises 'localhost', both will be in the
# resulting list.
# If we're connecting to the original cluster, we can
# extend the list given to the client with what we get
# from self.machines
self._machines_available = self._use_proxies
self._machines_cache = list(set(self._machines_cache))
if self._base_uri in self._machines_cache:
self._machines_cache.remove(self._base_uri)
_log.debug("Machines cache initialised to %s",
self._machines_cache)
else:
self._machines_available = True
if protocol == "https":
conn = aiohttp.TCPConnector(ssl_context=ssl_ctx, loop=loop)
self._client = aiohttp.ClientSession(connector=conn, loop=loop)
else:
self._client = aiohttp.ClientSession(loop=loop)
async def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs):
"""
Writes the value for a key, possibly doing atomic Compare-and-Swap
Args:
key (str): Key.
value (object): value to set
ttl (int): Time in seconds of expiration (optional).
dir (bool): Set to true if we are writing a directory; default is false.
append (bool): If true, it will post to append the new value to the dir, creating a sequential key. Defaults to false.
Other parameters modifying the write method are accepted:
prevValue (str): compare key to this value, and swap only if corresponding (optional).
prevIndex (int): modify key only if actual modifiedIndex matches the provided one (optional).
prevExist (bool): If false, only create key; if true, only update key.
refresh (bool): since 2.3.0, If true, only update the ttl, prev key must existed(prevExist=True).
Returns:
client.EtcdResult
>>> print client.write('/key', 'newValue', ttl=60, prevExist=False).value
'newValue'
"""
_log.debug("Writing %s to key %s ttl=%s dir=%s append=%s %s",
value, key, ttl, dir, append, kwdargs)
key = self._sanitize_key(key)
params = {}
if value is not None:
params['value'] = value
if ttl is not None:
params['ttl'] = ttl
if dir:
if value:
raise etcd.EtcdException(
'Cannot create a directory with a value')
params['dir'] = "true"
kw = {}
for (k, v) in kwdargs.items():
if k in self._comparison_conditions:
if type(v) == bool:
params[k] = v and "true" or "false"
else:
params[k] = v
else:
kw[k] = v
method = append and self._MPOST or self._MPUT
if '_endpoint' in kwdargs:
path = kwdargs['_endpoint'] + key
else:
path = self.key_endpoint + key
response = await self.api_execute(path, method, params=params, **kw)
return (await self._result_from_response(response))
