def api_execute(self, path, method, params=None, timeout=None):
""" Executes the query. """
if (method == self._MGET) or (method == self._MDELETE):
if params:
query_str = '?' + urlencode(params)
else:
query_str = ''
return self.request(
method,
path + query_str,
headers=self._get_headers(),
timeout=timeout,
)
elif (method == self._MPUT) or (method == self._MPOST):
req_path = path
encoded = b(urlencode(params)) if params else b''
headers = self._get_headers()
headers['content-type'] = 'application/x-www-form-urlencoded'
headers['content-length'] = len(encoded) if encoded else 0
return self.request(
method,
req_path,
body=encoded,
headers=headers,
timeout=timeout,
)
else:
raise etcd_gevent.EtcdException(
'HTTP method {} not supported'.format(method))
def _result_from_response(self, response):
""" Creates an EtcdResult from json dictionary """
raw_response = response.read()
try:
res = json.loads(raw_response.decode('utf-8'))
except (TypeError, ValueError, UnicodeError) as e:
raise etcd_gevent.EtcdException(
'Server response was not valid JSON: %r' % e)
try:
r = etcd_gevent.EtcdResult(**res)
if response.status_code == 201:
r.newKey = True
r.parse_headers(response)
return r
except Exception as e:
raise etcd_gevent.EtcdException(
'Unable to decode server response: %r' % e)
def _stats(self, what='self'):
""" Internal method to access the stats endpoints"""
data = self.api_execute(self.version_prefix + '/stats/' + what,
self._MGET).read().decode('utf-8')
try:
return json.loads(data)
except (TypeError, ValueError):
raise etcd_gevent.EtcdException(
"Cannot parse json data in the response")
def delete(self):
try:
_ = self.client.api_execute(self.uri, self.client._MDELETE)
except etcd_gevent.EtcdInsufficientPermissions as e:
_log.error(
"Any action on the authorization requires the root role")
raise
except etcd_gevent.EtcdKeyNotFound:
_log.info("%s '%s' not found", self.entity, self.name)
raise
except Exception as e:
_log.error("Failed to delete %s in %s%s: %r", self.entity,
self._base_uri, self.version_prefix, e)
raise etcd_gevent.EtcdException("Could not delete {} '{}'".format(
self.entity, self.name))
def leader(self):
"""
Returns:
dict. the leader of the cluster.
>>> print client.leader
{"id":"ce2a822cea30bfca","name":"default","peerURLs":["http://localhost:2380","http://localhost:7001"],"clientURLs":["http://127.0.0.1:4001"]}
"""
try:
leader = json.loads(
self.api_execute(self.version_prefix + '/stats/self',
self._MGET).read().decode('utf-8'))
return self.members[leader['leaderInfo']['leader']]
except Exception as e:
raise etcd_gevent.EtcdException("Cannot get leader data: %s" % e)
def read(self):
try:
response = self.client.api_execute(self.uri, self.client._MGET)
except etcd_gevent.EtcdInsufficientPermissions as e:
_log.error(
"Any action on the authorization requires the root role")
raise
except etcd_gevent.EtcdKeyNotFound:
_log.info("%s '%s' not found", self.entity, self.name)
raise
except Exception as e:
_log.error("Failed to fetch %s in %s%s: %r", self.entity,
self.client._base_uri, self.client.version_prefix, e)
raise etcd_gevent.EtcdException("Could not fetch {} '{}'".format(
self.entity, self.name))
self._from_net(response.read())
def members(self):
"""
A more structured view of peers in the cluster.
Note that while we have an internal DS called _members, accessing the public property will call etcd.
"""
# Empty the members list
self._members = {}
try:
data = self.api_execute(self.version_prefix + '/members',
self._MGET).read().decode('utf-8')
res = json.loads(data)
for member in res['members']:
self._members[member['id']] = member
return self._members
except:
raise etcd_gevent.EtcdException(
"Could not get the members list, maybe the cluster has gone away?"
)
def machines(self):
"""
Members of the cluster.
Returns:
list. str with all the nodes in the cluster.
>>> print client.machines
['http://127.0.0.1:4001', 'http://127.0.0.1:4002']
"""
# We can't use api_execute here, or it causes a logical loop
try:
response = self.request(
self._MGET,
self.version_prefix + '/machines',
headers=self._get_headers(),
timeout=self.read_timeout,
)
machines = [
node.strip() for node in self._handle_server_response(
response).read().decode('utf-8').split(',')
]
_log.debug("Retrieved list of machines: %s", machines)
return machines
except (HTTPException, socket.error) as e:
# We can't get the list of machines, if one server is in the
# machines cache, try on it
_log.error("Failed to get list of machines from %s%s: %r",
self._base_uri, self.version_prefix, e)
if self._machines_cache:
self._base_uri = self._machines_cache.pop(0)
_log.info("Retrying on %s", self._base_uri)
# Call myself
return self.machines
else:
raise etcd_gevent.EtcdException(
"Could not get the list of servers, "
"maybe you provided the wrong "
"host(s) to connect to?")
def write(self):
try:
r = self.__class__(self.client, self.name)
r.read()
except etcd_gevent.EtcdKeyNotFound:
r = None
try:
for payload in self._to_net(r):
response = self.client.api_execute_json(self.uri,
self.client._MPUT,
params=payload)
# This will fail if the response is an error
self._from_net(response.read())
except etcd_gevent.EtcdInsufficientPermissions as e:
_log.error(
"Any action on the authorization requires the root role")
raise
except Exception as e:
_log.error("Failed to write %s '%s'", self.entity, self.name)
# TODO: fine-grained exception handling
raise etcd_gevent.EtcdException(
"Could not write {} '{}': {}".format(self.entity, self.name,
e))
def __init__(self,
host='127.0.0.1',
port=4001,
srv_domain=None,
version_prefix='/v2',
read_timeout=60,
allow_redirect=True,
protocol='http',
cert=None,
ca_cert=None,
username=None,
password=None,
allow_reconnect=False,
use_proxies=False,
expected_cluster_id=None,
per_host_pool_size=10,
lock_prefix="/_locks"):
"""
Initialize the client.
Args:
host (mixed):
If a string, IP to connect to.
If a tuple ((host, port), (host, port), ...)
port (int): Port used to connect to etcd.
srv_domain (str): Domain to search the SRV record for cluster autodiscovery.
version_prefix (str): Url or version prefix in etcd url (default=/v2).
read_timeout (int): max seconds to wait for a read.
allow_redirect (bool): allow the client to connect to other nodes.
protocol (str): Protocol used to connect to etcd.
cert (mixed): If a string, the whole ssl client certificate;
if a tuple, the cert and key file names.
ca_cert (str): The ca certificate. If pressent it will enable
validation.
username (str): username for etcd authentication.
password (str): password for etcd authentication.
allow_reconnect (bool): allow the client to reconnect to another
etcd server in the cluster in the case the
default one does not respond.
use_proxies (bool): we are using a list of proxies to which we connect,
and don't want to connect to the original etcd cluster.
expected_cluster_id (str): If a string, recorded as the expected
UUID of the cluster (rather than
learning it from the first request),
reads will raise EtcdClusterIdChanged
if they receive a response with a
different cluster ID.
per_host_pool_size (int): specifies maximum number of connections to pool
by host. By default this will use up to 10
connections.
lock_prefix (str): Set the key prefix at etcd when client to lock object.
By default this will be use /_locks.
"""
# If a DNS record is provided, use it to get the hosts list
if srv_domain is not None:
try:
host = self._discover(srv_domain)
except Exception as e:
_log.error("Could not discover the etcd hosts from %s: %s",
srv_domain, e)
self._protocol = protocol
def uri(protocol, host, port):
return '%s://%s:%d' % (protocol, host, port)
if not isinstance(host, tuple):
self._machines_cache = []
self._base_uri = uri(self._protocol, host, port)
else:
if not allow_reconnect:
_log.error("List of hosts incompatible with allow_reconnect.")
raise etcd_gevent.EtcdException(
"A list of hosts to connect to was given, but reconnection not allowed?"
)
self._machines_cache = [
uri(self._protocol, *conn) for conn in host
]
self._base_uri = self._machines_cache.pop(0)
self.expected_cluster_id = expected_cluster_id
self.version_prefix = version_prefix
self._read_timeout = read_timeout
self._allow_redirect = allow_redirect
self._use_proxies = use_proxies
self._allow_reconnect = allow_reconnect
self._lock_prefix = lock_prefix
self._per_host_pool_size = per_host_pool_size
# SSL Client certificate support
self._ssl_options = dict()
if cert:
if isinstance(cert, tuple):
# Key and cert are separate
self._ssl_options['cert_file'] = cert[0]
self._ssl_options['key_file'] = cert[1]
else:
# combined certificate
self._ssl_options['cert_file'] = cert
if ca_cert:
self._ssl_options['ca_certs'] = ca_cert
self._ssl_options['cert_reqs'] = ssl.CERT_REQUIRED
self.username = None
self.password = None
if username and password:
self.username = username
self.password = password
elif username:
_log.warning(
'Username provided without password, both are required for authentication'
)
elif password:
_log.warning(
'Password provided without username, both are required for authentication'
)
self.http_clients = OrderedDict()
_log.debug("New etcd client created for %s", self.base_uri)
if self._allow_reconnect:
# we need the set of servers in the cluster in order to try
# reconnecting upon error. The cluster members will be
# added to the hosts list you provided. If you are using
# proxies, set all
#
# Beware though: if you input '127.0.0.1' as your host and
# etcd advertises 'localhost', both will be in the
# resulting list.
# If we're connecting to the original cluster, we can
# extend the list given to the client with what we get
# from self.machines
if not self._use_proxies:
self._machines_cache = list(
set(self._machines_cache) | set(self.machines))
if self._base_uri in self._machines_cache:
self._machines_cache.remove(self._base_uri)
_log.debug("Machines cache initialised to %s",
self._machines_cache)
# Versions set to None. They will be set upon first usage.
self._version = self._cluster_version = None
def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs):
"""
Writes the value for a key, possibly doing atomic Compare-and-Swap
Args:
key (str): Key.
value (object): value to set
ttl (int): Time in seconds of expiration (optional).
dir (bool): Set to true if we are writing a directory; default is false.
append (bool): If true, it will post to append the new value to the dir, creating a sequential key. Defaults to false.
Other parameters modifying the write method are accepted:
prevValue (str): compare key to this value, and swap only if corresponding (optional).
prevIndex (int): modify key only if actual modifiedIndex matches the provided one (optional).
prevExist (bool): If false, only create key; if true, only update key.
refresh (bool): since 2.3.0, If true, only update the ttl, prev key must existed(prevExist=True).
Returns:
client.EtcdResult
>>> print client.write('/key', 'newValue', ttl=60, prevExist=False).value
'newValue'
"""
_log.debug("Writing %s to key %s ttl=%s dir=%s append=%s", value, key,
ttl, dir, append)
key = self._sanitize_key(key)
params = {}
if value is not None:
params['value'] = value
if ttl is not None:
params['ttl'] = ttl
if dir:
if value:
raise etcd_gevent.EtcdException(
'Cannot create a directory with a value')
params['dir'] = "true"
for (k, v) in kwdargs.items():
if k in self._comparison_conditions:
if type(v) == bool:
params[k] = v and "true" or "false"
else:
params[k] = v
method = append and self._MPOST or self._MPUT
if '_endpoint' in kwdargs:
path = kwdargs['_endpoint'] + key
else:
path = self.key_endpoint + key
response = self.api_execute(path, method, params=params)
return self._result_from_response(response)