def api_execute(self, path, method, params=None, **kw): """ Executes the query. """ if not path.startswith('/'): raise ValueError('Path does not start with /') url = self._base_uri + path if (method == self._MGET) or (method == self._MDELETE): return self._client.request( method, url, params=params, auth=self._get_auth(), allow_redirects=self.allow_redirect, **kw ) elif (method == self._MPUT) or (method == self._MPOST): return self._client.request( method, url, data=params, auth=self._get_auth(), allow_redirects=self.allow_redirect, **kw ) else: raise etcd.EtcdException( 'HTTP method {} not supported'.format(method))
async def _result_from_response(self, response): """ Creates an EtcdResult from json dictionary """ raw_response = await response.read() try: res = await response.json() except (TypeError, ValueError, UnicodeError) as e: raise etcd.EtcdException('Server response was not valid JSON: %r', raw_response) _log.debug("result: %s", res) try: r = etcd.EtcdResult(**res) if response.status == 201: r.newKey = True r.parse_headers(response) return r except Exception as e: raise etcd.EtcdException('Unable to decode server response') from e
async def _stats(self, what='self'): """ Internal method to access the stats endpoints""" data = await self.api_execute(self.version_prefix + '/stats/' + what, self._MGET) data = await data.read() data = data.decode('utf-8') try: return json.loads(data) except (TypeError,ValueError) as e: raise etcd.EtcdException("Cannot parse json data in the response") from e
async def leader(self, **kw): """ Returns: dict. the leader of the cluster. >>> print (loop.run_until_complete(client.leader())) {"id":"ce2a822cea30bfca","name":"default","peerURLs":["http://localhost:2380","http://localhost:7001"],"clientURLs":["http://127.0.0.1:4001"]} """ try: response = await self.api_execute(self.version_prefix + '/stats/self', self._MGET, **kw) data = await response.read() leader = json.loads(data.decode('utf-8')) return (await self.members())[leader['leaderInfo']['leader']] except Exception as exc: raise etcd.EtcdException("Cannot get leader data") from exc
async def members(self, **kw): """ A more structured view of peers in the cluster. """ # Empty the members list self._members = {} try: response = await self.api_execute(self.version_prefix + '/members', self._MGET, **kw) data = await response.read() res = json.loads(data.decode('utf-8')) for member in res['members']: self._members[member['id']] = member return self._members except Exception as e: raise etcd.EtcdException("Could not get the members list, maybe the cluster has gone away?") from e
async def delete(self): try: await self.client.api_execute(self.uri, self.client._MDELETE) except etcd.EtcdInsufficientPermissions as e: _log.error( "Any action on the authorization requires the root role") raise except etcd.EtcdKeyNotFound: _log.info("%s '%s' not found", self.entity, self.name) raise except Exception as e: _log.error("Failed to delete %s in %s%s: %r", self.entity, self._base_uri, self.version_prefix, e) raise etcd.EtcdException("Could not delete {} '{}'".format( self.entity, self.name)) from e
async def read(self): try: response = await self.client.api_execute(self.uri, self.client._MGET) except etcd.EtcdInsufficientPermissions as e: _log.error( "Any action on the authorization requires the root role") raise except etcd.EtcdKeyNotFound: _log.info("%s '%s' not found", self.entity, self.name) raise except Exception as e: _log.error("Failed to fetch %s in %s%s: %r", self.entity, self.client._base_uri, self.client.version_prefix, e) raise etcd.EtcdException("Could not fetch {} '{}'".format( self.entity, self.name)) from e self._from_net((await response.read()))
async def machines(self): """ Members of the cluster. Returns: list. str with all the nodes in the cluster. >>> print client.machines ['http://127.0.0.1:4001', 'http://127.0.0.1:4002'] """ # We can't use api_execute here, or it causes a logical loop retries = 0 while retries < 5: for m in [self._base_uri] + self._machines_cache: try: uri = m + self.version_prefix + '/machines' response = await self._client.request( self._MGET, uri, allow_redirects=self.allow_redirect, ) response = await self._handle_server_response(response) response = await response.read() if response != b"": machines = [ node.strip() for node in response.decode('utf-8').split(',') ] _log.debug("Retrieved list of machines: %s", machines) return machines except (HTTPException, socket.error, DisconnectedError, ClientConnectionError) as e: # We can't get the list of machines, if one server is in the # machines cache, try on it _log.error("Failed to get list of machines from %s%s: %r", self._base_uri, self.version_prefix, e) retries += 1 await asyncio.sleep(retries / 10, loop=self._loop) raise etcd.EtcdException("Could not get the list of servers, " "maybe you provided the wrong " "host(s) to connect to?")
async def members(self): """ A more structured view of peers in the cluster. Note that while we have an internal DS called _members, accessing the public property will call etcd. """ # Empty the members list self._members = {} try: response = await self.api_execute(self.version_prefix + '/members', self._MGET) data = await response.read() res = json.loads(data.decode('utf-8')) for member in res['members']: self._members[member['id']] = member return self._members except Exception as e: raise etcd.EtcdException( "Could not get the members list, maybe the cluster has gone away?" ) from e
async def write(self): try: r = self.__class__(self.client, self.name) await r.read() except etcd.EtcdKeyNotFound: r = None try: for payload in self._to_net(r): response = await self.client.api_execute_json( self.uri, self.client._MPUT, params=payload) # This will fail if the response is an error self._from_net(await response.read()) except etcd.EtcdInsufficientPermissions as e: _log.error( "Any action on the authorization requires the root role") raise except Exception as e: _log.error("Failed to write %s '%s'", self.entity, self.name) # TODO: fine-grained exception handling raise raise etcd.EtcdException("Could not write {} '{}': {}".format( self.entity, self.name, e)) from e
def __init__( self, host='127.0.0.1', port=2379, srv_domain=None, version_prefix='/v2', allow_redirect=True, protocol='http', cert=None, ca_cert=None, username=None, password=None, allow_reconnect=False, use_proxies=False, expected_cluster_id=None, per_host_pool_size=10, ssl_verify=ssl.CERT_REQUIRED, loop=None, lock_prefix="/_locks" ): """ Initialize the client. Args: host (mixed): If a string, IP to connect to. If a tuple ((host, port), (host, port), ...) port (int): Port used to connect to etcd. srv_domain (str): Domain to search the SRV record for cluster autodiscovery. version_prefix (str): Url or version prefix in etcd url (default=/v2). allow_redirect (bool): allow the client to connect to other nodes. protocol (str): Protocol used to connect to etcd. cert (mixed): If a string, the whole ssl client certificate; if a tuple, the cert and key file names. ca_cert (str): The ca certificate. If pressent it will enable validation. username (str): username for etcd authentication. password (str): password for etcd authentication. allow_reconnect (bool): allow the client to reconnect to another etcd server in the cluster in the case the default one does not respond. use_proxies (bool): we are using a list of proxies to which we connect, and don't want to connect to the original etcd cluster. expected_cluster_id (str): If a string, recorded as the expected UUID of the cluster (rather than learning it from the first request), reads will raise EtcdClusterIdChanged if they receive a response with a different cluster ID. per_host_pool_size (int): specifies maximum number of connections to pool by host. By default this will use up to 10 connections. lock_prefix (str): Set the key prefix at etcd when client to lock object. By default this will be use /_locks. """ # If a DNS record is provided, use it to get the hosts list if srv_domain is not None: try: host = self._discover(srv_domain) except Exception as e: _log.error("Could not discover the etcd hosts from %s: %s", srv_domain, e) self._protocol = protocol self._loop = loop if loop is not None else asyncio.get_event_loop() def uri(protocol, host, port): return '%s://%s:%d' % (protocol, host, port) if not isinstance(host, tuple): self._machines_cache = [] self._base_uri = uri(self._protocol, host, port) else: if not allow_reconnect: _log.error("List of hosts incompatible with allow_reconnect.") raise etcd.EtcdException("A list of hosts to connect to was given, but reconnection not allowed?") self._machines_cache = [uri(self._protocol, *conn) for conn in host] self._base_uri = self._machines_cache.pop(0) self.expected_cluster_id = expected_cluster_id self.version_prefix = version_prefix self._allow_redirect = allow_redirect self._use_proxies = use_proxies self._allow_reconnect = allow_reconnect self._lock_prefix = lock_prefix # SSL Client certificate support ssl_ctx = ssl.create_default_context() if protocol == 'https': # If we don't allow TLSv1, clients using older version of OpenSSL # (<1.0) won't be able to connect. _log.debug("HTTPS enabled.") #kw['ssl_version'] = ssl.PROTOCOL_TLSv1 if ssl_verify == ssl.CERT_NONE: ssl_ctx.check_hostname = False ssl_ctx.verify_mode = ssl_verify if cert: if isinstance(cert, tuple): # Key and cert are separate ssl_ctx.load_cert_chain(*cert) else: ssl_ctx.load_cert_chain(cert) if ca_cert: ssl_ctx.load_verify_locations(ca_cert) self.username = None self.password = None if username and password: self.username = username self.password = password elif username: _log.warning('Username provided without password, both are required for authentication') elif password: _log.warning('Password provided without username, both are required for authentication') if self._allow_reconnect: # we need the set of servers in the cluster in order to try # reconnecting upon error. The cluster members will be # added to the hosts list you provided. If you are using # proxies, set all # # Beware though: if you input '127.0.0.1' as your host and # etcd advertises 'localhost', both will be in the # resulting list. # If we're connecting to the original cluster, we can # extend the list given to the client with what we get # from self.machines self._machines_available = self._use_proxies self._machines_cache = list(set(self._machines_cache)) if self._base_uri in self._machines_cache: self._machines_cache.remove(self._base_uri) _log.debug("Machines cache initialised to %s", self._machines_cache) else: self._machines_available = True if protocol == "https": conn = aiohttp.TCPConnector(ssl_context=ssl_ctx, loop=loop) self._client = aiohttp.ClientSession(connector=conn, loop=loop) else: self._client = aiohttp.ClientSession(loop=loop)
async def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs): """ Writes the value for a key, possibly doing atomic Compare-and-Swap Args: key (str): Key. value (object): value to set ttl (int): Time in seconds of expiration (optional). dir (bool): Set to true if we are writing a directory; default is false. append (bool): If true, it will post to append the new value to the dir, creating a sequential key. Defaults to false. Other parameters modifying the write method are accepted: prevValue (str): compare key to this value, and swap only if corresponding (optional). prevIndex (int): modify key only if actual modifiedIndex matches the provided one (optional). prevExist (bool): If false, only create key; if true, only update key. refresh (bool): since 2.3.0, If true, only update the ttl, prev key must existed(prevExist=True). Returns: client.EtcdResult >>> print client.write('/key', 'newValue', ttl=60, prevExist=False).value 'newValue' """ _log.debug("Writing %s to key %s ttl=%s dir=%s append=%s %s", value, key, ttl, dir, append, kwdargs) key = self._sanitize_key(key) params = {} if value is not None: params['value'] = value if ttl is not None: params['ttl'] = ttl if dir: if value: raise etcd.EtcdException( 'Cannot create a directory with a value') params['dir'] = "true" kw = {} for (k, v) in kwdargs.items(): if k in self._comparison_conditions: if type(v) == bool: params[k] = v and "true" or "false" else: params[k] = v else: kw[k] = v method = append and self._MPOST or self._MPUT if '_endpoint' in kwdargs: path = kwdargs['_endpoint'] + key else: path = self.key_endpoint + key response = await self.api_execute(path, method, params=params, **kw) return (await self._result_from_response(response))