def api_execute(self, path, method, params=None, timeout=None): """ Executes the query. """ if (method == self._MGET) or (method == self._MDELETE): if params: query_str = '?' + urlencode(params) else: query_str = '' return self.request( method, path + query_str, headers=self._get_headers(), timeout=timeout, ) elif (method == self._MPUT) or (method == self._MPOST): req_path = path encoded = b(urlencode(params)) if params else b'' headers = self._get_headers() headers['content-type'] = 'application/x-www-form-urlencoded' headers['content-length'] = len(encoded) if encoded else 0 return self.request( method, req_path, body=encoded, headers=headers, timeout=timeout, ) else: raise etcd_gevent.EtcdException( 'HTTP method {} not supported'.format(method))
def _result_from_response(self, response): """ Creates an EtcdResult from json dictionary """ raw_response = response.read() try: res = json.loads(raw_response.decode('utf-8')) except (TypeError, ValueError, UnicodeError) as e: raise etcd_gevent.EtcdException( 'Server response was not valid JSON: %r' % e) try: r = etcd_gevent.EtcdResult(**res) if response.status_code == 201: r.newKey = True r.parse_headers(response) return r except Exception as e: raise etcd_gevent.EtcdException( 'Unable to decode server response: %r' % e)
def _stats(self, what='self'): """ Internal method to access the stats endpoints""" data = self.api_execute(self.version_prefix + '/stats/' + what, self._MGET).read().decode('utf-8') try: return json.loads(data) except (TypeError, ValueError): raise etcd_gevent.EtcdException( "Cannot parse json data in the response")
def delete(self): try: _ = self.client.api_execute(self.uri, self.client._MDELETE) except etcd_gevent.EtcdInsufficientPermissions as e: _log.error( "Any action on the authorization requires the root role") raise except etcd_gevent.EtcdKeyNotFound: _log.info("%s '%s' not found", self.entity, self.name) raise except Exception as e: _log.error("Failed to delete %s in %s%s: %r", self.entity, self._base_uri, self.version_prefix, e) raise etcd_gevent.EtcdException("Could not delete {} '{}'".format( self.entity, self.name))
def leader(self): """ Returns: dict. the leader of the cluster. >>> print client.leader {"id":"ce2a822cea30bfca","name":"default","peerURLs":["http://localhost:2380","http://localhost:7001"],"clientURLs":["http://127.0.0.1:4001"]} """ try: leader = json.loads( self.api_execute(self.version_prefix + '/stats/self', self._MGET).read().decode('utf-8')) return self.members[leader['leaderInfo']['leader']] except Exception as e: raise etcd_gevent.EtcdException("Cannot get leader data: %s" % e)
def read(self): try: response = self.client.api_execute(self.uri, self.client._MGET) except etcd_gevent.EtcdInsufficientPermissions as e: _log.error( "Any action on the authorization requires the root role") raise except etcd_gevent.EtcdKeyNotFound: _log.info("%s '%s' not found", self.entity, self.name) raise except Exception as e: _log.error("Failed to fetch %s in %s%s: %r", self.entity, self.client._base_uri, self.client.version_prefix, e) raise etcd_gevent.EtcdException("Could not fetch {} '{}'".format( self.entity, self.name)) self._from_net(response.read())
def members(self): """ A more structured view of peers in the cluster. Note that while we have an internal DS called _members, accessing the public property will call etcd. """ # Empty the members list self._members = {} try: data = self.api_execute(self.version_prefix + '/members', self._MGET).read().decode('utf-8') res = json.loads(data) for member in res['members']: self._members[member['id']] = member return self._members except: raise etcd_gevent.EtcdException( "Could not get the members list, maybe the cluster has gone away?" )
def machines(self): """ Members of the cluster. Returns: list. str with all the nodes in the cluster. >>> print client.machines ['http://127.0.0.1:4001', 'http://127.0.0.1:4002'] """ # We can't use api_execute here, or it causes a logical loop try: response = self.request( self._MGET, self.version_prefix + '/machines', headers=self._get_headers(), timeout=self.read_timeout, ) machines = [ node.strip() for node in self._handle_server_response( response).read().decode('utf-8').split(',') ] _log.debug("Retrieved list of machines: %s", machines) return machines except (HTTPException, socket.error) as e: # We can't get the list of machines, if one server is in the # machines cache, try on it _log.error("Failed to get list of machines from %s%s: %r", self._base_uri, self.version_prefix, e) if self._machines_cache: self._base_uri = self._machines_cache.pop(0) _log.info("Retrying on %s", self._base_uri) # Call myself return self.machines else: raise etcd_gevent.EtcdException( "Could not get the list of servers, " "maybe you provided the wrong " "host(s) to connect to?")
def write(self): try: r = self.__class__(self.client, self.name) r.read() except etcd_gevent.EtcdKeyNotFound: r = None try: for payload in self._to_net(r): response = self.client.api_execute_json(self.uri, self.client._MPUT, params=payload) # This will fail if the response is an error self._from_net(response.read()) except etcd_gevent.EtcdInsufficientPermissions as e: _log.error( "Any action on the authorization requires the root role") raise except Exception as e: _log.error("Failed to write %s '%s'", self.entity, self.name) # TODO: fine-grained exception handling raise etcd_gevent.EtcdException( "Could not write {} '{}': {}".format(self.entity, self.name, e))
def __init__(self, host='127.0.0.1', port=4001, srv_domain=None, version_prefix='/v2', read_timeout=60, allow_redirect=True, protocol='http', cert=None, ca_cert=None, username=None, password=None, allow_reconnect=False, use_proxies=False, expected_cluster_id=None, per_host_pool_size=10, lock_prefix="/_locks"): """ Initialize the client. Args: host (mixed): If a string, IP to connect to. If a tuple ((host, port), (host, port), ...) port (int): Port used to connect to etcd. srv_domain (str): Domain to search the SRV record for cluster autodiscovery. version_prefix (str): Url or version prefix in etcd url (default=/v2). read_timeout (int): max seconds to wait for a read. allow_redirect (bool): allow the client to connect to other nodes. protocol (str): Protocol used to connect to etcd. cert (mixed): If a string, the whole ssl client certificate; if a tuple, the cert and key file names. ca_cert (str): The ca certificate. If pressent it will enable validation. username (str): username for etcd authentication. password (str): password for etcd authentication. allow_reconnect (bool): allow the client to reconnect to another etcd server in the cluster in the case the default one does not respond. use_proxies (bool): we are using a list of proxies to which we connect, and don't want to connect to the original etcd cluster. expected_cluster_id (str): If a string, recorded as the expected UUID of the cluster (rather than learning it from the first request), reads will raise EtcdClusterIdChanged if they receive a response with a different cluster ID. per_host_pool_size (int): specifies maximum number of connections to pool by host. By default this will use up to 10 connections. lock_prefix (str): Set the key prefix at etcd when client to lock object. By default this will be use /_locks. """ # If a DNS record is provided, use it to get the hosts list if srv_domain is not None: try: host = self._discover(srv_domain) except Exception as e: _log.error("Could not discover the etcd hosts from %s: %s", srv_domain, e) self._protocol = protocol def uri(protocol, host, port): return '%s://%s:%d' % (protocol, host, port) if not isinstance(host, tuple): self._machines_cache = [] self._base_uri = uri(self._protocol, host, port) else: if not allow_reconnect: _log.error("List of hosts incompatible with allow_reconnect.") raise etcd_gevent.EtcdException( "A list of hosts to connect to was given, but reconnection not allowed?" ) self._machines_cache = [ uri(self._protocol, *conn) for conn in host ] self._base_uri = self._machines_cache.pop(0) self.expected_cluster_id = expected_cluster_id self.version_prefix = version_prefix self._read_timeout = read_timeout self._allow_redirect = allow_redirect self._use_proxies = use_proxies self._allow_reconnect = allow_reconnect self._lock_prefix = lock_prefix self._per_host_pool_size = per_host_pool_size # SSL Client certificate support self._ssl_options = dict() if cert: if isinstance(cert, tuple): # Key and cert are separate self._ssl_options['cert_file'] = cert[0] self._ssl_options['key_file'] = cert[1] else: # combined certificate self._ssl_options['cert_file'] = cert if ca_cert: self._ssl_options['ca_certs'] = ca_cert self._ssl_options['cert_reqs'] = ssl.CERT_REQUIRED self.username = None self.password = None if username and password: self.username = username self.password = password elif username: _log.warning( 'Username provided without password, both are required for authentication' ) elif password: _log.warning( 'Password provided without username, both are required for authentication' ) self.http_clients = OrderedDict() _log.debug("New etcd client created for %s", self.base_uri) if self._allow_reconnect: # we need the set of servers in the cluster in order to try # reconnecting upon error. The cluster members will be # added to the hosts list you provided. If you are using # proxies, set all # # Beware though: if you input '127.0.0.1' as your host and # etcd advertises 'localhost', both will be in the # resulting list. # If we're connecting to the original cluster, we can # extend the list given to the client with what we get # from self.machines if not self._use_proxies: self._machines_cache = list( set(self._machines_cache) | set(self.machines)) if self._base_uri in self._machines_cache: self._machines_cache.remove(self._base_uri) _log.debug("Machines cache initialised to %s", self._machines_cache) # Versions set to None. They will be set upon first usage. self._version = self._cluster_version = None
def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs): """ Writes the value for a key, possibly doing atomic Compare-and-Swap Args: key (str): Key. value (object): value to set ttl (int): Time in seconds of expiration (optional). dir (bool): Set to true if we are writing a directory; default is false. append (bool): If true, it will post to append the new value to the dir, creating a sequential key. Defaults to false. Other parameters modifying the write method are accepted: prevValue (str): compare key to this value, and swap only if corresponding (optional). prevIndex (int): modify key only if actual modifiedIndex matches the provided one (optional). prevExist (bool): If false, only create key; if true, only update key. refresh (bool): since 2.3.0, If true, only update the ttl, prev key must existed(prevExist=True). Returns: client.EtcdResult >>> print client.write('/key', 'newValue', ttl=60, prevExist=False).value 'newValue' """ _log.debug("Writing %s to key %s ttl=%s dir=%s append=%s", value, key, ttl, dir, append) key = self._sanitize_key(key) params = {} if value is not None: params['value'] = value if ttl is not None: params['ttl'] = ttl if dir: if value: raise etcd_gevent.EtcdException( 'Cannot create a directory with a value') params['dir'] = "true" for (k, v) in kwdargs.items(): if k in self._comparison_conditions: if type(v) == bool: params[k] = v and "true" or "false" else: params[k] = v method = append and self._MPOST or self._MPUT if '_endpoint' in kwdargs: path = kwdargs['_endpoint'] + key else: path = self.key_endpoint + key response = self.api_execute(path, method, params=params) return self._result_from_response(response)