def main(reactor): # create an etcd client etcd = Client(reactor) # retrieve etcd cluster status status = yield etcd.status() print(status) # callback invoked for every change def on_change(kv): print('on_change: {}'.format(kv)) # start watching on given keys or key sets # when the key sets overlap, the callback might get called multiple # times, once for each key set matching an event keys = [KeySet(b'mykey2', b'mykey5'), KeySet(b'mykey', prefix=True)] d = etcd.watch(keys, on_change) # stop after 10 seconds print('watching for 1s ..') yield txaio.sleep(1) etcd.set(b'mykey10', b'Test') print('watching for 3s ..') yield txaio.sleep(3) d.cancel()
def main(reactor): etcd = Client(reactor, u'http://localhost:2379') # set key-value etcd.set(b'foo', os.urandom(8)) # set key-value, return revision including previous value revision = yield etcd.set(b'foo', os.urandom(8), return_previous=True) print('previous:', revision.previous) # set values on keys for i in range(10): etcd.set('mykey{}'.format(i).encode(), os.urandom(8)) # get value by key for key in [b'mykey1', KeySet(b'mykey1'), b'mykey13']: result = yield etcd.get(key) if result.kvs: kv = result.kvs[0] print(kv) else: print('key {} not found!'.format(key)) # iterate over keys in range result = yield etcd.get(KeySet(b'mykey1', b'mykey5')) print('KV pairs for range:') for kv in result.kvs: print(kv) # iterate over keys with given prefix result = yield etcd.get(KeySet(b'mykey', prefix=True)) print('KV pairs for prefix:') for kv in result.kvs: print(kv) # delete a single key deleted = yield etcd.delete(b'mykey0') print('deleted {} key-value pairs'.format(deleted.deleted)) # delete non-existing key deleted = yield etcd.delete(os.urandom(8)) print('deleted {} key-value pairs'.format(deleted.deleted)) # delete a key range deleted = yield etcd.delete(KeySet(b'mykey2', b'mykey7')) print('deleted {} key-value pairs'.format(deleted.deleted)) # delete a key set defined by prefix and return deleted key-value pairs deleted = yield etcd.delete(KeySet(b'mykey', prefix=True), return_previous=True) print('deleted {} key-value pairs:'.format(deleted.deleted)) for d in deleted.previous: print(d)
def _list(self, key): err = None list = [] resp = yield self.client.get(KeySet(key, prefix=True)) if resp.kvs is not None and len(resp.kvs) > 0: for kv in resp.kvs: list.append(KVPair(kv.key, kv.value, kv.mod_revision)) returnValue((list, err))
def _op_with_retry(self, operation, key, value, timeout, *args, **kw): log.debug('kv-op', operation=operation, key=key, timeout=timeout, args=args, kw=kw) err = None result = None if type(key) == str: key = bytes(key) if value is not None: value = bytes(value) while True: try: if operation == 'GET': result = yield self._get(key) elif operation == 'LIST': result, err = yield self._list(key) elif operation == 'PUT': # Put returns an object of type Revision result = yield self.client.set(key, value, **kw) elif operation == 'DELETE': # Delete returns an object of type Deleted result = yield self.client.delete(key) elif operation == 'RESERVE': result, err = yield self._reserve(key, value, **kw) elif operation == 'RENEW': result, err = yield self._renew_reservation(key) elif operation == 'RELEASE': result, err = yield self._release_reservation(key) elif operation == 'RELEASE-ALL': err = yield self._release_all_reservations() elif operation == 'WATCH': for name, val in kw.items(): if name == 'callback': callback = val break result = self.client.watch([KeySet(key, prefix=True)], callback) self._clear_backoff() break except ConnectionRefusedError as ex: log.error('comms-exception', ex=ex) yield self._backoff('etcd-not-up') except Exception as ex: log.error('etcd-exception', ex=ex) err = ex if timeout > 0 and self.retry_time > timeout: err = 'operation-timed-out' if err is not None: self._clear_backoff() break returnValue((result, err))
def __validate(self): if type(self._key) == six.binary_type: if self._range_end: self._key = KeySet(self._key, range_end=self._range_end) else: self._key = KeySet(self._key) elif isinstance(self._key, KeySet): pass else: raise TypeError( 'key must either be bytes or a KeySet object, not {}'.format(type(self._key))) if self._key.type == KeySet._SINGLE: self._range_end = None elif self._key.type == KeySet._PREFIX: self._range_end = _increment_last_byte(self._key.key) elif self._key.type == KeySet._RANGE: self._range_end = self._key.range_end else: raise Exception('logic error')
def main(reactor): etcd = Client(reactor) # # Example 1 # for val in [b'val1', b'val2']: yield etcd.set(b'test1', val) txn = Transaction( compare=[ # compute conjunction of all terms to # determine "success" vs "failure" CompValue(b'test1', '==', b'val1') ], success=[ # if true ("success"), run these ops OpSet(b'test1', b'val2'), OpSet(b'test2', b'success') ], failure=[ # if not true ("failure"), run these ops OpSet(b'test2', b'failure'), OpGet(b'test1') ]) try: result = yield etcd.submit(txn) except Failed as failed: print('transaction FAILED:') for response in failed.responses: print(response) else: print('transaction SUCCESS:') for response in result.responses: print(response) for key in [b'test1', b'test2']: value = yield etcd.get(key) print('{}: {}'.format(key, value)) # # Example 2 # rev = yield etcd.set(b'mykey1', os.urandom(8)) print(rev) result = yield etcd.get(b'mykey1') kv = result.kvs[0] print(kv) for version in [kv.version, kv.version - 1]: txn = Transaction( compare=[ # value equality comparison CompValue(b'mykey1', '==', kv.value), # version, and different comparison operators CompVersion(b'mykey1', '==', version), CompVersion(b'mykey1', '!=', version + 1), CompVersion(b'mykey1', '<', version + 1), CompVersion(b'mykey1', '>', version - 1), # created revision comparison CompCreated(b'mykey1', '==', kv.create_revision), # modified revision comparison CompModified(b'mykey1', '==', kv.mod_revision), ], success=[ OpSet(b'mykey2', b'success'), OpSet(b'mykey3', os.urandom(8)) ], failure=[ OpSet(b'mykey2', b'failure'), OpSet(b'mykey3', os.urandom(8)) ]) try: result = yield etcd.submit(txn) except Failed as failed: print('transaction FAILED:') for response in failed.responses: print(response) else: print('transaction SUCCESS:') for response in result.responses: print(response) result = yield etcd.get(KeySet(b'mykey2', b'mykey4')) for kv in result.kvs: print(kv)
def _start_watching(self, keys, on_watch, filters, start_revision, return_previous): data = [] headers = dict() url = ENDPOINT_WATCH.format(self._url).encode() # create watches for all key prefixes for key in keys: if type(key) == six.binary_type: key = KeySet(key) elif isinstance(key, KeySet): pass else: raise TypeError('key must be binary string or KeySet, not {}'.format(type(key))) if key.type == KeySet._SINGLE: range_end = None elif key.type == KeySet._PREFIX: range_end = _increment_last_byte(key.key) elif key.type == KeySet._RANGE: range_end = key.range_end else: raise Exception('logic error') obj = { 'create_request': { u'start_revision': start_revision, u'key': base64.b64encode(key.key).decode(), # range_end is the end of the range [key, range_end) to watch. # If range_end is not given,\nonly the key argument is watched. # If range_end is equal to '\\0', all keys greater than nor equal # to the key argument are watched. If the range_end is one bit # larger than the given key,\nthen all keys with the prefix (the # given key) will be watched. # progress_notify is set so that the etcd server will periodically # send a WatchResponse with\nno events to the new watcher if there # are no recent events. It is useful when clients wish to recover # a disconnected watcher starting from a recent known revision. # The etcd server may decide how often it will send notifications # based on current load. u'progress_notify': True, } } if range_end: obj[u'create_request'][u'range_end'] = base64.b64encode(range_end).decode() if filters: obj[u'create_request'][u'filters'] = filters if return_previous: # If prev_kv is set, created watcher gets the previous KV # before the event happens. # If the previous KV is already compacted, nothing will be # returned. obj[u'create_request'][u'prev_kv'] = True data.append(json.dumps(obj).encode('utf8')) data = b'\n'.join(data) # HTTP/POST request in one go, but response is streaming .. d = self._agent.request(b'POST', url, Headers(headers), _BufferedSender(data)) def handle_response(response): if response.code == 200: done = Deferred() response.deliverBody(_StreamingReceiver(on_watch, done)) return done else: raise Exception('unexpected response status {}'.format(response.code)) def handle_error(err): self.log.warn('could not start watching on etcd: {error}', error=err.value) return err d.addCallbacks(handle_response, handle_error) return d
def delete(self, key, return_previous=None, timeout=None): """ Delete value(s) from etcd. :param key: key is the first key to delete in the range. :type key: bytes or instance of :class:`txaioetcd.KeySet` :param return_previous: If enabled, return the deleted key-value pairs :type return_previous: bool or None :param timeout: Request timeout in seconds. :type timeout: int :returns: Deletion info :rtype: instance of :class:`txaioetcd.Deleted` """ if type(key) == six.binary_type: key = KeySet(key) elif isinstance(key, KeySet): pass else: raise TypeError('key must either be bytes or a KeySet object, not {}'.format(type(key))) if return_previous is not None and type(return_previous) != bool: raise TypeError('return_previous must be bool, not {}'.format(type(return_previous))) if key.type == KeySet._SINGLE: range_end = None elif key.type == KeySet._PREFIX: range_end = _increment_last_byte(key.key) elif key.type == KeySet._RANGE: range_end = key.range_end else: raise Exception('logic error') url = u'{}/v3alpha/kv/deleterange'.format(self._url).encode() obj = { u'key': base64.b64encode(key.key).decode(), } if range_end: # range_end is the key following the last key to delete # for the range [key, range_end). # If range_end is not given, the range is defined to contain only # the key argument. # If range_end is one bit larger than the given key, then the range # is all keys with the prefix (the given key). # If range_end is '\\0', the range is all keys greater # than or equal to the key argument. # obj[u'range_end'] = base64.b64encode(range_end).decode() if return_previous: # If prev_kv is set, etcd gets the previous key-value pairs # before deleting it. # The previous key-value pairs will be returned in the # delete response. # obj[u'prev_kv'] = True data = json.dumps(obj).encode('utf8') response = yield treq.post(url, data, headers=self._REQ_HEADERS, timeout=(timeout or self._timeout)) obj = yield treq.json_content(response) deleted = Deleted._parse(obj) returnValue(deleted)
def get(self, key, range_end=None, count_only=None, keys_only=None, limit=None, max_create_revision=None, min_create_revision=None, min_mod_revision=None, revision=None, serializable=None, sort_order=None, sort_target=None, timeout=None): """ Range gets the keys in the range from the key-value store. :param key: key is the first key for the range. If range_end is not given, the request only looks up key. :type key: bytes :param range_end: range_end is the upper bound on the requested range [key, range_end). If range_end is ``\\0``, the range is all keys ``\u003e=`` key. If the range_end is one bit larger than the given key, then the range requests get the all keys with the prefix (the given key). If both key and range_end are ``\\0``, then range requests returns all keys. :type range_end: bytes :param prefix: If set, and no range_end is given, compute range_end from key prefix. :type prefix: bool :param count_only: count_only when set returns only the count of the keys in the range. :type count_only: bool :param keys_only: keys_only when set returns only the keys and not the values. :type keys_only: bool :param limit: limit is a limit on the number of keys returned for the request. :type limit: int :param max_create_revision: max_create_revision is the upper bound for returned key create revisions; all keys with greater create revisions will be filtered away. :type max_create_revision: int :param max_mod_revision: max_mod_revision is the upper bound for returned key mod revisions; all keys with greater mod revisions will be filtered away. :type max_mod_revision: int :param min_create_revision: min_create_revision is the lower bound for returned key create revisions; all keys with lesser create trevisions will be filtered away. :type min_create_revision: int :param min_mod_revision: min_mod_revision is the lower bound for returned key mod revisions; all keys with lesser mod revisions will be filtered away. :type min_min_revision: int :param revision: revision is the point-in-time of the key-value store to use for the range. If revision is less or equal to zero, the range is over the newest key-value store. If the revision has been compacted, ErrCompacted is returned as a response. :type revision: int :param serializable: serializable sets the range request to use serializable member-local reads. Range requests are linearizable by default; linearizable requests have higher latency and lower throughput than serializable requests but reflect the current consensus of the cluster. For better performance, in exchange for possible stale reads, a serializable range request is served locally without needing to reach consensus with other nodes in the cluster. :type serializable: bool :param sort_order: Sort order for returned KVs, one of :class:`txaioetcd.OpGet.SORT_ORDERS`. :type sort_order: str :param sort_target: Sort target for sorting returned KVs, one of :class:`txaioetcd.OpGet.SORT_TARGETS`. :type sort_taget: str or None :param timeout: Request timeout in seconds. :type timeout: int or None """ if type(key) == six.binary_type: if range_end: key = KeySet(key, range_end=range_end) else: key = KeySet(key) elif isinstance(key, KeySet): pass else: raise TypeError('key must either be bytes or a KeySet object, not {}'.format(type(key))) if key.type == KeySet._SINGLE: range_end = None elif key.type == KeySet._PREFIX: range_end = _increment_last_byte(key.key) elif key.type == KeySet._RANGE: range_end = key.range_end else: raise Exception('logic error') url = u'{}/v3alpha/kv/range'.format(self._url).encode() obj = { u'key': base64.b64encode(key.key).decode() } if range_end: obj[u'range_end'] = base64.b64encode(range_end).decode() data = json.dumps(obj).encode('utf8') response = yield treq.post(url, data, headers=self._REQ_HEADERS, timeout=(timeout or self._timeout)) obj = yield treq.json_content(response) result = Range._parse(obj) # count = int(obj.get(u'count', 0)) returnValue(result)
def main(reactor): if True: with TemporaryDirectory() as dbpath: print('Using temporary directory {} for database'.format(dbpath)) schema = MySchema() with zlmdb.Database(dbpath) as db: # write records into zlmdb with db.begin(write=True) as txn: for i in range(10): key = 'key{}'.format(i) value = 'value{}'.format(random.randint(0, 1000)) schema.samples[txn, key] = value # read records from zlmdb with db.begin() as txn: for i in range(10): key = 'key{}'.format(i) value = schema.samples[txn, key] print('key={} : value={}'.format(key, value)) if True: # etcd database etcd = Client(reactor) status = yield etcd.status() print(status) # zlmdb database schema = MySchema() dbpath = '/tmp/.test-zlmdb' with zlmdb.Database(dbpath) as db: print('zlmdb open on {}'.format(dbpath)) # check current record count with db.begin() as txn: cnt = schema.samples.count(txn) print('currently {} rows in table'.format(cnt)) # watch changes in etcd and write to local zlmdb def on_change(kv): key = kv.key.decode() value = kv.value.decode() with db.begin(write=True) as txn: schema.samples[txn, key] = value print( 'on_change received from etcd and written to zlmdb: key={} value={}' .format(key, value)) # start watching for etcd changes .. ks = [KeySet('k'.encode(), prefix=True)] d = etcd.watch(ks, on_change) print('watching for 1s ..') yield txaio.sleep(1) # loop every 1s and write a key-value in etcd directly for i in range(5): print('watching for 1s ..') yield txaio.sleep(1) key = 'key{}'.format(i).encode() value = 'value{}'.format(random.randint(0, 1000)).encode() etcd.set(key, value) # cancel our watch d.cancel() yield util.sleep(1) # check current record count with db.begin() as txn: cnt = schema.samples.count(txn) print('currently {} rows in table'.format(cnt)) yield util.sleep(1)