def main(reactor): # create an etcd client etcd = Client(reactor) # retrieve etcd cluster status status = yield etcd.status() print(status) # callback invoked for every change def on_change(kv): print('on_change: {}'.format(kv)) # start watching on given keys or key sets # when the key sets overlap, the callback might get called multiple # times, once for each key set matching an event keys = [KeySet(b'mykey2', b'mykey5'), KeySet(b'mykey', prefix=True)] d = etcd.watch(keys, on_change) # stop after 10 seconds print('watching for 1s ..') yield txaio.sleep(1) etcd.set(b'mykey10', b'Test') print('watching for 3s ..') yield txaio.sleep(3) d.cancel()
class EtcdClient(KVClient): def __init__(self, kv_host, kv_port): KVClient.__init__(self, kv_host, kv_port) self.url = u'http://' + kv_host + u':' + str(kv_port) self.client = Client(reactor, self.url) @inlineCallbacks def watch(self, key, key_change_callback, timeout=DEFAULT_TIMEOUT): self.key_watches[key] = key_change_callback result = yield self._op_with_retry('WATCH', key, None, timeout, callback=self.key_changed) returnValue(result) def key_changed(self, kv): key = kv.key value = kv.value log.debug('key-changed', key=key, value=value) # Notify client of key change event if value is not None: evt = Event(Event.PUT, key, value) else: evt = Event(Event.DELETE, key, None) if key in self.key_watches: self.key_watches[key](evt) def close_watch(self, key, timeout=DEFAULT_TIMEOUT): log.debug('close-watch', key=key) if key in self.key_watches: self.key_watches.pop(key) @inlineCallbacks def _op_with_retry(self, operation, key, value, timeout, *args, **kw): log.debug('kv-op', operation=operation, key=key, timeout=timeout, args=args, kw=kw) err = None result = None if type(key) == str: key = bytes(key) if value is not None: value = bytes(value) while True: try: if operation == 'GET': result = yield self._get(key) elif operation == 'LIST': result, err = yield self._list(key) elif operation == 'PUT': # Put returns an object of type Revision result = yield self.client.set(key, value, **kw) elif operation == 'DELETE': # Delete returns an object of type Deleted result = yield self.client.delete(key) elif operation == 'RESERVE': result, err = yield self._reserve(key, value, **kw) elif operation == 'RENEW': result, err = yield self._renew_reservation(key) elif operation == 'RELEASE': result, err = yield self._release_reservation(key) elif operation == 'RELEASE-ALL': err = yield self._release_all_reservations() elif operation == 'WATCH': for name, val in kw.items(): if name == 'callback': callback = val break result = self.client.watch([KeySet(key, prefix=True)], callback) self._clear_backoff() break except ConnectionRefusedError as ex: log.error('comms-exception', ex=ex) yield self._backoff('etcd-not-up') except Exception as ex: log.error('etcd-exception', ex=ex) err = ex if timeout > 0 and self.retry_time > timeout: err = 'operation-timed-out' if err is not None: self._clear_backoff() break returnValue((result, err)) @inlineCallbacks def _get(self, key): kvp = None resp = yield self.client.get(key) if resp.kvs is not None and len(resp.kvs) == 1: kv = resp.kvs[0] kvp = KVPair(kv.key, kv.value, kv.mod_revision) returnValue(kvp) @inlineCallbacks def _list(self, key): err = None list = [] resp = yield self.client.get(KeySet(key, prefix=True)) if resp.kvs is not None and len(resp.kvs) > 0: for kv in resp.kvs: list.append(KVPair(kv.key, kv.value, kv.mod_revision)) returnValue((list, err)) @inlineCallbacks def _reserve(self, key, value, **kw): for name, val in kw.items(): if name == 'ttl': ttl = val break reserved = False err = 'reservation-failed' owner = None # Create a lease lease = yield self.client.lease(ttl) # Create a transaction txn = Transaction( compare=[ CompVersion(key, '==', 0) ], success=[ OpSet(key, bytes(value), lease=lease) ], failure=[ OpGet(key) ] ) newly_acquired = False try: result = yield self.client.submit(txn) except Failed as failed: log.debug('key-already-present', key=key) if len(failed.responses) > 0: response = failed.responses[0] if response.kvs is not None and len(response.kvs) > 0: kv = response.kvs[0] log.debug('key-already-present', value=kv.value) if kv.value == value: reserved = True log.debug('key-already-reserved', key = kv.key, value=kv.value) else: newly_acquired = True log.debug('key-was-absent', key=key, result=result) # Check if reservation succeeded resp = yield self.client.get(key) if resp.kvs is not None and len(resp.kvs) == 1: owner = resp.kvs[0].value if owner == value: if newly_acquired: log.debug('key-reserved', key=key, value=value, ttl=ttl, lease_id=lease.lease_id) reserved = True # Add key to reservation list self.key_reservations[key] = lease else: log.debug("reservation-still-held") else: log.debug('reservation-held-by-another', value=owner) if reserved: err = None returnValue((owner, err)) @inlineCallbacks def _renew_reservation(self, key): result = None err = None if key not in self.key_reservations: err = 'key-not-reserved' else: lease = self.key_reservations[key] # A successfully refreshed lease returns an object of type Header result = yield lease.refresh() if result is None: err = 'lease-refresh-failed' returnValue((result, err)) @inlineCallbacks def _release_reservation(self, key): err = None if key not in self.key_reservations: err = 'key-not-reserved' else: lease = self.key_reservations[key] time_left = yield lease.remaining() # A successfully revoked lease returns an object of type Header log.debug('release-reservation', key=key, lease_id=lease.lease_id, time_left_in_secs=time_left) result = yield lease.revoke() if result is None: err = 'lease-revoke-failed' self.key_reservations.pop(key) returnValue((result, err)) @inlineCallbacks def _release_all_reservations(self): err = None keys_to_delete = [] for key in self.key_reservations: lease = self.key_reservations[key] time_left = yield lease.remaining() # A successfully revoked lease returns an object of type Header log.debug('release-reservation', key=key, lease_id=lease.lease_id, time_left_in_secs=time_left) result = yield lease.revoke() if result is None: err = 'lease-revoke-failed' log.debug('lease-revoke', result=result) keys_to_delete.append(key) for key in keys_to_delete: self.key_reservations.pop(key) returnValue(err)
def main(reactor): if True: with TemporaryDirectory() as dbpath: print('Using temporary directory {} for database'.format(dbpath)) schema = MySchema() with zlmdb.Database(dbpath) as db: # write records into zlmdb with db.begin(write=True) as txn: for i in range(10): key = 'key{}'.format(i) value = 'value{}'.format(random.randint(0, 1000)) schema.samples[txn, key] = value # read records from zlmdb with db.begin() as txn: for i in range(10): key = 'key{}'.format(i) value = schema.samples[txn, key] print('key={} : value={}'.format(key, value)) if True: # etcd database etcd = Client(reactor) status = yield etcd.status() print(status) # zlmdb database schema = MySchema() dbpath = '/tmp/.test-zlmdb' with zlmdb.Database(dbpath) as db: print('zlmdb open on {}'.format(dbpath)) # check current record count with db.begin() as txn: cnt = schema.samples.count(txn) print('currently {} rows in table'.format(cnt)) # watch changes in etcd and write to local zlmdb def on_change(kv): key = kv.key.decode() value = kv.value.decode() with db.begin(write=True) as txn: schema.samples[txn, key] = value print( 'on_change received from etcd and written to zlmdb: key={} value={}' .format(key, value)) # start watching for etcd changes .. ks = [KeySet('k'.encode(), prefix=True)] d = etcd.watch(ks, on_change) print('watching for 1s ..') yield txaio.sleep(1) # loop every 1s and write a key-value in etcd directly for i in range(5): print('watching for 1s ..') yield txaio.sleep(1) key = 'key{}'.format(i).encode() value = 'value{}'.format(random.randint(0, 1000)).encode() etcd.set(key, value) # cancel our watch d.cancel() yield util.sleep(1) # check current record count with db.begin() as txn: cnt = schema.samples.count(txn) print('currently {} rows in table'.format(cnt)) yield util.sleep(1)