def main(reactor):
# create an etcd client
etcd = Client(reactor)
# retrieve etcd cluster status
status = yield etcd.status()
print(status)
# callback invoked for every change
def on_change(kv):
print('on_change: {}'.format(kv))
# start watching on given keys or key sets
# when the key sets overlap, the callback might get called multiple
# times, once for each key set matching an event
keys = [KeySet(b'mykey2', b'mykey5'), KeySet(b'mykey', prefix=True)]
d = etcd.watch(keys, on_change)
# stop after 10 seconds
print('watching for 1s ..')
yield txaio.sleep(1)
etcd.set(b'mykey10', b'Test')
print('watching for 3s ..')
yield txaio.sleep(3)
d.cancel()
class EtcdClient(KVClient):
def __init__(self, kv_host, kv_port):
KVClient.__init__(self, kv_host, kv_port)
self.url = u'http://' + kv_host + u':' + str(kv_port)
self.client = Client(reactor, self.url)
@inlineCallbacks
def watch(self, key, key_change_callback, timeout=DEFAULT_TIMEOUT):
self.key_watches[key] = key_change_callback
result = yield self._op_with_retry('WATCH', key, None, timeout, callback=self.key_changed)
returnValue(result)
def key_changed(self, kv):
key = kv.key
value = kv.value
log.debug('key-changed', key=key, value=value)
# Notify client of key change event
if value is not None:
evt = Event(Event.PUT, key, value)
else:
evt = Event(Event.DELETE, key, None)
if key in self.key_watches:
self.key_watches[key](evt)
def close_watch(self, key, timeout=DEFAULT_TIMEOUT):
log.debug('close-watch', key=key)
if key in self.key_watches:
self.key_watches.pop(key)
@inlineCallbacks
def _op_with_retry(self, operation, key, value, timeout, *args, **kw):
log.debug('kv-op', operation=operation, key=key, timeout=timeout, args=args, kw=kw)
err = None
result = None
if type(key) == str:
key = bytes(key)
if value is not None:
value = bytes(value)
while True:
try:
if operation == 'GET':
result = yield self._get(key)
elif operation == 'LIST':
result, err = yield self._list(key)
elif operation == 'PUT':
# Put returns an object of type Revision
result = yield self.client.set(key, value, **kw)
elif operation == 'DELETE':
# Delete returns an object of type Deleted
result = yield self.client.delete(key)
elif operation == 'RESERVE':
result, err = yield self._reserve(key, value, **kw)
elif operation == 'RENEW':
result, err = yield self._renew_reservation(key)
elif operation == 'RELEASE':
result, err = yield self._release_reservation(key)
elif operation == 'RELEASE-ALL':
err = yield self._release_all_reservations()
elif operation == 'WATCH':
for name, val in kw.items():
if name == 'callback':
callback = val
break
result = self.client.watch([KeySet(key, prefix=True)], callback)
self._clear_backoff()
break
except ConnectionRefusedError as ex:
log.error('comms-exception', ex=ex)
yield self._backoff('etcd-not-up')
except Exception as ex:
log.error('etcd-exception', ex=ex)
err = ex
if timeout > 0 and self.retry_time > timeout:
err = 'operation-timed-out'
if err is not None:
self._clear_backoff()
break
returnValue((result, err))
@inlineCallbacks
def _get(self, key):
kvp = None
resp = yield self.client.get(key)
if resp.kvs is not None and len(resp.kvs) == 1:
kv = resp.kvs[0]
kvp = KVPair(kv.key, kv.value, kv.mod_revision)
returnValue(kvp)
@inlineCallbacks
def _list(self, key):
err = None
list = []
resp = yield self.client.get(KeySet(key, prefix=True))
if resp.kvs is not None and len(resp.kvs) > 0:
for kv in resp.kvs:
list.append(KVPair(kv.key, kv.value, kv.mod_revision))
returnValue((list, err))
@inlineCallbacks
def _reserve(self, key, value, **kw):
for name, val in kw.items():
if name == 'ttl':
ttl = val
break
reserved = False
err = 'reservation-failed'
owner = None
# Create a lease
lease = yield self.client.lease(ttl)
# Create a transaction
txn = Transaction(
compare=[ CompVersion(key, '==', 0) ],
success=[ OpSet(key, bytes(value), lease=lease) ],
failure=[ OpGet(key) ]
)
newly_acquired = False
try:
result = yield self.client.submit(txn)
except Failed as failed:
log.debug('key-already-present', key=key)
if len(failed.responses) > 0:
response = failed.responses[0]
if response.kvs is not None and len(response.kvs) > 0:
kv = response.kvs[0]
log.debug('key-already-present', value=kv.value)
if kv.value == value:
reserved = True
log.debug('key-already-reserved', key = kv.key, value=kv.value)
else:
newly_acquired = True
log.debug('key-was-absent', key=key, result=result)
# Check if reservation succeeded
resp = yield self.client.get(key)
if resp.kvs is not None and len(resp.kvs) == 1:
owner = resp.kvs[0].value
if owner == value:
if newly_acquired:
log.debug('key-reserved', key=key, value=value, ttl=ttl,
lease_id=lease.lease_id)
reserved = True
# Add key to reservation list
self.key_reservations[key] = lease
else:
log.debug("reservation-still-held")
else:
log.debug('reservation-held-by-another', value=owner)
if reserved:
err = None
returnValue((owner, err))
@inlineCallbacks
def _renew_reservation(self, key):
result = None
err = None
if key not in self.key_reservations:
err = 'key-not-reserved'
else:
lease = self.key_reservations[key]
# A successfully refreshed lease returns an object of type Header
result = yield lease.refresh()
if result is None:
err = 'lease-refresh-failed'
returnValue((result, err))
@inlineCallbacks
def _release_reservation(self, key):
err = None
if key not in self.key_reservations:
err = 'key-not-reserved'
else:
lease = self.key_reservations[key]
time_left = yield lease.remaining()
# A successfully revoked lease returns an object of type Header
log.debug('release-reservation', key=key, lease_id=lease.lease_id,
time_left_in_secs=time_left)
result = yield lease.revoke()
if result is None:
err = 'lease-revoke-failed'
self.key_reservations.pop(key)
returnValue((result, err))
@inlineCallbacks
def _release_all_reservations(self):
err = None
keys_to_delete = []
for key in self.key_reservations:
lease = self.key_reservations[key]
time_left = yield lease.remaining()
# A successfully revoked lease returns an object of type Header
log.debug('release-reservation', key=key, lease_id=lease.lease_id,
time_left_in_secs=time_left)
result = yield lease.revoke()
if result is None:
err = 'lease-revoke-failed'
log.debug('lease-revoke', result=result)
keys_to_delete.append(key)
for key in keys_to_delete:
self.key_reservations.pop(key)
returnValue(err)
def main(reactor):
if True:
with TemporaryDirectory() as dbpath:
print('Using temporary directory {} for database'.format(dbpath))
schema = MySchema()
with zlmdb.Database(dbpath) as db:
# write records into zlmdb
with db.begin(write=True) as txn:
for i in range(10):
key = 'key{}'.format(i)
value = 'value{}'.format(random.randint(0, 1000))
schema.samples[txn, key] = value
# read records from zlmdb
with db.begin() as txn:
for i in range(10):
key = 'key{}'.format(i)
value = schema.samples[txn, key]
print('key={} : value={}'.format(key, value))
if True:
# etcd database
etcd = Client(reactor)
status = yield etcd.status()
print(status)
# zlmdb database
schema = MySchema()
dbpath = '/tmp/.test-zlmdb'
with zlmdb.Database(dbpath) as db:
print('zlmdb open on {}'.format(dbpath))
# check current record count
with db.begin() as txn:
cnt = schema.samples.count(txn)
print('currently {} rows in table'.format(cnt))
# watch changes in etcd and write to local zlmdb
def on_change(kv):
key = kv.key.decode()
value = kv.value.decode()
with db.begin(write=True) as txn:
schema.samples[txn, key] = value
print(
'on_change received from etcd and written to zlmdb: key={} value={}'
.format(key, value))
# start watching for etcd changes ..
ks = [KeySet('k'.encode(), prefix=True)]
d = etcd.watch(ks, on_change)
print('watching for 1s ..')
yield txaio.sleep(1)
# loop every 1s and write a key-value in etcd directly
for i in range(5):
print('watching for 1s ..')
yield txaio.sleep(1)
key = 'key{}'.format(i).encode()
value = 'value{}'.format(random.randint(0, 1000)).encode()
etcd.set(key, value)
# cancel our watch
d.cancel()
yield util.sleep(1)
# check current record count
with db.begin() as txn:
cnt = schema.samples.count(txn)
print('currently {} rows in table'.format(cnt))
yield util.sleep(1)