def main(reactor):
# create an etcd client
etcd = Client(reactor)
# retrieve etcd cluster status
status = yield etcd.status()
print(status)
# callback invoked for every change
def on_change(kv):
print('on_change: {}'.format(kv))
# start watching on given keys or key sets
# when the key sets overlap, the callback might get called multiple
# times, once for each key set matching an event
keys = [KeySet(b'mykey2', b'mykey5'), KeySet(b'mykey', prefix=True)]
d = etcd.watch(keys, on_change)
# stop after 10 seconds
print('watching for 1s ..')
yield txaio.sleep(1)
etcd.set(b'mykey10', b'Test')
print('watching for 3s ..')
yield txaio.sleep(3)
d.cancel()
def main(reactor):
etcd = Client(reactor, u'http://localhost:2379')
# set key-value
etcd.set(b'foo', os.urandom(8))
# set key-value, return revision including previous value
revision = yield etcd.set(b'foo', os.urandom(8), return_previous=True)
print('previous:', revision.previous)
# set values on keys
for i in range(10):
etcd.set('mykey{}'.format(i).encode(), os.urandom(8))
# get value by key
for key in [b'mykey1', KeySet(b'mykey1'), b'mykey13']:
result = yield etcd.get(key)
if result.kvs:
kv = result.kvs[0]
print(kv)
else:
print('key {} not found!'.format(key))
# iterate over keys in range
result = yield etcd.get(KeySet(b'mykey1', b'mykey5'))
print('KV pairs for range:')
for kv in result.kvs:
print(kv)
# iterate over keys with given prefix
result = yield etcd.get(KeySet(b'mykey', prefix=True))
print('KV pairs for prefix:')
for kv in result.kvs:
print(kv)
# delete a single key
deleted = yield etcd.delete(b'mykey0')
print('deleted {} key-value pairs'.format(deleted.deleted))
# delete non-existing key
deleted = yield etcd.delete(os.urandom(8))
print('deleted {} key-value pairs'.format(deleted.deleted))
# delete a key range
deleted = yield etcd.delete(KeySet(b'mykey2', b'mykey7'))
print('deleted {} key-value pairs'.format(deleted.deleted))
# delete a key set defined by prefix and return deleted key-value pairs
deleted = yield etcd.delete(KeySet(b'mykey', prefix=True),
return_previous=True)
print('deleted {} key-value pairs:'.format(deleted.deleted))
for d in deleted.previous:
print(d)
def _list(self, key):
err = None
list = []
resp = yield self.client.get(KeySet(key, prefix=True))
if resp.kvs is not None and len(resp.kvs) > 0:
for kv in resp.kvs:
list.append(KVPair(kv.key, kv.value, kv.mod_revision))
returnValue((list, err))
def _op_with_retry(self, operation, key, value, timeout, *args, **kw):
log.debug('kv-op',
operation=operation,
key=key,
timeout=timeout,
args=args,
kw=kw)
err = None
result = None
if type(key) == str:
key = bytes(key)
if value is not None:
value = bytes(value)
while True:
try:
if operation == 'GET':
result = yield self._get(key)
elif operation == 'LIST':
result, err = yield self._list(key)
elif operation == 'PUT':
# Put returns an object of type Revision
result = yield self.client.set(key, value, **kw)
elif operation == 'DELETE':
# Delete returns an object of type Deleted
result = yield self.client.delete(key)
elif operation == 'RESERVE':
result, err = yield self._reserve(key, value, **kw)
elif operation == 'RENEW':
result, err = yield self._renew_reservation(key)
elif operation == 'RELEASE':
result, err = yield self._release_reservation(key)
elif operation == 'RELEASE-ALL':
err = yield self._release_all_reservations()
elif operation == 'WATCH':
for name, val in kw.items():
if name == 'callback':
callback = val
break
result = self.client.watch([KeySet(key, prefix=True)],
callback)
self._clear_backoff()
break
except ConnectionRefusedError as ex:
log.error('comms-exception', ex=ex)
yield self._backoff('etcd-not-up')
except Exception as ex:
log.error('etcd-exception', ex=ex)
err = ex
if timeout > 0 and self.retry_time > timeout:
err = 'operation-timed-out'
if err is not None:
self._clear_backoff()
break
returnValue((result, err))
def __validate(self):
if type(self._key) == six.binary_type:
if self._range_end:
self._key = KeySet(self._key, range_end=self._range_end)
else:
self._key = KeySet(self._key)
elif isinstance(self._key, KeySet):
pass
else:
raise TypeError(
'key must either be bytes or a KeySet object, not {}'.format(type(self._key)))
if self._key.type == KeySet._SINGLE:
self._range_end = None
elif self._key.type == KeySet._PREFIX:
self._range_end = _increment_last_byte(self._key.key)
elif self._key.type == KeySet._RANGE:
self._range_end = self._key.range_end
else:
raise Exception('logic error')
def main(reactor):
etcd = Client(reactor)
#
# Example 1
#
for val in [b'val1', b'val2']:
yield etcd.set(b'test1', val)
txn = Transaction(
compare=[
# compute conjunction of all terms to
# determine "success" vs "failure"
CompValue(b'test1', '==', b'val1')
],
success=[
# if true ("success"), run these ops
OpSet(b'test1', b'val2'),
OpSet(b'test2', b'success')
],
failure=[
# if not true ("failure"), run these ops
OpSet(b'test2', b'failure'),
OpGet(b'test1')
])
try:
result = yield etcd.submit(txn)
except Failed as failed:
print('transaction FAILED:')
for response in failed.responses:
print(response)
else:
print('transaction SUCCESS:')
for response in result.responses:
print(response)
for key in [b'test1', b'test2']:
value = yield etcd.get(key)
print('{}: {}'.format(key, value))
#
# Example 2
#
rev = yield etcd.set(b'mykey1', os.urandom(8))
print(rev)
result = yield etcd.get(b'mykey1')
kv = result.kvs[0]
print(kv)
for version in [kv.version, kv.version - 1]:
txn = Transaction(
compare=[
# value equality comparison
CompValue(b'mykey1', '==', kv.value),
# version, and different comparison operators
CompVersion(b'mykey1', '==', version),
CompVersion(b'mykey1', '!=', version + 1),
CompVersion(b'mykey1', '<', version + 1),
CompVersion(b'mykey1', '>', version - 1),
# created revision comparison
CompCreated(b'mykey1', '==', kv.create_revision),
# modified revision comparison
CompModified(b'mykey1', '==', kv.mod_revision),
],
success=[
OpSet(b'mykey2', b'success'),
OpSet(b'mykey3', os.urandom(8))
],
failure=[
OpSet(b'mykey2', b'failure'),
OpSet(b'mykey3', os.urandom(8))
])
try:
result = yield etcd.submit(txn)
except Failed as failed:
print('transaction FAILED:')
for response in failed.responses:
print(response)
else:
print('transaction SUCCESS:')
for response in result.responses:
print(response)
result = yield etcd.get(KeySet(b'mykey2', b'mykey4'))
for kv in result.kvs:
print(kv)
def _start_watching(self, keys, on_watch, filters, start_revision, return_previous):
data = []
headers = dict()
url = ENDPOINT_WATCH.format(self._url).encode()
# create watches for all key prefixes
for key in keys:
if type(key) == six.binary_type:
key = KeySet(key)
elif isinstance(key, KeySet):
pass
else:
raise TypeError('key must be binary string or KeySet, not {}'.format(type(key)))
if key.type == KeySet._SINGLE:
range_end = None
elif key.type == KeySet._PREFIX:
range_end = _increment_last_byte(key.key)
elif key.type == KeySet._RANGE:
range_end = key.range_end
else:
raise Exception('logic error')
obj = {
'create_request': {
u'start_revision': start_revision,
u'key': base64.b64encode(key.key).decode(),
# range_end is the end of the range [key, range_end) to watch.
# If range_end is not given,\nonly the key argument is watched.
# If range_end is equal to '\\0', all keys greater than nor equal
# to the key argument are watched. If the range_end is one bit
# larger than the given key,\nthen all keys with the prefix (the
# given key) will be watched.
# progress_notify is set so that the etcd server will periodically
# send a WatchResponse with\nno events to the new watcher if there
# are no recent events. It is useful when clients wish to recover
# a disconnected watcher starting from a recent known revision.
# The etcd server may decide how often it will send notifications
# based on current load.
u'progress_notify': True,
}
}
if range_end:
obj[u'create_request'][u'range_end'] = base64.b64encode(range_end).decode()
if filters:
obj[u'create_request'][u'filters'] = filters
if return_previous:
# If prev_kv is set, created watcher gets the previous KV
# before the event happens.
# If the previous KV is already compacted, nothing will be
# returned.
obj[u'create_request'][u'prev_kv'] = True
data.append(json.dumps(obj).encode('utf8'))
data = b'\n'.join(data)
# HTTP/POST request in one go, but response is streaming ..
d = self._agent.request(b'POST', url, Headers(headers), _BufferedSender(data))
def handle_response(response):
if response.code == 200:
done = Deferred()
response.deliverBody(_StreamingReceiver(on_watch, done))
return done
else:
raise Exception('unexpected response status {}'.format(response.code))
def handle_error(err):
self.log.warn('could not start watching on etcd: {error}', error=err.value)
return err
d.addCallbacks(handle_response, handle_error)
return d
def delete(self, key, return_previous=None, timeout=None):
"""
Delete value(s) from etcd.
:param key: key is the first key to delete in the range.
:type key: bytes or instance of :class:`txaioetcd.KeySet`
:param return_previous: If enabled, return the deleted key-value pairs
:type return_previous: bool or None
:param timeout: Request timeout in seconds.
:type timeout: int
:returns: Deletion info
:rtype: instance of :class:`txaioetcd.Deleted`
"""
if type(key) == six.binary_type:
key = KeySet(key)
elif isinstance(key, KeySet):
pass
else:
raise TypeError('key must either be bytes or a KeySet object, not {}'.format(type(key)))
if return_previous is not None and type(return_previous) != bool:
raise TypeError('return_previous must be bool, not {}'.format(type(return_previous)))
if key.type == KeySet._SINGLE:
range_end = None
elif key.type == KeySet._PREFIX:
range_end = _increment_last_byte(key.key)
elif key.type == KeySet._RANGE:
range_end = key.range_end
else:
raise Exception('logic error')
url = u'{}/v3alpha/kv/deleterange'.format(self._url).encode()
obj = {
u'key': base64.b64encode(key.key).decode(),
}
if range_end:
# range_end is the key following the last key to delete
# for the range [key, range_end).
# If range_end is not given, the range is defined to contain only
# the key argument.
# If range_end is one bit larger than the given key, then the range
# is all keys with the prefix (the given key).
# If range_end is '\\0', the range is all keys greater
# than or equal to the key argument.
#
obj[u'range_end'] = base64.b64encode(range_end).decode()
if return_previous:
# If prev_kv is set, etcd gets the previous key-value pairs
# before deleting it.
# The previous key-value pairs will be returned in the
# delete response.
#
obj[u'prev_kv'] = True
data = json.dumps(obj).encode('utf8')
response = yield treq.post(url, data, headers=self._REQ_HEADERS, timeout=(timeout or self._timeout))
obj = yield treq.json_content(response)
deleted = Deleted._parse(obj)
returnValue(deleted)
def get(self,
key,
range_end=None,
count_only=None,
keys_only=None,
limit=None,
max_create_revision=None,
min_create_revision=None,
min_mod_revision=None,
revision=None,
serializable=None,
sort_order=None,
sort_target=None,
timeout=None):
"""
Range gets the keys in the range from the key-value store.
:param key: key is the first key for the range. If range_end is not given,
the request only looks up key.
:type key: bytes
:param range_end: range_end is the upper bound on the requested range
[key, range_end). If range_end is ``\\0``, the range is all keys ``\u003e=`` key.
If the range_end is one bit larger than the given key, then the range requests
get the all keys with the prefix (the given key). If both key and range_end
are ``\\0``, then range requests returns all keys.
:type range_end: bytes
:param prefix: If set, and no range_end is given, compute range_end from key prefix.
:type prefix: bool
:param count_only: count_only when set returns only the count of the keys in the range.
:type count_only: bool
:param keys_only: keys_only when set returns only the keys and not the values.
:type keys_only: bool
:param limit: limit is a limit on the number of keys returned for the request.
:type limit: int
:param max_create_revision: max_create_revision is the upper bound for returned
key create revisions; all keys with greater create revisions will be filtered away.
:type max_create_revision: int
:param max_mod_revision: max_mod_revision is the upper bound for returned key
mod revisions; all keys with greater mod revisions will be filtered away.
:type max_mod_revision: int
:param min_create_revision: min_create_revision is the lower bound for returned
key create revisions; all keys with lesser create trevisions will be filtered away.
:type min_create_revision: int
:param min_mod_revision: min_mod_revision is the lower bound for returned key
mod revisions; all keys with lesser mod revisions will be filtered away.
:type min_min_revision: int
:param revision: revision is the point-in-time of the key-value store to use for the
range. If revision is less or equal to zero, the range is over the newest
key-value store. If the revision has been compacted, ErrCompacted is returned as
a response.
:type revision: int
:param serializable: serializable sets the range request to use serializable
member-local reads. Range requests are linearizable by default; linearizable
requests have higher latency and lower throughput than serializable requests
but reflect the current consensus of the cluster. For better performance, in
exchange for possible stale reads, a serializable range request is served
locally without needing to reach consensus with other nodes in the cluster.
:type serializable: bool
:param sort_order: Sort order for returned KVs,
one of :class:`txaioetcd.OpGet.SORT_ORDERS`.
:type sort_order: str
:param sort_target: Sort target for sorting returned KVs,
one of :class:`txaioetcd.OpGet.SORT_TARGETS`.
:type sort_taget: str or None
:param timeout: Request timeout in seconds.
:type timeout: int or None
"""
if type(key) == six.binary_type:
if range_end:
key = KeySet(key, range_end=range_end)
else:
key = KeySet(key)
elif isinstance(key, KeySet):
pass
else:
raise TypeError('key must either be bytes or a KeySet object, not {}'.format(type(key)))
if key.type == KeySet._SINGLE:
range_end = None
elif key.type == KeySet._PREFIX:
range_end = _increment_last_byte(key.key)
elif key.type == KeySet._RANGE:
range_end = key.range_end
else:
raise Exception('logic error')
url = u'{}/v3alpha/kv/range'.format(self._url).encode()
obj = {
u'key': base64.b64encode(key.key).decode()
}
if range_end:
obj[u'range_end'] = base64.b64encode(range_end).decode()
data = json.dumps(obj).encode('utf8')
response = yield treq.post(url, data, headers=self._REQ_HEADERS, timeout=(timeout or self._timeout))
obj = yield treq.json_content(response)
result = Range._parse(obj)
# count = int(obj.get(u'count', 0))
returnValue(result)
def main(reactor):
if True:
with TemporaryDirectory() as dbpath:
print('Using temporary directory {} for database'.format(dbpath))
schema = MySchema()
with zlmdb.Database(dbpath) as db:
# write records into zlmdb
with db.begin(write=True) as txn:
for i in range(10):
key = 'key{}'.format(i)
value = 'value{}'.format(random.randint(0, 1000))
schema.samples[txn, key] = value
# read records from zlmdb
with db.begin() as txn:
for i in range(10):
key = 'key{}'.format(i)
value = schema.samples[txn, key]
print('key={} : value={}'.format(key, value))
if True:
# etcd database
etcd = Client(reactor)
status = yield etcd.status()
print(status)
# zlmdb database
schema = MySchema()
dbpath = '/tmp/.test-zlmdb'
with zlmdb.Database(dbpath) as db:
print('zlmdb open on {}'.format(dbpath))
# check current record count
with db.begin() as txn:
cnt = schema.samples.count(txn)
print('currently {} rows in table'.format(cnt))
# watch changes in etcd and write to local zlmdb
def on_change(kv):
key = kv.key.decode()
value = kv.value.decode()
with db.begin(write=True) as txn:
schema.samples[txn, key] = value
print(
'on_change received from etcd and written to zlmdb: key={} value={}'
.format(key, value))
# start watching for etcd changes ..
ks = [KeySet('k'.encode(), prefix=True)]
d = etcd.watch(ks, on_change)
print('watching for 1s ..')
yield txaio.sleep(1)
# loop every 1s and write a key-value in etcd directly
for i in range(5):
print('watching for 1s ..')
yield txaio.sleep(1)
key = 'key{}'.format(i).encode()
value = 'value{}'.format(random.randint(0, 1000)).encode()
etcd.set(key, value)
# cancel our watch
d.cancel()
yield util.sleep(1)
# check current record count
with db.begin() as txn:
cnt = schema.samples.count(txn)
print('currently {} rows in table'.format(cnt))
yield util.sleep(1)
