def get_all_keys(reactor, key_type, value_type, etcd_address):
"""Returns all keys from etcd.
:param reactor: reference to Twisted' reactor.
:param etcd_address: Address with port number where etcd is
running.
:return: An instance of txaioetcd.Range containing all keys and
their values.
"""
etcd = Client(reactor, etcd_address)
result = yield etcd.get(b'\x00', range_end=b'\x00')
res = {}
for item in result.kvs:
if key_type == u'utf8':
key = item.key.decode('utf8')
elif key_type == u'binary':
key = binascii.b2a_base64(item.key).decode().strip()
else:
raise Exception('logic error')
if value_type == u'json':
value = json.loads(item.value.decode('utf8'))
elif value_type == u'binary':
value = binascii.b2a_base64(item.value).decode().strip()
elif value_type == u'utf8':
value = item.value.decode('utf8')
else:
raise Exception('logic error')
res[key] = value
returnValue(res)
def import_to_db(reactor, key_type, value_type, input_format, input_file, etcd_address, dry_run,
dry_output, verbosity):
db_current = yield get_all_keys(reactor, key_type, value_type, etcd_address)
db_import = yield get_input_content(input_format, input_file, value_type)
db_diff = yield get_db_diff(db_current, db_import, key_type, value_type)
transaction = []
if dry_run:
pretty_print(db_diff, key_type, value_type, dry_output)
else:
for k, v in db_diff['to_update'].items():
if not isinstance(k, bytes):
k = k.encode()
if not isinstance(v, bytes):
v = v.encode()
transaction.append(OpSet(k, v))
for key in db_diff['to_delete']:
if not isinstance(key, bytes):
key = key.encode()
transaction.append(OpDel(key))
etcd = Client(reactor, etcd_address)
yield etcd.submit(Transaction(success=transaction))
if verbosity == 'verbose':
pretty_print(db_diff, key_type, value_type, dry_output)
elif verbosity == 'compact':
print('{} updated.'.format(len(db_diff['to_update'])))
print('{} deleted.'.format(len(db_diff['to_delete'])))
def main(reactor):
# create an etcd client
etcd = Client(reactor)
# retrieve etcd cluster status
status = yield etcd.status()
print(status)
def main(reactor):
# create an etcd client
etcd = Client(reactor, u'http://localhost:2379')
# retrieve etcd cluster status
status = yield etcd.status()
print(status)
def main(reactor):
etcd = Client(reactor, u'http://localhost:2379')
status = yield etcd.status()
print(status)
yield example1(reactor, etcd)
yield example2(reactor, etcd)
yield example3(reactor, etcd)
yield example4(reactor, etcd)
yield example5(reactor, etcd)
def main(reactor):
etcd = Client(reactor)
status = yield etcd.status()
print(status)
yield example1(reactor, etcd)
yield example2(reactor, etcd)
yield example3(reactor, etcd)
yield example4(reactor, etcd)
yield example5(reactor, etcd)
def main(reactor):
# create an etcd client
etcd = Client(reactor)
# retrieve etcd cluster status
status = yield etcd.status()
print(status)
# callback invoked for every change
def on_change(kv):
print('on_change: {}'.format(kv))
# start watching on given keys or key sets
# when the key sets overlap, the callback might get called multiple
# times, once for each key set matching an event
keys = [KeySet(b'mykey2', b'mykey5'), KeySet(b'mykey', prefix=True)]
d = etcd.watch(keys, on_change)
# stop after 10 seconds
print('watching for 1s ..')
yield txaio.sleep(1)
etcd.set(b'mykey10', b'Test')
print('watching for 3s ..')
yield txaio.sleep(3)
d.cancel()
async def main(reactor):
tab_users = MapUuidCbor(1,
marshal=lambda user: user.marshal(),
unmarshal=User.parse)
db = Database(Client(reactor))
revision = await db.status()
print('connected to etcd: revision', revision)
user_oids = []
# insert a couple of object in one etcd transaction
async with db.begin(write=True) as txn:
for i in range(10):
user = User.create_test_user(name='user{}'.format(i))
# INSERT: here we queue a key-value put operation with a native Python object
tab_users[txn, user.oid] = user
user_oids.append(user.oid)
print('new user object stored: name={}, oid={}'.format(
user.name, user.oid))
async with db.begin(write=True) as txn:
for oid in random.sample(user_oids, 5):
# DELETE
# tab_users.__delitem__((txn, oid))
# await tab_users.delete((txn, oid))
del tab_users[txn, oid]
print('user object deleted for oid={}'.format(oid))
async with db.begin() as txn:
for oid in user_oids:
# here we (directly/synchronously) get the value for a key as a native Python object
user = await tab_users[txn, oid]
if user:
print('user object loaded: name={}, oid={}'.format(
user.name, user.oid))
else:
print('no user object for oid={}'.format(oid))
print('etcd stats', db.stats())
async def main(reactor):
# Our main (asynchronous) entry point.
# users table schema (a table with UUID keys and CBOR values holding User objects)
tab_users = MapUuidCbor(1,
marshal=lambda user: user.marshal(),
unmarshal=User.parse)
# persistent KV database client using etcd backend
db = Database(Client(reactor))
revision = await db.status()
print('connected to etcd: revision', revision)
# new user ID
oid = uuid.uuid4()
# 1) store a native object in a UUID->CBOR table slot
async with db.begin(write=True) as txn:
user = User.create_test_user()
user.oid = oid
# the following enqueues the operation for when the transaction goes out of scope
tab_users[txn, user.oid] = user
print('new user object stored: name={}, oid={}'.format(
user.name, user.oid))
# 2) load a native object from above table slot
async with db.begin() as txn:
_user = await tab_users[txn, oid]
assert user
assert user == _user
print('user object loaded: name={}, oid={}:\n{}'.format(
_user.name, _user.oid, _user))
# 3) delete an object from above table slot
async with db.begin(write=True) as txn:
del tab_users[txn, oid]
print('user object deleted: oid={}'.format(oid))
print('etcd stats', db.stats())
def main(reactor):
etcd = Client(reactor)
#
# Example 1
#
for val in [b'val1', b'val2']:
yield etcd.set(b'test1', val)
txn = Transaction(
compare=[
# compute conjunction of all terms to
# determine "success" vs "failure"
CompValue(b'test1', '==', b'val1')
],
success=[
# if true ("success"), run these ops
OpSet(b'test1', b'val2'),
OpSet(b'test2', b'success')
],
failure=[
# if not true ("failure"), run these ops
OpSet(b'test2', b'failure'),
OpGet(b'test1')
])
try:
result = yield etcd.submit(txn)
except Failed as failed:
print('transaction FAILED:')
for response in failed.responses:
print(response)
else:
print('transaction SUCCESS:')
for response in result.responses:
print(response)
for key in [b'test1', b'test2']:
value = yield etcd.get(key)
print('{}: {}'.format(key, value))
#
# Example 2
#
rev = yield etcd.set(b'mykey1', os.urandom(8))
print(rev)
result = yield etcd.get(b'mykey1')
kv = result.kvs[0]
print(kv)
for version in [kv.version, kv.version - 1]:
txn = Transaction(
compare=[
# value equality comparison
CompValue(b'mykey1', '==', kv.value),
# version, and different comparison operators
CompVersion(b'mykey1', '==', version),
CompVersion(b'mykey1', '!=', version + 1),
CompVersion(b'mykey1', '<', version + 1),
CompVersion(b'mykey1', '>', version - 1),
# created revision comparison
CompCreated(b'mykey1', '==', kv.create_revision),
# modified revision comparison
CompModified(b'mykey1', '==', kv.mod_revision),
],
success=[
OpSet(b'mykey2', b'success'),
OpSet(b'mykey3', os.urandom(8))
],
failure=[
OpSet(b'mykey2', b'failure'),
OpSet(b'mykey3', os.urandom(8))
])
try:
result = yield etcd.submit(txn)
except Failed as failed:
print('transaction FAILED:')
for response in failed.responses:
print(response)
else:
print('transaction SUCCESS:')
for response in result.responses:
print(response)
result = yield etcd.get(KeySet(b'mykey2', b'mykey4'))
for kv in result.kvs:
print(kv)
def main(reactor):
etcd = Client(reactor, u'http://localhost:2379')
# set key-value
etcd.set(b'foo', os.urandom(8))
# set key-value, return revision including previous value
revision = yield etcd.set(b'foo', os.urandom(8), return_previous=True)
print('previous:', revision.previous)
# set values on keys
for i in range(10):
etcd.set('mykey{}'.format(i).encode(), os.urandom(8))
# get value by key
for key in [b'mykey1', KeySet(b'mykey1'), b'mykey13']:
result = yield etcd.get(key)
if result.kvs:
kv = result.kvs[0]
print(kv)
else:
print('key {} not found!'.format(key))
# iterate over keys in range
result = yield etcd.get(KeySet(b'mykey1', b'mykey5'))
print('KV pairs for range:')
for kv in result.kvs:
print(kv)
# iterate over keys with given prefix
result = yield etcd.get(KeySet(b'mykey', prefix=True))
print('KV pairs for prefix:')
for kv in result.kvs:
print(kv)
# delete a single key
deleted = yield etcd.delete(b'mykey0')
print('deleted {} key-value pairs'.format(deleted.deleted))
# delete non-existing key
deleted = yield etcd.delete(os.urandom(8))
print('deleted {} key-value pairs'.format(deleted.deleted))
# delete a key range
deleted = yield etcd.delete(KeySet(b'mykey2', b'mykey7'))
print('deleted {} key-value pairs'.format(deleted.deleted))
# delete a key set defined by prefix and return deleted key-value pairs
deleted = yield etcd.delete(KeySet(b'mykey', prefix=True),
return_previous=True)
print('deleted {} key-value pairs:'.format(deleted.deleted))
for d in deleted.previous:
print(d)
async def main(reactor):
tab_users = MapUuidCbor(1,
marshal=lambda user: user.marshal(),
unmarshal=User.parse)
idx_users_by_name = MapStringUuid(2)
tab_users.attach_index(idx_users_by_name, lambda user: user.name)
idx_users_by_email = MapStringUuid(3)
tab_users.attach_index(idx_users_by_email, lambda user: user.email)
db = Database(Client(reactor))
revision = await db.status()
print('connected to etcd: revision', revision)
users = []
removed = set()
async with db.begin(write=True) as txn:
for i in range(10):
user = User.create_test_user(name='user{}'.format(i))
tab_users[txn, user.oid] = user
users.append(user)
print('new user object stored: name={}, oid={}'.format(
user.name, user.oid))
async with db.begin() as txn:
for user in users:
_user = await tab_users[txn, user.oid]
assert _user
user_oid = await idx_users_by_name[txn, user.name]
assert user_oid == user.oid
user_oid = await idx_users_by_email[txn, user.email]
assert user_oid == user.oid
print('index lookups successful')
async with db.begin(write=True) as txn:
for user in random.sample(users, 5):
# DELETE
# tab_users.__delitem__((txn, user.oid))
# del tab_users[txn, user.oid]
await tab_users.delete((txn, user.oid))
print('user object deleted for oid={}'.format(user.oid))
removed.add(user.oid)
async with db.begin() as txn:
for user in users:
_user = await tab_users[txn, user.oid]
if user.oid in removed:
assert _user is None
user_oid = await idx_users_by_name[txn, user.name]
assert user_oid is None
user_oid = await idx_users_by_email[txn, user.email]
assert user_oid is None
else:
assert _user
assert _user == user
user_oid = await idx_users_by_name[txn, user.name]
assert user_oid == user.oid
user_oid = await idx_users_by_email[txn, user.email]
assert user_oid == user.oid
print('database structure for user oid={} verified successfully'.
format(user.oid))
print('etcd stats', db.stats())
async def main(reactor):
# our main (asynchronous) entry point ..
# persistent KV database client using etcd backend
etcd = Client(reactor)
db = Database(etcd)
revision = await db.status()
print('connected to etcd at revision {}'.format(revision))
# users table
users = await db.attach_table(Users)
users_by_name = await db.attach_table(IndexUsersByName)
users.attach_index(users_by_name, lambda user: user.name)
# select existing users
async with db.begin() as txn:
_users = await users.select(txn)
if _users:
print('yay, got {} users =)'.format(len(_users)))
name = _users[0].name
print('get user for name {}'.format(name))
oid = await users_by_name[txn, name]
if oid:
user = await users[txn, oid]
print('user: {}'.format(user))
else:
print('no users yet in database =(')
# delete user objects
oids = []
async with db.begin(write=True) as txn:
for i in range(5):
name = 'user{}'.format(random.randint(1, 10))
oid = await users_by_name[txn, name]
if oid:
print('deleting record')
#del users[txn, oid]
await users.delete((txn, oid))
oids.append(oid)
break
else:
print('missing data record!')
print('user objects deleted: oids={}'.format(oids))
# create and store new users
oids = []
async with db.begin(write=True) as txn:
for i in range(1, 10):
user = User.create_test_user(name='user{}'.format(i))
user_exists = await users_by_name[txn, user.name]
if True or not user_exists:
users[txn, user.oid] = user
oids.append(user.oid)
else:
print(user_exists)
print('new user objects stored: oids={}'.format(oids))
print('etcd stats', db.stats())
def main(reactor):
etcd = Client(reactor, u'http://192.168.0.172:2379')
status = yield etcd.status()
print(status)
async def main(reactor):
db = Database(Client(reactor))
revision = await db.status()
print('connected to etcd: revision', revision)
class EtcdClient(KVClient):
def __init__(self, kv_host, kv_port):
KVClient.__init__(self, kv_host, kv_port)
self.url = u'http://' + kv_host + u':' + str(kv_port)
self.client = Client(reactor, self.url)
@inlineCallbacks
def watch(self, key, key_change_callback, timeout=DEFAULT_TIMEOUT):
self.key_watches[key] = key_change_callback
result = yield self._op_with_retry('WATCH', key, None, timeout, callback=self.key_changed)
returnValue(result)
def key_changed(self, kv):
key = kv.key
value = kv.value
log.debug('key-changed', key=key, value=value)
# Notify client of key change event
if value is not None:
evt = Event(Event.PUT, key, value)
else:
evt = Event(Event.DELETE, key, None)
if key in self.key_watches:
self.key_watches[key](evt)
def close_watch(self, key, timeout=DEFAULT_TIMEOUT):
log.debug('close-watch', key=key)
if key in self.key_watches:
self.key_watches.pop(key)
@inlineCallbacks
def _op_with_retry(self, operation, key, value, timeout, *args, **kw):
log.debug('kv-op', operation=operation, key=key, timeout=timeout, args=args, kw=kw)
err = None
result = None
if type(key) == str:
key = bytes(key)
if value is not None:
value = bytes(value)
while True:
try:
if operation == 'GET':
result = yield self._get(key)
elif operation == 'LIST':
result, err = yield self._list(key)
elif operation == 'PUT':
# Put returns an object of type Revision
result = yield self.client.set(key, value, **kw)
elif operation == 'DELETE':
# Delete returns an object of type Deleted
result = yield self.client.delete(key)
elif operation == 'RESERVE':
result, err = yield self._reserve(key, value, **kw)
elif operation == 'RENEW':
result, err = yield self._renew_reservation(key)
elif operation == 'RELEASE':
result, err = yield self._release_reservation(key)
elif operation == 'RELEASE-ALL':
err = yield self._release_all_reservations()
elif operation == 'WATCH':
for name, val in kw.items():
if name == 'callback':
callback = val
break
result = self.client.watch([KeySet(key, prefix=True)], callback)
self._clear_backoff()
break
except ConnectionRefusedError as ex:
log.error('comms-exception', ex=ex)
yield self._backoff('etcd-not-up')
except Exception as ex:
log.error('etcd-exception', ex=ex)
err = ex
if timeout > 0 and self.retry_time > timeout:
err = 'operation-timed-out'
if err is not None:
self._clear_backoff()
break
returnValue((result, err))
@inlineCallbacks
def _get(self, key):
kvp = None
resp = yield self.client.get(key)
if resp.kvs is not None and len(resp.kvs) == 1:
kv = resp.kvs[0]
kvp = KVPair(kv.key, kv.value, kv.mod_revision)
returnValue(kvp)
@inlineCallbacks
def _list(self, key):
err = None
list = []
resp = yield self.client.get(KeySet(key, prefix=True))
if resp.kvs is not None and len(resp.kvs) > 0:
for kv in resp.kvs:
list.append(KVPair(kv.key, kv.value, kv.mod_revision))
returnValue((list, err))
@inlineCallbacks
def _reserve(self, key, value, **kw):
for name, val in kw.items():
if name == 'ttl':
ttl = val
break
reserved = False
err = 'reservation-failed'
owner = None
# Create a lease
lease = yield self.client.lease(ttl)
# Create a transaction
txn = Transaction(
compare=[ CompVersion(key, '==', 0) ],
success=[ OpSet(key, bytes(value), lease=lease) ],
failure=[ OpGet(key) ]
)
newly_acquired = False
try:
result = yield self.client.submit(txn)
except Failed as failed:
log.debug('key-already-present', key=key)
if len(failed.responses) > 0:
response = failed.responses[0]
if response.kvs is not None and len(response.kvs) > 0:
kv = response.kvs[0]
log.debug('key-already-present', value=kv.value)
if kv.value == value:
reserved = True
log.debug('key-already-reserved', key = kv.key, value=kv.value)
else:
newly_acquired = True
log.debug('key-was-absent', key=key, result=result)
# Check if reservation succeeded
resp = yield self.client.get(key)
if resp.kvs is not None and len(resp.kvs) == 1:
owner = resp.kvs[0].value
if owner == value:
if newly_acquired:
log.debug('key-reserved', key=key, value=value, ttl=ttl,
lease_id=lease.lease_id)
reserved = True
# Add key to reservation list
self.key_reservations[key] = lease
else:
log.debug("reservation-still-held")
else:
log.debug('reservation-held-by-another', value=owner)
if reserved:
err = None
returnValue((owner, err))
@inlineCallbacks
def _renew_reservation(self, key):
result = None
err = None
if key not in self.key_reservations:
err = 'key-not-reserved'
else:
lease = self.key_reservations[key]
# A successfully refreshed lease returns an object of type Header
result = yield lease.refresh()
if result is None:
err = 'lease-refresh-failed'
returnValue((result, err))
@inlineCallbacks
def _release_reservation(self, key):
err = None
if key not in self.key_reservations:
err = 'key-not-reserved'
else:
lease = self.key_reservations[key]
time_left = yield lease.remaining()
# A successfully revoked lease returns an object of type Header
log.debug('release-reservation', key=key, lease_id=lease.lease_id,
time_left_in_secs=time_left)
result = yield lease.revoke()
if result is None:
err = 'lease-revoke-failed'
self.key_reservations.pop(key)
returnValue((result, err))
@inlineCallbacks
def _release_all_reservations(self):
err = None
keys_to_delete = []
for key in self.key_reservations:
lease = self.key_reservations[key]
time_left = yield lease.remaining()
# A successfully revoked lease returns an object of type Header
log.debug('release-reservation', key=key, lease_id=lease.lease_id,
time_left_in_secs=time_left)
result = yield lease.revoke()
if result is None:
err = 'lease-revoke-failed'
log.debug('lease-revoke', result=result)
keys_to_delete.append(key)
for key in keys_to_delete:
self.key_reservations.pop(key)
returnValue(err)
def __init__(self, kv_host, kv_port):
KVClient.__init__(self, kv_host, kv_port)
self.url = u'http://' + kv_host + u':' + str(kv_port)
self.client = Client(reactor, self.url)
def main(reactor):
if True:
with TemporaryDirectory() as dbpath:
print('Using temporary directory {} for database'.format(dbpath))
schema = MySchema()
with zlmdb.Database(dbpath) as db:
# write records into zlmdb
with db.begin(write=True) as txn:
for i in range(10):
key = 'key{}'.format(i)
value = 'value{}'.format(random.randint(0, 1000))
schema.samples[txn, key] = value
# read records from zlmdb
with db.begin() as txn:
for i in range(10):
key = 'key{}'.format(i)
value = schema.samples[txn, key]
print('key={} : value={}'.format(key, value))
if True:
# etcd database
etcd = Client(reactor)
status = yield etcd.status()
print(status)
# zlmdb database
schema = MySchema()
dbpath = '/tmp/.test-zlmdb'
with zlmdb.Database(dbpath) as db:
print('zlmdb open on {}'.format(dbpath))
# check current record count
with db.begin() as txn:
cnt = schema.samples.count(txn)
print('currently {} rows in table'.format(cnt))
# watch changes in etcd and write to local zlmdb
def on_change(kv):
key = kv.key.decode()
value = kv.value.decode()
with db.begin(write=True) as txn:
schema.samples[txn, key] = value
print(
'on_change received from etcd and written to zlmdb: key={} value={}'
.format(key, value))
# start watching for etcd changes ..
ks = [KeySet('k'.encode(), prefix=True)]
d = etcd.watch(ks, on_change)
print('watching for 1s ..')
yield txaio.sleep(1)
# loop every 1s and write a key-value in etcd directly
for i in range(5):
print('watching for 1s ..')
yield txaio.sleep(1)
key = 'key{}'.format(i).encode()
value = 'value{}'.format(random.randint(0, 1000)).encode()
etcd.set(key, value)
# cancel our watch
d.cancel()
yield util.sleep(1)
# check current record count
with db.begin() as txn:
cnt = schema.samples.count(txn)
print('currently {} rows in table'.format(cnt))
yield util.sleep(1)