def import_to_db(reactor, key_type, value_type, input_format, input_file, etcd_address, dry_run,
dry_output, verbosity):
db_current = yield get_all_keys(reactor, key_type, value_type, etcd_address)
db_import = yield get_input_content(input_format, input_file, value_type)
db_diff = yield get_db_diff(db_current, db_import, key_type, value_type)
transaction = []
if dry_run:
pretty_print(db_diff, key_type, value_type, dry_output)
else:
for k, v in db_diff['to_update'].items():
if not isinstance(k, bytes):
k = k.encode()
if not isinstance(v, bytes):
v = v.encode()
transaction.append(OpSet(k, v))
for key in db_diff['to_delete']:
if not isinstance(key, bytes):
key = key.encode()
transaction.append(OpDel(key))
etcd = Client(reactor, etcd_address)
yield etcd.submit(Transaction(success=transaction))
if verbosity == 'verbose':
pretty_print(db_diff, key_type, value_type, dry_output)
elif verbosity == 'compact':
print('{} updated.'.format(len(db_diff['to_update'])))
print('{} deleted.'.format(len(db_diff['to_delete'])))
class EtcdClient(KVClient):
def __init__(self, kv_host, kv_port):
KVClient.__init__(self, kv_host, kv_port)
self.url = u'http://' + kv_host + u':' + str(kv_port)
self.client = Client(reactor, self.url)
@inlineCallbacks
def watch(self, key, key_change_callback, timeout=DEFAULT_TIMEOUT):
self.key_watches[key] = key_change_callback
result = yield self._op_with_retry('WATCH', key, None, timeout, callback=self.key_changed)
returnValue(result)
def key_changed(self, kv):
key = kv.key
value = kv.value
log.debug('key-changed', key=key, value=value)
# Notify client of key change event
if value is not None:
evt = Event(Event.PUT, key, value)
else:
evt = Event(Event.DELETE, key, None)
if key in self.key_watches:
self.key_watches[key](evt)
def close_watch(self, key, timeout=DEFAULT_TIMEOUT):
log.debug('close-watch', key=key)
if key in self.key_watches:
self.key_watches.pop(key)
@inlineCallbacks
def _op_with_retry(self, operation, key, value, timeout, *args, **kw):
log.debug('kv-op', operation=operation, key=key, timeout=timeout, args=args, kw=kw)
err = None
result = None
if type(key) == str:
key = bytes(key)
if value is not None:
value = bytes(value)
while True:
try:
if operation == 'GET':
result = yield self._get(key)
elif operation == 'LIST':
result, err = yield self._list(key)
elif operation == 'PUT':
# Put returns an object of type Revision
result = yield self.client.set(key, value, **kw)
elif operation == 'DELETE':
# Delete returns an object of type Deleted
result = yield self.client.delete(key)
elif operation == 'RESERVE':
result, err = yield self._reserve(key, value, **kw)
elif operation == 'RENEW':
result, err = yield self._renew_reservation(key)
elif operation == 'RELEASE':
result, err = yield self._release_reservation(key)
elif operation == 'RELEASE-ALL':
err = yield self._release_all_reservations()
elif operation == 'WATCH':
for name, val in kw.items():
if name == 'callback':
callback = val
break
result = self.client.watch([KeySet(key, prefix=True)], callback)
self._clear_backoff()
break
except ConnectionRefusedError as ex:
log.error('comms-exception', ex=ex)
yield self._backoff('etcd-not-up')
except Exception as ex:
log.error('etcd-exception', ex=ex)
err = ex
if timeout > 0 and self.retry_time > timeout:
err = 'operation-timed-out'
if err is not None:
self._clear_backoff()
break
returnValue((result, err))
@inlineCallbacks
def _get(self, key):
kvp = None
resp = yield self.client.get(key)
if resp.kvs is not None and len(resp.kvs) == 1:
kv = resp.kvs[0]
kvp = KVPair(kv.key, kv.value, kv.mod_revision)
returnValue(kvp)
@inlineCallbacks
def _list(self, key):
err = None
list = []
resp = yield self.client.get(KeySet(key, prefix=True))
if resp.kvs is not None and len(resp.kvs) > 0:
for kv in resp.kvs:
list.append(KVPair(kv.key, kv.value, kv.mod_revision))
returnValue((list, err))
@inlineCallbacks
def _reserve(self, key, value, **kw):
for name, val in kw.items():
if name == 'ttl':
ttl = val
break
reserved = False
err = 'reservation-failed'
owner = None
# Create a lease
lease = yield self.client.lease(ttl)
# Create a transaction
txn = Transaction(
compare=[ CompVersion(key, '==', 0) ],
success=[ OpSet(key, bytes(value), lease=lease) ],
failure=[ OpGet(key) ]
)
newly_acquired = False
try:
result = yield self.client.submit(txn)
except Failed as failed:
log.debug('key-already-present', key=key)
if len(failed.responses) > 0:
response = failed.responses[0]
if response.kvs is not None and len(response.kvs) > 0:
kv = response.kvs[0]
log.debug('key-already-present', value=kv.value)
if kv.value == value:
reserved = True
log.debug('key-already-reserved', key = kv.key, value=kv.value)
else:
newly_acquired = True
log.debug('key-was-absent', key=key, result=result)
# Check if reservation succeeded
resp = yield self.client.get(key)
if resp.kvs is not None and len(resp.kvs) == 1:
owner = resp.kvs[0].value
if owner == value:
if newly_acquired:
log.debug('key-reserved', key=key, value=value, ttl=ttl,
lease_id=lease.lease_id)
reserved = True
# Add key to reservation list
self.key_reservations[key] = lease
else:
log.debug("reservation-still-held")
else:
log.debug('reservation-held-by-another', value=owner)
if reserved:
err = None
returnValue((owner, err))
@inlineCallbacks
def _renew_reservation(self, key):
result = None
err = None
if key not in self.key_reservations:
err = 'key-not-reserved'
else:
lease = self.key_reservations[key]
# A successfully refreshed lease returns an object of type Header
result = yield lease.refresh()
if result is None:
err = 'lease-refresh-failed'
returnValue((result, err))
@inlineCallbacks
def _release_reservation(self, key):
err = None
if key not in self.key_reservations:
err = 'key-not-reserved'
else:
lease = self.key_reservations[key]
time_left = yield lease.remaining()
# A successfully revoked lease returns an object of type Header
log.debug('release-reservation', key=key, lease_id=lease.lease_id,
time_left_in_secs=time_left)
result = yield lease.revoke()
if result is None:
err = 'lease-revoke-failed'
self.key_reservations.pop(key)
returnValue((result, err))
@inlineCallbacks
def _release_all_reservations(self):
err = None
keys_to_delete = []
for key in self.key_reservations:
lease = self.key_reservations[key]
time_left = yield lease.remaining()
# A successfully revoked lease returns an object of type Header
log.debug('release-reservation', key=key, lease_id=lease.lease_id,
time_left_in_secs=time_left)
result = yield lease.revoke()
if result is None:
err = 'lease-revoke-failed'
log.debug('lease-revoke', result=result)
keys_to_delete.append(key)
for key in keys_to_delete:
self.key_reservations.pop(key)
returnValue(err)
def main(reactor):
etcd = Client(reactor)
#
# Example 1
#
for val in [b'val1', b'val2']:
yield etcd.set(b'test1', val)
txn = Transaction(
compare=[
# compute conjunction of all terms to
# determine "success" vs "failure"
CompValue(b'test1', '==', b'val1')
],
success=[
# if true ("success"), run these ops
OpSet(b'test1', b'val2'),
OpSet(b'test2', b'success')
],
failure=[
# if not true ("failure"), run these ops
OpSet(b'test2', b'failure'),
OpGet(b'test1')
])
try:
result = yield etcd.submit(txn)
except Failed as failed:
print('transaction FAILED:')
for response in failed.responses:
print(response)
else:
print('transaction SUCCESS:')
for response in result.responses:
print(response)
for key in [b'test1', b'test2']:
value = yield etcd.get(key)
print('{}: {}'.format(key, value))
#
# Example 2
#
rev = yield etcd.set(b'mykey1', os.urandom(8))
print(rev)
result = yield etcd.get(b'mykey1')
kv = result.kvs[0]
print(kv)
for version in [kv.version, kv.version - 1]:
txn = Transaction(
compare=[
# value equality comparison
CompValue(b'mykey1', '==', kv.value),
# version, and different comparison operators
CompVersion(b'mykey1', '==', version),
CompVersion(b'mykey1', '!=', version + 1),
CompVersion(b'mykey1', '<', version + 1),
CompVersion(b'mykey1', '>', version - 1),
# created revision comparison
CompCreated(b'mykey1', '==', kv.create_revision),
# modified revision comparison
CompModified(b'mykey1', '==', kv.mod_revision),
],
success=[
OpSet(b'mykey2', b'success'),
OpSet(b'mykey3', os.urandom(8))
],
failure=[
OpSet(b'mykey2', b'failure'),
OpSet(b'mykey3', os.urandom(8))
])
try:
result = yield etcd.submit(txn)
except Failed as failed:
print('transaction FAILED:')
for response in failed.responses:
print(response)
else:
print('transaction SUCCESS:')
for response in result.responses:
print(response)
result = yield etcd.get(KeySet(b'mykey2', b'mykey4'))
for kv in result.kvs:
print(kv)