def create(self, key, value):
"""Atomically create the given key only if the key doesn't exist.
This verifies that the create_revision of a key equales to 0, then
creates the key with the value.
This operation takes place in a transaction.
:param key: key in etcd to create
:param value: value of the key
:type value: bytes or string
:returns: status of transaction, ``True`` if the create was
successful, ``False`` otherwise
:rtype: bool
"""
base64_key = _encode(key)
base64_value = _encode(value)
txn = {
'compare': [{
'key': base64_key,
'result': 'EQUAL',
'target': 'CREATE',
'create_revision': 0
}],
'success': [{
'request_put': {
'key': base64_key,
'value': base64_value,
}
}],
'failure': []
}
result = self.transaction(txn)
if 'succeeded' in result:
return result['succeeded']
return False
def acquire(self):
"""Acquire the lock."""
self.lease = self.client.lease(self.ttl)
base64_key = _encode(self.key)
base64_value = _encode(self._uuid)
txn = {
'compare': [{
'key': base64_key,
'result': 'EQUAL',
'target': 'CREATE',
'create_revision': 0
}],
'success': [{
'request_put': {
'key': base64_key,
'value': base64_value,
'lease': self.lease.id
}
}],
'failure': [{
'request_range': {
'key': base64_key
}
}]
}
result = self.client.transaction(txn)
if 'succeeded' in result:
return result['succeeded']
return False
def delete(key, existing_value=None, mod_revision=None):
"""Delete a key/value pair from etcdv3.
- key (string): The key to delete.
- existing_value (string): If specified, indicates that the delete should
only proceed if the existing value is this.
- mod_revision (string): If specified, indicates that the delete should
only proceed if deleting an existing value with that mod_revision.
Returns True if the deletion was successful; False if not.
"""
client = _get_client()
LOG.debug("etcdv3 delete key=%s", key)
if mod_revision is not None:
base64_key = _encode(key)
txn = {
'compare': [{
'key': base64_key,
'result': 'EQUAL',
'target': 'MOD',
'mod_revision': mod_revision,
}],
'success': [{
'request_delete_range': {
'key': base64_key,
},
}],
'failure': [],
}
result = client.transaction(txn)
LOG.debug("transaction result %s", result)
deleted = result.get('succeeded', False)
elif existing_value is not None:
base64_key = _encode(key)
base64_existing = _encode(existing_value)
txn = {
'compare': [{
'key': base64_key,
'result': 'EQUAL',
'target': 'VALUE',
'value': base64_existing,
}],
'success': [{
'request_delete_range': {
'key': base64_key,
},
}],
'failure': [],
}
result = client.transaction(txn)
LOG.debug("transaction result %s", result)
deleted = result.get('succeeded', False)
else:
deleted = client.delete(key)
LOG.debug("etcdv3 deleted=%s", deleted)
return deleted
def get_all(self, sort_order=None, sort_target='key'):
"""Get all keys currently stored in etcd.
:returns: sequence of (value, metadata) tuples
"""
return self.get(
key=_encode(b'\0'),
metadata=True,
sort_order=sort_order,
sort_target=sort_target,
range_end=_encode(b'\0'),
)
def put(self, key, value, lease=None):
"""Put puts the given key into the key-value store.
A put request increments the revision of the key-value store
and generates one event in the event history.
:param key:
:param value:
:param lease:
:return: boolean
"""
payload = {"key": _encode(key), "value": _encode(value)}
if lease:
payload['lease'] = lease.id
self.post(self.get_url("/kv/put"), json=payload)
return True
def __init__(self, client, key, callback, **kwargs):
create_watch = {'key': _encode(key)}
for arg in kwargs:
if arg in self.KW_ARGS:
create_watch[arg] = kwargs[arg]
elif arg in self.KW_ENCODED_ARGS:
create_watch[arg] = _encode(kwargs[arg])
create_request = {"create_request": create_watch}
self._response = client.session.post(client.get_url('/watch'),
json=create_request,
stream=True)
clazz = _get_threadpool_executor()
self._executor = clazz(max_workers=2)
self._executor.submit(_watch, self._response, callback)
def get_prefix(self, key_prefix, sort_order=None, sort_target=None):
"""Get a range of keys with a prefix.
:param sort_order: 'ascend' or 'descend' or None
:param key_prefix: first key in range
:returns: sequence of (value, metadata) tuples
"""
return self.get(key_prefix,
metadata=True,
range_end=_encode(_increment_last_byte(key_prefix)),
sort_order=sort_order)
def replace(self, key, initial_value, new_value):
"""Atomically replace the value of a key with a new value.
This compares the current value of a key, then replaces it with a new
value if it is equal to a specified value. This operation takes place
in a transaction.
:param key: key in etcd to replace
:param initial_value: old value to replace
:type initial_value: bytes or string
:param new_value: new value of the key
:type new_value: bytes or string
:returns: status of transaction, ``True`` if the replace was
successful, ``False`` otherwise
:rtype: bool
"""
base64_key = _encode(key)
base64_initial_value = _encode(initial_value)
base64_new_value = _encode(new_value)
txn = {
'compare': [{
'key': base64_key,
'result': 'EQUAL',
'target': 'VALUE',
'value': base64_initial_value
}],
'success': [{
'request_put': {
'key': base64_key,
'value': base64_new_value,
}
}],
'failure': []
}
result = self.transaction(txn)
if 'succeeded' in result:
return result['succeeded']
return False
def get(self,
key,
metadata=False,
sort_order=None,
sort_target=None,
**kwargs):
"""Range gets the keys in the range from the key-value store.
:param key:
:param metadata:
:param sort_order: 'ascend' or 'descend' or None
:param sort_target: 'key' or 'version' or 'create' or 'mod' or 'value'
:param kwargs:
:return:
"""
try:
order = 0
if sort_order:
order = _SORT_ORDER.index(sort_order) + 1
except ValueError:
raise ValueError('sort_order must be one of "ascend" or "descend"')
try:
target = 0
if sort_target:
target = _SORT_TARGET.index(sort_target) + 1
except ValueError:
raise ValueError('sort_target must be one of "key", '
'"version", "create", "mod" or "value"')
payload = {
"key": _encode(key),
"sort_order": order,
"sort_target": target,
}
payload.update(kwargs)
result = self.post(self.get_url("/kv/range"), json=payload)
if 'kvs' not in result:
return []
if metadata:
def value_with_metadata(item):
item['key'] = _decode(item['key'])
value = _decode(item.pop('value'))
return value, item
return [value_with_metadata(item) for item in result['kvs']]
else:
return [_decode(item['value']) for item in result['kvs']]
def release(self):
"""Release the lock"""
base64_key = _encode(self.key)
base64_value = _encode(self._uuid)
txn = {
'compare': [{
'key': base64_key,
'result': 'EQUAL',
'target': 'VALUE',
'value': base64_value
}],
'success': [{
'request_delete_range': {
'key': base64_key
}
}]
}
result = self.client.transaction(txn)
if 'succeeded' in result:
return result['succeeded']
return False
def delete(self, key, **kwargs):
"""DeleteRange deletes the given range from the key-value store.
A delete request increments the revision of the key-value store and
generates a delete event in the event history for every deleted key.
:param key:
:param kwargs:
:return:
"""
payload = {
"key": _encode(key),
}
payload.update(kwargs)
result = self.post(self.get_url("/kv/deleterange"), json=payload)
if 'deleted' in result:
return True
return False
def delete_prefix(self, key_prefix):
"""Delete a range of keys with a prefix in etcd."""
return self.delete(
key_prefix, range_end=_encode(_increment_last_byte(key_prefix)))
def get_prefix(prefix, revision=None):
"""Read all etcdv3 data whose key begins with a given prefix.
- prefix (string): The prefix.
- revision: The revision to do the get at. If not specified then the
current revision is used.
Returns a list of tuples (key, value, mod_revision), one for each key-value
pair, in which:
- key is the etcd key (a string)
- value is the etcd value (also a string; note *not* JSON-decoded)
- mod_revision is the revision at which that key was last modified (an
integer represented as a string).
Note: this entrypoint is only used for data outside the Calico v3 data
model; specifically for legacy Calico v1 status notifications. This
entrypoint should be removed once those status notifications have been
reimplemented within the Calico v3 data model.
"""
client = _get_client()
if revision is None:
_, revision = get_status()
LOG.debug("Doing get at current revision: %r", revision)
# The JSON gateway can only return a certain number of bytes in a single
# response so we chunk up the read into blocks.
#
# Since etcd's get protocol has an inclusive range_start and an exclusive
# range_end, we load the keys in reverse order. That way, we can use the
# final key in each chunk as the next range_end.
range_end = _encode(_increment_last_byte(prefix))
results = []
while True:
# Note: originally, we included the sort_target parameter here but
# etcdgw has a bug (https://github.com/dims/etcd3-gateway/issues/18),
# which prevents that from working. In any case, sort-by-key is the
# default, which is what we want.
chunk = client.get(prefix,
metadata=True,
range_end=range_end,
sort_order='descend',
limit=CHUNK_SIZE_LIMIT,
revision=str(revision))
results.extend(chunk)
if len(chunk) < CHUNK_SIZE_LIMIT:
# Partial (or empty) chunk signals that we're done.
break
_, data = chunk[-1]
range_end = _encode(data["key"])
LOG.debug("etcdv3 get_prefix %s results=%s", prefix, len(results))
tuples = []
for result in results:
value, item = result
t = (item['key'].decode(), value.decode(), item['mod_revision'])
tuples.append(t)
return tuples
def put(key, value, mod_revision=None, lease=None, existing_value=None):
"""Write a key/value pair to etcdv3.
- key (string): The key to write.
- value (string): The value to write.
- mod_revision (string): If specified, indicates that the write should only
proceed if replacing an existing value with that mod_revision.
mod_revision=0 indicates that the key must not yet exist, i.e. that this
write will create it.
- lease: If specified, a Lease object to associate with the key.
- existing_value (string): If specified, indicates that the write should
only proceed if replacing that existing value.
Returns True if the write happened successfully; False if not.
"""
client = _get_client()
LOG.debug("etcdv3 put key=%s value=%s mod_revision=%r",
key, value, mod_revision)
txn = {}
if mod_revision == 0:
# Write operation must _create_ the KV entry.
base64_key = _encode(key)
txn['compare'] = [{
'key': base64_key,
'result': 'EQUAL',
'target': 'VERSION',
'version': 0,
}]
elif mod_revision == MUST_UPDATE:
# Write operation must update and _not_ create the KV entry.
base64_key = _encode(key)
txn['compare'] = [{
'key': base64_key,
'result': 'NOT_EQUAL',
'target': 'VERSION',
'version': 0,
}]
elif mod_revision is not None:
# Write operation must _replace_ a KV entry with the specified revision.
base64_key = _encode(key)
txn['compare'] = [{
'key': base64_key,
'result': 'EQUAL',
'target': 'MOD',
'mod_revision': mod_revision,
}]
elif existing_value is not None:
# Write operation must _replace_ a KV entry with the specified value.
base64_key = _encode(key)
base64_existing = _encode(existing_value)
txn['compare'] = [{
'key': base64_key,
'result': 'EQUAL',
'target': 'VALUE',
'value': base64_existing,
}]
if txn:
base64_value = _encode(value)
txn['success'] = [{
'request_put': {
'key': base64_key,
'value': base64_value,
},
}]
txn['failure'] = []
if lease is not None:
txn['success'][0]['request_put']['lease'] = lease.id
result = client.transaction(txn)
LOG.debug("transaction result %s", result)
succeeded = result.get('succeeded', False)
else:
succeeded = client.put(key, value, lease=lease)
return succeeded
def watch_prefix_once(self, key_prefix, timeout=None, **kwargs):
"""Watches a range of keys with a prefix, similar to watch_once"""
kwargs['range_end'] = \
_increment_last_byte(_encode(key_prefix))
return self.watch_once(key_prefix, timeout=timeout, **kwargs)
def watch_prefix(self, key_prefix, **kwargs):
"""The same as ``watch``, but watches a range of keys with a prefix."""
kwargs['range_end'] = \
_increment_last_byte(_encode(key_prefix))
return self.watch(key_prefix, **kwargs)
