def upsert_internal(self, kind, item):
r = redis.Redis(connection_pool=self._pool)
base_key = self._items_key(kind)
key = item['key']
item_json = json.dumps(item)
while True:
pipeline = r.pipeline()
pipeline.watch(base_key)
old = self.get_internal(kind, key)
if self.test_update_hook is not None:
self.test_update_hook(base_key, key)
if old and old['version'] >= item['version']:
log.debug(
'RedisFeatureStore: Attempted to %s key: %s version %d with a version that is the same or older: %d in "%s"',
'delete' if item.get('deleted') else 'update', key,
old['version'], item['version'], kind.namespace)
pipeline.unwatch()
return old
else:
pipeline.multi()
pipeline.hset(base_key, key, item_json)
try:
pipeline.execute()
# Unlike Redis implementations for other platforms, in redis-py a failed WATCH
# produces an exception rather than a null result from execute().
except redis.exceptions.WatchError:
log.debug(
"RedisFeatureStore: concurrent modification detected, retrying"
)
continue
return item
def _update_with_versioning(self, kind, item):
r = redis.Redis(connection_pool=self._pool)
base_key = self._items_key(kind)
key = item['key']
item_json = json.dumps(item)
while True:
pipeline = r.pipeline()
pipeline.watch(base_key)
old = self._get_even_if_deleted(kind, key, check_cache=False)
self._before_update_transaction(base_key, key)
if old and old['version'] >= item['version']:
log.debug('RedisFeatureStore: Attempted to %s key: %s version %d with a version that is the same or older: %d in "%s"',
'delete' if item.get('deleted') else 'update',
key, old['version'], item['version'], kind.namespace)
pipeline.unwatch()
break
else:
pipeline.multi()
pipeline.hset(base_key, key, item_json)
try:
pipeline.execute()
# Unlike Redis implementations for other platforms, in redis-py a failed WATCH
# produces an exception rather than a null result from execute().
except redis.exceptions.WatchError:
log.debug("RedisFeatureStore: concurrent modification detected, retrying")
continue
self._cache[self._cache_key(kind, key)] = item
break
def upsert_internal(self, kind, new_item):
key = self._item_key(kind, new_item['key'])
encoded_item = json.dumps(new_item)
# We will potentially keep retrying indefinitely until someone's write succeeds
while True:
index, old_value = self._client.kv.get(key)
if old_value is None:
mod_index = 0
else:
old_item = json.loads(old_value['Value'].decode('utf-8'))
# Check whether the item is stale. If so, don't do the update (and return the existing item to
# CachingStoreWrapper so it can be cached)
if old_item['version'] >= new_item['version']:
return old_item
mod_index = old_value['ModifyIndex']
# Otherwise, try to write. We will do a compare-and-set operation, so the write will only succeed if
# the key's ModifyIndex is still equal to the previous value. If the previous ModifyIndex was zero,
# it means the key did not previously exist and the write will only succeed if it still doesn't exist.
success = self._client.kv.put(key, encoded_item, cas=mod_index)
if success:
return new_item
log.debug('Concurrent modification detected, retrying')
def _get_even_if_deleted(self, kind, key, check_cache=True):
cacheKey = self._cache_key(kind, key)
if check_cache:
item = self._cache.get(cacheKey)
if item is not None:
# reset ttl
self._cache[cacheKey] = item
return item
try:
r = redis.Redis(connection_pool=self._pool)
item_json = r.hget(self._items_key(kind), key)
except BaseException as e:
log.error(
"RedisFeatureStore: Could not retrieve key %s from '%s' with error: %s",
key, kind.namespace, e.message)
return None
if item_json is None or item_json is "":
log.debug(
"RedisFeatureStore: key %s not found in '%s'. Returning None.",
key, kind.namespace)
return None
item = json.loads(item_json.decode('utf-8'))
self._cache[cacheKey] = item
return item
def get(self, kind, key, callback=lambda x: x):
item = self._get_even_if_deleted(kind, key, check_cache=True)
if item is not None and item.get('deleted', False) is True:
log.debug(
"RedisFeatureStore: get returned deleted item %s in '%s'. Returning None.",
key, kind.namespace)
return callback(None)
return callback(item)
def get_internal(self, kind, key):
r = redis.Redis(connection_pool=self._pool)
item_json = r.hget(self._items_key(kind), key)
if item_json is None or item_json == "":
log.debug(
"RedisFeatureStore: key %s not found in '%s'. Returning None.",
key, kind.namespace)
return None
return json.loads(item_json.decode('utf-8'))