def test_admin_garbage_collection(num=100):
rdz = Rediz(**REDIZ_TEST_CONFIG)
original_num = rdz.card()
names = [rdz.random_name() for _ in range(num)]
write_keys = [BELLEHOOD_BAT for _ in names]
values = ["from test_admin_garbage_collection" for _ in write_keys]
budgets = [1 for _ in range(num)]
mset_res = rdz.mset(names=names,
write_keys=write_keys,
values=values,
budgets=budgets)
assert len(mset_res) == len(names)
expire_pipe = rdz.client.pipeline()
for name in names:
expire_pipe.expire(name=name, time=0)
expire_pipe.execute()
time.sleep(0.15)
remaining = list()
for iter_no in range(5):
rdz.admin_garbage_collection(fraction=0.01)
remaining.append(rdz.card())
final_num = rdz.card()
rdz._delete_implementation(*names)
def test_mixed_log():
rdz = Rediz(**REDIZ_TEST_CONFIG)
names = [None, None, random_name()]
write_keys = [random_key(), None, random_key()]
values = [json.dumps([7.6 for _ in range(1000)]), "cat", json.dumps("dog")]
budgets = [1 for _ in names]
result = rdz._pipelined_set(names=names,
write_keys=write_keys,
values=values,
budgets=budgets)
rdz._delete_implementation(names)
def test_expire():
rdz = Rediz(**REDIZ_TEST_CONFIG)
name = rdz.random_name()
write_key = BELLEHOOD_BAT
assert rdz.set(value="44", name=name, write_key=write_key)
rdz.client.expire(name=name, time=0)
import time
time.sleep(0.1)
assert rdz.get(name) is None
assert rdz.client.sismember(name=rdz._NAMES, value=name)
rdz._delete_implementation(names=[name])
def test_mixed():
rdz = Rediz(**REDIZ_TEST_CONFIG)
# Create two new records ... one rejected due to a poor key
names = [None, None, random_name()]
write_keys = [BELLEHOOD_BAT, TASTEABLE_BEE, 'too-short']
values = [json.dumps(8), "cat", json.dumps("dog")]
budgets = [1, 10, 10000]
execution_log = rdz._pipelined_set(names=names,
write_keys=write_keys,
values=values,
budgets=budgets)
assert len(execution_log["executed"]) == 2, "Expected 2 to be executed"
assert len(execution_log["rejected"]) == 1, "Expected 1 rejection"
assert execution_log["executed"][0]["ttl"] > 25, "Expected ttl>25 seconds"
assert sum([int(t["obscure"] == True)
for t in execution_log["executed"]]) == 2, "Expected 2 obscure"
assert sum([int(t["new"] == True)
for t in execution_log["executed"]]) == 2, "Expected 2 new"
rdz._delete_implementation(names)
def test_set_integer():
rdz = Rediz(**REDIZ_TEST_CONFIG)
title = {
"name": "3912eb73-f5e6-4f5e-9674-1a320779b7d9.json",
"write_key": BELLEHOOD_BAT
}
rdz.delete(**title) # Previous run
res = rdz._pipelined_set(values=[25],
names=[title["name"]],
write_keys=[title["write_key"]],
budgets=[1])
dump(res)
assert res["executed"][0]["value"] == 25
access = {
"name": "3912eb73-f5e6-4f5e-9674-1a320779b7d9.json",
"write_key": TASTEABLE_BEE,
"value": 17
}
assert rdz.set(**access) is not None
rdz._delete_implementation(access["name"])
time.sleep(0.15)
assert not rdz.exists(access["name"])
if __name__ == '__main__':
rdz = Rediz(**REDIZ_COLLIDER_CONFIG)
ownership = rdz.client.hgetall(rdz._OWNERSHIP)
PLAYERS = {
'Hikaru': 'hikaru_nakamura',
'Firouzja2003': 'alireza_firouzja',
'GMWSO': 'wesley_so',
'LyonBeast': 'maxime_vachier_lagrave',
'nihalsarin': 'nihal_sarin',
'DanielNaroditsky': 'daniel_naroditsky',
'PinIsMightier': 'halloween_gambit'
}
URL_TEMPLATE = 'https://api.chess.com/pub/player/HANDLE/stats'
CATEGORIES = ['chess_blitz', 'chess_bullet']
if __name__ == '__main__':
# Chess.Com ratings
names_to_delete = list()
for category in CATEGORIES:
for handle, player in PLAYERS.items():
name = category + '_' + handle + '.json'
print(name)
names_to_delete.append(name)
if SURE:
for name in names_to_delete:
resultz = rdz._delete_implementation(names=names_to_delete)
def test_find_orphans_low_cardinality_test(num=20):
rdz = Rediz(**REDIZ_TEST_CONFIG)
original_num = rdz.card()
if original_num < 10000:
# This test won't ultimately scale as it calls smembers
original_set = rdz.client.smembers(rdz._NAMES)
for k in [5, 20, 400]:
some = rdz.client.srandmember(rdz._NAMES, k)
some_unique = list(set(some))
assert all(s in original_set for s in some)
assert len(some_unique) <= original_num
# Create some data with short ttl
names = [rdz.random_name() for _ in range(num)]
write_key = BELLEHOOD_BAT
value = "a string to store"
values = [value for _ in names]
budgets = [7 for _ in names]
title = rdz.mset(names=names,
write_key=write_key,
values=values,
budgets=budgets)
assert rdz.client.exists(
*names) == len(names), "Names were not created as expected."
for name in names:
rdz.client.expire(name=name, time=5 * 60)
expiring_names = [
n for n in names
if random.choice(['expired', 'living']) == 'expired'
]
for expiring_name in expiring_names:
rdz.client.expire(name=expiring_name, time=0)
time.sleep(0.1)
assert rdz.client.exists(
*expiring_names) == 0, "Names did not expire as expected"
some_orphans = rdz._randomly_find_orphans(num=50)
# Clean up most
almost_all_orphans = rdz._randomly_find_orphans(num=10 * num)
if almost_all_orphans:
rdz._delete_implementation(*almost_all_orphans)
# Clean up scraps
rdz._delete_implementation(*names)
final_num = rdz.card()
final_set = rdz.client.smembers(rdz._NAMES)
set_diff_1 = final_set.difference(original_set)
set_diff_2 = original_set.difference(final_set)
if False:
dump({
"original_num": original_num,
"num_added": len(names),
"final_num": final_num,
"num_orphans": len(some_orphans),
"num_orphans_wider_search": len(almost_all_orphans),
"set_diff_1": list(set_diff_1),
"set_diff_2": list(set_diff_2)
})
from rediz.client import Rediz
from rediz.collider_config_private import REDIZ_COLLIDER_CONFIG
from pprint import pprint
if __name__ == '__main__':
rdz = Rediz(**REDIZ_COLLIDER_CONFIG)
ownership = rdz.client.hgetall(rdz._OWNERSHIP)
for name, write_key in ownership.items():
print(name)
if 'z3~' in name and ('910' in name or '310' in name ):
print('Del '+name)
rdz._delete_implementation(name)
def subscription_example(plural=False, instant_recall=False):
""" We subscribe first to one publisher and then to 50 at once, and
verify propagation and cleanup after delete/unsubscribe
"""
rdz = Rediz(instant_recall=instant_recall, **REDIZ_TEST_CONFIG)
assert not rdz.get_subscriptions(name='non-existent-asdfaf.json')
PUBLISHER = 'PUBLISQHER_plural_' + str(
plural) + '3b4e229a-ffb4-4fc2-8370-c147944aa2b.json'
SUBSCRIBER = 'SUBSCRIIBER_plural_' + str(
plural) + 'ed2b4f6-c6bd-464c-a9e9-322e0c3147.json'
PUBLISHER_write_key = BELLEHOOD_BAT if plural else TASTEABLE_BEE
SUBSCRIBER_write_key = SHOOTABLE_CAT if plural else BABLOH_CATTLE
rdz._delete_implementation(PUBLISHER, SUBSCRIBER)
assert rdz.set(name=SUBSCRIBER,
value="some value",
write_key=SUBSCRIBER_write_key)
assert rdz.set(name=SUBSCRIBER,
value="some value",
write_key=SUBSCRIBER_write_key)
assert rdz.set(name=PUBLISHER,
value="some other value",
write_key=PUBLISHER_write_key)
if plural:
rdz.msubscribe(sources=[PUBLISHER],
name=SUBSCRIBER,
write_key=SUBSCRIBER_write_key)
else:
rdz.subscribe(source=PUBLISHER,
name=SUBSCRIBER,
write_key=SUBSCRIBER_write_key)
subscriptions = rdz.get_subscriptions(name=SUBSCRIBER)
assert PUBLISHER in subscriptions
subscribers = rdz.get_subscribers(name=PUBLISHER)
assert SUBSCRIBER in subscribers
# Check propagation
assert rdz.set(name=PUBLISHER,
value="propagate this",
write_key=PUBLISHER_write_key) # Should trigger propagation
messages = rdz.messages(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key)
assert messages[PUBLISHER] == "propagate this"
# Test removal
rdz.unsubscribe(name=SUBSCRIBER,
source=PUBLISHER,
write_key=SUBSCRIBER_write_key)
subscriptions = rdz.get_subscriptions(name=SUBSCRIBER)
assert PUBLISHER not in subscriptions
subscribers = rdz.get_subscribers(name=PUBLISHER)
assert SUBSCRIBER not in subscribers
# Test re-subscribe
rdz.subscribe(source=PUBLISHER,
name=SUBSCRIBER,
write_key=SUBSCRIBER_write_key)
# Check propagation
assert rdz.set(name=PUBLISHER,
value="propagate this",
write_key=PUBLISHER_write_key) # Should trigger propagation
messages = rdz.messages(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key)
assert messages[PUBLISHER] == "propagate this"
# Test removal with delete ...
subscribers = rdz.get_subscribers(name=PUBLISHER)
subscriptions = rdz.get_subscriptions(name=SUBSCRIBER)
rdz.delete(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key)
subscriptions = rdz.get_subscriptions(name=SUBSCRIBER)
subscribers = rdz.get_subscribers(name=PUBLISHER)
assert not subscriptions
assert SUBSCRIBER not in subscribers
# Multiple sources
publishers = dict()
NUM_PUBLISHERS = 50
for k in range(NUM_PUBLISHERS):
source = 'PUBLISHER_plural_' + (
str(plural).lower()) + '-number_' + str(k) + '__3b4e944aa2b.json'
write_key = rdz.create_key(difficulty=6)
publishers[source] = write_key
sources = list(publishers.keys())
write_keys = list(publishers.values())
values = list(range(NUM_PUBLISHERS))
assert rdz.set(
name=SUBSCRIBER,
value="I am back again",
write_key=SUBSCRIBER_write_key) # Should trigger propagation
budgets = [3 for _ in sources]
rdz.mset(names=sources,
values=values,
write_keys=write_keys,
budgets=budgets)
rdz.mset(names=sources,
values=values,
write_keys=write_keys,
budgets=budgets)
values_back = rdz.mget(names=sources)
assert all(int(v1) == int(v2) for v1, v2 in zip(values, values_back))
m_res = rdz.msubscribe(name=SUBSCRIBER,
sources=sources,
write_key=SUBSCRIBER_write_key)
assert m_res == len(sources)
subscriptions = rdz.get_subscriptions(name=SUBSCRIBER)
assert all(source in subscriptions for source in sources)
for source, write_key in publishers.items():
subscribers = rdz.get_subscribers(name=source)
assert SUBSCRIBER in subscribers
subscribers1 = rdz.get(
rdz.SUBSCRIBERS +
source) # 'publisher_plural_true-number_0__3b4e944aa2b.json'
assert SUBSCRIBER in subscribers1
# Propagate ...
changed_values = [int(2 * v) for v in values]
budgets = [2 for v in values]
rdz.mset(names=sources,
write_keys=write_keys,
values=changed_values,
budgets=budgets)
messages = rdz.messages(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key)
for source, v in zip(sources, changed_values):
assert messages[source] == str(v)
# One more time with feeling ....
changed_values = [int(3 * v) for v in values]
rdz.mset(names=sources,
write_keys=write_keys,
values=changed_values,
budgets=budgets)
messages = rdz.messages(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key)
for source, v in zip(sources, changed_values):
assert messages[source] == str(v)
# Multiple delete of sources
assert rdz.client.exists(*sources) == NUM_PUBLISHERS
rdz.mdelete(names=sources,
write_keys=write_keys) # Expires but does not delete
time.sleep(0.15) # Redis checks every 0.1 seconds (probably)
assert rdz.client.exists(
*
sources) < 10, "Fail could be bad luck" # Most will be deleted by now.
for source, write_key in publishers.items():
assert not rdz.get_subscribers(name=source)
subscriptions = rdz.get_subscriptions(SUBSCRIBER)
for source in sources:
assert source not in subscriptions
subscriptions = rdz.get_subscriptions(name=SUBSCRIBER)
for source in sources:
assert source not in subscriptions
# Messages may persist or not depending on settings
messages = rdz.messages(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key)
if rdz._INSTANT_RECALL:
for source in sources:
assert source not in messages
else:
assert len(messages) == NUM_PUBLISHERS
rdz.delete(name=PUBLISHER, write_key=PUBLISHER_write_key)
rdz.delete(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key)
from rediz.client import Rediz
from rediz.collider_config_private import REDIZ_COLLIDER_CONFIG
from pprint import pprint
STREAMS_TO_DELETE = [
'5m_cardano.json', '5m_ethereum.json', '5m_ripple.json', '5m_iota.json',
'ethereum_usd.json', 'ethereum_eur.json', 'bitcoin_usd.json',
'bitcoin_eur.json'
]
if __name__ == '__main__':
rdz = Rediz(**REDIZ_COLLIDER_CONFIG)
result = rdz._delete_implementation(names=STREAMS_TO_DELETE)
pprint(result)
resultz = rdz._delete_z1_implementation(names=STREAMS_TO_DELETE)
