def test_admin_garbage_collection(num=100): rdz = Rediz(**REDIZ_TEST_CONFIG) original_num = rdz.card() names = [rdz.random_name() for _ in range(num)] write_keys = [BELLEHOOD_BAT for _ in names] values = ["from test_admin_garbage_collection" for _ in write_keys] budgets = [1 for _ in range(num)] mset_res = rdz.mset(names=names, write_keys=write_keys, values=values, budgets=budgets) assert len(mset_res) == len(names) expire_pipe = rdz.client.pipeline() for name in names: expire_pipe.expire(name=name, time=0) expire_pipe.execute() time.sleep(0.15) remaining = list() for iter_no in range(5): rdz.admin_garbage_collection(fraction=0.01) remaining.append(rdz.card()) final_num = rdz.card() rdz._delete_implementation(*names)
def test_mixed_log(): rdz = Rediz(**REDIZ_TEST_CONFIG) names = [None, None, random_name()] write_keys = [random_key(), None, random_key()] values = [json.dumps([7.6 for _ in range(1000)]), "cat", json.dumps("dog")] budgets = [1 for _ in names] result = rdz._pipelined_set(names=names, write_keys=write_keys, values=values, budgets=budgets) rdz._delete_implementation(names)
def test_expire(): rdz = Rediz(**REDIZ_TEST_CONFIG) name = rdz.random_name() write_key = BELLEHOOD_BAT assert rdz.set(value="44", name=name, write_key=write_key) rdz.client.expire(name=name, time=0) import time time.sleep(0.1) assert rdz.get(name) is None assert rdz.client.sismember(name=rdz._NAMES, value=name) rdz._delete_implementation(names=[name])
def test_mixed(): rdz = Rediz(**REDIZ_TEST_CONFIG) # Create two new records ... one rejected due to a poor key names = [None, None, random_name()] write_keys = [BELLEHOOD_BAT, TASTEABLE_BEE, 'too-short'] values = [json.dumps(8), "cat", json.dumps("dog")] budgets = [1, 10, 10000] execution_log = rdz._pipelined_set(names=names, write_keys=write_keys, values=values, budgets=budgets) assert len(execution_log["executed"]) == 2, "Expected 2 to be executed" assert len(execution_log["rejected"]) == 1, "Expected 1 rejection" assert execution_log["executed"][0]["ttl"] > 25, "Expected ttl>25 seconds" assert sum([int(t["obscure"] == True) for t in execution_log["executed"]]) == 2, "Expected 2 obscure" assert sum([int(t["new"] == True) for t in execution_log["executed"]]) == 2, "Expected 2 new" rdz._delete_implementation(names)
def test_set_integer(): rdz = Rediz(**REDIZ_TEST_CONFIG) title = { "name": "3912eb73-f5e6-4f5e-9674-1a320779b7d9.json", "write_key": BELLEHOOD_BAT } rdz.delete(**title) # Previous run res = rdz._pipelined_set(values=[25], names=[title["name"]], write_keys=[title["write_key"]], budgets=[1]) dump(res) assert res["executed"][0]["value"] == 25 access = { "name": "3912eb73-f5e6-4f5e-9674-1a320779b7d9.json", "write_key": TASTEABLE_BEE, "value": 17 } assert rdz.set(**access) is not None rdz._delete_implementation(access["name"]) time.sleep(0.15) assert not rdz.exists(access["name"])
if __name__ == '__main__': rdz = Rediz(**REDIZ_COLLIDER_CONFIG) ownership = rdz.client.hgetall(rdz._OWNERSHIP) PLAYERS = { 'Hikaru': 'hikaru_nakamura', 'Firouzja2003': 'alireza_firouzja', 'GMWSO': 'wesley_so', 'LyonBeast': 'maxime_vachier_lagrave', 'nihalsarin': 'nihal_sarin', 'DanielNaroditsky': 'daniel_naroditsky', 'PinIsMightier': 'halloween_gambit' } URL_TEMPLATE = 'https://api.chess.com/pub/player/HANDLE/stats' CATEGORIES = ['chess_blitz', 'chess_bullet'] if __name__ == '__main__': # Chess.Com ratings names_to_delete = list() for category in CATEGORIES: for handle, player in PLAYERS.items(): name = category + '_' + handle + '.json' print(name) names_to_delete.append(name) if SURE: for name in names_to_delete: resultz = rdz._delete_implementation(names=names_to_delete)
def test_find_orphans_low_cardinality_test(num=20): rdz = Rediz(**REDIZ_TEST_CONFIG) original_num = rdz.card() if original_num < 10000: # This test won't ultimately scale as it calls smembers original_set = rdz.client.smembers(rdz._NAMES) for k in [5, 20, 400]: some = rdz.client.srandmember(rdz._NAMES, k) some_unique = list(set(some)) assert all(s in original_set for s in some) assert len(some_unique) <= original_num # Create some data with short ttl names = [rdz.random_name() for _ in range(num)] write_key = BELLEHOOD_BAT value = "a string to store" values = [value for _ in names] budgets = [7 for _ in names] title = rdz.mset(names=names, write_key=write_key, values=values, budgets=budgets) assert rdz.client.exists( *names) == len(names), "Names were not created as expected." for name in names: rdz.client.expire(name=name, time=5 * 60) expiring_names = [ n for n in names if random.choice(['expired', 'living']) == 'expired' ] for expiring_name in expiring_names: rdz.client.expire(name=expiring_name, time=0) time.sleep(0.1) assert rdz.client.exists( *expiring_names) == 0, "Names did not expire as expected" some_orphans = rdz._randomly_find_orphans(num=50) # Clean up most almost_all_orphans = rdz._randomly_find_orphans(num=10 * num) if almost_all_orphans: rdz._delete_implementation(*almost_all_orphans) # Clean up scraps rdz._delete_implementation(*names) final_num = rdz.card() final_set = rdz.client.smembers(rdz._NAMES) set_diff_1 = final_set.difference(original_set) set_diff_2 = original_set.difference(final_set) if False: dump({ "original_num": original_num, "num_added": len(names), "final_num": final_num, "num_orphans": len(some_orphans), "num_orphans_wider_search": len(almost_all_orphans), "set_diff_1": list(set_diff_1), "set_diff_2": list(set_diff_2) })
from rediz.client import Rediz from rediz.collider_config_private import REDIZ_COLLIDER_CONFIG from pprint import pprint if __name__ == '__main__': rdz = Rediz(**REDIZ_COLLIDER_CONFIG) ownership = rdz.client.hgetall(rdz._OWNERSHIP) for name, write_key in ownership.items(): print(name) if 'z3~' in name and ('910' in name or '310' in name ): print('Del '+name) rdz._delete_implementation(name)
def subscription_example(plural=False, instant_recall=False): """ We subscribe first to one publisher and then to 50 at once, and verify propagation and cleanup after delete/unsubscribe """ rdz = Rediz(instant_recall=instant_recall, **REDIZ_TEST_CONFIG) assert not rdz.get_subscriptions(name='non-existent-asdfaf.json') PUBLISHER = 'PUBLISQHER_plural_' + str( plural) + '3b4e229a-ffb4-4fc2-8370-c147944aa2b.json' SUBSCRIBER = 'SUBSCRIIBER_plural_' + str( plural) + 'ed2b4f6-c6bd-464c-a9e9-322e0c3147.json' PUBLISHER_write_key = BELLEHOOD_BAT if plural else TASTEABLE_BEE SUBSCRIBER_write_key = SHOOTABLE_CAT if plural else BABLOH_CATTLE rdz._delete_implementation(PUBLISHER, SUBSCRIBER) assert rdz.set(name=SUBSCRIBER, value="some value", write_key=SUBSCRIBER_write_key) assert rdz.set(name=SUBSCRIBER, value="some value", write_key=SUBSCRIBER_write_key) assert rdz.set(name=PUBLISHER, value="some other value", write_key=PUBLISHER_write_key) if plural: rdz.msubscribe(sources=[PUBLISHER], name=SUBSCRIBER, write_key=SUBSCRIBER_write_key) else: rdz.subscribe(source=PUBLISHER, name=SUBSCRIBER, write_key=SUBSCRIBER_write_key) subscriptions = rdz.get_subscriptions(name=SUBSCRIBER) assert PUBLISHER in subscriptions subscribers = rdz.get_subscribers(name=PUBLISHER) assert SUBSCRIBER in subscribers # Check propagation assert rdz.set(name=PUBLISHER, value="propagate this", write_key=PUBLISHER_write_key) # Should trigger propagation messages = rdz.messages(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key) assert messages[PUBLISHER] == "propagate this" # Test removal rdz.unsubscribe(name=SUBSCRIBER, source=PUBLISHER, write_key=SUBSCRIBER_write_key) subscriptions = rdz.get_subscriptions(name=SUBSCRIBER) assert PUBLISHER not in subscriptions subscribers = rdz.get_subscribers(name=PUBLISHER) assert SUBSCRIBER not in subscribers # Test re-subscribe rdz.subscribe(source=PUBLISHER, name=SUBSCRIBER, write_key=SUBSCRIBER_write_key) # Check propagation assert rdz.set(name=PUBLISHER, value="propagate this", write_key=PUBLISHER_write_key) # Should trigger propagation messages = rdz.messages(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key) assert messages[PUBLISHER] == "propagate this" # Test removal with delete ... subscribers = rdz.get_subscribers(name=PUBLISHER) subscriptions = rdz.get_subscriptions(name=SUBSCRIBER) rdz.delete(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key) subscriptions = rdz.get_subscriptions(name=SUBSCRIBER) subscribers = rdz.get_subscribers(name=PUBLISHER) assert not subscriptions assert SUBSCRIBER not in subscribers # Multiple sources publishers = dict() NUM_PUBLISHERS = 50 for k in range(NUM_PUBLISHERS): source = 'PUBLISHER_plural_' + ( str(plural).lower()) + '-number_' + str(k) + '__3b4e944aa2b.json' write_key = rdz.create_key(difficulty=6) publishers[source] = write_key sources = list(publishers.keys()) write_keys = list(publishers.values()) values = list(range(NUM_PUBLISHERS)) assert rdz.set( name=SUBSCRIBER, value="I am back again", write_key=SUBSCRIBER_write_key) # Should trigger propagation budgets = [3 for _ in sources] rdz.mset(names=sources, values=values, write_keys=write_keys, budgets=budgets) rdz.mset(names=sources, values=values, write_keys=write_keys, budgets=budgets) values_back = rdz.mget(names=sources) assert all(int(v1) == int(v2) for v1, v2 in zip(values, values_back)) m_res = rdz.msubscribe(name=SUBSCRIBER, sources=sources, write_key=SUBSCRIBER_write_key) assert m_res == len(sources) subscriptions = rdz.get_subscriptions(name=SUBSCRIBER) assert all(source in subscriptions for source in sources) for source, write_key in publishers.items(): subscribers = rdz.get_subscribers(name=source) assert SUBSCRIBER in subscribers subscribers1 = rdz.get( rdz.SUBSCRIBERS + source) # 'publisher_plural_true-number_0__3b4e944aa2b.json' assert SUBSCRIBER in subscribers1 # Propagate ... changed_values = [int(2 * v) for v in values] budgets = [2 for v in values] rdz.mset(names=sources, write_keys=write_keys, values=changed_values, budgets=budgets) messages = rdz.messages(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key) for source, v in zip(sources, changed_values): assert messages[source] == str(v) # One more time with feeling .... changed_values = [int(3 * v) for v in values] rdz.mset(names=sources, write_keys=write_keys, values=changed_values, budgets=budgets) messages = rdz.messages(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key) for source, v in zip(sources, changed_values): assert messages[source] == str(v) # Multiple delete of sources assert rdz.client.exists(*sources) == NUM_PUBLISHERS rdz.mdelete(names=sources, write_keys=write_keys) # Expires but does not delete time.sleep(0.15) # Redis checks every 0.1 seconds (probably) assert rdz.client.exists( * sources) < 10, "Fail could be bad luck" # Most will be deleted by now. for source, write_key in publishers.items(): assert not rdz.get_subscribers(name=source) subscriptions = rdz.get_subscriptions(SUBSCRIBER) for source in sources: assert source not in subscriptions subscriptions = rdz.get_subscriptions(name=SUBSCRIBER) for source in sources: assert source not in subscriptions # Messages may persist or not depending on settings messages = rdz.messages(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key) if rdz._INSTANT_RECALL: for source in sources: assert source not in messages else: assert len(messages) == NUM_PUBLISHERS rdz.delete(name=PUBLISHER, write_key=PUBLISHER_write_key) rdz.delete(name=SUBSCRIBER, write_key=SUBSCRIBER_write_key)
from rediz.client import Rediz from rediz.collider_config_private import REDIZ_COLLIDER_CONFIG from pprint import pprint STREAMS_TO_DELETE = [ '5m_cardano.json', '5m_ethereum.json', '5m_ripple.json', '5m_iota.json', 'ethereum_usd.json', 'ethereum_eur.json', 'bitcoin_usd.json', 'bitcoin_eur.json' ] if __name__ == '__main__': rdz = Rediz(**REDIZ_COLLIDER_CONFIG) result = rdz._delete_implementation(names=STREAMS_TO_DELETE) pprint(result) resultz = rdz._delete_z1_implementation(names=STREAMS_TO_DELETE)