def run_concurrent_readers_test(allocator_type): print('-- [%s] concurrent readers' % allocator_type) table = sharedstructures.PrefixTree('test-table', allocator_type) del table child_pids = set() while (len(child_pids) < 8) and (0 not in child_pids): child_pids.add(os.fork()) if 0 in child_pids: # child process: try up to a second to get the key table = sharedstructures.PrefixTree('test-table', allocator_type) value = 100 start_time = int(time.time() * 1000000) while (value < 110) and \ (int(time.time() * 1000000) < (start_time + 1000000)): time.sleep(0.001) try: res = table[b'key1'] except KeyError: pass else: if res == value: value += 1 os._exit(int(value != 110)) else: # parent process: write the key, then wait for children to terminate table = sharedstructures.PrefixTree('test-table', allocator_type) for value in range(100, 110): time.sleep(0.05) table[b'key1'] = value num_failures = 0 while child_pids: pid, exit_status = os.wait() child_pids.remove(pid) if os.WIFEXITED(exit_status) and (os.WEXITSTATUS(exit_status) == 0): print('-- [%s] child %d terminated successfully' % (allocator_type, pid)) else: print('-- [%s] child %d failed (%d)' % (allocator_type, pid, exit_status)) num_failures += 1 assert 0 == len(child_pids) assert 0 == num_failures
def run_complex_types_test(allocator_type): print('-- [%s] complex types' % allocator_type) table = sharedstructures.PrefixTree('test-table', allocator_type) expected = {} def insert_both(k, v): table[k] = v expected[k] = v def delete_both(k): del table[k] del expected[k] verify_state(expected, table) insert_both(b'key-list-empty', []) verify_state(expected, table) insert_both(b'key-list-ints', [1, 2, 3, 7]) verify_state(expected, table) insert_both(b'key-tuple-empty', ()) verify_state(expected, table) insert_both(b'key-tuple-ints', (1, 2, 3, 7)) verify_state(expected, table) insert_both(b'key-set-empty', set()) verify_state(expected, table) insert_both(b'key-set-ints', {1, 2, 3, 7}) verify_state(expected, table) insert_both(b'key-dict-empty', {}) verify_state(expected, table) insert_both(b'key-dict-ints', {1: 2, 3: 7}) verify_state(expected, table) table.clear() expected.clear() verify_state(expected, table)
def run_basic_test(allocator_type): print('-- [%s] basic' % allocator_type) before_lsof_count = len(get_current_process_lsof().splitlines()) table = sharedstructures.PrefixTree(POOL_NAME_PREFIX + allocator_type, allocator_type) expected = {} def insert_both(e, t, k, v): t[k] = v e[k] = v def delete_both(e, t, k): del t[k] del e[k] verify_state(expected, table) insert_both(expected, table, b'key1', b'value1') verify_state(expected, table) insert_both(expected, table, b'key2', b'value2') verify_state(expected, table) insert_both(expected, table, b'key3', b'value3') verify_state(expected, table) assert [b'key2', b'key3'] == list(table.keys_from(b'key2')) assert [(b'key2', b'value2'), (b'key3', b'value3')] == list(table.items_from(b'key2')) assert [b'key3'] == list(table.keys_from(b'key3')) assert [(b'key3', b'value3')] == list(table.items_from(b'key3')) assert [] == list(table.keys_from(b'key4')) assert [] == list(table.items_from(b'key4')) delete_both(expected, table, b'key2') verify_state(expected, table) try: del table[b'key2'] assert False, "del table[\'key2\'] did not raise KeyError" except KeyError: pass verify_state(expected, table) insert_both(expected, table, b'key1', b'value0') verify_state(expected, table) delete_both(expected, table, b'key1') verify_state(expected, table) delete_both(expected, table, b'key3') verify_state(expected, table) assert {} == expected del table # this should unmap the shared memory pool and close the fd sharedstructures.delete_pool(POOL_NAME_PREFIX + allocator_type) # make sure we didn't leak an fd assert before_lsof_count == len(get_current_process_lsof().splitlines())
def run_reorganization_test(allocator_type): print('-- [%s] reorganization' % allocator_type) table = sharedstructures.PrefixTree(POOL_NAME_PREFIX + allocator_type, allocator_type) expected = {} def insert_both(k): table[k] = k expected[k] = k def delete_both(k): del table[k] del expected[k] verify_state(expected, table) insert_both(b'abc') verify_state(expected, table) insert_both(b'ab') verify_state(expected, table) delete_both(b'abc') verify_state(expected, table) insert_both(b'') verify_state(expected, table) insert_both(b'abcd') verify_state(expected, table) delete_both(b'ab') verify_state(expected, table) insert_both(b'abcde') verify_state(expected, table) insert_both(b'abcdf') verify_state(expected, table) insert_both(b'abce') verify_state(expected, table) insert_both(b'abcef') verify_state(expected, table) table.clear() expected.clear() verify_state(expected, table)
def run_types_test(allocator_type): print('-- [%s] types' % allocator_type) table = sharedstructures.PrefixTree(POOL_NAME_PREFIX + allocator_type, allocator_type) expected = {} def insert_both(k, v): table[k] = v expected[k] = v def delete_both(k): del table[k] del expected[k] verify_state(expected, table) insert_both(b'key-string', b'value-string') verify_state(expected, table) insert_both(b'key-string-unicode', u'value-string-unicode') verify_state(expected, table) insert_both(b'key-int', 1024 * 1024 * -3) verify_state(expected, table) insert_both(b'key-int-long', 0x5555555555555555) verify_state(expected, table) insert_both(b'key-double', 2.38) verify_state(expected, table) insert_both(b'key-true', True) verify_state(expected, table) insert_both(b'key-false', False) verify_state(expected, table) insert_both(b'key-null', None) verify_state(expected, table) expect_key_missing(table, b'key-missing') # this calls exists() internally assert b'key-string' in table assert b'key-string-unicode' in table assert b'key-int' in table assert b'key-int-long' in table assert b'key-double' in table assert b'key-true' in table assert b'key-false' in table assert b'key-null' in table table.clear() expected.clear() verify_state(expected, table)
def get_shared_state(self) -> sharedstructures.PrefixTree: """Returns the shared state object, opening it if necessary.""" if self.shared_state is None: self.shared_state = sharedstructures.PrefixTree(self.shared_state_filename(), 'logarithmic', 0) return self.shared_state
def run_conditional_writes_test(allocator_type): print("-- [%s] conditional writes" % allocator_type) table = sharedstructures.PrefixTree('test-table', allocator_type) expected = {} def insert_both(e, t, k, v): t[k] = v e[k] = v def delete_both(e, t, k): del t[k] del e[k] def conditional_insert_both(e, t, check_k, check_v, target_k, target_v, written): if t.check_and_set(check_k, check_v, target_k, target_v): e[target_k] = target_v assert written else: assert not written def conditional_missing_insert_both(e, t, check_k, target_k, target_v, written): if t.check_missing_and_set(check_k, target_k, target_v): e[target_k] = target_v assert written else: assert not written def conditional_delete_both(e, t, check_k, check_v, target_k, written): if t.check_and_set(check_k, check_v, target_k): del e[target_k] assert written else: assert not written def conditional_missing_delete_both(e, t, check_k, target_k, written): if t.check_missing_and_set(check_k, target_k): del e[target_k] assert written else: assert not written verify_state(expected, table) insert_both(expected, table, b"key1", b"value1") verify_state(expected, table) insert_both(expected, table, b"key2", 10.0) verify_state(expected, table) insert_both(expected, table, b"key3", True) verify_state(expected, table) # check that conditions on the same key work for various types conditional_insert_both(expected, table, b"key1", b"value2", b"key1", b"value1_1", False) verify_state(expected, table) conditional_insert_both(expected, table, b"key1", b"value1", b"key1", b"value1_1", True) verify_state(expected, table) conditional_insert_both(expected, table, b"key2", 8.0, b"key2", 15.0, False) verify_state(expected, table) conditional_insert_both(expected, table, b"key2", 10.0, b"key2", 15.0, True) verify_state(expected, table) conditional_insert_both(expected, table, b"key3", False, b"key3", False, False) verify_state(expected, table) conditional_insert_both(expected, table, b"key3", True, b"key3", False, True) verify_state(expected, table) # now: # key1 = "value1_1" # key2 = 15.0 # key3 = False # check that conditions on other keys work conditional_insert_both(expected, table, b"key3", True, b"key1", b"value1", False) verify_state(expected, table) conditional_insert_both(expected, table, b"key3", False, b"key1", b"value1", True) verify_state(expected, table) conditional_insert_both(expected, table, b"key1", b"value2", b"key2", 10.0, False) verify_state(expected, table) conditional_insert_both(expected, table, b"key1", b"value1", b"key2", 10.0, True) verify_state(expected, table) conditional_insert_both(expected, table, b"key2", 20.0, b"key3", True, False) verify_state(expected, table) conditional_insert_both(expected, table, b"key2", 10.0, b"key3", True, True) verify_state(expected, table) # now: # key1 = "value1" # key2 = 10.0 # key3 = True # check that Missing conditions work conditional_insert_both(expected, table, b"key4", None, b"key4", None, False) verify_state(expected, table) conditional_missing_insert_both(expected, table, b"key2", b"key4", None, False) verify_state(expected, table) conditional_missing_insert_both(expected, table, b"key4", b"key4", None, True) verify_state(expected, table) # now: # key1 = "value1" # key2 = 10.0 # key3 = True # key4 = None # check that conditional deletes work conditional_delete_both(expected, table, b"key1", b"value2", b"key1", False) verify_state(expected, table) conditional_delete_both(expected, table, b"key1", b"value1", b"key1", True) verify_state(expected, table) conditional_delete_both(expected, table, b"key2", 20.0, b"key2", False) verify_state(expected, table) conditional_delete_both(expected, table, b"key2", 10.0, b"key2", True) verify_state(expected, table) conditional_delete_both(expected, table, b"key3", False, b"key3", False) verify_state(expected, table) conditional_delete_both(expected, table, b"key3", True, b"key3", True) verify_state(expected, table) conditional_missing_delete_both(expected, table, b"key4", b"key4", False) verify_state(expected, table) conditional_delete_both(expected, table, b"key4", None, b"key4", True) verify_state(expected, table) conditional_missing_delete_both(expected, table, b"key4", b"key4", False) verify_state(expected, table) assert expected == {}
def run_incr_test(allocator_type): print('-- [%s] incr' % allocator_type) table = sharedstructures.PrefixTree('test-table', allocator_type) expected = {} def insert_both(k, v): table[k] = v expected[k] = v def delete_both(k): del table[k] del expected[k] assert 0 == len(table) table[b'key-int'] = 10 table[b'key-int-long'] = 0x3333333333333333 table[b'key-double'] = 1.0 assert 3 == len(table) # giving garbage to incr() should cause a TypeError try: table.incr(b'key-missing', b'not a number, lolz') assert False except TypeError: pass try: table.incr(b'key-missing', {'still': 'not', 'a': 'number'}) assert False except TypeError: pass # incr should create the key if it doesn't exist assert 100 == table.incr(b'key-int2', 100) assert 0x5555555555555555 == table.incr(b'key-int-long2', 0x5555555555555555) assert 10.0 == table.incr(b'key-double2', 10.0) assert 100 == table[b'key-int2'] assert 0x5555555555555555 == table[b'key-int-long2'] assert 10.0 == table[b'key-double2'] assert 6 == len(table) # incr should return the new value of the key assert 99 == table.incr(b'key-int2', -1) assert 0.0 == table.incr(b'key-double2', -10.0) assert 99 == table[b'key-int2'] assert 0.0 == table[b'key-double2'] assert 6 == len(table) # test incr() on keys of the wrong type table[b'key-null'] = None table[b'key-string'] = b'value-string' assert 8 == len(table) try: table.incr(b'key-null', 13.0) assert False except ValueError: pass assert None == table[b'key-null'] try: table.incr(b'key-null', 13) assert False except ValueError: pass assert None == table[b'key-null'] try: table.incr(b'key-string', 13.0) assert False except ValueError: pass assert b'value-string' == table[b'key-string'] try: table.incr(b'key-string', 13) assert False except ValueError: pass assert b'value-string' == table[b'key-string'] try: table.incr(b'key-int', 13.0) assert False except ValueError: pass assert 10 == table[b'key-int'] try: table.incr(b'key-int-long', 13.0) assert False except ValueError: pass assert 0x3333333333333333 == table[b'key-int-long'] try: table.incr(b'key-int-long2', 13.0) assert False except ValueError: pass assert 0x5555555555555555 == table[b'key-int-long2'] try: table.incr(b'key-double', 13) assert False except ValueError: pass assert 1.0 == table[b'key-double'] # test converting integers between Int and Number assert 0x2AAAAAAAAAAAAAAA == table.incr(b'key-int', 0x2AAAAAAAAAAAAAA0) assert 8 == len(table) assert 3 == table.incr(b'key-int-long', -0x3333333333333330) assert 8 == len(table) # we're done here table.clear() assert len(table) == 0