def test_int2int_equal_when_same_keys_and_values():
a = Int2Int()
b = Int2Int()
for i in range(1, 7, 1):
a[i] = 100 + i
b[i] = 100 + i
assert a == b
def test_int2int_not_equal_when_same_size_same_keys_but_different_values():
a = Int2Int()
b = Int2Int()
for i in range(1, 7, 1):
a[i] = 100 + i
b[i] = 100 + i
b[6] = 200
assert a != b
def test_int2int_not_equal_when_different_size():
a = Int2Int()
b = Int2Int()
for i in range(1, 7, 1):
a[i] = 100 + i
b[i] = 100 + i
b[7] = 107
assert a != b
def test_int2int_repr_when_empty_and_default_kwarg():
int2int_map = Int2Int(default=100)
m = re.match(
r'\<cdatastructs.hashmap.Int2Int: object at 0x[0-9a-fA-F]+, '
r'used 0, default 100\>',
repr(int2int_map))
assert m is not None
def main():
# Random ids
ids = random.sample(range(1, IDS_COUNT + 1, 1), k=IDS_COUNT)
# Fill mapping ID to position
id2position = Int2Int()
for position, item_id in enumerate(ids):
id2position[item_id] = position
# Random values for calculate
a = array.array('d', (random.random() for unused in ids))
b = array.array('d', (random.random() for unused in ids))
# Random ids for calculate
ids_for_calculate = array.array(
'Q', random.choices(ids, k=random.randint(1, len(ids))))
# Array for results
result = array.array('d', (math.nan for unused in ids))
# Compute data
sum(ids_for_calculate, id2position.buffer_ptr, a, b, result)
for position, item_id in enumerate(ids):
if item_id in ids_for_calculate:
assert result[position] == a[position] + b[position]
else:
assert math.isnan(result[position])
print('PASS')
def worker(parentpid, shmkey, shmlock):
configure_logging()
logger = logging.getLogger('worker')
logger.info("Worker has been started with pid %d", os.getpid())
shm = sysv_ipc.SharedMemory(shmkey, flags=0, mode=0o600, size=0)
try:
signal.signal(signal.SIGINT, signal.SIG_IGN)
while 1:
try:
if shmlock.acquire(block=True, timeout=1):
try:
data = Int2Int.from_ptr(shm.address)
logger.info("Processing %d keys: %s", len(data),
list(data))
finally:
shmlock.release()
else:
raise logger.warning("SHM is not available")
except Exception:
logger.exception("Processing data error")
for unused in range(random.randint(0, 10)):
if os.getppid() != parentpid:
raise StopProcess
time.sleep(0.5)
except StopProcess:
logger.info("Stopping worker process")
finally:
shm.detach()
sys.exit(0)
def test_int2int_from_ptr(int2int_map):
int2int_map[1] = 101
int2int_map[2] = 102
addr = int2int_map.buffer_ptr
int2int_new_map = Int2Int.from_ptr(addr)
assert int2int_new_map[1] == 101
assert int2int_new_map[2] == 102
def test_int2int_pickle_dumps_loads_when_default():
int2int_map = Int2Int(default=0)
for i in (1, 2, 3, 4, 5):
int2int_map[i] = 100 + i
data = pickle.dumps(int2int_map)
new = pickle.loads(data)
assert int2int_map[6] == 0
assert new[6] == 0
assert new == int2int_map
assert set(new.keys()) == {1, 2, 3, 4, 5, 6}
assert set(new.values()) == {101, 102, 103, 104, 105, 0}
def test_int2int_new_when_prealloc_size_kwarg():
int2int_map = Int2Int(prealloc_size=1024)
class Int2IntHashTable_t(ctypes.Structure):
_fields_ = [
('size', ctypes.c_size_t),
('current_size', ctypes.c_size_t),
('table_size', ctypes.c_size_t),
('readonly', ctypes.c_bool),
]
t = Int2IntHashTable_t.from_address(int2int_map.buffer_ptr)
assert t.size == 1024
assert t.current_size == 0
def test_int2int_getitem_more_keys_does_not_exist_and_default_kwarg():
class Int2IntItem_t(ctypes.Structure):
_fields_ = [
('key', ctypes.c_ulonglong),
('value', ctypes.c_size_t),
('status', ctypes.c_int),
]
class Int2IntHashTable_t(ctypes.Structure):
_fields_ = [
('size', ctypes.c_size_t),
('current_size', ctypes.c_size_t),
('table_size', ctypes.c_size_t),
('readonly', ctypes.c_bool),
]
int2int_map = Int2Int(default=100)
t = Int2IntHashTable_t.from_address(int2int_map.buffer_ptr)
assert int2int_map.buffer_size == (
ctypes.sizeof(Int2IntHashTable_t)
+ (ctypes.sizeof(Int2IntItem_t) * t.table_size))
assert t.size == 8
assert t.current_size == 0
for i in range(16):
int2int_map[i]
t = Int2IntHashTable_t.from_address(int2int_map.buffer_ptr)
assert int2int_map.buffer_size == (
ctypes.sizeof(Int2IntHashTable_t)
+ (ctypes.sizeof(Int2IntItem_t) * t.table_size))
assert len(int2int_map) == 16
assert set(int2int_map.items()) == {
(0, 100), (1, 100), (2, 100), (3, 100),
(4, 100), (5, 100), (6, 100), (7, 100),
(8, 100), (9, 100), (10, 100), (11, 100),
(12, 100), (13, 100), (14, 100), (15, 100),
}
assert t.size == 16
assert t.current_size == 16
def indexer(parentpid, shmkey, shmlock):
configure_logging()
logger = logging.getLogger('indexer')
logger.info("Indexer has been started with pid %d", os.getpid())
shm_filled = False
shm = sysv_ipc.SharedMemory(shmkey, flags=0, mode=0o600, size=0)
try:
signal.signal(signal.SIGINT, signal.SIG_IGN)
while 1:
try:
data = Int2Int()
for i in range(random.randint(0, 16)):
entity_id = random.randint(100, 999)
data[entity_id] = i
data.make_readonly()
if shm_filled:
shmlock.acquire()
try:
if shm.size < data.buffer_size:
raise RuntimeError("SHM is too small")
ctypes.memmove(shm.address, data.buffer_ptr,
data.buffer_size)
shm_filled = True
finally:
if shm_filled:
shmlock.release()
logger.info("Created %d keys: %s", len(data), list(data))
except Exception:
logger.exception("Indexer error")
for unused in range(10):
if os.getppid() != parentpid:
raise StopProcess
time.sleep(1)
except StopProcess:
logger.info("Stopping indexer")
finally:
shm.detach()
sys.exit(0)
def test_int2int_new_fail_when_negative_default_kwarg():
with pytest.raises(TypeError, match="'default' must be positive int"):
Int2Int(default=-1)
def test_int2int_new_fail_when_invalid_default_kwarg_type():
with pytest.raises(TypeError, match="'default' must be positive int"):
Int2Int(default='8')
def test_int2int_new_fail_when_invalid_initializer(other, exc, msg):
with pytest.raises(exc, match=msg):
Int2Int(other)
def test_int2int_from_raw_data_fail_when_inconsistent_args(args, exc, msg):
with pytest.raises(exc, match=msg):
Int2Int._from_raw_data(*args)
def int2int_map():
return Int2Int()
def test_int2int_new_when_initializer_is_iterable(initializer):
int2int_map = Int2Int(initializer)
assert set(int2int_map.items()) == {(1, 101), (2, 102)}
def test_int2int_equal_when_empty():
a = Int2Int()
b = Int2Int()
assert a == b
def test_int2int_getitem_key_does_not_exist_and_default_kwarg():
int2int_map = Int2Int(default=100)
assert isinstance(int2int_map[1], int)
assert int2int_map[1] == 100
assert len(int2int_map) == 1
def test_int2int_getitem_key_does_not_exist_and_default_kwarg_is_none():
int2int_map = Int2Int(default=None)
with pytest.raises(KeyError) as exc_info:
int2int_map[1]
assert "'1'" in str(exc_info)
def test_int2int_new_when_initializer_is_mapping():
int2int_map = Int2Int({1: 101, 2: 102})
assert set(int2int_map.items()) == {(1, 101), (2, 102)}