def decode(msg, item_size, decoder):
frame = msg.next_frame()
type = FixSizedTypesCodec.decode_byte(frame.buf, 0)
count = FixSizedTypesCodec.decode_int(frame.buf, 1)
if type == ListCNFixedSizeCodec._TYPE_NULL_ONLY:
return [None] * count
elif type == ListCNFixedSizeCodec._TYPE_NOT_NULL_ONLY:
header_size = ListCNFixedSizeCodec._HEADER_SIZE
return [
decoder(frame.buf, header_size + i * item_size)
for i in range(count)
]
else:
response = [None] * count
position = ListCNFixedSizeCodec._HEADER_SIZE
read_count = 0
items_per_bitmask = ListCNFixedSizeCodec._ITEMS_PER_BITMASK
while read_count < count:
bitmask = FixSizedTypesCodec.decode_byte(frame.buf, position)
position += 1
batch_size = min(items_per_bitmask, count - read_count)
for i in range(batch_size):
mask = 1 << i
if (bitmask & mask) == mask:
response[read_count] = decoder(frame.buf, position)
position += item_size
read_count += 1
return response
def _fill_map(self, key_count=5, value_count=5):
map = {
"key-%d" % x: ["value-%d-%d" % (x, y) for y in range(0, value_count)]
for x in range(0, key_count)
}
for k, l in six.iteritems(map):
for v in l:
self.multi_map.put(k, v)
return map
def test_LRU_time(self):
near_cache = self.create_near_cache(self.service, IN_MEMORY_FORMAT.OBJECT, 1000, 1000, EVICTION_POLICY.LRU, 10000, 16, 16)
for i in range(0, 10000):
key = "key-{}".format(i)
value = "value-{}".format(i)
near_cache[key] = value
self.assertEqual("value-0", near_cache["key-0"])
for i in range(10001, 20000):
key = "key-{}".format(i)
value = "value-{}".format(i)
near_cache[key] = value
stats = near_cache.get_statistics()
evict, expire = stats["evictions"], stats["expirations"]
self.assertEqual(expire, 0)
self.assertLess(evict, 10000)
def test_expiry_time(self):
near_cache = self.create_near_cache(self.service, IN_MEMORY_FORMAT.OBJECT, 1, 1000, EVICTION_POLICY.LRU, 1000)
for i in range(0, 1000):
key = "key-{}".format(i)
value = "value-{}".format(i)
near_cache[key] = value
sleep(2)
for i in range(1001, 1010):
key = "key-{}".format(i)
value = "value-{}".format(i)
near_cache[key] = value
stats = near_cache.get_statistics()
evict, expire = stats["evictions"], stats["expirations"]
self.assertLess(evict, 2)
self.assertGreater(expire, 8)
def test_LRU_time_with_update(self):
near_cache = self.create_near_cache(self.service, IN_MEMORY_FORMAT.OBJECT, 1000, 1000, EVICTION_POLICY.LRU, 10, 10, 10)
for i in range(0, 10):
key = "key-{}".format(i)
value = "value-{}".format(i)
near_cache[key] = value
sleep(0.1)
for i in range(0, 9):
key = "key-{}".format(i)
value = "value-{}".format(i)
self.assertEqual(value, near_cache[key])
sleep(0.1)
near_cache["key-10"] = "value-10"
with self.assertRaises(KeyError):
val = near_cache["key-9"]
def decode_response(client_message, to_object=None):
""" Decode response from client message"""
parameters = dict(partitions=None)
partitions_size = client_message.read_int()
partitions = {}
for _ in range(0, partitions_size):
partitions_key = AddressCodec.decode(client_message, to_object)
partitions_val_size = client_message.read_int()
partitions_val = []
for _ in range(0, partitions_val_size):
partitions_val_item = client_message.read_int()
partitions_val.append(partitions_val_item)
partitions[partitions_key] = partitions_val
parameters['partitions'] = partitions
return parameters
def do_benchmark():
class ClientThread(threading.Thread):
def __init__(self, name):
threading.Thread.__init__(self, name=name)
self.ops = 0
self.decode = 0
self.setDaemon(True)
self.bench = Bench()
def run(self):
while True:
self.bench.encode()
self.bench.decode()
self.ops += 1
threads = [ClientThread("client-thread-%d" % i) for i in range(0, THREAD_COUNT)]
for t in threads:
t.start()
start = time.time()
counter = 1
while counter < 10:
time.sleep(5)
six.print_("ops per second : " + \
str(sum([t.ops for t in threads]) // (time.time() - start)))
# for t in threads:
# print ("%s: ops: %d " % (t.name, t.ops))
counter += 1
def decode(msg):
b = msg.next_frame().buf
n = len(b) // LONG_SIZE_IN_BYTES
result = []
for i in range(n):
result.append(FixSizedTypesCodec.decode_long(b, i * LONG_SIZE_IN_BYTES))
return result
def _fill_map_and_near_cache(self, count=10):
fill_content = {"key-%d" % x: "value-%d" % x for x in range(0, count)}
for k, v in six.iteritems(fill_content):
self.map.put(k, v)
for k, v in six.iteritems(fill_content):
self.map.get(k)
return fill_content
def bench(my_map):
start = time.time()
hit = my_map._near_cache._cache_hit
for key in range(0, ENTRY_COUNT):
my_map.get(key)
six.print_("op / sec :", ENTRY_COUNT // (time.time() - start), "hit:",
my_map._near_cache._cache_hit - hit)
def _read_array_fnc(self, read_item_fnc):
length = self.read_int()
if length == NULL_ARRAY_LENGTH:
return None
if length > 0:
return [read_item_fnc() for _ in range(0, length)]
return []
def test_LFU_time(self):
near_cache = self.create_near_cache(self.service, IN_MEMORY_FORMAT.BINARY, 1000, 1000, EVICTION_POLICY.LFU, 1000)
for i in range(0, 1000):
key = "key-{}".format(i)
value = "value-{}".format(i)
near_cache[key] = value
for j in range(0, i + 1):
v = near_cache[key]
for i in range(1001, 2000):
key = "key-{}".format(i)
value = "value-{}".format(i)
near_cache[key] = value
stats = near_cache.get_statistics()
evict, expire = stats["evictions"], stats["expirations"]
self.assertEqual(expire, 0)
self.assertLess(evict, 1000)
def test_put_all(self):
map = {"key-%d" % x: "value-%d" % x for x in range(0, 10)}
self.replicated_map.put_all(map)
self.assertTrueEventually(lambda: six.assertCountEqual(
self, six.iteritems(map), self.replicated_map.entry_set()))
def handle(client_message,
handle_event_member=None,
handle_event_memberlist=None,
handle_event_memberattributechange=None,
to_object=None):
""" Event handler """
message_type = client_message.get_message_type()
if message_type == EVENT_MEMBER and handle_event_member is not None:
member = MemberCodec.decode(client_message, to_object)
event_type = client_message.read_int()
handle_event_member(member=member, event_type=event_type)
if message_type == EVENT_MEMBERLIST and handle_event_memberlist is not None:
members_size = client_message.read_int()
members = []
for members_index in range(0, members_size):
members_item = MemberCodec.decode(client_message, to_object)
members.append(members_item)
handle_event_memberlist(members=members)
if message_type == EVENT_MEMBERATTRIBUTECHANGE and handle_event_memberattributechange is not None:
uuid = client_message.read_str()
key = client_message.read_str()
operation_type = client_message.read_int()
value = None
if not client_message.read_bool():
value = client_message.read_str()
handle_event_memberattributechange(uuid=uuid,
key=key,
operation_type=operation_type,
value=value)
def _fill_map(self, count=1000):
m = {
InnerPortable("key-%d" % x, x): InnerPortable("value-%d" % x, x)
for x in range(0, count)
}
self.map.put_all(m)
return m
def decode_response(client_message, to_object=None):
""" Decode response from client message"""
parameters = dict(status=None,
address=None,
uuid=None,
owner_uuid=None,
serialization_version=None,
server_hazelcast_version=None,
client_unregistered_members=None)
parameters['status'] = client_message.read_byte()
if not client_message.read_bool():
parameters['address'] = AddressCodec.decode(client_message, to_object)
if not client_message.read_bool():
parameters['uuid'] = client_message.read_str()
if not client_message.read_bool():
parameters['owner_uuid'] = client_message.read_str()
parameters['serialization_version'] = client_message.read_byte()
if client_message.is_complete():
return parameters
parameters['server_hazelcast_version'] = client_message.read_str()
if not client_message.read_bool():
client_unregistered_members_size = client_message.read_int()
client_unregistered_members = []
for _ in range(0, client_unregistered_members_size):
client_unregistered_members_item = MemberCodec.decode(
client_message, to_object)
client_unregistered_members.append(
client_unregistered_members_item)
parameters['client_unregistered_members'] = ImmutableLazyDataList(
client_unregistered_members, to_object)
return parameters
def _parse_address(address):
if ":" in address:
host, port = address.split(":")
return [hazelcast.core.Address(host, int(port))]
return [
hazelcast.core.Address(address, p)
for p in range(DEFAULT_PORT, DEFAULT_PORT + 3)
]
def _fill_map(self, count=1000):
map = {
InnerPortable("key-%d" % x, x): InnerPortable("value-%d" % x, x)
for x in range(0, count)
}
for k, v in six.iteritems(map):
self.map.put(k, v)
return map
def decode(msg):
values = ListMultiFrameCodec.decode(msg, ListIntegerCodec.decode)
keys = ListUUIDCodec.decode(msg)
result = []
n = len(keys)
for i in range(n):
result.append((keys[i], values[i]))
return result
def test_expiry_time(self):
near_cache = self.create_near_cache(self.service,
InMemoryFormat.OBJECT, 0.05, 100,
EvictionPolicy.LRU, 100)
for i in range(0, 100):
key = "key-{}".format(i)
value = "value-{}".format(i)
near_cache[key] = value
sleep(0.1)
for i in range(101, 110):
key = "key-{}".format(i)
value = "value-{}".format(i)
near_cache[key] = value
stats = near_cache.get_statistics()
evict, expire = stats["evictions"], stats["expirations"]
self.assertLess(evict, 2)
self.assertGreater(expire, 8)
def test_LRU_time_with_update(self):
near_cache = self.create_near_cache(self.service,
InMemoryFormat.OBJECT, 1000, 1000,
EvictionPolicy.LRU, 10, 10, 10)
for i in range(0, 10):
key = "key-{}".format(i)
value = "value-{}".format(i)
near_cache[key] = value
sleep(0.1)
for i in range(0, 9):
key = "key-{}".format(i)
value = "value-{}".format(i)
self.assertEqual(value, near_cache[key])
sleep(0.1)
near_cache["key-10"] = "value-10"
with self.assertRaises(KeyError):
val = near_cache["key-9"]
def read_class_definition(self, data_in, factory_id, class_id, version):
register = True
builder = ClassDefinitionBuilder(factory_id, class_id, version)
# final position after portable is read
data_in.read_int()
# field count
field_count = data_in.read_int()
offset = data_in.position()
for i in range(0, field_count):
pos = data_in.read_int(offset + i * bits.INT_SIZE_IN_BYTES)
data_in.set_position(pos)
_len = data_in.read_short()
field_name = bytearray(_len)
data_in.read_into(field_name)
field_type = data_in.read_byte()
field_factory_id = 0
field_class_id = 0
field_version = version
if field_type == FieldType.PORTABLE:
# is null
if data_in.read_boolean():
register = False
field_factory_id = data_in.read_int()
field_class_id = data_in.read_int()
# TODO: what there's a null inner Portable field
if register:
field_version = data_in.read_int()
self.read_class_definition(data_in, field_factory_id,
field_class_id, field_version)
elif field_type == FieldType.PORTABLE_ARRAY:
k = data_in.read_int()
field_factory_id = data_in.read_int()
field_class_id = data_in.read_int()
# TODO: what there's a null inner Portable field
if k > 0:
p = data_in.read_int()
data_in.set_position(p)
field_version = data_in.read_int()
self.read_class_definition(data_in, field_factory_id,
field_class_id, field_version)
else:
register = False
builder.add_field_def(
FieldDefinition(i, field_name.decode('ascii'), field_type,
field_version, field_factory_id,
field_class_id))
class_def = builder.build()
if register:
class_def = self.register_class_definition(class_def)
return class_def
def test_put_get(self):
near_cache = self.create_near_cache(self.service, IN_MEMORY_FORMAT.OBJECT, 1000, 1000, EVICTION_POLICY.LRU, 1000)
for i in range(0, 10000):
key = "key-{}".format(i)
value = "value-{}".format(i)
near_cache[key] = value
self.assertEqual(value, near_cache[key])
self.assertEqual("value-0", near_cache["key-0"]) # prevent its eviction
self.assertGreaterEqual(near_cache.eviction_max_size * 1.1, near_cache.__len__())
def test_operation_from_multiple_threads(self):
num_threads = 4
num_iterations = 5000
value_size = 1000
key_range = 50
timeout = 300
keys = list(range(0, key_range))
exceptions = []
value = "v" * value_size
def put_get_remove():
for i in range(0, num_iterations):
if i % 100 == 0:
self.logger.info("op %i", i)
try:
key = choice(keys)
self.map.lock(key)
self.map.put(key, value)
self.assertEqual(value, self.map.get(key))
self.assertEqual(value, self.map.remove(key))
self.map.unlock(key)
except:
self.logger.exception("Exception in thread")
exceptions.append(
(threading.currentThread().getName(), sys.exc_info()))
threads = [
self.start_new_thread(put_get_remove)
for _ in range(0, num_threads)
]
for t in threads:
t.join(timeout)
if t.isAlive():
self.fail("thread %s did not finish in %s seconds" %
(t.getName(), timeout))
if exceptions:
name, exception = exceptions[0]
self.logger.exception("Exception in thread %s", name)
six.reraise(exception[0].__class__, exception[0], exception[2])
def decode(msg):
b = msg.next_frame().buf
n = len(b) // _UUID_LONG_ENTRY_SIZE_IN_BYTES
result = []
for i in range(n):
o = i * _UUID_LONG_ENTRY_SIZE_IN_BYTES
key = FixSizedTypesCodec.decode_uuid(b, o)
value = FixSizedTypesCodec.decode_long(b, o + UUID_SIZE_IN_BYTES)
result.append((key, value))
return result
