def decode(msg, item_size, decoder): frame = msg.next_frame() type = FixSizedTypesCodec.decode_byte(frame.buf, 0) count = FixSizedTypesCodec.decode_int(frame.buf, 1) if type == ListCNFixedSizeCodec._TYPE_NULL_ONLY: return [None] * count elif type == ListCNFixedSizeCodec._TYPE_NOT_NULL_ONLY: header_size = ListCNFixedSizeCodec._HEADER_SIZE return [ decoder(frame.buf, header_size + i * item_size) for i in range(count) ] else: response = [None] * count position = ListCNFixedSizeCodec._HEADER_SIZE read_count = 0 items_per_bitmask = ListCNFixedSizeCodec._ITEMS_PER_BITMASK while read_count < count: bitmask = FixSizedTypesCodec.decode_byte(frame.buf, position) position += 1 batch_size = min(items_per_bitmask, count - read_count) for i in range(batch_size): mask = 1 << i if (bitmask & mask) == mask: response[read_count] = decoder(frame.buf, position) position += item_size read_count += 1 return response
def _fill_map(self, key_count=5, value_count=5): map = { "key-%d" % x: ["value-%d-%d" % (x, y) for y in range(0, value_count)] for x in range(0, key_count) } for k, l in six.iteritems(map): for v in l: self.multi_map.put(k, v) return map
def test_LRU_time(self): near_cache = self.create_near_cache(self.service, IN_MEMORY_FORMAT.OBJECT, 1000, 1000, EVICTION_POLICY.LRU, 10000, 16, 16) for i in range(0, 10000): key = "key-{}".format(i) value = "value-{}".format(i) near_cache[key] = value self.assertEqual("value-0", near_cache["key-0"]) for i in range(10001, 20000): key = "key-{}".format(i) value = "value-{}".format(i) near_cache[key] = value stats = near_cache.get_statistics() evict, expire = stats["evictions"], stats["expirations"] self.assertEqual(expire, 0) self.assertLess(evict, 10000)
def test_expiry_time(self): near_cache = self.create_near_cache(self.service, IN_MEMORY_FORMAT.OBJECT, 1, 1000, EVICTION_POLICY.LRU, 1000) for i in range(0, 1000): key = "key-{}".format(i) value = "value-{}".format(i) near_cache[key] = value sleep(2) for i in range(1001, 1010): key = "key-{}".format(i) value = "value-{}".format(i) near_cache[key] = value stats = near_cache.get_statistics() evict, expire = stats["evictions"], stats["expirations"] self.assertLess(evict, 2) self.assertGreater(expire, 8)
def test_LRU_time_with_update(self): near_cache = self.create_near_cache(self.service, IN_MEMORY_FORMAT.OBJECT, 1000, 1000, EVICTION_POLICY.LRU, 10, 10, 10) for i in range(0, 10): key = "key-{}".format(i) value = "value-{}".format(i) near_cache[key] = value sleep(0.1) for i in range(0, 9): key = "key-{}".format(i) value = "value-{}".format(i) self.assertEqual(value, near_cache[key]) sleep(0.1) near_cache["key-10"] = "value-10" with self.assertRaises(KeyError): val = near_cache["key-9"]
def decode_response(client_message, to_object=None): """ Decode response from client message""" parameters = dict(partitions=None) partitions_size = client_message.read_int() partitions = {} for _ in range(0, partitions_size): partitions_key = AddressCodec.decode(client_message, to_object) partitions_val_size = client_message.read_int() partitions_val = [] for _ in range(0, partitions_val_size): partitions_val_item = client_message.read_int() partitions_val.append(partitions_val_item) partitions[partitions_key] = partitions_val parameters['partitions'] = partitions return parameters
def do_benchmark(): class ClientThread(threading.Thread): def __init__(self, name): threading.Thread.__init__(self, name=name) self.ops = 0 self.decode = 0 self.setDaemon(True) self.bench = Bench() def run(self): while True: self.bench.encode() self.bench.decode() self.ops += 1 threads = [ClientThread("client-thread-%d" % i) for i in range(0, THREAD_COUNT)] for t in threads: t.start() start = time.time() counter = 1 while counter < 10: time.sleep(5) six.print_("ops per second : " + \ str(sum([t.ops for t in threads]) // (time.time() - start))) # for t in threads: # print ("%s: ops: %d " % (t.name, t.ops)) counter += 1
def decode(msg): b = msg.next_frame().buf n = len(b) // LONG_SIZE_IN_BYTES result = [] for i in range(n): result.append(FixSizedTypesCodec.decode_long(b, i * LONG_SIZE_IN_BYTES)) return result
def _fill_map_and_near_cache(self, count=10): fill_content = {"key-%d" % x: "value-%d" % x for x in range(0, count)} for k, v in six.iteritems(fill_content): self.map.put(k, v) for k, v in six.iteritems(fill_content): self.map.get(k) return fill_content
def bench(my_map): start = time.time() hit = my_map._near_cache._cache_hit for key in range(0, ENTRY_COUNT): my_map.get(key) six.print_("op / sec :", ENTRY_COUNT // (time.time() - start), "hit:", my_map._near_cache._cache_hit - hit)
def _read_array_fnc(self, read_item_fnc): length = self.read_int() if length == NULL_ARRAY_LENGTH: return None if length > 0: return [read_item_fnc() for _ in range(0, length)] return []
def test_LFU_time(self): near_cache = self.create_near_cache(self.service, IN_MEMORY_FORMAT.BINARY, 1000, 1000, EVICTION_POLICY.LFU, 1000) for i in range(0, 1000): key = "key-{}".format(i) value = "value-{}".format(i) near_cache[key] = value for j in range(0, i + 1): v = near_cache[key] for i in range(1001, 2000): key = "key-{}".format(i) value = "value-{}".format(i) near_cache[key] = value stats = near_cache.get_statistics() evict, expire = stats["evictions"], stats["expirations"] self.assertEqual(expire, 0) self.assertLess(evict, 1000)
def test_put_all(self): map = {"key-%d" % x: "value-%d" % x for x in range(0, 10)} self.replicated_map.put_all(map) self.assertTrueEventually(lambda: six.assertCountEqual( self, six.iteritems(map), self.replicated_map.entry_set()))
def handle(client_message, handle_event_member=None, handle_event_memberlist=None, handle_event_memberattributechange=None, to_object=None): """ Event handler """ message_type = client_message.get_message_type() if message_type == EVENT_MEMBER and handle_event_member is not None: member = MemberCodec.decode(client_message, to_object) event_type = client_message.read_int() handle_event_member(member=member, event_type=event_type) if message_type == EVENT_MEMBERLIST and handle_event_memberlist is not None: members_size = client_message.read_int() members = [] for members_index in range(0, members_size): members_item = MemberCodec.decode(client_message, to_object) members.append(members_item) handle_event_memberlist(members=members) if message_type == EVENT_MEMBERATTRIBUTECHANGE and handle_event_memberattributechange is not None: uuid = client_message.read_str() key = client_message.read_str() operation_type = client_message.read_int() value = None if not client_message.read_bool(): value = client_message.read_str() handle_event_memberattributechange(uuid=uuid, key=key, operation_type=operation_type, value=value)
def _fill_map(self, count=1000): m = { InnerPortable("key-%d" % x, x): InnerPortable("value-%d" % x, x) for x in range(0, count) } self.map.put_all(m) return m
def decode_response(client_message, to_object=None): """ Decode response from client message""" parameters = dict(status=None, address=None, uuid=None, owner_uuid=None, serialization_version=None, server_hazelcast_version=None, client_unregistered_members=None) parameters['status'] = client_message.read_byte() if not client_message.read_bool(): parameters['address'] = AddressCodec.decode(client_message, to_object) if not client_message.read_bool(): parameters['uuid'] = client_message.read_str() if not client_message.read_bool(): parameters['owner_uuid'] = client_message.read_str() parameters['serialization_version'] = client_message.read_byte() if client_message.is_complete(): return parameters parameters['server_hazelcast_version'] = client_message.read_str() if not client_message.read_bool(): client_unregistered_members_size = client_message.read_int() client_unregistered_members = [] for _ in range(0, client_unregistered_members_size): client_unregistered_members_item = MemberCodec.decode( client_message, to_object) client_unregistered_members.append( client_unregistered_members_item) parameters['client_unregistered_members'] = ImmutableLazyDataList( client_unregistered_members, to_object) return parameters
def _parse_address(address): if ":" in address: host, port = address.split(":") return [hazelcast.core.Address(host, int(port))] return [ hazelcast.core.Address(address, p) for p in range(DEFAULT_PORT, DEFAULT_PORT + 3) ]
def _fill_map(self, count=1000): map = { InnerPortable("key-%d" % x, x): InnerPortable("value-%d" % x, x) for x in range(0, count) } for k, v in six.iteritems(map): self.map.put(k, v) return map
def decode(msg): values = ListMultiFrameCodec.decode(msg, ListIntegerCodec.decode) keys = ListUUIDCodec.decode(msg) result = [] n = len(keys) for i in range(n): result.append((keys[i], values[i])) return result
def test_expiry_time(self): near_cache = self.create_near_cache(self.service, InMemoryFormat.OBJECT, 0.05, 100, EvictionPolicy.LRU, 100) for i in range(0, 100): key = "key-{}".format(i) value = "value-{}".format(i) near_cache[key] = value sleep(0.1) for i in range(101, 110): key = "key-{}".format(i) value = "value-{}".format(i) near_cache[key] = value stats = near_cache.get_statistics() evict, expire = stats["evictions"], stats["expirations"] self.assertLess(evict, 2) self.assertGreater(expire, 8)
def test_LRU_time_with_update(self): near_cache = self.create_near_cache(self.service, InMemoryFormat.OBJECT, 1000, 1000, EvictionPolicy.LRU, 10, 10, 10) for i in range(0, 10): key = "key-{}".format(i) value = "value-{}".format(i) near_cache[key] = value sleep(0.1) for i in range(0, 9): key = "key-{}".format(i) value = "value-{}".format(i) self.assertEqual(value, near_cache[key]) sleep(0.1) near_cache["key-10"] = "value-10" with self.assertRaises(KeyError): val = near_cache["key-9"]
def read_class_definition(self, data_in, factory_id, class_id, version): register = True builder = ClassDefinitionBuilder(factory_id, class_id, version) # final position after portable is read data_in.read_int() # field count field_count = data_in.read_int() offset = data_in.position() for i in range(0, field_count): pos = data_in.read_int(offset + i * bits.INT_SIZE_IN_BYTES) data_in.set_position(pos) _len = data_in.read_short() field_name = bytearray(_len) data_in.read_into(field_name) field_type = data_in.read_byte() field_factory_id = 0 field_class_id = 0 field_version = version if field_type == FieldType.PORTABLE: # is null if data_in.read_boolean(): register = False field_factory_id = data_in.read_int() field_class_id = data_in.read_int() # TODO: what there's a null inner Portable field if register: field_version = data_in.read_int() self.read_class_definition(data_in, field_factory_id, field_class_id, field_version) elif field_type == FieldType.PORTABLE_ARRAY: k = data_in.read_int() field_factory_id = data_in.read_int() field_class_id = data_in.read_int() # TODO: what there's a null inner Portable field if k > 0: p = data_in.read_int() data_in.set_position(p) field_version = data_in.read_int() self.read_class_definition(data_in, field_factory_id, field_class_id, field_version) else: register = False builder.add_field_def( FieldDefinition(i, field_name.decode('ascii'), field_type, field_version, field_factory_id, field_class_id)) class_def = builder.build() if register: class_def = self.register_class_definition(class_def) return class_def
def test_put_get(self): near_cache = self.create_near_cache(self.service, IN_MEMORY_FORMAT.OBJECT, 1000, 1000, EVICTION_POLICY.LRU, 1000) for i in range(0, 10000): key = "key-{}".format(i) value = "value-{}".format(i) near_cache[key] = value self.assertEqual(value, near_cache[key]) self.assertEqual("value-0", near_cache["key-0"]) # prevent its eviction self.assertGreaterEqual(near_cache.eviction_max_size * 1.1, near_cache.__len__())
def test_operation_from_multiple_threads(self): num_threads = 4 num_iterations = 5000 value_size = 1000 key_range = 50 timeout = 300 keys = list(range(0, key_range)) exceptions = [] value = "v" * value_size def put_get_remove(): for i in range(0, num_iterations): if i % 100 == 0: self.logger.info("op %i", i) try: key = choice(keys) self.map.lock(key) self.map.put(key, value) self.assertEqual(value, self.map.get(key)) self.assertEqual(value, self.map.remove(key)) self.map.unlock(key) except: self.logger.exception("Exception in thread") exceptions.append( (threading.currentThread().getName(), sys.exc_info())) threads = [ self.start_new_thread(put_get_remove) for _ in range(0, num_threads) ] for t in threads: t.join(timeout) if t.isAlive(): self.fail("thread %s did not finish in %s seconds" % (t.getName(), timeout)) if exceptions: name, exception = exceptions[0] self.logger.exception("Exception in thread %s", name) six.reraise(exception[0].__class__, exception[0], exception[2])
def decode(msg): b = msg.next_frame().buf n = len(b) // _UUID_LONG_ENTRY_SIZE_IN_BYTES result = [] for i in range(n): o = i * _UUID_LONG_ENTRY_SIZE_IN_BYTES key = FixSizedTypesCodec.decode_uuid(b, o) value = FixSizedTypesCodec.decode_long(b, o + UUID_SIZE_IN_BYTES) result.append((key, value)) return result