def bench(my_map): start = time.time() hit = my_map._near_cache._cache_hit for key in range(0, ENTRY_COUNT): my_map.get(key) six.print_("op / sec :", ENTRY_COUNT // (time.time() - start), "hit:", my_map._near_cache._cache_hit - hit)
def do_benchmark(): class ClientThread(threading.Thread): def __init__(self, name): threading.Thread.__init__(self, name=name) self.ops = 0 self.decode = 0 self.setDaemon(True) self.bench = Bench() def run(self): while True: self.bench.encode() self.bench.decode() self.ops += 1 threads = [ClientThread("client-thread-%d" % i) for i in range(0, THREAD_COUNT)] for t in threads: t.start() start = time.time() counter = 1 while counter < 10: time.sleep(5) six.print_("ops per second : " + \ str(sum([t.ops for t in threads]) // (time.time() - start))) # for t in threads: # print ("%s: ops: %d " % (t.name, t.ops)) counter += 1
def handle_exception(e, traceback): if isinstance(e, MemoryError): # TODO six.print_("OUT OF MEMORY") six.reraise(MemoryError, e, traceback) elif isinstance(e, HazelcastSerializationError): six.reraise(HazelcastSerializationError, e, traceback) else: six.reraise(HazelcastSerializationError, HazelcastSerializationError(e.args[0]), traceback)
def _call_all_func(self, class_type): for meth in class_type.__dict__.values(): try: if isinstance(meth, FunctionType): with self.assertRaises(NotImplementedError): params = [ i for i in range( 0, six.get_function_code(meth).co_argcount) ] meth(*params) except TypeError as e: six.print_(e)
def init(): logging.basicConfig( format='%(asctime)s%(msecs)03d [%(name)s] %(levelname)s: %(message)s', datefmt="%H:%M%:%S,") logging.getLogger().setLevel(logging.INFO) logger = logging.getLogger("main") config = hazelcast.ClientConfig() config.group_config.name = "dev" config.group_config.password = "******" config.network.addresses.append("127.0.0.1") near_cache_config = NearCacheConfig(MAP_NAME) near_cache_config.in_memory_format = IN_MEMORY_FORMAT.OBJECT config.add_near_cache_config(near_cache_config) try: from tests.hzrc.client import HzRemoteController rc = HzRemoteController('127.0.0.1', '9701') if not rc.ping(): logger.info("Remote Controller Server not running... exiting.") exit() logger.info("Remote Controller Server OK...") rc_cluster = rc.createCluster(None, None) rc_member = rc.startMember(rc_cluster.id) config.network.addresses.append('{}:{}'.format(rc_member.host, rc_member.port)) except (ImportError, NameError): config.network.addresses.append('127.0.0.1') client = hazelcast.HazelcastClient(config) my_map = client.get_map(MAP_NAME).blocking() six.print_("START INIT") for key in range(0, ENTRY_COUNT): my_map.put(key, VALUE) for key in range(0, ENTRY_COUNT): my_map.get(key) six.print_("INIT COMPLETE") return my_map
def do_benchmark(): PROCESS_COUNT = 10 ENTRY_COUNT = 10 * 1000 VALUE_SIZE = 10000 GET_PERCENTAGE = 40 PUT_PERCENTAGE = 40 logging.basicConfig( format='%(asctime)s%(msecs)03d [%(name)s] %(levelname)s: %(message)s', datefmt="%H:%M%:%S,") logging.getLogger().setLevel(logging.INFO) logger = logging.getLogger("main") config = hazelcast.ClientConfig() config.group_config.name = "dev" config.group_config.password = "******" try: from tests.hzrc.client import HzRemoteController rc = HzRemoteController('127.0.0.1', '9701') if not rc.ping(): logger.info("Remote Controller Server not running... exiting.") exit() logger.info("Remote Controller Server OK...") rc_cluster = rc.createCluster(None, None) rc_member = rc.startMember(rc_cluster.id) config.network.addresses.append('{}:{}'.format(rc_member.host, rc_member.port)) except (ImportError, NameError): config.network.addresses.append('127.0.0.1') client = hazelcast.HazelcastClient(config) class ClientProcess(multiprocessing.Process): def __init__(self, name, config, counts): multiprocessing.Process.__init__(self, name=name) self.counts = counts self.config = config self.daemon = True def run(self): client = hazelcast.HazelcastClient(config) my_map = client.get_map("default") while True: key = int(random.random() * ENTRY_COUNT) operation = int(random.random() * 100) if operation < GET_PERCENTAGE: my_map.get(key) self.counts[0] += 1 elif operation < GET_PERCENTAGE + PUT_PERCENTAGE: my_map.put(key, "x" * VALUE_SIZE) self.counts[1] += 1 else: my_map.remove(key) self.counts[2] += 1 processes = [ ClientProcess("client-process-%d" % i, config, multiprocessing.Array('i', 3)) for i in range(0, PROCESS_COUNT) ] for p in processes: p.start() start = time.time() counter = 1 while counter < 1000: time.sleep(5) six.print_("ops per second : " + \ str(sum([sum(p.counts) for p in processes]) // (time.time() - start))) # for p in processes: # print ("%s: put: %d get: %d: remove: %d" % (p.name, p.counts[0], p.counts[1], p.counts[2])) counter += 1
def do_benchmark(): MAP_NAME = "default" THREAD_COUNT = 1 ENTRY_COUNT = 10 * 1000 VALUE_SIZE = 100 GET_PERCENTAGE = 40 PUT_PERCENTAGE = 40 VALUE = "x" * VALUE_SIZE logging.basicConfig( format='%(asctime)s%(msecs)03d [%(name)s] %(levelname)s: %(message)s', datefmt="%H:%M%:%S,") logging.getLogger().setLevel(logging.INFO) logger = logging.getLogger("main") class ClientThread(threading.Thread): def __init__(self, name): threading.Thread.__init__(self, name=name) self.gets = 0 self.puts = 0 self.removes = 0 self.setDaemon(True) self.my_map = dict() self.ss = SerializationServiceV1( serialization_config=SerializationConfig()) def run(self): while True: key = int(random.random() * ENTRY_COUNT) operation = int(random.random() * 100) if operation < GET_PERCENTAGE: key_data = self.ss.to_data(key) val_dat = self.my_map.get(key_data, None) self.ss.to_object(val_dat) self.gets += 1 elif operation < GET_PERCENTAGE + PUT_PERCENTAGE: key_data = self.ss.to_data(key) self.my_map[key_data] = self.ss.to_data(VALUE) self.puts += 1 else: try: key_data = self.ss.to_data(key) del self.my_map[key_data] except KeyError: pass self.removes += 1 threads = [ ClientThread("client-thread-%d" % i) for i in range(0, THREAD_COUNT) ] for t in threads: t.start() start = time.time() counter = 1 while counter < 3: time.sleep(5) six.print_("ops per second : " + \ str(sum([t.gets + t.puts + t.removes for t in threads]) // (time.time() - start))) for t in threads: six.print_("{}: put: {} get: {}: remove: {}".format( t.name, t.puts, t.gets, t.removes)) counter += 1
def do_benchmark(): THREAD_COUNT = 1 ENTRY_COUNT = 10 * 1000 VALUE_SIZE = 100 GET_PERCENTAGE = 40 PUT_PERCENTAGE = 40 VALUE = "x" * VALUE_SIZE logging.basicConfig(format='%(asctime)s%(msecs)03d [%(name)s] %(levelname)s: %(message)s', datefmt="%H:%M%:%S,") logging.getLogger().setLevel(logging.INFO) logger = logging.getLogger("main") config = hazelcast.ClientConfig() config.group_config.name = "dev" config.group_config.password = "******" try: from tests.hzrc.client import HzRemoteController rc = HzRemoteController('127.0.0.1', '9701') if not rc.ping(): logger.info("Remote Controller Server not running... exiting.") exit() logger.info("Remote Controller Server OK...") rc_cluster = rc.createCluster(None, None) rc_member = rc.startMember(rc_cluster.id) config.network.addresses.append('{}:{}'.format(rc_member.host, rc_member.port)) except (ImportError, NameError): config.network.addresses.append('127.0.0.1') client = hazelcast.HazelcastClient(config) class ClientThread(threading.Thread): def __init__(self, name): threading.Thread.__init__(self, name=name) self.gets = 0 self.puts = 0 self.removes = 0 self.setDaemon(True) def run(self): my_map = client.get_map("default").blocking() while True: key = int(random.random() * ENTRY_COUNT) operation = int(random.random() * 100) if operation < GET_PERCENTAGE: my_map.get(key) self.gets += 1 elif operation < GET_PERCENTAGE + PUT_PERCENTAGE: my_map.put(key, VALUE) self.puts += 1 else: my_map.remove(key) self.removes += 1 threads = [ClientThread("client-thread-%d" % i) for i in range(0, THREAD_COUNT)] for t in threads: t.start() start = time.time() counter = 1 while counter < 1000: time.sleep(5) six.print_("ops per second : " + \ str(sum([t.gets + t.puts + t.removes for t in threads]) // (time.time() - start))) for t in threads: six.print_("{}: put: {} get: {}: remove: {}".format(t.name, t.puts, t.gets, t.removes)) counter += 1
def do_benchmark(): REQ_COUNT = 50000 ENTRY_COUNT = 10 * 1000 VALUE_SIZE = 10000 GET_PERCENTAGE = 40 PUT_PERCENTAGE = 40 logging.basicConfig( format='%(asctime)s%(msecs)03d [%(name)s] %(levelname)s: %(message)s', datefmt="%H:%M%:%S,") logging.getLogger().setLevel(logging.INFO) logger = logging.getLogger("main") config = hazelcast.ClientConfig() config.group_config.name = "dev" config.group_config.password = "******" try: from tests.hzrc.client import HzRemoteController rc = HzRemoteController('127.0.0.1', '9701') if not rc.ping(): logger.info("Remote Controller Server not running... exiting.") exit() logger.info("Remote Controller Server OK...") rc_cluster = rc.createCluster(None, None) rc_member = rc.startMember(rc_cluster.id) config.network_config.addresses.append('{}:{}'.format( rc_member.host, rc_member.port)) except (ImportError, NameError): config.network_config.addresses.append('127.0.0.1') client = hazelcast.HazelcastClient(config) class Test(object): def __init__(self): self.ops = 0 self.event = threading.Event() def incr(self, _): self.ops += 1 if self.ops == REQ_COUNT: self.event.set() def run(self): my_map = client.get_map("default") for _ in range(0, REQ_COUNT): key = int(random.random() * ENTRY_COUNT) operation = int(random.random() * 100) if operation < GET_PERCENTAGE: my_map.get(key).add_done_callback(self.incr) elif operation < GET_PERCENTAGE + PUT_PERCENTAGE: my_map.put(key, "x" * VALUE_SIZE).add_done_callback(self.incr) else: my_map.remove(key).add_done_callback(self.incr) t = Test() start = time.time() t.run() t.event.wait(1) time_taken = time.time() - start six.print_("Took {} seconds for {} requests".format(time_taken, REQ_COUNT)) six.print_("ops per second: {}".format(t.ops // time_taken))
def test_DataRecord_expire_time(self): now = current_time() six.print_(int(now), now) data_rec = DataRecord("key", "value", create_time=now, ttl_seconds=1) sleep(2) self.assertTrue(data_rec.is_expired(max_idle_seconds=1000))
def measure(self): six.print_("Encode time: {}".format(timeit.timeit(self.encode, number=100000)))
def encode(self): self.request = map_get_codec.encode_request(self.name, self.key, self.thread_id) def decode(self): self.response_message._read_index = 0 self.response = map_get_codec.decode_response(self.response_message, self.service.to_data) def measure(self): six.print_("Encode time: {}".format(timeit.timeit(self.encode, number=100000))) # print "Decode time: {}".format(timeit.timeit(self.decode, number=100000)) if __name__ == '__main__': global bench bench = Bench() setup = "from __main__ import Bench" # setup = "from __main__ import Bench;global bench;bench = Bench()" number = 100000 encode_time = timeit.timeit(bench.encode, setup=setup, number=number) decode_time = timeit.timeit(bench.decode, setup=setup, number=number) six.print_("--------------------------------------------------------------------------------") six.print_("Encode op/s: {}".format(number // encode_time)) six.print_("Decode op/s: {}".format(number // decode_time)) six.print_("Total op/s: {}".format(number // (encode_time + decode_time))) six.print_("--------------------------------------------------------------------------------\n\n") six.print_(bench.request) six.print_(bench.response)