def bench_client(rpc_type): N = 1 client = Client(addr, port, rpc_type=rpc_type) start = time.time() for i in range(N): client.call('echo', i) end = time.time() delta = end - start print(" sync: {0:.0f} qps".format(N / delta)) futures = [] start = time.time() for i in range(N): futures.append(client.call_async('echo', i)) for fut in futures: loop.run_until_complete(fut) end = time.time() delta = end - start print(" async: {0:.0f} qps".format(N / delta)) start = time.time() for i in range(N): client.notify('notify', i) client.call('echo', 0) end = time.time() delta = end - start print(" notify: {0:.0f} qps".format(N / delta)) client.close_sync()
def __init__(self, bridge_port): self.loop = asyncio.get_event_loop() self.client = Client('127.0.0.1', bridge_port, notification_handler=self._yarn_notification) self.launched_containers = 0 self.requested_containers = self.wait_containers = 100 self.sh_name = sys.argv[2][sys.argv[2].rindex('/') + 1:]
class AppMaster(object): def __init__(self, bridge_port): self.loop = asyncio.get_event_loop() self.client = Client('127.0.0.1', bridge_port, notification_handler=self._yarn_notification) self.launched_containers = 0 self.requested_containers = self.wait_containers = 100 self.sh_name = sys.argv[2][sys.argv[2].rindex('/') + 1:] def start(self): for _ in range(self.wait_containers): self.client.call('addContainerRequest', cpu=1, memory=512, priority=0) try: self.loop.run_forever() except: pass self.loop.close() print('[AM] terminated') def _onContainersAllocated(self, containers): for c in containers: if self.launched_containers == self.requested_containers: fut = self.client.call_async('releaseAssignedContainer', container_id=c['container_id']) fut.add_done_callback(lambda _: None) continue self.launched_containers += 1 fut = self.client.call_async('startContainer', container_id=c['container_id'], command='/bin/bash ' + self.sh_name) fut.add_done_callback(lambda _: None) print('[AM] launch {} ({}/{})'.format(c['container_id'], self.launched_containers, self.requested_containers)) def _onContainersCompleted(self, statuses): self.wait_containers -= len(statuses) for s in statuses: print('[AM] {} terminated'.format(s['container_id'])) if self.wait_containers <= 0: print('[AM] terminating...') self.loop.stop() else: print('[AM] waiting containers {}/{}'.format( self.wait_containers, self.requested_containers)) def _yarn_notification(self, name, *args, **kwargs): if name == 'onContainersAllocated': self._onContainersAllocated(**kwargs) elif name == 'onContainersCompleted': self._onContainersCompleted(**kwargs)
class AppMaster(object): def __init__(self, bridge_port): self.loop = asyncio.get_event_loop() self.client = Client('127.0.0.1', bridge_port, notification_handler=self._yarn_notification) self.launched_containers = 0 self.requested_containers = self.wait_containers = 100 self.sh_name = sys.argv[2][sys.argv[2].rindex('/')+1:] def start(self): for _ in range(self.wait_containers): self.client.call('addContainerRequest', cpu=1, memory=512, priority=0) try: self.loop.run_forever() except: pass self.loop.close() print('[AM] terminated') def _onContainersAllocated(self, containers): for c in containers: if self.launched_containers == self.requested_containers: fut = self.client.call_async( 'releaseAssignedContainer', container_id=c['container_id']) fut.add_done_callback(lambda _: None) continue self.launched_containers += 1 fut = self.client.call_async( 'startContainer', container_id=c['container_id'], command='/bin/bash ' + self.sh_name) fut.add_done_callback(lambda _: None) print('[AM] launch {} ({}/{})'.format(c['container_id'], self.launched_containers, self.requested_containers)) def _onContainersCompleted(self, statuses): self.wait_containers -= len(statuses) for s in statuses: print('[AM] {} terminated'.format(s['container_id'])) if self.wait_containers <= 0: print('[AM] terminating...') self.loop.stop() else: print('[AM] waiting containers {}/{}'.format( self.wait_containers, self.requested_containers)) def _yarn_notification(self, name, *args, **kwargs): if name == 'onContainersAllocated': self._onContainersAllocated(**kwargs) elif name == 'onContainersCompleted': self._onContainersCompleted(**kwargs)
def __init__(self, bridge_port): self.loop = asyncio.get_event_loop() self.client = Client('127.0.0.1', bridge_port, notification_handler=self._yarn_notification) self.launched_containers = 0 self.requested_containers = self.wait_containers = 100 self.sh_name = sys.argv[2][sys.argv[2].rindex('/')+1:]