def main(MASTER_ADDR, WORKER_ROUTER_ADDR, slave_addr, control_router_addr): global context global master_conn global worker_router global worker_manager global SLAVE_ADDR global CONTROL_ROUTER_ADDR SLAVE_ADDR = slave_addr CONTROL_ROUTER_ADDR = control_router_addr try: loop = ZMQEventLoop() asyncio.set_event_loop(loop) context = Context() master_conn = MasterConnection(context, MASTER_ADDR) worker_router = WorkerRouter(context, WORKER_ROUTER_ADDR) worker_manager = WorkerManager() #loop.set_default_executor(ProcessPoolExecutor()) loop.run_until_complete(run_server()) except KeyboardInterrupt: print('\nFinished (interrupted)') sys.exit(0)
def main(): try: loop = ZMQEventLoop() asyncio.set_event_loop(loop) loop.run_until_complete(run_server()) except KeyboardInterrupt: print('\nFinished (interrupted)') sys.exit(0)
def main(): args = sys.argv[1:] if len(args) != 0: sys.exit(__doc__) try: loop = ZMQEventLoop() asyncio.set_event_loop(loop) loop.run_until_complete(run(loop)) except KeyboardInterrupt: print('\nFinished (interrupted)')
def main(): args = sys.argv[1:] if len(args) > 1: sys.exit(__doc__) zip_filter = sys.argv[1] if len(sys.argv) > 1 else "10001" try: loop = ZMQEventLoop() asyncio.set_event_loop(loop) loop.run_until_complete(run(loop, zip_filter)) except KeyboardInterrupt: print('\nFinished (interrupted)')
def main(): args = sys.argv[1:] if len(args) > 1: sys.exit(__doc__) zip_filter = sys.argv[1] if len(sys.argv) > 1 else "10001" try: loop = ZMQEventLoop() asyncio.set_event_loop(loop) loop.run_until_complete(run(loop, zip_filter)) except KeyboardInterrupt: print('\nFinished (interrupted)')
def main(slave_addr, serialized_data : bytes): try: loop = ZMQEventLoop() asyncio.set_event_loop(loop) context = Context() loop.run_until_complete(run_worker(context, slave_addr, serialized_data)) except KeyboardInterrupt: print('\nFinished (interrupted)') sys.exit(0)
def main(client_router_addr, slave_router_addr): try: loop = ZMQEventLoop() asyncio.set_event_loop(loop) context = Context() loop.run_until_complete(run_master(context, client_router_addr, slave_router_addr)) except KeyboardInterrupt: print('\nFinished (interrupted)') sys.exit(0)
def main(): args = sys.argv[1:] if len(args) == 1: num_requests = int(args[0]) elif len(args) == 0: num_requests = 10 else: sys.exit(__doc__) try: loop = ZMQEventLoop() asyncio.set_event_loop(loop) loop.run_until_complete(run(loop, num_requests)) except KeyboardInterrupt: print('\nFinished (interrupted)')
def main(): args = sys.argv[1:] if len(args) == 1: num_requests = int(args[0]) elif len(args) == 0: num_requests = 10 else: sys.exit(__doc__) try: loop = ZMQEventLoop() asyncio.set_event_loop(loop) loop.run_until_complete(run(loop, num_requests)) except KeyboardInterrupt: print('\nFinished (interrupted)')
msg = yield from client.recv() if msg == b"Hello": logging.info("Ironhouse test OK") else: logging.error("Ironhouse test FAIL") # close sockets server.close() client.close() # stop auth task auth.stop() if __name__ == '__main__': if zmq.zmq_version_info() < (4, 0): raise RuntimeError( "Security is not supported in libzmq version < 4.0. libzmq version {0}" .format(zmq.zmq_version())) if '-v' in sys.argv: level = logging.DEBUG else: level = logging.INFO logging.basicConfig(level=level, format="[%(levelname)s] %(message)s") loop = ZMQEventLoop() asyncio.set_event_loop(loop) loop.run_until_complete(run()) loop.close()
class TestWorker(TestCase): @patch('alamo_worker.manage.WorkerManagement.__init__', MagicMock(return_value=None)) def setUp(self): self.loop = ZMQEventLoop() asyncio.set_event_loop(self.loop) self.worker = WorkerManagement() self.worker.loop = self.loop self.worker.manager = PluginManager() def tearDown(self): self.loop.close() asyncio.set_event_loop(None) @asyncio.coroutine def mark_soft_exit(self): self.worker.soft_exit = True def get_future(self, result): """Wrap `result` as Future object.""" fut = asyncio.Future(loop=self.loop) fut.set_result(result) return fut @patch('alamo_worker.manage.WorkerManagement.__init__', MagicMock(return_value=None)) @patch('alamo_worker.manage.WorkerManagement.close') def test_worker_manager_as_context_manager(self, close): with WorkerManagement(): pass self.assertTrue(close.called) @patch('alamo_worker.manage.PluginManager.load') def test_load_plugins_with_correct_plugin_list(self, load_mock): self.worker._load_plugins() self.assertTrue(load_mock.called) @patch('alamo_worker.manage.Queue') @patch('alamo_worker.manage.Sentinel.master_for') def test_connect_to_queue(self, zmq_mock, master_for): self.worker._connect_to_queue() self.assertTrue(master_for.called) self.assertTrue(zmq_mock.called) @patch('alamo_worker.manage.WorkerManagement.__init__', Mock()) @patch('alamo_worker.manage.Session.get') def test_get_environments(self, session_mock): mock_response = MagicMock(status_code=200) mock_response.raise_for_status.return_value = None mock_response.json.return_value = ENV_CONF session_mock.return_value = mock_response envs = self.worker._get_environments() self.assertEqual(envs, ENV_CONF) def test_send_method(self): self.worker.queue = MagicMock() self.worker.send('foo') self.worker.queue.enqueue.assert_called_with( 'alamo_alerter.job.analyze', args=('foo',), result_ttl=60 ) @patch('alamo_worker.manage.SchedulerConnection') def test_worker_open_method(self, mocked): mocked.return_value = MagicMock() self.worker.scheduler_pool = [] self.worker.open() self.assertEqual(len(self.worker.scheduler_pool), 1) def test_worker_close_method(self): scheduler = MagicMock() self.worker.scheduler_pool = [scheduler] self.worker.close() scheduler.close.assert_called_once_with() @unpack @data((False, 1), (True, 2)) def test_worker_generate_futures_method(self, monitor, expected): scheduler = MagicMock() self.worker.scheduler_pool = [scheduler] with override_settings(SCHEDULER_MONITOR=monitor): futures = [f for f in self.worker.generate_futures()] self.assertEqual(len(futures), expected) def test_worker_wait_and_kill_method(self): self.loop.run_until_complete(self.worker.wait_and_kill('SIGINT')) self.assertTrue(self.worker.soft_exit) @unpack @data( (None, None, False, False), ({'id': 999, 'type': 'graphite'}, dict(id=999), True, True), ({'id': 999, 'type': 'graphite'}, None, False, True), ) @patch('alamo_worker.manage.PluginManager.dispatch') @patch('alamo_worker.manage.WorkerManagement.send') def test_worker_receiver_method(self, event, dispatch_res, send_called, dispatch_called, send, dispatch): @asyncio.coroutine def _dispatch(*args, **kwargs): return dispatch_res dispatch.side_effect = _dispatch send._is_coroutine = False scheduler = MagicMock() scheduler.receive.side_effect = [ self.get_future(event), self.mark_soft_exit() ] self.worker.scheduler_pool = [scheduler] future = asyncio.async(self.worker.receiver(scheduler)) self.loop.run_until_complete(future) self.assertTrue(future.done()) self.assertEqual(send.called, send_called) self.assertEqual(dispatch.called, dispatch_called) @unpack @data( (True, dict(event=1), False, True), (True, dict(event=EVENT_DISCONNECTED), True, True), (True, dict(event=EVENT_MONITOR_STOPPED), False, True), (False, dict(event=1), False, False) ) @patch('alamo_worker.manage.logger') def test_worker_monitor_method(self, monitor, event, warn, info, logger): scheduler = MagicMock() scheduler.receive_event.side_effect = [ self.get_future(event), self.mark_soft_exit() ] self.worker.scheduler_pool = [scheduler] future = asyncio.async(self.worker.monitor(scheduler)) with override_settings(SCHEDULER_MONITOR=monitor): self.loop.run_until_complete(future) self.assertTrue(future.done()) self.assertEqual(logger.warning.called, warn) self.assertEqual(logger.info.called, info) @patch('alamo_worker.manage.WorkerManagement.generate_futures') def test_worker_execute_method(self, futures): futures.return_value = [self.mark_soft_exit()] self.worker.execute() self.assertTrue(self.loop.is_closed())
ws.send_bytes(message[-1]) # CLI def init_function(argv): rpc = RPC() app = web.Application() # Pass the websocket handler to the router, not the rpc method... app.router.add_route('GET', '/', rpc.websocket_handler) return app async def init(loop): rpc = RPC(loop=loop) app = web.Application(loop=loop) app.router.add_route('GET', '/', rpc.websocket_handler) handler = app.make_handler() srv = await loop.create_server(handler, '127.0.0.1', 8080) print('Server started at http://127.0.0.1:8080') return srv, handler if __name__ == '__main__': loop = ZMQEventLoop() asyncio.set_event_loop(loop) srv, handler = loop.run_until_complete(init(loop)) try: loop.run_forever() except KeyboardInterrupt: loop.run_until_complete(handler.finish_connections())
if (yield from client.poll(1000)): msg = yield from client.recv() if msg == b"Hello": logging.info("Ironhouse test OK") else: logging.error("Ironhouse test FAIL") # close sockets server.close() client.close() # stop auth task auth.stop() if __name__ == '__main__': if zmq.zmq_version_info() < (4,0): raise RuntimeError("Security is not supported in libzmq version < 4.0. libzmq version {0}".format(zmq.zmq_version())) if '-v' in sys.argv: level = logging.DEBUG else: level = logging.INFO logging.basicConfig(level=level, format="[%(levelname)s] %(message)s") loop = ZMQEventLoop() asyncio.set_event_loop(loop) loop.run_until_complete(run()) loop.close()
def _worker_main(id, slave_addr, task): print("_worker_main") import zmq from zmq.asyncio import Context, ZMQEventLoop import asyncio from ..common.task import SleepTaskResult from .task import SleepTask def _resolve_msg(msg): print(msg) #addr = msg[0] #assert msg[1] == b"" header = msg[0] assert msg[1] == b"" body = msg[2] return header, body def _dispatch_msg(header, body = b""): async def _dispatch_msg(msg): await socket.send_multipart(msg) msg = [id.encode(encoding='utf-8'), b'', header, b'', body] asyncio.ensure_future(_dispatch_msg(msg)) def __dispatch_msg(header, body=b""): def _dispatch_msg(msg): socket.send_multipart(msg) msg = [id.encode(encoding='utf-8'), b'', header, b'', body] _dispatch_msg(msg) def _process_sleep_task(task): async def __process_sleep_task(task): await asyncio.sleep(task.job.seconds) task.result = SleepTaskResult("Sleep " + str(task.job.seconds) + "By " + id) _dispatch_msg(b"TaskFinish", task.result.to_bytes()) asyncio.ensure_future(__process_sleep_task(task)) async def _run_worker(): _dispatch_msg(b"TaskStart") if isinstance(task, SleepTask): _process_sleep_task(task) else: raise ValueError("Invalid Task Type.") while True: msg = await socket.recv_multipart() header, body = _resolve_msg(msg) # some codes will be filled later. break print("[Worker {0}] I'm created!".format(id)) loop = ZMQEventLoop() asyncio.set_event_loop(loop) context = Context() socket = context.socket(zmq.DEALER) socket.connect(slave_addr) """ policy = asyncio.get_event_loop_policy() policy.set_event_loop(policy.new_event_loop()) loop = asyncio.get_event_loop() """ loop.run_until_complete(_run_worker())
@asyncio.coroutine def receiver(): """receive messages with polling""" pull = ctx.socket(zmq.PULL) pull.connect(url) poller = Poller() poller.register(pull, zmq.POLLIN) while True: events = yield from poller.poll() if pull in dict(events): print("recving", events) msg = yield from pull.recv_multipart() print('recvd', msg) @asyncio.coroutine def sender(): """send a message every second""" tic = time.time() push = ctx.socket(zmq.PUSH) push.bind(url) while True: print("sending") yield from push.send_multipart([str(time.time() - tic).encode('ascii')]) yield from asyncio.sleep(1) loop.run_until_complete(asyncio.wait([ ping(), receiver(), sender(), ]))
for key, value in self._cache.items(): self.send_multipart(key, self._cache[value]) elif _topic in self._cache: self.send_multipart( _topic, self._cache[_topic]) def test_md_send_multipart(): _server = MdPubServer(9988) _server.send_multipart("a", {"key":["a", "b", "c"], "data": "nihao"}) time.sleep(0.5) _server.send_multipart("b", {"key": ["a", "b", "c"], "data": "nihao"}) time.sleep(0.5) _server.send_multipart("c", {"key": ["a", "b", "c"], "data": "nihao"}) time.sleep(0.5) if __name__ == '__main__': _server = MdPubServer(9988) loop = ZMQEventLoop() asyncio.set_event_loop(loop) lvm_task = loop.create_task(_server.lvm()) _server.send_multipart("a", {"key":["a", "b", "c"], "data": "nihao"}) time.sleep(0.5) _server.send_multipart("b", {"key": ["a", "b", "c"], "data": "nihao"}) time.sleep(0.5) _server.send_multipart("c", {"key": ["a", "b", "c"], "data": "nihao"}) time.sleep(0.5) loop.run_until_complete(asyncio.wait([lvm_task]))
#!/usr/bin/env python