def main(MASTER_ADDR, WORKER_ROUTER_ADDR, slave_addr, control_router_addr):
global context
global master_conn
global worker_router
global worker_manager
global SLAVE_ADDR
global CONTROL_ROUTER_ADDR
SLAVE_ADDR = slave_addr
CONTROL_ROUTER_ADDR = control_router_addr
try:
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
context = Context()
master_conn = MasterConnection(context, MASTER_ADDR)
worker_router = WorkerRouter(context, WORKER_ROUTER_ADDR)
worker_manager = WorkerManager()
#loop.set_default_executor(ProcessPoolExecutor())
loop.run_until_complete(run_server())
except KeyboardInterrupt:
print('\nFinished (interrupted)')
sys.exit(0)
def main():
try:
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
loop.run_until_complete(run_server())
except KeyboardInterrupt:
print('\nFinished (interrupted)')
sys.exit(0)
def main():
args = sys.argv[1:]
if len(args) != 0:
sys.exit(__doc__)
try:
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
loop.run_until_complete(run(loop))
except KeyboardInterrupt:
print('\nFinished (interrupted)')
def main():
args = sys.argv[1:]
if len(args) > 1:
sys.exit(__doc__)
zip_filter = sys.argv[1] if len(sys.argv) > 1 else "10001"
try:
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
loop.run_until_complete(run(loop, zip_filter))
except KeyboardInterrupt:
print('\nFinished (interrupted)')
def main():
args = sys.argv[1:]
if len(args) > 1:
sys.exit(__doc__)
zip_filter = sys.argv[1] if len(sys.argv) > 1 else "10001"
try:
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
loop.run_until_complete(run(loop, zip_filter))
except KeyboardInterrupt:
print('\nFinished (interrupted)')
def main(slave_addr, serialized_data : bytes):
try:
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
context = Context()
loop.run_until_complete(run_worker(context, slave_addr, serialized_data))
except KeyboardInterrupt:
print('\nFinished (interrupted)')
sys.exit(0)
def main(client_router_addr, slave_router_addr):
try:
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
context = Context()
loop.run_until_complete(run_master(context, client_router_addr, slave_router_addr))
except KeyboardInterrupt:
print('\nFinished (interrupted)')
sys.exit(0)
def main():
args = sys.argv[1:]
if len(args) == 1:
num_requests = int(args[0])
elif len(args) == 0:
num_requests = 10
else:
sys.exit(__doc__)
try:
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
loop.run_until_complete(run(loop, num_requests))
except KeyboardInterrupt:
print('\nFinished (interrupted)')
def main():
args = sys.argv[1:]
if len(args) == 1:
num_requests = int(args[0])
elif len(args) == 0:
num_requests = 10
else:
sys.exit(__doc__)
try:
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
loop.run_until_complete(run(loop, num_requests))
except KeyboardInterrupt:
print('\nFinished (interrupted)')
msg = yield from client.recv()
if msg == b"Hello":
logging.info("Ironhouse test OK")
else:
logging.error("Ironhouse test FAIL")
# close sockets
server.close()
client.close()
# stop auth task
auth.stop()
if __name__ == '__main__':
if zmq.zmq_version_info() < (4, 0):
raise RuntimeError(
"Security is not supported in libzmq version < 4.0. libzmq version {0}"
.format(zmq.zmq_version()))
if '-v' in sys.argv:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(level=level, format="[%(levelname)s] %(message)s")
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
loop.run_until_complete(run())
loop.close()
class TestWorker(TestCase):
@patch('alamo_worker.manage.WorkerManagement.__init__',
MagicMock(return_value=None))
def setUp(self):
self.loop = ZMQEventLoop()
asyncio.set_event_loop(self.loop)
self.worker = WorkerManagement()
self.worker.loop = self.loop
self.worker.manager = PluginManager()
def tearDown(self):
self.loop.close()
asyncio.set_event_loop(None)
@asyncio.coroutine
def mark_soft_exit(self):
self.worker.soft_exit = True
def get_future(self, result):
"""Wrap `result` as Future object."""
fut = asyncio.Future(loop=self.loop)
fut.set_result(result)
return fut
@patch('alamo_worker.manage.WorkerManagement.__init__',
MagicMock(return_value=None))
@patch('alamo_worker.manage.WorkerManagement.close')
def test_worker_manager_as_context_manager(self, close):
with WorkerManagement():
pass
self.assertTrue(close.called)
@patch('alamo_worker.manage.PluginManager.load')
def test_load_plugins_with_correct_plugin_list(self, load_mock):
self.worker._load_plugins()
self.assertTrue(load_mock.called)
@patch('alamo_worker.manage.Queue')
@patch('alamo_worker.manage.Sentinel.master_for')
def test_connect_to_queue(self, zmq_mock, master_for):
self.worker._connect_to_queue()
self.assertTrue(master_for.called)
self.assertTrue(zmq_mock.called)
@patch('alamo_worker.manage.WorkerManagement.__init__', Mock())
@patch('alamo_worker.manage.Session.get')
def test_get_environments(self, session_mock):
mock_response = MagicMock(status_code=200)
mock_response.raise_for_status.return_value = None
mock_response.json.return_value = ENV_CONF
session_mock.return_value = mock_response
envs = self.worker._get_environments()
self.assertEqual(envs, ENV_CONF)
def test_send_method(self):
self.worker.queue = MagicMock()
self.worker.send('foo')
self.worker.queue.enqueue.assert_called_with(
'alamo_alerter.job.analyze',
args=('foo',),
result_ttl=60
)
@patch('alamo_worker.manage.SchedulerConnection')
def test_worker_open_method(self, mocked):
mocked.return_value = MagicMock()
self.worker.scheduler_pool = []
self.worker.open()
self.assertEqual(len(self.worker.scheduler_pool), 1)
def test_worker_close_method(self):
scheduler = MagicMock()
self.worker.scheduler_pool = [scheduler]
self.worker.close()
scheduler.close.assert_called_once_with()
@unpack
@data((False, 1), (True, 2))
def test_worker_generate_futures_method(self, monitor, expected):
scheduler = MagicMock()
self.worker.scheduler_pool = [scheduler]
with override_settings(SCHEDULER_MONITOR=monitor):
futures = [f for f in self.worker.generate_futures()]
self.assertEqual(len(futures), expected)
def test_worker_wait_and_kill_method(self):
self.loop.run_until_complete(self.worker.wait_and_kill('SIGINT'))
self.assertTrue(self.worker.soft_exit)
@unpack
@data(
(None, None, False, False),
({'id': 999, 'type': 'graphite'}, dict(id=999), True, True),
({'id': 999, 'type': 'graphite'}, None, False, True),
)
@patch('alamo_worker.manage.PluginManager.dispatch')
@patch('alamo_worker.manage.WorkerManagement.send')
def test_worker_receiver_method(self,
event, dispatch_res,
send_called, dispatch_called,
send, dispatch):
@asyncio.coroutine
def _dispatch(*args, **kwargs):
return dispatch_res
dispatch.side_effect = _dispatch
send._is_coroutine = False
scheduler = MagicMock()
scheduler.receive.side_effect = [
self.get_future(event),
self.mark_soft_exit()
]
self.worker.scheduler_pool = [scheduler]
future = asyncio.async(self.worker.receiver(scheduler))
self.loop.run_until_complete(future)
self.assertTrue(future.done())
self.assertEqual(send.called, send_called)
self.assertEqual(dispatch.called, dispatch_called)
@unpack
@data(
(True, dict(event=1), False, True),
(True, dict(event=EVENT_DISCONNECTED), True, True),
(True, dict(event=EVENT_MONITOR_STOPPED), False, True),
(False, dict(event=1), False, False)
)
@patch('alamo_worker.manage.logger')
def test_worker_monitor_method(self, monitor, event, warn, info, logger):
scheduler = MagicMock()
scheduler.receive_event.side_effect = [
self.get_future(event), self.mark_soft_exit()
]
self.worker.scheduler_pool = [scheduler]
future = asyncio.async(self.worker.monitor(scheduler))
with override_settings(SCHEDULER_MONITOR=monitor):
self.loop.run_until_complete(future)
self.assertTrue(future.done())
self.assertEqual(logger.warning.called, warn)
self.assertEqual(logger.info.called, info)
@patch('alamo_worker.manage.WorkerManagement.generate_futures')
def test_worker_execute_method(self, futures):
futures.return_value = [self.mark_soft_exit()]
self.worker.execute()
self.assertTrue(self.loop.is_closed())
ws.send_bytes(message[-1])
# CLI
def init_function(argv):
rpc = RPC()
app = web.Application()
# Pass the websocket handler to the router, not the rpc method...
app.router.add_route('GET', '/', rpc.websocket_handler)
return app
async def init(loop):
rpc = RPC(loop=loop)
app = web.Application(loop=loop)
app.router.add_route('GET', '/', rpc.websocket_handler)
handler = app.make_handler()
srv = await loop.create_server(handler, '127.0.0.1', 8080)
print('Server started at http://127.0.0.1:8080')
return srv, handler
if __name__ == '__main__':
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
srv, handler = loop.run_until_complete(init(loop))
try:
loop.run_forever()
except KeyboardInterrupt:
loop.run_until_complete(handler.finish_connections())
if (yield from client.poll(1000)):
msg = yield from client.recv()
if msg == b"Hello":
logging.info("Ironhouse test OK")
else:
logging.error("Ironhouse test FAIL")
# close sockets
server.close()
client.close()
# stop auth task
auth.stop()
if __name__ == '__main__':
if zmq.zmq_version_info() < (4,0):
raise RuntimeError("Security is not supported in libzmq version < 4.0. libzmq version {0}".format(zmq.zmq_version()))
if '-v' in sys.argv:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(level=level, format="[%(levelname)s] %(message)s")
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
loop.run_until_complete(run())
loop.close()
def _worker_main(id, slave_addr, task):
print("_worker_main")
import zmq
from zmq.asyncio import Context, ZMQEventLoop
import asyncio
from ..common.task import SleepTaskResult
from .task import SleepTask
def _resolve_msg(msg):
print(msg)
#addr = msg[0]
#assert msg[1] == b""
header = msg[0]
assert msg[1] == b""
body = msg[2]
return header, body
def _dispatch_msg(header, body = b""):
async def _dispatch_msg(msg):
await socket.send_multipart(msg)
msg = [id.encode(encoding='utf-8'), b'', header, b'', body]
asyncio.ensure_future(_dispatch_msg(msg))
def __dispatch_msg(header, body=b""):
def _dispatch_msg(msg):
socket.send_multipart(msg)
msg = [id.encode(encoding='utf-8'), b'', header, b'', body]
_dispatch_msg(msg)
def _process_sleep_task(task):
async def __process_sleep_task(task):
await asyncio.sleep(task.job.seconds)
task.result = SleepTaskResult("Sleep " + str(task.job.seconds) + "By " + id)
_dispatch_msg(b"TaskFinish", task.result.to_bytes())
asyncio.ensure_future(__process_sleep_task(task))
async def _run_worker():
_dispatch_msg(b"TaskStart")
if isinstance(task, SleepTask):
_process_sleep_task(task)
else:
raise ValueError("Invalid Task Type.")
while True:
msg = await socket.recv_multipart()
header, body = _resolve_msg(msg)
# some codes will be filled later.
break
print("[Worker {0}] I'm created!".format(id))
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
context = Context()
socket = context.socket(zmq.DEALER)
socket.connect(slave_addr)
"""
policy = asyncio.get_event_loop_policy()
policy.set_event_loop(policy.new_event_loop())
loop = asyncio.get_event_loop()
"""
loop.run_until_complete(_run_worker())
@asyncio.coroutine
def receiver():
"""receive messages with polling"""
pull = ctx.socket(zmq.PULL)
pull.connect(url)
poller = Poller()
poller.register(pull, zmq.POLLIN)
while True:
events = yield from poller.poll()
if pull in dict(events):
print("recving", events)
msg = yield from pull.recv_multipart()
print('recvd', msg)
@asyncio.coroutine
def sender():
"""send a message every second"""
tic = time.time()
push = ctx.socket(zmq.PUSH)
push.bind(url)
while True:
print("sending")
yield from push.send_multipart([str(time.time() - tic).encode('ascii')])
yield from asyncio.sleep(1)
loop.run_until_complete(asyncio.wait([
ping(),
receiver(),
sender(),
]))
for key, value in self._cache.items():
self.send_multipart(key, self._cache[value])
elif _topic in self._cache:
self.send_multipart( _topic, self._cache[_topic])
def test_md_send_multipart():
_server = MdPubServer(9988)
_server.send_multipart("a", {"key":["a", "b", "c"], "data": "nihao"})
time.sleep(0.5)
_server.send_multipart("b", {"key": ["a", "b", "c"], "data": "nihao"})
time.sleep(0.5)
_server.send_multipart("c", {"key": ["a", "b", "c"], "data": "nihao"})
time.sleep(0.5)
if __name__ == '__main__':
_server = MdPubServer(9988)
loop = ZMQEventLoop()
asyncio.set_event_loop(loop)
lvm_task = loop.create_task(_server.lvm())
_server.send_multipart("a", {"key":["a", "b", "c"], "data": "nihao"})
time.sleep(0.5)
_server.send_multipart("b", {"key": ["a", "b", "c"], "data": "nihao"})
time.sleep(0.5)
_server.send_multipart("c", {"key": ["a", "b", "c"], "data": "nihao"})
time.sleep(0.5)
loop.run_until_complete(asyncio.wait([lvm_task]))
#!/usr/bin/env python