def start(self): loop = ioloop.IOLoop() s = zmq.Context().socket(zmq.PULL) s.SNDTIMEO = SNDTIMEO s.set_hwm(ZMQ_HWM) s.setsockopt(zmq.LINGER, 3) socket = zmqstream.ZMQStream(s, loop) socket.on_recv(self._process_message) socket.connect(HUNTER_ADDR) # this needs to be done here self.router = Client(remote=HUNTER_SINK_ADDR, token=self.token, nowait=True, autoclose=False) try: loop.start() except KeyboardInterrupt as e: loop.stop() self.router.socket.close()
def start(self, delay_start=0): """Start the client loop.""" # # Create a new instance of the ioloop. # This is important for example when running from an ipython notebook (which # is itself an ioloop.) # ioloop.IOLoop.clear_current() ioloop.IOLoop.clear_instance( ) # or del IOLoop._instance in tornado < 3.3 # # Set the (Tornado) loop # self.loop = ioloop.IOLoop().instance() # # Start the MDP client. # super(Client, self).start() # # Start the ioloop. # time.sleep(delay_start) ioloop.PeriodicCallback( partial(self.send_mmi, service=MDP.MMI_SERVICES, msg=[]), self.mmi_period, self.loop).start() self.loop.start()
def run(self): """Run my loop, ignoring EINTR events in the poller""" if 'asyncio' in sys.modules: # tornado may be using asyncio, # ensure an eventloop exists for this thread import asyncio asyncio.set_event_loop(asyncio.new_event_loop()) self.ioloop = ioloop.IOLoop() # signal that self.ioloop is defined self._start_event.set() while True: try: self.ioloop.start() except ZMQError as e: if e.errno == errno.EINTR: continue else: raise except Exception: if self._exiting: break else: raise else: break
def __init__(self, args=None, loop=None): if args is None: args = {} super(ExternalRunner, self).__init__(args) self._current_step = 0 self._step_started_at = None self._duration = self.args.get('duration') self._timeout = args.get('external_process_timeout', 2) self._processes = [] self._processes_pending_cleanup = [] # hits and users are lists that can be None. hits, users = [1], [1] if self.args.get('hits') is not None: hits = self.args['hits'] if self.args.get('users') is not None: users = self.args['users'] self.args['hits'] = hits self.args['users'] = users self._nb_steps = max(len(hits), len(users)) self._loop = loop or ioloop.IOLoop() # Check the status of the processes every so-often.(500ms) cb = ioloop.PeriodicCallback(self._check_processes, 500, self._loop) cb.start() self._receiver_socket = (self.args.get('zmq_receiver') or DEFAULT_EXTERNAL_RUNNER_RECEIVER)
def __init__(self, target, backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT, ping_delay=1., ping_retries=3, params=None, timeout=DEFAULT_TIMEOUT_MOVF, max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA): logger.debug('Initializing the worker.') self.ctx = zmq.Context() self.backend = backend self._backend = self.ctx.socket(zmq.REP) self._backend.connect(self.backend) self.target = target self.running = False self.loop = ioloop.IOLoop() self._backstream = zmqstream.ZMQStream(self._backend, self.loop) self._backstream.on_recv(self._handle_recv_back) self.ping = Stethoscope(heartbeat, onbeatlost=self.lost, delay=ping_delay, retries=ping_retries, ctx=self.ctx) self.debug = logger.isEnabledFor(logging.DEBUG) self.params = params self.pid = os.getpid() self.timer = ExecutionTimer(timeout=timeout) self.max_age = max_age self.max_age_delta = max_age_delta self.delayed_exit = None self.lock = threading.RLock()
def test_add_job(self): arbiter = mock.MagicMock() class MockedController(Controller): called = False def _init_stream(self): pass # NO OP def initialize(self): pass # NO OP def dispatch(self, job): self.called = True self.loop.stop() loop = ioloop.IOLoop() controller = MockedController('endpoint', 'multicast_endpoint', mock.sentinel.context, loop, arbiter, check_delay=-1.0) controller.dispatch((None, 'something')) loop.add_timeout(loop.time() + 1, loop.stop) controller.start() loop.start() self.assertTrue(controller.called)
def start(self): pull_s = Context().socket(zmq.PULL) pull_s.setsockopt(zmq.LINGER, 3) pull_s.SNDTIMEO = SNDTIMEO pull_s.RCVTIMEO = RCVTIMEO self.push_s = Context().socket(zmq.PUSH) self.push_s.setsockopt(zmq.LINGER, 3) self.push_s.SNDTIMEO = SNDTIMEO self.push_s.RCVTIMEO = RCVTIMEO logger.info('connecting...') pull_s.connect(ENRICHER_ADDR) self.push_s.connect(ENRICHER_SINK_ADDR) logger.info('connected') loop = ioloop.IOLoop() s = zmqstream.ZMQStream(pull_s, loop) s.on_recv(self._process_message) s.connect(ENRICHER_ADDR) try: loop.start() except KeyboardInterrupt: loop.stop() for ss in [s, pull_s, self.push_s]: ss.close()
def __init__(self, local_domains=("localhost")): self.local_domains = local_domains self.loop = ioloop.IOLoop() self.ctx = zmq.Context() log.debug('Registering StanzaProcessor at forwarder..') # connect push socket to forwarder self.forwarder = self.ctx.socket(zmq.PUSH) self.forwarder.connect(config.get('ipc', 'forwarder')) pull_socket = self.ctx.socket(zmq.PULL) stream = zmqstream.ZMQStream(pull_socket, self.loop) stream.on_recv(self.handle_stanza, False) port = pull_socket.bind_to_random_port('tcp://127.0.0.1') # register connection at forwarder reg_msg = ZMQForwarder_message('REGISTER') reg_msg.attributes = (config.get('ipc', 'password'), 'tcp://127.0.0.1:' + str(port), local_domains) self.forwarder.send_pyobj(reg_msg) # init the handlers self.stanza_handlers = { 'iq': iq.Iq(), 'message': message.Message(), 'presence': presence.Presence() }
def __init__(self): # State self.running = False self.terminated = False self.ioloop = ioloop.IOLoop() self.shutdown_handlers = [] self.exception_handlers = {}
def setUp(self): self.fail_msg = None self.io_loop = ioloop.IOLoop() super(TestIOLoopAuthentication, self).setUp() self.server = self.socket(zmq.PUSH) self.client = self.socket(zmq.PULL) self.pushstream = zmqstream.ZMQStream(self.server, self.io_loop) self.pullstream = zmqstream.ZMQStream(self.client, self.io_loop)
def test_timeout_compare(self): """test timeout comparisons""" loop = ioloop.IOLoop() t = _Timeout(1, 2, loop) t2 = _Timeout(1, 3, loop) self.assertEqual(t < t2, id(t) < id(t2)) t2 = _Timeout(2,1, loop) self.assertTrue(t < t2)
def _backend_init(self, io_loop=None): self.reader = None self.internal_loop = False if io_loop is None: self.internal_loop = True self.io_loop = ioloop.IOLoop() else: self.io_loop = io_loop
def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, reg_addr, config=None, logname='root', log_url=None, loglevel=logging.DEBUG, identity=b'task', in_thread=False): ZMQStream = zmqstream.ZMQStream if config: # unwrap dict back into Config config = Config(config) if in_thread: # use instance() to get the same Context/Loop as our parent ctx = zmq.Context.instance() loop = ioloop.IOLoop.instance() else: # in a process, don't use instance() # for safety with multiprocessing ctx = zmq.Context() loop = ioloop.IOLoop() ins = ZMQStream(ctx.socket(zmq.ROUTER),loop) util.set_hwm(ins, 0) ins.setsockopt(zmq.IDENTITY, identity + b'_in') ins.bind(in_addr) outs = ZMQStream(ctx.socket(zmq.ROUTER),loop) util.set_hwm(outs, 0) outs.setsockopt(zmq.IDENTITY, identity + b'_out') outs.bind(out_addr) mons = zmqstream.ZMQStream(ctx.socket(zmq.PUB),loop) util.set_hwm(mons, 0) mons.connect(mon_addr) nots = zmqstream.ZMQStream(ctx.socket(zmq.SUB),loop) nots.setsockopt(zmq.SUBSCRIBE, b'') nots.connect(not_addr) querys = ZMQStream(ctx.socket(zmq.DEALER),loop) querys.connect(reg_addr) # setup logging. if in_thread: log = Application.instance().log else: if log_url: log = connect_logger(logname, ctx, log_url, root="scheduler", loglevel=loglevel) else: log = local_logger(logname, loglevel) scheduler = TaskScheduler(client_stream=ins, engine_stream=outs, mon_stream=mons, notifier_stream=nots, query_stream=querys, loop=loop, log=log, config=config) scheduler.start() if not in_thread: try: loop.start() except KeyboardInterrupt: scheduler.log.critical("Interrupted, exiting...")
def __init__(self, broker=DEFAULT_FRONTEND, ping_delay=10., ping_retries=3, params=None, timeout=DEFAULT_TIMEOUT_MOVF, max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA): logger.debug('Initializing the agent.') self.debug = logger.isEnabledFor(logging.DEBUG) self.params = params self.pid = os.getpid() self.agent_id = '%s-%s' % (get_hostname(), self.pid) self.timeout = timeout self.max_age = max_age self.max_age_delta = max_age_delta self.env = os.environ.copy() self.running = False self._workers = {} self._max_id = defaultdict(int) # Let's ask the broker its options self.broker = broker client = Client(self.broker) # this will timeout in case the broker is unreachable result = client.ping() self.endpoints = result['endpoints'] # Setup the zmq sockets self.loop = ioloop.IOLoop() self.ctx = zmq.Context() # backend socket - used to receive work from the broker self._backend = self.ctx.socket(zmq.ROUTER) self._backend.identity = self.agent_id self._backend.connect(self.endpoints['backend']) # register socket - used to register into the broker self._reg = self.ctx.socket(zmq.PUSH) self._reg.connect(self.endpoints['register']) # hearbeat socket - used to check if the broker is alive heartbeat = self.endpoints.get('heartbeat') if heartbeat is not None: logger.info("Hearbeat activated") self.ping = Stethoscope(heartbeat, onbeatlost=self.lost, delay=ping_delay, retries=ping_retries, ctx=self.ctx, io_loop=self.loop, onregister=self.register) else: self.ping = None # Setup the zmq streams. self._backstream = zmqstream.ZMQStream(self._backend, self.loop) self._backstream.on_recv(self._handle_recv_back) self._check = ioloop.PeriodicCallback(self._check_proc, ping_delay * 1000, io_loop=self.loop)
def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, config=None, logname='ZMQ', log_addr=None, loglevel=logging.DEBUG, scheme='lru', identity=b'task'): from zmq.eventloop import ioloop from zmq.eventloop.zmqstream import ZMQStream if config: # unwrap dict back into Config config = Config(config) ctx = zmq.Context() loop = ioloop.IOLoop() ins = ZMQStream(ctx.socket(zmq.XREP), loop) ins.setsockopt(zmq.IDENTITY, identity) ins.bind(in_addr) outs = ZMQStream(ctx.socket(zmq.XREP), loop) outs.setsockopt(zmq.IDENTITY, identity) outs.bind(out_addr) mons = ZMQStream(ctx.socket(zmq.PUB), loop) mons.connect(mon_addr) nots = ZMQStream(ctx.socket(zmq.SUB), loop) nots.setsockopt(zmq.SUBSCRIBE, '') nots.connect(not_addr) scheme = globals().get(scheme, None) # setup logging if log_addr: connect_logger(logname, ctx, log_addr, root="scheduler", loglevel=loglevel) else: local_logger(logname, loglevel) scheduler = TaskScheduler(client_stream=ins, engine_stream=outs, mon_stream=mons, notifier_stream=nots, scheme=scheme, loop=loop, logname=logname, config=config) scheduler.start() try: loop.start() except KeyboardInterrupt: print("interrupted, exiting...", file=sys.__stderr__)
def test_execute(self): loop = ioloop.IOLoop() loop.start = mock.Mock() runner = ExternalRunner({'hits': [2], 'users': [2]}, loop) runner._prepare_filesystem = mock.Mock() runner.spawn_external_runner = mock.Mock() runner._execute() self.assertTrue(runner._prepare_filesystem.called) self.assertEquals(runner.spawn_external_runner.call_count, 4)
def run(): loop = ioloop.IOLoop() context = zmq.Context() pub = context.socket(zmq.PUB) pub.bind('tcp://0.0.0.0:5555') router = context.socket(zmq.ROUTER) router.bind('tcp://0.0.0.0:15556') outstream = zmqstream.ZMQStream(pub, loop) instream = zmqstream.ZMQStream(router, loop) hb = HeartBeater(loop, outstream, instream) loop.start()
def heartbeat_run(): loop = ioloop.IOLoop() context = zmq.Context() pub = context.socket(zmq.PUB) pub.bind('tcp://0.0.0.0:{}'.format(HEART_PUB_PORT)) router = context.socket(zmq.ROUTER) router.bind('tcp://0.0.0.0:{}'.format(HEART_ROUTER_PORT)) outstream = zmqstream.ZMQStream(pub, loop) instream = zmqstream.ZMQStream(router, loop) hb = HeartBeater(loop, outstream, instream) loop.start()
def __init__(self, name, frontend_point, backend_point, context=None, loop=None): self._context = context or zmq.Context.instance() self._loop = loop or ioloop.IOLoop().instance() self._name = name self._frontend_point = frontend_point self._backend_point = backend_point
def __init__(self, endpoint, pubsub_endpoint, check_delay, **config): self.daemon = True self.config = config self.active = to_bool(config.get('active', True)) self.context = zmq.Context() self.pubsub_endpoint = pubsub_endpoint self.endpoint = endpoint self.check_delay = check_delay self.loop = ioloop.IOLoop() self._id = uuid.uuid4().hex # XXX os.getpid()+thread id is enough... self.running = False
def __init__(self, forwarder_url): self.loop = ioloop.IOLoop() self.ctx = zmq.Context() # Create our PULL listener to listen to the world ;) self.pull_sock = self.ctx.socket(zmq.PULL) log.debug('Listening at ' + forwarder_url) self.pull_sock.bind(forwarder_url) self.stream = zmqstream.ZMQStream(self.pull_sock, self.loop) self.stream.on_recv(self.handle_stanza, False) self.peers = dict()
def __init__(self, context, endpoint, pubsub_endpoint, check_delay): super(Flapping, self).__init__() self.daemon = True self.context = context self.pubsub_endpoint = pubsub_endpoint self.endpoint = endpoint self.check_delay = check_delay self.loop = ioloop.IOLoop() self._id = uuid.uuid4().hex self.timelines = {} self.timers = {} self.configs = {} self.tries = {}
def setUp(self): if tornado is None: pytest.skip() if asyncio: asyncio.set_event_loop(asyncio.new_event_loop()) self.context = zmq.Context() self.loop = ioloop.IOLoop() self.loop.make_current() self.push = zmqstream.ZMQStream(self.context.socket(zmq.PUSH)) self.pull = zmqstream.ZMQStream(self.context.socket(zmq.PULL)) port = self.push.bind_to_random_port('tcp://127.0.0.1') self.pull.connect('tcp://127.0.0.1:%i' % port) self.stream = self.push
def __init__(self, watchers, endpoint, pubsub_endpoint, check_delay=1., prereload_fn=None, context=None, loop=None, stats_endpoint=None, plugins=None, sockets=None): self.watchers = watchers self.endpoint = endpoint self.check_delay = check_delay self.prereload_fn = prereload_fn self.pubsub_endpoint = pubsub_endpoint # initialize zmq context self.context = context or zmq.Context.instance() self.loop = loop or ioloop.IOLoop() self.ctrl = Controller(endpoint, self.context, self.loop, self, check_delay) self.pid = os.getpid() self._watchers_names = {} self.alive = True self._lock = RLock() # initializing circusd-stats as a watcher when configured self.stats_endpoint = stats_endpoint if self.stats_endpoint is not None: cmd = "%s -c 'from circus import stats; stats.main()'" % \ sys.executable cmd += ' --endpoint %s' % self.endpoint cmd += ' --pubsub %s' % self.pubsub_endpoint cmd += ' --statspoint %s' % self.stats_endpoint stats_watcher = Watcher('circusd-stats', cmd) self.watchers.append(stats_watcher) # adding each plugin as a watcher if plugins is not None: for plugin in plugins: fqnd = plugin['use'] name = 'plugin:%s' % fqnd.replace('.', '-') cmd = get_plugin_cmd(plugin, self.endpoint, self.pubsub_endpoint, self.check_delay) plugin_watcher = Watcher(name, cmd, priority=1, singleton=True) self.watchers.append(plugin_watcher) self.sockets = CircusSockets(sockets)
def instance_runner(self): loop = ioloop.IOLoop() app = server.Application(port=self.port) http_server = tornado.httpserver.HTTPServer(app, max_buffer_size=1024**3) http_server.listen(self.port) logging.info('Started instance') print('\nSearch "<server-id>:'+str(self.port)+'" in browser for a '\ 'training visualization.') print('If running locally, <server-id> = localhost.\n') loop.start() http_server.stop() logging.info('Ended instance')
def test_simple(self): """simple IOLoop creation test""" loop = ioloop.IOLoop() dc = ioloop.DelayedCallback(loop.stop, 200, loop) pc = ioloop.DelayedCallback(lambda : None, 10, loop) pc.start() dc.start() t = Delay(loop.stop,1) t.start() loop.start() if t.isAlive(): t.abort() else: self.assert_(False, "IOLoop failed to exit")
def __init__(self, *args, **kwargs): self.loop = ioloop.IOLoop().instance() data_dir = config.home + '/run/' self.open_storage(data_dir + 'bot.db') self.prefix = 'Tom' self.all_candidates = None super(TomBot, self).__init__(*args, **kwargs) # def update_dynamic_plugins(self): # all_candidates, errors = update_plugin_places([self.plugin_dir + os.sep + d for d in self.get(REPOS, {}).keys()]) # self.all_candidates = all_candidates # return errors self.update_dynamic_plugins() self.inject_commands_from(self)
def __init__(self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND, heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG, io_threads=DEFAULT_IOTHREADS, worker_timeout=DEFAULT_TIMEOUT_MOVF): # before doing anything, we verify if a broker is already up and # running logger.debug('Verifying if there is a running broker') pid = verify_broker(frontend) if pid is not None: # oops. can't do this ! logger.debug('Ooops, we have a running broker on that socket') raise DuplicateBrokerError(pid) logger.debug('Initializing the broker.') for endpoint in (frontend, backend, heartbeat): if endpoint.startswith('ipc'): register_ipc_file(endpoint) self.context = zmq.Context(io_threads=io_threads) # setting up the three sockets self._frontend = self.context.socket(zmq.ROUTER) self._frontend.identity = 'broker-' + frontend self._frontend.bind(frontend) self._backend = self.context.socket(zmq.ROUTER) self._backend.bind(backend) self._registration = self.context.socket(zmq.PULL) self._registration.bind(register) # setting up the streams self.loop = ioloop.IOLoop() self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop) self._frontstream.on_recv(self._handle_recv_front) self._backstream = zmqstream.ZMQStream(self._backend, self.loop) self._backstream.on_recv(self._handle_recv_back) self._regstream = zmqstream.ZMQStream(self._registration, self.loop) self._regstream.on_recv(self._handle_reg) # heartbeat self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context) # status self.started = False self.poll_timeout = None # workers registration and timers self._workers = [] self._worker_times = {} self.worker_timeout = worker_timeout
def run(self): """Run my loop, ignoring EINTR events in the poller""" loop = ioloop.IOLoop(make_current=True) self.client = IOLoopKernelClient(self.connection_info, manager=self.manager) self.client.ioloop.add_callback(self.started.set) self._ready_fut = self.client.wait_for_ready() loop.add_future(self._ready_fut, lambda future: self.kernel_responding.set()) try: self._run_loop() finally: self.client.close() self.client.ioloop.close() self.client = None
def test_simple(self): """simple IOLoop creation test""" loop = ioloop.IOLoop() loop.make_current() dc = ioloop.PeriodicCallback(loop.stop, 200) pc = ioloop.PeriodicCallback(lambda: None, 10) pc.start() dc.start() t = Delay(loop.stop, 1) t.start() loop.start() if t.isAlive(): t.abort() else: self.fail("IOLoop failed to exit")