def start(self):
loop = ioloop.IOLoop()
s = zmq.Context().socket(zmq.PULL)
s.SNDTIMEO = SNDTIMEO
s.set_hwm(ZMQ_HWM)
s.setsockopt(zmq.LINGER, 3)
socket = zmqstream.ZMQStream(s, loop)
socket.on_recv(self._process_message)
socket.connect(HUNTER_ADDR)
# this needs to be done here
self.router = Client(remote=HUNTER_SINK_ADDR,
token=self.token,
nowait=True,
autoclose=False)
try:
loop.start()
except KeyboardInterrupt as e:
loop.stop()
self.router.socket.close()
def start(self, delay_start=0):
"""Start the client loop."""
#
# Create a new instance of the ioloop.
# This is important for example when running from an ipython notebook (which
# is itself an ioloop.)
#
ioloop.IOLoop.clear_current()
ioloop.IOLoop.clear_instance(
) # or del IOLoop._instance in tornado < 3.3
#
# Set the (Tornado) loop
#
self.loop = ioloop.IOLoop().instance()
#
# Start the MDP client.
#
super(Client, self).start()
#
# Start the ioloop.
#
time.sleep(delay_start)
ioloop.PeriodicCallback(
partial(self.send_mmi, service=MDP.MMI_SERVICES, msg=[]),
self.mmi_period, self.loop).start()
self.loop.start()
def run(self):
"""Run my loop, ignoring EINTR events in the poller"""
if 'asyncio' in sys.modules:
# tornado may be using asyncio,
# ensure an eventloop exists for this thread
import asyncio
asyncio.set_event_loop(asyncio.new_event_loop())
self.ioloop = ioloop.IOLoop()
# signal that self.ioloop is defined
self._start_event.set()
while True:
try:
self.ioloop.start()
except ZMQError as e:
if e.errno == errno.EINTR:
continue
else:
raise
except Exception:
if self._exiting:
break
else:
raise
else:
break
def __init__(self, args=None, loop=None):
if args is None:
args = {}
super(ExternalRunner, self).__init__(args)
self._current_step = 0
self._step_started_at = None
self._duration = self.args.get('duration')
self._timeout = args.get('external_process_timeout', 2)
self._processes = []
self._processes_pending_cleanup = []
# hits and users are lists that can be None.
hits, users = [1], [1]
if self.args.get('hits') is not None:
hits = self.args['hits']
if self.args.get('users') is not None:
users = self.args['users']
self.args['hits'] = hits
self.args['users'] = users
self._nb_steps = max(len(hits), len(users))
self._loop = loop or ioloop.IOLoop()
# Check the status of the processes every so-often.(500ms)
cb = ioloop.PeriodicCallback(self._check_processes, 500, self._loop)
cb.start()
self._receiver_socket = (self.args.get('zmq_receiver')
or DEFAULT_EXTERNAL_RUNNER_RECEIVER)
def __init__(self, target, backend=DEFAULT_BACKEND,
heartbeat=DEFAULT_HEARTBEAT, ping_delay=1., ping_retries=3,
params=None, timeout=DEFAULT_TIMEOUT_MOVF,
max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA):
logger.debug('Initializing the worker.')
self.ctx = zmq.Context()
self.backend = backend
self._backend = self.ctx.socket(zmq.REP)
self._backend.connect(self.backend)
self.target = target
self.running = False
self.loop = ioloop.IOLoop()
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self.ping = Stethoscope(heartbeat, onbeatlost=self.lost,
delay=ping_delay, retries=ping_retries,
ctx=self.ctx)
self.debug = logger.isEnabledFor(logging.DEBUG)
self.params = params
self.pid = os.getpid()
self.timer = ExecutionTimer(timeout=timeout)
self.max_age = max_age
self.max_age_delta = max_age_delta
self.delayed_exit = None
self.lock = threading.RLock()
def test_add_job(self):
arbiter = mock.MagicMock()
class MockedController(Controller):
called = False
def _init_stream(self):
pass # NO OP
def initialize(self):
pass # NO OP
def dispatch(self, job):
self.called = True
self.loop.stop()
loop = ioloop.IOLoop()
controller = MockedController('endpoint',
'multicast_endpoint',
mock.sentinel.context,
loop,
arbiter,
check_delay=-1.0)
controller.dispatch((None, 'something'))
loop.add_timeout(loop.time() + 1, loop.stop)
controller.start()
loop.start()
self.assertTrue(controller.called)
def start(self):
pull_s = Context().socket(zmq.PULL)
pull_s.setsockopt(zmq.LINGER, 3)
pull_s.SNDTIMEO = SNDTIMEO
pull_s.RCVTIMEO = RCVTIMEO
self.push_s = Context().socket(zmq.PUSH)
self.push_s.setsockopt(zmq.LINGER, 3)
self.push_s.SNDTIMEO = SNDTIMEO
self.push_s.RCVTIMEO = RCVTIMEO
logger.info('connecting...')
pull_s.connect(ENRICHER_ADDR)
self.push_s.connect(ENRICHER_SINK_ADDR)
logger.info('connected')
loop = ioloop.IOLoop()
s = zmqstream.ZMQStream(pull_s, loop)
s.on_recv(self._process_message)
s.connect(ENRICHER_ADDR)
try:
loop.start()
except KeyboardInterrupt:
loop.stop()
for ss in [s, pull_s, self.push_s]:
ss.close()
def __init__(self, local_domains=("localhost")):
self.local_domains = local_domains
self.loop = ioloop.IOLoop()
self.ctx = zmq.Context()
log.debug('Registering StanzaProcessor at forwarder..')
# connect push socket to forwarder
self.forwarder = self.ctx.socket(zmq.PUSH)
self.forwarder.connect(config.get('ipc', 'forwarder'))
pull_socket = self.ctx.socket(zmq.PULL)
stream = zmqstream.ZMQStream(pull_socket, self.loop)
stream.on_recv(self.handle_stanza, False)
port = pull_socket.bind_to_random_port('tcp://127.0.0.1')
# register connection at forwarder
reg_msg = ZMQForwarder_message('REGISTER')
reg_msg.attributes = (config.get('ipc', 'password'),
'tcp://127.0.0.1:' + str(port), local_domains)
self.forwarder.send_pyobj(reg_msg)
# init the handlers
self.stanza_handlers = {
'iq': iq.Iq(),
'message': message.Message(),
'presence': presence.Presence()
}
def __init__(self):
# State
self.running = False
self.terminated = False
self.ioloop = ioloop.IOLoop()
self.shutdown_handlers = []
self.exception_handlers = {}
def setUp(self):
self.fail_msg = None
self.io_loop = ioloop.IOLoop()
super(TestIOLoopAuthentication, self).setUp()
self.server = self.socket(zmq.PUSH)
self.client = self.socket(zmq.PULL)
self.pushstream = zmqstream.ZMQStream(self.server, self.io_loop)
self.pullstream = zmqstream.ZMQStream(self.client, self.io_loop)
def test_timeout_compare(self):
"""test timeout comparisons"""
loop = ioloop.IOLoop()
t = _Timeout(1, 2, loop)
t2 = _Timeout(1, 3, loop)
self.assertEqual(t < t2, id(t) < id(t2))
t2 = _Timeout(2,1, loop)
self.assertTrue(t < t2)
def _backend_init(self, io_loop=None):
self.reader = None
self.internal_loop = False
if io_loop is None:
self.internal_loop = True
self.io_loop = ioloop.IOLoop()
else:
self.io_loop = io_loop
def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, reg_addr, config=None,
logname='root', log_url=None, loglevel=logging.DEBUG,
identity=b'task', in_thread=False):
ZMQStream = zmqstream.ZMQStream
if config:
# unwrap dict back into Config
config = Config(config)
if in_thread:
# use instance() to get the same Context/Loop as our parent
ctx = zmq.Context.instance()
loop = ioloop.IOLoop.instance()
else:
# in a process, don't use instance()
# for safety with multiprocessing
ctx = zmq.Context()
loop = ioloop.IOLoop()
ins = ZMQStream(ctx.socket(zmq.ROUTER),loop)
util.set_hwm(ins, 0)
ins.setsockopt(zmq.IDENTITY, identity + b'_in')
ins.bind(in_addr)
outs = ZMQStream(ctx.socket(zmq.ROUTER),loop)
util.set_hwm(outs, 0)
outs.setsockopt(zmq.IDENTITY, identity + b'_out')
outs.bind(out_addr)
mons = zmqstream.ZMQStream(ctx.socket(zmq.PUB),loop)
util.set_hwm(mons, 0)
mons.connect(mon_addr)
nots = zmqstream.ZMQStream(ctx.socket(zmq.SUB),loop)
nots.setsockopt(zmq.SUBSCRIBE, b'')
nots.connect(not_addr)
querys = ZMQStream(ctx.socket(zmq.DEALER),loop)
querys.connect(reg_addr)
# setup logging.
if in_thread:
log = Application.instance().log
else:
if log_url:
log = connect_logger(logname, ctx, log_url, root="scheduler", loglevel=loglevel)
else:
log = local_logger(logname, loglevel)
scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
mon_stream=mons, notifier_stream=nots,
query_stream=querys,
loop=loop, log=log,
config=config)
scheduler.start()
if not in_thread:
try:
loop.start()
except KeyboardInterrupt:
scheduler.log.critical("Interrupted, exiting...")
def __init__(self, broker=DEFAULT_FRONTEND,
ping_delay=10., ping_retries=3,
params=None, timeout=DEFAULT_TIMEOUT_MOVF,
max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA):
logger.debug('Initializing the agent.')
self.debug = logger.isEnabledFor(logging.DEBUG)
self.params = params
self.pid = os.getpid()
self.agent_id = '%s-%s' % (get_hostname(), self.pid)
self.timeout = timeout
self.max_age = max_age
self.max_age_delta = max_age_delta
self.env = os.environ.copy()
self.running = False
self._workers = {}
self._max_id = defaultdict(int)
# Let's ask the broker its options
self.broker = broker
client = Client(self.broker)
# this will timeout in case the broker is unreachable
result = client.ping()
self.endpoints = result['endpoints']
# Setup the zmq sockets
self.loop = ioloop.IOLoop()
self.ctx = zmq.Context()
# backend socket - used to receive work from the broker
self._backend = self.ctx.socket(zmq.ROUTER)
self._backend.identity = self.agent_id
self._backend.connect(self.endpoints['backend'])
# register socket - used to register into the broker
self._reg = self.ctx.socket(zmq.PUSH)
self._reg.connect(self.endpoints['register'])
# hearbeat socket - used to check if the broker is alive
heartbeat = self.endpoints.get('heartbeat')
if heartbeat is not None:
logger.info("Hearbeat activated")
self.ping = Stethoscope(heartbeat, onbeatlost=self.lost,
delay=ping_delay, retries=ping_retries,
ctx=self.ctx, io_loop=self.loop,
onregister=self.register)
else:
self.ping = None
# Setup the zmq streams.
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._check = ioloop.PeriodicCallback(self._check_proc,
ping_delay * 1000,
io_loop=self.loop)
def launch_scheduler(in_addr,
out_addr,
mon_addr,
not_addr,
config=None,
logname='ZMQ',
log_addr=None,
loglevel=logging.DEBUG,
scheme='lru',
identity=b'task'):
from zmq.eventloop import ioloop
from zmq.eventloop.zmqstream import ZMQStream
if config:
# unwrap dict back into Config
config = Config(config)
ctx = zmq.Context()
loop = ioloop.IOLoop()
ins = ZMQStream(ctx.socket(zmq.XREP), loop)
ins.setsockopt(zmq.IDENTITY, identity)
ins.bind(in_addr)
outs = ZMQStream(ctx.socket(zmq.XREP), loop)
outs.setsockopt(zmq.IDENTITY, identity)
outs.bind(out_addr)
mons = ZMQStream(ctx.socket(zmq.PUB), loop)
mons.connect(mon_addr)
nots = ZMQStream(ctx.socket(zmq.SUB), loop)
nots.setsockopt(zmq.SUBSCRIBE, '')
nots.connect(not_addr)
scheme = globals().get(scheme, None)
# setup logging
if log_addr:
connect_logger(logname,
ctx,
log_addr,
root="scheduler",
loglevel=loglevel)
else:
local_logger(logname, loglevel)
scheduler = TaskScheduler(client_stream=ins,
engine_stream=outs,
mon_stream=mons,
notifier_stream=nots,
scheme=scheme,
loop=loop,
logname=logname,
config=config)
scheduler.start()
try:
loop.start()
except KeyboardInterrupt:
print("interrupted, exiting...", file=sys.__stderr__)
def test_execute(self):
loop = ioloop.IOLoop()
loop.start = mock.Mock()
runner = ExternalRunner({'hits': [2], 'users': [2]}, loop)
runner._prepare_filesystem = mock.Mock()
runner.spawn_external_runner = mock.Mock()
runner._execute()
self.assertTrue(runner._prepare_filesystem.called)
self.assertEquals(runner.spawn_external_runner.call_count, 4)
def run():
loop = ioloop.IOLoop()
context = zmq.Context()
pub = context.socket(zmq.PUB)
pub.bind('tcp://0.0.0.0:5555')
router = context.socket(zmq.ROUTER)
router.bind('tcp://0.0.0.0:15556')
outstream = zmqstream.ZMQStream(pub, loop)
instream = zmqstream.ZMQStream(router, loop)
hb = HeartBeater(loop, outstream, instream)
loop.start()
def heartbeat_run():
loop = ioloop.IOLoop()
context = zmq.Context()
pub = context.socket(zmq.PUB)
pub.bind('tcp://0.0.0.0:{}'.format(HEART_PUB_PORT))
router = context.socket(zmq.ROUTER)
router.bind('tcp://0.0.0.0:{}'.format(HEART_ROUTER_PORT))
outstream = zmqstream.ZMQStream(pub, loop)
instream = zmqstream.ZMQStream(router, loop)
hb = HeartBeater(loop, outstream, instream)
loop.start()
def __init__(self,
name,
frontend_point,
backend_point,
context=None,
loop=None):
self._context = context or zmq.Context.instance()
self._loop = loop or ioloop.IOLoop().instance()
self._name = name
self._frontend_point = frontend_point
self._backend_point = backend_point
def __init__(self, endpoint, pubsub_endpoint, check_delay, **config):
self.daemon = True
self.config = config
self.active = to_bool(config.get('active', True))
self.context = zmq.Context()
self.pubsub_endpoint = pubsub_endpoint
self.endpoint = endpoint
self.check_delay = check_delay
self.loop = ioloop.IOLoop()
self._id = uuid.uuid4().hex # XXX os.getpid()+thread id is enough...
self.running = False
def __init__(self, forwarder_url):
self.loop = ioloop.IOLoop()
self.ctx = zmq.Context()
# Create our PULL listener to listen to the world ;)
self.pull_sock = self.ctx.socket(zmq.PULL)
log.debug('Listening at ' + forwarder_url)
self.pull_sock.bind(forwarder_url)
self.stream = zmqstream.ZMQStream(self.pull_sock, self.loop)
self.stream.on_recv(self.handle_stanza, False)
self.peers = dict()
def __init__(self, context, endpoint, pubsub_endpoint, check_delay):
super(Flapping, self).__init__()
self.daemon = True
self.context = context
self.pubsub_endpoint = pubsub_endpoint
self.endpoint = endpoint
self.check_delay = check_delay
self.loop = ioloop.IOLoop()
self._id = uuid.uuid4().hex
self.timelines = {}
self.timers = {}
self.configs = {}
self.tries = {}
def setUp(self):
if tornado is None:
pytest.skip()
if asyncio:
asyncio.set_event_loop(asyncio.new_event_loop())
self.context = zmq.Context()
self.loop = ioloop.IOLoop()
self.loop.make_current()
self.push = zmqstream.ZMQStream(self.context.socket(zmq.PUSH))
self.pull = zmqstream.ZMQStream(self.context.socket(zmq.PULL))
port = self.push.bind_to_random_port('tcp://127.0.0.1')
self.pull.connect('tcp://127.0.0.1:%i' % port)
self.stream = self.push
def __init__(self,
watchers,
endpoint,
pubsub_endpoint,
check_delay=1.,
prereload_fn=None,
context=None,
loop=None,
stats_endpoint=None,
plugins=None,
sockets=None):
self.watchers = watchers
self.endpoint = endpoint
self.check_delay = check_delay
self.prereload_fn = prereload_fn
self.pubsub_endpoint = pubsub_endpoint
# initialize zmq context
self.context = context or zmq.Context.instance()
self.loop = loop or ioloop.IOLoop()
self.ctrl = Controller(endpoint, self.context, self.loop, self,
check_delay)
self.pid = os.getpid()
self._watchers_names = {}
self.alive = True
self._lock = RLock()
# initializing circusd-stats as a watcher when configured
self.stats_endpoint = stats_endpoint
if self.stats_endpoint is not None:
cmd = "%s -c 'from circus import stats; stats.main()'" % \
sys.executable
cmd += ' --endpoint %s' % self.endpoint
cmd += ' --pubsub %s' % self.pubsub_endpoint
cmd += ' --statspoint %s' % self.stats_endpoint
stats_watcher = Watcher('circusd-stats', cmd)
self.watchers.append(stats_watcher)
# adding each plugin as a watcher
if plugins is not None:
for plugin in plugins:
fqnd = plugin['use']
name = 'plugin:%s' % fqnd.replace('.', '-')
cmd = get_plugin_cmd(plugin, self.endpoint,
self.pubsub_endpoint, self.check_delay)
plugin_watcher = Watcher(name, cmd, priority=1, singleton=True)
self.watchers.append(plugin_watcher)
self.sockets = CircusSockets(sockets)
def instance_runner(self):
loop = ioloop.IOLoop()
app = server.Application(port=self.port)
http_server = tornado.httpserver.HTTPServer(app,
max_buffer_size=1024**3)
http_server.listen(self.port)
logging.info('Started instance')
print('\nSearch "<server-id>:'+str(self.port)+'" in browser for a '\
'training visualization.')
print('If running locally, <server-id> = localhost.\n')
loop.start()
http_server.stop()
logging.info('Ended instance')
def test_simple(self):
"""simple IOLoop creation test"""
loop = ioloop.IOLoop()
dc = ioloop.DelayedCallback(loop.stop, 200, loop)
pc = ioloop.DelayedCallback(lambda : None, 10, loop)
pc.start()
dc.start()
t = Delay(loop.stop,1)
t.start()
loop.start()
if t.isAlive():
t.abort()
else:
self.assert_(False, "IOLoop failed to exit")
def __init__(self, *args, **kwargs):
self.loop = ioloop.IOLoop().instance()
data_dir = config.home + '/run/'
self.open_storage(data_dir + 'bot.db')
self.prefix = 'Tom'
self.all_candidates = None
super(TomBot, self).__init__(*args, **kwargs)
# def update_dynamic_plugins(self):
# all_candidates, errors = update_plugin_places([self.plugin_dir + os.sep + d for d in self.get(REPOS, {}).keys()])
# self.all_candidates = all_candidates
# return errors
self.update_dynamic_plugins()
self.inject_commands_from(self)
def __init__(self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND,
heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
worker_timeout=DEFAULT_TIMEOUT_MOVF):
# before doing anything, we verify if a broker is already up and
# running
logger.debug('Verifying if there is a running broker')
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug('Ooops, we have a running broker on that socket')
raise DuplicateBrokerError(pid)
logger.debug('Initializing the broker.')
for endpoint in (frontend, backend, heartbeat):
if endpoint.startswith('ipc'):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the three sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = 'broker-' + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
# heartbeat
self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context)
# status
self.started = False
self.poll_timeout = None
# workers registration and timers
self._workers = []
self._worker_times = {}
self.worker_timeout = worker_timeout
def run(self):
"""Run my loop, ignoring EINTR events in the poller"""
loop = ioloop.IOLoop(make_current=True)
self.client = IOLoopKernelClient(self.connection_info,
manager=self.manager)
self.client.ioloop.add_callback(self.started.set)
self._ready_fut = self.client.wait_for_ready()
loop.add_future(self._ready_fut,
lambda future: self.kernel_responding.set())
try:
self._run_loop()
finally:
self.client.close()
self.client.ioloop.close()
self.client = None
def test_simple(self):
"""simple IOLoop creation test"""
loop = ioloop.IOLoop()
loop.make_current()
dc = ioloop.PeriodicCallback(loop.stop, 200)
pc = ioloop.PeriodicCallback(lambda: None, 10)
pc.start()
dc.start()
t = Delay(loop.stop, 1)
t.start()
loop.start()
if t.isAlive():
t.abort()
else:
self.fail("IOLoop failed to exit")
