def _setup(self):
"""Setup all the attributes.
Bind the server, connect to remote peers and initiate the timers.
"""
self._zmq_context = zmq.Context()
self._socket_for_commands = self._zmq_context.socket(zmq.ROUTER)
self._socket_for_consensus = self._zmq_context.socket(zmq.ROUTER)
for remote_endpoint in self._remote_endpoints:
remote_peer = peer.Peer(self._zmq_context,
self._private_endpoint,
remote_endpoint)
peer_id = six.b(remote_endpoint)
self._remote_peers[peer_id] = remote_peer
self._server_state.init_indexes(peer_id)
self._zmq_ioloop = ioloop.ZMQIOLoop().instance()
self._zmq_ioloop.add_handler(self._socket_for_commands,
self._process_command_message,
zmq.POLLIN)
self._zmq_ioloop.add_handler(self._socket_for_consensus,
self._process_internal_message,
zmq.POLLIN)
self.checking_leader_timeout = ioloop.PeriodicCallback(
self._election_timeout_task,
random.randint(self._min_election_timeout,
self._max_election_timeout),
io_loop=self._zmq_ioloop)
self.heartbeating = ioloop.PeriodicCallback(
self.broadcast_append_entries,
self._leader_heartbeat_interval,
io_loop=self._zmq_ioloop)
def __init__(self):
LoggerMixin.configure()
db.configure()
Compute.configure()
APIContainer.configure()
JBoxAsyncJob.configure()
JBoxAsyncJob.init(JBoxAsyncJob.MODE_PUB)
self.application = tornado.web.Application(
handlers=[(r"^/", APIInfoHandler), (r"^/.*/.*", APIHandler)])
self.application.settings["cookie_secret"] = JBoxCfg.get('sesskey')
self.application.listen(JBoxCfg.get('api.manager_port'),
address=socket.gethostname())
self.application.listen(JBoxCfg.get('api.manager_port'),
address='localhost')
self.ioloop = ioloop.IOLoop.instance()
# run container maintainence every 5 minutes
run_interval = 5 * 60 * 1000
self.log_info("Container maintenance every " +
str(run_interval / (60 * 1000)) + " minutes")
self.ct = ioloop.PeriodicCallback(JBoxAPI.do_housekeeping,
run_interval, self.ioloop)
self.sigct = ioloop.PeriodicCallback(JBoxAPI.do_signals, 1000,
self.ioloop)
def initialize(self):
"""Initialize method called at plugin start.
The method register the periodic callback of the ping() method
and fill the monitored_watchers dict.
"""
super(CircusAutorestartPlugin, self).initialize()
self.fill_watchers(debug_output=True)
self.periodic = ioloop.PeriodicCallback(self.ping, 1000, self.loop)
self.periodic.start()
self.periodic10 = ioloop.PeriodicCallback(self.fill_watchers, 10000,
self.loop)
self.periodic10.start()
def initialize(self):
"""Initialize method called at plugin start.
The method register the periodic callback of the ping() method
and fill the watchers set.
"""
super(CircusAutostart, self).initialize()
self.periodic = ioloop.PeriodicCallback(self.ping, 1000, self.loop)
self.periodic5 = ioloop.PeriodicCallback(self.ping_every5s, 5000,
self.loop)
self.periodic.start()
self.periodic5.start()
self.fill_watchers()
def test_simple(self):
"""simple IOLoop creation test"""
loop = ioloop.IOLoop()
dc = ioloop.PeriodicCallback(loop.stop, 200, loop)
pc = ioloop.PeriodicCallback(lambda : None, 10, loop)
pc.start()
dc.start()
t = Delay(loop.stop,1)
t.start()
loop.start()
if t.isAlive():
t.abort()
else:
self.fail("IOLoop failed to exit")
def setup(self, pop_server_address, broker_server_address):
super().setup()
pop_subscription = self.context.socket(zmq.SUB)
pop_subscription.connect("ipc://{}".format(pop_server_address))
logger.info('Connected to population subscription: {}'.format(
pop_server_address))
pop_subscription.setsockopt(zmq.SUBSCRIBE, b"")
self.population_subscription = zmqstream.ZMQStream(
pop_subscription, self.loop)
self.population_subscription.on_recv(
PropagatorMessageHandler(None, self.propagator))
# Connection to broker
broker_connection = self.context.socket(zmq.REQ)
broker_connection.connect("ipc://{}".format(broker_server_address))
logger.info('Connected to broker: {}'.format(broker_server_address))
self.broker_connection = zmqstream.ZMQStream(broker_connection,
self.loop)
self.broker_connection.on_recv(
PropagatorBrokerMessageHandler(self.broker_connection,
self.propagator))
# Setup tick callback
self.ticker = ioloop.PeriodicCallback(self.tick, 1000, self.loop)
def start(self):
""" Start the Notebook server app, after initialization
This method takes no arguments so all configuration and initialization
must be done prior to calling this method."""
if not self.allow_root:
# check if we are running as root, and abort if it's not allowed
try:
uid = os.geteuid()
except AttributeError:
uid = -1 # anything nonzero here, since we can't check UID assume non-root
if uid == 0:
self.log.critical("Running as root is not recommended. Use --allow-root to bypass.")
self.exit(1)
super(NotebookApp, self).start()
info = self.log.info
for line in self.notebook_info().split("\n"):
info(line)
info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).")
self.write_server_info_file()
if self.open_browser or self.file_to_run:
try:
browser = webbrowser.get(self.browser or None)
except webbrowser.Error as e:
self.log.warning('No web browser found: %s.' % e)
browser = None
if self.file_to_run:
if not os.path.exists(self.file_to_run):
self.log.critical("%s does not exist" % self.file_to_run)
self.exit(1)
relpath = os.path.relpath(self.file_to_run, self.notebook_dir)
uri = url_escape(url_path_join('notebooks', *relpath.split(os.sep)))
else:
# default_url contains base_url, but so does connection_url
uri = self.default_url[len(self.base_url):]
if browser:
b = lambda : browser.open(url_path_join(self.connection_url, uri),
new=2)
threading.Thread(target=b).start()
self.io_loop = ioloop.IOLoop.current()
if sys.platform.startswith('win'):
# add no-op to wake every 5s
# to handle signals that may be ignored by the inner loop
pc = ioloop.PeriodicCallback(lambda : None, 5000)
pc.start()
try:
self.io_loop.start()
except KeyboardInterrupt:
info("Interrupted...")
finally:
self.cleanup_kernels()
self.remove_server_info_file()
def __init__(self,
socket_type,
service_name,
service_port=None,
service_type=None):
self.context = zmq.Context()
self.socket = self.context.socket(socket_type)
if not service_port:
service_port = self.socket.bind_to_random_port('tcp://*',
min_port=49152,
max_port=65535,
max_tries=100)
else:
self.socket.bind("tcp://*:%d" % service_port)
print "Bound to port %d" % service_port
self.stream = ZMQStream(self.socket)
if not service_type:
service_type = socket_type_to_service(socket_type)
if socket_type == zmq.PUB:
# TODO: how to handle this with ROUTER/DEALER combinations...
self.heartbeat_timer = ioloop.PeriodicCallback(
self._hearbeat, 1000)
self.heartbeat_timer.start()
if socket_type == zmq.ROUTER:
self.stream.on_recv(self._method_callback_wrapper)
bonjour_utilities.register_ioloop(ioloop.IOLoop.instance(),
service_type, service_name,
service_port)
def __init__(self, args=None, loop=None):
if args is None:
args = {}
super(ExternalRunner, self).__init__(args)
self._current_step = 0
self._step_started_at = None
self._duration = self.args.get('duration')
self._timeout = args.get('external_process_timeout', 2)
self._processes = []
self._processes_pending_cleanup = []
# hits and users are lists that can be None.
hits, users = [1], [1]
if self.args.get('hits') is not None:
hits = self.args['hits']
if self.args.get('users') is not None:
users = self.args['users']
self.args['hits'] = hits
self.args['users'] = users
self._nb_steps = max(len(hits), len(users))
self._loop = loop or ioloop.IOLoop()
# Check the status of the processes every so-often.(500ms)
cb = ioloop.PeriodicCallback(self._check_processes, 500, self._loop)
cb.start()
self._receiver_socket = (self.args.get('zmq_receiver')
or DEFAULT_EXTERNAL_RUNNER_RECEIVER)
def __init__(self, io_loop=None):
self.io_loop = io_loop or ioloop.IOLoop.instance()
self._sockets = {} # fd -> socket object
self._started = False
self._connections = {}
self.checker = ioloop.PeriodicCallback(
self.check_for_closed_connections, 30000)
def __init__(self, **kwargs):
super(SQLiteDB, self).__init__(**kwargs)
if sqlite3 is None:
raise ImportError("SQLiteDB requires sqlite3")
if not self.table:
# use session, and prefix _, since starting with # is illegal
self.table = '_' + self.session.replace('-', '_')
if not self.location:
# get current profile
from IPython.core.application import BaseIPythonApplication
if BaseIPythonApplication.initialized():
app = BaseIPythonApplication.instance()
if app.profile_dir is not None:
self.location = app.profile_dir.location
else:
self.location = u'.'
else:
self.location = u'.'
self._init_db()
# register db commit as 2s periodic callback
# to prevent clogging pipes
# assumes we are being run in a zmq ioloop app
loop = ioloop.IOLoop.instance()
pc = ioloop.PeriodicCallback(self._db.commit, 2000, loop)
pc.start()
def main():
import optparse
parser = optparse.OptionParser('usage: %prog')
ZmqPublisher.addOptions(parser, 'zmqPublish')
parser.add_option(
'-p',
'--prefix',
default='',
help=
'Prefix to prepend to incoming lines before publishing them (DO include trailing "." if you want it)'
)
opts, args = parser.parse_args()
if args:
parser.error('expected no args')
logging.basicConfig(level=logging.DEBUG)
# set up networking
p = ZmqPublisher(**ZmqPublisher.getOptionValues(opts))
p.start()
# start publishing an arbitrary message that central should forward
pubTimer = ioloop.PeriodicCallback(lambda: pubMessage(opts.prefix, p), 0.1)
pubTimer.start()
zmqLoop()
def start(self):
super(FormgradeApp, self).start()
if self.logfile:
self.init_logging(logging.FileHandler, [self.logfile], color=False)
else:
self.init_logging()
self.init_tornado_settings()
self.init_handlers()
self.init_tornado_application()
# Create the application
self.io_loop = ioloop.IOLoop.current()
self.tornado_application.listen(self.port, address=self.ip)
url = "http://{:s}:{:d}/".format(self.ip, self.port)
self.log.info("Form grader running at {}".format(url))
self.log.info("Use Control-C to stop this server")
if sys.platform.startswith('win'):
# add no-op to wake every 1s
# to handle signals that may be ignored by the inner loop
pc = ioloop.PeriodicCallback(lambda: None, 1000)
pc.start()
# Start the loop
self.io_loop.start()
def start(self):
if self.state == 'before':
self.process = Popen(self.args,
stdout=PIPE,
stderr=PIPE,
stdin=PIPE,
env=os.environ,
cwd=self.work_dir)
if WINDOWS:
self.stdout = forward_read_events(self.process.stdout)
self.stderr = forward_read_events(self.process.stderr)
else:
self.stdout = self.process.stdout.fileno()
self.stderr = self.process.stderr.fileno()
self.loop.add_handler(self.stdout, self.handle_stdout,
self.loop.READ)
self.loop.add_handler(self.stderr, self.handle_stderr,
self.loop.READ)
self.poller = ioloop.PeriodicCallback(self.poll,
self.poll_frequency,
self.loop)
self.poller.start()
self.notify_start(self.process.pid)
else:
s = 'The process was already started and has state: %r' % self.state
raise ProcessStateError(s)
def start_hb(self, callback):
"""Start the heartbeating and call the callback if the kernel dies."""
if not self._beating:
self._kernel_alive = True
def ping_or_dead():
self.hb_stream.flush()
if self._kernel_alive:
self._kernel_alive = False
self.hb_stream.send(b'ping')
# flush stream to force immediate socket send
self.hb_stream.flush()
else:
try:
callback()
except:
pass
finally:
self.stop_hb()
def beat_received(msg):
self._kernel_alive = True
self.hb_stream.on_recv(beat_received)
loop = ioloop.IOLoop.instance()
self._hb_periodic_callback = ioloop.PeriodicCallback(
ping_or_dead, self.time_to_dead * 1000, loop)
loop.add_timeout(time.time() + self.first_beat,
self._really_start_hb)
self._beating = True
def init_dynamic_configs(self) -> None:
"""
Initialize the set of configurables that should participate in dynamic updates. We should
also log that we're performing dynamic configuration updates, along with the list of CLI
options - that are not privy to dynamic updates.
:return:
"""
if self.dynamic_config_interval > 0:
self.add_dynamic_configurable("EnterpriseGatewayApp", self)
self.add_dynamic_configurable("MappingKernelManager",
self.kernel_manager)
self.add_dynamic_configurable("KernelSpecManager",
self.kernel_spec_manager)
self.add_dynamic_configurable("KernelSessionManager",
self.kernel_session_manager)
self.log.info(
"Dynamic updates have been configured. Checking every {} seconds."
.format(self.dynamic_config_interval))
self.log.info(
"The following configuration options will not be subject to dynamic updates "
"(configured via CLI):")
for config, options in self.cli_config.items():
for option, value in options.items():
self.log.info(f" '{config}.{option}': '{value}'")
if self.dynamic_config_poller is None:
self.dynamic_config_poller = ioloop.PeriodicCallback(
self.update_dynamic_configurables,
self.dynamic_config_interval * 1000)
self.dynamic_config_poller.start()
def start_hb(self, callback):
"""Start the heartbeating and call the callback if the kernel dies."""
if not self._beating:
self._kernel_alive = True
def ping_or_dead():
if self._kernel_alive:
self._kernel_alive = False
self.hb_stream.send(b'ping')
else:
try:
callback()
except:
pass
finally:
self._hb_periodic_callback.stop()
def beat_received(msg):
self._kernel_alive = True
self.hb_stream.on_recv(beat_received)
self._hb_periodic_callback = ioloop.PeriodicCallback(
ping_or_dead, self.time_to_dead * 1000)
self._hb_periodic_callback.start()
self._beating = True
def __init__(self, loop, pingstream, pongstream, config):
self.pingstream = pingstream
self.pongstream = pongstream
self.pongstream.on_recv(self.handle_input)
self.hearts = set()
self.responses = set()
self.lifetime = 0
self.tic = time.time()
self.logger = config.get_logger(__name__)
self.checkin_count = config.get_checkin_count()
self.caller = ioloop.PeriodicCallback(
self.beat,
config.get_ping_interval() * 1000, loop)
self.action_poll_interval = config.get_action_poll_interval()
self.last_action_poll = time.time()
self.smdb = smdb.SMDB()
self.changed_state = []
self.logger.info("Starting OSAD server.")
self.caller.start()
def start(self):
"""Start the polling of the kernel."""
if self._pcallback is None:
self._pcallback = ioloop.PeriodicCallback(
self.poll, 1000*self.time_to_dead,
)
self._pcallback.start()
def start(self, delay_start=0):
"""Start the client loop."""
#
# Create a new instance of the ioloop.
# This is important for example when running from an ipython notebook (which
# is itself an ioloop.)
#
ioloop.IOLoop.clear_current()
ioloop.IOLoop.clear_instance(
) # or del IOLoop._instance in tornado < 3.3
#
# Set the (Tornado) loop
#
self.loop = ioloop.IOLoop().instance()
#
# Start the MDP client.
#
super(Client, self).start()
#
# Start the ioloop.
#
time.sleep(delay_start)
ioloop.PeriodicCallback(
partial(self.send_mmi, service=MDP.MMI_SERVICES, msg=[]),
self.mmi_period, self.loop).start()
self.loop.start()
def __init__(self, streamer, name, callback_time=1., io_loop=None):
super(SocketStatsCollector, self).__init__(streamer, name,
callback_time, io_loop)
self._rstats = defaultdict(int)
self.sockets = [sock for sock, address, fd in self.streamer.sockets]
self._p = ioloop.PeriodicCallback(self._select,
_LOOP_RES,
io_loop=io_loop)
def run(self, func):
context = zmq.Context()
self.socket = context.socket(zmq.PAIR)
self.socket.set_hwm(self.hwm)
self.socket.bind(self.address)
self.receive_timer = ioloop.PeriodicCallback(func, self.interval)
self._start_timer()
def __init__(self, *args, **config):
super(ApprcWatcher, self).__init__(*args, **config)
self.loop_rate = config.get("loop_rate", 3) # in seconds
self.apprc = config.get("apprc", "/home/application/apprc")
self.port = config.get("port", "8888")
self.period = ioloop.PeriodicCallback(
FileWatcher(self.apprc, self.reload_env), self.loop_rate * 1000,
self.loop)
def __init__(self, broker=DEFAULT_FRONTEND,
ping_delay=10., ping_retries=3,
params=None, timeout=DEFAULT_TIMEOUT_MOVF,
max_age=DEFAULT_MAX_AGE, max_age_delta=DEFAULT_MAX_AGE_DELTA):
logger.debug('Initializing the agent.')
self.debug = logger.isEnabledFor(logging.DEBUG)
self.params = params
self.pid = os.getpid()
self.agent_id = '%s-%s' % (get_hostname(), self.pid)
self.timeout = timeout
self.max_age = max_age
self.max_age_delta = max_age_delta
self.env = os.environ.copy()
self.running = False
self._workers = {}
self._max_id = defaultdict(int)
# Let's ask the broker its options
self.broker = broker
client = Client(self.broker)
# this will timeout in case the broker is unreachable
result = client.ping()
self.endpoints = result['endpoints']
# Setup the zmq sockets
self.loop = ioloop.IOLoop()
self.ctx = zmq.Context()
# backend socket - used to receive work from the broker
self._backend = self.ctx.socket(zmq.ROUTER)
self._backend.identity = self.agent_id
self._backend.connect(self.endpoints['backend'])
# register socket - used to register into the broker
self._reg = self.ctx.socket(zmq.PUSH)
self._reg.connect(self.endpoints['register'])
# hearbeat socket - used to check if the broker is alive
heartbeat = self.endpoints.get('heartbeat')
if heartbeat is not None:
logger.info("Hearbeat activated")
self.ping = Stethoscope(heartbeat, onbeatlost=self.lost,
delay=ping_delay, retries=ping_retries,
ctx=self.ctx, io_loop=self.loop,
onregister=self.register)
else:
self.ping = None
# Setup the zmq streams.
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._check = ioloop.PeriodicCallback(self._check_proc,
ping_delay * 1000,
io_loop=self.loop)
def __init__(self):
super(mcp, self).__init__(METHODS_SERVICE_NAME, service_port=METHODS_PORT)
# This is low-level ZMQStream, to be used only in special cases
self.signals_stream = zmqdecorators.server_tracker.get_by_name(SIGNALS_SERVICE_NAME, zmq.PUB).stream
# periodically reap dead workers from our registry
self.reaper_pcb = ioloop_mod.PeriodicCallback(self.reap_dead_workers, 1000)
self.reaper_pcb.start()
def registerAlarm(self, interval, callback, UUID=None):
if UUID is None:
UUID = uuid4()
if UUID in self.alarms:
raise KeyError("Alarm UUID already exist")
self.alarms[UUID] = ioloop.PeriodicCallback(callback, interval)
self.alarms[UUID].start()
return UUID
def start(self):
self.initialize()
if self.check_delay > 0:
# The specific case (check_delay < 0)
# so with no period callback to manage_watchers
# is probably "unit tests only"
self.caller = ioloop.PeriodicCallback(self.manage_watchers,
self.check_delay, self.loop)
self.caller.start()
self.started = True
def start(self):
self.engine_stream.on_recv(self.dispatch_result, copy=False)
self._notification_handlers = dict(
registration_notification=self._register_engine,
unregistration_notification=self._unregister_engine)
self.notifier_stream.on_recv(self.dispatch_notification)
self.auditor = ioloop.PeriodicCallback(self.audit_timeouts, 2e3,
self.loop) # 1 Hz
self.auditor.start()
self.log.info("Scheduler started...%r" % self)
def handle_init(self):
"""Initialization of plugin
- set the periodic call back for the process monitoring (at loop_rate)
- create the listening UDP socket
"""
self.period = ioloop.PeriodicCallback(self.look_after,
self.loop_rate * 1000, self.loop)
self.period.start()
self._bind_socket()
def start(self):
""" Start the Notebook server app, after initialization
This method takes no arguments so all configuration and initialization
must be done prior to calling this method."""
super(NotebookApp, self).start()
info = self.log.info
for line in self.notebook_info().split("\n"):
info(line)
info(
"Use Control-C to stop this server and shut down all kernels (twice to skip confirmation)."
)
self.write_server_info_file()
if self.open_browser or self.file_to_run:
try:
browser = webbrowser.get(self.browser or None)
except webbrowser.Error as e:
self.log.warn('No web browser found: %s.' % e)
browser = None
if self.file_to_run:
if not os.path.exists(self.file_to_run):
self.log.critical("%s does not exist" % self.file_to_run)
self.exit(1)
relpath = os.path.relpath(self.file_to_run, self.notebook_dir)
uri = url_escape(
url_path_join('notebooks', *relpath.split(os.sep)))
else:
# default_url contains base_url, but so does connection_url
uri = self.default_url[len(self.base_url):]
if self.one_time_token:
uri = url_concat(uri, {'token': self.one_time_token})
if browser:
b = lambda: browser.open(
url_path_join(self.connection_url, uri), new=2)
threading.Thread(target=b).start()
self.io_loop = ioloop.IOLoop.current()
if sys.platform.startswith('win'):
# add no-op to wake every 5s
# to handle signals that may be ignored by the inner loop
pc = ioloop.PeriodicCallback(lambda: None, 5000)
pc.start()
try:
self.io_loop.start()
except KeyboardInterrupt:
info("Interrupted...")
finally:
self.remove_server_info_file()
self.cleanup_kernels()
