def run_worker(q, ip, center_ip, center_port, ncores, nanny_port,
worker_port, local_dir, services, name):
""" Function run by the Nanny when creating the worker """
from distributed import Worker # pragma: no cover
from tornado.ioloop import IOLoop # pragma: no cover
IOLoop.clear_instance() # pragma: no cover
loop = IOLoop() # pragma: no cover
loop.make_current() # pragma: no cover
worker = Worker(center_ip, center_port, ncores=ncores, ip=ip,
service_ports={'nanny': nanny_port}, local_dir=local_dir,
services=services, name=name) # pragma: no cover
@gen.coroutine # pragma: no cover
def start():
try: # pragma: no cover
yield worker._start(worker_port) # pragma: no cover
except Exception as e: # pragma: no cover
logger.exception(e) # pragma: no cover
q.put(e) # pragma: no cover
else:
assert worker.port # pragma: no cover
q.put({'port': worker.port, 'dir': worker.local_dir}) # pragma: no cover
loop.add_callback(start) # pragma: no cover
with ignoring(KeyboardInterrupt):
loop.start() # pragma: no cover
def run_worker(q, ip, center_ip, center_port, ncores, nanny_port,
local_dir, services):
""" Function run by the Nanny when creating the worker """
from distributed import Worker
from tornado.ioloop import IOLoop
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
worker = Worker(center_ip, center_port, ncores=ncores, ip=ip,
service_ports={'nanny': nanny_port}, local_dir=local_dir,
services=services)
@gen.coroutine
def start():
try:
yield worker._start()
except Exception as e:
logger.exception(e)
q.put(e)
else:
assert worker.port
q.put({'port': worker.port, 'dir': worker.local_dir})
loop.add_callback(start)
loop.start()
def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port,
worker_port, local_dir, services, name, memory_limit):
""" Function run by the Nanny when creating the worker """
from distributed import Worker # pragma: no cover
from tornado.ioloop import IOLoop # pragma: no cover
IOLoop.clear_instance() # pragma: no cover
loop = IOLoop() # pragma: no cover
loop.make_current() # pragma: no cover
worker = Worker(scheduler_ip, scheduler_port, ncores=ncores, ip=ip,
service_ports={'nanny': nanny_port}, local_dir=local_dir,
services=services, name=name, memory_limit=memory_limit,
loop=loop) # pragma: no cover
@gen.coroutine # pragma: no cover
def start():
try: # pragma: no cover
yield worker._start(worker_port) # pragma: no cover
except Exception as e: # pragma: no cover
logger.exception(e) # pragma: no cover
q.put(e) # pragma: no cover
else:
assert worker.port # pragma: no cover
q.put({'port': worker.port, 'dir': worker.local_dir}) # pragma: no cover
loop.add_callback(start) # pragma: no cover
try:
loop.start() # pragma: no cover
finally:
loop.stop()
loop.close(all_fds=True)
def loop():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
yield loop
loop.stop()
loop.close()
def pristine_loop():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
try:
yield loop
finally:
loop.close(all_fds=True)
IOLoop.clear_instance()
def test_sync_closed_loop():
loop = IOLoop.current()
loop.close()
IOLoop.clear_current()
IOLoop.clear_instance()
with pytest.raises(RuntimeError) as exc_info:
sync(loop, inc, 1)
exc_info.match("IOLoop is clos(ed|ing)")
def loop():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
yield loop
sync(loop, loop.stop)
for i in range(5):
with ignoring(Exception):
loop.close(all_fds=True)
break
def run_nanny(port, center_port, **kwargs):
from distributed import Nanny
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Nanny('127.0.0.1', port, port + 1000, '127.0.0.1', center_port, **kwargs)
loop.run_sync(worker._start)
loop.start()
def run_center(port):
from distributed import Center
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = Center('127.0.0.1', port)
center.listen(port)
loop.start()
def test_func():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
cor = gen.coroutine(func)
try:
loop.run_sync(cor, timeout=timeout)
finally:
loop.stop()
loop.close(all_fds=True)
def test_func():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
s, workers = loop.run_sync(lambda: start_cluster(ncores))
try:
loop.run_sync(lambda: cor(s, *workers), timeout=timeout)
finally:
loop.run_sync(lambda: end_cluster(s, workers))
loop.stop()
loop.close()
def run_worker(q, center_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
with log_errors():
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Worker('127.0.0.1', center_port, ip='127.0.0.1', **kwargs)
loop.run_sync(lambda: worker._start(0))
q.put(worker.port)
loop.start()
def run_worker(port, center_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Worker("127.0.0.1", port, "127.0.0.1", center_port, **kwargs)
worker.start()
loop.start()
def run_worker(q, scheduler_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop, PeriodicCallback
with log_errors():
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
worker = Worker('127.0.0.1', scheduler_port, ip='127.0.0.1',
loop=loop, validate=True, **kwargs)
loop.run_sync(lambda: worker._start(0))
q.put(worker.port)
try:
loop.start()
finally:
loop.close(all_fds=True)
def run_scheduler(q, scheduler_port=0, **kwargs):
from distributed import Scheduler
from tornado.ioloop import IOLoop, PeriodicCallback
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
scheduler = Scheduler(loop=loop, validate=True, **kwargs)
done = scheduler.start(scheduler_port)
q.put(scheduler.port)
try:
loop.start()
finally:
loop.close(all_fds=True)
def current_loop():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
yield loop
if loop._running:
sync(loop, loop.stop)
for i in range(5):
try:
loop.close(all_fds=True)
return
except Exception as e:
f = e
print(f)
IOLoop.clear_instance()
def pristine_loop():
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
assert IOLoop.current() is loop
try:
yield loop
finally:
try:
loop.close(all_fds=True)
except (KeyError, ValueError):
pass
IOLoop.clear_instance()
IOLoop.clear_current()
def run_worker(q, ip, port, center_ip, center_port, ncores, nanny_port):
""" Function run by the Nanny when creating the worker """
from distributed import Worker
from tornado.ioloop import IOLoop
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
worker = Worker(ip, port, center_ip, center_port, ncores,
nanny_port=nanny_port)
@gen.coroutine
def start():
yield worker._start()
q.put(worker.port)
loop.add_callback(start)
loop.start()
def run_nanny(q, center_port, **kwargs):
from distributed import Nanny
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
with log_errors():
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Nanny('127.0.0.1', center_port, ip='127.0.0.1', loop=loop, **kwargs)
loop.run_sync(lambda: worker._start(0))
q.put(worker.port)
try:
loop.start()
finally:
loop.run_sync(worker._close)
loop.close(all_fds=True)
def run_scheduler(q, scheduler_port=0, center_port=None, **kwargs):
from distributed import Scheduler
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = ('127.0.0.1', center_port) if center_port else None
scheduler = Scheduler(center=center, loop=loop, validate=True, **kwargs)
done = scheduler.start(scheduler_port)
q.put(scheduler.port)
try:
loop.start()
finally:
loop.close(all_fds=True)
def run_scheduler(q, center_port=None, **kwargs):
from distributed import Scheduler
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = ('127.0.0.1', center_port) if center_port else None
scheduler = Scheduler(center=center, **kwargs)
scheduler.listen(0)
if center_port:
loop.run_sync(scheduler.sync_center)
done = scheduler.start(0)
q.put(scheduler.port)
loop.start()
def run_center(q):
from distributed import Center
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = Center('127.0.0.1')
while True:
try:
center.listen(0)
break
except Exception as e:
logging.info("Could not start center on port. Retrying",
exc_info=True)
q.put(center.port)
loop.start()
def start(self):
"""Start the whole thing"""
# start the proxy
self.start_proxy()
# start the webserver
http_server = tornado.httpserver.HTTPServer(self.tornado_application)
http_server.listen(self.hub_port)
loop = IOLoop.instance()
try:
loop.start()
except KeyboardInterrupt:
print("\nInterrupted")
finally:
# have to install/start a new IOLoop briefly,
# to allow for async cleanup code.
IOLoop.clear_instance()
cleanup_loop = IOLoop.instance()
cleanup_loop.add_callback(self.cleanup)
cleanup_loop.start()
def test_func():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
s, workers = loop.run_sync(lambda: start_cluster(ncores,
Worker=Worker))
args = [s] + workers
if executor:
e = Executor((s.ip, s.port), loop=loop, start=False)
loop.run_sync(e._start)
args = [e] + args
try:
loop.run_sync(lambda: cor(*args), timeout=timeout)
finally:
if executor:
loop.run_sync(e._shutdown)
loop.run_sync(lambda: end_cluster(s, workers))
loop.stop()
loop.close(all_fds=True)
def _start_ipython(self):
from IPython import get_ipython
if get_ipython() is not None:
raise RuntimeError("Cannot start IPython, it's already running.")
from zmq.eventloop.ioloop import ZMQIOLoop
from ipykernel.kernelapp import IPKernelApp
# save the global IOLoop instance
# since IPython relies on it, but we are going to put it in a thread.
save_inst = IOLoop.instance()
IOLoop.clear_instance()
zmq_loop = ZMQIOLoop()
zmq_loop.install()
# start IPython, disabling its signal handlers that won't work due to running in a thread:
app = self._ipython_kernel = IPKernelApp.instance(log=logger)
# Don't connect to the history database
app.config.HistoryManager.hist_file = ':memory:'
# listen on all interfaces, so remote clients can connect:
app.ip = self.ip
app.init_signal = lambda : None
app.initialize([])
app.kernel.pre_handler_hook = lambda : None
app.kernel.post_handler_hook = lambda : None
app.kernel.start()
# save self in the IPython namespace as 'worker'
app.kernel.shell.user_ns['worker'] = self
# start IPython's IOLoop in a thread
zmq_loop_thread = Thread(target=zmq_loop.start)
zmq_loop_thread.start()
# put the global IOLoop instance back:
IOLoop.clear_instance()
save_inst.install()
return app
def run_in_pool(_pid, _f, _has_context, context, *args, **kwargs):
# globals from the parent process in the
# IOLoop so clear them.
subprocess = os.getpid() != _pid
if subprocess and IOLoop.current(False):
LOGGER.debug("clearing tornado globals")
IOLoop.clear_current()
IOLoop.clear_instance()
LOGGER.debug("running %s %s", os.getpid(), context)
if _has_context:
kwargs[_has_context] = context
result = _f(*args, **kwargs)
if not subprocess:
return result
if isinstance(result, Future):
LOGGER.debug('running up tornado to complete')
def done(*args, **kwargs):
LOGGER.debug('stopping tornado')
IOLoop.current().stop()
result.add_done_callback(done)
IOLoop.current().start()
result = result.result()
return context, result
def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port,
worker_port, local_dir, services, name, memory_limit, reconnect,
resources, validate):
""" Function run by the Nanny when creating the worker """
from distributed import Worker # pragma: no cover
from tornado.ioloop import IOLoop # pragma: no cover
IOLoop.clear_instance() # pragma: no cover
loop = IOLoop() # pragma: no cover
loop.make_current() # pragma: no cover
worker = Worker(scheduler_ip, scheduler_port, ncores=ncores, ip=ip,
service_ports={'nanny': nanny_port}, local_dir=local_dir,
services=services, name=name, memory_limit=memory_limit,
reconnect=reconnect, validate=validate,
resources=resources, loop=loop) # pragma: no cover
@gen.coroutine # pragma: no cover
def run():
try: # pragma: no cover
yield worker._start(worker_port) # pragma: no cover
except Exception as e: # pragma: no cover
logger.exception(e) # pragma: no cover
q.put(e) # pragma: no cover
else:
assert worker.port # pragma: no cover
q.put({'port': worker.port, 'dir': worker.local_dir}) # pragma: no cover
while worker.status != 'closed':
yield gen.sleep(0.1)
logger.info("Worker closed")
try:
loop.run_sync(run)
finally:
loop.stop()
loop.close(all_fds=True)
def __init__(
self,
max_workers: Optional[int] = 2,
storage_uri: Optional[str] = None,
storage_project_name: str = "temporary_snowflake",
max_active_services: int = 20,
logging: Union[bool, str] = False,
start_server: bool = True,
reset_database: bool = False,
):
"""A temporary FractalServer that can be used to run complex workflows or try new computations.
! Warning ! All data is lost when the server is shutdown.
Parameters
----------
max_workers : Optional[int], optional
The maximum number of multiprocessing.Pool processes to spin up.
storage_uri : Optional[str], optional
A database URI to connect to, otherwise builds a default instance in a
temporary directory
storage_project_name : str, optional
The database name
max_active_services : int, optional
The maximum number of active services
logging : Union[bool, str], optional
If True, prints logging information to stdout. If False, hides all logging output. If a filename string is provided the logging will be
written to this file.
start_server : bool, optional
Starts the background asyncio loop or not.
reset_database : bool, optional
Resets the database or not if a storage_uri is provided
"""
# Startup a MongoDB in background thread and in custom folder.
if storage_uri is None:
self._storage = TemporaryPostgres(
database_name=storage_project_name)
self._storage_uri = self._storage.database_uri(safe=False,
database="")
else:
self._storage = None
self._storage_uri = storage_uri
if reset_database:
socket = storage_socket_factory(
self._storage_uri,
project_name=storage_project_name,
skip_version_check=True)
socket._clear_db(socket._project_name)
del socket
# Boot workers if needed
self.queue_socket = None
if max_workers:
self.queue_socket = Pool(
processes=max_workers,
initializer=_initialize_signals_process_pool)
# Add the loop to a background thread and init the server
self.aioloop = asyncio.new_event_loop()
asyncio.set_event_loop(self.aioloop)
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
self.loop = loop
self.loop_thread = ThreadPoolExecutor(max_workers=2)
if logging is False:
self.logfile = tempfile.NamedTemporaryFile()
log_prefix = self.logfile.name
elif logging is True:
self.logfile = None
log_prefix = None
elif isinstance(logging, str):
self.logfile = logging
log_prefix = self.logfile
else:
raise KeyError(f"Logfile type not recognized {type(logging)}.")
self._view_tempdir = tempfile.TemporaryDirectory()
super().__init__(
name="QCFractal Snowflake Instance",
port=find_port(),
loop=self.loop,
storage_uri=self._storage_uri,
storage_project_name=storage_project_name,
ssl_options=False,
max_active_services=max_active_services,
queue_socket=self.queue_socket,
logfile_prefix=log_prefix,
service_frequency=2,
query_limit=int(1.0e6),
view_enabled=True,
view_path=self._view_tempdir.name,
)
if self._storage:
self.logger.warning(
"Warning! This is a temporary instance, data will be lost upon shutdown. "
"For information about how to set up a permanent QCFractal instance, see "
"http://docs.qcarchive.molssi.org/projects/qcfractal/en/latest/setup_quickstart.html"
)
if start_server:
self.start(start_loop=False)
self.loop_future = self.loop_thread.submit(self.loop.start)
self._active = True
# We need to call before threadings cleanup
atexit.register(self.stop)
def after_scenario(context, scenario):
IOLoop.clear_instance()
del context.send_message
del context.users
del context.test_users
def _run(
cls,
worker_args,
worker_kwargs,
worker_start_args,
silence_logs,
init_result_q,
child_stop_q,
uid,
env,
Worker,
): # pragma: no cover
os.environ.update(env)
try:
from dask.multiprocessing import initialize_worker_process
except ImportError: # old Dask version
pass
else:
initialize_worker_process()
if silence_logs:
logger.setLevel(silence_logs)
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
worker = Worker(*worker_args, **worker_kwargs)
@gen.coroutine
def do_stop(timeout=5, executor_wait=True):
try:
yield worker.close(
report=False,
nanny=False,
executor_wait=executor_wait,
timeout=timeout,
)
finally:
loop.stop()
def watch_stop_q():
"""
Wait for an incoming stop message and then stop the
worker cleanly.
"""
while True:
try:
msg = child_stop_q.get(timeout=1000)
except Empty:
pass
else:
child_stop_q.close()
assert msg.pop("op") == "stop"
loop.add_callback(do_stop, **msg)
break
t = threading.Thread(target=watch_stop_q, name="Nanny stop queue watch")
t.daemon = True
t.start()
@gen.coroutine
def run():
"""
Try to start worker and inform parent of outcome.
"""
try:
yield worker._start(*worker_start_args)
except Exception as e:
logger.exception("Failed to start worker")
init_result_q.put({"uid": uid, "exception": e})
init_result_q.close()
else:
assert worker.address
init_result_q.put(
{"address": worker.address, "dir": worker.local_dir, "uid": uid}
)
init_result_q.close()
yield worker.wait_until_closed()
logger.info("Worker closed")
try:
loop.run_sync(run)
except TimeoutError:
# Loop was stopped before wait_until_closed() returned, ignore
pass
except KeyboardInterrupt:
pass
def tearDown(self):
super(TestFutureSocket, self).tearDown()
if self.loop:
self.loop.close(all_fds=True)
IOLoop.clear_current()
IOLoop.clear_instance()
def _run(
cls,
worker_kwargs,
worker_start_args,
silence_logs,
init_result_q,
child_stop_q,
uid,
env,
config,
Worker,
): # pragma: no cover
try:
os.environ.update(env)
dask.config.set(config)
try:
from dask.multiprocessing import initialize_worker_process
except ImportError: # old Dask version
pass
else:
initialize_worker_process()
if silence_logs:
logger.setLevel(silence_logs)
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
worker = Worker(**worker_kwargs)
async def do_stop(timeout=5, executor_wait=True):
try:
await worker.close(
report=True,
nanny=False,
safe=True, # TODO: Graceful or not?
executor_wait=executor_wait,
timeout=timeout,
)
finally:
loop.stop()
def watch_stop_q():
"""
Wait for an incoming stop message and then stop the
worker cleanly.
"""
msg = child_stop_q.get()
child_stop_q.close()
assert msg.pop("op") == "stop"
loop.add_callback(do_stop, **msg)
t = threading.Thread(target=watch_stop_q,
name="Nanny stop queue watch")
t.daemon = True
t.start()
async def run():
"""
Try to start worker and inform parent of outcome.
"""
try:
await worker
except Exception as e:
logger.exception("Failed to start worker")
init_result_q.put({"uid": uid, "exception": e})
init_result_q.close()
# If we hit an exception here we need to wait for a least
# one interval for the outside to pick up this message.
# Otherwise we arrive in a race condition where the process
# cleanup wipes the queue before the exception can be
# properly handled. See also
# WorkerProcess._wait_until_connected (the 2 is for good
# measure)
sync_sleep(cls._init_msg_interval * 2)
else:
try:
assert worker.address
except ValueError:
pass
else:
init_result_q.put({
"address": worker.address,
"dir": worker.local_directory,
"uid": uid,
})
init_result_q.close()
await worker.finished()
logger.info("Worker closed")
except Exception as e:
logger.exception("Failed to initialize Worker")
init_result_q.put({"uid": uid, "exception": e})
init_result_q.close()
# If we hit an exception here we need to wait for a least one
# interval for the outside to pick up this message. Otherwise we
# arrive in a race condition where the process cleanup wipes the
# queue before the exception can be properly handled. See also
# WorkerProcess._wait_until_connected (the 2 is for good measure)
sync_sleep(cls._init_msg_interval * 2)
else:
try:
loop.run_sync(run)
except (TimeoutError, gen.TimeoutError):
# Loop was stopped before wait_until_closed() returned, ignore
pass
except KeyboardInterrupt:
# At this point the loop is not running thus we have to run
# do_stop() explicitly.
loop.run_sync(do_stop)
def _run(
cls,
silence_logs,
init_result_q,
child_info_stop_q,
parent_info_q,
proc_cls,
kwargs,
env,
): # pragma: no cover
os.environ.update(env)
if silence_logs:
logger.setLevel(silence_logs)
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
scheduler = proc_cls(**kwargs)
async def do_stop(timeout=5):
try:
await scheduler.close(comm=None,
fast=False,
close_workers=False)
finally:
loop.stop()
def watch_stop_q():
"""
Wait for an incoming stop message and then stop the
scheduler cleanly.
"""
while True:
try:
msg = child_info_stop_q.get(timeout=1000)
except Empty:
pass
else:
op = msg.pop("op")
assert op == "stop" or op == "info"
if op == "stop":
child_info_stop_q.close()
loop.add_callback(do_stop, **msg)
break
elif op == "info":
parent_info_q.put({
"op":
"info",
"workers":
len(scheduler.workers),
"total_nthreads":
scheduler.total_nthreads,
})
t = threading.Thread(target=watch_stop_q,
name="Scheduler stop queue watch")
t.daemon = True
t.start()
async def run():
"""
Try to start scheduler and inform parent of outcome.
"""
try:
await scheduler.start()
except Exception as e:
logger.exception("Failed to start scheduler")
init_result_q.put({"exception": e})
init_result_q.close()
else:
try:
assert scheduler.address
except ValueError:
pass
else:
init_result_q.put({"address": scheduler.address})
init_result_q.close()
await scheduler.finished()
logger.info("Scheduler closed")
try:
loop.run_sync(run)
except TimeoutError:
# Loop was stopped before wait_until_closed() returned, ignore
pass
except KeyboardInterrupt:
pass
def start_ipython(ip=None, ns=None, log=None):
"""Start an IPython kernel in a thread
Parameters
----------
ip: str
The IP address to listen on (likely the parent object's ip).
ns: dict
Any names that should be injected into the IPython namespace.
log: logger instance
Hook up IPython's logging to an existing logger instead of the default.
"""
from IPython import get_ipython
if get_ipython() is not None:
raise RuntimeError("Cannot start IPython, it's already running.")
from zmq.eventloop.ioloop import ZMQIOLoop
from ipykernel.kernelapp import IPKernelApp
# save the global IOLoop instance
# since IPython relies on it, but we are going to put it in a thread.
save_inst = IOLoop.instance()
IOLoop.clear_instance()
zmq_loop = ZMQIOLoop()
zmq_loop.install()
# start IPython, disabling its signal handlers that won't work due to running in a thread:
app = IPKernelApp.instance(log=log)
# Don't connect to the history database
app.config.HistoryManager.hist_file = ":memory:"
# listen on all interfaces, so remote clients can connect:
if ip:
app.ip = ip
# disable some signal handling, logging
def noop():
return None
app.init_signal = noop
app.log_connection_info = noop
# start IPython in a thread
# initialization happens in the thread to avoid threading problems
# with the sqlite history
evt = Event()
def _start():
app.initialize([])
app.kernel.pre_handler_hook = noop
app.kernel.post_handler_hook = noop
app.kernel.start()
app.kernel.loop = IOLoop.instance()
# save self in the IPython namespace as 'worker'
# inject things into the IPython namespace
if ns:
app.kernel.shell.user_ns.update(ns)
evt.set()
zmq_loop.start()
zmq_loop_thread = Thread(target=_start)
zmq_loop_thread.daemon = True
zmq_loop_thread.start()
assert evt.wait(
timeout=5), "IPython didn't start in a reasonable amount of time."
# put the global IOLoop instance back:
IOLoop.clear_instance()
save_inst.install()
return app
def tearDown(self):
super(TestIOLoop, self).tearDown()
BaseIOLoop.clear_current()
BaseIOLoop.clear_instance()
def server_close(self):
ioloop = IOLoop.instance()
ioloop.add_callback(ioloop.stop)
ioloop.clear_current()
IOLoop.clear_instance()
def server_close(self):
IOLoop.instance().stop()
IOLoop.clear_instance()
def tearDown(self):
super(TestFutureSocket, self).tearDown()
if self.loop:
self.loop.close(all_fds=True)
IOLoop.clear_current()
IOLoop.clear_instance()
def _run(cls, worker_args, worker_kwargs, worker_start_args,
silence_logs, init_result_q, child_stop_q): # pragma: no cover
from distributed import Worker
try:
from dask.multiprocessing import initialize_worker_process
except ImportError: # old Dask version
pass
else:
initialize_worker_process()
if silence_logs:
logger.setLevel(silence_logs)
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
worker = Worker(*worker_args, **worker_kwargs)
@gen.coroutine
def do_stop(timeout):
try:
yield worker._close(report=False, nanny=False)
finally:
loop.stop()
def watch_stop_q():
"""
Wait for an incoming stop message and then stop the
worker cleanly.
"""
while True:
try:
msg = child_stop_q.get(timeout=1000)
except Empty:
pass
else:
assert msg['op'] == 'stop'
loop.add_callback(do_stop, msg['timeout'])
break
t = threading.Thread(target=watch_stop_q, name="Nanny stop queue watch")
t.daemon = True
t.start()
@gen.coroutine
def run():
"""
Try to start worker and inform parent of outcome.
"""
try:
yield worker._start(*worker_start_args)
except Exception as e:
logger.exception("Failed to start worker")
init_result_q.put(e)
else:
assert worker.address
init_result_q.put({'address': worker.address,
'dir': worker.local_dir})
yield worker.wait_until_closed()
logger.info("Worker closed")
try:
loop.run_sync(run)
except TimeoutError:
# Loop was stopped before wait_until_closed() returned, ignore
pass
except KeyboardInterrupt:
pass
def tearDown(self):
super(TestIOLoop, self).tearDown()
BaseIOLoop.clear_current()
BaseIOLoop.clear_instance()