Esempio n. 1
0
def run_worker(q, ip, center_ip, center_port, ncores, nanny_port,
        worker_port, local_dir, services, name):
    """ Function run by the Nanny when creating the worker """
    from distributed import Worker  # pragma: no cover
    from tornado.ioloop import IOLoop  # pragma: no cover
    IOLoop.clear_instance()  # pragma: no cover
    loop = IOLoop()  # pragma: no cover
    loop.make_current()  # pragma: no cover
    worker = Worker(center_ip, center_port, ncores=ncores, ip=ip,
                    service_ports={'nanny': nanny_port}, local_dir=local_dir,
                    services=services, name=name)  # pragma: no cover

    @gen.coroutine  # pragma: no cover
    def start():
        try:  # pragma: no cover
            yield worker._start(worker_port)  # pragma: no cover
        except Exception as e:  # pragma: no cover
            logger.exception(e)  # pragma: no cover
            q.put(e)  # pragma: no cover
        else:
            assert worker.port  # pragma: no cover
            q.put({'port': worker.port, 'dir': worker.local_dir})  # pragma: no cover

    loop.add_callback(start)  # pragma: no cover
    with ignoring(KeyboardInterrupt):
        loop.start()  # pragma: no cover
Esempio n. 2
0
def run_worker(q, ip, center_ip, center_port, ncores, nanny_port,
        local_dir, services):
    """ Function run by the Nanny when creating the worker """
    from distributed import Worker
    from tornado.ioloop import IOLoop
    IOLoop.clear_instance()
    loop = IOLoop()
    loop.make_current()
    worker = Worker(center_ip, center_port, ncores=ncores, ip=ip,
                    service_ports={'nanny': nanny_port}, local_dir=local_dir,
                    services=services)

    @gen.coroutine
    def start():
        try:
            yield worker._start()
        except Exception as e:
            logger.exception(e)
            q.put(e)
        else:
            assert worker.port
            q.put({'port': worker.port, 'dir': worker.local_dir})

    loop.add_callback(start)
    loop.start()
Esempio n. 3
0
def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port,
        worker_port, local_dir, services, name, memory_limit):
    """ Function run by the Nanny when creating the worker """
    from distributed import Worker  # pragma: no cover
    from tornado.ioloop import IOLoop  # pragma: no cover
    IOLoop.clear_instance()  # pragma: no cover
    loop = IOLoop()  # pragma: no cover
    loop.make_current()  # pragma: no cover
    worker = Worker(scheduler_ip, scheduler_port, ncores=ncores, ip=ip,
                    service_ports={'nanny': nanny_port}, local_dir=local_dir,
                    services=services, name=name, memory_limit=memory_limit,
                    loop=loop)  # pragma: no cover

    @gen.coroutine  # pragma: no cover
    def start():
        try:  # pragma: no cover
            yield worker._start(worker_port)  # pragma: no cover
        except Exception as e:  # pragma: no cover
            logger.exception(e)  # pragma: no cover
            q.put(e)  # pragma: no cover
        else:
            assert worker.port  # pragma: no cover
            q.put({'port': worker.port, 'dir': worker.local_dir})  # pragma: no cover

    loop.add_callback(start)  # pragma: no cover
    try:
        loop.start()  # pragma: no cover
    finally:
        loop.stop()
        loop.close(all_fds=True)
Esempio n. 4
0
def loop():
    IOLoop.clear_instance()
    loop = IOLoop()
    loop.make_current()
    yield loop
    loop.stop()
    loop.close()
Esempio n. 5
0
def pristine_loop():
    IOLoop.clear_instance()
    loop = IOLoop()
    loop.make_current()
    try:
        yield loop
    finally:
        loop.close(all_fds=True)
        IOLoop.clear_instance()
Esempio n. 6
0
def test_sync_closed_loop():
    loop = IOLoop.current()
    loop.close()
    IOLoop.clear_current()
    IOLoop.clear_instance()

    with pytest.raises(RuntimeError) as exc_info:
        sync(loop, inc, 1)
    exc_info.match("IOLoop is clos(ed|ing)")
Esempio n. 7
0
def loop():
    IOLoop.clear_instance()
    loop = IOLoop()
    loop.make_current()
    yield loop
    sync(loop, loop.stop)
    for i in range(5):
        with ignoring(Exception):
            loop.close(all_fds=True)
            break
Esempio n. 8
0
def run_nanny(port, center_port, **kwargs):
    from distributed import Nanny
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging
    IOLoop.clear_instance()
    loop = IOLoop(); loop.make_current()
    PeriodicCallback(lambda: None, 500).start()
    logging.getLogger("tornado").setLevel(logging.CRITICAL)
    worker = Nanny('127.0.0.1', port, port + 1000, '127.0.0.1', center_port, **kwargs)
    loop.run_sync(worker._start)
    loop.start()
Esempio n. 9
0
def run_center(port):
    from distributed import Center
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging
    IOLoop.clear_instance()
    loop = IOLoop(); loop.make_current()
    PeriodicCallback(lambda: None, 500).start()
    logging.getLogger("tornado").setLevel(logging.CRITICAL)
    center = Center('127.0.0.1', port)
    center.listen(port)
    loop.start()
Esempio n. 10
0
        def test_func():
            IOLoop.clear_instance()
            loop = IOLoop()
            loop.make_current()

            cor = gen.coroutine(func)
            try:
                loop.run_sync(cor, timeout=timeout)
            finally:
                loop.stop()
                loop.close(all_fds=True)
Esempio n. 11
0
        def test_func():
            IOLoop.clear_instance()
            loop = IOLoop()
            loop.make_current()

            s, workers = loop.run_sync(lambda: start_cluster(ncores))
            try:
                loop.run_sync(lambda: cor(s, *workers), timeout=timeout)
            finally:
                loop.run_sync(lambda: end_cluster(s, workers))
                loop.stop()
                loop.close()
Esempio n. 12
0
def run_worker(q, center_port, **kwargs):
    from distributed import Worker
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging
    with log_errors():
        IOLoop.clear_instance()
        loop = IOLoop(); loop.make_current()
        PeriodicCallback(lambda: None, 500).start()
        logging.getLogger("tornado").setLevel(logging.CRITICAL)
        worker = Worker('127.0.0.1', center_port, ip='127.0.0.1', **kwargs)
        loop.run_sync(lambda: worker._start(0))
        q.put(worker.port)
        loop.start()
Esempio n. 13
0
def run_worker(port, center_port, **kwargs):
    from distributed import Worker
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging

    IOLoop.clear_instance()
    loop = IOLoop()
    loop.make_current()
    PeriodicCallback(lambda: None, 500).start()
    logging.getLogger("tornado").setLevel(logging.CRITICAL)
    worker = Worker("127.0.0.1", port, "127.0.0.1", center_port, **kwargs)
    worker.start()
    loop.start()
Esempio n. 14
0
def run_worker(q, scheduler_port, **kwargs):
    from distributed import Worker
    from tornado.ioloop import IOLoop, PeriodicCallback
    with log_errors():
        IOLoop.clear_instance()
        loop = IOLoop(); loop.make_current()
        PeriodicCallback(lambda: None, 500).start()
        worker = Worker('127.0.0.1', scheduler_port, ip='127.0.0.1',
                        loop=loop, validate=True, **kwargs)
        loop.run_sync(lambda: worker._start(0))
        q.put(worker.port)
        try:
            loop.start()
        finally:
            loop.close(all_fds=True)
Esempio n. 15
0
def run_scheduler(q, scheduler_port=0, **kwargs):
    from distributed import Scheduler
    from tornado.ioloop import IOLoop, PeriodicCallback
    IOLoop.clear_instance()
    loop = IOLoop(); loop.make_current()
    PeriodicCallback(lambda: None, 500).start()

    scheduler = Scheduler(loop=loop, validate=True, **kwargs)
    done = scheduler.start(scheduler_port)

    q.put(scheduler.port)
    try:
        loop.start()
    finally:
        loop.close(all_fds=True)
Esempio n. 16
0
def current_loop():
    IOLoop.clear_instance()
    loop = IOLoop()
    loop.make_current()
    yield loop
    if loop._running:
        sync(loop, loop.stop)
    for i in range(5):
        try:
            loop.close(all_fds=True)
            return
        except Exception as e:
            f = e
            print(f)
    IOLoop.clear_instance()
Esempio n. 17
0
def pristine_loop():
    IOLoop.clear_instance()
    IOLoop.clear_current()
    loop = IOLoop()
    loop.make_current()
    assert IOLoop.current() is loop
    try:
        yield loop
    finally:
        try:
            loop.close(all_fds=True)
        except (KeyError, ValueError):
            pass
        IOLoop.clear_instance()
        IOLoop.clear_current()
Esempio n. 18
0
def run_worker(q, ip, port, center_ip, center_port, ncores, nanny_port):
    """ Function run by the Nanny when creating the worker """
    from distributed import Worker
    from tornado.ioloop import IOLoop
    IOLoop.clear_instance()
    loop = IOLoop()
    loop.make_current()
    worker = Worker(ip, port, center_ip, center_port, ncores,
                    nanny_port=nanny_port)

    @gen.coroutine
    def start():
        yield worker._start()
        q.put(worker.port)
    loop.add_callback(start)
    loop.start()
Esempio n. 19
0
def run_nanny(q, center_port, **kwargs):
    from distributed import Nanny
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging
    with log_errors():
        IOLoop.clear_instance()
        loop = IOLoop(); loop.make_current()
        PeriodicCallback(lambda: None, 500).start()
        logging.getLogger("tornado").setLevel(logging.CRITICAL)
        worker = Nanny('127.0.0.1', center_port, ip='127.0.0.1', loop=loop, **kwargs)
        loop.run_sync(lambda: worker._start(0))
        q.put(worker.port)
        try:
            loop.start()
        finally:
            loop.run_sync(worker._close)
            loop.close(all_fds=True)
Esempio n. 20
0
def run_scheduler(q, scheduler_port=0, center_port=None, **kwargs):
    from distributed import Scheduler
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging
    IOLoop.clear_instance()
    loop = IOLoop(); loop.make_current()
    PeriodicCallback(lambda: None, 500).start()
    logging.getLogger("tornado").setLevel(logging.CRITICAL)

    center = ('127.0.0.1', center_port) if center_port else None
    scheduler = Scheduler(center=center, loop=loop, validate=True, **kwargs)
    done = scheduler.start(scheduler_port)

    q.put(scheduler.port)
    try:
        loop.start()
    finally:
        loop.close(all_fds=True)
Esempio n. 21
0
def run_scheduler(q, center_port=None, **kwargs):
    from distributed import Scheduler
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging
    IOLoop.clear_instance()
    loop = IOLoop(); loop.make_current()
    PeriodicCallback(lambda: None, 500).start()
    logging.getLogger("tornado").setLevel(logging.CRITICAL)

    center = ('127.0.0.1', center_port) if center_port else None
    scheduler = Scheduler(center=center, **kwargs)
    scheduler.listen(0)

    if center_port:
        loop.run_sync(scheduler.sync_center)
    done = scheduler.start(0)

    q.put(scheduler.port)
    loop.start()
Esempio n. 22
0
def run_center(q):
    from distributed import Center
    from tornado.ioloop import IOLoop, PeriodicCallback
    import logging
    IOLoop.clear_instance()
    loop = IOLoop(); loop.make_current()
    PeriodicCallback(lambda: None, 500).start()
    logging.getLogger("tornado").setLevel(logging.CRITICAL)
    center = Center('127.0.0.1')

    while True:
        try:
            center.listen(0)
            break
        except Exception as e:
            logging.info("Could not start center on port.  Retrying",
                    exc_info=True)

    q.put(center.port)
    loop.start()
Esempio n. 23
0
 def start(self):
     """Start the whole thing"""
     # start the proxy
     self.start_proxy()
     # start the webserver
     http_server = tornado.httpserver.HTTPServer(self.tornado_application)
     http_server.listen(self.hub_port)
     
     loop = IOLoop.instance()
     try:
         loop.start()
     except KeyboardInterrupt:
         print("\nInterrupted")
     finally:
         # have to install/start a new IOLoop briefly,
         # to allow for async cleanup code.
         IOLoop.clear_instance()
         cleanup_loop = IOLoop.instance()
         cleanup_loop.add_callback(self.cleanup)
         cleanup_loop.start()
Esempio n. 24
0
        def test_func():
            IOLoop.clear_instance()
            loop = IOLoop()
            loop.make_current()

            s, workers = loop.run_sync(lambda: start_cluster(ncores,
                                                             Worker=Worker))
            args = [s] + workers

            if executor:
                e = Executor((s.ip, s.port), loop=loop, start=False)
                loop.run_sync(e._start)
                args = [e] + args
            try:
                loop.run_sync(lambda: cor(*args), timeout=timeout)
            finally:
                if executor:
                    loop.run_sync(e._shutdown)
                loop.run_sync(lambda: end_cluster(s, workers))
                loop.stop()
                loop.close(all_fds=True)
Esempio n. 25
0
    def _start_ipython(self):
        from IPython import get_ipython
        if get_ipython() is not None:
            raise RuntimeError("Cannot start IPython, it's already running.")

        from zmq.eventloop.ioloop import ZMQIOLoop
        from ipykernel.kernelapp import IPKernelApp
        # save the global IOLoop instance
        # since IPython relies on it, but we are going to put it in a thread.
        save_inst = IOLoop.instance()
        IOLoop.clear_instance()
        zmq_loop = ZMQIOLoop()
        zmq_loop.install()

        # start IPython, disabling its signal handlers that won't work due to running in a thread:
        app = self._ipython_kernel = IPKernelApp.instance(log=logger)
        # Don't connect to the history database
        app.config.HistoryManager.hist_file = ':memory:'
        # listen on all interfaces, so remote clients can connect:
        app.ip = self.ip
        app.init_signal = lambda : None
        app.initialize([])
        app.kernel.pre_handler_hook = lambda : None
        app.kernel.post_handler_hook = lambda : None
        app.kernel.start()

        # save self in the IPython namespace as 'worker'
        app.kernel.shell.user_ns['worker'] = self

        # start IPython's IOLoop in a thread
        zmq_loop_thread = Thread(target=zmq_loop.start)
        zmq_loop_thread.start()

        # put the global IOLoop instance back:
        IOLoop.clear_instance()
        save_inst.install()
        return app
Esempio n. 26
0
def run_in_pool(_pid, _f, _has_context, context, *args, **kwargs):
    # globals from the parent process in the
    # IOLoop so clear them.
    subprocess = os.getpid() != _pid
    if subprocess and IOLoop.current(False):
        LOGGER.debug("clearing tornado globals")
        IOLoop.clear_current()
        IOLoop.clear_instance()
    LOGGER.debug("running %s %s", os.getpid(), context)
    if _has_context:
        kwargs[_has_context] = context
    result = _f(*args, **kwargs)
    if not subprocess:
        return result
    if isinstance(result, Future):
        LOGGER.debug('running up tornado to complete')

        def done(*args, **kwargs):
            LOGGER.debug('stopping tornado')
            IOLoop.current().stop()
        result.add_done_callback(done)
        IOLoop.current().start()
        result = result.result()
    return context, result
Esempio n. 27
0
def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port,
        worker_port, local_dir, services, name, memory_limit, reconnect,
        resources, validate):
    """ Function run by the Nanny when creating the worker """
    from distributed import Worker  # pragma: no cover
    from tornado.ioloop import IOLoop  # pragma: no cover
    IOLoop.clear_instance()  # pragma: no cover
    loop = IOLoop()  # pragma: no cover
    loop.make_current()  # pragma: no cover
    worker = Worker(scheduler_ip, scheduler_port, ncores=ncores, ip=ip,
                    service_ports={'nanny': nanny_port}, local_dir=local_dir,
                    services=services, name=name, memory_limit=memory_limit,
                    reconnect=reconnect, validate=validate,
                    resources=resources, loop=loop)  # pragma: no cover

    @gen.coroutine  # pragma: no cover
    def run():
        try:  # pragma: no cover
            yield worker._start(worker_port)  # pragma: no cover
        except Exception as e:  # pragma: no cover
            logger.exception(e)  # pragma: no cover
            q.put(e)  # pragma: no cover
        else:
            assert worker.port  # pragma: no cover
            q.put({'port': worker.port, 'dir': worker.local_dir})  # pragma: no cover

        while worker.status != 'closed':
            yield gen.sleep(0.1)

        logger.info("Worker closed")

    try:
        loop.run_sync(run)
    finally:
        loop.stop()
        loop.close(all_fds=True)
Esempio n. 28
0
    def __init__(
        self,
        max_workers: Optional[int] = 2,
        storage_uri: Optional[str] = None,
        storage_project_name: str = "temporary_snowflake",
        max_active_services: int = 20,
        logging: Union[bool, str] = False,
        start_server: bool = True,
        reset_database: bool = False,
    ):
        """A temporary FractalServer that can be used to run complex workflows or try new computations.

        ! Warning ! All data is lost when the server is shutdown.

        Parameters
        ----------
        max_workers : Optional[int], optional
            The maximum number of multiprocessing.Pool processes to spin up.
        storage_uri : Optional[str], optional
            A database URI to connect to, otherwise builds a default instance in a
            temporary directory
        storage_project_name : str, optional
            The database name
        max_active_services : int, optional
            The maximum number of active services
        logging : Union[bool, str], optional
            If True, prints logging information to stdout. If False, hides all logging output. If a filename string is provided the logging will be
            written to this file.
        start_server : bool, optional
            Starts the background asyncio loop or not.
        reset_database : bool, optional
            Resets the database or not if a storage_uri is provided

        """

        # Startup a MongoDB in background thread and in custom folder.
        if storage_uri is None:
            self._storage = TemporaryPostgres(
                database_name=storage_project_name)
            self._storage_uri = self._storage.database_uri(safe=False,
                                                           database="")
        else:
            self._storage = None
            self._storage_uri = storage_uri

            if reset_database:
                socket = storage_socket_factory(
                    self._storage_uri,
                    project_name=storage_project_name,
                    skip_version_check=True)
                socket._clear_db(socket._project_name)
                del socket

        # Boot workers if needed
        self.queue_socket = None
        if max_workers:
            self.queue_socket = Pool(
                processes=max_workers,
                initializer=_initialize_signals_process_pool)

        # Add the loop to a background thread and init the server
        self.aioloop = asyncio.new_event_loop()
        asyncio.set_event_loop(self.aioloop)
        IOLoop.clear_instance()
        IOLoop.clear_current()
        loop = IOLoop()
        self.loop = loop
        self.loop_thread = ThreadPoolExecutor(max_workers=2)

        if logging is False:
            self.logfile = tempfile.NamedTemporaryFile()
            log_prefix = self.logfile.name
        elif logging is True:
            self.logfile = None
            log_prefix = None
        elif isinstance(logging, str):
            self.logfile = logging
            log_prefix = self.logfile
        else:
            raise KeyError(f"Logfile type not recognized {type(logging)}.")

        self._view_tempdir = tempfile.TemporaryDirectory()

        super().__init__(
            name="QCFractal Snowflake Instance",
            port=find_port(),
            loop=self.loop,
            storage_uri=self._storage_uri,
            storage_project_name=storage_project_name,
            ssl_options=False,
            max_active_services=max_active_services,
            queue_socket=self.queue_socket,
            logfile_prefix=log_prefix,
            service_frequency=2,
            query_limit=int(1.0e6),
            view_enabled=True,
            view_path=self._view_tempdir.name,
        )

        if self._storage:
            self.logger.warning(
                "Warning! This is a temporary instance, data will be lost upon shutdown. "
                "For information about how to set up a permanent QCFractal instance, see "
                "http://docs.qcarchive.molssi.org/projects/qcfractal/en/latest/setup_quickstart.html"
            )

        if start_server:
            self.start(start_loop=False)

        self.loop_future = self.loop_thread.submit(self.loop.start)

        self._active = True

        # We need to call before threadings cleanup
        atexit.register(self.stop)
Esempio n. 29
0
def after_scenario(context, scenario):
    IOLoop.clear_instance()
    del context.send_message
    del context.users
    del context.test_users
Esempio n. 30
0
    def _run(
        cls,
        worker_args,
        worker_kwargs,
        worker_start_args,
        silence_logs,
        init_result_q,
        child_stop_q,
        uid,
        env,
        Worker,
    ):  # pragma: no cover
        os.environ.update(env)
        try:
            from dask.multiprocessing import initialize_worker_process
        except ImportError:  # old Dask version
            pass
        else:
            initialize_worker_process()

        if silence_logs:
            logger.setLevel(silence_logs)

        IOLoop.clear_instance()
        loop = IOLoop()
        loop.make_current()
        worker = Worker(*worker_args, **worker_kwargs)

        @gen.coroutine
        def do_stop(timeout=5, executor_wait=True):
            try:
                yield worker.close(
                    report=False,
                    nanny=False,
                    executor_wait=executor_wait,
                    timeout=timeout,
                )
            finally:
                loop.stop()

        def watch_stop_q():
            """
            Wait for an incoming stop message and then stop the
            worker cleanly.
            """
            while True:
                try:
                    msg = child_stop_q.get(timeout=1000)
                except Empty:
                    pass
                else:
                    child_stop_q.close()
                    assert msg.pop("op") == "stop"
                    loop.add_callback(do_stop, **msg)
                    break

        t = threading.Thread(target=watch_stop_q, name="Nanny stop queue watch")
        t.daemon = True
        t.start()

        @gen.coroutine
        def run():
            """
            Try to start worker and inform parent of outcome.
            """
            try:
                yield worker._start(*worker_start_args)
            except Exception as e:
                logger.exception("Failed to start worker")
                init_result_q.put({"uid": uid, "exception": e})
                init_result_q.close()
            else:
                assert worker.address
                init_result_q.put(
                    {"address": worker.address, "dir": worker.local_dir, "uid": uid}
                )
                init_result_q.close()
                yield worker.wait_until_closed()
                logger.info("Worker closed")

        try:
            loop.run_sync(run)
        except TimeoutError:
            # Loop was stopped before wait_until_closed() returned, ignore
            pass
        except KeyboardInterrupt:
            pass
Esempio n. 31
0
 def tearDown(self):
     super(TestFutureSocket, self).tearDown()
     if self.loop:
         self.loop.close(all_fds=True)
     IOLoop.clear_current()
     IOLoop.clear_instance()
Esempio n. 32
0
    def _run(
        cls,
        worker_kwargs,
        worker_start_args,
        silence_logs,
        init_result_q,
        child_stop_q,
        uid,
        env,
        config,
        Worker,
    ):  # pragma: no cover
        try:
            os.environ.update(env)
            dask.config.set(config)
            try:
                from dask.multiprocessing import initialize_worker_process
            except ImportError:  # old Dask version
                pass
            else:
                initialize_worker_process()

            if silence_logs:
                logger.setLevel(silence_logs)

            IOLoop.clear_instance()
            loop = IOLoop()
            loop.make_current()
            worker = Worker(**worker_kwargs)

            async def do_stop(timeout=5, executor_wait=True):
                try:
                    await worker.close(
                        report=True,
                        nanny=False,
                        safe=True,  # TODO: Graceful or not?
                        executor_wait=executor_wait,
                        timeout=timeout,
                    )
                finally:
                    loop.stop()

            def watch_stop_q():
                """
                Wait for an incoming stop message and then stop the
                worker cleanly.
                """
                msg = child_stop_q.get()
                child_stop_q.close()
                assert msg.pop("op") == "stop"
                loop.add_callback(do_stop, **msg)

            t = threading.Thread(target=watch_stop_q,
                                 name="Nanny stop queue watch")
            t.daemon = True
            t.start()

            async def run():
                """
                Try to start worker and inform parent of outcome.
                """
                try:
                    await worker
                except Exception as e:
                    logger.exception("Failed to start worker")
                    init_result_q.put({"uid": uid, "exception": e})
                    init_result_q.close()
                    # If we hit an exception here we need to wait for a least
                    # one interval for the outside to pick up this message.
                    # Otherwise we arrive in a race condition where the process
                    # cleanup wipes the queue before the exception can be
                    # properly handled. See also
                    # WorkerProcess._wait_until_connected (the 2 is for good
                    # measure)
                    sync_sleep(cls._init_msg_interval * 2)
                else:
                    try:
                        assert worker.address
                    except ValueError:
                        pass
                    else:
                        init_result_q.put({
                            "address": worker.address,
                            "dir": worker.local_directory,
                            "uid": uid,
                        })
                        init_result_q.close()
                        await worker.finished()
                        logger.info("Worker closed")

        except Exception as e:
            logger.exception("Failed to initialize Worker")
            init_result_q.put({"uid": uid, "exception": e})
            init_result_q.close()
            # If we hit an exception here we need to wait for a least one
            # interval for the outside to pick up this message. Otherwise we
            # arrive in a race condition where the process cleanup wipes the
            # queue before the exception can be properly handled. See also
            # WorkerProcess._wait_until_connected (the 2 is for good measure)
            sync_sleep(cls._init_msg_interval * 2)
        else:
            try:
                loop.run_sync(run)
            except (TimeoutError, gen.TimeoutError):
                # Loop was stopped before wait_until_closed() returned, ignore
                pass
            except KeyboardInterrupt:
                # At this point the loop is not running thus we have to run
                # do_stop() explicitly.
                loop.run_sync(do_stop)
Esempio n. 33
0
    def _run(
        cls,
        silence_logs,
        init_result_q,
        child_info_stop_q,
        parent_info_q,
        proc_cls,
        kwargs,
        env,
    ):  # pragma: no cover
        os.environ.update(env)

        if silence_logs:
            logger.setLevel(silence_logs)

        IOLoop.clear_instance()
        loop = IOLoop()
        loop.make_current()
        scheduler = proc_cls(**kwargs)

        async def do_stop(timeout=5):
            try:
                await scheduler.close(comm=None,
                                      fast=False,
                                      close_workers=False)
            finally:
                loop.stop()

        def watch_stop_q():
            """
            Wait for an incoming stop message and then stop the
            scheduler cleanly.
            """
            while True:
                try:
                    msg = child_info_stop_q.get(timeout=1000)
                except Empty:
                    pass
                else:
                    op = msg.pop("op")
                    assert op == "stop" or op == "info"
                    if op == "stop":
                        child_info_stop_q.close()
                        loop.add_callback(do_stop, **msg)
                        break
                    elif op == "info":
                        parent_info_q.put({
                            "op":
                            "info",
                            "workers":
                            len(scheduler.workers),
                            "total_nthreads":
                            scheduler.total_nthreads,
                        })

        t = threading.Thread(target=watch_stop_q,
                             name="Scheduler stop queue watch")
        t.daemon = True
        t.start()

        async def run():
            """
            Try to start scheduler and inform parent of outcome.
            """
            try:
                await scheduler.start()
            except Exception as e:
                logger.exception("Failed to start scheduler")
                init_result_q.put({"exception": e})
                init_result_q.close()
            else:
                try:
                    assert scheduler.address
                except ValueError:
                    pass
                else:
                    init_result_q.put({"address": scheduler.address})
                    init_result_q.close()
                    await scheduler.finished()
                    logger.info("Scheduler closed")

        try:
            loop.run_sync(run)
        except TimeoutError:
            # Loop was stopped before wait_until_closed() returned, ignore
            pass
        except KeyboardInterrupt:
            pass
Esempio n. 34
0
def start_ipython(ip=None, ns=None, log=None):
    """Start an IPython kernel in a thread

    Parameters
    ----------

    ip: str
        The IP address to listen on (likely the parent object's ip).
    ns: dict
        Any names that should be injected into the IPython namespace.
    log: logger instance
        Hook up IPython's logging to an existing logger instead of the default.
    """
    from IPython import get_ipython

    if get_ipython() is not None:
        raise RuntimeError("Cannot start IPython, it's already running.")

    from zmq.eventloop.ioloop import ZMQIOLoop
    from ipykernel.kernelapp import IPKernelApp

    # save the global IOLoop instance
    # since IPython relies on it, but we are going to put it in a thread.
    save_inst = IOLoop.instance()
    IOLoop.clear_instance()
    zmq_loop = ZMQIOLoop()
    zmq_loop.install()

    # start IPython, disabling its signal handlers that won't work due to running in a thread:
    app = IPKernelApp.instance(log=log)
    # Don't connect to the history database
    app.config.HistoryManager.hist_file = ":memory:"
    # listen on all interfaces, so remote clients can connect:
    if ip:
        app.ip = ip
    # disable some signal handling, logging

    def noop():
        return None

    app.init_signal = noop
    app.log_connection_info = noop

    # start IPython in a thread
    # initialization happens in the thread to avoid threading problems
    # with the sqlite history
    evt = Event()

    def _start():
        app.initialize([])
        app.kernel.pre_handler_hook = noop
        app.kernel.post_handler_hook = noop
        app.kernel.start()
        app.kernel.loop = IOLoop.instance()
        # save self in the IPython namespace as 'worker'
        # inject things into the IPython namespace
        if ns:
            app.kernel.shell.user_ns.update(ns)
        evt.set()
        zmq_loop.start()

    zmq_loop_thread = Thread(target=_start)
    zmq_loop_thread.daemon = True
    zmq_loop_thread.start()
    assert evt.wait(
        timeout=5), "IPython didn't start in a reasonable amount of time."

    # put the global IOLoop instance back:
    IOLoop.clear_instance()
    save_inst.install()
    return app
Esempio n. 35
0
 def tearDown(self):
     super(TestIOLoop, self).tearDown()
     BaseIOLoop.clear_current()
     BaseIOLoop.clear_instance()
Esempio n. 36
0
 def server_close(self):
     ioloop = IOLoop.instance()
     ioloop.add_callback(ioloop.stop)
     ioloop.clear_current()
     IOLoop.clear_instance()
Esempio n. 37
0
 def server_close(self):
     IOLoop.instance().stop()
     IOLoop.clear_instance()
Esempio n. 38
0
 def tearDown(self):
     super(TestFutureSocket, self).tearDown()
     if self.loop:
         self.loop.close(all_fds=True)
     IOLoop.clear_current()
     IOLoop.clear_instance()
    def _run(cls, worker_args, worker_kwargs, worker_start_args,
             silence_logs, init_result_q, child_stop_q):  # pragma: no cover
        from distributed import Worker

        try:
            from dask.multiprocessing import initialize_worker_process
        except ImportError:   # old Dask version
            pass
        else:
            initialize_worker_process()

        if silence_logs:
            logger.setLevel(silence_logs)

        IOLoop.clear_instance()
        loop = IOLoop()
        loop.make_current()
        worker = Worker(*worker_args, **worker_kwargs)

        @gen.coroutine
        def do_stop(timeout):
            try:
                yield worker._close(report=False, nanny=False)
            finally:
                loop.stop()

        def watch_stop_q():
            """
            Wait for an incoming stop message and then stop the
            worker cleanly.
            """
            while True:
                try:
                    msg = child_stop_q.get(timeout=1000)
                except Empty:
                    pass
                else:
                    assert msg['op'] == 'stop'
                    loop.add_callback(do_stop, msg['timeout'])
                    break

        t = threading.Thread(target=watch_stop_q, name="Nanny stop queue watch")
        t.daemon = True
        t.start()

        @gen.coroutine
        def run():
            """
            Try to start worker and inform parent of outcome.
            """
            try:
                yield worker._start(*worker_start_args)
            except Exception as e:
                logger.exception("Failed to start worker")
                init_result_q.put(e)
            else:
                assert worker.address
                init_result_q.put({'address': worker.address,
                                   'dir': worker.local_dir})
                yield worker.wait_until_closed()
                logger.info("Worker closed")

        try:
            loop.run_sync(run)
        except TimeoutError:
            # Loop was stopped before wait_until_closed() returned, ignore
            pass
        except KeyboardInterrupt:
            pass
Esempio n. 40
0
 def tearDown(self):
     super(TestIOLoop, self).tearDown()
     BaseIOLoop.clear_current()
     BaseIOLoop.clear_instance()