コード例 #1
0
ファイル: test_comms.py プロジェクト: tomMoral/distributed
def test_get_local_address_for():
    f = get_local_address_for

    assert f('tcp://127.0.0.1:80') == 'tcp://127.0.0.1'
    assert f('tcp://8.8.8.8:4444') == 'tcp://' + get_ip()
    if has_ipv6():
        assert f('tcp://[::1]:123') == 'tcp://[::1]'

    inproc_arg = 'inproc://%s/%d/444' % (get_ip(), os.getpid())
    inproc_res = f(inproc_arg)
    assert inproc_res.startswith('inproc://')
    assert inproc_res != inproc_arg
コード例 #2
0
def test_bokeh():
    pytest.importorskip('bokeh')

    try:
        proc = Popen(['dscheduler'], stdout=PIPE, stderr=PIPE)
        e = Executor('127.0.0.1:%d' % Scheduler.default_port)

        while True:
            line = proc.stderr.readline()
            if b'Bokeh UI' in line:
                break

        start = time()
        while True:
            try:
                for name in [socket.gethostname(), 'localhost', '127.0.0.1', get_ip()]:
                    response = requests.get('http://%s:8787/status/' % name)
                    assert response.ok
                break
            except:
                sleep(0.1)
                assert time() < start + 5

    finally:
        with ignoring(Exception):
            e.shutdown()
        with ignoring(Exception):
            os.kill(proc.pid, signal.SIGINT)
コード例 #3
0
ファイル: test_core.py プロジェクト: tomMoral/distributed
    def f():
        server = Server({'ping': pingpong})
        with pytest.raises(ValueError):
            server.port
        server.listen(8881)
        assert server.port == 8881
        assert server.address == ('tcp://%s:8881' % get_ip())

        for addr in ('127.0.0.1:8881', 'tcp://127.0.0.1:8881', server.address):
            comm = yield connect(addr)

            n = yield comm.write({'op': 'ping'})
            assert isinstance(n, int)
            assert 4 <= n <= 1000

            response = yield comm.read()
            assert response == b'pong'

            yield comm.write({'op': 'ping', 'close': True})
            response = yield comm.read()
            assert response == b'pong'

            yield comm.close()

        server.stop()
コード例 #4
0
ファイル: submit.py プロジェクト: amosonn/distributed
 def __init__(self, ip=get_ip(), local_dir=tempfile.mkdtemp(prefix='client-'),
              loop=None, **kwargs):
     self.ip = ip
     self.loop = loop or IOLoop.current()
     self.local_dir = local_dir
     handlers = {'upload_file': self.upload_file, 'execute': self.execute}
     super(RemoteClient, self).__init__(handlers, io_loop=self.loop, **kwargs)
コード例 #5
0
ファイル: dscheduler.py プロジェクト: sonlia/distributed
def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist):
    given_host = host
    host = host or get_ip()
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(center, ip=ip,
                          services={('http', http_port): HTTPScheduler})
    if center:
        loop.run_sync(scheduler.sync_center)
    scheduler.start(port)

    if _bokeh:
        try:
            import bokeh
            import distributed.bokeh
            hosts = ['%s:%d' % (h, bokeh_port) for h in
                     ['localhost', '127.0.0.1', ip, socket.gethostname(),
                      host] + list(bokeh_whitelist)]
            dirname = os.path.dirname(distributed.__file__)
            paths = [os.path.join(dirname, 'bokeh', name)
                     for name in ['status', 'tasks']]
            binname = sys.argv[0][:-len('dscheduler')] + 'bokeh'
            args = ([binname, 'serve'] + paths +
                    ['--log-level', 'warning',
                     '--check-unused-sessions=50',
                     '--unused-session-lifetime=1',
                     '--port', str(bokeh_port)] +
                     sum([['--host', host] for host in hosts], []))
            if show:
                args.append('--show')

            bokeh_options = {'host': host if given_host else '127.0.0.1',
                             'http-port': http_port,
                             'tcp-port': port,
                             'bokeh-port': bokeh_port}
            with open('.dask-web-ui.json', 'w') as f:
                json.dump(bokeh_options, f, indent=2)

            if sys.version_info[0] >= 3:
                from bokeh.command.bootstrap import main
                ctx = multiprocessing.get_context('spawn')
                bokeh_proc = ctx.Process(target=main, args=(args,))
                bokeh_proc.daemon = True
                bokeh_proc.start()
            else:
                bokeh_proc = subprocess.Popen(args)

            logger.info(" Bokeh UI at:  http://%s:%d/status/"
                        % (ip, bokeh_port))
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    loop.start()
    loop.close()
    scheduler.stop()
    bokeh_proc.terminate()

    logger.info("End scheduler at %s:%d", ip, port)
コード例 #6
0
def test_bokeh(loop):
    pytest.importorskip('bokeh')

    with popen(['dask-scheduler']) as proc:
        with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c:
            pass

        while True:
            line = proc.stderr.readline()
            if b'Bokeh UI' in line:
                break

        names = ['localhost', '127.0.0.1', get_ip()]
        if 'linux' in sys.platform:
            names.append(socket.gethostname())

        start = time()
        while True:
            try:
                for name in names:
                    response = requests.get('http://%s:8787/status/' % name)
                    assert response.ok
                break
            except Exception as f:
                print(f)
                sleep(0.1)
                assert time() < start + 10
    with pytest.raises(Exception):
        requests.get('http://127.0.0.1:8787/status/')
コード例 #7
0
def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist):
    given_host = host
    host = host or get_ip()
    if ':' in host and port == 8786:
        host, port = host.rsplit(':', 1)
        port = int(port)
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(center, ip=ip,
                          services={('http', http_port): HTTPScheduler},
                          loop=loop)
    scheduler.start(port)

    bokeh_proc = None
    if _bokeh:
        try:
            from distributed.bokeh.application import BokehWebInterface
            bokeh_proc = BokehWebInterface(host=host, http_port=http_port,
                    tcp_port=port, bokeh_port=bokeh_port,
                    bokeh_whitelist=bokeh_whitelist, show=show)
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    loop.start()
    loop.close()
    scheduler.stop()
    if bokeh_proc:
        bokeh_proc.close()

    logger.info("End scheduler at %s:%d", ip, port)
コード例 #8
0
ファイル: dworker.py プロジェクト: LiuXiaozeeee/distributed
def main(center, host, worker_port, http_port, nanny_port, nthreads, nprocs,
        no_nanny, name, port):
    if port:
        logger.info("--port is deprecated, use --nanny-port instead")
        assert not nanny_port
        nanny_port = port
    try:
        center_host, center_port = center.split(':')
        center_ip = socket.gethostbyname(center_host)
        center_port = int(center_port)
    except IndexError:
        logger.info("Usage:  dworker center_host:center_port")

    if nprocs > 1 and worker_port != 0:
        logger.error("Failed to launch worker.  You cannot use the --port argument when nprocs > 1.")
        exit(1)

    if nprocs > 1 and name:
        logger.error("Failed to launch worker.  You cannot use the --name argument when nprocs > 1.")
        exit(1)

    if not nthreads:
        nthreads = _ncores // nprocs

    services = {('http', http_port): HTTPWorker}

    loop = IOLoop.current()
    t = Worker if no_nanny else Nanny
    if host is not None:
        ip = socket.gethostbyname(host)
    else:
        # lookup the ip address of a local interface on a network that
        # reach the center
        ip = get_ip(center_ip, center_port)
    nannies = [t(center_ip, center_port, ncores=nthreads, ip=ip,
                 services=services, name=name, loop=loop,
                 worker_port=worker_port)
               for i in range(nprocs)]

    for nanny in nannies:
        nanny.start(nanny_port)
    try:
        loop.start()
    except KeyboardInterrupt:
        if not no_nanny:
            @gen.coroutine
            def stop():
                results = yield [nanny._kill() for nanny in nannies]
                raise gen.Return(results)

            results = IOLoop().run_sync(stop)

        for r, nanny in zip(results, nannies):
            if r == b'OK':
                logger.info("End nanny at %s:%d", nanny.ip, nanny.port)
            else:
                logger.error("Failed to cleanly kill worker %s:%d",
                             nanny.ip, nanny.port)
コード例 #9
0
ファイル: test_comms.py プロジェクト: tomMoral/distributed
def inproc_check():
    expected_ip = get_ip()
    expected_pid = os.getpid()

    def checker(loc):
        ip, pid, suffix = loc.split('/')
        assert ip == expected_ip
        assert int(pid) == expected_pid

    return checker
コード例 #10
0
ファイル: dcenter.py プロジェクト: canavandl/distributed
def main(host, port):
    if host is None:
        host = get_ip()

    logger.info("Start center at %s:%d", host, port)
    center = Center(host)
    center.listen(port)
    IOLoop.current().start()
    IOLoop.current().close()
    logger.info("\nEnd center at %s:%d", host, port)
コード例 #11
0
ファイル: submit.py プロジェクト: amosonn/distributed
def _remote(host, port, loop=IOLoop.current(), client=RemoteClient):
    host = host or get_ip()
    if ':' in host and port == 8788:
        host, port = host.rsplit(':', 1)
        port = int(port)
    ip = socket.gethostbyname(host)
    remote_client = client(ip=ip, loop=loop)
    remote_client.start(port=port)
    loop.start()
    loop.close()
    remote_client.stop()
    logger.info("End remote client at %s:%d", host, port)
コード例 #12
0
def main(host, port, http_port, bokeh_port, show, _bokeh,
         bokeh_whitelist, prefix, use_xheaders, pid_file):

    if pid_file:
        with open(pid_file, 'w') as f:
            f.write(str(os.getpid()))

        def del_pid_file():
            if os.path.exists(pid_file):
                os.remove(pid_file)
        atexit.register(del_pid_file)

    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
    limit = max(soft, hard // 2)
    resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))

    given_host = host
    host = host or get_ip()
    if ':' in host and port == 8786:
        host, port = host.rsplit(':', 1)
        port = int(port)
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    scheduler = Scheduler(ip=ip, loop=loop,
                          services={('http', http_port): HTTPScheduler})
    scheduler.start(port)

    bokeh_proc = None
    if _bokeh:
        try:
            from distributed.bokeh.application import BokehWebInterface
            bokeh_proc = BokehWebInterface(host=host, http_port=http_port,
                    tcp_port=port, bokeh_port=bokeh_port,
                    bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix,
                    use_xheaders=use_xheaders, quiet=False)
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    loop.start()
    loop.close()
    scheduler.stop()
    if bokeh_proc:
        bokeh_proc.close()

    logger.info("End scheduler at %s:%d", ip, port)
コード例 #13
0
ファイル: test_utils_test.py プロジェクト: dask/distributed
def test_wait_for_port():
    t1 = time()
    with pytest.raises(RuntimeError):
        wait_for_port((get_ip(), 9999), 0.5)
    t2 = time()
    assert t2 - t1 >= 0.5

    with _listen(0) as s1:
        t1 = time()
        wait_for_port(s1.getsockname())
        t2 = time()
        assert t2 - t1 <= 1.0

    with _listen(1) as s1:
        t1 = time()
        wait_for_port(s1.getsockname())
        t2 = time()
        assert t2 - t1 <= 2.0
コード例 #14
0
def test_bokeh(loop):
    pytest.importorskip('bokeh')

    with popen(['dask-scheduler']) as proc:
        with Executor('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as e:
            pass

        while True:
            line = proc.stderr.readline()
            if b'Bokeh UI' in line:
                break

        start = time()
        while True:
            try:
                for name in [socket.gethostname(), 'localhost', '127.0.0.1', get_ip()]:
                    response = requests.get('http://%s:8787/status/' % name)
                    assert response.ok
                break
            except Exception as f:
                print(f)
                sleep(0.1)
                assert time() < start + 10
コード例 #15
0
ファイル: dask_scheduler.py プロジェクト: dask/distributed
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file):

    if pid_file:
        with open(pid_file, "w") as f:
            f.write(str(os.getpid()))

        def del_pid_file():
            if os.path.exists(pid_file):
                os.remove(pid_file)

        atexit.register(del_pid_file)

    if sys.platform.startswith("linux"):
        import resource  # module fails importing on Windows

        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        limit = max(soft, hard // 2)
        resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))

    given_host = host
    host = host or get_ip()
    if ":" in host and port == 8786:
        host, port = host.rsplit(":", 1)
        port = int(port)
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    logger.info("-" * 47)

    services = {("http", http_port): HTTPScheduler}
    if _bokeh:
        from distributed.bokeh.scheduler import BokehScheduler

        services[("bokeh", 8788)] = BokehScheduler
    scheduler = Scheduler(ip=ip, loop=loop, services=services)
    scheduler.start(port)

    bokeh_proc = None
    if _bokeh:
        try:
            from distributed.bokeh.application import BokehWebInterface

            bokeh_proc = BokehWebInterface(
                host=host,
                http_port=http_port,
                tcp_port=port,
                bokeh_port=bokeh_port,
                bokeh_whitelist=bokeh_whitelist,
                show=show,
                prefix=prefix,
                use_xheaders=use_xheaders,
                quiet=False,
            )
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    logger.info("-" * 47)
    try:
        loop.start()
        loop.close()
    finally:
        scheduler.stop()
        if bokeh_proc:
            bokeh_proc.close()

        logger.info("End scheduler at %s:%d", ip, port)
コード例 #16
0
ファイル: test_comms.py プロジェクト: tomMoral/distributed
def test_get_address_host():
    f = get_address_host

    assert f('tcp://127.0.0.1:123') == '127.0.0.1'
    assert f('inproc://%s/%d/123' % (get_ip(), os.getpid())) == get_ip()
コード例 #17
0
ファイル: test_comms.py プロジェクト: tomMoral/distributed
from distributed.utils import get_ip, get_ipv6
from distributed.utils_test import (gen_test, requires_ipv6, has_ipv6,
                                    get_cert, get_server_ssl_context,
                                    get_client_ssl_context)
from distributed.utils_test import loop  # noqa: F401

from distributed.protocol import (to_serialize, Serialized, serialize,
                                  deserialize)

from distributed.comm import (tcp, inproc, connect, listen, CommClosedError,
                              parse_address, parse_host_port,
                              unparse_host_port, resolve_address,
                              get_address_host, get_local_address_for)


EXTERNAL_IP4 = get_ip()
if has_ipv6():
    with warnings.catch_warnings(record=True):
        warnings.simplefilter('always')
        EXTERNAL_IP6 = get_ipv6()


ca_file = get_cert('tls-ca-cert.pem')

# The Subject field of our test certs
cert_subject = (
    (('countryName', 'XY'),),
    (('localityName', 'Dask-distributed'),),
    (('organizationName', 'Dask'),),
    (('commonName', 'localhost'),)
)
コード例 #18
0
ファイル: test_cli_utils.py プロジェクト: slawler/distributed
from __future__ import print_function, division, absolute_import

import pytest

pytest.importorskip('requests')

from distributed.cli.utils import uri_from_host_port
from distributed.utils import get_ip

external_ip = get_ip()


def test_uri_from_host_port():
    f = uri_from_host_port

    assert f('', 456, None) == 'tcp://:456'
    assert f('', 456, 123) == 'tcp://:456'
    assert f('', None, 123) == 'tcp://:123'
    assert f('', None, 0) == 'tcp://'
    assert f('', 0, 123) == 'tcp://'

    assert f('localhost', 456, None) == 'tcp://localhost:456'
    assert f('localhost', 456, 123) == 'tcp://localhost:456'
    assert f('localhost', None, 123) == 'tcp://localhost:123'
    assert f('localhost', None, 0) == 'tcp://localhost'

    assert f('192.168.1.2', 456, None) == 'tcp://192.168.1.2:456'
    assert f('192.168.1.2', 456, 123) == 'tcp://192.168.1.2:456'
    assert f('192.168.1.2', None, 123) == 'tcp://192.168.1.2:123'
    assert f('192.168.1.2', None, 0) == 'tcp://192.168.1.2'
コード例 #19
0
ファイル: test_core.py プロジェクト: tomMoral/distributed
def test_server_listen():
    """
    Test various Server.listen() arguments and their effect.
    """

    @contextmanager
    def listen_on(cls, *args, **kwargs):
        server = cls({})
        server.listen(*args, **kwargs)
        try:
            yield server
        finally:
            server.stop()

    # Note server.address is the concrete, contactable address

    with listen_on(Server, 7800) as server:
        assert server.port == 7800
        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
        yield assert_can_connect(server.address)
        yield assert_can_connect_from_everywhere_4_6(server.port)

    with listen_on(Server) as server:
        assert server.port > 0
        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
        yield assert_can_connect(server.address)
        yield assert_can_connect_from_everywhere_4_6(server.port)

    with listen_on(MyServer) as server:
        assert server.port == MyServer.default_port
        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
        yield assert_can_connect(server.address)
        yield assert_can_connect_from_everywhere_4_6(server.port)

    with listen_on(Server, ('', 7801)) as server:
        assert server.port == 7801
        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
        yield assert_can_connect(server.address)
        yield assert_can_connect_from_everywhere_4_6(server.port)

    with listen_on(Server, 'tcp://:7802') as server:
        assert server.port == 7802
        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
        yield assert_can_connect(server.address)
        yield assert_can_connect_from_everywhere_4_6(server.port)

    # Only IPv4

    with listen_on(Server, ('0.0.0.0', 7810)) as server:
        assert server.port == 7810
        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
        yield assert_can_connect(server.address)
        yield assert_can_connect_from_everywhere_4(server.port)

    with listen_on(Server, ('127.0.0.1', 7811)) as server:
        assert server.port == 7811
        assert server.address == 'tcp://127.0.0.1:%d' % server.port
        yield assert_can_connect(server.address)
        yield assert_can_connect_locally_4(server.port)

    with listen_on(Server, 'tcp://127.0.0.1:7812') as server:
        assert server.port == 7812
        assert server.address == 'tcp://127.0.0.1:%d' % server.port
        yield assert_can_connect(server.address)
        yield assert_can_connect_locally_4(server.port)

    # Only IPv6

    if has_ipv6():
        with listen_on(Server, ('::', 7813)) as server:
            assert server.port == 7813
            assert server.address == 'tcp://[%s]:%d' % (EXTERNAL_IP6, server.port)
            yield assert_can_connect(server.address)
            yield assert_can_connect_from_everywhere_6(server.port)

        with listen_on(Server, ('::1', 7814)) as server:
            assert server.port == 7814
            assert server.address == 'tcp://[::1]:%d' % server.port
            yield assert_can_connect(server.address)
            yield assert_can_connect_locally_6(server.port)

        with listen_on(Server, 'tcp://[::1]:7815') as server:
            assert server.port == 7815
            assert server.address == 'tcp://[::1]:%d' % server.port
            yield assert_can_connect(server.address)
            yield assert_can_connect_locally_6(server.port)

    # TLS

    sec = tls_security()
    with listen_on(Server, 'tls://',
                   listen_args=sec.get_listen_args('scheduler')) as server:
        assert server.address.startswith('tls://')
        yield assert_can_connect(server.address,
                                 connection_args=sec.get_connection_args('client'))

    # InProc

    with listen_on(Server, 'inproc://') as server:
        inproc_addr1 = server.address
        assert inproc_addr1.startswith('inproc://%s/%d/' % (get_ip(), os.getpid()))
        yield assert_can_connect(inproc_addr1)

        with listen_on(Server, 'inproc://') as server2:
            inproc_addr2 = server2.address
            assert inproc_addr2.startswith('inproc://%s/%d/' % (get_ip(), os.getpid()))
            yield assert_can_connect(inproc_addr2)

        yield assert_can_connect(inproc_addr1)
        yield assert_cannot_connect(inproc_addr2)
コード例 #20
0
ファイル: dask_worker.py プロジェクト: amosonn/distributed
def main(scheduler, host, worker_port, http_port, nanny_port, nthreads, nprocs,
        no_nanny, name, memory_limit, pid_file, temp_filename):
    if no_nanny:
        port = worker_port
    else:
        port = nanny_port

    try:
        scheduler_host, scheduler_port = scheduler.split(':')
        scheduler_ip = socket.gethostbyname(scheduler_host)
        scheduler_port = int(scheduler_port)
    except IndexError:
        logger.info("Usage:  dask-worker scheduler_host:scheduler_port")

    if nprocs > 1 and worker_port != 0:
        logger.error("Failed to launch worker.  You cannot use the --port argument when nprocs > 1.")
        exit(1)

    if nprocs > 1 and name:
        logger.error("Failed to launch worker.  You cannot use the --name argument when nprocs > 1.")
        exit(1)

    if not nthreads:
        nthreads = _ncores // nprocs

    if pid_file:
        with open(pid_file, 'w') as f:
            f.write(str(os.getpid()))

        def del_pid_file():
            if os.path.exists(pid_file):
                os.remove(pid_file)
        atexit.register(del_pid_file)

    services = {('http', http_port): HTTPWorker}

    loop = IOLoop.current()

    if memory_limit == 'auto':
        import psutil
        memory_limit = psutil.virtual_memory().total * 0.60

    if memory_limit:
        memory_limit = float(memory_limit)
        if memory_limit < 1.0:
            import psutil
            memory_limit = psutil.virtual_memory().total * memory_limit
        memory_limit /= nprocs
        memory_limit = int(memory_limit)

    if no_nanny:
        kwargs = {}
        if nanny_port:
            kwargs['service_ports'] = {'nanny': nanny_port}
        t = Worker
    else:
        kwargs = {'worker_port': worker_port}
        t = Nanny

    if host is not None:
        ip = socket.gethostbyname(host)
    else:
        # lookup the ip address of a local interface on a network that
        # reach the scheduler
        ip = get_ip(scheduler_ip, scheduler_port)
    nannies = [t(scheduler_ip, scheduler_port, ncores=nthreads, ip=ip,
                 services=services, name=name, loop=loop,
                 memory_limit=memory_limit, **kwargs)
               for i in range(nprocs)]

    for nanny in nannies:
        nanny.start(port)
        if t is Nanny:
            global_nannies.append(nanny)

    if temp_filename:
        @gen.coroutine
        def f():
            while nannies[0].status != 'running':
                yield gen.sleep(0.01)
            import json
            msg = {'port': nannies[0].port,
                   'local_directory': nannies[0].local_dir}
            with open(temp_filename, 'w') as f:
                json.dump(msg, f)
        loop.add_callback(f)

    loop.start()
    logger.info("End worker")
    loop.close()

    loop2 = IOLoop()

    @gen.coroutine
    def f():
        scheduler = rpc(ip=nannies[0].scheduler.ip,
                        port=nannies[0].scheduler.port)
        if not no_nanny:
            yield gen.with_timeout(timedelta(seconds=2),
                    All([scheduler.unregister(address=n.worker_address, close=True)
                        for n in nannies if n.process and n.worker_port]), io_loop=loop2)

    loop2.run_sync(f)

    if not no_nanny:
        for n in nannies:
            n.process.terminate()

    if not no_nanny:
        start = time()
        while (any(n.process.poll() is None for n in nannies)
                and time() < start + 1):
            sleep(0.1)

    for nanny in nannies:
        nanny.stop()
コード例 #21
0
ファイル: dask_worker.py プロジェクト: dask/distributed
def main(scheduler, host, worker_port, http_port, nanny_port, nthreads, nprocs,
        nanny, name, memory_limit, pid_file, temp_filename, reconnect,
        resources, bokeh, bokeh_port):
    if nanny:
        port = nanny_port
    else:
        port = worker_port

    try:
        scheduler_host, scheduler_port = scheduler.split(':')
        scheduler_ip = socket.gethostbyname(scheduler_host)
        scheduler_port = int(scheduler_port)
    except IndexError:
        logger.info("Usage:  dask-worker scheduler_host:scheduler_port")

    if nprocs > 1 and worker_port != 0:
        logger.error("Failed to launch worker.  You cannot use the --port argument when nprocs > 1.")
        exit(1)

    if nprocs > 1 and name:
        logger.error("Failed to launch worker.  You cannot use the --name argument when nprocs > 1.")
        exit(1)

    if not nthreads:
        nthreads = _ncores // nprocs

    if pid_file:
        with open(pid_file, 'w') as f:
            f.write(str(os.getpid()))

        def del_pid_file():
            if os.path.exists(pid_file):
                os.remove(pid_file)
        atexit.register(del_pid_file)

    services = {('http', http_port): HTTPWorker}

    if bokeh:
        try:
            from distributed.bokeh.worker import BokehWorker
        except ImportError:
            pass
        else:
            services[('bokeh', bokeh_port)] = BokehWorker

    if resources:
        resources = resources.replace(',', ' ').split()
        resources = dict(pair.split('=') for pair in resources)
        resources = valmap(float, resources)
    else:
        resources = None

    loop = IOLoop.current()

    if nanny:
        kwargs = {'worker_port': worker_port}
        t = Nanny
    else:
        kwargs = {}
        if nanny_port:
            kwargs['service_ports'] = {'nanny': nanny_port}
        t = Worker

    if host is not None:
        ip = socket.gethostbyname(host)
    else:
        # lookup the ip address of a local interface on a network that
        # reach the scheduler
        ip = get_ip(scheduler_ip, scheduler_port)
    nannies = [t(scheduler_ip, scheduler_port, ncores=nthreads, ip=ip,
                 services=services, name=name, loop=loop, resources=resources,
                 memory_limit=memory_limit, reconnect=reconnect, **kwargs)
               for i in range(nprocs)]

    for n in nannies:
        n.start(port)
        if t is Nanny:
            global_nannies.append(n)

    if temp_filename:
        @gen.coroutine
        def f():
            while nannies[0].status != 'running':
                yield gen.sleep(0.01)
            import json
            msg = {'port': nannies[0].port,
                   'local_directory': nannies[0].local_dir}
            with open(temp_filename, 'w') as f:
                json.dump(msg, f)
        loop.add_callback(f)

    @gen.coroutine
    def run():
        while all(n.status != 'closed' for n in nannies):
            yield gen.sleep(0.2)

    try:
        loop.run_sync(run)
    except (KeyboardInterrupt, TimeoutError):
        pass
    finally:
        logger.info("End worker")
        loop.close()

    loop2 = IOLoop()

    @gen.coroutine
    def f():
        scheduler = rpc(ip=nannies[0].scheduler.ip,
                        port=nannies[0].scheduler.port)
        if nanny:
            yield gen.with_timeout(timedelta(seconds=2),
                    All([scheduler.unregister(address=n.worker_address, close=True)
                        for n in nannies if n.process and n.worker_port]), io_loop=loop2)

    loop2.run_sync(f)

    if nanny:
        for n in nannies:
            if isalive(n.process):
                n.process.terminate()

    if nanny:
        start = time()
        while (any(isalive(n.process) for n in nannies)
                and time() < start + 1):
            sleep(0.1)

    for nanny in nannies:
        nanny.stop()
コード例 #22
0
ファイル: dworker.py プロジェクト: canavandl/distributed
import logging
from sys import argv, exit
import socket

import click
from distributed import Nanny, Worker, sync
from distributed.utils import get_ip
from distributed.worker import _ncores
from distributed.http import HTTPWorker
from distributed.cli.utils import check_python_3
from tornado.ioloop import IOLoop
from tornado import gen

logger = logging.getLogger('distributed.dworker')

ip = get_ip()


@click.command()
@click.argument('center', type=str)
@click.option('--port', type=int, default=0,
              help="Serving port, defaults to randomly assigned")
@click.option('--host', type=str, default=None,
              help="Serving host defaults to %s" % ip)
@click.option('--nthreads', type=int, default=0,
              help="Number of threads per process. Defaults to number of cores")
@click.option('--nprocs', type=int, default=1,
              help="Number of worker processes.  Defaults to one.")
@click.option('--no-nanny', is_flag=True)
def main(center, host, port, nthreads, nprocs, no_nanny):
    try:
コード例 #23
0
from __future__ import print_function, division, absolute_import

import pytest
pytest.importorskip('requests')

from distributed.cli.utils import uri_from_host_port
from distributed.utils import get_ip


external_ip = get_ip()


def test_uri_from_host_port():
    f = uri_from_host_port

    assert f('', 456, None) == 'tcp://:456'
    assert f('', 456, 123) == 'tcp://:456'
    assert f('', None, 123) == 'tcp://:123'
    assert f('', None, 0) == 'tcp://'
    assert f('', 0, 123) == 'tcp://'

    assert f('localhost', 456, None) == 'tcp://localhost:456'
    assert f('localhost', 456, 123) == 'tcp://localhost:456'
    assert f('localhost', None, 123) == 'tcp://localhost:123'
    assert f('localhost', None, 0) == 'tcp://localhost'

    assert f('192.168.1.2', 456, None) == 'tcp://192.168.1.2:456'
    assert f('192.168.1.2', 456, 123) == 'tcp://192.168.1.2:456'
    assert f('192.168.1.2', None, 123) == 'tcp://192.168.1.2:123'
    assert f('192.168.1.2', None, 0) == 'tcp://192.168.1.2'
コード例 #24
0
ファイル: dask_worker.py プロジェクト: HugoTian/distributed
def main(scheduler, host, worker_port, http_port, nanny_port, nthreads, nprocs,
        no_nanny, name, port):
    if port:
        logger.info("--port is deprecated, use --nanny-port instead")
        assert not nanny_port
        nanny_port = port
    try:
        scheduler_host, scheduler_port = scheduler.split(':')
        scheduler_ip = socket.gethostbyname(scheduler_host)
        scheduler_port = int(scheduler_port)
    except IndexError:
        logger.info("Usage:  dask-worker scheduler_host:scheduler_port")

    if nprocs > 1 and worker_port != 0:
        logger.error("Failed to launch worker.  You cannot use the --port argument when nprocs > 1.")
        exit(1)

    if nprocs > 1 and name:
        logger.error("Failed to launch worker.  You cannot use the --name argument when nprocs > 1.")
        exit(1)

    if not nthreads:
        nthreads = _ncores // nprocs

    services = {('http', http_port): HTTPWorker}

    loop = IOLoop.current()
    t = Worker if no_nanny else Nanny
    if host is not None:
        ip = socket.gethostbyname(host)
    else:
        # lookup the ip address of a local interface on a network that
        # reach the scheduler
        ip = get_ip(scheduler_ip, scheduler_port)
    nannies = [t(scheduler_ip, scheduler_port, ncores=nthreads, ip=ip,
                 services=services, name=name, loop=loop,
                 worker_port=worker_port)
               for i in range(nprocs)]

    for nanny in nannies:
        nanny.start(nanny_port)

    loop.start()
    logger.info("End worker")
    loop.close()

    loop2 = IOLoop()

    @gen.coroutine
    def f():
        scheduler = rpc(ip=nannies[0].center.ip, port=nannies[0].center.port)
        yield gen.with_timeout(timedelta(seconds=2),
                All([scheduler.unregister(address=n.worker_address, close=True)
                    for n in nannies if n.process]), io_loop=loop2)

    loop2.run_sync(f)

    for n in nannies:
        n.process.terminate()

    for n in nannies:
        n.process.join(timeout=1)

    for nanny in nannies:
        nanny.stop()
コード例 #25
0
ファイル: dask_scheduler.py プロジェクト: zegami/distributed
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist,
         prefix, use_xheaders, pid_file):

    if pid_file:
        with open(pid_file, 'w') as f:
            f.write(str(os.getpid()))

        def del_pid_file():
            if os.path.exists(pid_file):
                os.remove(pid_file)

        atexit.register(del_pid_file)

    if sys.platform.startswith('linux'):
        import resource  # module fails importing on Windows
        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        limit = max(soft, hard // 2)
        resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))

    given_host = host
    host = host or get_ip()
    if ':' in host and port == 8786:
        host, port = host.rsplit(':', 1)
        port = int(port)
    ip = socket.gethostbyname(host)
    loop = IOLoop.current()
    logger.info('-' * 47)

    services = {('http', http_port): HTTPScheduler}
    if _bokeh:
        from distributed.bokeh.scheduler import BokehScheduler
        services[('bokeh', 8788)] = BokehScheduler
    scheduler = Scheduler(ip=ip, loop=loop, services=services)
    scheduler.start(port)

    bokeh_proc = None
    if _bokeh:
        try:
            from distributed.bokeh.application import BokehWebInterface
            bokeh_proc = BokehWebInterface(host=host,
                                           http_port=http_port,
                                           tcp_port=port,
                                           bokeh_port=bokeh_port,
                                           bokeh_whitelist=bokeh_whitelist,
                                           show=show,
                                           prefix=prefix,
                                           use_xheaders=use_xheaders,
                                           quiet=False)
        except ImportError:
            logger.info("Please install Bokeh to get Web UI")
        except Exception as e:
            logger.warn("Could not start Bokeh web UI", exc_info=True)

    logger.info('-' * 47)
    try:
        loop.start()
        loop.close()
    finally:
        scheduler.stop()
        if bokeh_proc:
            bokeh_proc.close()

        logger.info("End scheduler at %s:%d", ip, port)