def test_automatic_startup(image_name, loop, ns): test_yaml = { "kind": "Pod", "metadata": { "labels": { "foo": "bar", } }, "spec": { "containers": [{ "args": [ "dask-worker", "$(DASK_SCHEDULER_ADDRESS)", "--nthreads", "1" ], "image": image_name, "name": "dask-worker" }] } } with tmpfile(extension='yaml') as fn: with open(fn, mode='w') as f: yaml.dump(test_yaml, f) with set_config(**{'kubernetes-worker-template-path': fn}): with KubeCluster(loop=loop, namespace=ns) as cluster: assert cluster.pod_template.metadata.labels['foo'] == 'bar'
def test_tick_logging(s, a, b): pytest.importorskip('crick') with set_config(**{'tick-maximum-delay': 10}): with captured_logger('distributed.core') as sio: yield gen.sleep(0.1) text = sio.getvalue() assert "unresponsive" in text
def test_diagnostics_link_env_variable(pod_spec, loop, ns): pytest.importorskip('bokeh') with set_config(**{'diagnostics-link': 'foo-{USER}-{port}'}): with KubeCluster(pod_spec, loop=loop, namespace=ns) as cluster: port = cluster.scheduler.services['bokeh'].port cluster._ipython_display_() box = cluster._cached_widget assert 'foo-' + getpass.getuser() + '-' + str(port) in str(box)
def test_non_existent_worker(c, s): with set_config({'connect-timeout': '100ms'}): s.add_worker(address='127.0.0.1:5738', ncores=2, nbytes={}, host_info={}) futures = c.map(inc, range(10)) yield gen.sleep(0.300) assert not s.workers assert all(ts.state == 'no-worker' for ts in s.tasks.values())
def test_mising_data_errant_worker(c, s, w1, w2, w3): with set_config({'connect-timeout': '1s'}): np = pytest.importorskip('numpy') x = c.submit(np.random.random, 10000000, workers=w1.address) yield wait(x) yield c.replicate(x, workers=[w1.address, w2.address]) y = c.submit(len, x, workers=w3.address) while not w3.tasks: yield gen.sleep(0.001) w1._close() yield wait(y)
def test_coerce_address(): with set_config({'connect-timeout': '100ms'}): s = Scheduler(validate=True) s.start(0) print("scheduler:", s.address, s.listen_address) a = Worker(s.ip, s.port, name='alice') b = Worker(s.ip, s.port, name=123) c = Worker('127.0.0.1', s.port, name='charlie') yield [a._start(), b._start(), c._start()] assert s.coerce_address('127.0.0.1:8000') == 'tcp://127.0.0.1:8000' assert s.coerce_address('[::1]:8000') == 'tcp://[::1]:8000' assert s.coerce_address( 'tcp://127.0.0.1:8000') == 'tcp://127.0.0.1:8000' assert s.coerce_address('tcp://[::1]:8000') == 'tcp://[::1]:8000' assert s.coerce_address('localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000') assert s.coerce_address(u'localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000') assert s.coerce_address(a.address) == a.address # Aliases assert s.coerce_address('alice') == a.address assert s.coerce_address(123) == b.address assert s.coerce_address('charlie') == c.address assert s.coerce_hostname('127.0.0.1') == '127.0.0.1' assert s.coerce_hostname('alice') == a.ip assert s.coerce_hostname(123) == b.ip assert s.coerce_hostname('charlie') == c.ip assert s.coerce_hostname('jimmy') == 'jimmy' assert s.coerce_address('zzzt:8000', resolve=False) == 'tcp://zzzt:8000' yield s.close() yield [w._close() for w in [a, b, c]]
def test_set_config(): assert 'foo' not in config with set_config(foo=1): assert config['foo'] == 1 assert 'foo' not in config
def test_dask_worker_name_env_variable(pod_spec, loop, ns): with set_config(**{'kubernetes-worker-name': 'foo-{USER}-{uuid}'}): with KubeCluster(pod_spec, loop=loop, namespace=ns) as cluster: assert 'foo-' + getpass.getuser() in cluster.name