def test_automatic_startup(image_name, loop, ns):
test_yaml = {
"kind": "Pod",
"metadata": {
"labels": {
"foo": "bar",
}
},
"spec": {
"containers": [{
"args": [
"dask-worker", "$(DASK_SCHEDULER_ADDRESS)", "--nthreads",
"1"
],
"image":
image_name,
"name":
"dask-worker"
}]
}
}
with tmpfile(extension='yaml') as fn:
with open(fn, mode='w') as f:
yaml.dump(test_yaml, f)
with set_config(**{'kubernetes-worker-template-path': fn}):
with KubeCluster(loop=loop, namespace=ns) as cluster:
assert cluster.pod_template.metadata.labels['foo'] == 'bar'
def test_tick_logging(s, a, b):
pytest.importorskip('crick')
with set_config(**{'tick-maximum-delay': 10}):
with captured_logger('distributed.core') as sio:
yield gen.sleep(0.1)
text = sio.getvalue()
assert "unresponsive" in text
def test_diagnostics_link_env_variable(pod_spec, loop, ns):
pytest.importorskip('bokeh')
with set_config(**{'diagnostics-link': 'foo-{USER}-{port}'}):
with KubeCluster(pod_spec, loop=loop, namespace=ns) as cluster:
port = cluster.scheduler.services['bokeh'].port
cluster._ipython_display_()
box = cluster._cached_widget
assert 'foo-' + getpass.getuser() + '-' + str(port) in str(box)
def test_non_existent_worker(c, s):
with set_config({'connect-timeout': '100ms'}):
s.add_worker(address='127.0.0.1:5738',
ncores=2,
nbytes={},
host_info={})
futures = c.map(inc, range(10))
yield gen.sleep(0.300)
assert not s.workers
assert all(ts.state == 'no-worker' for ts in s.tasks.values())
def test_mising_data_errant_worker(c, s, w1, w2, w3):
with set_config({'connect-timeout': '1s'}):
np = pytest.importorskip('numpy')
x = c.submit(np.random.random, 10000000, workers=w1.address)
yield wait(x)
yield c.replicate(x, workers=[w1.address, w2.address])
y = c.submit(len, x, workers=w3.address)
while not w3.tasks:
yield gen.sleep(0.001)
w1._close()
yield wait(y)
def test_coerce_address():
with set_config({'connect-timeout': '100ms'}):
s = Scheduler(validate=True)
s.start(0)
print("scheduler:", s.address, s.listen_address)
a = Worker(s.ip, s.port, name='alice')
b = Worker(s.ip, s.port, name=123)
c = Worker('127.0.0.1', s.port, name='charlie')
yield [a._start(), b._start(), c._start()]
assert s.coerce_address('127.0.0.1:8000') == 'tcp://127.0.0.1:8000'
assert s.coerce_address('[::1]:8000') == 'tcp://[::1]:8000'
assert s.coerce_address(
'tcp://127.0.0.1:8000') == 'tcp://127.0.0.1:8000'
assert s.coerce_address('tcp://[::1]:8000') == 'tcp://[::1]:8000'
assert s.coerce_address('localhost:8000') in ('tcp://127.0.0.1:8000',
'tcp://[::1]:8000')
assert s.coerce_address(u'localhost:8000') in ('tcp://127.0.0.1:8000',
'tcp://[::1]:8000')
assert s.coerce_address(a.address) == a.address
# Aliases
assert s.coerce_address('alice') == a.address
assert s.coerce_address(123) == b.address
assert s.coerce_address('charlie') == c.address
assert s.coerce_hostname('127.0.0.1') == '127.0.0.1'
assert s.coerce_hostname('alice') == a.ip
assert s.coerce_hostname(123) == b.ip
assert s.coerce_hostname('charlie') == c.ip
assert s.coerce_hostname('jimmy') == 'jimmy'
assert s.coerce_address('zzzt:8000',
resolve=False) == 'tcp://zzzt:8000'
yield s.close()
yield [w._close() for w in [a, b, c]]
def test_set_config():
assert 'foo' not in config
with set_config(foo=1):
assert config['foo'] == 1
assert 'foo' not in config
def test_dask_worker_name_env_variable(pod_spec, loop, ns):
with set_config(**{'kubernetes-worker-name': 'foo-{USER}-{uuid}'}):
with KubeCluster(pod_spec, loop=loop, namespace=ns) as cluster:
assert 'foo-' + getpass.getuser() in cluster.name