def test_http_client_proxy_true(monkeypatch):
http_proxy = "http://*****:*****@host:80/path"
proxy_sol = yarl.URL("http://host:80/path")
proxy_auth_sol = aiohttp.BasicAuth("alice", "password")
with dask.config.set(gateway__http_client__proxy=True):
with monkeypatch.context() as m:
for k in ["http_proxy", "https_proxy"]:
m.delenv(k, raising=False)
m.delenv(k.upper(), raising=False)
with m.context() as m2:
m2.setenv("http_proxy", http_proxy)
# Properly inferred from environment
g = Gateway("http://myhost:80")
assert g._request_kwargs["proxy"] == proxy_sol
assert g._request_kwargs["proxy_auth"] == proxy_auth_sol
# No HTTPS proxy set
g = Gateway("https://myhost:80")
assert g._request_kwargs == {"proxy": None, "proxy_auth": None}
# No HTTP proxy set
g = Gateway("http://myhost:80")
assert g._request_kwargs == {"proxy": None, "proxy_auth": None}
def test_gateway_proxy_address_infer_port():
with dask.config.set(gateway__proxy_address=None):
g = Gateway("http://localhost")
assert g.proxy_address == "gateway://localhost:80"
g = Gateway("https://localhost")
assert g.proxy_address == "gateway://localhost:443"
def test_client_init():
config = {
"gateway": {
"address": "http://127.0.0.1:8888",
"auth": {"type": "basic", "kwargs": {"username": "******"}},
}
}
with dask.config.set(config):
# Defaults
gateway = Gateway()
assert gateway.address == "http://127.0.0.1:8888"
assert gateway._auth.username == "bruce"
# Address override
gateway = Gateway(address="http://127.0.0.1:9999")
assert gateway.address == "http://127.0.0.1:9999"
assert gateway._auth.username == "bruce"
# Auth override
gateway = Gateway(auth="kerberos")
assert gateway.address == "http://127.0.0.1:8888"
assert isinstance(gateway._auth, KerberosAuth)
config = {"gateway": {"address": None, "auth": {"type": "basic", "kwargs": {}}}}
with dask.config.set(config):
# No address provided
with pytest.raises(ValueError):
Gateway()
def test_gateway_addresses_template_environment_vars(monkeypatch):
monkeypatch.setenv("TEST", "foobar")
with dask.config.set(gateway__address="http://{TEST}:80",
gateway__proxy_address=8785):
g = Gateway()
assert g.address == "http://foobar:80"
assert g.proxy_address == "gateway://foobar:8785"
with dask.config.set(gateway__proxy_address="gateway://{TEST}:8787"):
g = Gateway("http://test.com")
assert g.address == "http://test.com"
assert g.proxy_address == "gateway://foobar:8787"
def test_http_client_proxy_false(monkeypatch):
with dask.config.set(gateway__http_client__proxy=False):
monkeypatch.setenv("http_proxy", "http://*****:*****@host:80/path")
# http_proxy environment variable ignored
g = Gateway("http://myhost:80")
assert g._request_kwargs == {"proxy": None, "proxy_auth": None}
async def test_GatewayCluster_shutdown_on_close(tmpdir):
async with temp_gateway(cluster_manager_class=InProcessClusterManager,
temp_dir=str(tmpdir)) as gateway_proc:
def test():
cluster = GatewayCluster(
address=gateway_proc.public_urls.connect_url,
proxy_address=gateway_proc.gateway_urls.connect_url,
)
assert cluster.shutdown_on_close
assert cluster in GatewayCluster._instances
loop = get_running_loop()
await loop.run_in_executor(None, test)
assert len(GatewayCluster._instances) == 0
async with Gateway(
address=gateway_proc.public_urls.connect_url,
proxy_address=gateway_proc.gateway_urls.connect_url,
asynchronous=True,
) as gateway:
# No cluster running
clusters = await gateway.list_clusters()
assert not clusters
async def test_create_cluster_with_GatewayCluster_constructor(tmpdir):
async with temp_gateway(cluster_manager_class=InProcessClusterManager,
temp_dir=str(tmpdir)) as gateway_proc:
async with GatewayCluster(
address=gateway_proc.public_urls.connect_url,
proxy_address=gateway_proc.gateway_urls.connect_url,
asynchronous=True,
) as cluster:
# Cluster is now present in list
clusters = await cluster.gateway.list_clusters()
assert len(clusters)
assert clusters[0].name == cluster.name
await cluster.scale(1)
with cluster.get_client(set_as_default=False) as client:
res = await client.submit(lambda x: x + 1, 1)
assert res == 2
assert cluster.status == "closed"
async with Gateway(
address=gateway_proc.public_urls.connect_url,
proxy_address=gateway_proc.gateway_urls.connect_url,
asynchronous=True,
) as gateway:
# No cluster running
clusters = await gateway.list_clusters()
assert not clusters
async def test_client_reprs(tmpdir):
async with temp_gateway(
cluster_manager_class=InProcessClusterManager,
temp_dir=str(tmpdir.join("dask-gateway")),
) as gateway_proc:
async with Gateway(
address=gateway_proc.public_urls.connect_url,
proxy_address=gateway_proc.gateway_urls.connect_url,
asynchronous=True,
) as gateway:
cluster = await gateway.new_cluster()
# Plain repr
assert cluster.name in repr(cluster)
# HTML repr with dashboard
cluster.dashboard_link = "%s/gateway/clusters/%s/status" % (
gateway_proc.public_url,
cluster.name,
)
assert cluster.name in cluster._repr_html_()
assert cluster.dashboard_link in cluster._repr_html_()
# HTML repr with no dashboard
cluster.dashboard_link = None
assert "Not Available" in cluster._repr_html_()
def test_client_init():
config = {
"gateway": {
"address": "http://127.0.0.1:8888",
"public-address": None,
"proxy-address": 8786,
"auth": {
"type": "basic",
"kwargs": {
"username": "******"
}
},
}
}
with dask.config.set(config):
# Defaults
gateway = Gateway()
assert gateway.address == "http://127.0.0.1:8888"
assert gateway._public_address == "http://127.0.0.1:8888"
assert gateway.proxy_address == "gateway://127.0.0.1:8786"
assert gateway.auth.username == "bruce"
# Address override
gateway = Gateway(address="http://127.0.0.1:9999")
assert gateway.address == "http://127.0.0.1:9999"
# Proxy address override
gateway = Gateway(proxy_address="gateway://123.4.5.6:9999")
assert gateway.proxy_address == "gateway://123.4.5.6:9999"
# Auth override
gateway = Gateway(auth="kerberos")
assert isinstance(gateway.auth, KerberosAuth)
config = {
"gateway": {
"address": None,
"public-address": None,
"proxy-address": 8786,
"auth": {
"type": "basic",
"kwargs": {}
},
}
}
with dask.config.set(config):
# No address provided
with pytest.raises(ValueError):
Gateway()
config["gateway"]["address"] = "http://127.0.0.1:8888"
config["gateway"]["proxy-address"] = None
with dask.config.set(config):
# No proxy-address provided
with pytest.raises(ValueError):
Gateway()
def test_http_client_proxy_explicit(monkeypatch):
http_proxy = "http://*****:*****@host:80/path"
proxy_sol = yarl.URL("http://*****:*****@otherhost:90/path")
# Loaded from config, not environment variables
for scheme in ["http", "https"]:
g = Gateway(f"{scheme}://myhost:80")
assert g._request_kwargs["proxy"] == proxy_sol
assert g._request_kwargs["proxy_auth"] == proxy_auth_sol
async def test_GatewayCluster_cleanup_atexit(tmpdir):
async with temp_gateway(
cluster_manager_class=InProcessClusterManager,
temp_dir=str(tmpdir.join("dask-gateway")),
) as gateway_proc:
def test():
return GatewayCluster(
address=gateway_proc.public_urls.connect_url,
proxy_address=gateway_proc.gateway_urls.connect_url,
)
loop = get_running_loop()
cluster = await loop.run_in_executor(None, test)
assert len(GatewayCluster._instances) == 1
def test_cleanup():
# No warnings raised by cleanup function
with pytest.warns(None) as rec:
cleanup_lingering_clusters()
for r in rec:
assert not issubclass(r.category, UserWarning)
# Cluster is now closed
assert cluster.status == "closed"
# No harm in double running
with pytest.warns(None) as rec:
cleanup_lingering_clusters()
for r in rec:
assert not issubclass(r.category, UserWarning)
await loop.run_in_executor(None, test_cleanup)
async with Gateway(
address=gateway_proc.public_urls.connect_url,
proxy_address=gateway_proc.gateway_urls.connect_url,
asynchronous=True,
) as gateway:
# No cluster running
clusters = await gateway.list_clusters()
assert not clusters
async def test_dashboard_link_from_public_address():
pytest.importorskip("bokeh")
async with temp_gateway() as g:
with dask.config.set(
gateway__address=g.address,
gateway__public_address="/services/dask-gateway/",
gateway__proxy_address=g.proxy_address,
):
async with Gateway(asynchronous=True) as gateway:
assert gateway._public_address == "/services/dask-gateway"
cluster = await gateway.new_cluster()
sol = "/services/dask-gateway/clusters/%s/status" % cluster.name
assert cluster.dashboard_link == sol
clusters = await gateway.list_clusters()
for c in clusters:
assert c.dashboard_link.startswith("/services/dask-gateway")
async def test_dashboard_link_from_public_address(tmpdir):
pytest.importorskip("bokeh")
async with temp_gateway(cluster_manager_class=InProcessClusterManager,
temp_dir=str(tmpdir)) as gateway_proc:
with dask.config.set(
gateway__address=gateway_proc.public_urls.connect_url,
gateway__public_address="/services/dask-gateway/",
gateway__proxy_address=gateway_proc.gateway_urls.connect_url,
):
async with Gateway(asynchronous=True) as gateway:
assert gateway._public_address == "/services/dask-gateway"
cluster = await gateway.new_cluster()
sol = "/services/dask-gateway/gateway/clusters/%s/status" % cluster.name
assert cluster.dashboard_link == sol
clusters = await gateway.list_clusters()
for c in clusters:
assert c.dashboard_link.startswith(
"/services/dask-gateway")
async def test_client_fetch_timeout():
async with slow_server() as server:
gateway = Gateway(server.address, auth=BasicAuth("alice"))
with pytest.raises(TimeoutError):
await gateway._fetch(HTTPRequest(url=server.address, request_timeout=1))