def test_format_dashboard_link():
with dask.config.set({"distributed.dashboard.link": "foo"}):
assert format_dashboard_link("host", 1234) == "foo"
assert "host" in format_dashboard_link("host", 1234)
assert "1234" in format_dashboard_link("host", 1234)
try:
os.environ["host"] = "hello"
assert "hello" not in format_dashboard_link("host", 1234)
finally:
del os.environ["host"]
def test_https_support(c, s, a, b):
assert isinstance(s.services["dashboard"], BokehScheduler)
port = s.services["dashboard"].port
assert (format_dashboard_link(
"localhost", port) == "https://localhost:%d/status" % port)
ctx = ssl.create_default_context()
ctx.load_verify_locations(get_cert("tls-ca-cert.pem"))
http_client = AsyncHTTPClient()
response = yield http_client.fetch(
"https://localhost:%d/individual-plots.json" % port, ssl_options=ctx)
response = json.loads(response.body.decode())
for suffix in [
"system",
"counters",
"workers",
"status",
"tasks",
"stealing",
"graph",
] + [url.strip("/") for url in response.values()]:
req = HTTPRequest(url="https://localhost:%d/%s" % (port, suffix),
ssl_options=ctx)
response = yield http_client.fetch(req)
assert response.code < 300
body = response.body.decode()
assert not re.search("href=./", body) # no absolute links
def test_https_support(c, s, a, b):
assert isinstance(s.services["dashboard"], BokehScheduler)
port = s.services["dashboard"].port
assert (format_dashboard_link(
"localhost", port) == "https://localhost:%d/status" % port)
ctx = ssl.create_default_context()
ctx.load_verify_locations(get_cert("tls-ca-cert.pem"))
http_client = AsyncHTTPClient()
for suffix in [
"system",
"counters",
"workers",
"status",
"tasks",
"stealing",
"graph",
"individual-task-stream",
"individual-progress",
"individual-graph",
"individual-nbytes",
"individual-nprocessing",
"individual-profile",
]:
req = HTTPRequest(url="https://localhost:%d/%s" % (port, suffix),
ssl_options=ctx)
response = yield http_client.fetch(req)
body = response.body.decode()
assert "bokeh" in body.lower()
assert not re.search("href=./", body) # no absolute links
def _format_dashboard_address(server):
try:
host = (server["host"] if "host" in server else urlparse(
server["address"]).hostname)
return format_dashboard_link(host,
server["services"]["dashboard"])
except KeyError:
return None
def dashboard_link(self):
try:
port = self.scheduler_info["services"]["dashboard"]
except KeyError:
return ""
else:
host = self.scheduler_address.split("://")[1].split("/")[0].split(
":")[0]
return format_dashboard_link(host, port)
def dashboard_link(self):
"""Link to the dask dashboard. None if dashboard isn't running"""
if self._dashboard_address is None:
return None
dashboard = urlparse(self._dashboard_address)
return format_dashboard_link(dashboard.hostname, dashboard.port)
def dashboard_link(self):
host = self.scheduler.address.split("://")[1].split(":")[0]
port = self.scheduler.services["dashboard"].port
return format_dashboard_link(host, port)
def _repr_html_(self):
dashboard_address = None
if "dashboard" in self["services"]:
host = urlparse(self["address"]).hostname
dashboard_address = format_dashboard_link(
host, self["services"]["dashboard"])
scheduler = f"""
<div>
<div style="
width: 24px;
height: 24px;
background-color: #FFF7E5;
border: 3px solid #FF6132;
border-radius: 5px;
position: absolute;"> </div>
<div style="margin-left: 48px;">
<h3 style="margin-bottom: 0px;">{self["type"]}</h3>
<p style="color: #9D9D9D; margin-bottom: 0px;">{self["id"]}</p>
<table style="width: 100%; text-align: left;">
<tr>
<td style="text-align: left;"><strong>Comm:</strong> {self["address"]}</td>
<td style="text-align: left;"><strong>Workers:</strong> {len(self["workers"])}</td>
</tr>
<tr>
<td style="text-align: left;">
<strong>Dashboard:</strong> <a href="{dashboard_address}">{dashboard_address}</a>
</td>
<td style="text-align: left;">
<strong>Total threads:</strong>
{sum([w["nthreads"] for w in self["workers"].values()])}
</td>
</tr>
<tr>
<td style="text-align: left;">
<strong>Started:</strong>
{format_time_ago(datetime.datetime.fromtimestamp(self["started"]))}
</td>
<td style="text-align: left;">
<strong>Total memory:</strong>
{format_bytes(sum([w["memory_limit"] for w in self["workers"].values()]))}
</td>
</tr>
</table>
</div>
</div>
"""
workers = ""
for worker_name in self["workers"]:
self["workers"][worker_name]["comm"] = worker_name
for worker in sorted(self["workers"].values(),
key=lambda k: k["name"]):
dashboard_address = None
if "dashboard" in worker["services"]:
host = urlparse(worker["comm"]).hostname
dashboard_address = format_dashboard_link(
host, worker["services"]["dashboard"])
metrics = ""
if "metrics" in worker:
metrics = f"""
<tr>
<td style="text-align: left;">
<strong>Tasks executing: </strong> {worker["metrics"]["executing"]}
</td>
<td style="text-align: left;">
<strong>Tasks in memory: </strong> {worker["metrics"]["in_memory"]}
</td>
</tr>
<tr>
<td style="text-align: left;">
<strong>Tasks ready: </strong> {worker["metrics"]["ready"]}
</td>
<td style="text-align: left;">
<strong>Tasks in flight: </strong>{worker["metrics"]["in_flight"]}
</td>
</tr>
<tr>
<td style="text-align: left;"><strong>CPU usage:</strong> {worker["metrics"]["cpu"]}%</td>
<td style="text-align: left;">
<strong>Last seen: </strong>
{format_time_ago(datetime.datetime.fromtimestamp(worker["last_seen"]))}
</td>
</tr>
<tr>
<td style="text-align: left;">
<strong>Memory usage: </strong>
{format_bytes(worker["metrics"]["memory"])}
</td>
<td style="text-align: left;">
<strong>Spilled bytes: </strong>
{format_bytes(worker["metrics"]["spilled_nbytes"])}
</td>
</tr>
<tr>
<td style="text-align: left;">
<strong>Read bytes: </strong>
{format_bytes(worker["metrics"]["read_bytes"])}
</td>
<td style="text-align: left;">
<strong>Write bytes: </strong>
{format_bytes(worker["metrics"]["write_bytes"])}
</td>
</tr>
"""
gpu = ""
if "gpu" in worker:
gpu = f"""
<tr>
<td style="text-align: left;">
<strong>GPU: </strong>{worker["gpu"]["name"]}
</td>
<td style="text-align: left;">
<strong>GPU memory: </strong>
{format_bytes(worker["gpu"]["memory-total"])}
</td>
</tr>
"""
workers += f"""
<div style="margin-bottom: 20px;">
<div style="width: 24px;
height: 24px;
background-color: #DBF5FF;
border: 3px solid #4CC9FF;
border-radius: 5px;
position: absolute;"> </div>
<div style="margin-left: 48px;">
<details>
<summary>
<h4 style="margin-bottom: 0px; display: inline;">{worker["type"]}: {worker["name"]}</h4>
</summary>
<table style="width: 100%; text-align: left;">
<tr>
<td style="text-align: left;"><strong>Comm: </strong> {worker["comm"]}</td>
<td style="text-align: left;"><strong>Total threads: </strong> {worker["nthreads"]}</td>
</tr>
<tr>
<td style="text-align: left;">
<strong>Dashboard: </strong>
<a href="{dashboard_address}">{dashboard_address}</a>
</td>
<td style="text-align: left;">
<strong>Memory: </strong>
{format_bytes(worker["memory_limit"])}
</td>
</tr>
<tr>
<td style="text-align: left;"><strong>Nanny: </strong> {worker["nanny"]}</td>
<td style="text-align: left;"></td>
</tr>
<tr>
<td colspan="2" style="text-align: left;">
<strong>Local directory: </strong>
{worker["local_directory"]}
</td>
</tr>
{gpu}
{metrics}
</table>
</details>
</div>
</div>
"""
return f"""
