def update(self):
with log_errors():
outgoing = self.worker.outgoing_transfer_log
n = self.worker.outgoing_count - self.last_outgoing
n = min(n, 1000)
outgoing = [outgoing[-i].copy() for i in range(1, n)]
self.last_outgoing = self.worker.outgoing_count
incoming = self.worker.incoming_transfer_log
n = self.worker.incoming_count - self.last_incoming
n = min(n, 1000)
incoming = [incoming[-i].copy() for i in range(1, n)]
self.last_incoming = self.worker.incoming_count
out = []
for msg in incoming:
if msg["keys"]:
d = self.process_msg(msg)
d["inout-color"] = "red"
out.append(d)
for msg in outgoing:
if msg["keys"]:
d = self.process_msg(msg)
d["inout-color"] = "blue"
out.append(d)
if out:
out = transpose(out)
if (len(self.source.data["stop"]) and
min(out["start"]) > self.source.data["stop"][-1] + 10):
update(self.source, out)
else:
self.source.stream(out, rollover=1000)
def trigger_update(self):
self.state = profile.get_profile(self.log, start=self.start, stop=self.stop)
data = profile.plot_data(self.state, profile_interval)
self.states = data.pop("states")
update(self.source, data)
times = [t * 1000 for t, _ in self.log]
counts = list(toolz.pluck("count", toolz.pluck(1, self.log)))
self.ts_source.data.update({"time": times, "count": counts})
def update(self, messages):
with log_errors():
msg = messages["progress"]
if not msg:
return
nb = nbytes_bar(msg["nbytes"])
update(self.source, nb)
self.root.title.text = "Memory Use: %0.2f MB" % (
sum(msg["nbytes"].values()) / 1e6)
def cb(attr, old, new):
if changing[0] or len(new) == 0:
return
with log_errors():
data = profile.plot_data(self.states[new[0]], profile_interval)
del self.states[:]
self.states.extend(data.pop("states"))
changing[0] = True # don't recursively trigger callback
update(self.source, data)
self.source.selected.indices = old
changing[0] = False
def cb(attr, old, new):
with log_errors():
try:
ind = new.indices[0]
except IndexError:
return
data = profile.plot_data(self.states[ind], profile_interval)
del self.states[:]
self.states.extend(data.pop("states"))
update(self.source, data)
self.source.selected = old
def update(self):
with log_errors():
w = self.worker
d = {
"Stored": [len(w.data)],
"Executing": ["%d / %d" % (len(w.executing), w.nthreads)],
"Ready": [len(w.ready)],
"Waiting": [len(w.waiting_for_data)],
"Connections": [len(w.in_flight_workers)],
"Serving": [len(w._comms)],
}
update(self.source, d)
def update(self):
with log_errors():
workers = list(self.scheduler.workers.values())
utilization = []
memory = []
gpu_index = []
y = []
memory_total = 0
memory_max = 0
worker = []
i = 0
for ws in workers:
info = ws.extra["gpu"]
metrics = ws.metrics["gpu"]
for j, (u, mem_used, mem_total) in enumerate(
zip(
metrics["utilization"],
metrics["memory-used"],
info["memory-total"],
)):
memory_max = max(memory_max, mem_total)
memory_total += mem_total
utilization.append(int(u))
memory.append(mem_used)
worker.append(ws.address)
gpu_index.append(j)
y.append(i)
i += 1
memory_text = [format_bytes(m) for m in memory]
result = {
"memory": memory,
"memory-half": [m / 2 for m in memory],
"memory_text": memory_text,
"utilization": utilization,
"utilization-half": [u / 2 for u in utilization],
"worker": worker,
"gpu-index": gpu_index,
"y": y,
"escaped_worker": [escape.url_escape(w) for w in worker],
}
self.memory_figure.title.text = "GPU Memory: %s / %s" % (
format_bytes(sum(memory)),
format_bytes(memory_total),
)
self.memory_figure.x_range.end = memory_max
update(self.source, result)
def update(self):
with log_errors():
workers = list(self.scheduler.workers.values())
utilization = []
memory = []
gpu_index = []
y = []
memory_total = 0
memory_max = 0
worker = []
for idx, ws in enumerate(workers):
try:
info = ws.extra["gpu"]
except KeyError:
continue
metrics = ws.metrics["gpu"]
u = metrics["utilization"]
mem_used = metrics["memory-used"]
mem_total = info["memory-total"]
memory_max = max(memory_max, mem_total)
memory_total += mem_total
utilization.append(int(u))
memory.append(mem_used)
worker.append(ws.address)
gpu_index.append(idx)
y.append(idx)
memory_text = [format_bytes(m) for m in memory]
result = {
"memory": memory,
"memory-half": [m / 2 for m in memory],
"memory_text": memory_text,
"utilization": utilization,
"utilization-half": [u / 2 for u in utilization],
"worker": worker,
"gpu-index": gpu_index,
"y": y,
"escaped_worker": [escape.url_escape(w) for w in worker],
}
self.memory_figure.title.text = "GPU Memory: {} / {}".format(
format_bytes(sum(memory)),
format_bytes(memory_total),
)
self.memory_figure.x_range.end = memory_max
update(self.source, result)
def update(self, messages):
with log_errors():
msg = messages["processing"]
if not msg.get("nthreads"):
return
data = self.processing_update(msg)
x_range = self.root.x_range
max_right = max(data["right"])
cores = max(data["nthreads"])
if x_range.end < max_right:
x_range.end = max_right + 2
elif x_range.end > 2 * max_right + cores: # way out there, walk back
x_range.end = x_range.end * 0.95 + max_right * 0.05
update(self.source, data)
def update(self, state, metadata=None):
with log_errors():
self.state = state
data = profile.plot_data(self.state, profile_interval)
self.states = data.pop("states")
update(self.source, data)
if metadata is not None and metadata["counts"]:
self.task_names = ["All"] + sorted(metadata["keys"])
self.select.options = self.task_names
if self.key:
ts = metadata["keys"][self.key]
else:
ts = metadata["counts"]
times, counts = zip(*ts)
self.ts = {"count": counts, "time": [t * 1000 for t in times]}
self.ts_source.data.update(self.ts)
def cb(attr, old, new):
if changing[0]:
return
with log_errors():
if isinstance(new, list): # bokeh >= 1.0
selected = new
else:
selected = new["1d"]["indices"]
try:
ind = selected[0]
except IndexError:
return
data = profile.plot_data(self.states[ind], profile_interval)
del self.states[:]
self.states.extend(data.pop("states"))
changing[0] = True # don't recursively trigger callback
update(self.source, data)
if isinstance(new, list): # bokeh >= 1.0
self.source.selected.indices = old
else:
self.source.selected = old
changing[0] = False
def __init__(self, worker, height=150, **kwargs):
self.worker = worker
names = worker.monitor.quantities
self.last = 0
self.source = ColumnDataSource({name: [] for name in names})
update(self.source, self.get_data())
x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
tools = "reset,xpan,xwheel_zoom"
self.cpu = figure(
title="CPU",
x_axis_type="datetime",
height=height,
tools=tools,
x_range=x_range,
**kwargs
)
self.cpu.line(source=self.source, x="time", y="cpu")
self.cpu.yaxis.axis_label = "Percentage"
self.mem = figure(
title="Memory",
x_axis_type="datetime",
height=height,
tools=tools,
x_range=x_range,
**kwargs
)
self.mem.line(source=self.source, x="time", y="memory")
self.mem.yaxis.axis_label = "Bytes"
self.bandwidth = figure(
title="Bandwidth",
x_axis_type="datetime",
height=height,
x_range=x_range,
tools=tools,
**kwargs
)
self.bandwidth.line(source=self.source, x="time", y="read_bytes", color="red")
self.bandwidth.line(source=self.source, x="time", y="write_bytes", color="blue")
self.bandwidth.yaxis.axis_label = "Bytes / second"
# self.cpu.yaxis[0].formatter = NumeralTickFormatter(format='0%')
self.bandwidth.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
self.mem.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
plots = [self.cpu, self.mem, self.bandwidth]
if not WINDOWS:
self.num_fds = figure(
title="Number of File Descriptors",
x_axis_type="datetime",
height=height,
x_range=x_range,
tools=tools,
**kwargs
)
self.num_fds.line(source=self.source, x="time", y="num_fds")
plots.append(self.num_fds)
if "sizing_mode" in kwargs:
kw = {"sizing_mode": kwargs["sizing_mode"]}
else:
kw = {}
if not WINDOWS:
self.num_fds.y_range.start = 0
self.mem.y_range.start = 0
self.cpu.y_range.start = 0
self.bandwidth.y_range.start = 0
self.root = column(*plots, **kw)
self.worker.monitor.update()
def update(self, state):
with log_errors():
self.state = state
data = profile.plot_data(self.state, profile_interval)
self.states = data.pop("states")
update(self.source, data)
def __init__(self, worker, height=150, last_count=None, **kwargs):
self.worker = worker
names = worker.monitor.quantities
self.last_count = 0
if last_count is not None:
names = worker.monitor.range_query(start=last_count)
self.last_count = last_count
self.source = ColumnDataSource({name: [] for name in names})
self.label_source = ColumnDataSource(
{
"x": [5] * 3,
"y": [70, 55, 40],
"cpu": ["max: 45%", "min: 45%", "mean: 45%"],
"memory": ["max: 133.5MiB", "min: 23.6MiB", "mean: 115.4MiB"],
}
)
update(self.source, self.get_data())
x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
tools = "reset,xpan,xwheel_zoom"
self.cpu = figure(
title="CPU",
x_axis_type="datetime",
height=height,
tools=tools,
toolbar_location="above",
x_range=x_range,
**kwargs,
)
self.cpu.line(source=self.source, x="time", y="cpu")
self.cpu.yaxis.axis_label = "Percentage"
self.cpu.add_layout(
LabelSet(
x="x",
y="y",
x_units="screen",
y_units="screen",
text="cpu",
text_font_size="1em",
render_mode="css",
source=self.label_source,
)
)
self.mem = figure(
title="Memory",
x_axis_type="datetime",
height=height,
tools=tools,
toolbar_location="above",
x_range=x_range,
**kwargs,
)
self.mem.line(source=self.source, x="time", y="memory")
self.mem.yaxis.axis_label = "Bytes"
self.mem.add_layout(
LabelSet(
x="x",
y="y",
x_units="screen",
y_units="screen",
text="memory",
text_font_size="1em",
render_mode="css",
source=self.label_source,
)
)
self.bandwidth = figure(
title="Bandwidth",
x_axis_type="datetime",
height=height,
x_range=x_range,
tools=tools,
toolbar_location="above",
**kwargs,
)
self.bandwidth.line(
source=self.source,
x="time",
y="read_bytes",
color="red",
legend_label="read",
)
self.bandwidth.line(
source=self.source,
x="time",
y="write_bytes",
color="blue",
legend_label="write",
)
self.bandwidth.yaxis.axis_label = "Bytes / second"
# self.cpu.yaxis[0].formatter = NumeralTickFormatter(format='0%')
self.bandwidth.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
self.mem.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
plots = [self.cpu, self.mem, self.bandwidth]
if not WINDOWS:
self.num_fds = figure(
title="Number of File Descriptors",
x_axis_type="datetime",
height=height,
x_range=x_range,
tools=tools,
toolbar_location="above",
**kwargs,
)
self.num_fds.line(source=self.source, x="time", y="num_fds")
plots.append(self.num_fds)
if "sizing_mode" in kwargs:
kw = {"sizing_mode": kwargs["sizing_mode"]}
else:
kw = {}
if not WINDOWS:
self.num_fds.y_range.start = 0
self.mem.y_range.start = 0
self.cpu.y_range.start = 0
self.bandwidth.y_range.start = 0
self.root = column(*plots, **kw)
self.worker.monitor.update()
def _update_data_source(self):
self.logger.info(
f"update data source with data from {self.from_datepk.value} to {self.to_datepk.value}"
)
try:
from_ts = np.datetime64(self.from_datepk.value)
to_ts = np.datetime64(self.to_datepk.value)
stats_data = {
'ccypair': [pair for pair in self.pairs],
'sample_mean': [],
'variance': [],
'std_dev': [],
'skew': [],
'kurtosis': [],
't_test': [],
'hurst': [],
}
for pair in self.pairs:
self.logger.debug(
f"Calculate data health metrics for pair{pair}")
data = self.tiledb.get_ts_dataframe("health",
f"{pair}_DAILY_METRICS",
from_ts, to_ts)
data.fillna(method='bfill', axis=0, inplace=True)
if data is not None and len(data.index) > 0:
new_data = {
col: data[col].tolist()
for col in data.columns
}
update(self.ts_source[pair], new_data)
logret_ema = data['logret_ema'].to_numpy()
stats_metrics = stats.describe(logret_ema,
nan_policy='omit')
min, max = stats_metrics.minmax
t_test = stats.ttest_1samp(logret_ema,
popmean=0.0,
nan_policy='omit')
if len(data.index) > 100:
logret_ema += (0 if min > 0 else ((-min) + 1.0))
H, c, data = compute_Hc(logret_ema,
kind='price',
simplified=True)
stats_data['hurst'].append(f"H={H}, c={c}")
else:
stats_data['hurst'].append(f"data too small")
stats_data['sample_mean'].append(f"{stats_metrics.mean}")
stats_data['variance'].append(f"{stats_metrics.variance}")
stats_data['std_dev'].append(
f"{math.sqrt(stats_metrics.variance)}")
stats_data['skew'].append(f"{stats_metrics.skewness}")
stats_data['kurtosis'].append(f"{stats_metrics.kurtosis}")
stats_data['t_test'].append(
f"t-stats={t_test.statistic}, p-value={(t_test.pvalue/2.0)}"
)
update(self.table_source, stats_data)
self.logger.debug("Finished update health data source")
except Exception as ex:
self.logger.error("Failed to update data source", exc_info=ex)
def __init__(self, worker, height=200, **kwargs):
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.debug("Initialize DataMonitorDashboard")
self.cache = DataMonitorCache.instance()
names = self.cache.data
self.pairs = self._DEFAULT_PAIRS
self.last = {}
self.source = {}
self.figure = {}
plots = []
for pair in self.pairs:
self.last[pair] = 0
self.source[pair] = ColumnDataSource({name: [] for name in names})
update(self.source[pair], self.get_data(pair))
pair_figure = figure(title=pair,
x_axis_type="datetime",
height=height,
x_range=DataRange1d(follow="end",
follow_interval=20000000,
range_padding=0),
y_range=DataRange1d(follow="end",
follow_interval=1,
range_padding=0.15),
output_backend="webgl",
**kwargs)
pair_figure.line(source=self.source[pair],
x="date",
y="bidclose",
color="red",
legend_label='bid close')
pair_figure.line(source=self.source[pair],
x="date",
y="askclose",
color="blue",
legend_label='ask close')
pair_figure.legend.location = "bottom_right"
pair_figure.legend.click_policy = "hide"
pair_figure.legend.background_fill_alpha = 0.0
pair_figure.yaxis.axis_label = "Exchange Rate"
pair_figure.xaxis.axis_label = "Time"
pair_figure.xaxis.major_label_orientation = pi / 4
pair_figure.xaxis.formatter = DatetimeTickFormatter(
microseconds=['%fus'],
milliseconds=['%3Nms', '%S.%3Ns'],
seconds=['%H:%M:%S'],
minsec=['%H:%M:%S'],
minutes=['%d/%m/%y %H:%M:%S'],
hourmin=['%d/%m/%y %H:%M:%S'],
hours=['%d/%m/%y %H:%M:%S'],
days=['%d/%m/%y %H:%M:%S'],
months=['%d/%m/%y %H:%M:%S'],
years=['%d/%m/%y %H:%M:%S'],
)
self.figure[pair] = pair_figure
plots.append(self.figure[pair])
if "sizing_mode" in kwargs:
kw = {"sizing_mode": kwargs["sizing_mode"]}
else:
kw = {}
self.root = column(*plots, **kw)