def show_overview(self, queue, group=None, delimiter=None):
# Try to show an overview of the entire system
if self.aggregate is None:
try:
self.aggregate = self.client.aggregate_monitor()
except Exception as e:
print(e)
return self.list_namespaces(queue)
data = defaultdict(lambda: dict(unread=0, unactioned=0, actioned=0, lost=0, failed=0, agent=0, runtime=0))
insert(data, self.aggregate.unread_count(queue, group, delimiter=delimiter), 'unread')
insert(data, self.aggregate.unactioned_count(queue, group, delimiter=delimiter), 'unactioned')
insert(data, self.aggregate.actioned_count(queue, group, delimiter=delimiter), 'actioned')
insert(data, self.aggregate.lost_count(queue, group, delimiter=delimiter), 'lost')
insert(data, self.aggregate.failed_count(queue, group, delimiter=delimiter), 'failed')
for group, info in data.items():
print(group, info)
name = 'experiment'
if delimiter is not None:
name = 'study'
status, agents = prepare_overview_altair(data)
chart = aggregate_overview_altair(status, name)
return html.div(
html.header('Status', level=3),
html.div_row(
html.div_col(html.altair_plot(chart), style='height:100vh;', size=5),
html.div_col(list_experiment_stats(self.base_path, queue, data.keys(), data, nolink=delimiter is not None))
)
)
def main(self, uid, keys, module=None, param=None, uid2=None):
self.uid = uid
self.keys = keys
self.module = module
self.param = param
model2 = None
keys = keys.split('/')
try:
model = self.load_model(uid, keys)
if uid2 is not None:
model2 = self.load_model(uid2, keys)
except:
return html.div(
f'Could not load model in {uid}.state {".".join(keys)}',
html.pre(traceback.format_exc()))
if module is None:
return self.module_selection(model, uid, keys)
if param is None:
return self.parameter_selection(model, module)
return self.show_param(model, module, param, model2, uid2)
def show_namespace(self, name, status, agents):
fig = work_status(status)
fig.update_layout(template='plotly_dark')
return html.div(
html.header(f'{name} {agents}', level=5),
html.div_col(html.plotly_plot(fig), size=4))
def show_queue(self, queue, namespace, delimiter=None):
args = (queue, namespace)
if delimiter:
args += delimiter,
messages = self.client.messages(*args)
return html.div(html.header('/'.join([queue, namespace]), level=4),
html.show_messages(messages))
def main(self):
routes = [
html.link(html.chain(html.span(name), ':', html.code(spec)), spec)
for spec, name in self.dash.routes
]
page = self.env.get_template('routes.html')
return page.render(routes=html.div(html.header('Routes', level=4),
html.ul(routes)),
state=self.app.state)
def module_selection(self, model: nn.Module, uid, keys):
layers = []
for k, v in list_layers(model):
layers.append(' '.join(
[html.link(k, self.module_subroute(k)),
html.code(type(v))]))
return html.div(
html.header(
f'Modules of {html.code(".".join(keys))} in {html.code(uid)}',
level=4), html.ol(layers))
def select_study(self):
def redirect_on_enter(data):
if not data:
return
self.prefix = data
redirect(f'/{self.base_path}/{data}')
bind('study_prefix', 'change', redirect_on_enter, property='value')
return html.div(
html.header('Study prefix', level=5),
'A study is a group of experiment, experiment belonging to the same study should'
'start with the same name for example <code>mystudy_exp1</code>, <code>my_study_exp2</code>',
html.text_input(id='study_prefix', placeholder='Study prefix'))
def show_state(self, uid):
if self.meta is None:
self.meta = self.storage.meta.load(self.storage.folder)
meta_uid = uid.split('_')[-1]
meta = self.meta.get(meta_uid, dict())
state = self.storage.load(uid)
display = self.generic_display(uid, state)
return html.div(
html.header(f'State {uid}', level=4),
html.header('Meta', level=5),
html.pre(json.dumps(meta, indent=2)),
html.header('State', level=5),
display
)
def show_key(self, uid, *keys):
if self.meta is None:
self.meta = self.storage.meta.load(self.storage.folder)
state = self.storage.load(uid)
for k in keys:
t = state.get(k, dict())
if isinstance(t, dict):
state = t
else:
break
display = self.generic_display(uid, state, keys)
return html.div(
html.header(f'State {uid}/{"/".join(keys)}', level=4),
display
)
def form(self, options, sample):
import json
options.insert(0, None)
mark_opt = list(map(lambda i: i[0], self.mark_options))
form_html = html.div(
html.div(
'Mark',
html.select_dropdown(mark_opt, 'mark')),
html.div(
'X axis',
html.select_dropdown(options, 'x-label')),
html.div(
'Y axis',
html.select_dropdown(options, 'y-label')),
html.div(
'Y2 axis',
html.select_dropdown(options, 'y2-label')),
html.div(
'Colors',
html.select_dropdown(options, 'colors')),
html.div(
'Shape',
html.select_dropdown(options, 'shape')),
html.div(
'Data Example',
html.pre(json.dumps(sample, indent=2))
)
)
# Request callbacks when those two values change
bind('mark' , 'change', self.set_attribute_callback('mark'), property='selectedIndex')
bind('x-label' , 'change', self.set_attribute_callback('xlabel'), property='selectedIndex')
bind('y-label' , 'change', self.set_attribute_callback('ylabel'), property='selectedIndex')
bind('y2-label', 'change', self.set_attribute_callback('y2label'), property='selectedIndex')
bind('colors' , 'change', self.set_attribute_callback('colors'), property='selectedIndex')
bind('shape' , 'change', self.set_attribute_callback('shape'), property='selectedIndex')
return form_html
def show_queue(self, queue, namespace):
# unread_messages = self.client.unread_messages(queue, namespace)
# unactioned = self.client.unactioned_messages(queue, namespace)
unread = self.client.unread_count(queue, namespace)
unactioned = self.client.unactioned_count(queue, namespace)
finished = self.client.actioned_count(queue, namespace)
lost = self.client.lost_messages(queue, namespace)
failed = self.client.failed_messages(queue, namespace)
agents = self.get_unique_agents(queue, namespace)
data = {
'pending': unread,
'running': unactioned,
'finished': finished,
'lost': len(lost),
'failed': len(failed)
}
fig = work_status(data)
fig.update_layout(template='plotly_dark')
return html.div(
html.header(f'Status', level=3),
html.div_row(
html.div_col(
html.header('Tasks', level=5),
html.plotly_plot(fig), size=4),
html.div_col(
html.header('Agents', level=5),
html.show_agent(agents))),
html.div_row(
html.div_col(
html.header('Lost', level=5),
html.show_messages(lost[:10])),
html.div_col(
html.header('Failed', level=5),
html.show_messages(failed[:10]))))
def list_experiment_stats(base_path, queue, experiments, data, nolink):
def show_expriment(n):
status = data.get(n, {})
if not nolink:
link = html.link(n, ref=f'/{base_path}/{queue}/{n}')
else:
link = n
row = f"""
<tbody>
<tr>
<td>{link}</td>
<td>{status.get("lost", 0)}</td>
<td>{status.get("failed", 0)}</td>
<td>{status.get("agent", 0)}</td>
<td>{status.get("runtime_actioned", 0):0.2f}</td>
</tr>
</tbody>"""
return row
rows = ''.join([show_expriment(name) for name in filter(lambda e: e is not None, experiments)])
return html.div(
f"""
<table class="table table-hover table-striped table-sm">
<thead>
<tr>
<th>Experiment</th>
<th>Lost</th>
<th>Failed</th>
<th>Agents</th>
<th>Runtime</th>
</tr>
</thead>
<tbody>
{rows}
</tbody>
</table>""")
def list_namespaces(self, queue):
return html.div(
html.header(queue, level=4),
html.ul(
html.link(n, f'/{self.base_path}/{queue}/{n}')
for n in self.client.namespaces(queue)))
def show_queue(self, namespace, agent_id, ltype):
log = self.client.log(agent_id, ltype=ltype)
return html.div(
html.header(f'Logs for agent {agent_id[:8]}'),
html.pre(log))
def list_agents(self, queue, namespace):
return html.div(
html.header('Agents', level=4),
html.ul(
html.link(q, f'/{self.base_path}/{queue}/{namespace}/{q.uid}') for q in self.client.agents(namespace)))
def list_namespaces(self, queue=WORK_QUEUE):
return html.div(
html.header('Experiments', level=4),
html.ul(
html.link(n, f'/{self.base_path}/{n}')
for n in self.client.namespaces(queue)))
def main(self):
routes = [
html.link(html.chain(html.span(name), ':', html.code(spec)), spec)
for spec, name in self.dash.routes
]
return html.div(html.header('Routes', level=4), html.ul(routes))