def view_dashboard():
session = create_session()
clusters = session.query(Cluster).all()
jobs = session.query(JobRun).all()
states = concat_dfs(cluster.state_df() for cluster in clusters)
level_info_data = get_level_info_data()
numbjobs_dict = get_running_jobs(jobs)
last7dbu_dict = aggregate_by_types(states, get_last_7_days_dbu)
time_stats_dict = {
'interactive': empty_timeseries(),
'job': empty_timeseries()
}
if not states.empty:
results = aggregate_by_types(states, aggregate_for_entity)
# cost_summary_dict = {}
for key, (cost_summary, time_stats) in results.items():
time_stats['dbu_cumsum'] = time_stats['interval_dbu_sum'].cumsum()
time_stats_dict[key] = time_stats.to_dict("records")
# cost_summary_dict[key] = cost_summary.to_dict()
return render_template('dashboard.html',
time_stats=time_stats_dict,
last7dbu=last7dbu_dict,
numjobs=numbjobs_dict,
data=level_info_data)
def view_clusters():
session = create_session()
clusters = session.query(Cluster).all()
states = concat_dfs(cluster.state_df() for cluster in clusters)
level_info_data = get_level_info_data()
price_settings = get_price_settings()
time_stats_dict = {
'interactive': empty_timeseries(),
'job': empty_timeseries()
}
if not states.empty:
results = aggregate_by_types(states, aggregate_for_entity)
for key, (_, time_stats) in results.items():
time_stats_dict[key] = time_stats.to_dict("records")
cluster_dbus = (aggregate(df=states,
col="interval_dbu",
by="cluster_id",
since_days=7).rename(columns={
'interval_dbu': 'dbu'
}).dbu.to_dict())
else:
cluster_dbus = {cluster.cluster_id: 0.0 for cluster in clusters}
clusters_by_type = {}
for cluster in clusters:
clusters_by_type.setdefault(cluster.cluster_type(), []).append(cluster)
return render_template('clusters.html',
clusters_by_type=clusters_by_type,
price_settings=price_settings,
data=level_info_data,
cluster_dbus=cluster_dbus,
time_stats=time_stats_dict)
def view_user(username):
session = create_session()
try:
user = (session.query(User).filter(User.username == username).one())
except Exception:
return view_missing(type="user", id=username)
states = user.state_df()
time_stats_dict = {
'interactive': empty_timeseries(),
'job': empty_timeseries()
}
if not states.empty:
workspaces = (concat_dfs({
(w.workspace.id, w.workspace.name): w.workspace.state_df()
for w in user.user_workspaces
}).reset_index([0, 1]).rename(columns={
'level_0': 'workspace_id',
'level_1': 'workspace_name'
}))
last7_workspaces = (aggregate(
df=workspaces,
col='interval_dbu',
by=['workspace_id', 'workspace_name'],
since_days=7).rename(columns={'interval_dbu': 'last7dbu'}))
all_workspaces = (aggregate(
df=workspaces,
col='interval_dbu',
by=['workspace_id', 'workspace_name'
]).rename(columns={'interval_dbu': 'alltimedbu'}))
workspaces_dict = (pd.merge(
all_workspaces,
last7_workspaces,
how='left',
left_index=True,
right_index=True).fillna(0.0).reset_index().sort_values(
'last7dbu').to_dict('records'))
price_settings = get_price_settings()
results = aggregate_by_types(states, aggregate_for_entity)
cost_summary_dict = {}
for key, (cost_summary, time_stats) in results.items():
time_stats_dict[key] = time_stats.to_dict("records")
cost_summary = cost_summary.to_dict()
cost = cost_summary['interval_dbu'] * price_settings[key]
weekly_cost = (cost_summary['weekly_interval_dbu_sum'] *
price_settings[key])
cost_summary['cost'] = cost
cost_summary['weekly_cost'] = weekly_cost
cost_summary_dict[key] = cost_summary
# We aren't sure if we have both interactive and job
present_key = list(cost_summary_dict.keys())[0]
cost_summary_dict = {
key: sum([cost_summary_dict[type][key] for type in results.keys()])
for key in cost_summary_dict[present_key]
}
else:
workspaces_dict = [{
'workspace_id': w.workspace.id,
'workspace_name': w.workspace.name,
'last7dbu': 0.0,
'alltimedbu': 0.0
} for w in user.user_workspaces]
cost_summary_dict = {
"interval": 0.0,
"interval_dbu": 0.0,
"weekly_interval_sum": 0.0,
"weekly_interval_dbu_sum": 0.0,
"cost": 0.0,
"weekly_cost": 0.0
}
return render_template('user.html',
user=user,
workspaces=workspaces_dict,
cost=cost_summary_dict,
time_stats=time_stats_dict)
def view_workspace(workspace_id):
session = create_session()
try:
workspace = (session.query(Workspace).filter(
Workspace.id == workspace_id).one())
except Exception:
return view_missing(type="workspace", id=workspace_id)
states = workspace.state_df()
numbjobs_dict = get_running_jobs(workspace.jobruns)
price_settings = get_price_settings()
time_stats_dict = {
'interactive': empty_timeseries(),
'job': empty_timeseries()
}
if not states.empty:
results = aggregate_by_types(states, aggregate_for_entity)
cost_summary_dict = {}
for key, (cost_summary, time_stats) in results.items():
time_stats_dict[key] = time_stats.to_dict("records")
cost_summary = cost_summary.to_dict()
cost = cost_summary['interval_dbu'] * price_settings[key]
weekly_cost = (cost_summary['weekly_interval_dbu_sum'] *
price_settings[key])
cost_summary['cost'] = cost
cost_summary['weekly_cost'] = weekly_cost
cost_summary_dict[key] = cost_summary
# We aren't sure if we have both interactive and job
present_key = list(cost_summary_dict.keys())[0]
cost_summary_dict = {
key: sum([cost_summary_dict[type][key] for type in results.keys()])
for key in cost_summary_dict[present_key]
}
top_users = (aggregate(
df=states, col='interval_dbu', by='user_id',
since_days=7).reset_index().rename(columns={
'interval_dbu': 'dbu'
}).sort_values('dbu', ascending=False))
top_users_dict = (top_users.loc[~top_users.user_id.isin(['UNKONWN'])].
to_dict("records")[:3])
else:
cost_summary_dict = {
"interval": 0.0,
"interval_dbu": 0.0,
"weekly_interval_sum": 0.0,
"weekly_interval_dbu_sum": 0.0,
"cost": 0.0,
"weekly_cost": 0.0
}
top_users_dict = {}
clusters_by_type = {}
for cluster in workspace.clusters:
clusters_by_type.setdefault(cluster.cluster_type(), []).append(cluster)
return render_template('workspace.html',
workspace=workspace,
clusters_by_type=clusters_by_type,
cost=cost_summary_dict,
time_stats=time_stats_dict,
top_users=top_users_dict,
numjobs=numbjobs_dict,
empty=states.empty)