def gb_status(q, mode='runs'):
#bystatus
if GROUPING_MODE == 'python':
if mode == 'runs':
gb_status_rows = dbs(q).select(sr.status,
orderby=sr.status,
**ANALYZE_CACHE_KWARGS)
gb_status_series = defaultdict(int)
for row in gb_status_rows:
status = row.status
gb_status_series[status] += 1
else:
gb_status_series = defaultdict(int)
gb_status_rows = dbs(q).select(st.status, st.times_run,
**ANALYZE_CACHE_KWARGS)
for row in gb_status_rows:
gb_status_series[row.status] += row.times_run
else:
if mode == 'runs':
status_ = sr.status
count_ = sr.id.count()
gb_status_rows = dbs(q).select(count_,
status_,
groupby=status_,
orderby=status_,
**ANALYZE_CACHE_KWARGS)
gb_status_series = defaultdict(int)
for row in gb_status_rows:
status = row.scheduler_run.status
howmany = row[count_]
gb_status_series[status] += howmany
else:
status_ = st.status
count_ = st.times_run.sum()
gb_status_rows = dbs(q).select(count_,
status_,
groupby=status_,
orderby=status_,
**ANALYZE_CACHE_KWARGS)
gb_status_series = defaultdict(int)
for row in gb_status_rows:
status = row.scheduler_task.status
howmany = row[count_]
gb_status_series[status] += howmany
jgb_status_series = []
for k, v in gb_status_series.items():
jgb_status_series.append({
'label': k,
'color': graph_colors_task_status(k),
'data': [v]
})
return gb_status_rows, jgb_status_series
def jobs_render():
session.forget(response)
if request.vars.job_name:
q = sd.job_name == request.vars.job_name
all_jobs = {request.vars.job_name: 'a_0'}
newjob = None
else:
q = sd.id > 0
all_jobs = dbs(q).select(sd.job_name, groupby=sd.job_name)
all_jobs = dict([(row.job_name, "a_%s" % k)
for k, row in enumerate(all_jobs)])
newjob = str(int(time.mktime(datetime.datetime.utcnow().timetuple())))
all_deps = dbs(q).select()
all_nodes = {}
all_edges = {}
for row in all_deps:
jname_slug = all_jobs[row.job_name]
if jname_slug not in all_nodes:
all_nodes[jname_slug] = {}
all_nodes[jname_slug][row.task_parent] = None
all_nodes[jname_slug][row.task_child] = None
if jname_slug not in all_edges:
all_edges[jname_slug] = []
all_edges[jname_slug].append(
(row.task_child, row.task_parent,
dict(style="stroke: %s" %
(row.can_visit and '#5CB85C' or '#D9534F'))))
all_tasks_ids = {}
for k, v in all_nodes.iteritems():
for id in v.keys():
all_tasks_ids[id] = None
all_tasks_labels = dbs(st.id.belongs(all_tasks_ids.keys())).select(
st.id, st.function_name, st.task_name, st.status).as_dict()
for k, v in all_nodes.iteritems():
for id in v.keys():
all_nodes[k][id] = dict(
label=all_tasks_labels[id]['task_name'],
style="fill: %s;" %
graph_colors_task_status(all_tasks_labels[id]['status']),
title="%(id)s (%(function_name)s): %(status)s" %
all_tasks_labels[id],
linkto=URL('task_details', args=id, user_signature=True))
return dict(all_jobs=all_jobs,
all_edges=dumps(all_edges),
all_nodes=dumps(all_nodes),
newjob=newjob)
def gb_duration(q):
#byduration
count_ = sr.id.count()
status_ = sr.status
duration_g = sr.stop_time.epoch() - sr.start_time.epoch()
if GROUPING_MODE == 'python':
gb_duration_rows = dbs(q).select(status_,
sr.start_time,
sr.stop_time,
orderby=status_ | duration_g)
gb_duration_series = {}
for row in gb_duration_rows:
status = row.status
duration_ = row.stop_time - row.start_time
duration = (duration_.seconds + duration_.days * 24 * 3600)
if status not in gb_duration_series:
gb_duration_series[status] = defaultdict(int, {duration: 1})
else:
gb_duration_series[status][duration] += 1
else:
gb_duration_rows = dbs(q).select(count_,
status_,
duration_g,
groupby=status_ | duration_g,
orderby=status_ | duration_g)
#convert to duration series
gb_duration_series = {}
for row in gb_duration_rows:
status = row.scheduler_run.status
duration = row[duration_g]
howmany = row[count_]
if status not in gb_duration_series:
gb_duration_series[status] = {duration: howmany}
else:
if duration not in gb_duration_series[status]:
gb_duration_series[status][duration] = howmany
jgb_duration_series = []
for k, v in gb_duration_series.items():
jgb_duration_series.append({
'label': k,
'data': [[kk, vv] for kk, vv in v.items()],
'color': graph_colors_task_status(k)
})
return gb_duration_rows, jgb_duration_series
def jobs_render():
session.forget(response)
if request.vars.job_name:
q = sd.job_name == request.vars.job_name
all_jobs = {request.vars.job_name : 'a_0'}
newjob = None
else:
q = sd.id > 0
all_jobs = dbs(q).select(sd.job_name, groupby=sd.job_name)
all_jobs = dict([(row.job_name, "a_%s" % k) for k, row in enumerate(all_jobs)])
newjob = str(int(time.mktime(datetime.datetime.utcnow().timetuple())))
all_deps = dbs(q).select()
all_nodes = {}
all_edges = {}
for row in all_deps:
jname_slug = all_jobs[row.job_name]
if jname_slug not in all_nodes:
all_nodes[jname_slug] = {}
all_nodes[jname_slug][row.task_parent] = None
all_nodes[jname_slug][row.task_child] = None
if jname_slug not in all_edges:
all_edges[jname_slug] = []
all_edges[jname_slug].append((row.task_child, row.task_parent,
dict(style="stroke: %s" % (row.can_visit and '#5CB85C' or '#D9534F'))))
all_tasks_ids = {}
for k, v in all_nodes.iteritems():
for id in v.keys():
all_tasks_ids[id] = None
all_tasks_labels = dbs(st.id.belongs(all_tasks_ids.keys())).select(
st.id, st.function_name, st.task_name, st.status
).as_dict()
for k, v in all_nodes.iteritems():
for id in v.keys():
all_nodes[k][id] = dict(label=all_tasks_labels[id]['task_name'],
style="fill: %s;" % graph_colors_task_status(all_tasks_labels[id]['status']),
title="%(id)s (%(function_name)s): %(status)s" % all_tasks_labels[id],
linkto=URL('task_details', args=id, user_signature=True))
return dict(all_jobs=all_jobs, all_edges=dumps(all_edges), all_nodes=dumps(all_nodes), newjob=newjob)
def gb_status(q, mode='runs'):
#bystatus
if GROUPING_MODE == 'python':
if mode == 'runs':
gb_status_rows = dbs(q).select(sr.status, orderby=sr.status, **ANALYZE_CACHE_KWARGS)
gb_status_series = defaultdict(int)
for row in gb_status_rows:
status = row.status
gb_status_series[status] += 1
else:
gb_status_series = defaultdict(int)
gb_status_rows = dbs(q).select(st.status, st.times_run, **ANALYZE_CACHE_KWARGS)
for row in gb_status_rows:
gb_status_series[row.status] += row.times_run
else:
if mode == 'runs':
status_ = sr.status
count_ = sr.id.count()
gb_status_rows = dbs(q).select(count_, status_, groupby=status_, orderby=status_, **ANALYZE_CACHE_KWARGS)
gb_status_series = defaultdict(int)
for row in gb_status_rows:
status = row.scheduler_run.status
howmany = row[count_]
gb_status_series[status] += howmany
else:
status_ = st.status
count_ = st.times_run.sum()
gb_status_rows = dbs(q).select(count_, status_, groupby=status_, orderby=status_, **ANALYZE_CACHE_KWARGS)
gb_status_series = defaultdict(int)
for row in gb_status_rows:
status = row.scheduler_task.status
howmany = row[count_]
gb_status_series[status] += howmany
jgb_status_series = []
for k,v in gb_status_series.items():
jgb_status_series.append(
{'label' : k, 'color' : graph_colors_task_status(k), 'data' : [v]}
)
return gb_status_rows, jgb_status_series
def gb_duration(q):
#byduration
count_ = sr.id.count()
status_ = sr.status
duration_g = sr.stop_time.epoch() - sr.start_time.epoch()
if GROUPING_MODE == 'python':
gb_duration_rows = dbs(q).select(status_, sr.start_time, sr.stop_time, orderby=status_|duration_g)
gb_duration_series = {}
for row in gb_duration_rows:
status = row.status
duration_ = row.stop_time - row.start_time
duration = (duration_.seconds + duration_.days * 24 * 3600)
if status not in gb_duration_series:
gb_duration_series[status] = defaultdict(int, {duration : 1})
else:
gb_duration_series[status][duration] += 1
else:
gb_duration_rows = dbs(q).select(count_, status_, duration_g, groupby=status_|duration_g, orderby=status_|duration_g)
#convert to duration series
gb_duration_series = {}
for row in gb_duration_rows:
status = row.scheduler_run.status
duration = row[duration_g]
howmany = row[count_]
if status not in gb_duration_series:
gb_duration_series[status] = {duration : howmany}
else:
if duration not in gb_duration_series[status]:
gb_duration_series[status][duration] = howmany
jgb_duration_series = []
for k,v in gb_duration_series.items():
jgb_duration_series.append(
{'label': k, 'data' : [[kk,vv] for kk,vv in v.items()], 'color' : graph_colors_task_status(k)}
)
return gb_duration_rows, jgb_duration_series
def byday(q, day, mode):
#by period
if GROUPING_MODE == 'python':
if mode == 'runs':
gb_whend_rows = dbs(q).select(sr.status,
sr.start_time,
orderby=sr.status | sr.start_time,
**ANALYZE_CACHE_KWARGS)
gb_whend_series = {}
for row in gb_whend_rows:
status = row.status
refdate = row.start_time.strftime('%Y-%m-%d %H:%M')
if status not in gb_whend_series:
gb_whend_series[status] = defaultdict(int, {refdate: 1})
else:
gb_whend_series[status][refdate] += 1
else:
gb_whend_rows = dbs(q).select(st.times_run,
st.status,
st.last_run_time,
orderby=st.status | st.last_run_time,
**ANALYZE_CACHE_KWARGS)
gb_whend_series = {}
for row in gb_whend_rows:
status = row.status
refdate = row.last_run_time.strftime('%Y-%m-%d %H:%M')
if status not in gb_whend_series:
gb_whend_series[status] = defaultdict(
int, {refdate: row.times_run})
else:
gb_whend_series[status][refdate] += row.times_run
else:
if mode == 'runs':
count_ = sr.id.count()
status_ = sr.status
d = sr.start_time.hour() | sr.start_time.minutes()
gb_whend_rows = dbs(q).select(count_,
status_,
sr.start_time.hour(),
sr.start_time.minutes(),
groupby=status_ | d,
orderby=status_ | d,
**ANALYZE_CACHE_KWARGS)
gb_whend_series = {}
for row in gb_whend_rows:
status = row.scheduler_run.status
howmany = row[count_]
refdate = day.year, day.month, day.day, row[
sr.start_time.hour()], row[sr.start_time.minutes()], 0
refdate = datetime.datetime(
*refdate).strftime('%Y-%m-%d %H:%M')
if status not in gb_whend_series:
gb_whend_series[status] = {refdate: howmany}
else:
gb_whend_series[status][refdate] = howmany
else:
count_ = st.times_run.sum()
status_ = st.status
d = st.last_run_time.hour() | st.last_run_time.minutes()
gb_whend_rows = dbs(q).select(count_,
status_,
st.last_run_time.hour(),
st.last_run_time.minutes(),
groupby=status_ | d,
orderby=status_ | d,
**ANALYZE_CACHE_KWARGS)
gb_whend_series = {}
for row in gb_whend_rows:
status = row.scheduler_task.status
howmany = row[count_]
refdate = day.year, day.month, day.day, row[
st.last_run_time.hour()], row[
st.last_run_time.minutes()], 0
refdate = datetime.datetime(
*refdate).strftime('%Y-%m-%d %H:%M')
if status not in gb_whend_series:
gb_whend_series[status] = {refdate: howmany}
else:
gb_whend_series[status][refdate] = howmany
jgb_whend_series = []
for k, v in gb_whend_series.items():
jgb_whend_series.append({
'label': k,
'data': [[kk, vv] for kk, vv in v.items()],
'color': graph_colors_task_status(k)
})
return gb_whend_rows, jgb_whend_series
def byday(q, day, mode):
#by period
if GROUPING_MODE == 'python':
if mode == 'runs':
gb_whend_rows = dbs(q).select(sr.status, sr.start_time, orderby=sr.status|sr.start_time, **ANALYZE_CACHE_KWARGS)
gb_whend_series = {}
for row in gb_whend_rows:
status = row.status
refdate = row.start_time.strftime('%Y-%m-%d %H:%M')
if status not in gb_whend_series:
gb_whend_series[status] = defaultdict(int, {refdate : 1})
else:
gb_whend_series[status][refdate] += 1
else:
gb_whend_rows = dbs(q).select(st.times_run, st.status, st.last_run_time, orderby=st.status|st.last_run_time, **ANALYZE_CACHE_KWARGS)
gb_whend_series = {}
for row in gb_whend_rows:
status = row.status
refdate = row.last_run_time.strftime('%Y-%m-%d %H:%M')
if status not in gb_whend_series:
gb_whend_series[status] = defaultdict(int, {refdate : row.times_run})
else:
gb_whend_series[status][refdate] += row.times_run
else:
if mode == 'runs':
count_ = sr.id.count()
status_ = sr.status
d = sr.start_time.hour()|sr.start_time.minutes()
gb_whend_rows = dbs(q).select(count_, status_, sr.start_time.hour(), sr.start_time.minutes(), groupby=status_|d, orderby=status_|d, **ANALYZE_CACHE_KWARGS)
gb_whend_series = {}
for row in gb_whend_rows:
status = row.scheduler_run.status
howmany = row[count_]
refdate = day.year, day.month, day.day, row[sr.start_time.hour()], row[sr.start_time.minutes()], 0
refdate = datetime.datetime(*refdate).strftime('%Y-%m-%d %H:%M')
if status not in gb_whend_series:
gb_whend_series[status] = {refdate : howmany}
else:
gb_whend_series[status][refdate] = howmany
else:
count_ = st.times_run.sum()
status_ = st.status
d = st.last_run_time.hour()|st.last_run_time.minutes()
gb_whend_rows = dbs(q).select(count_, status_, st.last_run_time.hour(), st.last_run_time.minutes(), groupby=status_|d, orderby=status_|d, **ANALYZE_CACHE_KWARGS)
gb_whend_series = {}
for row in gb_whend_rows:
status = row.scheduler_task.status
howmany = row[count_]
refdate = day.year, day.month, day.day, row[st.last_run_time.hour()], row[st.last_run_time.minutes()], 0
refdate = datetime.datetime(*refdate).strftime('%Y-%m-%d %H:%M')
if status not in gb_whend_series:
gb_whend_series[status] = {refdate : howmany}
else:
gb_whend_series[status][refdate] = howmany
jgb_whend_series = []
for k, v in gb_whend_series.items():
jgb_whend_series.append(
{'label': k, 'data' : [[kk,vv] for kk,vv in v.items()], 'color' : graph_colors_task_status(k)}
)
return gb_whend_rows, jgb_whend_series
