def gb_status(q, mode='runs'): #bystatus if GROUPING_MODE == 'python': if mode == 'runs': gb_status_rows = dbs(q).select(sr.status, orderby=sr.status, **ANALYZE_CACHE_KWARGS) gb_status_series = defaultdict(int) for row in gb_status_rows: status = row.status gb_status_series[status] += 1 else: gb_status_series = defaultdict(int) gb_status_rows = dbs(q).select(st.status, st.times_run, **ANALYZE_CACHE_KWARGS) for row in gb_status_rows: gb_status_series[row.status] += row.times_run else: if mode == 'runs': status_ = sr.status count_ = sr.id.count() gb_status_rows = dbs(q).select(count_, status_, groupby=status_, orderby=status_, **ANALYZE_CACHE_KWARGS) gb_status_series = defaultdict(int) for row in gb_status_rows: status = row.scheduler_run.status howmany = row[count_] gb_status_series[status] += howmany else: status_ = st.status count_ = st.times_run.sum() gb_status_rows = dbs(q).select(count_, status_, groupby=status_, orderby=status_, **ANALYZE_CACHE_KWARGS) gb_status_series = defaultdict(int) for row in gb_status_rows: status = row.scheduler_task.status howmany = row[count_] gb_status_series[status] += howmany jgb_status_series = [] for k, v in gb_status_series.items(): jgb_status_series.append({ 'label': k, 'color': graph_colors_task_status(k), 'data': [v] }) return gb_status_rows, jgb_status_series
def jobs_render(): session.forget(response) if request.vars.job_name: q = sd.job_name == request.vars.job_name all_jobs = {request.vars.job_name: 'a_0'} newjob = None else: q = sd.id > 0 all_jobs = dbs(q).select(sd.job_name, groupby=sd.job_name) all_jobs = dict([(row.job_name, "a_%s" % k) for k, row in enumerate(all_jobs)]) newjob = str(int(time.mktime(datetime.datetime.utcnow().timetuple()))) all_deps = dbs(q).select() all_nodes = {} all_edges = {} for row in all_deps: jname_slug = all_jobs[row.job_name] if jname_slug not in all_nodes: all_nodes[jname_slug] = {} all_nodes[jname_slug][row.task_parent] = None all_nodes[jname_slug][row.task_child] = None if jname_slug not in all_edges: all_edges[jname_slug] = [] all_edges[jname_slug].append( (row.task_child, row.task_parent, dict(style="stroke: %s" % (row.can_visit and '#5CB85C' or '#D9534F')))) all_tasks_ids = {} for k, v in all_nodes.iteritems(): for id in v.keys(): all_tasks_ids[id] = None all_tasks_labels = dbs(st.id.belongs(all_tasks_ids.keys())).select( st.id, st.function_name, st.task_name, st.status).as_dict() for k, v in all_nodes.iteritems(): for id in v.keys(): all_nodes[k][id] = dict( label=all_tasks_labels[id]['task_name'], style="fill: %s;" % graph_colors_task_status(all_tasks_labels[id]['status']), title="%(id)s (%(function_name)s): %(status)s" % all_tasks_labels[id], linkto=URL('task_details', args=id, user_signature=True)) return dict(all_jobs=all_jobs, all_edges=dumps(all_edges), all_nodes=dumps(all_nodes), newjob=newjob)
def gb_duration(q): #byduration count_ = sr.id.count() status_ = sr.status duration_g = sr.stop_time.epoch() - sr.start_time.epoch() if GROUPING_MODE == 'python': gb_duration_rows = dbs(q).select(status_, sr.start_time, sr.stop_time, orderby=status_ | duration_g) gb_duration_series = {} for row in gb_duration_rows: status = row.status duration_ = row.stop_time - row.start_time duration = (duration_.seconds + duration_.days * 24 * 3600) if status not in gb_duration_series: gb_duration_series[status] = defaultdict(int, {duration: 1}) else: gb_duration_series[status][duration] += 1 else: gb_duration_rows = dbs(q).select(count_, status_, duration_g, groupby=status_ | duration_g, orderby=status_ | duration_g) #convert to duration series gb_duration_series = {} for row in gb_duration_rows: status = row.scheduler_run.status duration = row[duration_g] howmany = row[count_] if status not in gb_duration_series: gb_duration_series[status] = {duration: howmany} else: if duration not in gb_duration_series[status]: gb_duration_series[status][duration] = howmany jgb_duration_series = [] for k, v in gb_duration_series.items(): jgb_duration_series.append({ 'label': k, 'data': [[kk, vv] for kk, vv in v.items()], 'color': graph_colors_task_status(k) }) return gb_duration_rows, jgb_duration_series
def jobs_render(): session.forget(response) if request.vars.job_name: q = sd.job_name == request.vars.job_name all_jobs = {request.vars.job_name : 'a_0'} newjob = None else: q = sd.id > 0 all_jobs = dbs(q).select(sd.job_name, groupby=sd.job_name) all_jobs = dict([(row.job_name, "a_%s" % k) for k, row in enumerate(all_jobs)]) newjob = str(int(time.mktime(datetime.datetime.utcnow().timetuple()))) all_deps = dbs(q).select() all_nodes = {} all_edges = {} for row in all_deps: jname_slug = all_jobs[row.job_name] if jname_slug not in all_nodes: all_nodes[jname_slug] = {} all_nodes[jname_slug][row.task_parent] = None all_nodes[jname_slug][row.task_child] = None if jname_slug not in all_edges: all_edges[jname_slug] = [] all_edges[jname_slug].append((row.task_child, row.task_parent, dict(style="stroke: %s" % (row.can_visit and '#5CB85C' or '#D9534F')))) all_tasks_ids = {} for k, v in all_nodes.iteritems(): for id in v.keys(): all_tasks_ids[id] = None all_tasks_labels = dbs(st.id.belongs(all_tasks_ids.keys())).select( st.id, st.function_name, st.task_name, st.status ).as_dict() for k, v in all_nodes.iteritems(): for id in v.keys(): all_nodes[k][id] = dict(label=all_tasks_labels[id]['task_name'], style="fill: %s;" % graph_colors_task_status(all_tasks_labels[id]['status']), title="%(id)s (%(function_name)s): %(status)s" % all_tasks_labels[id], linkto=URL('task_details', args=id, user_signature=True)) return dict(all_jobs=all_jobs, all_edges=dumps(all_edges), all_nodes=dumps(all_nodes), newjob=newjob)
def gb_status(q, mode='runs'): #bystatus if GROUPING_MODE == 'python': if mode == 'runs': gb_status_rows = dbs(q).select(sr.status, orderby=sr.status, **ANALYZE_CACHE_KWARGS) gb_status_series = defaultdict(int) for row in gb_status_rows: status = row.status gb_status_series[status] += 1 else: gb_status_series = defaultdict(int) gb_status_rows = dbs(q).select(st.status, st.times_run, **ANALYZE_CACHE_KWARGS) for row in gb_status_rows: gb_status_series[row.status] += row.times_run else: if mode == 'runs': status_ = sr.status count_ = sr.id.count() gb_status_rows = dbs(q).select(count_, status_, groupby=status_, orderby=status_, **ANALYZE_CACHE_KWARGS) gb_status_series = defaultdict(int) for row in gb_status_rows: status = row.scheduler_run.status howmany = row[count_] gb_status_series[status] += howmany else: status_ = st.status count_ = st.times_run.sum() gb_status_rows = dbs(q).select(count_, status_, groupby=status_, orderby=status_, **ANALYZE_CACHE_KWARGS) gb_status_series = defaultdict(int) for row in gb_status_rows: status = row.scheduler_task.status howmany = row[count_] gb_status_series[status] += howmany jgb_status_series = [] for k,v in gb_status_series.items(): jgb_status_series.append( {'label' : k, 'color' : graph_colors_task_status(k), 'data' : [v]} ) return gb_status_rows, jgb_status_series
def gb_duration(q): #byduration count_ = sr.id.count() status_ = sr.status duration_g = sr.stop_time.epoch() - sr.start_time.epoch() if GROUPING_MODE == 'python': gb_duration_rows = dbs(q).select(status_, sr.start_time, sr.stop_time, orderby=status_|duration_g) gb_duration_series = {} for row in gb_duration_rows: status = row.status duration_ = row.stop_time - row.start_time duration = (duration_.seconds + duration_.days * 24 * 3600) if status not in gb_duration_series: gb_duration_series[status] = defaultdict(int, {duration : 1}) else: gb_duration_series[status][duration] += 1 else: gb_duration_rows = dbs(q).select(count_, status_, duration_g, groupby=status_|duration_g, orderby=status_|duration_g) #convert to duration series gb_duration_series = {} for row in gb_duration_rows: status = row.scheduler_run.status duration = row[duration_g] howmany = row[count_] if status not in gb_duration_series: gb_duration_series[status] = {duration : howmany} else: if duration not in gb_duration_series[status]: gb_duration_series[status][duration] = howmany jgb_duration_series = [] for k,v in gb_duration_series.items(): jgb_duration_series.append( {'label': k, 'data' : [[kk,vv] for kk,vv in v.items()], 'color' : graph_colors_task_status(k)} ) return gb_duration_rows, jgb_duration_series
def byday(q, day, mode): #by period if GROUPING_MODE == 'python': if mode == 'runs': gb_whend_rows = dbs(q).select(sr.status, sr.start_time, orderby=sr.status | sr.start_time, **ANALYZE_CACHE_KWARGS) gb_whend_series = {} for row in gb_whend_rows: status = row.status refdate = row.start_time.strftime('%Y-%m-%d %H:%M') if status not in gb_whend_series: gb_whend_series[status] = defaultdict(int, {refdate: 1}) else: gb_whend_series[status][refdate] += 1 else: gb_whend_rows = dbs(q).select(st.times_run, st.status, st.last_run_time, orderby=st.status | st.last_run_time, **ANALYZE_CACHE_KWARGS) gb_whend_series = {} for row in gb_whend_rows: status = row.status refdate = row.last_run_time.strftime('%Y-%m-%d %H:%M') if status not in gb_whend_series: gb_whend_series[status] = defaultdict( int, {refdate: row.times_run}) else: gb_whend_series[status][refdate] += row.times_run else: if mode == 'runs': count_ = sr.id.count() status_ = sr.status d = sr.start_time.hour() | sr.start_time.minutes() gb_whend_rows = dbs(q).select(count_, status_, sr.start_time.hour(), sr.start_time.minutes(), groupby=status_ | d, orderby=status_ | d, **ANALYZE_CACHE_KWARGS) gb_whend_series = {} for row in gb_whend_rows: status = row.scheduler_run.status howmany = row[count_] refdate = day.year, day.month, day.day, row[ sr.start_time.hour()], row[sr.start_time.minutes()], 0 refdate = datetime.datetime( *refdate).strftime('%Y-%m-%d %H:%M') if status not in gb_whend_series: gb_whend_series[status] = {refdate: howmany} else: gb_whend_series[status][refdate] = howmany else: count_ = st.times_run.sum() status_ = st.status d = st.last_run_time.hour() | st.last_run_time.minutes() gb_whend_rows = dbs(q).select(count_, status_, st.last_run_time.hour(), st.last_run_time.minutes(), groupby=status_ | d, orderby=status_ | d, **ANALYZE_CACHE_KWARGS) gb_whend_series = {} for row in gb_whend_rows: status = row.scheduler_task.status howmany = row[count_] refdate = day.year, day.month, day.day, row[ st.last_run_time.hour()], row[ st.last_run_time.minutes()], 0 refdate = datetime.datetime( *refdate).strftime('%Y-%m-%d %H:%M') if status not in gb_whend_series: gb_whend_series[status] = {refdate: howmany} else: gb_whend_series[status][refdate] = howmany jgb_whend_series = [] for k, v in gb_whend_series.items(): jgb_whend_series.append({ 'label': k, 'data': [[kk, vv] for kk, vv in v.items()], 'color': graph_colors_task_status(k) }) return gb_whend_rows, jgb_whend_series
def byday(q, day, mode): #by period if GROUPING_MODE == 'python': if mode == 'runs': gb_whend_rows = dbs(q).select(sr.status, sr.start_time, orderby=sr.status|sr.start_time, **ANALYZE_CACHE_KWARGS) gb_whend_series = {} for row in gb_whend_rows: status = row.status refdate = row.start_time.strftime('%Y-%m-%d %H:%M') if status not in gb_whend_series: gb_whend_series[status] = defaultdict(int, {refdate : 1}) else: gb_whend_series[status][refdate] += 1 else: gb_whend_rows = dbs(q).select(st.times_run, st.status, st.last_run_time, orderby=st.status|st.last_run_time, **ANALYZE_CACHE_KWARGS) gb_whend_series = {} for row in gb_whend_rows: status = row.status refdate = row.last_run_time.strftime('%Y-%m-%d %H:%M') if status not in gb_whend_series: gb_whend_series[status] = defaultdict(int, {refdate : row.times_run}) else: gb_whend_series[status][refdate] += row.times_run else: if mode == 'runs': count_ = sr.id.count() status_ = sr.status d = sr.start_time.hour()|sr.start_time.minutes() gb_whend_rows = dbs(q).select(count_, status_, sr.start_time.hour(), sr.start_time.minutes(), groupby=status_|d, orderby=status_|d, **ANALYZE_CACHE_KWARGS) gb_whend_series = {} for row in gb_whend_rows: status = row.scheduler_run.status howmany = row[count_] refdate = day.year, day.month, day.day, row[sr.start_time.hour()], row[sr.start_time.minutes()], 0 refdate = datetime.datetime(*refdate).strftime('%Y-%m-%d %H:%M') if status not in gb_whend_series: gb_whend_series[status] = {refdate : howmany} else: gb_whend_series[status][refdate] = howmany else: count_ = st.times_run.sum() status_ = st.status d = st.last_run_time.hour()|st.last_run_time.minutes() gb_whend_rows = dbs(q).select(count_, status_, st.last_run_time.hour(), st.last_run_time.minutes(), groupby=status_|d, orderby=status_|d, **ANALYZE_CACHE_KWARGS) gb_whend_series = {} for row in gb_whend_rows: status = row.scheduler_task.status howmany = row[count_] refdate = day.year, day.month, day.day, row[st.last_run_time.hour()], row[st.last_run_time.minutes()], 0 refdate = datetime.datetime(*refdate).strftime('%Y-%m-%d %H:%M') if status not in gb_whend_series: gb_whend_series[status] = {refdate : howmany} else: gb_whend_series[status][refdate] = howmany jgb_whend_series = [] for k, v in gb_whend_series.items(): jgb_whend_series.append( {'label': k, 'data' : [[kk,vv] for kk,vv in v.items()], 'color' : graph_colors_task_status(k)} ) return gb_whend_rows, jgb_whend_series