def graph(self, id: int):
tasks = self.query(Task). \
filter(Task.dag == id). \
filter(Task.type <= TaskType.Train.value). \
all()
task_ids = [t.id for t in tasks]
dep = self.query(TaskDependence).filter(
TaskDependence.task_id.in_(task_ids)).all()
task_by_id = {t.id: t for t in tasks}
def label(t: Task):
res = [t.executor]
if t.status >= TaskStatus.InProgress.value:
res.append(self.duration(t))
res.append(f'{t.current_step if t.current_step else ""}/'
f'{t.steps if t.steps else ""}')
return '\n'.join(res)
nodes = [{
'id': t.id,
'label': label(t),
'name': t.name,
'status': to_snake(TaskStatus(t.status).name)
} for t in tasks]
edges = [{
'from':
d.depend_id,
'to':
d.task_id,
'status':
to_snake(TaskStatus(task_by_id[d.depend_id].status).name)
} for d in dep]
return {'nodes': nodes, 'edges': edges}
def load_tasks(self):
self.tasks = self.provider.by_status(TaskStatus.NotRan,
TaskStatus.InProgress,
TaskStatus.Queued)
not_ran_tasks = [t for t in self.tasks if
t.status == TaskStatus.NotRan.value]
self.not_ran_tasks = [task for task in not_ran_tasks if not task.debug]
self.not_ran_tasks = sorted(
self.not_ran_tasks, key=lambda x: x.gpu or 0,
reverse=True)
self.logger.debug(
f'Found {len(not_ran_tasks)} not ran tasks',
ComponentType.Supervisor
)
self.dep_status = self.provider.dependency_status(self.not_ran_tasks)
self.auxiliary['not_ran_tasks'] = [
{
'id': t.id,
'name': t.name,
'dep_status': [
TaskStatus(s).name
for s in self.dep_status.get(t.id, set())
]
} for t in not_ran_tasks[:5]
]
def task_stop():
data = request_data()
provider = TaskProvider(_write_session)
task = provider.by_id(data['id'], joinedload(Task.dag_rel, innerjoin=True))
dag = task.dag_rel
status = celery_tasks.stop(logger, _write_session, task, dag)
child_tasks = provider.children(task.id)
for t in child_tasks:
celery_tasks.stop(logger, _write_session, t, dag)
return {'status': to_snake(TaskStatus(status).name)}
def _get_filter(self, query, filter: dict):
if filter.get('dag'):
query = query.filter(Task.dag == filter['dag'])
if filter.get('name'):
query = query.filter(Task.name.like(f'%{filter["name"]}%'))
if filter.get('status'):
status = [
TaskStatus.from_name(k) for k, v in filter['status'].items()
if v
]
if len(status) > 0:
query = query.filter(Task.status.in_(status))
if filter.get('id_min'):
query = query.filter(Task.id >= filter['id_min'])
if filter.get('id_max'):
query = query.filter(Task.id <= filter['id_max'])
if filter.get('id'):
query = query.filter(Task.id == filter['id'])
if filter.get('project'):
query = query.filter(Dag.project == filter['project'])
if filter.get('created_min'):
created_min = parse_time(filter['created_min'])
query = query.filter(Dag.created >= created_min)
if filter.get('created_max'):
created_max = parse_time(filter['created_max'])
query = query.filter(Dag.created <= created_max)
if filter.get('last_activity_min'):
last_activity_min = parse_time(filter['last_activity_min'])
query = query.filter(Task.last_activity >= last_activity_min)
if filter.get('last_activity_max'):
last_activity_max = parse_time(filter['last_activity_max'])
query = query.filter(Task.last_activity <= last_activity_max)
if filter.get('report'):
query = query.filter(Task.report is not None)
if filter.get('parent'):
query = query.filter(Task.parent == filter['parent'])
types = filter.get('type', ['User', 'Train'])
types = [TaskType.from_name(t) for t in types]
query = query.filter(Task.type.in_(types))
return query
def load_tasks(self):
not_ran_tasks = self.provider.by_status(TaskStatus.NotRan)
self.not_ran_tasks = [task for task in not_ran_tasks if not task.debug]
self.logger.debug(f'Found {len(not_ran_tasks)} not ran tasks',
ComponentType.Supervisor)
self.dep_status = self.provider.dependency_status(self.not_ran_tasks)
self.auxiliary['not_ran_tasks'] = [{
'id':
t.id,
'name':
t.name,
'dep_status':
[TaskStatus(s).name for s in self.dep_status.get(t.id, set())]
} for t in not_ran_tasks]
def get(self, filter: dict, options: PaginatorOptions):
query = self.query(Task, Project.name). \
join(Dag, Dag.id == Task.dag). \
join(Project, Project.id == Dag.project). \
options(joinedload(Task.dag_rel, innerjoin=True))
query = self._get_filter(query, filter)
total = query.count()
paginator = self.paginator(query, options)
res = []
for p, project_name in paginator.all():
if p.dag_rel is None:
continue
item = {**self.to_dict(p, rules=('-additional_info', ))}
item['status'] = to_snake(TaskStatus(item['status']).name)
item['type'] = to_snake(TaskType(item['type']).name)
item['dag_rel']['project'] = {
'id': item['dag_rel']['project'],
'name': project_name
}
if p.started is None:
delta = 0
elif p.status == TaskStatus.InProgress.value:
delta = (now() - p.started).total_seconds()
else:
finish = (p.finished or p.last_activity)
delta = (finish - p.started).total_seconds()
item['duration'] = duration_format(delta)
if p.dag_rel is not None:
res.append(item)
if filter.get('report'):
tasks_within_report = self.query(
ReportTasks.task
).filter(ReportTasks.report == int(filter['report']))
tasks_within_report = {t[0] for t in tasks_within_report}
for r in res:
r['report_full'] = r['id'] in tasks_within_report
projects = self.query(Project.name, Project.id). \
order_by(Project.id.desc()). \
limit(20). \
all()
dags = self.query(Dag.name, Dag.id). \
order_by(Dag.id.desc()). \
limit(20). \
all()
projects = [{'name': name, 'id': id} for name, id in projects]
dags = [{'name': name, 'id': id} for name, id in dags]
dags_model = self.query(Dag.name, Dag.id, Dag.project). \
filter(Dag.type == DagType.Pipe.value). \
order_by(Dag.id.desc()). \
all()
dags_model_dict = []
used_dag_names = set()
for name, id, project in dags_model:
if name in used_dag_names:
continue
dag = {'name': name, 'id': id, 'project': project}
dags_model_dict.append(dag)
used_dag_names.add(name)
return {
'total': total,
'data': res,
'projects': projects,
'dags': dags,
'dags_model': dags_model_dict
}
def describe_tasks(dag: int, axis):
provider = TaskProvider()
columns = ['Id', 'Started', 'Duration', 'Step', 'Status']
cells = []
cells_colours = []
tasks = provider.by_dag(dag)
status_colors = {
'not_ran': 'gray',
'queued': 'lightblue',
'in_progress': 'lime',
'failed': '#e83217',
'stopped': '#cb88ea',
'skipped': 'orange',
'success': 'green'
}
finish = True
for task in tasks:
started = ''
duration = ''
if task.status <= TaskStatus.InProgress.value:
finish = False
if task.started:
started = task.started.strftime('%m.%d %H:%M:%S')
if task.finished:
duration = (task.finished - task.started).total_seconds()
else:
duration = (now() - task.started).total_seconds()
if duration > 3600:
duration = f'{int(duration // 3600)} hours ' \
f'{int((duration % 3600) // 60)} min' \
f' {int(duration % 60)} sec'
elif duration > 60:
duration = f'{int(duration // 60)} min' \
f' {int(duration % 60)} sec'
else:
duration = f'{int(duration)} sec'
status = to_snake(TaskStatus(task.status).name)
status_color = status_colors[status]
task_cells = [
str(task.id), started, duration, task.current_step or '1', status
]
task_colors = ['white', 'white', 'white', 'white', status_color]
cells.append(task_cells)
cells_colours.append(task_colors)
table = axis.table(cellText=cells,
colLabels=columns,
cellColours=cells_colours,
cellLoc='center',
colWidths=[0.2, 0.3, 0.4, 0.1, 0.2],
bbox=[0, 0, 1.0, 1.0],
loc='center')
table.auto_set_font_size(False)
table.set_fontsize(14)
axis.set_xticks([])
axis.axis('off')
axis.set_title('Tasks')
return finish