def duration(self, t: Task):
if not t.started:
return duration_format(0)
finished = (
t.finished if t.finished else now()
)
delta = (finished - t.started).total_seconds()
return duration_format(delta)
def get(self, filter: dict, options: PaginatorOptions):
query = self.query(Task, Project.name). \
join(Dag, Dag.id == Task.dag). \
join(Project, Project.id == Dag.project). \
options(joinedload(Task.dag_rel, innerjoin=True))
query = self._get_filter(query, filter)
total = query.count()
paginator = self.paginator(query, options)
res = []
for p, project_name in paginator.all():
if p.dag_rel is None:
continue
item = {**self.to_dict(p, rules=('-additional_info', ))}
item['status'] = to_snake(TaskStatus(item['status']).name)
item['type'] = to_snake(TaskType(item['type']).name)
item['dag_rel']['project'] = {
'id': item['dag_rel']['project'],
'name': project_name
}
if p.started is None:
delta = 0
elif p.status == TaskStatus.InProgress.value:
delta = (now() - p.started).total_seconds()
else:
finish = (p.finished or p.last_activity)
delta = (finish - p.started).total_seconds()
item['duration'] = duration_format(delta)
if p.dag_rel is not None:
res.append(item)
if filter.get('report'):
tasks_within_report = self.query(
ReportTasks.task
).filter(ReportTasks.report == int(filter['report']))
tasks_within_report = {t[0] for t in tasks_within_report}
for r in res:
r['report_full'] = r['id'] in tasks_within_report
projects = self.query(Project.name, Project.id). \
order_by(Project.id.desc()). \
limit(20). \
all()
dags = self.query(Dag.name, Dag.id). \
order_by(Dag.id.desc()). \
limit(20). \
all()
projects = [{'name': name, 'id': id} for name, id in projects]
dags = [{'name': name, 'id': id} for name, id in dags]
dags_model = self.query(Dag.name, Dag.id, Dag.project). \
filter(Dag.type == DagType.Pipe.value). \
order_by(Dag.id.desc()). \
all()
dags_model_dict = []
used_dag_names = set()
for name, id, project in dags_model:
if name in used_dag_names:
continue
dag = {'name': name, 'id': id, 'project': project}
dags_model_dict.append(dag)
used_dag_names.add(name)
return {
'total': total,
'data': res,
'projects': projects,
'dags': dags,
'dags_model': dags_model_dict
}
def get(self, filter: dict, options: PaginatorOptions = None):
task_status = []
for e in TaskStatus:
task_status.append(
func.sum(
case(
whens=[(Task.status == e.value, 1)],
else_=0
).label(e.name)
)
)
last_activity = func.max(Task.last_activity).label('last_activity')
funcs = [
func.count(Task.id).label('task_count'), last_activity,
func.min(Task.started).label('started'),
func.max(Task.finished).label('finished')
]
query = self.query(Dag, Project.name, *funcs,
*task_status).join(Project)
query = self._get_filter(query, filter, last_activity)
status_clauses = []
for agg, e in zip(task_status, TaskStatus):
if filter.get('status', {}).get(to_snake(e.name)):
status_clauses.append(agg > 0)
if len(status_clauses) > 0:
query = query.having(or_(*status_clauses))
query = query.join(Task, isouter=True).group_by(Dag.id, Project.name)
# Do not include service tasks
query = query.filter(Task.type < TaskType.Service.value)
total = query.count()
paginator = self.paginator(query, options) if options else query
res = []
rules = ('-tasks.dag_rel', )
for dag, \
project_name, \
task_count, \
last_activity, \
started, \
finished, \
*(task_status) in paginator.all():
items = self.to_dict(dag, rules=rules).items()
# noinspection PyDictCreation
r = {
'task_count': task_count,
'last_activity': last_activity,
'started': started,
'finished': finished,
**{k: v
for k, v in items if k not in ['tasks', 'config']}
}
r['project'] = {'name': project_name}
r['task_statuses'] = [
{
'name': to_snake(e.name),
'count': s
} for e, s in zip(TaskStatus, task_status)
]
r['last_activity'] = self.serializer.serialize_datetime(
r['last_activity']
) if r['last_activity'] else None
r['started'] = self.serializer.serialize_datetime(r['started']) \
if r['started'] else None
r['finished'] = self.serializer.serialize_datetime(
r['finished']
) if r['finished'] else None
if task_status[TaskStatus.InProgress.value] > 0:
delta = (now() - started).total_seconds()
elif sum(
task_status[TaskStatus.InProgress.value:]
) == 0 or not started or not last_activity:
delta = 0
else:
delta = (last_activity - started).total_seconds()
r['duration'] = duration_format(delta)
res.append(r)
if filter.get('report'):
dag_ids = [r['id'] for r in res]
tasks_dags = self.query(Task.id, Task.dag). \
filter(Task.type <= TaskType.Train.value). \
filter(Task.dag.in_(dag_ids)). \
all()
tasks_within_report = self.query(ReportTasks.task). \
filter(ReportTasks.report == int(filter['report']))
tasks_within_report = {t[0] for t in tasks_within_report}
dags_not_full_included = {
d
for t, d in tasks_dags if t not in tasks_within_report
}
for r in res:
r['report_full'] = r['id'] not in dags_not_full_included
projects = self.query(Project.name, Project.id). \
order_by(Project.id.desc()). \
limit(20). \
all()
projects = [{'name': name, 'id': id} for name, id in projects]
return {'total': total, 'data': res, 'projects': projects}