def _get_filter(self, query, filter: dict, last_activity): if filter.get('project'): query = query.filter(Dag.project == filter['project']) if filter.get('name'): query = query.filter(Dag.name.like(f'%{filter["name"]}%')) if filter.get('id'): query = query.filter(Dag.id == int(filter['id'])) if filter.get('id_min'): query = query.filter(Dag.id >= filter['id_min']) if filter.get('id_max'): query = query.filter(Dag.id <= filter['id_max']) if filter.get('created_min'): created_min = parse_time(filter['created_min']) query = query.filter(Dag.created >= created_min) if filter.get('created_max'): created_max = parse_time(filter['created_max']) query = query.filter(Dag.created <= created_max) if filter.get('last_activity_min'): last_activity_min = parse_time(filter['last_activity_min']) query = query.having(last_activity >= last_activity_min) if filter.get('last_activity_max'): last_activity_max = parse_time(filter['last_activity_max']) query = query.having(last_activity <= last_activity_max) if filter.get('report'): query = query.filter(Dag.report is not None) return query
def _get_filter(self, query, filter: dict): if filter.get('dag'): query = query.filter(Task.dag == filter['dag']) if filter.get('name'): query = query.filter(Task.name.like(f'%{filter["name"]}%')) if filter.get('status'): status = [ TaskStatus.from_name(k) for k, v in filter['status'].items() if v ] if len(status) > 0: query = query.filter(Task.status.in_(status)) if filter.get('id_min'): query = query.filter(Task.id >= filter['id_min']) if filter.get('id_max'): query = query.filter(Task.id <= filter['id_max']) if filter.get('id'): query = query.filter(Task.id == filter['id']) if filter.get('project'): query = query.filter(Dag.project == filter['project']) if filter.get('created_min'): created_min = parse_time(filter['created_min']) query = query.filter(Dag.created >= created_min) if filter.get('created_max'): created_max = parse_time(filter['created_max']) query = query.filter(Dag.created <= created_max) if filter.get('last_activity_min'): last_activity_min = parse_time(filter['last_activity_min']) query = query.filter(Task.last_activity >= last_activity_min) if filter.get('last_activity_max'): last_activity_max = parse_time(filter['last_activity_max']) query = query.filter(Task.last_activity <= last_activity_max) if filter.get('report'): query = query.filter(Task.report is not None) if filter.get('parent'): query = query.filter(Task.parent == filter['parent']) types = filter.get('type', ['User', 'Train']) types = [TaskType.from_name(t) for t in types] query = query.filter(Task.type.in_(types)) return query
def get(self, filter: dict, options: PaginatorOptions = None): query = self.query(Computer) query = query.filter( Computer.last_synced >= now() - datetime.timedelta(days=1)) total = query.count() if options: query = self.paginator(query, options) res = [] for c in query.all(): item = self.to_dict(c) default_usage = { 'cpu': 0, 'memory': 0, 'gpu': [{ 'memory': 0, 'load': 0 } for i in range(item['gpu'])] } sync_status = 'Not synced' sync_date = None if c.last_synced: sync_date = self.serialize_datetime(c.last_synced) sync_status = f'Last synced' if c.syncing_computer: if c.last_synced is None or \ (now() - c.last_synced).total_seconds() >= 5: sync_status = f'Syncing with {c.syncing_computer}' if c.last_synced: sync_status += f' from ' sync_date = self.serialize_datetime(c.last_synced) item['sync_status'] = sync_status item['sync_date'] = sync_date item['usage'] = json.loads(item['usage']) \ if item['usage'] else default_usage item['memory'] = int(item['memory'] / 1000) item['usage']['cpu'] = int(item['usage']['cpu']) item['usage']['memory'] = int(item['usage']['memory']) for gpu in item['usage']['gpu']: gpu['memory'] = 0 if np.isnan(gpu['memory']) else int( gpu['memory']) gpu['load'] = 0 if np.isnan(gpu['load']) else int(gpu['load']) min_time = parse_time(filter.get('usage_min_time')) item['usage_history'] = self.usage_history( c.name, min_time ) item['dockers'] = self.dockers(c.name, c.cpu) res.append(item) return {'data': res, 'total': total}
def get(self, filter, options: PaginatorOptions): query = self.query(Model). \ options(joinedload(Model.dag_rel, innerjoin=False)). \ options(joinedload(Model.project_rel, innerjoin=True)) if filter.get('project'): query = query.filter(Model.project == filter['project']) if filter.get('name'): query = query.filter(Model.name.like(f'%{filter["name"]}%')) if filter.get('created_min'): created_min = parse_time(filter['created_min']) query = query.filter(Model.created >= created_min) if filter.get('created_max'): created_max = parse_time(filter['created_max']) query = query.filter(Model.created <= created_max) total = query.count() paginator = self.paginator(query, options) if options else query res = [] models = paginator.all() models_projects = set() for model in models: if not model.project_rel: model.project_rel = self.query(Project).filter( Project.id == model.project).one() row = self.to_dict(model, rules=('-project_rel.class_names', )) res.append(row) models_projects.add(model.project) last_activity = func.max(Task.last_activity).label('last_activity') projects = self.query(Project.name, Project.id, last_activity). \ join(Dag, Dag.project == Project.id, isouter=True). \ join(Task, isouter=True). \ group_by(Project.id). \ order_by(last_activity.desc()). \ all() projects = [{'name': name, 'id': id} for name, id, _ in projects] return {'total': total, 'data': res, 'projects': projects}
def get(self, filter, options: PaginatorOptions): query = self.query(Model). \ options(joinedload(Model.dag_rel, innerjoin=True)). \ options(joinedload(Model.project_rel, innerjoin=True)) if filter.get('project'): query = query.filter(Model.project == filter['project']) if filter.get('name'): query = query.filter(Model.name.like(f'%{filter["name"]}%')) if filter.get('created_min'): created_min = parse_time(filter['created_min']) query = query.filter(Model.created >= created_min) if filter.get('created_max'): created_max = parse_time(filter['created_max']) query = query.filter(Model.created <= created_max) total = query.count() paginator = self.paginator(query, options) if options else query res = [] models = paginator.all() models_projects = set() for model in models: row = self.to_dict(model, rules=('-project_rel.class_names', )) res.append(row) models_projects.add(model.project) models_dags = self.query(Dag). \ filter(Dag.type == DagType.Pipe.value). \ filter(Dag.project.in_(list(models_projects))). \ order_by(Dag.id.desc()). \ all() dags_by_project = defaultdict(list) used_dag_names = set() for dag in models_dags: if dag.name in used_dag_names: continue config = Config.from_yaml(dag.config) slots = [] for pipe in config['pipes'].values(): for k, v in pipe.items(): if 'slot' in v: if v['slot'] not in slots: slots.append(v['slot']) elif 'slots' in v: for slot in v['slots']: if slot not in slots: slots.append(slot) d = { 'name': dag.name, 'id': dag.id, 'slots': slots, 'interfaces': list(config['interfaces']), 'pipes': list(config['pipes']) } dags_by_project[dag.project].append(d) used_dag_names.add(dag.name) for row in res: row['dags'] = dags_by_project[row['project']] projects = self.query(Project.name, Project.id). \ order_by(Project.id.desc()). \ limit(20). \ all() projects = [{'name': name, 'id': id} for name, id in projects] return {'total': total, 'data': res, 'projects': projects}