def to_graph(self, engine_name, entity_models): visited_nodes = set() visited_edges = set() nodes = [] edges = [] for entity in entity_models: if isinstance(entity, Dataset): self._add_node(nodes, visited_nodes, 'dataset', entity.id, entity.data.name, None, None) if isinstance(entity, Action): self._add_node(nodes, visited_nodes, 'action', entity.id, entity.data.name, entity.data.state, entity.data.action_type_name) if entity.data.args and 'dataset_id' in entity.data.args: self._add_edge(edges, visited_edges, 'dataset', entity.data.args['dataset_id'], 'action', entity.id) if entity.data.args and 'subscription_id' in entity.data.args: self._add_edge(edges, visited_edges, 'subscription', entity.data.args['subscription_id'], 'action', entity.id) if entity.data.workflow_id: self._add_edge(edges, visited_edges, 'workflow', entity.data.workflow_id, 'action', entity.id) if entity.data.datastore_id: self._add_edge(edges, visited_edges, 'datastore', entity.data.datastore_id, 'action', entity.id) if isinstance(entity, Workflow): self._add_node(nodes, visited_nodes, 'workflow', entity.id, entity.data.name, entity.data.state, None) if entity.data.datastore_id: self._add_edge(edges, visited_edges, 'datastore', entity.data.datastore_id, 'workflow', entity.id) if isinstance(entity, Trigger): self._add_node(nodes, visited_nodes, 'trigger', entity.id, entity.data.name, entity.data.state, entity.data.trigger_type_name) if entity.data.workflow_ids: for wf_id in entity.data.workflow_ids: self._add_edge(edges, visited_edges, 'trigger', entity.id, 'workflow', wf_id) if entity.data.args and 'completed_trigger_ids' in entity.data.args: for t_id in entity.data.args['completed_trigger_ids']: self._add_edge(edges, visited_edges, 'trigger', t_id, 'trigger', entity.id) if entity.data.args and 'event_id' in entity.data.args: self._add_edge(edges, visited_edges, 'event', entity.data.args['event_id'], 'trigger', entity.id) if entity.data.args and 'completed_workflow_id' in entity.data.args: self._add_edge(edges, visited_edges, 'workflow', entity.data.args['completed_workflow_id'], 'trigger', entity.id) if entity.data.args and 'subscription_id' in entity.data.args: self._add_edge(edges, visited_edges, 'subscription', entity.data.args['subscription_id'], 'trigger', entity.id) if isinstance(entity, Subscription): self._add_node(nodes, visited_nodes, 'subscription', entity.id, entity.data.name, entity.data.state, None) if entity.data.dataset_id: self._add_edge(edges, visited_edges, 'dataset', entity.data.dataset_id, 'subscription', entity.id) if isinstance(entity, Event): self._add_node(nodes, visited_nodes, 'event', entity.id, entity.data.name, entity.data.state, None) if isinstance(entity, Datastore): self._add_node(nodes, visited_nodes, 'datastore', entity.id, entity.data.name, entity.data.state, engine_name) return Graph(nodes, edges)
def get_entity_graph(self, entity): """ :type entity: dart.model.graph.GraphEntity :rtype: dart.model.graph.Graph """ statement = text(RECURSIVE_SQL).bindparams( entity_type=entity.entity_type, entity_id=entity.entity_id, name=entity.name, state=entity.state, sub_type=entity.sub_type) visited_nodes = set() visited_edges = set() nodes = [] edges = [] for e in [GraphEntity(*r) for r in db.session.execute(statement)]: self._add_node(nodes, visited_nodes, e.entity_type, e.entity_id, e.name, e.state, e.sub_type) if e.related_id: if e.related_is_a == Relationship.CHILD: src_type = e.entity_type src_id = e.entity_id dst_type = e.related_type dst_id = e.related_id else: src_type = e.related_type src_id = e.related_id dst_type = e.entity_type dst_id = e.entity_id self._add_edge(edges, visited_edges, src_type, src_id, dst_type, dst_id) # now that we have the base graph, add in the most recent workflow_instances and their actions, # as well as recent datastore actions that are not associated with any workflows (if any) wf_ids, wf_sql = self._get_workflow_instance_sql(nodes) d_ids, d_sql = self._get_datastore_one_offs(nodes) sql_parts = [] if wf_sql: sql_parts.append('(' + wf_sql + ')') if d_sql: sql_parts.append('(' + d_sql + ')') if len(sql_parts) == 0: return Graph(nodes, edges) sql = '\nUNION ALL\n'.join(sql_parts) statement = text( sql.format(wf_ids=', '.join( ["'" + wf_id + "'" for wf_id in wf_ids]), d_ids=', '.join(["'" + d_id + "'" for d_id in d_ids]))) for r in db.session.execute(statement): if r[0] == 'workflow': entity_type, wf_id, wfi_id, wfi_progress, wfi_state, a_id, a_name, a_state, a_sub_type = r name = 'workflow_instance - %s%%' % wfi_progress if wfi_state == 'RUNNING' else 'workflow_instance' self._add_node(nodes, visited_nodes, 'workflow_instance', wfi_id, name, wfi_state, None) self._add_edge(edges, visited_edges, 'workflow', wf_id, 'workflow_instance', wfi_id) if a_id: self._add_node(nodes, visited_nodes, 'action', a_id, a_name, a_state, a_sub_type) self._add_edge(edges, visited_edges, 'workflow_instance', wfi_id, 'action', a_id) else: entity_type, d_id, skip1, skip2, skip3, a_id, a_name, a_state, a_sub_type = r self._add_node(nodes, visited_nodes, 'action', a_id, a_name, a_state, a_sub_type) self._add_edge(edges, visited_edges, 'datastore', d_id, 'action', a_id) return Graph(nodes, edges)
def get_entity_graph(self, entity): """ :type entity: dart.model.graph.GraphEntity :rtype: dart.model.graph.Graph """ key = "graph:{0}".format(entity.entity_id) cached_graph = self.redis_client.get(key) if self.redis_client else None # String representation of each Graph record if cached_graph: json_records = json.loads(cached_graph) ### read state from database: # we need to update the current state of object in the graph get_state_sql = ["select '{0}' as type, '{1}' as id,'{2}' as name, data->>'state' as state, '{4}' as sub_type, '{5}' as related_type, '{6}' as related_id, '{7}' as related_is_a FROM {0} where id = '{1}'".format(*rec) for rec in json_records] # add union all between sql statements state_sql = " union all ".join(get_state_sql) # execute sql: (dart_depth was already removed, so we use r instead of r[0:-1]) records = [(r) for r in db.session.execute(text(state_sql))] else: records = [ (r[:-1]) # remove depth parameter for r in db.session.execute(text(RECURSIVE_SQL).bindparams( entity_type=entity.entity_type, entity_id=entity.entity_id, name=entity.name, state=entity.state, sub_type=entity.sub_type )) ] string_records = json.dumps(records) if self.redis_client: self.redis_client.setex(key, self.redis_expiration_ttl, string_records) # Keep this graph cached for 10 minutes # rebuilding graph from string entities = [GraphEntity(*r) for r in records] nodes, visited_nodes = [], set() edges, visited_edges = [], set() for e in entities: self._add_node(nodes, visited_nodes, e.entity_type, e.entity_id, e.name, e.state, e.sub_type) if e.related_id: if e.related_is_a == Relationship.CHILD: src_type = e.entity_type src_id = e.entity_id dst_type = e.related_type dst_id = e.related_id else: src_type = e.related_type src_id = e.related_id dst_type = e.entity_type dst_id = e.entity_id self._add_edge(edges, visited_edges, src_type, src_id, dst_type, dst_id) # now that we have the base graph, add in the most recent workflow_instances and their actions, # as well as recent datastore actions that are not associated with any workflows (if any) wf_ids, wf_sql = self._get_workflow_instance_sql(nodes) d_ids, d_sql = self._get_datastore_one_offs(nodes) sql_parts = [] if wf_sql: sql_parts.append('(' + wf_sql + ')') if d_sql: sql_parts.append('(' + d_sql + ')') if len(sql_parts) == 0: return Graph(nodes, edges) sql = '\nUNION ALL\n'.join(sql_parts) statement = text(sql.format( wf_ids=', '.join(["'" + wf_id + "'" for wf_id in wf_ids]), d_ids=', '.join(["'" + d_id + "'" for d_id in d_ids]) )) for r in db.session.execute(statement): if r[0] == 'workflow': entity_type, wf_id, wfi_id, wfi_state, a_id, a_name, a_state, a_sub_type = r name = 'workflow_instance' if wfi_state == 'RUNNING' and ('workflow_instance' + wfi_id) not in visited_nodes: actions = self._action_service.find_actions(workflow_instance_id=wfi_id) if None not in [a.data.avg_runtime for a in actions]: wfi_progress = 0 wf_runtime = 0 for a in actions: a_runtime = a.data.avg_runtime.total_seconds() wf_runtime += a_runtime if a.data.state == ActionState.COMPLETED or a.data.state == ActionState.FAILED: wfi_progress += a_runtime elif (a.data.state == ActionState.RUNNING or a.data.state == ActionState.FINISHING) and a.data.progress: wfi_progress += a_runtime * a.data.progress if wf_runtime: wfi_progress /= wf_runtime name += ' - {:.0%}'.format(wfi_progress) self._add_node(nodes, visited_nodes, 'workflow_instance', wfi_id, name, wfi_state, None) self._add_edge(edges, visited_edges, 'workflow', wf_id, 'workflow_instance', wfi_id) if a_id: self._add_node(nodes, visited_nodes, 'action', a_id, a_name, a_state, a_sub_type) self._add_edge(edges, visited_edges, 'workflow_instance', wfi_id, 'action', a_id) else: entity_type, d_id, skip1, skip2, a_id, a_name, a_state, a_sub_type = r self._add_node(nodes, visited_nodes, 'action', a_id, a_name, a_state, a_sub_type) self._add_edge(edges, visited_edges, 'datastore', d_id, 'action', a_id) return Graph(nodes, edges)
def get_entity_graph(self, entity): """ :type entity: dart.model.graph.GraphEntity :rtype: dart.model.graph.Graph """ statement = text(RECURSIVE_SQL).bindparams( entity_type=entity.entity_type, entity_id=entity.entity_id, name=entity.name, state=entity.state, sub_type=entity.sub_type) visited_nodes = set() visited_edges = set() nodes = [] edges = [] for e in [GraphEntity(*r) for r in db.session.execute(statement)]: self._add_node(nodes, visited_nodes, e.entity_type, e.entity_id, e.name, e.state, e.sub_type) if e.related_id: if e.related_is_a == Relationship.CHILD: src_type = e.entity_type src_id = e.entity_id dst_type = e.related_type dst_id = e.related_id else: src_type = e.related_type src_id = e.related_id dst_type = e.entity_type dst_id = e.entity_id self._add_edge(edges, visited_edges, src_type, src_id, dst_type, dst_id) # now that we have the base graph, add in the most recent workflow_instances and their actions, # as well as recent datastore actions that are not associated with any workflows (if any) wf_ids, wf_sql = self._get_workflow_instance_sql(nodes) d_ids, d_sql = self._get_datastore_one_offs(nodes) sql_parts = [] if wf_sql: sql_parts.append('(' + wf_sql + ')') if d_sql: sql_parts.append('(' + d_sql + ')') if len(sql_parts) == 0: return Graph(nodes, edges) sql = '\nUNION ALL\n'.join(sql_parts) statement = text( sql.format(wf_ids=', '.join( ["'" + wf_id + "'" for wf_id in wf_ids]), d_ids=', '.join(["'" + d_id + "'" for d_id in d_ids]))) for r in db.session.execute(statement): if r[0] == 'workflow': entity_type, wf_id, wfi_id, wfi_state, a_id, a_name, a_state, a_sub_type = r name = 'workflow_instance' if wfi_state == 'RUNNING' and ('workflow_instance' + wfi_id) not in visited_nodes: actions = self._action_service.find_actions( workflow_instance_id=wfi_id) if None not in [a.data.avg_runtime for a in actions]: wfi_progress = 0 wf_runtime = 0 for a in actions: a_runtime = a.data.avg_runtime.total_seconds() wf_runtime += a_runtime if a.data.state == ActionState.COMPLETED or a.data.state == ActionState.FAILED: wfi_progress += a_runtime elif (a.data.state == ActionState.RUNNING or a.data.state == ActionState.FINISHING ) and a.data.progress: wfi_progress += a_runtime * a.data.progress if wf_runtime: wfi_progress /= wf_runtime name += ' - {:.0%}'.format(wfi_progress) self._add_node(nodes, visited_nodes, 'workflow_instance', wfi_id, name, wfi_state, None) self._add_edge(edges, visited_edges, 'workflow', wf_id, 'workflow_instance', wfi_id) if a_id: self._add_node(nodes, visited_nodes, 'action', a_id, a_name, a_state, a_sub_type) self._add_edge(edges, visited_edges, 'workflow_instance', wfi_id, 'action', a_id) else: entity_type, d_id, skip1, skip2, a_id, a_name, a_state, a_sub_type = r self._add_node(nodes, visited_nodes, 'action', a_id, a_name, a_state, a_sub_type) self._add_edge(edges, visited_edges, 'datastore', d_id, 'action', a_id) return Graph(nodes, edges)