def to_graph(self, engine_name, entity_models):
visited_nodes = set()
visited_edges = set()
nodes = []
edges = []
for entity in entity_models:
if isinstance(entity, Dataset):
self._add_node(nodes, visited_nodes, 'dataset', entity.id, entity.data.name, None, None)
if isinstance(entity, Action):
self._add_node(nodes, visited_nodes, 'action', entity.id, entity.data.name, entity.data.state, entity.data.action_type_name)
if entity.data.args and 'dataset_id' in entity.data.args:
self._add_edge(edges, visited_edges, 'dataset', entity.data.args['dataset_id'], 'action', entity.id)
if entity.data.args and 'subscription_id' in entity.data.args:
self._add_edge(edges, visited_edges, 'subscription', entity.data.args['subscription_id'], 'action', entity.id)
if entity.data.workflow_id:
self._add_edge(edges, visited_edges, 'workflow', entity.data.workflow_id, 'action', entity.id)
if entity.data.datastore_id:
self._add_edge(edges, visited_edges, 'datastore', entity.data.datastore_id, 'action', entity.id)
if isinstance(entity, Workflow):
self._add_node(nodes, visited_nodes, 'workflow', entity.id, entity.data.name, entity.data.state, None)
if entity.data.datastore_id:
self._add_edge(edges, visited_edges, 'datastore', entity.data.datastore_id, 'workflow', entity.id)
if isinstance(entity, Trigger):
self._add_node(nodes, visited_nodes, 'trigger', entity.id, entity.data.name, entity.data.state, entity.data.trigger_type_name)
if entity.data.workflow_ids:
for wf_id in entity.data.workflow_ids:
self._add_edge(edges, visited_edges, 'trigger', entity.id, 'workflow', wf_id)
if entity.data.args and 'completed_trigger_ids' in entity.data.args:
for t_id in entity.data.args['completed_trigger_ids']:
self._add_edge(edges, visited_edges, 'trigger', t_id, 'trigger', entity.id)
if entity.data.args and 'event_id' in entity.data.args:
self._add_edge(edges, visited_edges, 'event', entity.data.args['event_id'], 'trigger', entity.id)
if entity.data.args and 'completed_workflow_id' in entity.data.args:
self._add_edge(edges, visited_edges, 'workflow', entity.data.args['completed_workflow_id'], 'trigger', entity.id)
if entity.data.args and 'subscription_id' in entity.data.args:
self._add_edge(edges, visited_edges, 'subscription', entity.data.args['subscription_id'], 'trigger', entity.id)
if isinstance(entity, Subscription):
self._add_node(nodes, visited_nodes, 'subscription', entity.id, entity.data.name, entity.data.state, None)
if entity.data.dataset_id:
self._add_edge(edges, visited_edges, 'dataset', entity.data.dataset_id, 'subscription', entity.id)
if isinstance(entity, Event):
self._add_node(nodes, visited_nodes, 'event', entity.id, entity.data.name, entity.data.state, None)
if isinstance(entity, Datastore):
self._add_node(nodes, visited_nodes, 'datastore', entity.id, entity.data.name, entity.data.state, engine_name)
return Graph(nodes, edges)
def get_entity_graph(self, entity):
""" :type entity: dart.model.graph.GraphEntity
:rtype: dart.model.graph.Graph """
statement = text(RECURSIVE_SQL).bindparams(
entity_type=entity.entity_type,
entity_id=entity.entity_id,
name=entity.name,
state=entity.state,
sub_type=entity.sub_type)
visited_nodes = set()
visited_edges = set()
nodes = []
edges = []
for e in [GraphEntity(*r) for r in db.session.execute(statement)]:
self._add_node(nodes, visited_nodes, e.entity_type, e.entity_id,
e.name, e.state, e.sub_type)
if e.related_id:
if e.related_is_a == Relationship.CHILD:
src_type = e.entity_type
src_id = e.entity_id
dst_type = e.related_type
dst_id = e.related_id
else:
src_type = e.related_type
src_id = e.related_id
dst_type = e.entity_type
dst_id = e.entity_id
self._add_edge(edges, visited_edges, src_type, src_id,
dst_type, dst_id)
# now that we have the base graph, add in the most recent workflow_instances and their actions,
# as well as recent datastore actions that are not associated with any workflows (if any)
wf_ids, wf_sql = self._get_workflow_instance_sql(nodes)
d_ids, d_sql = self._get_datastore_one_offs(nodes)
sql_parts = []
if wf_sql:
sql_parts.append('(' + wf_sql + ')')
if d_sql:
sql_parts.append('(' + d_sql + ')')
if len(sql_parts) == 0:
return Graph(nodes, edges)
sql = '\nUNION ALL\n'.join(sql_parts)
statement = text(
sql.format(wf_ids=', '.join(
["'" + wf_id + "'" for wf_id in wf_ids]),
d_ids=', '.join(["'" + d_id + "'" for d_id in d_ids])))
for r in db.session.execute(statement):
if r[0] == 'workflow':
entity_type, wf_id, wfi_id, wfi_progress, wfi_state, a_id, a_name, a_state, a_sub_type = r
name = 'workflow_instance - %s%%' % wfi_progress if wfi_state == 'RUNNING' else 'workflow_instance'
self._add_node(nodes, visited_nodes, 'workflow_instance',
wfi_id, name, wfi_state, None)
self._add_edge(edges, visited_edges, 'workflow', wf_id,
'workflow_instance', wfi_id)
if a_id:
self._add_node(nodes, visited_nodes, 'action', a_id,
a_name, a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'workflow_instance',
wfi_id, 'action', a_id)
else:
entity_type, d_id, skip1, skip2, skip3, a_id, a_name, a_state, a_sub_type = r
self._add_node(nodes, visited_nodes, 'action', a_id, a_name,
a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'datastore', d_id,
'action', a_id)
return Graph(nodes, edges)
def get_entity_graph(self, entity):
""" :type entity: dart.model.graph.GraphEntity
:rtype: dart.model.graph.Graph """
key = "graph:{0}".format(entity.entity_id)
cached_graph = self.redis_client.get(key) if self.redis_client else None # String representation of each Graph record
if cached_graph:
json_records = json.loads(cached_graph)
### read state from database:
# we need to update the current state of object in the graph
get_state_sql = ["select '{0}' as type, '{1}' as id,'{2}' as name, data->>'state' as state, '{4}' as sub_type, '{5}' as related_type, '{6}' as related_id, '{7}' as related_is_a FROM {0} where id = '{1}'".format(*rec) for rec in json_records]
# add union all between sql statements
state_sql = " union all ".join(get_state_sql)
# execute sql: (dart_depth was already removed, so we use r instead of r[0:-1])
records = [(r) for r in db.session.execute(text(state_sql))]
else:
records = [
(r[:-1]) # remove depth parameter
for r in db.session.execute(text(RECURSIVE_SQL).bindparams(
entity_type=entity.entity_type,
entity_id=entity.entity_id,
name=entity.name,
state=entity.state,
sub_type=entity.sub_type
))
]
string_records = json.dumps(records)
if self.redis_client:
self.redis_client.setex(key, self.redis_expiration_ttl, string_records) # Keep this graph cached for 10 minutes
# rebuilding graph from string
entities = [GraphEntity(*r) for r in records]
nodes, visited_nodes = [], set()
edges, visited_edges = [], set()
for e in entities:
self._add_node(nodes, visited_nodes, e.entity_type, e.entity_id, e.name, e.state, e.sub_type)
if e.related_id:
if e.related_is_a == Relationship.CHILD:
src_type = e.entity_type
src_id = e.entity_id
dst_type = e.related_type
dst_id = e.related_id
else:
src_type = e.related_type
src_id = e.related_id
dst_type = e.entity_type
dst_id = e.entity_id
self._add_edge(edges, visited_edges, src_type, src_id, dst_type, dst_id)
# now that we have the base graph, add in the most recent workflow_instances and their actions,
# as well as recent datastore actions that are not associated with any workflows (if any)
wf_ids, wf_sql = self._get_workflow_instance_sql(nodes)
d_ids, d_sql = self._get_datastore_one_offs(nodes)
sql_parts = []
if wf_sql:
sql_parts.append('(' + wf_sql + ')')
if d_sql:
sql_parts.append('(' + d_sql + ')')
if len(sql_parts) == 0:
return Graph(nodes, edges)
sql = '\nUNION ALL\n'.join(sql_parts)
statement = text(sql.format(
wf_ids=', '.join(["'" + wf_id + "'" for wf_id in wf_ids]),
d_ids=', '.join(["'" + d_id + "'" for d_id in d_ids])
))
for r in db.session.execute(statement):
if r[0] == 'workflow':
entity_type, wf_id, wfi_id, wfi_state, a_id, a_name, a_state, a_sub_type = r
name = 'workflow_instance'
if wfi_state == 'RUNNING' and ('workflow_instance' + wfi_id) not in visited_nodes:
actions = self._action_service.find_actions(workflow_instance_id=wfi_id)
if None not in [a.data.avg_runtime for a in actions]:
wfi_progress = 0
wf_runtime = 0
for a in actions:
a_runtime = a.data.avg_runtime.total_seconds()
wf_runtime += a_runtime
if a.data.state == ActionState.COMPLETED or a.data.state == ActionState.FAILED:
wfi_progress += a_runtime
elif (a.data.state == ActionState.RUNNING or a.data.state == ActionState.FINISHING) and a.data.progress:
wfi_progress += a_runtime * a.data.progress
if wf_runtime:
wfi_progress /= wf_runtime
name += ' - {:.0%}'.format(wfi_progress)
self._add_node(nodes, visited_nodes, 'workflow_instance', wfi_id, name, wfi_state, None)
self._add_edge(edges, visited_edges, 'workflow', wf_id, 'workflow_instance', wfi_id)
if a_id:
self._add_node(nodes, visited_nodes, 'action', a_id, a_name, a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'workflow_instance', wfi_id, 'action', a_id)
else:
entity_type, d_id, skip1, skip2, a_id, a_name, a_state, a_sub_type = r
self._add_node(nodes, visited_nodes, 'action', a_id, a_name, a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'datastore', d_id, 'action', a_id)
return Graph(nodes, edges)
def get_entity_graph(self, entity):
""" :type entity: dart.model.graph.GraphEntity
:rtype: dart.model.graph.Graph """
statement = text(RECURSIVE_SQL).bindparams(
entity_type=entity.entity_type,
entity_id=entity.entity_id,
name=entity.name,
state=entity.state,
sub_type=entity.sub_type)
visited_nodes = set()
visited_edges = set()
nodes = []
edges = []
for e in [GraphEntity(*r) for r in db.session.execute(statement)]:
self._add_node(nodes, visited_nodes, e.entity_type, e.entity_id,
e.name, e.state, e.sub_type)
if e.related_id:
if e.related_is_a == Relationship.CHILD:
src_type = e.entity_type
src_id = e.entity_id
dst_type = e.related_type
dst_id = e.related_id
else:
src_type = e.related_type
src_id = e.related_id
dst_type = e.entity_type
dst_id = e.entity_id
self._add_edge(edges, visited_edges, src_type, src_id,
dst_type, dst_id)
# now that we have the base graph, add in the most recent workflow_instances and their actions,
# as well as recent datastore actions that are not associated with any workflows (if any)
wf_ids, wf_sql = self._get_workflow_instance_sql(nodes)
d_ids, d_sql = self._get_datastore_one_offs(nodes)
sql_parts = []
if wf_sql:
sql_parts.append('(' + wf_sql + ')')
if d_sql:
sql_parts.append('(' + d_sql + ')')
if len(sql_parts) == 0:
return Graph(nodes, edges)
sql = '\nUNION ALL\n'.join(sql_parts)
statement = text(
sql.format(wf_ids=', '.join(
["'" + wf_id + "'" for wf_id in wf_ids]),
d_ids=', '.join(["'" + d_id + "'" for d_id in d_ids])))
for r in db.session.execute(statement):
if r[0] == 'workflow':
entity_type, wf_id, wfi_id, wfi_state, a_id, a_name, a_state, a_sub_type = r
name = 'workflow_instance'
if wfi_state == 'RUNNING' and ('workflow_instance' +
wfi_id) not in visited_nodes:
actions = self._action_service.find_actions(
workflow_instance_id=wfi_id)
if None not in [a.data.avg_runtime for a in actions]:
wfi_progress = 0
wf_runtime = 0
for a in actions:
a_runtime = a.data.avg_runtime.total_seconds()
wf_runtime += a_runtime
if a.data.state == ActionState.COMPLETED or a.data.state == ActionState.FAILED:
wfi_progress += a_runtime
elif (a.data.state == ActionState.RUNNING
or a.data.state == ActionState.FINISHING
) and a.data.progress:
wfi_progress += a_runtime * a.data.progress
if wf_runtime:
wfi_progress /= wf_runtime
name += ' - {:.0%}'.format(wfi_progress)
self._add_node(nodes, visited_nodes, 'workflow_instance',
wfi_id, name, wfi_state, None)
self._add_edge(edges, visited_edges, 'workflow', wf_id,
'workflow_instance', wfi_id)
if a_id:
self._add_node(nodes, visited_nodes, 'action', a_id,
a_name, a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'workflow_instance',
wfi_id, 'action', a_id)
else:
entity_type, d_id, skip1, skip2, a_id, a_name, a_state, a_sub_type = r
self._add_node(nodes, visited_nodes, 'action', a_id, a_name,
a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'datastore', d_id,
'action', a_id)
return Graph(nodes, edges)