def get_graph_from_subscription(subscription):
""" :type subscription: dart.model.subscription.Subscription """
entity = GraphEntity('subscription', subscription.id,
subscription.data.name, subscription.data.state)
return {
'results': graph_entity_service().get_entity_graph(entity).to_dict()
}
def get_graph_from_trigger(trigger):
""" :type trigger: dart.model.trigger.Trigger """
entity = GraphEntity('trigger', trigger.id, trigger.data.name,
trigger.data.state, trigger.data.trigger_type_name)
return {
'results': graph_entity_service().get_entity_graph(entity).to_dict()
}
def get_graph_from_datastore(datastore):
""" :type datastore: dart.model.datastore.Datastore """
entity = GraphEntity('datastore', datastore.id, datastore.data.name,
datastore.data.state)
return {
'results': graph_entity_service().get_entity_graph(entity).to_dict()
}
def get_graph_from_action(action):
""" :type action: dart.model.action.Action """
entity = GraphEntity('action', action.id, action.data.action_type_name,
action.data.state)
return {
'results': graph_entity_service().get_entity_graph(entity).to_dict()
}
def get_graph_from_workflow(workflow):
""" :type workflow: dart.model.workflow.Workflow """
entity = GraphEntity('workflow', workflow.id, workflow.data.name,
workflow.data.state)
return {
'results': graph_entity_service().get_entity_graph(entity).to_dict()
}
def get_entity_graph(self, entity):
""" :type entity: dart.model.graph.GraphEntity
:rtype: dart.model.graph.Graph """
statement = text(RECURSIVE_SQL).bindparams(
entity_type=entity.entity_type,
entity_id=entity.entity_id,
name=entity.name,
state=entity.state,
sub_type=entity.sub_type)
visited_nodes = set()
visited_edges = set()
nodes = []
edges = []
for e in [GraphEntity(*r) for r in db.session.execute(statement)]:
self._add_node(nodes, visited_nodes, e.entity_type, e.entity_id,
e.name, e.state, e.sub_type)
if e.related_id:
if e.related_is_a == Relationship.CHILD:
src_type = e.entity_type
src_id = e.entity_id
dst_type = e.related_type
dst_id = e.related_id
else:
src_type = e.related_type
src_id = e.related_id
dst_type = e.entity_type
dst_id = e.entity_id
self._add_edge(edges, visited_edges, src_type, src_id,
dst_type, dst_id)
# now that we have the base graph, add in the most recent workflow_instances and their actions,
# as well as recent datastore actions that are not associated with any workflows (if any)
wf_ids, wf_sql = self._get_workflow_instance_sql(nodes)
d_ids, d_sql = self._get_datastore_one_offs(nodes)
sql_parts = []
if wf_sql:
sql_parts.append('(' + wf_sql + ')')
if d_sql:
sql_parts.append('(' + d_sql + ')')
if len(sql_parts) == 0:
return Graph(nodes, edges)
sql = '\nUNION ALL\n'.join(sql_parts)
statement = text(
sql.format(wf_ids=', '.join(
["'" + wf_id + "'" for wf_id in wf_ids]),
d_ids=', '.join(["'" + d_id + "'" for d_id in d_ids])))
for r in db.session.execute(statement):
if r[0] == 'workflow':
entity_type, wf_id, wfi_id, wfi_progress, wfi_state, a_id, a_name, a_state, a_sub_type = r
name = 'workflow_instance - %s%%' % wfi_progress if wfi_state == 'RUNNING' else 'workflow_instance'
self._add_node(nodes, visited_nodes, 'workflow_instance',
wfi_id, name, wfi_state, None)
self._add_edge(edges, visited_edges, 'workflow', wf_id,
'workflow_instance', wfi_id)
if a_id:
self._add_node(nodes, visited_nodes, 'action', a_id,
a_name, a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'workflow_instance',
wfi_id, 'action', a_id)
else:
entity_type, d_id, skip1, skip2, skip3, a_id, a_name, a_state, a_sub_type = r
self._add_node(nodes, visited_nodes, 'action', a_id, a_name,
a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'datastore', d_id,
'action', a_id)
return Graph(nodes, edges)
def get_graph_from_event(event):
""" :type event: dart.model.event.Event """
entity = GraphEntity('event', event.id, event.data.name, event.data.state)
return {
'results': graph_entity_service().get_entity_graph(entity).to_dict()
}
def get_graph_from_dataset(dataset):
""" :type dataset: dart.model.dataset.Dataset """
entity = GraphEntity('dataset', dataset.id, dataset.data.name)
return {
'results': graph_entity_service().get_entity_graph(entity).to_dict()
}
def get_entity_graph(self, entity):
""" :type entity: dart.model.graph.GraphEntity
:rtype: dart.model.graph.Graph """
key = "graph:{0}".format(entity.entity_id)
cached_graph = self.redis_client.get(key) if self.redis_client else None # String representation of each Graph record
if cached_graph:
json_records = json.loads(cached_graph)
### read state from database:
# we need to update the current state of object in the graph
get_state_sql = ["select '{0}' as type, '{1}' as id,'{2}' as name, data->>'state' as state, '{4}' as sub_type, '{5}' as related_type, '{6}' as related_id, '{7}' as related_is_a FROM {0} where id = '{1}'".format(*rec) for rec in json_records]
# add union all between sql statements
state_sql = " union all ".join(get_state_sql)
# execute sql: (dart_depth was already removed, so we use r instead of r[0:-1])
records = [(r) for r in db.session.execute(text(state_sql))]
else:
records = [
(r[:-1]) # remove depth parameter
for r in db.session.execute(text(RECURSIVE_SQL).bindparams(
entity_type=entity.entity_type,
entity_id=entity.entity_id,
name=entity.name,
state=entity.state,
sub_type=entity.sub_type
))
]
string_records = json.dumps(records)
if self.redis_client:
self.redis_client.setex(key, self.redis_expiration_ttl, string_records) # Keep this graph cached for 10 minutes
# rebuilding graph from string
entities = [GraphEntity(*r) for r in records]
nodes, visited_nodes = [], set()
edges, visited_edges = [], set()
for e in entities:
self._add_node(nodes, visited_nodes, e.entity_type, e.entity_id, e.name, e.state, e.sub_type)
if e.related_id:
if e.related_is_a == Relationship.CHILD:
src_type = e.entity_type
src_id = e.entity_id
dst_type = e.related_type
dst_id = e.related_id
else:
src_type = e.related_type
src_id = e.related_id
dst_type = e.entity_type
dst_id = e.entity_id
self._add_edge(edges, visited_edges, src_type, src_id, dst_type, dst_id)
# now that we have the base graph, add in the most recent workflow_instances and their actions,
# as well as recent datastore actions that are not associated with any workflows (if any)
wf_ids, wf_sql = self._get_workflow_instance_sql(nodes)
d_ids, d_sql = self._get_datastore_one_offs(nodes)
sql_parts = []
if wf_sql:
sql_parts.append('(' + wf_sql + ')')
if d_sql:
sql_parts.append('(' + d_sql + ')')
if len(sql_parts) == 0:
return Graph(nodes, edges)
sql = '\nUNION ALL\n'.join(sql_parts)
statement = text(sql.format(
wf_ids=', '.join(["'" + wf_id + "'" for wf_id in wf_ids]),
d_ids=', '.join(["'" + d_id + "'" for d_id in d_ids])
))
for r in db.session.execute(statement):
if r[0] == 'workflow':
entity_type, wf_id, wfi_id, wfi_state, a_id, a_name, a_state, a_sub_type = r
name = 'workflow_instance'
if wfi_state == 'RUNNING' and ('workflow_instance' + wfi_id) not in visited_nodes:
actions = self._action_service.find_actions(workflow_instance_id=wfi_id)
if None not in [a.data.avg_runtime for a in actions]:
wfi_progress = 0
wf_runtime = 0
for a in actions:
a_runtime = a.data.avg_runtime.total_seconds()
wf_runtime += a_runtime
if a.data.state == ActionState.COMPLETED or a.data.state == ActionState.FAILED:
wfi_progress += a_runtime
elif (a.data.state == ActionState.RUNNING or a.data.state == ActionState.FINISHING) and a.data.progress:
wfi_progress += a_runtime * a.data.progress
if wf_runtime:
wfi_progress /= wf_runtime
name += ' - {:.0%}'.format(wfi_progress)
self._add_node(nodes, visited_nodes, 'workflow_instance', wfi_id, name, wfi_state, None)
self._add_edge(edges, visited_edges, 'workflow', wf_id, 'workflow_instance', wfi_id)
if a_id:
self._add_node(nodes, visited_nodes, 'action', a_id, a_name, a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'workflow_instance', wfi_id, 'action', a_id)
else:
entity_type, d_id, skip1, skip2, a_id, a_name, a_state, a_sub_type = r
self._add_node(nodes, visited_nodes, 'action', a_id, a_name, a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'datastore', d_id, 'action', a_id)
return Graph(nodes, edges)
def get_entity_graph(self, entity):
""" :type entity: dart.model.graph.GraphEntity
:rtype: dart.model.graph.Graph """
statement = text(RECURSIVE_SQL).bindparams(
entity_type=entity.entity_type,
entity_id=entity.entity_id,
name=entity.name,
state=entity.state,
sub_type=entity.sub_type)
visited_nodes = set()
visited_edges = set()
nodes = []
edges = []
for e in [GraphEntity(*r) for r in db.session.execute(statement)]:
self._add_node(nodes, visited_nodes, e.entity_type, e.entity_id,
e.name, e.state, e.sub_type)
if e.related_id:
if e.related_is_a == Relationship.CHILD:
src_type = e.entity_type
src_id = e.entity_id
dst_type = e.related_type
dst_id = e.related_id
else:
src_type = e.related_type
src_id = e.related_id
dst_type = e.entity_type
dst_id = e.entity_id
self._add_edge(edges, visited_edges, src_type, src_id,
dst_type, dst_id)
# now that we have the base graph, add in the most recent workflow_instances and their actions,
# as well as recent datastore actions that are not associated with any workflows (if any)
wf_ids, wf_sql = self._get_workflow_instance_sql(nodes)
d_ids, d_sql = self._get_datastore_one_offs(nodes)
sql_parts = []
if wf_sql:
sql_parts.append('(' + wf_sql + ')')
if d_sql:
sql_parts.append('(' + d_sql + ')')
if len(sql_parts) == 0:
return Graph(nodes, edges)
sql = '\nUNION ALL\n'.join(sql_parts)
statement = text(
sql.format(wf_ids=', '.join(
["'" + wf_id + "'" for wf_id in wf_ids]),
d_ids=', '.join(["'" + d_id + "'" for d_id in d_ids])))
for r in db.session.execute(statement):
if r[0] == 'workflow':
entity_type, wf_id, wfi_id, wfi_state, a_id, a_name, a_state, a_sub_type = r
name = 'workflow_instance'
if wfi_state == 'RUNNING' and ('workflow_instance' +
wfi_id) not in visited_nodes:
actions = self._action_service.find_actions(
workflow_instance_id=wfi_id)
if None not in [a.data.avg_runtime for a in actions]:
wfi_progress = 0
wf_runtime = 0
for a in actions:
a_runtime = a.data.avg_runtime.total_seconds()
wf_runtime += a_runtime
if a.data.state == ActionState.COMPLETED or a.data.state == ActionState.FAILED:
wfi_progress += a_runtime
elif (a.data.state == ActionState.RUNNING
or a.data.state == ActionState.FINISHING
) and a.data.progress:
wfi_progress += a_runtime * a.data.progress
if wf_runtime:
wfi_progress /= wf_runtime
name += ' - {:.0%}'.format(wfi_progress)
self._add_node(nodes, visited_nodes, 'workflow_instance',
wfi_id, name, wfi_state, None)
self._add_edge(edges, visited_edges, 'workflow', wf_id,
'workflow_instance', wfi_id)
if a_id:
self._add_node(nodes, visited_nodes, 'action', a_id,
a_name, a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'workflow_instance',
wfi_id, 'action', a_id)
else:
entity_type, d_id, skip1, skip2, a_id, a_name, a_state, a_sub_type = r
self._add_node(nodes, visited_nodes, 'action', a_id, a_name,
a_state, a_sub_type)
self._add_edge(edges, visited_edges, 'datastore', d_id,
'action', a_id)
return Graph(nodes, edges)