def insert_end(node, decision):
"""Insert DecisionEnd between node and node parents"""
parent_links = node.get_parent_links().exclude(name='default')
decision_end = decision.get_child_end()
# Find parent decision node for every end's parent.
# If the decision node is the one passed,
# change the parent to link to the Decision node's DecisionEnd node.
# Skip embedded decisions and forks along the way.
decision_end_used = False
for parent_link in parent_links:
parent = parent_link.parent.get_full_node()
node_temp = parent
while node_temp and not isinstance(node_temp, Decision):
if isinstance(node_temp, Join):
node_temp = node_temp.get_parent_fork().get_parent()
elif isinstance(node_temp, DecisionEnd):
node_temp = node_temp.get_parent_decision().get_parent()
else:
node_temp = node_temp.get_parent()
if node_temp.id == decision.id and parent.node_type != Decision.node_type:
links = Link.objects.filter(parent=parent).exclude(name__in=['related', 'kill', 'error'])
if len(links) != 1:
raise RuntimeError(_('Cannot import workflows that have decision DAG leaf nodes with multiple children or no children.'))
link = links[0]
link.child = decision_end
link.save()
decision_end_used = True
# Create link between DecisionEnd and terminal node.
if decision_end_used and not Link.objects.filter(name='to', parent=decision_end, child=node).exists():
link = Link(name='to', parent=decision_end, child=node)
link.save()
def insert_end(node, decision):
"""Insert DecisionEnd between node and node parents"""
parent_links = node.get_parent_links().exclude(name='default')
decision_end = decision.get_child_end()
# Find parent decision node for every end's parent.
# If the decision node is the one passed,
# change the parent to link to the Decision node's DecisionEnd node.
# Skip embedded decisions and forks along the way.
decision_end_used = False
for parent_link in parent_links:
parent = parent_link.parent.get_full_node()
node_temp = parent
while node_temp and not isinstance(node_temp, Decision):
if isinstance(node_temp, Join):
node_temp = node_temp.get_parent_fork().get_parent()
elif isinstance(node_temp, DecisionEnd):
node_temp = node_temp.get_parent_decision().get_parent()
else:
node_temp = node_temp.get_parent()
if node_temp.id == decision.id and parent.node_type != Decision.node_type:
links = Link.objects.filter(parent=parent).exclude(name__in=['related', 'kill', 'error'])
if len(links) != 1:
raise RuntimeError(_('Cannot import workflows that have decision DAG leaf nodes with multiple children or no children.'))
link = links[0]
link.child = decision_end
link.save()
decision_end_used = True
# Create link between DecisionEnd and terminal node.
if decision_end_used and not Link.objects.filter(name='to', parent=decision_end, child=node).exists():
link = Link(name='to', parent=decision_end, child=node)
link.save()
def workflow_save(request, workflow):
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be POST request.'),
error_code=405)
json_workflow = format_dict_field_values(
json.loads(request.POST.get('workflow')))
json_workflow.setdefault('schema_version', workflow.schema_version)
form = WorkflowForm(data=json_workflow)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving workflow'),
data={'errors': form.errors},
error_code=400)
json_nodes = json_workflow['nodes']
id_map = {}
errors = {}
if not _validate_nodes_json(json_nodes, errors, request.user, workflow):
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving workflow'),
data={'errors': errors},
error_code=400)
workflow = _update_workflow_json(json_workflow)
nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map,
request.user)
# Update links
index = 0
for json_node in json_nodes:
child_links = json_node['child_links']
Link.objects.filter(parent=nodes[index]).delete()
for child_link in child_links:
link = Link()
link.id = getattr(child_link, 'id', None)
link.name = child_link['name']
id = str(child_link['parent'])
link.parent = Node.objects.get(id=id_map[id])
id = str(child_link['child'])
link.child = Node.objects.get(id=id_map[id])
link.comment = child_link.get('comment', '')
link.save()
index += 1
# Make sure workflow HDFS permissions are correct
Workflow.objects.check_workspace(workflow, request.fs)
return _workflow(request, workflow=workflow)
def decision_helper(decision, subgraphs):
"""
Iterates through children, waits for ends.
When an end is found, finish the decision.
If the end has more parents than the decision has branches, bubble the end upwards.
"""
# Create decision end if it does not exist.
if not Link.objects.filter(parent=decision, name='related').exists():
end = DecisionEnd(workflow=workflow, node_type=DecisionEnd.node_type)
end.save()
link = Link(name='related', parent=decision, child=end)
link.save()
children = [_link.child.get_full_node() for _link in decision.get_children_links().exclude(name__in=['error','default'])]
ends = set()
for child in children:
end = helper(child, subgraphs)
if end:
ends.add(end)
# A single end means that we've found a unique end for this decision.
# Multiple ends mean that we've found a bad decision.
if len(ends) > 1:
raise RuntimeError(_('Cannot import workflows that have decisions paths with multiple terminal nodes that converge on a single terminal node.'))
elif len(ends) == 1:
end = ends.pop()
# Branch count will vary with each call if we have multiple decision nodes embedded within decision paths.
# This is because parents are replaced with DecisionEnd nodes.
fan_in_count = len(end.get_parent_links().exclude(name__in=['error','default']))
# IF it covers all branches, then it is an end that perfectly matches this decision.
# ELSE it is an end for a decision path that the current decision node is a part of as well.
# The unhandled case is multiple ends for a single decision that converge on a single end.
# This is not handled in Hue.
fan_out_count = len(decision.get_children_links().exclude(name__in=['error','default']))
if fan_in_count > fan_out_count:
insert_end(end, decision)
return end
elif fan_in_count == fan_out_count:
insert_end(end, decision)
# End node is a decision node.
# This means that there are multiple decision nodes in sequence.
# If both decision nodes are within a single decision path,
# then the end may need to be returned, if found.
if isinstance(end, Decision):
end = decision_helper(end, subgraphs)
if end:
return end
# Can do this because we've replace all its parents with a single DecisionEnd node.
return helper(end, subgraphs)
else:
raise RuntimeError(_('Cannot import workflows that have decisions paths with multiple terminal nodes that converge on a single terminal node.'))
else:
raise RuntimeError(_('Cannot import workflows that have decisions paths that never end.'))
return None
def decision_helper(decision):
"""
Iterates through children, waits for ends.
When an end is found, finish the decision.
If the end has more parents than the decision has branches, bubble the end upwards.
"""
# Create decision end if it does not exist.
if not Link.objects.filter(parent=decision, name='related').exists():
end = DecisionEnd(workflow=workflow, node_type=DecisionEnd.node_type)
end.save()
link = Link(name='related', parent=decision, child=end)
link.save()
children = [link.child.get_full_node() for link in decision.get_children_links().exclude(name__in=['error','default'])]
ends = set()
for child in children:
end = helper(child)
if end:
ends.add(end)
# A single end means that we've found a unique end for this decision.
# Multiple ends mean that we've found a bad decision.
if len(ends) > 1:
raise RuntimeError(_('Cannot import workflows that have decisions paths with multiple terminal nodes that converge on a single terminal node.'))
elif len(ends) == 1:
end = ends.pop()
# Branch count will vary with each call if we have multiple decision nodes embedded within decision paths.
# This is because parents are replaced with DecisionEnd nodes.
fan_in_count = len(end.get_parent_links().exclude(name__in=['error','default']))
# IF it covers all branches, then it is an end that perfectly matches this decision.
# ELSE it is an end for a decision path that the current decision node is a part of as well.
# The unhandled case is multiple ends for a single decision that converge on a single end.
# This is not handled in Hue.
fan_out_count = len(decision.get_children_links().exclude(name__in=['error','default']))
if fan_in_count > fan_out_count:
insert_end(end, decision)
return end
elif fan_in_count == fan_out_count:
insert_end(end, decision)
# End node is a decision node.
# This means that there are multiple decision nodes in sequence.
# If both decision nodes are within a single decision path,
# then the end may need to be returned, if found.
if isinstance(end, Decision):
end = decision_helper(end)
if end:
return end
# Can do this because we've replace all its parents with a single DecisionEnd node.
return helper(end)
else:
raise RuntimeError(_('Cannot import workflows that have decisions paths with multiple terminal nodes that converge on a single terminal node.'))
else:
raise RuntimeError(_('Cannot import workflows that have decisions paths that never end.'))
return None
def workflow_save(request, workflow):
if request.method != "POST":
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_("Must be POST request."), error_code=405)
json_workflow = format_dict_field_values(json.loads(request.POST.get("workflow")))
json_workflow.setdefault("schema_version", workflow.schema_version)
form = WorkflowForm(data=json_workflow)
if not form.is_valid():
raise StructuredException(
code="INVALID_REQUEST_ERROR",
message=_("Error saving workflow"),
data={"errors": form.errors},
error_code=400,
)
json_nodes = json_workflow["nodes"]
id_map = {}
errors = {}
if not _validate_nodes_json(json_nodes, errors, request.user, workflow):
raise StructuredException(
code="INVALID_REQUEST_ERROR", message=_("Error saving workflow"), data={"errors": errors}, error_code=400
)
workflow = _update_workflow_json(json_workflow)
nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user)
# Update links
index = 0
for json_node in json_nodes:
child_links = json_node["child_links"]
Link.objects.filter(parent=nodes[index]).delete()
for child_link in child_links:
link = Link()
link.id = getattr(child_link, "id", None)
link.name = child_link["name"]
id = str(child_link["parent"])
link.parent = Node.objects.get(id=id_map[id])
id = str(child_link["child"])
link.child = Node.objects.get(id=id_map[id])
link.comment = child_link.get("comment", "")
link.save()
index += 1
# Make sure workflow HDFS permissions are correct
Workflow.objects.check_workspace(workflow, request.fs)
return _workflow(request, workflow=workflow)
def workflow_save(request, workflow):
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405)
json_workflow = format_dict_field_values(json.loads(str(request.POST.get('workflow'))))
json_workflow.setdefault('schema_version', workflow.schema_version)
form = WorkflowForm(data=json_workflow)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': form.errors}, error_code=400)
json_nodes = json_workflow['nodes']
id_map = {}
errors = {}
if not _validate_nodes_json(json_nodes, errors, request.user, workflow):
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': errors}, error_code=400)
workflow = _update_workflow_json(json_workflow)
nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user)
# Update links
index = 0
for json_node in json_nodes:
child_links = json_node['child_links']
Link.objects.filter(parent=nodes[index]).delete()
for child_link in child_links:
link = Link()
link.id = getattr(child_link, 'id', None)
link.name = child_link['name']
id = str(child_link['parent'])
link.parent = Node.objects.get(id=id_map[id])
id = str(child_link['child'])
link.child = Node.objects.get(id=id_map[id])
link.comment = child_link.get('comment', '')
link.save()
index += 1
# Make sure workflow HDFS permissions are correct
Workflow.objects.check_workspace(workflow, request.fs)
return _workflow(request, workflow=workflow)
def helper(workflow, node, last_fork):
if isinstance(node, Fork):
join = None
children = node.get_children()
for child in children:
join = helper(workflow, child.get_full_node(), node) or join
link = Link(name='related', parent=node, child=join)
link.save()
node = join
elif isinstance(node, Join):
return node
join = None
children = node.get_children()
for child in children:
join = helper(workflow, child.get_full_node(), last_fork) or join
return join
def helper(workflow, node, last_fork):
if isinstance(node, Fork):
join = None
children = node.get_children()
for child in children:
join = helper(workflow, child.get_full_node(), node) or join
link = Link(name='related', parent=node, child=join)
link.save()
node = join
elif isinstance(node, Join):
return node
join = None
children = node.get_children()
for child in children:
join = helper(workflow, child.get_full_node(), last_fork) or join
return join
Link.objects.filter(parent=nodes[index]).delete()
for child_link in child_links:
link = Link()
link.id = getattr(child_link, 'id', None)
link.name = child_link['name']
id = str(child_link['parent'])
link.parent = Node.objects.get(id=id_map[id])
id = str(child_link['child'])
link.child = Node.objects.get(id=id_map[id])
link.comment = child_link.get('comment', '')
link.save()
index += 1
# Make sure workflow is shared
Workflow.objects.check_workspace(workflow, request.fs)
return _workflow(request, workflow=workflow)
def _workflow(request, workflow):
response = {'status': -1, 'data': 'None'}
workflow_dict = model_to_dict(workflow)
node_list = [node.get_full_node() for node in workflow.node_list]
nodes = [model_to_dict(node) for node in node_list]
