def insert_end(node, decision): """Insert DecisionEnd between node and node parents""" parent_links = node.get_parent_links().exclude(name='default') decision_end = decision.get_child_end() # Find parent decision node for every end's parent. # If the decision node is the one passed, # change the parent to link to the Decision node's DecisionEnd node. # Skip embedded decisions and forks along the way. decision_end_used = False for parent_link in parent_links: parent = parent_link.parent.get_full_node() node_temp = parent while node_temp and not isinstance(node_temp, Decision): if isinstance(node_temp, Join): node_temp = node_temp.get_parent_fork().get_parent() elif isinstance(node_temp, DecisionEnd): node_temp = node_temp.get_parent_decision().get_parent() else: node_temp = node_temp.get_parent() if node_temp.id == decision.id and parent.node_type != Decision.node_type: links = Link.objects.filter(parent=parent).exclude(name__in=['related', 'kill', 'error']) if len(links) != 1: raise RuntimeError(_('Cannot import workflows that have decision DAG leaf nodes with multiple children or no children.')) link = links[0] link.child = decision_end link.save() decision_end_used = True # Create link between DecisionEnd and terminal node. if decision_end_used and not Link.objects.filter(name='to', parent=decision_end, child=node).exists(): link = Link(name='to', parent=decision_end, child=node) link.save()
def insert_end(node, decision): """Insert DecisionEnd between node and node parents""" parent_links = node.get_parent_links().exclude(name='default') decision_end = decision.get_child_end() # Find parent decision node for every end's parent. # If the decision node is the one passed, # change the parent to link to the Decision node's DecisionEnd node. # Skip embedded decisions and forks along the way. decision_end_used = False for parent_link in parent_links: parent = parent_link.parent.get_full_node() node_temp = parent while node_temp and not isinstance(node_temp, Decision): if isinstance(node_temp, Join): node_temp = node_temp.get_parent_fork().get_parent() elif isinstance(node_temp, DecisionEnd): node_temp = node_temp.get_parent_decision().get_parent() else: node_temp = node_temp.get_parent() if node_temp.id == decision.id and parent.node_type != Decision.node_type: links = Link.objects.filter(parent=parent).exclude(name__in=['related', 'kill', 'error']) if len(links) != 1: raise RuntimeError(_('Cannot import workflows that have decision DAG leaf nodes with multiple children or no children.')) link = links[0] link.child = decision_end link.save() decision_end_used = True # Create link between DecisionEnd and terminal node. if decision_end_used and not Link.objects.filter(name='to', parent=decision_end, child=node).exists(): link = Link(name='to', parent=decision_end, child=node) link.save()
def workflow_save(request, workflow): if request.method != 'POST': raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405) json_workflow = format_dict_field_values( json.loads(request.POST.get('workflow'))) json_workflow.setdefault('schema_version', workflow.schema_version) form = WorkflowForm(data=json_workflow) if not form.is_valid(): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': form.errors}, error_code=400) json_nodes = json_workflow['nodes'] id_map = {} errors = {} if not _validate_nodes_json(json_nodes, errors, request.user, workflow): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': errors}, error_code=400) workflow = _update_workflow_json(json_workflow) nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user) # Update links index = 0 for json_node in json_nodes: child_links = json_node['child_links'] Link.objects.filter(parent=nodes[index]).delete() for child_link in child_links: link = Link() link.id = getattr(child_link, 'id', None) link.name = child_link['name'] id = str(child_link['parent']) link.parent = Node.objects.get(id=id_map[id]) id = str(child_link['child']) link.child = Node.objects.get(id=id_map[id]) link.comment = child_link.get('comment', '') link.save() index += 1 # Make sure workflow HDFS permissions are correct Workflow.objects.check_workspace(workflow, request.fs) return _workflow(request, workflow=workflow)
def decision_helper(decision, subgraphs): """ Iterates through children, waits for ends. When an end is found, finish the decision. If the end has more parents than the decision has branches, bubble the end upwards. """ # Create decision end if it does not exist. if not Link.objects.filter(parent=decision, name='related').exists(): end = DecisionEnd(workflow=workflow, node_type=DecisionEnd.node_type) end.save() link = Link(name='related', parent=decision, child=end) link.save() children = [_link.child.get_full_node() for _link in decision.get_children_links().exclude(name__in=['error','default'])] ends = set() for child in children: end = helper(child, subgraphs) if end: ends.add(end) # A single end means that we've found a unique end for this decision. # Multiple ends mean that we've found a bad decision. if len(ends) > 1: raise RuntimeError(_('Cannot import workflows that have decisions paths with multiple terminal nodes that converge on a single terminal node.')) elif len(ends) == 1: end = ends.pop() # Branch count will vary with each call if we have multiple decision nodes embedded within decision paths. # This is because parents are replaced with DecisionEnd nodes. fan_in_count = len(end.get_parent_links().exclude(name__in=['error','default'])) # IF it covers all branches, then it is an end that perfectly matches this decision. # ELSE it is an end for a decision path that the current decision node is a part of as well. # The unhandled case is multiple ends for a single decision that converge on a single end. # This is not handled in Hue. fan_out_count = len(decision.get_children_links().exclude(name__in=['error','default'])) if fan_in_count > fan_out_count: insert_end(end, decision) return end elif fan_in_count == fan_out_count: insert_end(end, decision) # End node is a decision node. # This means that there are multiple decision nodes in sequence. # If both decision nodes are within a single decision path, # then the end may need to be returned, if found. if isinstance(end, Decision): end = decision_helper(end, subgraphs) if end: return end # Can do this because we've replace all its parents with a single DecisionEnd node. return helper(end, subgraphs) else: raise RuntimeError(_('Cannot import workflows that have decisions paths with multiple terminal nodes that converge on a single terminal node.')) else: raise RuntimeError(_('Cannot import workflows that have decisions paths that never end.')) return None
def decision_helper(decision): """ Iterates through children, waits for ends. When an end is found, finish the decision. If the end has more parents than the decision has branches, bubble the end upwards. """ # Create decision end if it does not exist. if not Link.objects.filter(parent=decision, name='related').exists(): end = DecisionEnd(workflow=workflow, node_type=DecisionEnd.node_type) end.save() link = Link(name='related', parent=decision, child=end) link.save() children = [link.child.get_full_node() for link in decision.get_children_links().exclude(name__in=['error','default'])] ends = set() for child in children: end = helper(child) if end: ends.add(end) # A single end means that we've found a unique end for this decision. # Multiple ends mean that we've found a bad decision. if len(ends) > 1: raise RuntimeError(_('Cannot import workflows that have decisions paths with multiple terminal nodes that converge on a single terminal node.')) elif len(ends) == 1: end = ends.pop() # Branch count will vary with each call if we have multiple decision nodes embedded within decision paths. # This is because parents are replaced with DecisionEnd nodes. fan_in_count = len(end.get_parent_links().exclude(name__in=['error','default'])) # IF it covers all branches, then it is an end that perfectly matches this decision. # ELSE it is an end for a decision path that the current decision node is a part of as well. # The unhandled case is multiple ends for a single decision that converge on a single end. # This is not handled in Hue. fan_out_count = len(decision.get_children_links().exclude(name__in=['error','default'])) if fan_in_count > fan_out_count: insert_end(end, decision) return end elif fan_in_count == fan_out_count: insert_end(end, decision) # End node is a decision node. # This means that there are multiple decision nodes in sequence. # If both decision nodes are within a single decision path, # then the end may need to be returned, if found. if isinstance(end, Decision): end = decision_helper(end) if end: return end # Can do this because we've replace all its parents with a single DecisionEnd node. return helper(end) else: raise RuntimeError(_('Cannot import workflows that have decisions paths with multiple terminal nodes that converge on a single terminal node.')) else: raise RuntimeError(_('Cannot import workflows that have decisions paths that never end.')) return None
def workflow_save(request, workflow): if request.method != "POST": raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_("Must be POST request."), error_code=405) json_workflow = format_dict_field_values(json.loads(request.POST.get("workflow"))) json_workflow.setdefault("schema_version", workflow.schema_version) form = WorkflowForm(data=json_workflow) if not form.is_valid(): raise StructuredException( code="INVALID_REQUEST_ERROR", message=_("Error saving workflow"), data={"errors": form.errors}, error_code=400, ) json_nodes = json_workflow["nodes"] id_map = {} errors = {} if not _validate_nodes_json(json_nodes, errors, request.user, workflow): raise StructuredException( code="INVALID_REQUEST_ERROR", message=_("Error saving workflow"), data={"errors": errors}, error_code=400 ) workflow = _update_workflow_json(json_workflow) nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user) # Update links index = 0 for json_node in json_nodes: child_links = json_node["child_links"] Link.objects.filter(parent=nodes[index]).delete() for child_link in child_links: link = Link() link.id = getattr(child_link, "id", None) link.name = child_link["name"] id = str(child_link["parent"]) link.parent = Node.objects.get(id=id_map[id]) id = str(child_link["child"]) link.child = Node.objects.get(id=id_map[id]) link.comment = child_link.get("comment", "") link.save() index += 1 # Make sure workflow HDFS permissions are correct Workflow.objects.check_workspace(workflow, request.fs) return _workflow(request, workflow=workflow)
def workflow_save(request, workflow): if request.method != 'POST': raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405) json_workflow = format_dict_field_values(json.loads(str(request.POST.get('workflow')))) json_workflow.setdefault('schema_version', workflow.schema_version) form = WorkflowForm(data=json_workflow) if not form.is_valid(): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': form.errors}, error_code=400) json_nodes = json_workflow['nodes'] id_map = {} errors = {} if not _validate_nodes_json(json_nodes, errors, request.user, workflow): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': errors}, error_code=400) workflow = _update_workflow_json(json_workflow) nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user) # Update links index = 0 for json_node in json_nodes: child_links = json_node['child_links'] Link.objects.filter(parent=nodes[index]).delete() for child_link in child_links: link = Link() link.id = getattr(child_link, 'id', None) link.name = child_link['name'] id = str(child_link['parent']) link.parent = Node.objects.get(id=id_map[id]) id = str(child_link['child']) link.child = Node.objects.get(id=id_map[id]) link.comment = child_link.get('comment', '') link.save() index += 1 # Make sure workflow HDFS permissions are correct Workflow.objects.check_workspace(workflow, request.fs) return _workflow(request, workflow=workflow)
def helper(workflow, node, last_fork): if isinstance(node, Fork): join = None children = node.get_children() for child in children: join = helper(workflow, child.get_full_node(), node) or join link = Link(name='related', parent=node, child=join) link.save() node = join elif isinstance(node, Join): return node join = None children = node.get_children() for child in children: join = helper(workflow, child.get_full_node(), last_fork) or join return join
def helper(workflow, node, last_fork): if isinstance(node, Fork): join = None children = node.get_children() for child in children: join = helper(workflow, child.get_full_node(), node) or join link = Link(name='related', parent=node, child=join) link.save() node = join elif isinstance(node, Join): return node join = None children = node.get_children() for child in children: join = helper(workflow, child.get_full_node(), last_fork) or join return join
Link.objects.filter(parent=nodes[index]).delete() for child_link in child_links: link = Link() link.id = getattr(child_link, 'id', None) link.name = child_link['name'] id = str(child_link['parent']) link.parent = Node.objects.get(id=id_map[id]) id = str(child_link['child']) link.child = Node.objects.get(id=id_map[id]) link.comment = child_link.get('comment', '') link.save() index += 1 # Make sure workflow is shared Workflow.objects.check_workspace(workflow, request.fs) return _workflow(request, workflow=workflow) def _workflow(request, workflow): response = {'status': -1, 'data': 'None'} workflow_dict = model_to_dict(workflow) node_list = [node.get_full_node() for node in workflow.node_list] nodes = [model_to_dict(node) for node in node_list]