def load_from_links(links, task_name_mappings):
"""Converts a Links Dictionary into a Links object
Args:
links (list): List of Link descriptions matching app_schemas.LinkSchema
task_name_mappings (dict): Mapping from node ID to task name
Raises:
DagHandlerValidationError: Raised if the resulting graph is not a valid DAG
Returns:
Links: Instantiated DAG object
"""
graph = DiGraph()
for link_dict in links:
try:
# Link goes from out nodes to in node
# Front end flow chart allows link to start at our in node
'''
if link_dict["from_node"]["portId"] == "out_port":
from_dict = link_dict["from_node"]
to_dict = link_dict["to_node"]
else:
from_dict = link_dict["to_node"]
to_dict = link_dict["from_node"]'''
if link_dict["fromNode"] == "out_port":
from_dict = link_dict["fromNode"]
to_dict = link_dict["toNode"]
else:
from_dict = link_dict["toNode"]
to_dict = link_dict["fromNode"]
graph.add_edge(
#task_name_mappings[from_dict["nodeId"]],
#task_name_mappings[to_dict["nodeId"]],
task_name_mappings[from_dict],
task_name_mappings[to_dict],
)
except KeyError as e:
raise DagHandlerValidationError(
f"Unable to find node(s) [{e}] referenced by links"
)
if not is_directed_acyclic_graph(graph):
raise DagHandlerValidationError("Links do not form a valid DAG")
return Links(graph)
def load_from_wml(cls, wml_dict):
"""Returns a validated DagHandler from a Wml Dict
Args:
wml_dict (dict): Full WML file converted to a Dict
Raises:
DagHandlerValidationError: If instance params are invalid
Returns:
[DagHandler]: The converted DagHandler object
"""
dag_params = cls.parameter_list_to_dict(wml_dict["dag"]["parameters"])
if not dag_params.get("dag_id"):
raise DagHandlerValidationError(
f"DAG is missing required parameter dag_id")
tasks = [
TaskHandler.load_from_node(node)
for node in wml_dict["nodes"].values()
]
task_name_mappings = {t.node_id: t.snake_name for t in tasks}
links = Links.load_from_links(wml_dict["links"].values(),
task_name_mappings)
return DagHandler(dag_params, tasks, links)
def load_from_node(cls, node_dict):
"""Creates a TaskHandler from a WML Node Dictionary
Args:
node_dict (dict): Dictionary that matches app_schemas.NodeSchema
Returns:
[TaskHandler]: The converted TaskHandler object
"""
node_id = node_dict["id"]
operator_type = node_dict["type"]
module = node_dict["properties"]["module"]
task_params = cls.parameter_list_to_dict(node_dict["properties"]["parameters"])
if not task_params.get("task_id"):
raise DagHandlerValidationError(
f"Node {node_id} missing required parameter task_id"
)
return TaskHandler(node_id, operator_type, module, task_params)
def parameter_list_to_dict(params):
"""Validates a list of App Parameters and returns a dict of parameter keys and values
Args:
params (list): List of Parameters matching OperatorParameterSchema
Raises:
DagHandlerValidationError: If a required parameter doesn't have a value
Returns:
[dict]: Mapping between non-default fields and values
"""
params = [_op_schema.load(param) for param in params]
for param in params:
if param.get("required", False) and param.get("value") == None:
raise DagHandlerValidationError(
f"'{param['id']}' is a required parameter'")
return {
param["id"]: param
for param in params
if param.get("value") and param["value"] != param.get("default")
}