def test_create_relations(self):
op1 = parsed_node.ParsedNode(dummy_mapper.DummyMapper(None, 'task1'))
op1.downstream_names = ['task2', 'task3']
op1.error_xml = 'fail1'
op2 = parsed_node.ParsedNode(dummy_mapper.DummyMapper(None, 'task2'))
op2.downstream_names = ['task3']
op2.error_xml = 'fail1'
op3 = parsed_node.ParsedNode(dummy_mapper.DummyMapper(None, 'task3'))
op3.downstream_names = ['end1']
op3.error_xml = 'fail1'
end = parsed_node.ParsedNode(dummy_mapper.DummyMapper(None, 'end1'))
fail = parsed_node.ParsedNode(dummy_mapper.DummyMapper(None, 'fail1'))
op_dict = {
'task1': op1,
'task2': op2,
'task3': op3,
'end1': end,
'fail1': fail
}
self.parser.OPERATORS.update(op_dict)
self.parser.create_relations()
self.assertIn('task3.set_downstream(fail1)', self.parser.relations)
self.assertIn('task3.set_downstream(end1)', self.parser.relations)
self.assertIn('task1.set_downstream(task2)', self.parser.relations)
self.assertIn('task1.set_downstream(task3)', self.parser.relations)
self.assertIn('task2.set_downstream(task3)', self.parser.relations)
self.assertIn('task2.set_downstream(fail1)', self.parser.relations)
self.assertIn('task1.set_downstream(fail1)', self.parser.relations)
def test_update_trigger_rules(self):
op1 = parsed_node.ParsedNode(dummy_mapper.DummyMapper(None, 'task1'))
op1.downstream_names = ['task2', 'task3']
op1.error_xml = 'fail1'
op2 = parsed_node.ParsedNode(dummy_mapper.DummyMapper(None, 'task2'))
op2.downstream_names = ['task3']
op2.error_xml = 'fail1'
op3 = parsed_node.ParsedNode(dummy_mapper.DummyMapper(None, 'task3'))
op3.downstream_names = ['end1']
op3.error_xml = 'fail1'
end = parsed_node.ParsedNode(dummy_mapper.DummyMapper(None, 'end1'))
fail = parsed_node.ParsedNode(dummy_mapper.DummyMapper(None, 'fail1'))
op_dict = {'task1': op1, 'task2': op2, 'task3': op3, 'end1': end,
'fail1': fail}
self.parser.OPERATORS.update(op_dict)
self.parser.create_relations()
self.parser.update_trigger_rules()
self.assertFalse(op1.is_ok)
self.assertFalse(op1.is_error)
self.assertTrue(op2.is_ok)
self.assertFalse(op2.is_error)
self.assertTrue(op3.is_ok)
self.assertFalse(op2.is_error)
self.assertTrue(end.is_ok)
self.assertFalse(end.is_error)
self.assertFalse(fail.is_ok)
self.assertTrue(fail.is_error)
def test_create_relations(self):
oozie_node = ET.Element("dummy")
op1 = parsed_node.ParsedNode(
dummy_mapper.DummyMapper(oozie_node=oozie_node, name="task1"))
op1.downstream_names = ["task2", "task3"]
op1.error_xml = "fail1"
op2 = parsed_node.ParsedNode(
dummy_mapper.DummyMapper(oozie_node=oozie_node, name="task2"))
op2.downstream_names = ["task3", "task4"]
op2.error_xml = "fail1"
op3 = parsed_node.ParsedNode(
dummy_mapper.DummyMapper(oozie_node=oozie_node, name="task3"))
op3.downstream_names = ["end1"]
op3.error_xml = "fail1"
op4 = mock.Mock(
**{
"first_task_id": "task4_first",
"last_task_id": "task4_last",
"get_downstreams.return_value": ["task1", "task2", "task3"],
"get_error_downstream_name.return_value": "fail1",
})
end = parsed_node.ParsedNode(
dummy_mapper.DummyMapper(oozie_node=oozie_node, name="end1"))
fail = parsed_node.ParsedNode(
dummy_mapper.DummyMapper(oozie_node=oozie_node, name="fail1"))
op_dict = {
"task1": op1,
"task2": op2,
"task3": op3,
"task4": op4,
"end1": end,
"fail1": fail
}
self.parser.workflow.nodes.update(op_dict)
self.parser.create_relations()
self.assertEqual(
self.parser.workflow.relations,
{
Relation(from_task_id="task1", to_task_id="fail1"),
Relation(from_task_id="task1", to_task_id="task2"),
Relation(from_task_id="task1", to_task_id="task3"),
Relation(from_task_id="task2", to_task_id="fail1"),
Relation(from_task_id="task2", to_task_id="task3"),
Relation(from_task_id="task2", to_task_id="task4_first"),
Relation(from_task_id="task3", to_task_id="end1"),
Relation(from_task_id="task3", to_task_id="fail1"),
Relation(from_task_id="task4_last", to_task_id="fail1"),
Relation(from_task_id="task4_last", to_task_id="task1"),
Relation(from_task_id="task4_last", to_task_id="task2"),
Relation(from_task_id="task4_last", to_task_id="task3"),
},
)
def _parse_action_node(self, action_node):
"""
Action nodes are the mechanism by which a workflow triggers the
execution of a computation/processing task.
Action nodes are required to have an action-choice (map-reduce, etc.),
ok, and error node in the xml.
"""
# The 0th element of the node is the actual action tag.
# In the form of 'action'
action_name = action_node[0].tag
if action_name not in self.ACTION_MAP:
action_name = 'unknown'
map_class = self.ACTION_MAP[action_name]
operator = map_class(action_node[0],
action_node.attrib['name'],
params=self.PARAMS)
p_node = parsed_node.ParsedNode(operator)
p_node.add_downstream_node_name(action_node[1].attrib['to'])
p_node.set_error_node_name(action_node[2].attrib['to'])
logging.info("Parsed {} as Action Node of type {}.".format(
operator.task_id, action_name))
self.DEPENDENCIES.update(operator.required_imports())
# TODO A hacky way to get the correct control flow for now, fix
if operator.has_prepare():
print(operator.task_id)
self.OPERATORS[operator.task_id] = ParsedNode(
mappers.null_mapper.NullMapper(operator.task_id))
self.OPERATORS[operator.get_task_id()] = p_node
def _parse_decision_node(self, decision_node):
"""
A decision node enables a workflow to make a selection on the execution
path to follow.
The behavior of a decision node can be seen as a switch-case statement.
A decision node consists of a list of predicates-transition pairs plus
a default transition. Predicates are evaluated in order or appearance
until one of them evaluates to true and the corresponding transition is
taken. If none of the predicates evaluates to true the default
transition is taken.
example oozie wf decision node:
<decision name="[NODE-NAME]">
<switch>
<case to="[NODE_NAME]">[PREDICATE]</case>
...
<case to="[NODE_NAME]">[PREDICATE]</case>
<default to="[NODE_NAME]"/>
</switch>
</decision>
"""
map_class = self.CONTROL_MAP['decision']
operator = map_class(decision_node, decision_node.attrib['name'])
p_node = parsed_node.ParsedNode(operator)
for cases in decision_node[0]:
p_node.add_downstream_node_name(cases.attrib['to'])
logging.info("Parsed {} as Decision Node.".format(operator.task_id))
self.OPERATORS[decision_node.attrib['name']] = p_node
self.DEPENDENCIES.update(operator.required_imports())
def test_write_operators(self):
node = parsed_node.ParsedNode(dummy_mapper.DummyMapper(None, 'task1'))
ops = {'task1': node}
fp = io.StringIO()
oozie_converter.write_operators(fp, ops, indent=0)
fp.seek(0)
self.assertEqual(node.operator.convert_to_text(), fp.read())
def _parse_end_node(self, end_node):
"""
Upon reaching the end node, the workflow is considered completed successfully.
Thus it gets mapped to a dummy node that always completes.
"""
map_class = self.CONTROL_MAP['end']
operator = map_class(end_node, end_node.attrib['name'])
p_node = parsed_node.ParsedNode(operator)
logging.info("Parsed {} as End Node.".format(operator.task_id))
self.OPERATORS[end_node.attrib['name']] = p_node
self.DEPENDENCIES.update(operator.required_imports())
def test_update_trigger_rules(self):
oozie_node = ET.Element("dummy")
op1 = parsed_node.ParsedNode(
dummy_mapper.DummyMapper(oozie_node=oozie_node, name="task1"))
op1.downstream_names = ["task2", "task3"]
op1.error_xml = "fail1"
op2 = parsed_node.ParsedNode(
dummy_mapper.DummyMapper(oozie_node=oozie_node, name="task2"))
op2.downstream_names = ["task3"]
op2.error_xml = "fail1"
op3 = parsed_node.ParsedNode(
dummy_mapper.DummyMapper(oozie_node=oozie_node, name="task3"))
op3.downstream_names = ["end1"]
op3.error_xml = "fail1"
end = parsed_node.ParsedNode(
dummy_mapper.DummyMapper(oozie_node=oozie_node, name="end1"))
fail = parsed_node.ParsedNode(
dummy_mapper.DummyMapper(oozie_node=oozie_node, name="fail1"))
op_dict = {
"task1": op1,
"task2": op2,
"task3": op3,
"end1": end,
"fail1": fail
}
self.parser.workflow.nodes.update(op_dict)
self.parser.create_relations()
self.parser.update_trigger_rules()
self.assertFalse(op1.is_ok)
self.assertFalse(op1.is_error)
self.assertTrue(op2.is_ok)
self.assertFalse(op2.is_error)
self.assertTrue(op3.is_ok)
self.assertFalse(op2.is_error)
self.assertTrue(end.is_ok)
self.assertFalse(end.is_error)
self.assertFalse(fail.is_ok)
self.assertTrue(fail.is_error)
def _parse_kill_node(self, kill_node):
"""
When a workflow node reaches the `kill` node, it finishes in an error
A workflow definition may have zero or more kill nodes.
"""
map_class = self.CONTROL_MAP['kill']
operator = map_class(None, kill_node.attrib['name'],
TriggerRule.ONE_FAILED)
p_node = parsed_node.ParsedNode(operator)
logging.info("Parsed {} as Kill Node.".format(operator.task_id))
self.OPERATORS[kill_node.attrib['name']] = p_node
self.DEPENDENCIES.update(operator.required_imports())
def _parse_join_node(self, join_node):
"""
Join nodes wait for the corresponding beginning fork node paths to
finish. As the parser we are assuming the Oozie workflow follows the
schema perfectly.
"""
map_class = self.CONTROL_MAP['join']
operator = map_class(join_node, join_node.attrib['name'])
p_node = parsed_node.ParsedNode(operator)
p_node.add_downstream_node_name(join_node.attrib['to'])
logging.info("Parsed {} as Join Node.".format(operator.task_id))
self.OPERATORS[join_node.attrib['name']] = p_node
self.DEPENDENCIES.update(operator.required_imports())
def _parse_start_node(self, start_node):
"""
The start node is the entry point for a workflow job, it indicates the
first workflow node the workflow job must transition to.
When a workflow is started, it automatically transitions to the
node specified in the start.
A workflow definition must have one start node.
"""
map_class = self.CONTROL_MAP['start']
# Theoretically this could cause conflicts, but it is very unlikely
start_name = 'start_node_' + str(uuid.uuid4())[:4]
operator = map_class(start_node, task_id=start_name)
p_node = parsed_node.ParsedNode(operator)
p_node.add_downstream_node_name(start_node.attrib['to'])
logging.info("Parsed {} as Start Node.".format(operator.task_id))
self.OPERATORS[start_name] = p_node
self.DEPENDENCIES.update(operator.required_imports())
def _parse_fork_node(self, root, fork_node):
"""
Fork nodes need to be dummy operators with multiple parallel downstream
tasks.
This parses the fork node, the action nodes that it references and then
the join node at the end.
This will only parse well-formed xml-adhering workflows where all paths
end at the join node.
"""
map_class = self.CONTROL_MAP['fork']
fork_name = fork_node.attrib['name']
fork_start_op = map_class(fork_node, fork_name)
p_node = parsed_node.ParsedNode(fork_start_op)
logging.info("Parsed {} as Fork Node.".format(fork_start_op.task_id))
paths = []
for node in fork_node:
if 'path' in node.tag:
# Parse all the downstream tasks that can run in parallel.
curr_name = node.attrib['start']
paths.append(utils.xml_utils.find_node_by_name(
root, curr_name))
self.OPERATORS[fork_name] = p_node
self.DEPENDENCIES.update(fork_start_op.required_imports())
for path in paths:
p_node.add_downstream_node_name(path.attrib['name'])
logging.info("Added {}'s downstream: {}".format(
fork_start_op.task_id, path.attrib['name']))
# Theoretically these will all be action nodes, however I don't
# think that is guaranteed.
# The end of the execution path has not been reached
self.parse_node(root, path)
if path.attrib['name'] not in self.OPERATORS:
root.remove(path)
def setUp(self):
op1 = dummy_mapper.DummyMapper(None, 'task1')
self.p_node = parsed_node.ParsedNode(op1)
def test_last_task_id(self):
op1 = mock.Mock(**{"last_task_id": "last_task_id"})
p_node = parsed_node.ParsedNode(op1)
self.assertEqual("last_task_id", p_node.last_task_id)
def setUp(self):
oozie_node = Element("dummy")
op1 = dummy_mapper.DummyMapper(oozie_node=oozie_node, name="task1")
self.p_node = parsed_node.ParsedNode(op1)
