class SubWorkflowTemplateTestCase(BaseTestCases.BaseTemplateTestCase):
TEMPLATE_NAME = "subworkflow.tpl"
DEFAULT_TEMPLATE_PARAMS = dict(
dependencies={"import awesome_stuff"},
task_groups=[
TaskGroup(
name="AAA",
tasks=[
Task(task_id="first_task", template_name="dummy.tpl"),
Task(task_id="second_task", template_name="dummy.tpl"),
],
relations=[
Relation(from_task_id="first_task",
to_task_id="second_task")
],
)
],
job_properties={"user.name": "USER"},
config={"key": "value"},
relations={Relation(from_task_id="TASK_1", to_task_id="TASK_2")},
)
def test_green_path(self):
res = render_template(self.TEMPLATE_NAME,
**self.DEFAULT_TEMPLATE_PARAMS)
self.assertValidPython(res)
def add_state_handler_if_needed(self):
"""
Add additional tasks and relations to handle error and ok flow.
If the error path is specified, additional relations and task are added to handle
the error state.
If the error path and the ok path is specified, additional relations and task are added
to handle the ok path and the error path.
If the error path and the ok path is not-specified, no action is performed.
"""
if not self.error_downstream_name:
return
error_handler_task_id = self.name + "_error"
error_handler = Task(
task_id=error_handler_task_id, template_name="dummy.tpl", trigger_rule=TriggerRule.ONE_FAILED
)
self.error_handler_task = error_handler
new_relations = (
Relation(from_task_id=t.task_id, to_task_id=error_handler_task_id, is_error=True)
for t in self.tasks
)
self.relations.extend(new_relations)
if not self.downstream_names:
return
ok_handler_task_id = self.name + "_ok"
ok_handler = Task(
task_id=ok_handler_task_id, template_name="dummy.tpl", trigger_rule=TriggerRule.ONE_SUCCESS
)
self.ok_handler_task = ok_handler
self.relations.append(Relation(from_task_id=self.tasks[-1].task_id, to_task_id=ok_handler_task_id))
def to_tasks_and_relations(self):
prepare_command = self.get_prepare_command(self.oozie_node,
self.params)
tasks = [
Task(
task_id=self.name + "_prepare",
template_name="prepare.tpl",
trigger_rule=self.trigger_rule,
template_params=dict(prepare_command=prepare_command),
),
Task(
task_id=self.name,
template_name="pig.tpl",
trigger_rule=self.trigger_rule,
template_params=dict(
properties=self.properties,
params_dict=self.params_dict,
script_file_name=self.script_file_name,
),
),
]
relations = [
Relation(from_task_id=self.name + "_prepare", to_task_id=self.name)
]
return tasks, relations
def to_tasks_and_relations(self):
tasks = [
Task(
task_id=self.name,
template_name="mapreduce.tpl",
trigger_rule=self.trigger_rule,
template_params=dict(
properties=self.properties,
params_dict=self.params_dict,
hdfs_files=self.hdfs_files,
hdfs_archives=self.hdfs_archives,
),
)
]
relations = []
if self.has_prepare(self.oozie_node):
prepare_command = self.get_prepare_command(self.oozie_node, self.params)
tasks.insert(
0,
Task(
task_id=self.name + "_prepare",
template_name="prepare.tpl",
trigger_rule=self.trigger_rule,
template_params=dict(prepare_command=prepare_command),
),
)
relations = [Relation(from_task_id=self.name + "_prepare", to_task_id=self.name)]
return tasks, relations
def test_to_tasks_and_relations(self):
params = {
"dataproc_cluster": "my-cluster",
"gcp_region": "europe-west3",
"nameNode": "hdfs://localhost:9020/",
}
mapper = self._get_shell_mapper(params=params)
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(
tasks,
[
Task(
task_id="test_id_prepare",
template_name="prepare.tpl",
template_params={
"prepare_command":
"$DAGS_FOLDER/../data/prepare.sh -c my-cluster -r europe-west3 "
'-d "//examples/output-data/demo/pig-node //examples/output-data'
'/demo/pig-node2" -m "//examples/input-data/demo/pig-node '
'//examples/input-data/demo/pig-node2"'
},
),
Task(
task_id="test_id",
template_name="shell.tpl",
template_params={"pig_command": "sh 'echo arg1 arg2'"},
),
],
)
self.assertEqual(
relations,
[Relation(from_task_id="test_id_prepare", to_task_id="test_id")])
class SubWorkflowTemplateTestCase(TestCase, TemplateTestMixin):
TEMPLATE_NAME = "subworkflow.tpl"
DEFAULT_TEMPLATE_PARAMS = dict(
dependencies={"import awesome_stuff"},
nodes=[
ParsedActionNode(
mock.MagicMock(spec=DummyMapper),
tasks=[
Task(task_id="first_task", template_name="dummy.tpl"),
Task(task_id="second_task", template_name="dummy.tpl"),
],
relations=[
Relation(from_task_id="first_task",
to_task_id="second_task")
],
)
],
job_properties={"user.name": "USER"},
config={"key": "value"},
relations={Relation(from_task_id="TASK_1", to_task_id="TASK_2")},
)
def test_green_path(self):
res = render_template(self.TEMPLATE_NAME,
**self.DEFAULT_TEMPLATE_PARAMS)
self.assertValidPython(res)
class WorkflowTemplateTestCase(TestCase, TemplateTestMixin):
TEMPLATE_NAME = "workflow.tpl"
DEFAULT_TEMPLATE_PARAMS = dict(
dag_name="test_dag",
dependencies={"import awesome_stuff"},
task_groups=[
TaskGroup(
name="TASK_GROUP",
tasks=[
Task(task_id="first_task", template_name="dummy.tpl"),
Task(task_id="second_task", template_name="dummy.tpl"),
],
)
],
job_properties={"user.name": "USER"},
config={},
relations={Relation(from_task_id="TASK_1", to_task_id="TASK_2")},
schedule_interval=3,
start_days_ago=3,
task_map={"oozie-task": ["airflow-task"]},
)
def test_green_path(self):
res = render_template(self.TEMPLATE_NAME, **self.DEFAULT_TEMPLATE_PARAMS)
self.assertValidPython(res)
def test_task_and_relations(self):
# Given
mapper = _get_distcp_mapper(self.distcp_node,
job_properties=EXAMPLE_JOB_PROPERTIES,
config=EXAMPLE_CONFIG_PROPERTIES)
# When
mapper.on_parse_node()
tasks, relations = mapper.to_tasks_and_relations()
# Then
self.assertEqual(mapper.oozie_node, self.distcp_node)
self.assertIsNotNone(tasks)
self.assertIsNotNone(relations)
self.assertEqual(2, len(tasks))
self.assertEqual(1, len(relations))
self.assertEqual(
[
Task(
task_id="distcp_prepare",
template_name="prepare.tpl",
trigger_rule="one_success",
template_params={
"delete": "/tmp/d_path",
"mkdir": None
},
),
Task(
task_id="distcp",
template_name="distcp.tpl",
trigger_rule="one_success",
template_params={
"props":
PropertySet(
config={
"dataproc_cluster": "my-cluster",
"gcp_region": "europe-west3"
},
job_properties={
"nameNode1": "hdfs://localhost:8081",
"nameNode2": "hdfs://localhost:8082",
},
action_node_properties={
"oozie.launcher.mapreduce.job.hdfs-servers":
"{{nameNode1}} ,{{nameNode2}}"
},
),
"distcp_command":
"--class=org.apache.hadoop.tools.DistCp -- -update -skipcrccheck "
"-strategy dynamic '{{nameNode1}}/path/to/input file.txt' "
"'{{nameNode2}}/path/to/output-file.txt'",
},
),
],
tasks,
)
self.assertEqual([
Relation(from_task_id=f"{mapper.name}_prepare",
to_task_id=mapper.name)
], relations)
def test_convert_nodes(self):
tasks_1 = [
Task(task_id="first_task", template_name="dummy.tpl"),
Task(task_id="second_task", template_name="dummy.tpl"),
]
relations_1 = {
Relation(from_task_id="first_task", to_task_id="tasks_2")
}
tasks_2 = [Task(task_id="third_task", template_name="dummy.tpl")]
relations_2 = {}
mapper_1 = mock.MagicMock(
**{"to_tasks_and_relations.return_value": (tasks_1, relations_1)})
mapper_2 = mock.MagicMock(
**{"to_tasks_and_relations.return_value": (tasks_2, relations_2)})
node_1 = ParsedNode(mapper=mapper_1)
node_2 = ParsedNode(mapper=mapper_2)
nodes = dict(TASK_1=node_1, TASK_2=node_2)
self.converter.convert_nodes(nodes=nodes)
self.assertIs(node_1.tasks, tasks_1)
self.assertIs(node_2.tasks, tasks_2)
self.assertIs(node_1.relations, relations_1)
self.assertIs(node_2.relations, relations_2)
def test_to_tasks_and_relations(self):
job_properties = {"nameNode": "hdfs://localhost:9020/", "queueName": "default"}
config = {"dataproc_cluster": "my-cluster", "gcp_region": "europe-west3"}
mapper = self._get_shell_mapper(job_properties=job_properties, config=config)
mapper.on_parse_node()
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(
[
Task(
task_id="test_id_prepare",
template_name="prepare.tpl",
template_params={
"delete": "//examples/output-data/demo/pig-node "
"//examples/output-data/demo/pig-node2",
"mkdir": "//examples/input-data/demo/pig-node "
"//examples/input-data/demo/pig-node2",
},
),
Task(
task_id="test_id",
template_name="shell.tpl",
template_params={
"pig_command": "sh echo arg1 arg2",
"action_node_properties": {
"mapred.job.queue.name": "default",
"mapred.map.output.compress": "false",
},
},
),
],
tasks,
)
self.assertEqual(relations, [Relation(from_task_id="test_id_prepare", to_task_id="test_id")])
class SubWorkflowTemplateTestCase(TestCase, TemplateTestMixin):
TEMPLATE_NAME = "subworkflow.tpl"
DEFAULT_TEMPLATE_PARAMS = dict(
dependencies=["import awesome_stuff"],
nodes=[
ParsedNode(
mock.MagicMock(spec=DummyMapper),
tasks=[
Task(task_id="first_task", template_name="dummy.tpl"),
Task(task_id="second_task", template_name="dummy.tpl"),
],
relations=[
Relation(from_task_id="first_task",
to_task_id="second_task")
],
)
],
params={"user.name": "USER"},
relations={Relation(from_task_id="TASK_1", to_task_id="TASK_2")},
schedule_interval=None,
start_days_ago=None,
)
def test_green_path(self):
res = render_template(self.TEMPLATE_NAME,
**self.DEFAULT_TEMPLATE_PARAMS)
self.assertValidPython(res)
def _get_tasks(self):
"""
Returns the list of Airflow tasks that are the result of mapping
:return: list of Airflow tasks
"""
action_task = Task(
task_id=self.name,
template_name="spark.tpl",
trigger_rule=self.trigger_rule,
template_params=dict(
main_jar=self.java_jar,
main_class=self.java_class,
arguments=self.application_args,
archives=self.hdfs_archives,
files=self.hdfs_files,
job_name=self.job_name,
dataproc_spark_properties=self.properties,
dataproc_spark_jars=self.dataproc_jars,
),
)
if not self.has_prepare(self.oozie_node):
return [action_task]
prepare_task = Task(
task_id=self.name + "_prepare",
template_name="prepare.tpl",
template_params=dict(prepare_command=self.prepare_command),
)
return [prepare_task, action_task]
def test_should_add_end_success_workflow_node(self):
# Given
transformer = AddWorkflowNotificationTransformer()
workflow = Workflow(input_directory_path="",
output_directory_path="",
dag_name="DAG_NAME_B")
props = PropertySet(
job_properties={
PROP_WORKFLOW_NOTIFICATION_URL: "http://example.com/workflow"
})
first_task_group = TaskGroup(
name="first_task",
tasks=[Task(task_id="first_task", template_name="dummy.tpl")])
# When
workflow.task_groups[first_task_group.name] = first_task_group
# Then
transformer.process_workflow_after_convert_nodes(workflow, props)
self.assertIn(END_SUCCESS_TASK_GROUP_NAME, workflow.task_groups.keys())
self.assertIn(END_SUCCESS_TASK_GROUP_NAME,
first_task_group.downstream_names)
self.assertEqual(
[
Task(
task_id=END_SUCCESS_TASK_GROUP_NAME,
template_name="http.tpl",
trigger_rule="one_success",
template_params={"url": "http://example.com/workflow"},
)
],
workflow.task_groups[END_SUCCESS_TASK_GROUP_NAME].tasks,
)
def to_tasks_and_relations(self):
tasks = [
Task(
task_id=self.name,
template_name="git.tpl",
template_params=dict(bash_command=self.bash_command),
)
]
relations = []
if self.has_prepare(self.oozie_node):
prepare_command = self.get_prepare_command(self.oozie_node,
self.params)
tasks.insert(
0,
Task(
task_id=self.name + "_prepare",
template_name="prepare.tpl",
template_params=dict(prepare_command=prepare_command),
),
)
relations = [
Relation(from_task_id=self.name + "_prepare",
to_task_id=self.name)
]
return tasks, relations
def test_to_tasks_and_relations(self):
job_properties = {"nameNode": "hdfs://"}
config = {
"dataproc_cluster": "my-cluster",
"gcp_region": "europe-west3"
}
mapper = self._get_pig_mapper(job_properties=job_properties,
config=config)
mapper.on_parse_node()
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(
tasks,
[
Task(
task_id="test_id_prepare",
template_name="prepare.tpl",
template_params={
"delete":
"/examples/output-data/demo/pig-node /examples/output-data/demo/pig-node2",
"mkdir":
"/examples/input-data/demo/pig-node /examples/input-data/demo/pig-node2",
},
),
Task(
task_id="test_id",
template_name="pig.tpl",
template_params={
"props":
PropertySet(
config={
"dataproc_cluster": "my-cluster",
"gcp_region": "europe-west3"
},
job_properties={"nameNode": "hdfs://"},
action_node_properties={
"mapred.job.queue.name": "${queueName}",
"mapred.map.output.compress": "false",
},
),
"params_dict": {
"INPUT":
"/user/${wf:user()}/${examplesRoot}/input-data/text",
"OUTPUT":
"/user/${wf:user()}/${examplesRoot}/output-data/demo/pig-node",
},
"script_file_name":
"id.pig",
"action_node_properties": {
"mapred.job.queue.name": "${queueName}",
"mapred.map.output.compress": "false",
},
},
),
],
)
self.assertEqual(
[Relation(from_task_id="test_id_prepare", to_task_id="test_id")],
relations)
def test_should_handle_join_type_case(self):
"""
Input:
CONTROL > < ACTION
| |
CONTROL
Expected output:
STATUS
|
CONTROL > < ACTION
| |
TRANSITION TRANSITION
| |
CONTROL
"""
# Given
control_task_group = ControlTaskGroup(
name="control_task_group",
tasks=[Task(task_id="control_task", template_name="dummy.tpl")])
join_task_group = ControlTaskGroup(
name="join_task_group",
tasks=[Task(task_id="join_task", template_name="dummy.tpl")])
self.workflow.task_groups[control_task_group.name] = control_task_group
self.workflow.task_groups[join_task_group.name] = join_task_group
self.action_task_group.downstream_names = [join_task_group.name]
control_task_group.downstream_names = [join_task_group.name]
exp_action_status_notification_name = f"{self.action_task_group.name}{NODE_STATUS_SUFFIX}"
exp_action_control_transition_notification_name = (
f"{self.action_task_group.name}{NODE_TRANSITION_SUFFIX}_T_{join_task_group.name}"
)
exp_control_control_transition_notification_name = (
f"{control_task_group.name}{NODE_TRANSITION_SUFFIX}_T_{join_task_group.name}"
)
# When
self.transformer.process_workflow_after_convert_nodes(
self.workflow, self.props)
# Then
self.assertEqual(6, len(self.workflow.task_groups))
self.assertEqual(
{
exp_action_status_notification_name,
control_task_group.name,
self.action_task_group.name,
exp_action_control_transition_notification_name,
exp_control_control_transition_notification_name,
join_task_group.name,
},
self.workflow.task_groups.keys(),
)
def test_convert_to_text_with_prepare_node(self):
git_node = ET.fromstring(EXAMPLE_XML)
mapper = self._get_git_mapper(git_node)
mapper.on_parse_node()
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(
[
Task(
task_id="test_id_prepare",
template_name="prepare.tpl",
trigger_rule="one_success",
template_params={
"delete": "/tmp/d_path",
"mkdir": "/tmp/mk_path"
},
),
Task(
task_id="test_id",
template_name="git.tpl",
trigger_rule="one_success",
template_params={
"git_uri":
"https://github.com/apache/oozie",
"git_branch":
"{{branch}}",
"destination_path":
"/my_git_repo_directory",
"key_path":
"/awesome-key/",
"props":
PropertySet(
config={
"dataproc_cluster": "my-cluster",
"gcp_region": "europe-west3"
},
job_properties={
"branch": "my-awesome-branch",
"nameNode": "hdfs://",
"userName": "******",
"examplesRoot": "examples",
},
action_node_properties={},
),
},
),
],
tasks,
)
self.assertEqual(
[Relation(from_task_id="test_id_prepare", to_task_id="test_id")],
relations)
def test_prepend_task_empty_relations(self):
task_1 = Task(task_id=TEST_MAPPER_NAME + "_1", template_name="pig.tpl")
task_2 = Task(task_id=TEST_MAPPER_NAME + "_2", template_name="pig.tpl")
tasks, relations = ActionMapper.prepend_task(task_to_prepend=task_1,
tasks=[task_2],
relations=[])
self.assertEqual([task_1, task_2], tasks)
self.assertEqual([
Relation(from_task_id="mapper_name_1", to_task_id="mapper_name_2")
], relations)
def test_to_tasks_and_relations_with_prepare_node(self):
spark_node = ET.fromstring(EXAMPLE_XML_WITH_PREPARE)
mapper = self._get_spark_mapper(spark_node)
mapper.on_parse_node()
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(
tasks,
[
Task(
task_id="test_id_prepare",
template_name="prepare.tpl",
template_params={
"prepare_command":
"$DAGS_FOLDER/../data/prepare.sh -c my-cluster -r europe-west3 "
'-d "/tmp/d_path" -m "/tmp/mk_path"'
},
),
Task(
task_id="test_id",
template_name="spark.tpl",
template_params={
"main_jar":
None,
"main_class":
"org.apache.spark.examples.mllib.JavaALS",
"arguments":
["inputpath=hdfs:///input/file.txt", "value=2"],
"archives": [],
"files": [],
"job_name":
"Spark Examples",
"dataproc_spark_properties": {
"mapred.compress.map.output":
"true",
"spark.executor.extraJavaOptions":
"-XX:+HeapDumpOnOutOfMemoryError "
"-XX:HeapDumpPath=/tmp",
},
"dataproc_spark_jars":
["/lib/spark-examples_2.10-1.1.0.jar"],
},
),
],
)
self.assertEqual(
relations,
[Relation(from_task_id="test_id_prepare", to_task_id="test_id")])
def test_multiple(self):
relations = fs_mapper.chain([
Task(task_id="task_1", template_name=""),
Task(task_id="task_2", template_name=""),
Task(task_id="task_3", template_name=""),
Task(task_id="task_4", template_name=""),
])
self.assertEqual(
[
Relation(from_task_id="task_1", to_task_id="task_2"),
Relation(from_task_id="task_2", to_task_id="task_3"),
Relation(from_task_id="task_3", to_task_id="task_4"),
],
relations,
)
def test_convert_to_text_without_prepare_node(self):
spark_node = ET.fromstring(EXAMPLE_XML)
prepare_node = spark_node.find("prepare")
spark_node.remove(prepare_node)
mapper = self._get_git_mapper(spark_node)
mapper.on_parse_node()
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(
tasks,
[
Task(
task_id="test_id",
template_name="git.tpl",
trigger_rule="dummy",
template_params={
"bash_command":
"$DAGS_FOLDER/../data/git.sh --cluster {dataproc_cluster} "
"--region {gcp_region} --git-uri https://github.com/apache/oozie "
"--destination-path /my_git_repo_directory --branch my-awesome-branch "
"--key-path /awesome-key/"
},
)
],
)
self.assertEqual(relations, [])
def test_convert_to_text_without_prepare_node(self):
spark_node = ET.fromstring(EXAMPLE_XML)
prepare_node = spark_node.find("prepare")
spark_node.remove(prepare_node)
mapper = self._get_git_mapper(spark_node)
mapper.on_parse_node()
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(
[
Task(
task_id="test_id",
template_name="git.tpl",
template_params={
"git_uri": "https://github.com/apache/oozie",
"git_branch": "my-awesome-branch",
"destination_path": "/my_git_repo_directory",
"key_path": "/awesome-key/",
"props": PropertySet(
config={"dataproc_cluster": "my-cluster", "gcp_region": "europe-west3"},
job_properties={
"branch": "my-awesome-branch",
"nameNode": "hdfs://",
"userName": "******",
"examplesRoot": "examples",
},
action_node_properties={},
),
},
)
],
tasks,
)
self.assertEqual([], relations)
def test_to_tasks_and_relations(self):
mapper = self._get_ssh_mapper(job_properties={}, config={})
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(
[
Task(
task_id="test_id",
template_name="ssh.tpl",
template_params={
"props":
PropertySet(config={},
job_properties={},
action_node_properties={}),
"command":
"'ls -l -a'",
"user":
"******",
"host":
"apache.org",
},
)
],
tasks,
)
self.assertEqual(relations, [])
def test_to_tasks_and_relations(self):
mapper = self._get_email_mapper(job_properties={"userName": "******"},
config={})
mapper.on_parse_node()
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(
[
Task(
task_id="test_id",
template_name="email.tpl",
trigger_rule="one_success",
template_params={
"props":
PropertySet(config={},
job_properties={"userName": "******"},
action_node_properties={}),
"to_addr":
"[email protected],[email protected]",
"cc_addr":
"[email protected],[email protected]",
"bcc_addr":
"[email protected],[email protected]",
"subject":
"Email notifications for {{run_id}}",
"body":
"Hi {{userName}} , the wf {{run_id}} successfully "
"completed. Bye {{userName}}",
},
)
],
tasks,
)
self.assertEqual(relations, [])
def test_to_tasks_and_relations_should_parse_prepare_element(self):
self.hive_node.append(ET.fromstring(FRAGMENT_QUERY))
self.hive_node.append(ET.fromstring(FRAGMENT_PREPARE))
mapper = self._get_hive_mapper(job_properties=self.job_properties,
config=self.config)
mapper.on_parse_node()
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(2, len(tasks))
self.assertEqual(
Task(
task_id="test_id_prepare",
template_name="prepare.tpl",
template_params={
"delete":
"/user/TEST_USERNAME/TEST_EXAMPLE_ROOT/apps/pig/output",
"mkdir":
"/user/TEST_USERNAME/TEST_EXAMPLE_ROOT/apps/pig/created-folder",
},
),
tasks[0],
)
self.assertEqual(
[Relation(from_task_id="test_id_prepare", to_task_id="test_id")],
relations)
def test_with_prepare(self):
cluster = "my-cluster"
region = "europe-west3"
job_properties = {"nameNode": "hdfs://localhost:8020"}
config = {"dataproc_cluster": cluster, "gcp_region": region}
# language=XML
pig_node_prepare_str = """
<pig>
<name-node>hdfs://</name-node>
<prepare>
<delete path="${nameNode}/examples/output-data/demo/pig-node" />
<delete path="${nameNode}/examples/output-data/demo/pig-node2" />
<mkdir path="${nameNode}/examples/input-data/demo/pig-node" />
<mkdir path="${nameNode}/examples/input-data/demo/pig-node2" />
</prepare>
</pig>
"""
pig_node_prepare = ET.fromstring(pig_node_prepare_str)
extension = self.get_mapper_extension(
pig_node_prepare, props=PropertySet(config=config, job_properties=job_properties)
)
self.assertTrue(extension.has_prepare())
task = extension.get_prepare_task()
self.assertEqual(
Task(
task_id="mapper_prepare",
template_name="prepare.tpl",
template_params={
"delete": "/examples/output-data/demo/pig-node /examples/output-data/demo/pig-node2",
"mkdir": "/examples/input-data/demo/pig-node /examples/input-data/demo/pig-node2",
},
),
task,
)
def test_last_task_id_of_ok_flow(self):
task_group = TaskGroup(
name="task1",
error_downstream_name="AAAA",
tasks=[Task(task_id="TASK", template_name="dummy.tpl")],
)
self.assertEqual("TASK", task_group.last_task_id_of_ok_flow)
def test_to_tasks_and_relations_with_prepare_node(self):
spark_node = ET.fromstring(EXAMPLE_XML_WITH_PREPARE)
mapper = self._get_spark_mapper(spark_node)
mapper.on_parse_node()
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(
[
Task(
task_id="test_id_prepare",
template_name="prepare.tpl",
template_params={
"delete": "/tmp/d_path",
"mkdir": "/tmp/mk_path"
},
),
Task(
task_id="test_id",
template_name="spark.tpl",
template_params={
"main_jar":
None,
"main_class":
"org.apache.spark.examples.mllib.JavaALS",
"arguments":
["inputpath=hdfs:///input/file.txt", "value=2"],
"hdfs_archives": [],
"hdfs_files": [],
"job_name":
"Spark Examples",
"spark_opts": {
"spark.executor.extraJavaOptions":
"-XX:+HeapDumpOnOutOfMemoryError "
"-XX:HeapDumpPath=/tmp"
},
"dataproc_spark_jars":
["/lib/spark-examples_2.10-1.1.0.jar"],
},
),
],
tasks,
)
self.assertEqual(
relations,
[Relation(from_task_id="test_id_prepare", to_task_id="test_id")])
def to_tasks_and_relations(self) -> Tuple[List[Task], List[Relation]]:
tasks: List[Task] = [
Task(task_id="TEST_TASK",
template_name="dummy.tpl",
template_params={})
]
relations: List[Relation] = []
return tasks, relations
def test_convert_tasks_and_relations(self):
mapper = self._get_dummy_mapper()
tasks, relations = mapper.to_tasks_and_relations()
self.assertEqual(tasks,
[Task(task_id="test_id", template_name="dummy.tpl")])
self.assertEqual(relations, [])
