def test_workflow_with_reused_identifier():
with pytest.raises(AssertionError) as assertion_info:
tags.WorkflowApp(
name='descriptive-name',
job_tracker='job-tracker',
name_node='name-node',
entities=tags.Action(
name='build', action=tags.Shell(exec_command='echo', arguments=['build']),
on_error=tags.Action(name='build', action=tags.Shell(exec_command='echo', arguments=['error']))
)
)
assert str(assertion_info.value) == 'Name(s) reused: action-build'
with pytest.raises(AssertionError) as assertion_info:
tags.WorkflowApp(
name='descriptive-name',
job_tracker='job-tracker',
name_node='name-node',
entities=tags.Serial(
tags.Action(name='build', action=tags.Shell(exec_command='echo', arguments=['build'])),
tags.Action(name='resolve', action=tags.Shell(exec_command='echo', arguments=['resolve'])),
on_error=tags.Serial(
tags.Action(name='build', action=tags.Shell(exec_command='echo', arguments=['error'])),
tags.Kill('A bad thing happened')
)
)
)
assert str(assertion_info.value) == 'Name(s) reused: action-build'
def test_builder_raises_on_bad_action_name():
# Does it throw an exception on a bad action name?
with pytest.raises(AssertionError) as assertion_info:
xml.WorkflowBuilder(
name='descriptive-name'
).add_action(
name='Action name with invalid characters',
action=tags.Shell(exec_command='echo "test"'),
action_on_error=tags.Email(to='*****@*****.**', subject='Error', body='A bad thing happened'),
kill_on_error='Failure message',
)
assert "Identifier must match " in str(assertion_info.value) and \
"Action name with invalid characters" in str(assertion_info.value)
# Does it throw an exception on an action name that's too long?
with pytest.raises(AssertionError) as assertion_info:
xml.WorkflowBuilder(
name='descriptive-name'
).add_action(
name='l' * (tags.MAX_IDENTIFIER_LENGTH + 1),
action=tags.Shell(exec_command='echo "test"'),
action_on_error=tags.Email(to='*****@*****.**', subject='Error', body='A bad thing happened'),
kill_on_error='Failure message',
)
assert "Identifier must be less than " in str(assertion_info.value)
def test_workflow_app_inherit_parent_error(request):
workflow_app = tags.WorkflowApp(
name='descriptive-name',
job_tracker='job-tracker',
name_node='name-node',
entities=tags.Serial(
tags.Parallel(
tags.Action(name='build_a', action=tags.Shell(exec_command='echo', arguments=['build_a'])),
tags.Action(name='build_b', action=tags.Shell(exec_command='echo', arguments=['build_b'])),
name='builders'
),
tags.Parallel(
tags.Action(name='resolve_a', action=tags.Shell(exec_command='echo', arguments=['resolve_a'])),
tags.Action(name='resolve_b', action=tags.Shell(exec_command='echo', arguments=['resolve_b'])),
name='resolvers'
),
on_error=tags.Serial(
tags.Action(name='error', action=tags.Shell(exec_command='echo', arguments=['error'])),
tags.Kill(name='error', message='A bad thing happened')
)
)
)
assert_workflow(request, workflow_app)
app = tests.utils.ParsedXml(workflow_app.xml())
app.assert_node("/action[@name='action-build_a']/error", to='action-error')
app.assert_node("/action[@name='action-build_b']/error", to='action-error')
app.assert_node("/action[@name='action-resolve_a']/error", to='action-error')
app.assert_node("/action[@name='action-resolve_a']/error", to='action-error')
app.assert_node("/action[@name='action-error']/error", to='end')
app.assert_node("/action[@name='action-error']/ok", to='kill-error')
def test_workflow_app_serial_entities(request):
# Create a serial collection of entities with a serial collection as an error condition
entities = tags.Serial(
tags.Action(tags.Shell(exec_command='echo', arguments=['build'])),
tags.Action(tags.Shell(exec_command='echo', arguments=['resolve'])),
on_error=tags.Serial(
tags.Action(tags.Shell(exec_command='echo', arguments=['error'])),
tags.Kill('A bad thing happened')
)
)
assert len(set(entities)) == 4
assert bool(entities)
workflow_app = tags.WorkflowApp(
name='descriptive-name',
job_tracker='job-tracker',
name_node='name-node',
entities=entities
)
assert_workflow(request, workflow_app, """
<workflow-app xmlns="uri:oozie:workflow:0.5" name="descriptive-name">
<global>
<job-tracker>job-tracker</job-tracker>
<name-node>name-node</name-node>
</global>
<start to="action-00000000" />
<action name="action-00000002">
<shell xmlns="uri:oozie:shell-action:0.3">
<exec>echo</exec>
<argument>error</argument>
</shell>
<ok to="kill-00000003" />
<error to="end" />
</action>
<kill name="kill-00000003">
<message>A bad thing happened</message>
</kill>
<action name="action-00000000">
<shell xmlns="uri:oozie:shell-action:0.3">
<exec>echo</exec>
<argument>build</argument>
</shell>
<ok to="action-00000001" />
<error to="action-00000002" />
</action>
<action name="action-00000001">
<shell xmlns="uri:oozie:shell-action:0.3">
<exec>echo</exec>
<argument>resolve</argument>
</shell>
<ok to="end" />
<error to="action-00000002" />
</action>
<end name="end" />
</workflow-app>
""")
def test_shell():
actual = tags.Shell(
exec_command='${EXEC}',
job_tracker='${jobTracker}',
name_node='${nameNode}',
prepare=None,
job_xml_files=['/user/${wf:user()}/job.xml'],
configuration={
'mapred.job.queue.name': '${queueName}'
},
arguments=['A', 'B'],
env_vars={
'ENVIRONMENT': 'production',
'RESOURCES': 'large',
},
files=['/users/blabla/testfile.sh#testfile'],
archives=['/users/blabla/testarchive.jar#testarchive'],
capture_output=True,
).xml(indent=True)
assert tests.utils.xml_to_comparable_dict('''
<shell xmlns="uri:oozie:shell-action:0.3">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<job-xml>/user/${wf:user()}/job.xml</job-xml>
<configuration>
<property>
<name>mapred.job.queue.name</name>
<value>${queueName}</value>
</property>
</configuration>
<exec>${EXEC}</exec>
<argument>A</argument>
<argument>B</argument>
<file>/users/blabla/testfile.sh#testfile</file>
<archive>/users/blabla/testarchive.jar#testarchive</archive>
<env-var>ENVIRONMENT=production</env-var>
<env-var>RESOURCES=large</env-var>
<capture-output />
</shell>''') == tests.utils.xml_to_comparable_dict(actual)
# Test using prepare fails
with pytest.raises(NotImplementedError) as assertion_info:
tags.Shell(
exec_command='${EXEC}',
prepare=['anything'],
).xml()
assert str(assertion_info.value) == "Shell action's prepare has not yet been implemented"
def workflow_builder():
return xml.WorkflowBuilder(
name='descriptive-name'
).add_action(
name='payload',
action=tags.Shell(exec_command='echo "test"'),
action_on_error=tags.Email(to='*****@*****.**', subject='Error', body='A bad thing happened'),
kill_on_error='Failure message 😢',
)
def test_builder_raises_on_multiple_actions(workflow_builder):
# Does it raise an exception when you try to add multiple actions?
with pytest.raises(NotImplementedError) as assertion_info:
workflow_builder.add_action(
name='payload',
action=tags.Shell(exec_command='echo "test"'),
action_on_error=tags.Email(to='*****@*****.**', subject='Error', body='A bad thing happened'),
kill_on_error='Failure message',
)
assert str(assertion_info.value) == 'Can only add one action in this version'
def test_builder_raises_on_bad_workflow_name():
# Does it throw an exception on a bad workflow name?
with pytest.raises(AssertionError) as assertion_info:
xml.WorkflowBuilder(
name='l' * (tags.MAX_NAME_LENGTH + 1)
).add_action(
name='payload',
action=tags.Shell(exec_command='echo "test"'),
action_on_error=tags.Email(to='*****@*****.**', subject='Error', body='A bad thing happened'),
kill_on_error='Failure message',
)
assert "Name must be less than " in str(assertion_info.value)
def workflow_app_xml(**kwargs):
workflow_app = tags.WorkflowApp(
name='descriptive-name',
job_tracker='job-tracker',
name_node='name-node',
entities=tags.Action(
name='name',
action=tags.Shell(exec_command='echo', arguments=['build']),
**kwargs
)
)
assert_workflow(request, workflow_app)
return tests.utils.ParsedXml(workflow_app.xml())
def test_workflow_app_empty_decision_entities():
with pytest.raises(AssertionError) as assertion_info:
tags.Decision(
default=None,
choices={
'${wf:lastErrorNode() eq null}': tags.Action(
tags.Shell(exec_command='echo', arguments=['"arg"'])
),
})
assert str(assertion_info.value) == 'A default must be supplied'
with pytest.raises(AssertionError) as assertion_info:
tags.Decision(
default=tags.Shell(exec_command='echo', arguments=['"arg"']),
choices=None
)
assert str(assertion_info.value) == 'At least one choice required'
with pytest.raises(AssertionError) as assertion_info:
tags.Decision(
default=tags.Shell(exec_command='echo', arguments=['"arg"']),
choices={}
)
assert str(assertion_info.value) == 'At least one choice required'
def test_workflow_action_without_credential():
with pytest.raises(AssertionError) as assertion_info:
tags.WorkflowApp(
name='descriptive-name',
job_tracker='job-tracker',
name_node='name-node',
entities=tags.Action(
name='action-name',
action=tags.Shell(exec_command='echo', arguments=['build']),
credential='my-hcat-creds',
retry_max=10,
retry_interval=20,
)
)
assert str(assertion_info.value) == str('Missing credentials: my-hcat-creds')
def test_workflow_app_decision_entities(request):
entities = tags.Decision(
default=tags.Action(tags.Shell(exec_command='echo', arguments=['default'])),
choices={
'${wf:lastErrorNode() eq null}': tags.Action(
tags.Shell(exec_command='echo', arguments=['"No last error node"'])),
},
on_error=tags.Serial(
tags.Action(tags.Shell(exec_command='echo', arguments=['error'])),
tags.Kill('A bad thing happened')
)
)
assert len(set(entities)) == 5
workflow_app = tags.WorkflowApp(
name='descriptive-name',
job_tracker='job-tracker',
name_node='name-node',
entities=entities
)
assert_workflow(request, workflow_app, """
<workflow-app xmlns="uri:oozie:workflow:0.5" name="descriptive-name">
<global>
<job-tracker>job-tracker</job-tracker>
<name-node>name-node</name-node>
</global>
<start to="decision-00000005" />
<action name="action-00000002">
<shell xmlns="uri:oozie:shell-action:0.3">
<exec>echo</exec>
<argument>error</argument>
</shell>
<ok to="kill-00000003" />
<error to="end" />
</action>
<kill name="kill-00000003">
<message>A bad thing happened</message>
</kill>
<decision name="decision-00000005">
<switch>
<case to="action-00000001">${wf:lastErrorNode() eq null}</case>
<default to="action-00000000" />
</switch>
</decision>
<action name="action-00000000">
<shell xmlns="uri:oozie:shell-action:0.3">
<exec>echo</exec>
<argument>default</argument>
</shell>
<ok to="end" />
<error to="action-00000002" />
</action>
<action name="action-00000001">
<shell xmlns="uri:oozie:shell-action:0.3">
<exec>echo</exec>
<argument>"No last error node"</argument>
</shell>
<ok to="end" />
<error to="action-00000002" />
</action>
<end name="end" />
</workflow-app>
""")
def test_workflow_action(request):
entities = tags.Action(
name='action-name',
action=tags.Shell(exec_command='echo', arguments=['build']),
)
assert len(set(entities)) == 1
assert bool(entities)
workflow_app = tags.WorkflowApp(
name='descriptive-name',
job_tracker='job-tracker',
name_node='name-node',
entities=entities
)
assert_workflow(request, workflow_app, """
<workflow-app xmlns="uri:oozie:workflow:0.5" name="descriptive-name">
<global>
<job-tracker>job-tracker</job-tracker>
<name-node>name-node</name-node>
</global>
<start to="action-action-name" />
<action name="action-action-name">
<shell xmlns="uri:oozie:shell-action:0.3">
<exec>echo</exec>
<argument>build</argument>
</shell>
<ok to="end" />
<error to="end" />
</action>
<end name="end" />
</workflow-app>
""")
entities = tags.Action(
name='action-name',
action=tags.Shell(exec_command='echo', arguments=['build']),
credential='my-hcat-creds',
retry_max=10,
retry_interval=20,
on_error=tags.Kill(name='error', message='A bad thing happened'),
)
assert len(set(entities)) == 2
assert bool(entities)
workflow_app = tags.WorkflowApp(
name='descriptive-name',
job_tracker='job-tracker',
name_node='name-node',
credentials=[tags.Credential(
{'cred_name': 'cred_value'},
credential_name='my-hcat-creds',
credential_type='hcat')],
entities=entities
)
assert_workflow(request, workflow_app, """
<workflow-app xmlns="uri:oozie:workflow:0.5" name="descriptive-name">
<global>
<job-tracker>job-tracker</job-tracker>
<name-node>name-node</name-node>
</global>
<credentials>
<credential type="hcat" name="my-hcat-creds">
<property>
<name>cred_name</name>
<value>cred_value</value>
</property>
</credential>
</credentials>
<start to="action-action-name" />
<action retry-max="10" cred="my-hcat-creds" name="action-action-name" retry-interval="20">
<shell xmlns="uri:oozie:shell-action:0.3">
<exec>echo</exec>
<argument>build</argument>
</shell>
<ok to="end" />
<error to="kill-error" />
</action>
<kill name="kill-error">
<message>A bad thing happened</message>
</kill>
<end name="end" />
</workflow-app>
""")
