def test_workflow_with_reused_identifier(): with pytest.raises(AssertionError) as assertion_info: tags.WorkflowApp( name='descriptive-name', job_tracker='job-tracker', name_node='name-node', entities=tags.Action( name='build', action=tags.Shell(exec_command='echo', arguments=['build']), on_error=tags.Action(name='build', action=tags.Shell(exec_command='echo', arguments=['error'])) ) ) assert str(assertion_info.value) == 'Name(s) reused: action-build' with pytest.raises(AssertionError) as assertion_info: tags.WorkflowApp( name='descriptive-name', job_tracker='job-tracker', name_node='name-node', entities=tags.Serial( tags.Action(name='build', action=tags.Shell(exec_command='echo', arguments=['build'])), tags.Action(name='resolve', action=tags.Shell(exec_command='echo', arguments=['resolve'])), on_error=tags.Serial( tags.Action(name='build', action=tags.Shell(exec_command='echo', arguments=['error'])), tags.Kill('A bad thing happened') ) ) ) assert str(assertion_info.value) == 'Name(s) reused: action-build'
def test_builder_raises_on_bad_action_name(): # Does it throw an exception on a bad action name? with pytest.raises(AssertionError) as assertion_info: xml.WorkflowBuilder( name='descriptive-name' ).add_action( name='Action name with invalid characters', action=tags.Shell(exec_command='echo "test"'), action_on_error=tags.Email(to='*****@*****.**', subject='Error', body='A bad thing happened'), kill_on_error='Failure message', ) assert "Identifier must match " in str(assertion_info.value) and \ "Action name with invalid characters" in str(assertion_info.value) # Does it throw an exception on an action name that's too long? with pytest.raises(AssertionError) as assertion_info: xml.WorkflowBuilder( name='descriptive-name' ).add_action( name='l' * (tags.MAX_IDENTIFIER_LENGTH + 1), action=tags.Shell(exec_command='echo "test"'), action_on_error=tags.Email(to='*****@*****.**', subject='Error', body='A bad thing happened'), kill_on_error='Failure message', ) assert "Identifier must be less than " in str(assertion_info.value)
def test_workflow_app_inherit_parent_error(request): workflow_app = tags.WorkflowApp( name='descriptive-name', job_tracker='job-tracker', name_node='name-node', entities=tags.Serial( tags.Parallel( tags.Action(name='build_a', action=tags.Shell(exec_command='echo', arguments=['build_a'])), tags.Action(name='build_b', action=tags.Shell(exec_command='echo', arguments=['build_b'])), name='builders' ), tags.Parallel( tags.Action(name='resolve_a', action=tags.Shell(exec_command='echo', arguments=['resolve_a'])), tags.Action(name='resolve_b', action=tags.Shell(exec_command='echo', arguments=['resolve_b'])), name='resolvers' ), on_error=tags.Serial( tags.Action(name='error', action=tags.Shell(exec_command='echo', arguments=['error'])), tags.Kill(name='error', message='A bad thing happened') ) ) ) assert_workflow(request, workflow_app) app = tests.utils.ParsedXml(workflow_app.xml()) app.assert_node("/action[@name='action-build_a']/error", to='action-error') app.assert_node("/action[@name='action-build_b']/error", to='action-error') app.assert_node("/action[@name='action-resolve_a']/error", to='action-error') app.assert_node("/action[@name='action-resolve_a']/error", to='action-error') app.assert_node("/action[@name='action-error']/error", to='end') app.assert_node("/action[@name='action-error']/ok", to='kill-error')
def test_workflow_app_serial_entities(request): # Create a serial collection of entities with a serial collection as an error condition entities = tags.Serial( tags.Action(tags.Shell(exec_command='echo', arguments=['build'])), tags.Action(tags.Shell(exec_command='echo', arguments=['resolve'])), on_error=tags.Serial( tags.Action(tags.Shell(exec_command='echo', arguments=['error'])), tags.Kill('A bad thing happened') ) ) assert len(set(entities)) == 4 assert bool(entities) workflow_app = tags.WorkflowApp( name='descriptive-name', job_tracker='job-tracker', name_node='name-node', entities=entities ) assert_workflow(request, workflow_app, """ <workflow-app xmlns="uri:oozie:workflow:0.5" name="descriptive-name"> <global> <job-tracker>job-tracker</job-tracker> <name-node>name-node</name-node> </global> <start to="action-00000000" /> <action name="action-00000002"> <shell xmlns="uri:oozie:shell-action:0.3"> <exec>echo</exec> <argument>error</argument> </shell> <ok to="kill-00000003" /> <error to="end" /> </action> <kill name="kill-00000003"> <message>A bad thing happened</message> </kill> <action name="action-00000000"> <shell xmlns="uri:oozie:shell-action:0.3"> <exec>echo</exec> <argument>build</argument> </shell> <ok to="action-00000001" /> <error to="action-00000002" /> </action> <action name="action-00000001"> <shell xmlns="uri:oozie:shell-action:0.3"> <exec>echo</exec> <argument>resolve</argument> </shell> <ok to="end" /> <error to="action-00000002" /> </action> <end name="end" /> </workflow-app> """)
def test_shell(): actual = tags.Shell( exec_command='${EXEC}', job_tracker='${jobTracker}', name_node='${nameNode}', prepare=None, job_xml_files=['/user/${wf:user()}/job.xml'], configuration={ 'mapred.job.queue.name': '${queueName}' }, arguments=['A', 'B'], env_vars={ 'ENVIRONMENT': 'production', 'RESOURCES': 'large', }, files=['/users/blabla/testfile.sh#testfile'], archives=['/users/blabla/testarchive.jar#testarchive'], capture_output=True, ).xml(indent=True) assert tests.utils.xml_to_comparable_dict(''' <shell xmlns="uri:oozie:shell-action:0.3"> <job-tracker>${jobTracker}</job-tracker> <name-node>${nameNode}</name-node> <job-xml>/user/${wf:user()}/job.xml</job-xml> <configuration> <property> <name>mapred.job.queue.name</name> <value>${queueName}</value> </property> </configuration> <exec>${EXEC}</exec> <argument>A</argument> <argument>B</argument> <file>/users/blabla/testfile.sh#testfile</file> <archive>/users/blabla/testarchive.jar#testarchive</archive> <env-var>ENVIRONMENT=production</env-var> <env-var>RESOURCES=large</env-var> <capture-output /> </shell>''') == tests.utils.xml_to_comparable_dict(actual) # Test using prepare fails with pytest.raises(NotImplementedError) as assertion_info: tags.Shell( exec_command='${EXEC}', prepare=['anything'], ).xml() assert str(assertion_info.value) == "Shell action's prepare has not yet been implemented"
def workflow_builder(): return xml.WorkflowBuilder( name='descriptive-name' ).add_action( name='payload', action=tags.Shell(exec_command='echo "test"'), action_on_error=tags.Email(to='*****@*****.**', subject='Error', body='A bad thing happened'), kill_on_error='Failure message 😢', )
def test_builder_raises_on_multiple_actions(workflow_builder): # Does it raise an exception when you try to add multiple actions? with pytest.raises(NotImplementedError) as assertion_info: workflow_builder.add_action( name='payload', action=tags.Shell(exec_command='echo "test"'), action_on_error=tags.Email(to='*****@*****.**', subject='Error', body='A bad thing happened'), kill_on_error='Failure message', ) assert str(assertion_info.value) == 'Can only add one action in this version'
def test_builder_raises_on_bad_workflow_name(): # Does it throw an exception on a bad workflow name? with pytest.raises(AssertionError) as assertion_info: xml.WorkflowBuilder( name='l' * (tags.MAX_NAME_LENGTH + 1) ).add_action( name='payload', action=tags.Shell(exec_command='echo "test"'), action_on_error=tags.Email(to='*****@*****.**', subject='Error', body='A bad thing happened'), kill_on_error='Failure message', ) assert "Name must be less than " in str(assertion_info.value)
def workflow_app_xml(**kwargs): workflow_app = tags.WorkflowApp( name='descriptive-name', job_tracker='job-tracker', name_node='name-node', entities=tags.Action( name='name', action=tags.Shell(exec_command='echo', arguments=['build']), **kwargs ) ) assert_workflow(request, workflow_app) return tests.utils.ParsedXml(workflow_app.xml())
def test_workflow_app_empty_decision_entities(): with pytest.raises(AssertionError) as assertion_info: tags.Decision( default=None, choices={ '${wf:lastErrorNode() eq null}': tags.Action( tags.Shell(exec_command='echo', arguments=['"arg"']) ), }) assert str(assertion_info.value) == 'A default must be supplied' with pytest.raises(AssertionError) as assertion_info: tags.Decision( default=tags.Shell(exec_command='echo', arguments=['"arg"']), choices=None ) assert str(assertion_info.value) == 'At least one choice required' with pytest.raises(AssertionError) as assertion_info: tags.Decision( default=tags.Shell(exec_command='echo', arguments=['"arg"']), choices={} ) assert str(assertion_info.value) == 'At least one choice required'
def test_workflow_action_without_credential(): with pytest.raises(AssertionError) as assertion_info: tags.WorkflowApp( name='descriptive-name', job_tracker='job-tracker', name_node='name-node', entities=tags.Action( name='action-name', action=tags.Shell(exec_command='echo', arguments=['build']), credential='my-hcat-creds', retry_max=10, retry_interval=20, ) ) assert str(assertion_info.value) == str('Missing credentials: my-hcat-creds')
def test_workflow_app_decision_entities(request): entities = tags.Decision( default=tags.Action(tags.Shell(exec_command='echo', arguments=['default'])), choices={ '${wf:lastErrorNode() eq null}': tags.Action( tags.Shell(exec_command='echo', arguments=['"No last error node"'])), }, on_error=tags.Serial( tags.Action(tags.Shell(exec_command='echo', arguments=['error'])), tags.Kill('A bad thing happened') ) ) assert len(set(entities)) == 5 workflow_app = tags.WorkflowApp( name='descriptive-name', job_tracker='job-tracker', name_node='name-node', entities=entities ) assert_workflow(request, workflow_app, """ <workflow-app xmlns="uri:oozie:workflow:0.5" name="descriptive-name"> <global> <job-tracker>job-tracker</job-tracker> <name-node>name-node</name-node> </global> <start to="decision-00000005" /> <action name="action-00000002"> <shell xmlns="uri:oozie:shell-action:0.3"> <exec>echo</exec> <argument>error</argument> </shell> <ok to="kill-00000003" /> <error to="end" /> </action> <kill name="kill-00000003"> <message>A bad thing happened</message> </kill> <decision name="decision-00000005"> <switch> <case to="action-00000001">${wf:lastErrorNode() eq null}</case> <default to="action-00000000" /> </switch> </decision> <action name="action-00000000"> <shell xmlns="uri:oozie:shell-action:0.3"> <exec>echo</exec> <argument>default</argument> </shell> <ok to="end" /> <error to="action-00000002" /> </action> <action name="action-00000001"> <shell xmlns="uri:oozie:shell-action:0.3"> <exec>echo</exec> <argument>"No last error node"</argument> </shell> <ok to="end" /> <error to="action-00000002" /> </action> <end name="end" /> </workflow-app> """)
def test_workflow_action(request): entities = tags.Action( name='action-name', action=tags.Shell(exec_command='echo', arguments=['build']), ) assert len(set(entities)) == 1 assert bool(entities) workflow_app = tags.WorkflowApp( name='descriptive-name', job_tracker='job-tracker', name_node='name-node', entities=entities ) assert_workflow(request, workflow_app, """ <workflow-app xmlns="uri:oozie:workflow:0.5" name="descriptive-name"> <global> <job-tracker>job-tracker</job-tracker> <name-node>name-node</name-node> </global> <start to="action-action-name" /> <action name="action-action-name"> <shell xmlns="uri:oozie:shell-action:0.3"> <exec>echo</exec> <argument>build</argument> </shell> <ok to="end" /> <error to="end" /> </action> <end name="end" /> </workflow-app> """) entities = tags.Action( name='action-name', action=tags.Shell(exec_command='echo', arguments=['build']), credential='my-hcat-creds', retry_max=10, retry_interval=20, on_error=tags.Kill(name='error', message='A bad thing happened'), ) assert len(set(entities)) == 2 assert bool(entities) workflow_app = tags.WorkflowApp( name='descriptive-name', job_tracker='job-tracker', name_node='name-node', credentials=[tags.Credential( {'cred_name': 'cred_value'}, credential_name='my-hcat-creds', credential_type='hcat')], entities=entities ) assert_workflow(request, workflow_app, """ <workflow-app xmlns="uri:oozie:workflow:0.5" name="descriptive-name"> <global> <job-tracker>job-tracker</job-tracker> <name-node>name-node</name-node> </global> <credentials> <credential type="hcat" name="my-hcat-creds"> <property> <name>cred_name</name> <value>cred_value</value> </property> </credential> </credentials> <start to="action-action-name" /> <action retry-max="10" cred="my-hcat-creds" name="action-action-name" retry-interval="20"> <shell xmlns="uri:oozie:shell-action:0.3"> <exec>echo</exec> <argument>build</argument> </shell> <ok to="end" /> <error to="kill-error" /> </action> <kill name="kill-error"> <message>A bad thing happened</message> </kill> <end name="end" /> </workflow-app> """)