def submit_single_action(request, doc_id, node_id):
parent_doc = Document2.objects.get(id=doc_id)
workflow_data = Workflow(
document=parent_doc).create_single_action_workflow_data(node_id)
_data = json.loads(workflow_data)
# Create separate wf object for the submit node with new deployment_dir
workflow = Workflow(data=workflow_data)
workflow.set_workspace(request.user)
workflow.check_workspace(request.fs, request.user)
workflow.document = wf_doc = Document2.objects.create(
name=_data['workflow']['name'],
type='oozie-workflow2',
owner=parent_doc.owner,
data=workflow_data)
Document.objects.link(wf_doc,
owner=wf_doc.owner,
name=wf_doc.name,
description=wf_doc.description,
extra='workflow2')
return _submit_workflow_helper(request,
workflow,
submit_action=reverse(
'oozie:submit_single_action',
kwargs={
'doc_id': doc_id,
'node_id': node_id
}))
def test_workflow_create_single_action_data(self):
workflow = Workflow(
data=
"{\"layout\": [{\"oozieRows\": [{\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"MapReduce job\", \"widgetType\": \"mapreduce-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"0cf2d5d5-2315-0bda-bd53-0eec257e943f\", \"size\": 12}], \"id\": \"e2caca14-8afc-d7e0-287c-88accd0b4253\", \"columns\": []}], \"rows\": [{\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Start\", \"widgetType\": \"start-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"size\": 12}], \"id\": \"ff63ee3f-df54-2fa3-477b-65f5e0f0632c\", \"columns\": []}, {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"MapReduce job\", \"widgetType\": \"mapreduce-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"0cf2d5d5-2315-0bda-bd53-0eec257e943f\", \"size\": 12}], \"id\": \"e2caca14-8afc-d7e0-287c-88accd0b4253\", \"columns\": []}, {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"End\", \"widgetType\": \"end-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"size\": 12}], \"id\": \"6a13d869-d04c-8431-6c5c-dbe67ea33889\", \"columns\": []}, {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Kill\", \"widgetType\": \"kill-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"size\": 12}], \"id\": \"e3b56553-7a4f-43d2-b1e2-4dc433280095\", \"columns\": []}], \"oozieEndRow\": {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"End\", \"widgetType\": \"end-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"size\": 12}], \"id\": \"6a13d869-d04c-8431-6c5c-dbe67ea33889\", \"columns\": []}, \"oozieKillRow\": {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Kill\", \"widgetType\": \"kill-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"size\": 12}], \"id\": \"e3b56553-7a4f-43d2-b1e2-4dc433280095\", \"columns\": []}, \"enableOozieDropOnAfter\": true, \"oozieStartRow\": {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Start\", \"widgetType\": \"start-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"size\": 12}], \"id\": \"ff63ee3f-df54-2fa3-477b-65f5e0f0632c\", \"columns\": []}, \"klass\": \"card card-home card-column span12\", \"enableOozieDropOnBefore\": true, \"drops\": [\"temp\"], \"id\": \"0c1908e7-0096-46e7-a16b-b17b1142a730\", \"size\": 12}], \"workflow\": {\"properties\": {\"job_xml\": \"\", \"description\": \"\", \"wf1_id\": null, \"sla_enabled\": false, \"deployment_dir\": \"/user/hue/oozie/workspaces/hue-oozie-1430228904.58\", \"schema_version\": \"uri:oozie:workflow:0.5\", \"sla\": [{\"key\": \"enabled\", \"value\": false}, {\"key\": \"nominal-time\", \"value\": \"${nominal_time}\"}, {\"key\": \"should-start\", \"value\": \"\"}, {\"key\": \"should-end\", \"value\": \"${30 * MINUTES}\"}, {\"key\": \"max-duration\", \"value\": \"\"}, {\"key\": \"alert-events\", \"value\": \"\"}, {\"key\": \"alert-contact\", \"value\": \"\"}, {\"key\": \"notification-msg\", \"value\": \"\"}, {\"key\": \"upstream-apps\", \"value\": \"\"}], \"show_arrows\": true, \"parameters\": [{\"name\": \"oozie.use.system.libpath\", \"value\": true}], \"properties\": []}, \"name\": \"My Workflow\", \"versions\": [\"uri:oozie:workflow:0.4\", \"uri:oozie:workflow:0.4.5\", \"uri:oozie:workflow:0.5\"], \"isDirty\": true, \"movedNode\": null, \"linkMapping\": {\"0cf2d5d5-2315-0bda-bd53-0eec257e943f\": [\"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\"], \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\": [], \"3f107997-04cc-8733-60a9-a4bb62cebffc\": [\"0cf2d5d5-2315-0bda-bd53-0eec257e943f\"], \"17c9c895-5a16-7443-bb81-f34b30b21548\": []}, \"nodeIds\": [\"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"0cf2d5d5-2315-0bda-bd53-0eec257e943f\"], \"nodes\": [{\"properties\": {}, \"name\": \"Start\", \"children\": [{\"to\": \"0cf2d5d5-2315-0bda-bd53-0eec257e943f\"}], \"actionParametersFetched\": false, \"type\": \"start-widget\", \"id\": \"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"actionParameters\": []}, {\"properties\": {}, \"name\": \"End\", \"children\": [], \"actionParametersFetched\": false, \"type\": \"end-widget\", \"id\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"actionParameters\": []}, {\"properties\": {\"message\": \"Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]\"}, \"name\": \"Kill\", \"children\": [], \"actionParametersFetched\": false, \"type\": \"kill-widget\", \"id\": \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"actionParameters\": []}, {\"properties\": {\"retry_max\": [{\"value\": \"5\"}], \"files\": [], \"job_xml\": \"\", \"jar_path\": \"my_jar\", \"job_properties\": [{\"name\": \"prop_1_name\", \"value\": \"prop_1_value\"}], \"archives\": [], \"prepares\": [], \"credentials\": [], \"sla\": [{\"key\": \"enabled\", \"value\": false}, {\"key\": \"nominal-time\", \"value\": \"${nominal_time}\"}, {\"key\": \"should-start\", \"value\": \"\"}, {\"key\": \"should-end\", \"value\": \"${30 * MINUTES}\"}, {\"key\": \"max-duration\", \"value\": \"\"}, {\"key\": \"alert-events\", \"value\": \"\"}, {\"key\": \"alert-contact\", \"value\": \"\"}, {\"key\": \"notification-msg\", \"value\": \"\"}, {\"key\": \"upstream-apps\", \"value\": \"\"}]}, \"name\": \"mapreduce-0cf2\", \"children\": [{\"to\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\"}, {\"error\": \"17c9c895-5a16-7443-bb81-f34b30b21548\"}], \"actionParametersFetched\": false, \"type\": \"mapreduce-widget\", \"id\": \"0cf2d5d5-2315-0bda-bd53-0eec257e943f\", \"actionParameters\": []}], \"id\": 50019, \"nodeNamesMapping\": {\"0cf2d5d5-2315-0bda-bd53-0eec257e943f\": \"mapreduce-0cf2\", \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\": \"End\", \"3f107997-04cc-8733-60a9-a4bb62cebffc\": \"Start\", \"17c9c895-5a16-7443-bb81-f34b30b21548\": \"Kill\"}, \"uuid\": \"084f4d4c-00f1-62d2-e27e-e153c1f9acfb\"}}"
)
single_action_wf_data = workflow.create_single_action_workflow_data(
'0cf2d5d5-2315-0bda-bd53-0eec257e943f')
single_action_wf = Workflow(data=single_action_wf_data)
assert_true(len(single_action_wf.nodes) == 4)
# Validating DAG: Start -> node -> Kill/End
_data = json.loads(single_action_wf_data)
start_node = [
node for node in _data['workflow']['nodes']
if node['name'] == 'Start'
][0]
submit_node = [
node for node in _data['workflow']['nodes']
if node['id'] == '0cf2d5d5-2315-0bda-bd53-0eec257e943f'
][0]
end_node = [
node for node in _data['workflow']['nodes']
if node['name'] == 'End'
][0]
kill_node = [
node for node in _data['workflow']['nodes']
if node['name'] == 'Kill'
][0]
assert_true(submit_node['id'] in str(start_node['children']))
assert_true(end_node['id'] in str(submit_node['children']))
assert_true(kill_node['id'] in str(submit_node['children']))
def submit_single_action(request, doc_id, node_id):
parent_doc = Document2.objects.get(id=doc_id)
parent_wf = Workflow(document=parent_doc)
workflow_data = parent_wf.create_single_action_workflow_data(node_id)
_data = json.loads(workflow_data)
# Create separate wf object for the submit node with new deployment_dir
workflow = Workflow(data=workflow_data)
workflow.set_workspace(request.user)
workflow.check_workspace(request.fs, request.user)
# The imported wf deployment directory might not neccessarily exist on first submission
if not request.fs.exists(parent_wf.deployment_dir):
request.fs.do_as_user(request.user.username, request.fs.mkdir,
parent_wf.deployment_dir)
workflow.import_workspace(request.fs, parent_wf.deployment_dir,
request.user)
workflow.document = parent_doc
return _submit_workflow_helper(request,
workflow,
submit_action=reverse(
'oozie:submit_single_action',
kwargs={
'doc_id': doc_id,
'node_id': node_id
}))
def submit_single_action(request, doc_id, node_id):
parent_doc = Document2.objects.get(id=doc_id)
parent_wf = Workflow(document=parent_doc)
workflow_data = parent_wf.create_single_action_workflow_data(node_id)
_data = json.loads(workflow_data)
# Create separate wf object for the submit node with new deployment_dir
workflow = Workflow(data=workflow_data)
workflow.set_workspace(request.user)
workflow.check_workspace(request.fs, request.user)
workflow.import_workspace(request.fs, parent_wf.deployment_dir, request.user)
workflow.document = parent_doc
return _submit_workflow_helper(request, workflow, submit_action=reverse('oozie:submit_single_action', kwargs={'doc_id': doc_id, 'node_id': node_id}))
def execute(self, notebook, snippet):
# Get document from notebook
if not notebook.get('uuid', ''):
raise PopupException(
_('Notebook is missing a uuid, please save the notebook before executing as a batch job.'
))
notebook_doc = Document2.objects.get_by_uuid(user=self.user,
uuid=notebook['uuid'],
perm_type='read')
# Create a managed workflow from the notebook doc
workflow_doc = WorkflowBuilder().create_workflow(
document=notebook_doc,
user=self.user,
managed=True,
name=_("Batch job for %s") % notebook_doc.name
or notebook_doc.type)
workflow = Workflow(document=workflow_doc, user=self.user)
# Submit workflow
job_id = _submit_workflow(user=self.user,
fs=self.fs,
jt=self.jt,
workflow=workflow,
mapping=None)
return {
'id': job_id,
'has_result_set': True,
}
def test_workflow_map_reduce_gen_xml(self):
wf = Workflow(data="{\"layout\": [{\"oozieRows\": [{\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"MapReduce job\", \"widgetType\": \"mapreduce-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"0cf2d5d5-2315-0bda-bd53-0eec257e943f\", \"size\": 12}], \"id\": \"e2caca14-8afc-d7e0-287c-88accd0b4253\", \"columns\": []}], \"rows\": [{\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Start\", \"widgetType\": \"start-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"size\": 12}], \"id\": \"ff63ee3f-df54-2fa3-477b-65f5e0f0632c\", \"columns\": []}, {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"MapReduce job\", \"widgetType\": \"mapreduce-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"0cf2d5d5-2315-0bda-bd53-0eec257e943f\", \"size\": 12}], \"id\": \"e2caca14-8afc-d7e0-287c-88accd0b4253\", \"columns\": []}, {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"End\", \"widgetType\": \"end-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"size\": 12}], \"id\": \"6a13d869-d04c-8431-6c5c-dbe67ea33889\", \"columns\": []}, {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Kill\", \"widgetType\": \"kill-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"size\": 12}], \"id\": \"e3b56553-7a4f-43d2-b1e2-4dc433280095\", \"columns\": []}], \"oozieEndRow\": {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"End\", \"widgetType\": \"end-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"size\": 12}], \"id\": \"6a13d869-d04c-8431-6c5c-dbe67ea33889\", \"columns\": []}, \"oozieKillRow\": {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Kill\", \"widgetType\": \"kill-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"size\": 12}], \"id\": \"e3b56553-7a4f-43d2-b1e2-4dc433280095\", \"columns\": []}, \"enableOozieDropOnAfter\": true, \"oozieStartRow\": {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Start\", \"widgetType\": \"start-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"size\": 12}], \"id\": \"ff63ee3f-df54-2fa3-477b-65f5e0f0632c\", \"columns\": []}, \"klass\": \"card card-home card-column span12\", \"enableOozieDropOnBefore\": true, \"drops\": [\"temp\"], \"id\": \"0c1908e7-0096-46e7-a16b-b17b1142a730\", \"size\": 12}], \"workflow\": {\"properties\": {\"job_xml\": \"\", \"description\": \"\", \"wf1_id\": null, \"sla_enabled\": false, \"deployment_dir\": \"/user/hue/oozie/workspaces/hue-oozie-1430228904.58\", \"schema_version\": \"uri:oozie:workflow:0.5\", \"sla\": [{\"key\": \"enabled\", \"value\": false}, {\"key\": \"nominal-time\", \"value\": \"${nominal_time}\"}, {\"key\": \"should-start\", \"value\": \"\"}, {\"key\": \"should-end\", \"value\": \"${30 * MINUTES}\"}, {\"key\": \"max-duration\", \"value\": \"\"}, {\"key\": \"alert-events\", \"value\": \"\"}, {\"key\": \"alert-contact\", \"value\": \"\"}, {\"key\": \"notification-msg\", \"value\": \"\"}, {\"key\": \"upstream-apps\", \"value\": \"\"}], \"show_arrows\": true, \"parameters\": [{\"name\": \"oozie.use.system.libpath\", \"value\": true}], \"properties\": []}, \"name\": \"My Workflow\", \"versions\": [\"uri:oozie:workflow:0.4\", \"uri:oozie:workflow:0.4.5\", \"uri:oozie:workflow:0.5\"], \"isDirty\": true, \"movedNode\": null, \"linkMapping\": {\"0cf2d5d5-2315-0bda-bd53-0eec257e943f\": [\"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\"], \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\": [], \"3f107997-04cc-8733-60a9-a4bb62cebffc\": [\"0cf2d5d5-2315-0bda-bd53-0eec257e943f\"], \"17c9c895-5a16-7443-bb81-f34b30b21548\": []}, \"nodeIds\": [\"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"0cf2d5d5-2315-0bda-bd53-0eec257e943f\"], \"nodes\": [{\"properties\": {}, \"name\": \"Start\", \"children\": [{\"to\": \"0cf2d5d5-2315-0bda-bd53-0eec257e943f\"}], \"actionParametersFetched\": false, \"type\": \"start-widget\", \"id\": \"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"actionParameters\": []}, {\"properties\": {}, \"name\": \"End\", \"children\": [], \"actionParametersFetched\": false, \"type\": \"end-widget\", \"id\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"actionParameters\": []}, {\"properties\": {\"message\": \"Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]\"}, \"name\": \"Kill\", \"children\": [], \"actionParametersFetched\": false, \"type\": \"kill-widget\", \"id\": \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"actionParameters\": []}, {\"properties\": {\"files\": [], \"job_xml\": \"\", \"jar_path\": \"my_jar\", \"job_properties\": [{\"name\": \"prop_1_name\", \"value\": \"prop_1_value\"}], \"archives\": [], \"prepares\": [], \"credentials\": [], \"sla\": [{\"key\": \"enabled\", \"value\": false}, {\"key\": \"nominal-time\", \"value\": \"${nominal_time}\"}, {\"key\": \"should-start\", \"value\": \"\"}, {\"key\": \"should-end\", \"value\": \"${30 * MINUTES}\"}, {\"key\": \"max-duration\", \"value\": \"\"}, {\"key\": \"alert-events\", \"value\": \"\"}, {\"key\": \"alert-contact\", \"value\": \"\"}, {\"key\": \"notification-msg\", \"value\": \"\"}, {\"key\": \"upstream-apps\", \"value\": \"\"}]}, \"name\": \"mapreduce-0cf2\", \"children\": [{\"to\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\"}, {\"error\": \"17c9c895-5a16-7443-bb81-f34b30b21548\"}], \"actionParametersFetched\": false, \"type\": \"mapreduce-widget\", \"id\": \"0cf2d5d5-2315-0bda-bd53-0eec257e943f\", \"actionParameters\": []}], \"id\": 50019, \"nodeNamesMapping\": {\"0cf2d5d5-2315-0bda-bd53-0eec257e943f\": \"mapreduce-0cf2\", \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\": \"End\", \"3f107997-04cc-8733-60a9-a4bb62cebffc\": \"Start\", \"17c9c895-5a16-7443-bb81-f34b30b21548\": \"Kill\"}, \"uuid\": \"084f4d4c-00f1-62d2-e27e-e153c1f9acfb\"}}")
assert_equal([
u'<workflow-app', u'name="My_Workflow"', u'xmlns="uri:oozie:workflow:0.5">',
u'<start', u'to="mapreduce-0cf2"/>',
u'<kill', u'name="Kill">', u'<message>Action', u'failed,', u'error', u'message[${wf:errorMessage(wf:lastErrorNode())}]</message>', u'</kill>',
u'<action', u'name="mapreduce-0cf2">',
u'<map-reduce>',
u'<job-tracker>${jobTracker}</job-tracker>',
u'<name-node>${nameNode}</name-node>',
u'<configuration>',
u'<property>',
u'<name>prop_1_name</name>',
u'<value>prop_1_value</value>',
u'</property>',
u'</configuration>',
u'</map-reduce>',
u'<ok', u'to="End"/>',
u'<error', u'to="Kill"/>',
u'</action>',
u'<end', u'name="End"/>',
u'</workflow-app>'
],
wf.to_xml({'output': '/path'}).split()
)
def save_workflow(request):
response = {'status': -1}
workflow = json.loads(request.POST.get('workflow', '{}'))
layout = json.loads(request.POST.get('layout', '{}'))
if workflow.get('id'):
workflow_doc = Document2.objects.get(id=workflow['id'])
else:
workflow_doc = Document2.objects.create(name=workflow['name'], uuid=workflow['uuid'], type='oozie-workflow2', owner=request.user, description=workflow['properties']['description'])
Document.objects.link(workflow_doc, owner=workflow_doc.owner, name=workflow_doc.name, description=workflow_doc.description, extra='workflow2')
subworkflows = [node['properties']['workflow'] for node in workflow['nodes'] if node['type'] == 'subworkflow-widget']
if subworkflows:
dependencies = Document2.objects.filter(uuid__in=subworkflows)
workflow_doc.dependencies = dependencies
if workflow['properties'].get('imported'): # Old workflow format
workflow['properties']['imported'] = False
response['url'] = reverse('oozie:edit_workflow') + '?workflow=' + str(workflow_doc.id)
workflow_doc.update_data({'workflow': workflow})
workflow_doc.update_data({'layout': layout})
workflow_doc.name = workflow['name']
workflow_doc.save()
workflow_instance = Workflow(document=workflow_doc)
response['status'] = 0
response['id'] = workflow_doc.id
response['doc1_id'] = workflow_doc.doc.get().id
response['message'] = _('Page saved !')
return HttpResponse(json.dumps(response), mimetype="application/json")
def _submit_coordinator(request, coordinator, mapping):
try:
wf_doc = Document2.objects.get(
uuid=coordinator.data['properties']['workflow'])
wf_dir = Submission(
request.user,
Workflow(document=wf_doc),
request.fs,
request.jt,
mapping,
local_tz=coordinator.data['properties']['timezone']).deploy()
properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
properties.update(mapping)
submission = Submission(request.user,
coordinator,
request.fs,
request.jt,
properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting coordinator')
raise PopupException(_("Error submitting coordinator %s") %
(coordinator, ),
detail=ex._headers.get('oozie-error-message', ex))
def _submit_bundle(request, bundle, properties):
try:
deployment_mapping = {}
coords = dict([(c.uuid, c) for c in Document2.objects.filter(
type='oozie-coordinator2',
uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])
for i, bundled in enumerate(bundle.data['coordinators']):
coord = coords[bundled['coordinator']]
workflow = Workflow(document=coord.dependencies.all()[0])
wf_dir = Submission(request.user, workflow, request.fs, request.jt,
properties).deploy()
deployment_mapping['wf_%s_dir' %
i] = request.fs.get_hdfs_path(wf_dir)
coordinator = Coordinator(document=coord)
coord_dir = Submission(request.user, coordinator, request.fs,
request.jt, properties).deploy()
deployment_mapping['coord_%s_dir' % i] = coord_dir
deployment_mapping['coord_%s' % i] = coord
properties.update(deployment_mapping)
submission = Submission(request.user,
bundle,
request.fs,
request.jt,
properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
raise PopupException(_("Error submitting bundle %s") % (bundle, ),
detail=ex._headers.get('oozie-error-message', ex))
def copy_workflow(request):
if request.method != 'POST':
raise PopupException(_('A POST request is required.'))
jobs = json.loads(request.POST.get('selection'))
for job in jobs:
doc2 = Document2.objects.get(type='oozie-workflow2', id=job['id'])
name = doc2.name + '-copy'
copy_doc = doc2.doc.get().copy(name=name, owner=request.user)
doc2.pk = None
doc2.id = None
doc2.uuid = str(uuid.uuid4())
doc2.name = name
doc2.owner = request.user
doc2.save()
doc2.doc.all().delete()
doc2.doc.add(copy_doc)
workflow = Workflow(document=doc2)
workflow.update_name(name)
doc2.update_data({'workflow': workflow.get_data()['workflow']})
doc2.save()
workflow.set_workspace(request.user)
workflow.check_workspace(request.fs, request.user)
response = {}
request.info(_('Workflows copied.') if len(jobs) > 1 else _('Workflow copied.'))
return JsonResponse(response)
def new_workflow(request):
doc = None
workflow = Workflow()
workflow.set_workspace(request.user)
workflow.check_workspace(request.fs, request.user)
return _edit_workflow(request, doc, workflow)
def _submit_bundle(request, bundle, properties):
try:
deployment_mapping = {}
coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])
for i, bundled in enumerate(bundle.data['coordinators']):
coord = coords[bundled['coordinator']]
workflow = Workflow(document=coord.dependencies.filter(type='oozie-workflow2')[0])
wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy()
deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir)
coordinator = Coordinator(document=coord)
coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy()
deployment_mapping['coord_%s_dir' % i] = request.fs.get_hdfs_path(coord_dir)
deployment_mapping['coord_%s' % i] = coord
# Convert start/end dates of coordinator to server timezone
for prop in bundled['properties']:
if prop['name'] in ('end_date', 'start_date'):
prop['value'] = convert_to_server_timezone(prop['value'], local_tz=coordinator.data['properties']['timezone'])
properties.update(deployment_mapping)
submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting bundle')
raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex))
def copy_workflow(request):
if request.method != 'POST':
raise PopupException(_('A POST request is required.'))
jobs = json.loads(request.POST.get('selection'))
for job in jobs:
doc2 = Document2.objects.get(type='oozie-workflow2', id=job['id'])
doc = doc2.doc.get()
name = doc2.name + '-copy'
doc2 = doc2.copy(name=name, owner=request.user)
doc.copy(content_object=doc2, name=name, owner=request.user)
workflow = Workflow(document=doc2)
workflow.update_name(name)
_import_workspace(request.fs, request.user, workflow)
doc2.update_data({'workflow': workflow.get_data()['workflow']})
doc2.save()
response = {}
request.info(
_('Workflows copied.') if len(jobs) > 1 else _('Workflow copied.'))
return JsonResponse(response)
def submit_workflow(request, doc_id):
workflow = Workflow(document=Document2.objects.get(id=doc_id))
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value'])
for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
try:
job_id = _submit_workflow(request.user, request.fs, request.jt,
workflow, mapping)
except Exception, e:
raise PopupException(_('Workflow submission failed'),
detail=smart_str(e))
request.info(_('Workflow submitted'))
return redirect(
reverse('oozie:list_oozie_workflow', kwargs={'job_id':
job_id}))
else:
request.error(_('Invalid submission form: %s' %
params_form.errors))
def submit_workflow(request, doc_id):
workflow = Workflow(document=Document2.objects.get(id=doc_id))
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping)
request.info(_('Workflow submitted'))
return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
else:
parameters = workflow.find_all_parameters()
initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
params_form = ParametersFormSet(initial=initial_params)
popup = render('editor2/submit_job_popup.mako', request, {
'params_form': params_form,
'name': workflow.name,
'action': reverse('oozie:editor_submit_workflow', kwargs={'doc_id': workflow.id})
}, force_template=True).content
return JsonResponse(popup, safe=False)
def refresh_action_parameters(request):
response = {'status': -1}
try:
coord_uuid = request.POST.get('uuid')
workflow_doc = Document2.objects.get(type='oozie-workflow2', owner=request.user, is_managed=True, dependents__uuid__in=[coord_uuid])
# Refresh the action parameters of a document action in case the document changed
workflow = Workflow(document=workflow_doc, user=request.user)
_data = workflow.get_data()
hive_node = _data['workflow']['nodes'][3]
query_document = Document2.objects.get_by_uuid(user=request.user, uuid=hive_node['properties']['uuid'])
parameters = WorkflowBuilder().get_document_parameters(query_document)
changed = set([p['value'] for p in parameters]) != set([p['value'] for p in hive_node['properties']['parameters']])
if changed:
hive_node['properties']['parameters'] = parameters
workflow.data = json.dumps(_data)
workflow_doc.update_data({'workflow': _data['workflow']})
workflow_doc.save()
response['status'] = 0
response['parameters'] = parameters
response['changed'] = changed
except Exception, e:
response['message'] = str(e)
def copy_document(request):
uuid = json.loads(request.POST.get('uuid'), '""')
if not uuid:
raise PopupException(_('copy_document requires uuid'))
document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
if document.type == 'directory':
raise PopupException(_('Directory copy is not supported'))
name = document.name + '-copy'
# Make the copy of the new Document
copy_document = document.copy(name=name, owner=request.user)
# Import workspace for all oozie jobs
if document.type == 'oozie-workflow2' or document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2':
from oozie.models2 import Workflow, Coordinator, Bundle, _import_workspace
# Update the name field in the json 'data' field
if document.type == 'oozie-workflow2':
workflow = Workflow(document=document)
workflow.update_name(name)
workflow.update_uuid(copy_document.uuid)
_import_workspace(request.fs, request.user, workflow)
copy_document.update_data(
{'workflow': workflow.get_data()['workflow']})
copy_document.save()
if document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2':
if document.type == 'oozie-bundle2':
bundle_or_coordinator = Bundle(document=document)
else:
bundle_or_coordinator = Coordinator(document=document)
json_data = bundle_or_coordinator.get_data_for_json()
json_data['name'] = name
json_data['uuid'] = copy_document.uuid
copy_document.update_data(json_data)
copy_document.save()
_import_workspace(request.fs, request.user, bundle_or_coordinator)
elif document.type == 'search-dashboard':
from dashboard.models import Collection2
collection = Collection2(request.user, document=document)
collection.data['collection']['label'] = name
collection.data['collection']['uuid'] = copy_document.uuid
copy_document.update_data(
{'collection': collection.data['collection']})
copy_document.save()
# Keep the document and data in sync
else:
copy_data = copy_document.data_dict
if 'name' in copy_data:
copy_data['name'] = name
if 'uuid' in copy_data:
copy_data['uuid'] = copy_document.uuid
copy_document.update_data(copy_data)
copy_document.save()
return JsonResponse({'status': 0, 'document': copy_document.to_dict()})
def save_workflow(request):
response = {'status': -1}
workflow = json.loads(request.POST.get('workflow', '{}'))
layout = json.loads(request.POST.get('layout', '{}'))
if workflow.get('id'):
workflow_doc = Document2.objects.get(id=workflow['id'])
else:
workflow_doc = Document2.objects.create(
name=workflow['name'],
uuid=workflow['uuid'],
type='oozie-workflow2',
owner=request.user,
description=workflow['properties']['description'])
Document.objects.link(workflow_doc,
owner=workflow_doc.owner,
name=workflow_doc.name,
description=workflow_doc.description,
extra='workflow2')
# Excludes all the sub-workflow dependencies. Contains list of history and coordinator dependencies.
workflow_doc.dependencies = workflow_doc.dependencies.exclude(
Q(is_history=False) & Q(type='oozie-workflow2'))
subworkflows = [
node['properties']['workflow'] for node in workflow['nodes']
if node['type'] == 'subworkflow-widget'
]
if subworkflows:
subworkflow_dependencies = Document2.objects.filter(
uuid__in=subworkflows)
workflow_doc.dependencies.add(*subworkflow_dependencies)
if workflow['properties'].get(
'imported'): # We save and old format workflow to the latest
workflow['properties']['imported'] = False
workflow_instance = Workflow(workflow=workflow)
_import_workspace(request.fs, request.user, workflow_instance)
workflow['properties'][
'deployment_dir'] = workflow_instance.deployment_dir
response['url'] = reverse('oozie:edit_workflow') + '?workflow=' + str(
workflow_doc.id)
workflow_doc.update_data({'workflow': workflow})
workflow_doc.update_data({'layout': layout})
workflow_doc1 = workflow_doc.doc.get()
workflow_doc.name = workflow_doc1.name = workflow['name']
workflow_doc.description = workflow_doc1.description = workflow[
'properties']['description']
workflow_doc.save()
workflow_doc1.save()
response['status'] = 0
response['id'] = workflow_doc.id
response['doc1_id'] = workflow_doc1.id
response['message'] = _('Page saved !')
return JsonResponse(response)
def submit_workflow(request, doc_id):
workflow = Workflow(document=Document2.objects.get(id=doc_id))
return _submit_workflow_helper(request,
workflow,
submit_action=reverse(
'oozie:editor_submit_workflow',
kwargs={'doc_id': workflow.id}))
def workflow_parameters(request):
response = {'status': -1}
try:
workflow_doc = Document2.objects.get(uuid=request.GET.get('uuid') or request.GET.get('document'))
if workflow_doc.type == 'oozie-workflow2':
workflow = Workflow(document=workflow_doc, user=request.user)
else:
wf_doc = WorkflowBuilder().create_workflow(document=workflow_doc, user=request.user, managed=True)
workflow = Workflow(data=wf_doc.data)
wf_doc.delete()
response['status'] = 0
response['parameters'] = workflow.find_all_parameters(with_lib_path=False)
except Exception, e:
response['message'] = str(e)
def workflow_parameters(request):
response = {'status': -1}
try:
workflow = Workflow(document=Document2.objects.get(type='oozie-workflow2', uuid=request.GET.get('uuid')))
response['status'] = 0
response['parameters'] = workflow.find_all_parameters(with_lib_path=False)
except Exception, e:
response['message'] = str(e)
def new_workflow(request):
doc = None
workflow = Workflow(user=request.user)
workflow.set_workspace(request.user)
try:
workflow.check_workspace(request.fs, request.user)
except Exception as e:
raise PopupException(_('Could not create workflow workspace'), detail=e)
return _edit_workflow(request, doc, workflow)
def edit_workflow(request):
workflow_id = request.GET.get('workflow')
wid = {}
if workflow_id.isdigit():
wid['id'] = workflow_id
else:
wid['uuid'] = workflow_id
doc = Document2.objects.get(type='oozie-workflow2', **wid)
workflow = Workflow(document=doc)
return _edit_workflow(request, doc, workflow)
def gen_xml_workflow(request):
response = {'status': -1}
try:
workflow_json = json.loads(request.POST.get('workflow', '{}'))
workflow = Workflow(workflow=workflow_json)
response['status'] = 0
response['xml'] = workflow.to_xml()
except Exception, e:
response['message'] = str(e)
def execute(self, notebook, snippet):
# Get document from notebook
if not notebook.get('uuid', ''):
raise PopupException(
_('Notebook is missing a uuid, please save the notebook before executing as a batch job.'
))
if notebook['type'] == 'notebook' or notebook['type'] == 'query-java':
# Convert notebook to workflow
workflow_doc = WorkflowBuilder().create_notebook_workflow(
notebook=notebook,
user=self.user,
managed=True,
name=_("%s for %s") % (OozieApi.BATCH_JOB_PREFIX,
notebook['name'] or notebook['type']))
workflow = Workflow(document=workflow_doc, user=self.user)
else:
notebook_doc = Document2.objects.get_by_uuid(user=self.user,
uuid=notebook['uuid'],
perm_type='read')
# Create a managed workflow from the notebook doc
workflow_doc = WorkflowBuilder().create_workflow(
document=notebook_doc,
user=self.user,
managed=True,
name=_("Batch job for %s") %
(notebook_doc.name or notebook_doc.type))
workflow = Workflow(document=workflow_doc, user=self.user)
# Submit workflow
job_id = _submit_workflow(user=self.user,
fs=self.fs,
jt=self.jt,
workflow=workflow,
mapping=None)
return {
'id': job_id,
'has_result_set': True,
}
def run_morphline(self, collection_name, morphline, input_path):
workspace_path = self._upload_workspace(morphline)
snippet_properties = {
u'files': [{
u'path': u'%s/log4j.properties' % workspace_path,
u'type': u'file'
}, {
u'path': u'%s/morphline.conf' % workspace_path,
u'type': u'file'
}],
u'class':
u'org.apache.solr.hadoop.MapReduceIndexerTool',
u'app_jar':
CONFIG_INDEXER_LIBS_PATH.get(),
u'arguments': [
u'--morphline-file',
u'morphline.conf',
u'--output-dir',
u'${nameNode}/user/%s/indexer' % self.username,
u'--log4j',
u'log4j.properties',
u'--go-live',
u'--zk-host',
zkensemble(),
u'--collection',
collection_name,
u'${nameNode}%s' % input_path,
],
u'archives': [],
}
notebook = make_notebook(
name='Indexer',
editor_type='java',
snippet_properties=snippet_properties).get_data()
notebook_doc, created = _save_notebook(notebook, self.user)
workflow_doc = WorkflowBuilder().create_workflow(
document=notebook_doc,
user=self.user,
managed=True,
name=_("Batch job for %s") % notebook_doc.name)
workflow = Workflow(document=workflow_doc, user=self.user)
job_id = _submit_workflow(user=self.user,
fs=self.fs,
jt=self.jt,
workflow=workflow,
mapping=None)
return job_id
def test_job_validate_xml_name(self):
job = Workflow()
job.update_name('a')
assert_equal('a', job.validated_name)
job.update_name('aa')
assert_equal('aa', job.validated_name)
job.update_name('%a')
assert_equal('_a', job.validated_name)
job.update_name('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaz')
assert_equal(len('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'), len(job.validated_name))
job.update_name('My <...> 1st W$rkflow [With] (Bad) letter$')
assert_equal('My_______1st_W$rkflow__With___Bad__lette', job.validated_name)
def test_upgrade_nodes_in_workflow(self):
wf = Workflow(
data=
"{\"layout\": [{\"oozieRows\": [{\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Sqoop 1\", \"widgetType\": \"sqoop-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"79774a62-94e3-2ddb-554f-b83640fa5b03\", \"size\": 12}], \"id\": \"0f54ae72-7122-ad7c-fb31-aa715e15a707\", \"columns\": []}], \"rows\": [{\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Start\", \"widgetType\": \"start-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"size\": 12}], \"id\": \"371cf19e-0c45-1e40-2887-5de4033c2a01\", \"columns\": []}, {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Sqoop 1\", \"widgetType\": \"sqoop-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"79774a62-94e3-2ddb-554f-b83640fa5b03\", \"size\": 12}], \"id\": \"0f54ae72-7122-ad7c-fb31-aa715e15a707\", \"columns\": []}, {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"End\", \"widgetType\": \"end-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"size\": 12}], \"id\": \"40cfacb5-0622-4305-1473-8f70e287668b\", \"columns\": []}, {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Kill\", \"widgetType\": \"kill-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"size\": 12}], \"id\": \"373c9cc8-c64a-f1ef-5486-f18ec52620e3\", \"columns\": []}], \"oozieEndRow\": {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"End\", \"widgetType\": \"end-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"size\": 12}], \"id\": \"40cfacb5-0622-4305-1473-8f70e287668b\", \"columns\": []}, \"oozieKillRow\": {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Kill\", \"widgetType\": \"kill-widget\", \"oozieMovable\": true, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"size\": 12}], \"id\": \"373c9cc8-c64a-f1ef-5486-f18ec52620e3\", \"columns\": []}, \"enableOozieDropOnAfter\": true, \"oozieStartRow\": {\"enableOozieDropOnBefore\": true, \"enableOozieDropOnSide\": true, \"enableOozieDrop\": false, \"widgets\": [{\"status\": \"\", \"logsURL\": \"\", \"name\": \"Start\", \"widgetType\": \"start-widget\", \"oozieMovable\": false, \"ooziePropertiesExpanded\": false, \"properties\": {}, \"isLoading\": true, \"offset\": 0, \"actionURL\": \"\", \"progress\": 0, \"klass\": \"card card-widget span12\", \"oozieExpanded\": false, \"id\": \"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"size\": 12}], \"id\": \"371cf19e-0c45-1e40-2887-5de4033c2a01\", \"columns\": []}, \"klass\": \"card card-home card-column span12\", \"enableOozieDropOnBefore\": true, \"drops\": [\"temp\"], \"id\": \"a8549012-ec27-4686-d71a-c6ff95785ff9\", \"size\": 12}], \"workflow\": {\"properties\": {\"job_xml\": \"\", \"description\": \"\", \"wf1_id\": null, \"sla_enabled\": false, \"deployment_dir\": \"/user/hue/oozie/workspaces/hue-oozie-1438808722.99\", \"schema_version\": \"uri:oozie:workflow:0.5\", \"properties\": [], \"show_arrows\": true, \"parameters\": [{\"name\": \"oozie.use.system.libpath\", \"value\": true}], \"sla\": [{\"value\": false, \"key\": \"enabled\"}, {\"value\": \"${nominal_time}\", \"key\": \"nominal-time\"}, {\"value\": \"\", \"key\": \"should-start\"}, {\"value\": \"${30 * MINUTES}\", \"key\": \"should-end\"}, {\"value\": \"\", \"key\": \"max-duration\"}, {\"value\": \"\", \"key\": \"alert-events\"}, {\"value\": \"\", \"key\": \"alert-contact\"}, {\"value\": \"\", \"key\": \"notification-msg\"}, {\"value\": \"\", \"key\": \"upstream-apps\"}]}, \"name\": \"My Workflow\", \"versions\": [\"uri:oozie:workflow:0.4\", \"uri:oozie:workflow:0.4.5\", \"uri:oozie:workflow:0.5\"], \"isDirty\": true, \"movedNode\": null, \"linkMapping\": {\"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\": [], \"3f107997-04cc-8733-60a9-a4bb62cebffc\": [\"79774a62-94e3-2ddb-554f-b83640fa5b03\"], \"79774a62-94e3-2ddb-554f-b83640fa5b03\": [\"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\"], \"17c9c895-5a16-7443-bb81-f34b30b21548\": []}, \"nodeIds\": [\"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"79774a62-94e3-2ddb-554f-b83640fa5b03\"], \"nodes\": [{\"properties\": {}, \"name\": \"Start\", \"children\": [{\"to\": \"79774a62-94e3-2ddb-554f-b83640fa5b03\"}], \"actionParametersFetched\": false, \"type\": \"start-widget\", \"id\": \"3f107997-04cc-8733-60a9-a4bb62cebffc\", \"actionParameters\": []}, {\"properties\": {}, \"name\": \"End\", \"children\": [], \"actionParametersFetched\": false, \"type\": \"end-widget\", \"id\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\", \"actionParameters\": []}, {\"properties\": {\"message\": \"Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]\"}, \"name\": \"Kill\", \"children\": [], \"actionParametersFetched\": false, \"type\": \"kill-widget\", \"id\": \"17c9c895-5a16-7443-bb81-f34b30b21548\", \"actionParameters\": []}, {\"name\": \"sqoop-7977\", \"actionParametersUI\": [], \"children\": [{\"to\": \"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\"}, {\"error\": \"17c9c895-5a16-7443-bb81-f34b30b21548\"}], \"properties\": {\"files\": [], \"job_xml\": \"\", \"parameters\": [], \"job_properties\": [], \"command\": \"import --connect jdbc:hsqldb:file:db.hsqldb --table TT --target-dir hdfs://localhost:8020/user/foo -m 1\", \"archives\": [], \"prepares\": [], \"credentials\": [], \"sla\": [{\"value\": false, \"key\": \"enabled\"}, {\"value\": \"${nominal_time}\", \"key\": \"nominal-time\"}, {\"value\": \"\", \"key\": \"should-start\"}, {\"value\": \"${30 * MINUTES}\", \"key\": \"should-end\"}, {\"value\": \"\", \"key\": \"max-duration\"}, {\"value\": \"\", \"key\": \"alert-events\"}, {\"value\": \"\", \"key\": \"alert-contact\"}, {\"value\": \"\", \"key\": \"notification-msg\"}, {\"value\": \"\", \"key\": \"upstream-apps\"}]}, \"actionParametersFetched\": true, \"type\": \"sqoop-widget\", \"id\": \"79774a62-94e3-2ddb-554f-b83640fa5b03\", \"actionParameters\": []}], \"id\": null, \"nodeNamesMapping\": {\"33430f0f-ebfa-c3ec-f237-3e77efa03d0a\": \"End\", \"3f107997-04cc-8733-60a9-a4bb62cebffc\": \"Start\", \"79774a62-94e3-2ddb-554f-b83640fa5b03\": \"sqoop-7977\", \"17c9c895-5a16-7443-bb81-f34b30b21548\": \"Kill\"}, \"uuid\": \"b5511e29-c9cc-7f40-0d3a-6dd768f3b1e9\"}}"
)
assert_true(
'parameters'
in json.loads(wf.data)['workflow']['nodes'][3]['properties'],
wf.data)
assert_false('arguments' in json.loads(
wf.data)['workflow']['nodes'][3]['properties'],
wf.data) # Does not exist yet
data = wf.get_data()
assert_true('parameters' in data['workflow']['nodes'][3]['properties'],
wf.data)
assert_true('arguments' in data['workflow']['nodes'][3]['properties'],
wf.data) # New field transparently added
class Submission(object):
"""
Represents one unique Oozie submission.
Actions are:
- submit
- rerun
"""
def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None, local_tz=None):
self.job = job
self.user = user
self.fs = fs
self.jt = jt # Deprecated with YARN, we now use logical names only for RM
self.oozie_id = oozie_id
self.api = get_oozie(self.user)
if properties is not None:
self.properties = properties
else:
self.properties = {}
if local_tz and isinstance(self.job.data, dict):
local_tz = self.job.data.get('properties')['timezone']
# Modify start_date & end_date only when it's a coordinator
from oozie.models2 import Coordinator
if type(self.job) is Coordinator:
if 'start_date' in self.properties:
properties['start_date'] = convert_to_server_timezone(self.properties['start_date'], local_tz)
if 'end_date' in self.properties:
properties['end_date'] = convert_to_server_timezone(self.properties['end_date'], local_tz)
if 'nominal_time' in self.properties:
properties['nominal_time'] = convert_to_server_timezone(self.properties['nominal_time'], local_tz)
self.properties['security_enabled'] = self.api.security_enabled
def __str__(self):
if self.oozie_id:
res = "Submission for job '%s'." % (self.oozie_id,)
else:
res = "Submission for job '%s' (id %s, owner %s)." % (self.job.name, self.job.id, self.user)
if self.oozie_id:
res += " -- " + self.oozie_id
return res
@submit_dryrun
def run(self, deployment_dir=None):
"""
Take care of all the actions of submitting a Oozie workflow.
Returns the oozie job id if all goes well.
"""
if self.properties and 'oozie.use.system.libpath' not in self.properties:
self.properties['oozie.use.system.libpath'] = 'true'
self.oozie_id = self.api.submit_job(self.properties)
LOG.info("Submitted: %s" % (self,))
if self._is_workflow():
self.api.job_control(self.oozie_id, 'start')
LOG.info("Started: %s" % (self,))
return self.oozie_id
def rerun(self, deployment_dir, fail_nodes=None, skip_nodes=None):
jt_address = cluster.get_cluster_addr_for_job_submission()
self._update_properties(jt_address, deployment_dir)
self.properties.update({'oozie.wf.application.path': deployment_dir})
if 'oozie.coord.application.path' in self.properties:
self.properties.pop('oozie.coord.application.path')
if 'oozie.bundle.application.path' in self.properties:
self.properties.pop('oozie.bundle.application.path')
if fail_nodes:
self.properties.update({'oozie.wf.rerun.failnodes': fail_nodes})
elif not skip_nodes:
self.properties.update({'oozie.wf.rerun.failnodes': 'false'}) # Case empty 'skip_nodes' list
else:
self.properties.update({'oozie.wf.rerun.skip.nodes': skip_nodes})
self.api.rerun(self.oozie_id, properties=self.properties)
LOG.info("Rerun: %s" % (self,))
return self.oozie_id
def rerun_coord(self, deployment_dir, params):
jt_address = cluster.get_cluster_addr_for_job_submission()
self._update_properties(jt_address, deployment_dir)
self.properties.update({'oozie.coord.application.path': deployment_dir})
self.api.job_control(self.oozie_id, action='coord-rerun', properties=self.properties, parameters=params)
LOG.info("Rerun: %s" % (self,))
return self.oozie_id
def update_coord(self):
self.api = get_oozie(self.user, api_version="v2")
self.api.job_control(self.oozie_id, action='update', properties=self.properties, parameters=None)
LOG.info("Update: %s" % (self,))
return self.oozie_id
def rerun_bundle(self, deployment_dir, params):
jt_address = cluster.get_cluster_addr_for_job_submission()
self._update_properties(jt_address, deployment_dir)
self.properties.update({'oozie.bundle.application.path': deployment_dir})
self.api.job_control(self.oozie_id, action='bundle-rerun', properties=self.properties, parameters=params)
LOG.info("Rerun: %s" % (self,))
return self.oozie_id
def deploy(self, deployment_dir=None):
try:
if not deployment_dir:
deployment_dir = self._create_deployment_dir()
except Exception, ex:
msg = _("Failed to create deployment directory: %s" % ex)
LOG.exception(msg)
raise PopupException(message=msg, detail=str(ex))
if self.api.security_enabled:
jt_address = cluster.get_cluster_addr_for_job_submission()
self._update_properties(jt_address) # Needed for coordinator deploying workflows with credentials
if hasattr(self.job, 'nodes'):
for action in self.job.nodes:
# Make sure XML is there
# Don't support more than one level sub-workflow
if action.data['type'] == 'subworkflow':
from oozie.models2 import Workflow
workflow = Workflow(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['workflow']))
sub_deploy = Submission(self.user, workflow, self.fs, self.jt, self.properties)
workspace = sub_deploy.deploy()
self.job.override_subworkflow_id(action, workflow.id) # For displaying the correct graph
self.properties['workspace_%s' % workflow.uuid] = workspace # For pointing to the correct workspace
elif action.data['type'] == 'impala' or action.data['type'] == 'impala-document':
from oozie.models2 import _get_impala_url
from impala.impala_flags import get_ssl_server_certificate
if action.data['type'] == 'impala-document':
from notebook.models import Notebook
if action.data['properties'].get('uuid'):
notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['uuid']))
statements = notebook.get_str()
statements = Template(statements).safe_substitute(**self.properties)
script_name = action.data['name'] + '.sql'
self._create_file(deployment_dir, script_name, statements)
else:
script_name = os.path.basename(action.data['properties'].get('script_path'))
if self.api.security_enabled:
kinit = 'kinit -k -t *.keytab %(user_principal)s' % {
'user_principal': self.properties.get('user_principal', action.data['properties'].get('user_principal'))
}
else:
kinit = ''
shell_script = """#!/bin/bash
# Needed to launch impala shell in oozie
export PYTHON_EGG_CACHE=./myeggs
%(kinit)s
impala-shell %(kerberos_option)s %(ssl_option)s -i %(impalad_host)s -f %(query_file)s""" % {
'impalad_host': action.data['properties'].get('impalad_host') or _get_impala_url(),
'kerberos_option': '-k' if self.api.security_enabled else '',
'ssl_option': '--ssl' if get_ssl_server_certificate() else '',
'query_file': script_name,
'kinit': kinit
}
self._create_file(deployment_dir, action.data['name'] + '.sh', shell_script)
elif action.data['type'] == 'hive-document':
from notebook.models import Notebook
if action.data['properties'].get('uuid'):
notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['uuid']))
statements = notebook.get_str()
else:
statements = action.data['properties'].get('statements')
if self.properties.get('send_result_path'):
statements = """
INSERT OVERWRITE DIRECTORY '%s'
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
WITH SERDEPROPERTIES (
"separatorChar" = "\t",
"quoteChar" = "'",
"escapeChar" = "\\"
)
STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.join([snippet['statement_raw'] for snippet in notebook.get_data()['snippets']]))
if statements is not None:
self._create_file(deployment_dir, action.data['name'] + '.sql', statements)
elif action.data['type'] in ('java-document', 'java', 'mapreduce-document'):
if action.data['type'] == 'java-document' or action.data['type'] == 'mapreduce-document':
from notebook.models import Notebook
notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['uuid']))
properties = notebook.get_data()['snippets'][0]['properties']
else:
properties = action.data['properties']
if properties.get('app_jar'):
LOG.debug("Adding to oozie.libpath %s" % properties['app_jar'])
paths = [properties['app_jar']]
if self.properties.get('oozie.libpath'):
paths.append(self.properties['oozie.libpath'])
self.properties['oozie.libpath'] = ','.join(paths)
elif action.data['type'] == 'pig-document':
from notebook.models import Notebook
notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['uuid']))
statements = notebook.get_data()['snippets'][0]['statement_raw']
self._create_file(deployment_dir, action.data['name'] + '.pig', statements)
oozie_xml = self.job.to_xml(self.properties)
self._do_as(self.user.username, self._copy_files, deployment_dir, oozie_xml, self.properties)
return deployment_dir
class Submission(object):
"""
Represents one unique Oozie submission.
Actions are:
- submit
- rerun
"""
def __init__(self,
user,
job=None,
fs=None,
jt=None,
properties=None,
oozie_id=None,
local_tz=None):
self.job = job
self.user = user
self.fs = fs
self.jt = jt # Deprecated with YARN, we now use logical names only for RM
self.oozie_id = oozie_id
self.api = get_oozie(self.user)
if properties is not None:
self.properties = properties
else:
self.properties = {}
if local_tz and isinstance(self.job.data, dict):
local_tz = self.job.data.get('properties')['timezone']
# Modify start_date & end_date only when it's a coordinator
from oozie.models2 import Coordinator
if type(self.job) is Coordinator:
if 'start_date' in self.properties:
properties['start_date'] = convert_to_server_timezone(
self.properties['start_date'], local_tz)
if 'end_date' in self.properties:
properties['end_date'] = convert_to_server_timezone(
self.properties['end_date'], local_tz)
self.properties['security_enabled'] = self.api.security_enabled
def __str__(self):
if self.oozie_id:
res = "Submission for job '%s'." % (self.oozie_id, )
else:
res = "Submission for job '%s' (id %s, owner %s)." % (
self.job.name, self.job.id, self.user)
if self.oozie_id:
res += " -- " + self.oozie_id
return res
@submit_dryrun
def run(self, deployment_dir=None):
"""
Take care of all the actions of submitting a Oozie workflow.
Returns the oozie job id if all goes well.
"""
if self.properties and 'oozie.use.system.libpath' not in self.properties:
self.properties['oozie.use.system.libpath'] = 'true'
self.oozie_id = self.api.submit_job(self.properties)
LOG.info("Submitted: %s" % (self, ))
if self._is_workflow():
self.api.job_control(self.oozie_id, 'start')
LOG.info("Started: %s" % (self, ))
return self.oozie_id
def rerun(self, deployment_dir, fail_nodes=None, skip_nodes=None):
jt_address = cluster.get_cluster_addr_for_job_submission()
self._update_properties(jt_address, deployment_dir)
self.properties.update({'oozie.wf.application.path': deployment_dir})
if 'oozie.coord.application.path' in self.properties:
self.properties.pop('oozie.coord.application.path')
if fail_nodes:
self.properties.update({'oozie.wf.rerun.failnodes': fail_nodes})
elif not skip_nodes:
self.properties.update({'oozie.wf.rerun.failnodes':
'false'}) # Case empty 'skip_nodes' list
else:
self.properties.update({'oozie.wf.rerun.skip.nodes': skip_nodes})
self.api.rerun(self.oozie_id, properties=self.properties)
LOG.info("Rerun: %s" % (self, ))
return self.oozie_id
def rerun_coord(self, deployment_dir, params):
jt_address = cluster.get_cluster_addr_for_job_submission()
self._update_properties(jt_address, deployment_dir)
self.properties.update(
{'oozie.coord.application.path': deployment_dir})
self.api.job_control(self.oozie_id,
action='coord-rerun',
properties=self.properties,
parameters=params)
LOG.info("Rerun: %s" % (self, ))
return self.oozie_id
def update_coord(self):
self.api = get_oozie(self.user, api_version="v2")
self.api.job_control(self.oozie_id,
action='update',
properties=self.properties,
parameters=None)
LOG.info("Update: %s" % (self, ))
return self.oozie_id
def rerun_bundle(self, deployment_dir, params):
jt_address = cluster.get_cluster_addr_for_job_submission()
self._update_properties(jt_address, deployment_dir)
self.properties.update(
{'oozie.bundle.application.path': deployment_dir})
self.api.job_control(self.oozie_id,
action='bundle-rerun',
properties=self.properties,
parameters=params)
LOG.info("Rerun: %s" % (self, ))
return self.oozie_id
def deploy(self):
try:
deployment_dir = self._create_deployment_dir()
except Exception, ex:
msg = _("Failed to create deployment directory: %s" % ex)
LOG.exception(msg)
raise PopupException(message=msg, detail=str(ex))
if self.api.security_enabled:
jt_address = cluster.get_cluster_addr_for_job_submission()
self._update_properties(
jt_address
) # Needed for coordinator deploying workflows with credentials
if hasattr(self.job, 'nodes'):
for action in self.job.nodes:
# Make sure XML is there
# Don't support more than one level sub-workflow
if action.data['type'] == 'subworkflow':
from oozie.models2 import Workflow
workflow = Workflow(document=Document2.objects.get_by_uuid(
user=self.user,
uuid=action.data['properties']['workflow']))
sub_deploy = Submission(self.user, workflow, self.fs,
self.jt, self.properties)
workspace = sub_deploy.deploy()
self.job.override_subworkflow_id(
action,
workflow.id) # For displaying the correct graph
self.properties[
'workspace_%s' % workflow.
uuid] = workspace # For pointing to the correct workspace
elif action.data['type'] == 'hive-document':
from notebook.models import Notebook
notebook = Notebook(document=Document2.objects.get_by_uuid(
user=self.user, uuid=action.data['properties']
['uuid']))
self._create_file(deployment_dir,
action.data['name'] + '.sql',
notebook.get_str())
#self.data['properties']['script_path'] = _generate_hive_script(self.data['uuid']) #'workspace_%s' % workflow.uui
oozie_xml = self.job.to_xml(self.properties)
self._do_as(self.user.username, self._copy_files, deployment_dir,
oozie_xml, self.properties)
return deployment_dir
