def _submit_bundle(request, bundle, properties):
try:
deployment_dirs = {}
for bundled in bundle.coordinators.all():
wf_dir = Submission(request.user, bundled.coordinator.workflow,
request.fs, request.jt, properties).deploy()
deployment_dirs['wf_%s_dir' % bundled.coordinator.workflow.
id] = request.fs.get_hdfs_path(wf_dir)
coord_dir = Submission(request.user, bundled.coordinator,
request.fs, request.jt,
properties).deploy()
deployment_dirs['coord_%s_dir' %
bundled.coordinator.id] = coord_dir
properties.update(deployment_dirs)
submission = Submission(request.user,
bundle,
request.fs,
request.jt,
properties=properties)
job_id = submission.run()
History.objects.create_from_submission(submission)
return job_id
except RestException, ex:
raise PopupException(_("Error submitting bundle %s") % (bundle, ),
detail=ex._headers.get('oozie-error-message', ex))
def test_update_properties():
finish = []
finish.append(MR_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
try:
properties = {
'user.name': 'hue',
'test.1': 'http://localhost/test?test1=test&test2=test'
}
final_properties = properties.copy()
submission = Submission(None, properties=properties, oozie_id='test')
assert_equal(properties, submission.properties)
submission._update_properties('jtaddress', 'deployment-directory')
assert_equal(final_properties, submission.properties)
cluster.clear_caches()
fs = cluster.get_hdfs()
jt = cluster.get_next_ha_mrcluster()[1]
final_properties = properties.copy()
final_properties.update({
'jobTracker': 'jtaddress',
'nameNode': fs.fs_defaultfs
})
submission = Submission(None,
properties=properties,
oozie_id='test',
fs=fs,
jt=jt)
assert_equal(properties, submission.properties)
submission._update_properties('jtaddress', 'deployment-directory')
assert_equal(final_properties, submission.properties)
finish.append(
HDFS_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('namenode'))
finish.append(
MR_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('jobtracker'))
cluster.clear_caches()
fs = cluster.get_hdfs()
jt = cluster.get_next_ha_mrcluster()[1]
final_properties = properties.copy()
final_properties.update({
'jobTracker': 'jobtracker',
'nameNode': 'namenode'
})
submission = Submission(None,
properties=properties,
oozie_id='test',
fs=fs,
jt=jt)
assert_equal(properties, submission.properties)
submission._update_properties('jtaddress', 'deployment-directory')
assert_equal(final_properties, submission.properties)
finally:
cluster.clear_caches()
for reset in finish:
reset()
def submit_external_job(request, application_path):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
application_name = os.path.basename(application_path)
application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else Workflow
mapping[application_class.get_application_path_key()] = application_path
try:
submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
job_id = submission.run(application_path)
except RestException, ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.error(smart_str(detail))
raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)
request.info(_('Oozie job submitted'))
view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
def _submit_coordinator(request, coordinator, mapping):
try:
wf_dir = Submission(request.user, coordinator.workflow, request.fs, request.jt, mapping).deploy()
properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
properties.update(mapping)
submission = Submission(request.user, coordinator, request.fs, request.jt, properties=properties)
job_id = submission.run()
History.objects.create_from_submission(submission)
return job_id
except RestException, ex:
raise PopupException(_("Error submitting coordinator %s") % (coordinator,),
detail=ex._headers.get('oozie-error-message', ex))
def test_get_external_parameters(self):
xml = """
<workflow-app name="Pig" xmlns="uri:oozie:workflow:0.4">
<start to="Pig"/>
<action name="Pig">
<pig>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<prepare>
<delete path="${output}"/>
</prepare>
<script>aggregate.pig</script>
<argument>-param</argument>
<argument>INPUT=${input}</argument>
<argument>-param</argument>
<argument>OUTPUT=${output}</argument>
<configuration>
<property>
<name>mapred.input.format.class</name>
<value>org.apache.hadoop.examples.SleepJob$SleepInputFormat</value>
</property>
</configuration>
</pig>
<ok to="end"/>
<error to="kill"/>
</action>
<kill name="kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
</kill>
<end name="end"/>
</workflow-app>
"""
properties = """
#
# Licensed to the Hue
#
nameNode=hdfs://localhost:8020
jobTracker=localhost:8021
queueName=default
examplesRoot=examples
oozie.use.system.libpath=true
oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pig
"""
parameters = Submission(self.user)._get_external_parameters(xml, properties)
assert_equal({'oozie.use.system.libpath': 'true',
'input': '',
'jobTracker': 'localhost:8021',
'oozie.wf.application.path': '${nameNode}/user/${user.name}/${examplesRoot}/apps/pig',
'examplesRoot': 'examples',
'output': '',
'nameNode': 'hdfs://localhost:8020',
'queueName': 'default'
},
parameters)
def _rerun_bundle(request, oozie_id, args, params, properties):
try:
submission = Submission(user=request.user, fs=request.fs, jt=request.jt, oozie_id=oozie_id, properties=properties)
job_id = submission.rerun_bundle(params=params, **args)
return job_id
except RestException, ex:
raise PopupException(_("Error re-running bundle %s.") % (oozie_id,),
detail=ex._headers.get('oozie-error-message', ex))
def _rerun_workflow(request, oozie_id, run_args, mapping):
try:
submission = Submission(user=request.user, fs=request.fs, jt=request.jt, properties=mapping, oozie_id=oozie_id)
job_id = submission.rerun(**run_args)
return job_id
except RestException, ex:
raise PopupException(_("Error re-running workflow %s.") % (oozie_id,),
detail=ex._headers.get('oozie-error-message', ex))
def _submit_workflow(request, workflow, mapping):
try:
submission = Submission(request.user, workflow, request.fs, mapping)
job_id = submission.run()
History.objects.create_from_submission(submission)
return job_id
except RestException, ex:
raise PopupException(_("Error submitting workflow %s") % (workflow, ),
detail=ex._headers.get('oozie-error-message', ex))
def delete_coordinator(request, coordinator):
if request.method != 'POST':
raise PopupException(_('A POST request is required.'))
coordinator.delete()
Submission(request.user, coordinator, request.fs,
{}).remove_deployment_dir()
request.info(_('Coordinator deleted.'))
return redirect(reverse('oozie:list_coordinators'))
def _rerun_coordinator(request, oozie_id, args, params, properties):
try:
submission = Submission(user=request.user, fs=request.fs, jt=request.jt, oozie_id=oozie_id, properties=properties)
job_id = submission.rerun_coord(params=params, **args)
return job_id
except RestException, ex:
msg = _("Error re-running coordinator %s.") % (oozie_id,)
LOG.exception(msg)
raise PopupException(msg, detail=ex._headers.get('oozie-error-message', ex))
def test_get_properties(self):
submission = Submission(self.user, fs=MockFs())
assert_equal({}, submission.properties)
submission._update_properties('curacao:8032', '/deployment_dir')
assert_equal({
'jobTracker': 'curacao:8032',
'nameNode': 'hdfs://curacao:8020'
}, submission.properties)
def test_get_logical_properties(self):
submission = Submission(self.user, fs=MockFs(logical_name='fsname'), jt=MockJt(logical_name='jtname'))
assert_equal({}, submission.properties)
submission._update_properties('curacao:8032', '/deployment_dir')
assert_equal({
'jobTracker': 'jtname',
'nameNode': 'fsname'
}, submission.properties)
def _submit_workflow(user, fs, workflow, mapping):
try:
submission = Submission(user, workflow, fs, mapping)
job_id = submission.run()
History.objects.create_from_submission(submission)
return job_id
except RestException, ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'urlopen error' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
raise PopupException(_("Error submitting workflow %s") % (workflow, ),
detail=detail)
def _submit_workflow(user, fs, jt, workflow, mapping):
try:
submission = Submission(user, workflow, fs, jt, mapping)
job_id = submission.run()
History.objects.create_from_submission(submission)
return job_id
except RestException as ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.error(smart_str(detail))
raise PopupException(_("Error submitting workflow %s") % (workflow,), detail=detail)
return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
def delete_bundle(request):
if request.method != 'POST':
raise PopupException(_('A POST request is required.'))
job_ids = request.POST.getlist('job_selection')
for job_id in job_ids:
job = Job.objects.is_accessible_or_exception(request, job_id)
Job.objects.can_edit_or_exception(request, job)
Submission(request.user, job, request.fs, {}).remove_deployment_dir()
job.delete()
request.info(_('Bundle(s) deleted.'))
return redirect(reverse('oozie:list_bundles'))
def delete_coordinator(request):
if request.method != 'POST':
raise PopupException(_('A POST request is required.'))
skip_trash = 'skip_trash' in request.GET
job_ids = request.POST.getlist('job_selection')
for job_id in job_ids:
job = Job.objects.can_read_or_exception(request, job_id)
Job.objects.can_edit_or_exception(request, job)
if skip_trash:
Submission(request.user, job, request.fs, {}).remove_deployment_dir()
job.delete(skip_trash=skip_trash)
if skip_trash:
request.info(_('Coordinator(s) deleted.'))
else:
request.info(_('Coordinator(s) trashed.'))
return redirect(reverse('oozie:list_coordinators'))
def test_copy_files():
cluster = pseudo_hdfs4.shared_cluster()
try:
c = make_logged_in_client()
user = User.objects.get(username='******')
ensure_home_directory(cluster.fs, user)
prefix = '/tmp/test_copy_files'
if cluster.fs.exists(prefix):
cluster.fs.rmtree(
prefix,
skip_trash=True) # admin user might not have a home directory
# Jars in various locations
deployment_dir = '%s/workspace' % prefix
external_deployment_dir = '%s/deployment' % prefix
jar_1 = '%s/udf1.jar' % prefix
jar_2 = '%s/lib/udf2.jar' % prefix
jar_3 = '%s/udf3.jar' % deployment_dir
jar_4 = '%s/lib/udf4.jar' % deployment_dir # Never move
cluster.fs.mkdir(prefix)
cluster.fs.create(jar_1)
cluster.fs.create(jar_2)
cluster.fs.create(jar_3)
cluster.fs.create(jar_4)
class MockNode():
def __init__(self, jar_path):
self.jar_path = jar_path
class MockJob():
def __init__(self):
self.node_list = [
MockNode(jar_1),
MockNode(jar_2),
MockNode(jar_3),
MockNode(jar_4),
]
def get_application_filename(self):
return 'workflow.xml'
submission = Submission(user,
job=MockJob(),
fs=cluster.fs,
jt=cluster.jt)
submission._copy_files(deployment_dir, "<xml>My XML</xml>")
submission._copy_files(external_deployment_dir, "<xml>My XML</xml>")
# All sources still there
assert_true(cluster.fs.exists(jar_1))
assert_true(cluster.fs.exists(jar_2))
assert_true(cluster.fs.exists(jar_3))
assert_true(cluster.fs.exists(jar_4))
deployment_dir = deployment_dir + '/lib'
external_deployment_dir = external_deployment_dir + '/lib'
list_dir_workspace = cluster.fs.listdir(deployment_dir)
list_dir_deployement = cluster.fs.listdir(external_deployment_dir)
# All destinations there
assert_true(cluster.fs.exists(deployment_dir + '/udf1.jar'),
list_dir_workspace)
assert_true(cluster.fs.exists(deployment_dir + '/udf2.jar'),
list_dir_workspace)
assert_true(cluster.fs.exists(deployment_dir + '/udf3.jar'),
list_dir_workspace)
assert_true(cluster.fs.exists(deployment_dir + '/udf4.jar'),
list_dir_workspace)
assert_true(cluster.fs.exists(external_deployment_dir + '/udf1.jar'),
list_dir_deployement)
assert_true(cluster.fs.exists(external_deployment_dir + '/udf2.jar'),
list_dir_deployement)
assert_true(cluster.fs.exists(external_deployment_dir + '/udf3.jar'),
list_dir_deployement)
assert_true(cluster.fs.exists(external_deployment_dir + '/udf4.jar'),
list_dir_deployement)
stats_udf1 = cluster.fs.stats(deployment_dir + '/udf1.jar')
stats_udf2 = cluster.fs.stats(deployment_dir + '/udf2.jar')
stats_udf3 = cluster.fs.stats(deployment_dir + '/udf3.jar')
stats_udf4 = cluster.fs.stats(deployment_dir + '/udf4.jar')
submission._copy_files('%s/workspace' % prefix, "<xml>My XML</xml>")
assert_not_equal(
stats_udf1['fileId'],
cluster.fs.stats(deployment_dir + '/udf1.jar')['fileId'])
assert_not_equal(
stats_udf2['fileId'],
cluster.fs.stats(deployment_dir + '/udf2.jar')['fileId'])
assert_not_equal(
stats_udf3['fileId'],
cluster.fs.stats(deployment_dir + '/udf3.jar')['fileId'])
assert_equal(stats_udf4['fileId'],
cluster.fs.stats(deployment_dir + '/udf4.jar')['fileId'])
finally:
try:
cluster.fs.rmtree(prefix, skip_trash=True)
except:
LOG.exception('failed to remove %s' % prefix)
detail)
LOG.error(smart_str(detail))
raise PopupException(_("Error submitting job %s") %
(application_path, ),
detail=detail)
request.info(_('Oozie job submitted'))
view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
return redirect(
reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' %
params_form.errors))
else:
parameters = Submission(
request.user, fs=request.fs,
jt=request.jt).get_external_parameters(application_path)
initial_params = ParameterForm.get_initial_params(parameters)
params_form = ParametersFormSet(initial=initial_params)
popup = render('editor/submit_job_popup.mako',
request, {
'params_form':
params_form,
'action':
reverse('oozie:submit_external_job',
kwargs={'application_path': application_path})
},
force_template=True).content
return HttpResponse(json.dumps(popup), mimetype="application/json")