def run_job(job_execution):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, job_execution.cluster_id)
if cluster.status != 'Active':
return job_execution
job = conductor.job_get(ctx, job_execution.job_id)
if not edp.compare_job_type(job.type, 'Java'):
input_source = conductor.data_source_get(ctx, job_execution.input_id)
output_source = conductor.data_source_get(ctx, job_execution.output_id)
else:
input_source = None
output_source = None
#TODO(nprivalova): should be removed after all features implemented
validate(input_source, output_source, job)
for data_source in [input_source, output_source]:
if data_source and data_source.type == 'hdfs':
h.configure_cluster_for_hdfs(cluster, data_source)
hdfs_user = _get_hdfs_user(cluster)
oozie_server = _get_oozie_server(cluster)
wf_dir = create_workflow_dir(oozie_server, job, hdfs_user)
upload_job_files(oozie_server, wf_dir, job, hdfs_user)
creator = workflow_factory.get_creator(job)
wf_xml = creator.get_workflow_xml(cluster, job_execution,
input_source, output_source)
path_to_workflow = upload_workflow_file(oozie_server,
wf_dir, wf_xml, hdfs_user)
rm_path = _get_resource_manager_path(cluster)
nn_path = cluster['info']['HDFS']['NameNode']
client = o.OozieClient(cluster['info']['JobFlow']['Oozie'] + "/oozie/",
_get_oozie_server(cluster))
job_parameters = {"jobTracker": rm_path,
"nameNode": nn_path,
"user.name": hdfs_user,
"oozie.wf.application.path":
"%s%s" % (nn_path, path_to_workflow),
"oozie.use.system.libpath": "true"}
oozie_job_id = client.add_job(x.create_hadoop_xml(job_parameters),
job_execution)
job_execution = conductor.job_execution_update(ctx, job_execution,
{'oozie_job_id':
oozie_job_id,
'start_time':
datetime.datetime.now()})
client.run_job(job_execution, oozie_job_id)
return job_execution
def cancel_job(job_execution_id):
ctx = context.ctx()
job_execution = conductor.job_execution_get(ctx, job_execution_id)
cluster = conductor.cluster_get(ctx, job_execution.cluster_id)
client = o.OozieClient(cluster['info']['JobFlow']['Oozie'] + "/oozie/",
_get_oozie_server(cluster))
client.kill_job(job_execution)
job_info = client.get_job_status(job_execution)
update = {"info": job_info,
"end_time": datetime.datetime.now()}
job_execution = conductor.job_execution_update(ctx, job_execution,
update)
return job_execution
def get_job_status(job_execution_id):
ctx = context.ctx()
job_execution = conductor.job_execution_get(ctx, job_execution_id)
if job_execution.oozie_job_id is None:
# We don't have an Oozie id yet for this job, that's okay
return job_execution
cluster = conductor.cluster_get(ctx, job_execution.cluster_id)
if cluster is None or cluster.status != 'Active':
return job_execution
client = o.OozieClient(cluster['info']['JobFlow']['Oozie'] + "/oozie",
_get_oozie_server(cluster))
job_info = client.get_job_status(job_execution)
update = {"info": job_info}
if job_info['status'] in terminated_job_states:
update['end_time'] = datetime.datetime.now()
job_execution = conductor.job_execution_update(ctx, job_execution,
update)
return job_execution
def _create_oozie_client(cluster):
plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name)
return o.OozieClient(plugin.get_oozie_server_uri(cluster),
plugin.get_oozie_server(cluster))