def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None, local_tz=None):
self.job = job
self.user = user
self.fs = fs
self.jt = jt # Deprecated with YARN, we now use logical names only for RM
self.oozie_id = oozie_id
self.api = get_oozie(self.user)
if properties is not None:
self.properties = properties
else:
self.properties = {}
if local_tz and isinstance(self.job.data, dict):
local_tz = self.job.data.get('properties')['timezone']
# Modify start_date & end_date only when it's a coordinator
from oozie.models2 import Coordinator
if type(self.job) is Coordinator:
if 'start_date' in self.properties:
properties['start_date'] = convert_to_server_timezone(self.properties['start_date'], local_tz)
if 'end_date' in self.properties:
properties['end_date'] = convert_to_server_timezone(self.properties['end_date'], local_tz)
if 'nominal_time' in self.properties:
properties['nominal_time'] = convert_to_server_timezone(self.properties['nominal_time'], local_tz)
self.properties['security_enabled'] = self.api.security_enabled
def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None, local_tz=None):
self.job = job
self.user = user
self.fs = fs
self.jt = jt # Deprecated with YARN, we now use logical names only for RM
self.oozie_id = oozie_id
self.api = get_oozie(self.user)
if properties is not None:
self.properties = properties
else:
self.properties = {}
if local_tz and isinstance(self.job.data, dict):
local_tz = self.job.data.get('properties')['timezone']
# Modify start_date & end_date only when it's a coordinator
from oozie.models2 import Coordinator
if type(self.job) is Coordinator:
if 'start_date' in self.properties:
properties['start_date'] = convert_to_server_timezone(self.properties['start_date'], local_tz)
if 'end_date' in self.properties:
properties['end_date'] = convert_to_server_timezone(self.properties['end_date'], local_tz)
self.properties['security_enabled'] = self.api.security_enabled
def __init__(self,
user,
job=None,
fs=None,
jt=None,
properties=None,
oozie_id=None,
local_tz=None):
self.job = job
self.user = user
self.fs = fs
self.jt = jt # Deprecated with YARN, we now use logical names only for RM
self.oozie_id = oozie_id
self.api = get_oozie(self.user)
if properties is not None:
self.properties = properties
else:
self.properties = {}
if local_tz and isinstance(self.job.data, dict):
local_tz = self.job.data.get('properties')['timezone']
if 'start_date' in self.properties:
properties['start_date'] = convert_to_server_timezone(
self.properties['start_date'], local_tz)
if 'end_date' in self.properties:
properties['end_date'] = convert_to_server_timezone(
self.properties['end_date'], local_tz)
self.properties['security_enabled'] = self.api.security_enabled
def manage_oozie_jobs(request, job_id, action):
if request.method != 'POST':
raise PopupException(_('Use a POST request to manage an Oozie job.'))
job = check_job_access_permission(request, job_id)
check_job_edition_permission(job, request.user)
response = {'status': -1, 'data': ''}
try:
oozie_api = get_oozie(request.user)
params = None
if action == 'change':
pause_time_val = request.POST.get('pause_time')
if request.POST.get('clear_pause_time') == 'true':
pause_time_val = ''
end_time_val = request.POST.get('end_time')
if end_time_val:
end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get())
if pause_time_val:
pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get())
params = {'value': 'endtime=%s' % (end_time_val) + ';'
'pausetime=%s' % (pause_time_val) + ';'
'concurrency=%s' % (request.POST.get('concurrency'))}
elif action == 'ignore':
oozie_api = get_oozie(request.user, api_version="v2")
params = {
'type': 'action',
'scope': ','.join(job.aggreate(request.POST.get('actions').split())),
}
response['data'] = oozie_api.job_control(job_id, action, parameters=params)
response['status'] = 0
if 'notification' in request.POST:
request.info(_(request.POST.get('notification')))
except RestException as ex:
ex_message = ex.message
if ex._headers.get('oozie-error-message'):
ex_message = ex._headers.get('oozie-error-message')
msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message)
LOG.exception(msg)
response['data'] = _(msg)
return JsonResponse(response)
def _submit_bundle(request, bundle, properties):
try:
deployment_mapping = {}
coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])
for i, bundled in enumerate(bundle.data['coordinators']):
coord = coords[bundled['coordinator']]
workflow = Workflow(document=coord.dependencies.filter(type='oozie-workflow2')[0])
wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy()
deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir)
coordinator = Coordinator(document=coord)
coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy()
deployment_mapping['coord_%s_dir' % i] = request.fs.get_hdfs_path(coord_dir)
deployment_mapping['coord_%s' % i] = coord
# Convert start/end dates of coordinator to server timezone
for prop in bundled['properties']:
if prop['name'] in ('end_date', 'start_date'):
prop['value'] = convert_to_server_timezone(prop['value'], local_tz=coordinator.data['properties']['timezone'])
properties.update(deployment_mapping)
submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting bundle')
raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex), error_code=200)
def _submit_bundle(request, bundle, properties):
try:
deployment_mapping = {}
coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])
for i, bundled in enumerate(bundle.data['coordinators']):
coord = coords[bundled['coordinator']]
workflow = Workflow(document=coord.dependencies.filter(type='oozie-workflow2')[0])
wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy()
deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir)
coordinator = Coordinator(document=coord)
coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy()
deployment_mapping['coord_%s_dir' % i] = request.fs.get_hdfs_path(coord_dir)
deployment_mapping['coord_%s' % i] = coord
# Convert start/end dates of coordinator to server timezone
for prop in bundled['properties']:
if prop['name'] in ('end_date', 'start_date'):
prop['value'] = convert_to_server_timezone(prop['value'], local_tz=coordinator.data['properties']['timezone'])
properties.update(deployment_mapping)
submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting bundle')
raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex))
def manage_oozie_jobs(request, job_id, action):
if request.method != 'POST':
raise PopupException(_('Use a POST request to manage an Oozie job.'))
job = check_job_access_permission(request, job_id)
check_job_edition_permission(job, request.user)
response = {'status': -1, 'data': ''}
try:
oozie_api = get_oozie(request.user)
params = None
if action == 'change':
pause_time_val = request.POST.get('pause_time')
if request.POST.get('clear_pause_time') == 'true':
pause_time_val = ''
end_time_val = request.POST.get('end_time')
if end_time_val:
end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get())
if pause_time_val:
pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get())
params = {'value': 'endtime=%s' % (end_time_val) + ';'
'pausetime=%s' % (pause_time_val) + ';'
'concurrency=%s' % (request.POST.get('concurrency'))}
elif action == 'ignore':
oozie_api = get_oozie(request.user, api_version="v2")
params = {
'type': 'action',
'scope': ','.join(job.aggreate(request.POST.get('actions').split())),
}
response['data'] = oozie_api.job_control(job_id, action, parameters=params)
response['status'] = 0
if 'notification' in request.POST:
request.info(_(request.POST.get('notification')))
except RestException, ex:
ex_message = ex.message
if ex._headers.get('oozie-error-message'):
ex_message = ex._headers.get('oozie-error-message')
msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message)
LOG.exception(msg)
response['data'] = _(msg)
def manage_oozie_jobs(request, job_id, action):
if request.method != "POST":
raise PopupException(_("Use a POST request to manage an Oozie job."))
job = check_job_access_permission(request, job_id)
check_job_edition_permission(job, request.user)
response = {"status": -1, "data": ""}
try:
oozie_api = get_oozie(request.user)
params = None
if action == "change":
pause_time_val = request.POST.get("pause_time")
if request.POST.get("clear_pause_time") == "true":
pause_time_val = ""
end_time_val = request.POST.get("end_time")
if end_time_val:
end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get())
if pause_time_val:
pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get())
params = {
"value": "endtime=%s" % (end_time_val) + ";"
"pausetime=%s" % (pause_time_val) + ";"
"concurrency=%s" % (request.POST.get("concurrency"))
}
elif action == "ignore":
oozie_api = get_oozie(request.user, api_version="v2")
params = {"type": "action", "scope": ",".join(job.aggreate(request.POST.get("actions").split()))}
response["data"] = oozie_api.job_control(job_id, action, parameters=params)
response["status"] = 0
if "notification" in request.POST:
request.info(_(request.POST.get("notification")))
except RestException, ex:
ex_message = ex.message
if ex._headers.get("oozie-error-message"):
ex_message = ex._headers.get("oozie-error-message")
msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message)
LOG.exception(msg)
response["data"] = _(msg)
