def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None, local_tz=None): self.job = job self.user = user self.fs = fs self.jt = jt # Deprecated with YARN, we now use logical names only for RM self.oozie_id = oozie_id self.api = get_oozie(self.user) if properties is not None: self.properties = properties else: self.properties = {} if local_tz and isinstance(self.job.data, dict): local_tz = self.job.data.get('properties')['timezone'] # Modify start_date & end_date only when it's a coordinator from oozie.models2 import Coordinator if type(self.job) is Coordinator: if 'start_date' in self.properties: properties['start_date'] = convert_to_server_timezone(self.properties['start_date'], local_tz) if 'end_date' in self.properties: properties['end_date'] = convert_to_server_timezone(self.properties['end_date'], local_tz) if 'nominal_time' in self.properties: properties['nominal_time'] = convert_to_server_timezone(self.properties['nominal_time'], local_tz) self.properties['security_enabled'] = self.api.security_enabled
def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None, local_tz=None): self.job = job self.user = user self.fs = fs self.jt = jt # Deprecated with YARN, we now use logical names only for RM self.oozie_id = oozie_id self.api = get_oozie(self.user) if properties is not None: self.properties = properties else: self.properties = {} if local_tz and isinstance(self.job.data, dict): local_tz = self.job.data.get('properties')['timezone'] # Modify start_date & end_date only when it's a coordinator from oozie.models2 import Coordinator if type(self.job) is Coordinator: if 'start_date' in self.properties: properties['start_date'] = convert_to_server_timezone(self.properties['start_date'], local_tz) if 'end_date' in self.properties: properties['end_date'] = convert_to_server_timezone(self.properties['end_date'], local_tz) self.properties['security_enabled'] = self.api.security_enabled
def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None, local_tz=None): self.job = job self.user = user self.fs = fs self.jt = jt # Deprecated with YARN, we now use logical names only for RM self.oozie_id = oozie_id self.api = get_oozie(self.user) if properties is not None: self.properties = properties else: self.properties = {} if local_tz and isinstance(self.job.data, dict): local_tz = self.job.data.get('properties')['timezone'] if 'start_date' in self.properties: properties['start_date'] = convert_to_server_timezone( self.properties['start_date'], local_tz) if 'end_date' in self.properties: properties['end_date'] = convert_to_server_timezone( self.properties['end_date'], local_tz) self.properties['security_enabled'] = self.api.security_enabled
def manage_oozie_jobs(request, job_id, action): if request.method != 'POST': raise PopupException(_('Use a POST request to manage an Oozie job.')) job = check_job_access_permission(request, job_id) check_job_edition_permission(job, request.user) response = {'status': -1, 'data': ''} try: oozie_api = get_oozie(request.user) params = None if action == 'change': pause_time_val = request.POST.get('pause_time') if request.POST.get('clear_pause_time') == 'true': pause_time_val = '' end_time_val = request.POST.get('end_time') if end_time_val: end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get()) if pause_time_val: pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get()) params = {'value': 'endtime=%s' % (end_time_val) + ';' 'pausetime=%s' % (pause_time_val) + ';' 'concurrency=%s' % (request.POST.get('concurrency'))} elif action == 'ignore': oozie_api = get_oozie(request.user, api_version="v2") params = { 'type': 'action', 'scope': ','.join(job.aggreate(request.POST.get('actions').split())), } response['data'] = oozie_api.job_control(job_id, action, parameters=params) response['status'] = 0 if 'notification' in request.POST: request.info(_(request.POST.get('notification'))) except RestException as ex: ex_message = ex.message if ex._headers.get('oozie-error-message'): ex_message = ex._headers.get('oozie-error-message') msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message) LOG.exception(msg) response['data'] = _(msg) return JsonResponse(response)
def _submit_bundle(request, bundle, properties): try: deployment_mapping = {} coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])]) for i, bundled in enumerate(bundle.data['coordinators']): coord = coords[bundled['coordinator']] workflow = Workflow(document=coord.dependencies.filter(type='oozie-workflow2')[0]) wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy() deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir) coordinator = Coordinator(document=coord) coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy() deployment_mapping['coord_%s_dir' % i] = request.fs.get_hdfs_path(coord_dir) deployment_mapping['coord_%s' % i] = coord # Convert start/end dates of coordinator to server timezone for prop in bundled['properties']: if prop['name'] in ('end_date', 'start_date'): prop['value'] = convert_to_server_timezone(prop['value'], local_tz=coordinator.data['properties']['timezone']) properties.update(deployment_mapping) submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties) job_id = submission.run() return job_id except RestException, ex: LOG.exception('Error submitting bundle') raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex), error_code=200)
def _submit_bundle(request, bundle, properties): try: deployment_mapping = {} coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])]) for i, bundled in enumerate(bundle.data['coordinators']): coord = coords[bundled['coordinator']] workflow = Workflow(document=coord.dependencies.filter(type='oozie-workflow2')[0]) wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy() deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir) coordinator = Coordinator(document=coord) coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy() deployment_mapping['coord_%s_dir' % i] = request.fs.get_hdfs_path(coord_dir) deployment_mapping['coord_%s' % i] = coord # Convert start/end dates of coordinator to server timezone for prop in bundled['properties']: if prop['name'] in ('end_date', 'start_date'): prop['value'] = convert_to_server_timezone(prop['value'], local_tz=coordinator.data['properties']['timezone']) properties.update(deployment_mapping) submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties) job_id = submission.run() return job_id except RestException, ex: LOG.exception('Error submitting bundle') raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex))
def manage_oozie_jobs(request, job_id, action): if request.method != 'POST': raise PopupException(_('Use a POST request to manage an Oozie job.')) job = check_job_access_permission(request, job_id) check_job_edition_permission(job, request.user) response = {'status': -1, 'data': ''} try: oozie_api = get_oozie(request.user) params = None if action == 'change': pause_time_val = request.POST.get('pause_time') if request.POST.get('clear_pause_time') == 'true': pause_time_val = '' end_time_val = request.POST.get('end_time') if end_time_val: end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get()) if pause_time_val: pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get()) params = {'value': 'endtime=%s' % (end_time_val) + ';' 'pausetime=%s' % (pause_time_val) + ';' 'concurrency=%s' % (request.POST.get('concurrency'))} elif action == 'ignore': oozie_api = get_oozie(request.user, api_version="v2") params = { 'type': 'action', 'scope': ','.join(job.aggreate(request.POST.get('actions').split())), } response['data'] = oozie_api.job_control(job_id, action, parameters=params) response['status'] = 0 if 'notification' in request.POST: request.info(_(request.POST.get('notification'))) except RestException, ex: ex_message = ex.message if ex._headers.get('oozie-error-message'): ex_message = ex._headers.get('oozie-error-message') msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message) LOG.exception(msg) response['data'] = _(msg)
def manage_oozie_jobs(request, job_id, action): if request.method != "POST": raise PopupException(_("Use a POST request to manage an Oozie job.")) job = check_job_access_permission(request, job_id) check_job_edition_permission(job, request.user) response = {"status": -1, "data": ""} try: oozie_api = get_oozie(request.user) params = None if action == "change": pause_time_val = request.POST.get("pause_time") if request.POST.get("clear_pause_time") == "true": pause_time_val = "" end_time_val = request.POST.get("end_time") if end_time_val: end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get()) if pause_time_val: pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get()) params = { "value": "endtime=%s" % (end_time_val) + ";" "pausetime=%s" % (pause_time_val) + ";" "concurrency=%s" % (request.POST.get("concurrency")) } elif action == "ignore": oozie_api = get_oozie(request.user, api_version="v2") params = {"type": "action", "scope": ",".join(job.aggreate(request.POST.get("actions").split()))} response["data"] = oozie_api.job_control(job_id, action, parameters=params) response["status"] = 0 if "notification" in request.POST: request.info(_(request.POST.get("notification"))) except RestException, ex: ex_message = ex.message if ex._headers.get("oozie-error-message"): ex_message = ex._headers.get("oozie-error-message") msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message) LOG.exception(msg) response["data"] = _(msg)