def list_oozie_info(request): api = get_oozie(request.user) configuration = api.get_configuration() oozie_status = api.get_oozie_status() instrumentation = {} metrics = {} if "org.apache.oozie.service.MetricsInstrumentationService" in [ c.strip() for c in configuration.get("oozie.services.ext", "").split(",") ]: api2 = get_oozie(request.user, api_version="v2") metrics = api2.get_metrics() else: instrumentation = api.get_instrumentation() return render( "dashboard/list_oozie_info.mako", request, { "instrumentation": instrumentation, "metrics": metrics, "configuration": configuration, "oozie_status": oozie_status, }, )
def manage_oozie_jobs(request, job_id, action): if request.method != 'POST': raise PopupException(_('Use a POST request to manage an Oozie job.')) job = check_job_access_permission(request, job_id) check_job_edition_permission(job, request.user) response = {'status': -1, 'data': ''} try: oozie_api = get_oozie(request.user) params = None if action == 'change': params = {'value': 'endtime=%s' % (request.POST.get('end_time'))} elif action == 'ignore': oozie_api = get_oozie(request.user, api_version="v2") params = { 'type': 'action', 'scope': ','.join(job.aggreate(request.POST.get('actions').split())), } response['data'] = oozie_api.job_control(job_id, action, parameters=params) response['status'] = 0 if 'notification' in request.POST: request.info(_(request.POST.get('notification'))) except RestException, ex: ex_message = ex.message if ex._headers.get('oozie-error-message'): ex_message = ex._headers.get('oozie-error-message') msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message) LOG.exception(msg) response['data'] = _(msg)
def massaged_oozie_jobs_for_json(oozie_jobs, user): jobs = [] for job in oozie_jobs: if job.is_running(): if job.type == 'Workflow': job = get_oozie().get_job(job.id) else: job = get_oozie().get_coordinator(job.id) massaged_job = { 'id': job.id, 'lastModTime': time.mktime(job.lastModTime), 'endTime': time.mktime(job.endTime), 'status': job.status, 'isRunning': job.is_running(), 'duration': job.endTime and job.startTime and ( time.mktime(job.endTime) - time.mktime(job.startTime) ) * 1000 or None, 'appName': escapejs(job.appName), 'progress': job.get_progress(), 'user': job.user, 'absoluteUrl': job.get_absolute_url(), 'canEdit': has_job_edition_permission(job, user), } jobs.append(massaged_job) return jobs
def massaged_oozie_jobs_for_json(oozie_jobs, user): jobs = [] for job in oozie_jobs: if job.is_running(): if job.type == 'Workflow': job = get_oozie().get_job(job.id) else: job = get_oozie().get_coordinator(job.id) massaged_job = { 'id': job.id, 'lastModTime': hasattr(job, 'lastModTime') and job.lastModTime and format_time(job.lastModTime) or None, 'endTime': job.endTime and format_time(job.endTime) or None, 'status': job.status, 'isRunning': job.is_running(), 'duration': job.endTime and job.startTime and format_duration_in_millis(( time.mktime(job.endTime) - time.mktime(job.startTime) ) * 1000) or None, 'appName': escapejs(job.appName), 'progress': job.get_progress(), 'user': job.user, 'absoluteUrl': job.get_absolute_url(), 'canEdit': has_job_edition_permission(job, user), 'killUrl': reverse('oozie:manage_oozie_jobs', kwargs={'job_id':job.id, 'action':'kill'}), } jobs.append(massaged_job) return jobs
def list_oozie_info(request): instrumentation = get_oozie().get_instrumentation() configuration = get_oozie().get_configuration() oozie_status = get_oozie().get_oozie_status() return render('dashboard/list_oozie_info.mako', request, { 'instrumentation': instrumentation, 'configuration': configuration, 'oozie_status': oozie_status, })
def profile(self, appid, app_type, app_property): if app_property == 'xml': oozie_api = get_oozie(self.user) workflow = oozie_api.get_coordinator(jobid=appid) return { 'xml': workflow.definition, } elif app_property == 'properties': oozie_api = get_oozie(self.user) workflow = oozie_api.get_coordinator(jobid=appid) return { 'properties': workflow.conf_dict, }
def rerun_bundle(self, deployment_dir, params): jobtracker = cluster.get_cluster_addr_for_job_submission() try: prev = get_oozie().setuser(self.user.username) self._update_properties(jobtracker, deployment_dir) self.properties.update({'oozie.bundle.application.path': deployment_dir}) get_oozie().job_control(self.oozie_id, action='bundle-rerun', properties=self.properties, parameters=params) LOG.info("Rerun: %s" % (self,)) finally: get_oozie().setuser(prev) return self.oozie_id
def list_oozie_coordinators(request): kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []} if not has_dashboard_jobs_access(request.user): kwargs['filters'].append(('user', request.user.username)) oozie_api = get_oozie(request.user) enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get() if request.GET.get('format') == 'json': if request.GET.get('type') in ('running', 'progress'): kwargs['filters'].extend([('status', status) for status in CoordinatorWorkflow.RUNNING_STATUSES]) elif request.GET.get('type') == 'completed': kwargs['filters'].extend([('status', status) for status in CoordinatorWorkflow.FINISHED_STATUSES]) json_jobs = oozie_api.get_coordinators(**kwargs).jobs if request.GET.get('type') == 'progress': json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs] return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(json_jobs, request.user)).replace('\\\\', '\\'), content_type="application/json") return render('dashboard/list_oozie_coordinators.mako', request, { 'jobs': [], 'has_job_edition_permission': has_job_edition_permission, 'enable_cron_scheduling': enable_cron_scheduling, })
def profile(self, appid, app_type, app_property, app_filters): if app_property == 'xml': oozie_api = get_oozie(self.user) coordinator = oozie_api.get_coordinator(jobid=appid) return { 'xml': coordinator.definition, } elif app_property == 'properties': oozie_api = get_oozie(self.user) coordinator = oozie_api.get_coordinator(jobid=appid) return { 'properties': coordinator.conf_dict, } elif app_property == 'tasks': coordinator = self.app(appid) return coordinator['properties']['tasks']
def list_oozie_workflows(request): kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []} if not has_dashboard_jobs_access(request.user): kwargs['filters'].append(('user', request.user.username)) oozie_api = get_oozie(request.user) if request.GET.get('format') == 'json': just_sla = request.GET.get('justsla') == 'true' if request.GET.get('startcreatedtime'): kwargs['filters'].extend([('startcreatedtime', request.GET.get('startcreatedtime'))]) if request.GET.get('offset'): kwargs['offset'] = request.GET.get('offset') json_jobs = [] total_jobs = 0 if request.GET.getlist('status'): kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')]) wf_list = oozie_api.get_workflows(**kwargs) json_jobs = wf_list.jobs total_jobs = wf_list.total if request.GET.get('type') == 'progress': json_jobs = [oozie_api.get_job(job.id) for job in json_jobs] response = massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla) response['total_jobs'] = total_jobs return JsonResponse(response, encoder=JSONEncoderForHTML) return render('dashboard/list_oozie_workflows.mako', request, { 'user': request.user, 'jobs': [], 'has_job_edition_permission': has_job_edition_permission, })
def manage_oozie_jobs(request, job_id, action): if request.method != 'POST': raise PopupException(_('Use a POST request to manage an Oozie job.')) job = check_job_access_permission(request, job_id) check_job_edition_permission(job, request.user) response = {'status': -1, 'data': ''} try: oozie_api = get_oozie(request.user) if action == 'change': end_time = 'endtime=%s' % (request.POST.get('end_time')) response['data'] = oozie_api.job_control(job_id, action, parameters={'value': end_time}) else: response['data'] = oozie_api.job_control(job_id, action) response['status'] = 0 if 'notification' in request.POST: request.info(_(request.POST.get('notification'))) except RestException, ex: msg = _("Error performing %s on Oozie job %s: %s.") % (action, job_id, ex.message) LOG.exception(msg) response['data'] = msg
def list_oozie_bundles(request): kwargs = {"cnt": OOZIE_JOBS_COUNT.get(), "filters": []} if not has_dashboard_jobs_access(request.user): kwargs["filters"].append(("user", request.user.username)) oozie_api = get_oozie(request.user) if request.GET.get("format") == "json": if request.GET.get("offset"): kwargs["offset"] = request.GET.get("offset") json_jobs = [] total_jobs = 0 if request.GET.getlist("status"): kwargs["filters"].extend([("status", status) for status in request.GET.getlist("status")]) bundle_list = oozie_api.get_bundles(**kwargs) json_jobs = bundle_list.jobs total_jobs = bundle_list.total if request.GET.get("type") == "progress": json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs] response = massaged_oozie_jobs_for_json(json_jobs, request.user) response["total_jobs"] = total_jobs return JsonResponse(response, encoder=JSONEncoderForHTML) return render( "dashboard/list_oozie_bundles.mako", request, {"jobs": [], "has_job_edition_permission": has_job_edition_permission}, )
def convert_to_server_timezone(date, local_tz='UTC', server_tz=None, user=DEFAULT_USER): api = get_oozie(user) if server_tz is None: oozie_conf = api.get_configuration() server_tz = oozie_conf.get('oozie.processing.timezone') or 'UTC' if date and date.startswith('$'): return date # To support previously created jobs if date.endswith('Z'): date = date[:-1] local_tz = 'UTC' try: date_local_tz = parser.parse(date) date_local_tz = date_local_tz.replace(tzinfo=tz.gettz(local_tz)) date_server_tz = date_local_tz.astimezone(tz.gettz(server_tz)) date_server_tz = date_server_tz.strftime('%Y-%m-%dT%H:%M') # Oozie timezone is either UTC or GMT(+/-)#### if 'UTC' == server_tz: return date_server_tz + u'Z' else: return date_server_tz + u'+' + re.split('[+-]', server_tz)[1] except TypeError, ValueError: LOG.error("Failed to convert Oozie timestamp: %s" % date)
def schedule_document(request): if request.method != 'POST': raise PopupException(_('A POST request is required.')) uuid = request.POST.get('uuid') document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid) notebook = Notebook(document=document) parameters = find_dollar_braced_variables(notebook.get_str()) name = _('Schedule of ') + document.name params = [{u'value': u'%s=${%s}' % (p, p)} for p in parameters] api = get_oozie(request.user) credentials = [HiveDocumentAction.DEFAULT_CREDENTIALS] if api.security_enabled else [] data = json.dumps({ 'workflow': {u'name': name, u'versions': [u'uri:oozie:workflow:0.4', u'uri:oozie:workflow:0.4.5', u'uri:oozie:workflow:0.5'], u'isDirty': False, u'movedNode': None, u'linkMapping': {u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a': [], u'3f107997-04cc-8733-60a9-a4bb62cebffc': [u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c'], u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c': [u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a'], u'17c9c895-5a16-7443-bb81-f34b30b21548': []}, u'nodeIds': [u'3f107997-04cc-8733-60a9-a4bb62cebffc', u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a', u'17c9c895-5a16-7443-bb81-f34b30b21548', u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c'], u'id': 47, u'nodes': [{u'name': u'Start', u'properties': {}, u'actionParametersFetched': False, u'id': u'3f107997-04cc-8733-60a9-a4bb62cebffc', u'type': u'start-widget', u'children': [{u'to': u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c'}], u'actionParameters': []}, {u'name': u'End', u'properties': {}, u'actionParametersFetched': False, u'id': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a', u'type': u'end-widget', u'children': [], u'actionParameters': []}, {u'name': u'Kill', u'properties': {u'body': u'', u'cc': u'', u'to': u'', u'enableMail': False, u'message': u'Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]', u'subject': u''}, u'actionParametersFetched': False, u'id': u'17c9c895-5a16-7443-bb81-f34b30b21548', u'type': u'kill-widget', u'children': [], u'actionParameters': []}, {u'name': u'hive-0aec', u'actionParametersUI': [], u'properties': {u'files': [], u'job_xml': u'', u'uuid': uuid, u'parameters': params, u'retry_interval': [], u'retry_max': [], u'job_properties': [], u'sla': [{u'key': u'enabled', u'value': False}, {u'key': u'nominal-time', u'value': u'${nominal_time}'}, {u'key': u'should-start', u'value': u''}, {u'key': u'should-end', u'value': u'${30 * MINUTES}'}, {u'key': u'max-duration', u'value': u''}, {u'key': u'alert-events', u'value': u''}, {u'key': u'alert-contact', u'value': u''}, {u'key': u'notification-msg', u'value': u''}, {u'key': u'upstream-apps', u'value': u''}], u'archives': [], u'prepares': [], u'credentials': credentials, u'password': u'', u'jdbc_url': u''}, u'actionParametersFetched': False, u'id': u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c', u'type': u'hive-document-widget', u'children': [{u'to': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a'}, {u'error': u'17c9c895-5a16-7443-bb81-f34b30b21548'}], u'actionParameters': []}], u'properties': {u'job_xml': u'', u'description': u'', u'wf1_id': None, u'sla_enabled': False, u'deployment_dir': u'/user/hue/oozie/workspaces/hue-oozie-1459474214.27', u'schema_version': u'uri:oozie:workflow:0.5', u'sla': [{u'key': u'enabled', u'value': False}, {u'key': u'nominal-time', u'value': u'${nominal_time}'}, {u'key': u'should-start', u'value': u''}, {u'key': u'should-end', u'value': u'${30 * MINUTES}'}, {u'key': u'max-duration', u'value': u''}, {u'key': u'alert-events', u'value': u''}, {u'key': u'alert-contact', u'value': u''}, {u'key': u'notification-msg', u'value': u''}, {u'key': u'upstream-apps', u'value': u''}], u'show_arrows': True, u'parameters': [{u'name': u'oozie.use.system.libpath', u'value': True}], u'properties': []}, u'nodeNamesMapping': {u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a': u'End', u'3f107997-04cc-8733-60a9-a4bb62cebffc': u'Start', u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c': u'hive-0aec', u'17c9c895-5a16-7443-bb81-f34b30b21548': u'Kill'}, u'uuid': u'433922e5-e616-dfe0-1cba-7fe744c9305c'}, 'layout': [{u'oozieRows': [{u'enableOozieDropOnBefore': True, u'enableOozieDropOnSide': True, u'enableOozieDrop': False, u'widgets': [{u'status': u'', u'logsURL': u'', u'name': u'Hive', u'widgetType': u'hive-document-widget', u'oozieMovable': True, u'ooziePropertiesExpanded': False, u'externalIdUrl': u'', u'properties': {}, u'isLoading': True, u'offset': 0, u'actionURL': u'', u'progress': 0, u'klass': u'card card-widget span12', u'oozieExpanded': False, u'id': u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c', u'size': 12}], u'id': u'32e1ea1a-812b-6878-9719-ff7b8407bf46', u'columns': []}], u'rows': [{u'enableOozieDropOnBefore': True, u'enableOozieDropOnSide': True, u'enableOozieDrop': False, u'widgets': [{u'status': u'', u'logsURL': u'', u'name': u'Start', u'widgetType': u'start-widget', u'oozieMovable': False, u'ooziePropertiesExpanded': False, u'externalIdUrl': u'', u'properties': {}, u'isLoading': True, u'offset': 0, u'actionURL': u'', u'progress': 0, u'klass': u'card card-widget span12', u'oozieExpanded': False, u'id': u'3f107997-04cc-8733-60a9-a4bb62cebffc', u'size': 12}], u'id': u'798dc16a-d366-6305-d2b3-2d5a6f6c4f4b', u'columns': []}, {u'enableOozieDropOnBefore': True, u'enableOozieDropOnSide': True, u'enableOozieDrop': False, u'widgets': [{u'status': u'', u'logsURL': u'', u'name': u'Hive', u'widgetType': u'hive-document-widget', u'oozieMovable': True, u'ooziePropertiesExpanded': False, u'externalIdUrl': u'', u'properties': {}, u'isLoading': True, u'offset': 0, u'actionURL': u'', u'progress': 0, u'klass': u'card card-widget span12', u'oozieExpanded': False, u'id': u'0aec471d-2b7c-d93d-b22c-2110fd17ea2c', u'size': 12}], u'id': u'32e1ea1a-812b-6878-9719-ff7b8407bf46', u'columns': []}, {u'enableOozieDropOnBefore': True, u'enableOozieDropOnSide': True, u'enableOozieDrop': False, u'widgets': [{u'status': u'', u'logsURL': u'', u'name': u'End', u'widgetType': u'end-widget', u'oozieMovable': False, u'ooziePropertiesExpanded': False, u'externalIdUrl': u'', u'properties': {}, u'isLoading': True, u'offset': 0, u'actionURL': u'', u'progress': 0, u'klass': u'card card-widget span12', u'oozieExpanded': False, u'id': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a', u'size': 12}], u'id': u'f2cf152d-8c82-2f4f-5d67-2e18c99e59c4', u'columns': []}, {u'enableOozieDropOnBefore': True, u'enableOozieDropOnSide': True, u'enableOozieDrop': False, u'widgets': [{u'status': u'', u'logsURL': u'', u'name': u'Kill', u'widgetType': u'kill-widget', u'oozieMovable': True, u'ooziePropertiesExpanded': False, u'externalIdUrl': u'', u'properties': {}, u'isLoading': True, u'offset': 0, u'actionURL': u'', u'progress': 0, u'klass': u'card card-widget span12', u'oozieExpanded': False, u'id': u'17c9c895-5a16-7443-bb81-f34b30b21548', u'size': 12}], u'id': u'01afcf1b-fa7a-e093-b613-ce52c5531a04', u'columns': []}], u'oozieEndRow': {u'enableOozieDropOnBefore': True, u'enableOozieDropOnSide': True, u'enableOozieDrop': False, u'widgets': [{u'status': u'', u'logsURL': u'', u'name': u'End', u'widgetType': u'end-widget', u'oozieMovable': False, u'ooziePropertiesExpanded': False, u'externalIdUrl': u'', u'properties': {}, u'isLoading': True, u'offset': 0, u'actionURL': u'', u'progress': 0, u'klass': u'card card-widget span12', u'oozieExpanded': False, u'id': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a', u'size': 12}], u'id': u'f2cf152d-8c82-2f4f-5d67-2e18c99e59c4', u'columns': []}, u'oozieKillRow': {u'enableOozieDropOnBefore': True, u'enableOozieDropOnSide': True, u'enableOozieDrop': False, u'widgets': [{u'status': u'', u'logsURL': u'', u'name': u'Kill', u'widgetType': u'kill-widget', u'oozieMovable': True, u'ooziePropertiesExpanded': False, u'externalIdUrl': u'', u'properties': {}, u'isLoading': True, u'offset': 0, u'actionURL': u'', u'progress': 0, u'klass': u'card card-widget span12', u'oozieExpanded': False, u'id': u'17c9c895-5a16-7443-bb81-f34b30b21548', u'size': 12}], u'id': u'01afcf1b-fa7a-e093-b613-ce52c5531a04', u'columns': []}, u'enableOozieDropOnAfter': True, u'oozieStartRow': {u'enableOozieDropOnBefore': True, u'enableOozieDropOnSide': True, u'enableOozieDrop': False, u'widgets': [{u'status': u'', u'logsURL': u'', u'name': u'Start', u'widgetType': u'start-widget', u'oozieMovable': False, u'ooziePropertiesExpanded': False, u'externalIdUrl': u'', u'properties': {}, u'isLoading': True, u'offset': 0, u'actionURL': u'', u'progress': 0, u'klass': u'card card-widget span12', u'oozieExpanded': False, u'id': u'3f107997-04cc-8733-60a9-a4bb62cebffc', u'size': 12}], u'id': u'798dc16a-d366-6305-d2b3-2d5a6f6c4f4b', u'columns': []}, u'klass': u'card card-home card-column span12', u'enableOozieDropOnBefore': True, u'drops': [u'temp'], u'id': u'672ff75a-d841-72c3-c616-c9d45ec97649', u'size': 12}] }) workflow_doc = Document2.objects.create(name=name, type='oozie-workflow2', owner=request.user, data=data) Document.objects.link(workflow_doc, owner=workflow_doc.owner, name=workflow_doc.name, description=workflow_doc.description, extra='workflow2') workflow_doc.dependencies.add(document) response = { 'status': 0, 'url': reverse('oozie:new_coordinator') + '?workflow=' + workflow_doc.uuid } return JsonResponse(response)
def apps(self, filters): oozie_api = get_oozie(self.user) kwargs = {'cnt': OOZIE_JOBS_COUNT_LIMIT, 'filters': []} _filter_oozie_jobs(self.user, filters, kwargs) wf_list = oozie_api.get_workflows(**kwargs) return { 'apps': [{ 'id': app['id'], 'name': app['appName'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'type': 'workflow', 'user': app['user'], 'progress': app['progress'], 'queue': app['group'], 'duration': app['durationInMillis'], 'submitted': app['startTimeInMillis'] * 1000, 'canWrite': app['canEdit'] } for app in massaged_oozie_jobs_for_json( wf_list.jobs, self.user)['jobs']], 'total': wf_list.total }
def app(self, appid): if '@' in appid: return WorkflowActionApi(self.user).app(appid) oozie_api = get_oozie(self.user) workflow = oozie_api.get_job(jobid=appid) common = { 'id': workflow.id, 'name': workflow.appName, 'status': workflow.status, 'apiStatus': self._api_status(workflow.status), 'progress': workflow.get_progress(), 'type': 'workflow', } request = MockDjangoRequest(self.user) response = list_oozie_workflow(request, job_id=appid) common['properties'] = json.loads(response.content) common['properties']['xml'] = '' common['properties']['properties'] = '' common['properties']['coordinator_id'] = workflow.get_parent_job_id() common['properties']['bundle_id'] = workflow.conf_dict.get( 'oozie.bundle.id') return common
def list_oozie_workflows(request): kwargs = { 'cnt': OOZIE_JOBS_COUNT.get(), } if not has_dashboard_jobs_access(request.user): kwargs['user'] = request.user.username workflows = get_oozie(request.user).get_workflows(**kwargs) if request.GET.get('format') == 'json': json_jobs = workflows.jobs just_sla = request.GET.get('justsla') == 'true' if request.GET.get('type') == 'running': json_jobs = split_oozie_jobs(request.user, workflows.jobs)['running_jobs'] if request.GET.get('type') == 'completed': json_jobs = split_oozie_jobs(request.user, workflows.jobs)['completed_jobs'] return HttpResponse(encode_json_for_js( massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla)), mimetype="application/json") return render( 'dashboard/list_oozie_workflows.mako', request, { 'user': request.user, 'jobs': split_oozie_jobs(request.user, workflows.jobs), 'has_job_edition_permission': has_job_edition_permission, })
def list_oozie_bundles(request): kwargs = { 'cnt': OOZIE_JOBS_COUNT.get(), } if not has_dashboard_jobs_access(request.user): kwargs['user'] = request.user.username bundles = get_oozie(request.user).get_bundles(**kwargs) if request.GET.get('format') == 'json': json_jobs = bundles.jobs if request.GET.get('type') == 'running': json_jobs = split_oozie_jobs(request.user, bundles.jobs)['running_jobs'] if request.GET.get('type') == 'completed': json_jobs = split_oozie_jobs(request.user, bundles.jobs)['completed_jobs'] return HttpResponse(json.dumps( massaged_oozie_jobs_for_json(json_jobs, request.user)).replace('\\\\', '\\'), mimetype="application/json") return render( 'dashboard/list_oozie_bundles.mako', request, { 'jobs': split_oozie_jobs(request.user, bundles.jobs), 'has_job_edition_permission': has_job_edition_permission, })
def app(self, appid): oozie_api = get_oozie(self.user) coordinator = oozie_api.get_coordinator(jobid=appid) request = MockDjangoRequest(self.user, get=MockGet()) response = list_oozie_coordinator(request, job_id=appid) common = { 'id': coordinator.coordJobId, 'name': coordinator.coordJobName, 'status': coordinator.status, 'apiStatus': self._api_status(coordinator.status), 'progress': coordinator.get_progress(), 'type': 'schedule', 'submitted': format_time(coordinator.startTime), 'user': coordinator.user, 'canWrite': has_job_edition_permission(coordinator, self.user), } common['properties'] = json.loads(response.content) common['properties']['tasks'] = common['properties']['actions'] common['properties']['xml'] = '' common['properties']['properties'] = '' common['properties']['bundle_id'] = coordinator.conf_dict.get( 'oozie.bundle.id') common['doc_url'] = common['properties'].get('doc_url') return common
def apps(self, filters): oozie_api = get_oozie(self.user) kwargs = { 'cnt': hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get(), 'filters': [] } _filter_oozie_jobs(self.user, filters, kwargs) jobs = oozie_api.get_coordinators(**kwargs) return { 'apps': [{ 'id': app['id'], 'name': app['appName'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'type': 'schedule', 'user': app['user'], 'progress': app['progress'], 'queue': app['group'], 'duration': app['durationInMillis'], 'submitted': app['lastActionInMillis'] * 1000, 'canWrite': app['canEdit'] } for app in massaged_oozie_jobs_for_json( jobs.jobs, self.user)['jobs']], 'total': jobs.total }
def app(self, appid): if '@' in appid: return WorkflowActionApi(self.user).app(appid) oozie_api = get_oozie(self.user) workflow = oozie_api.get_job(jobid=appid) common = { 'id': workflow.id, 'name': workflow.appName, 'status': workflow.status, 'apiStatus': self._api_status(workflow.status), 'progress': workflow.get_progress(), 'type': 'workflow', 'user': workflow.user, 'duration': workflow.durationTime, 'submitted': workflow.submissionTime * 1000, 'canWrite': has_job_edition_permission(workflow, self.user), } request = MockDjangoRequest(self.user) response = list_oozie_workflow(request, job_id=appid) common['properties'] = json.loads(response.content) common['properties']['xml'] = '' common['properties']['properties'] = '' common['properties']['coordinator_id'] = workflow.get_parent_job_id() common['properties']['bundle_id'] = workflow.conf_dict.get( 'oozie.bundle.id') common['properties']['parameters'] = self._get_variables(workflow) common['doc_url'] = common['properties'].get('doc_url') return common
def list_oozie_workflows(request): kwargs = { 'cnt': OOZIE_JOBS_COUNT.get(), } if not has_dashboard_jobs_access(request.user): kwargs['user'] = request.user.username oozie_api = get_oozie(request.user) if request.GET.get('format') == 'json': just_sla = request.GET.get('justsla') == 'true' if request.GET.get('type') in ('running', 'progress'): kwargs['filters'] = [('status', status) for status in OozieWorkflow.RUNNING_STATUSES] elif request.GET.get('type') == 'completed': kwargs['filters'] = [('status', status) for status in OozieWorkflow.FINISHED_STATUSES] json_jobs = oozie_api.get_workflows(**kwargs).jobs if request.GET.get('type') == 'progress': json_jobs = [oozie_api.get_job(job.id) for job in json_jobs] return JsonResponse(massaged_oozie_jobs_for_json( json_jobs, request.user, just_sla), encoder=JSONEncoderForHTML) return render( 'dashboard/list_oozie_workflows.mako', request, { 'user': request.user, 'jobs': [], 'has_job_edition_permission': has_job_edition_permission, })
def config_validator(): """ config_validator() -> [ (config_variable, error_message) ] Called by core check_config() view. """ from hadoop.cluster import get_all_hdfs from liboozie.oozie_api import get_oozie res = [] status = 'down' try: status = str(get_oozie().get_oozie_status()) except: pass if 'NORMAL' not in status: res.append((status, _('The Oozie server is not available'))) class ConfigMock: def __init__(self, value): self.value = value def get(self): return self.value def get_fully_qualifying_key(self): return self.value for cluster in get_all_hdfs().values(): res.extend(validate_path(REMOTE_DEPLOYMENT_DIR, is_dir=True, fs=cluster, message=_('The deployment directory of Oozie workflows does not exist. ' 'Please run "Setup App" on the Oozie workflow page.'))) res.extend(validate_path(ConfigMock('/user/oozie/share/lib'), is_dir=True, fs=cluster, message=_('Oozie Share Lib not installed in default location.'))) return res
def list_oozie_workflow_action(request, action, coordinator_job_id=None, bundle_job_id=None): try: action = get_oozie().get_action(action) workflow = check_job_access_permission(request, action.id.split('@')[0]) except RestException, ex: raise PopupException(_("Error accessing Oozie action %s.") % (action,), detail=ex.message)
def list_oozie_workflows(request): kwargs = {"cnt": OOZIE_JOBS_COUNT.get(), "filters": []} if not has_dashboard_jobs_access(request.user): kwargs["filters"].append(("user", request.user.username)) oozie_api = get_oozie(request.user) if request.GET.get("format") == "json": just_sla = request.GET.get("justsla") == "true" if request.GET.get("startcreatedtime"): kwargs["filters"].extend([("startcreatedtime", request.GET.get("startcreatedtime"))]) if request.GET.get("offset"): kwargs["offset"] = request.GET.get("offset") json_jobs = [] total_jobs = 0 if request.GET.getlist("status"): kwargs["filters"].extend([("status", status) for status in request.GET.getlist("status")]) wf_list = oozie_api.get_workflows(**kwargs) json_jobs = wf_list.jobs total_jobs = wf_list.total if request.GET.get("type") == "progress": json_jobs = [oozie_api.get_job(job.id) for job in json_jobs] response = massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla) response["total_jobs"] = total_jobs return JsonResponse(response, encoder=JSONEncoderForHTML) return render( "dashboard/list_oozie_workflows.mako", request, {"user": request.user, "jobs": [], "has_job_edition_permission": has_job_edition_permission}, )
def list_oozie_coordinators(request): kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []} if not has_dashboard_jobs_access(request.user): kwargs['filters'].append(('user', request.user.username)) oozie_api = get_oozie(request.user) enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get() if request.GET.get('format') == 'json': if request.GET.get('offset'): kwargs['offset'] = request.GET.get('offset') if request.GET.get('text') and ENABLE_OOZIE_BACKEND_FILTERING.get(): kwargs['filters'].extend([('text', request.GET.get('text'))]) json_jobs = [] total_jobs = 0 if request.GET.getlist('status'): kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')]) co_list = oozie_api.get_coordinators(**kwargs) json_jobs = co_list.jobs total_jobs = co_list.total if request.GET.get('type') == 'progress': json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs] response = massaged_oozie_jobs_for_json(json_jobs, request.user) response['total_jobs'] = total_jobs return JsonResponse(response, encoder=JSONEncoderForHTML) return render('dashboard/list_oozie_coordinators.mako', request, { 'jobs': [], 'has_job_edition_permission': has_job_edition_permission, 'enable_cron_scheduling': enable_cron_scheduling, })
def check_job_access_permission(request, job_id, actions_offset=1): """ Decorator ensuring that the user has access to the job submitted to Oozie. Arg: Oozie 'workflow', 'coordinator' or 'bundle' ID. Return: the Oozie workflow, coordinator or bundle or raise an exception Notice: its gets an id in input and returns the full object in output (not an id). """ if job_id is not None: oozie_api = get_oozie(request.user) if job_id.endswith('W'): get_job = oozie_api.get_job elif job_id.endswith('C'): get_job = oozie_api.get_coordinator else: get_job = oozie_api.get_bundle try: if job_id.endswith('C'): oozie_job = get_job(job_id, actions_offset) else: oozie_job = get_job(job_id) except RestException, ex: msg = _("Error accessing Oozie job %s.") % (job_id,) LOG.exception(msg) raise PopupException(msg, detail=ex._headers['oozie-error-message', ''])
def apps(self, filters): oozie_api = get_oozie(self.user) kwargs = {'cnt': hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get(), 'filters': []} filters.pop('time') _filter_oozie_jobs(self.user, filters, kwargs) jobs = oozie_api.get_coordinators(**kwargs) return { 'apps':[{ 'id': app['id'], 'name': app['appName'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'type': 'schedule', 'user': app['user'], 'progress': app['progress'], 'queue': app['group'], 'duration': app['durationInMillis'], 'submitted': app['lastActionInMillis'] * 1000, 'canWrite': app['canEdit'] } for app in massaged_oozie_jobs_for_json(jobs.jobs, self.user)['jobs']], 'total': jobs.total }
def apps(self, filters): oozie_api = get_oozie(self.user) kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []} filters.pop('time') _filter_oozie_jobs(self.user, filters, kwargs) jobs = oozie_api.get_bundles(**kwargs) return { 'apps': [{ 'id': app['id'], 'name': app['appName'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'type': 'bundle', 'user': app['user'], 'progress': app['progress'], 'queue': app['group'], 'duration': app['durationInMillis'], 'submitted': app['kickoffTimeInMillis'] * 1000, 'canWrite': app['canEdit'] } for app in massaged_oozie_jobs_for_json( jobs.jobs, self.user)['jobs']], 'total': jobs.total }
def list_oozie_workflow_action(request, action): try: action = get_oozie().get_action(action) workflow = check_access_and_get_oozie_job(request, action.id.split('@')[0]) except RestException, ex: raise PopupException(_("Error accessing Oozie action %s") % (action,), detail=ex.message)
def list_oozie_bundles(request): kwargs = { 'cnt': OOZIE_JOBS_COUNT.get(), } if not has_dashboard_jobs_access(request.user): kwargs['user'] = request.user.username oozie_api = get_oozie(request.user) if request.GET.get('format') == 'json': if request.GET.get('type') in ('running', 'progress'): kwargs['filters'] = [('status', status) for status in BundleWorkflow.RUNNING_STATUSES] elif request.GET.get('type') == 'completed': kwargs['filters'] = [('status', status) for status in BundleWorkflow.FINISHED_STATUSES ] json_jobs = oozie_api.get_bundles(**kwargs).jobs if request.GET.get('type') == 'progress': json_jobs = [oozie_api.get_bundle(job.id) for job in json_jobs] return HttpResponse(json.dumps( massaged_oozie_jobs_for_json(json_jobs, request.user)).replace('\\\\', '\\'), content_type="application/json") return render('dashboard/list_oozie_bundles.mako', request, { 'jobs': [], 'has_job_edition_permission': has_job_edition_permission, })
def app(self, appid): if '@' in appid: return WorkflowActionApi(self.user).app(appid) oozie_api = get_oozie(self.user) workflow = oozie_api.get_job(jobid=appid) common = { 'id': workflow.id, 'name': workflow.appName, 'status': workflow.status, 'apiStatus': self._api_status(workflow.status), 'progress': workflow.get_progress(), 'type': 'workflow', 'user': workflow.user, 'duration': workflow.durationTime, 'submitted': workflow.submissionTime * 1000, 'canWrite': has_job_edition_permission(workflow, self.user), } request = MockDjangoRequest(self.user) response = list_oozie_workflow(request, job_id=appid) common['properties'] = json.loads(response.content) common['properties']['xml'] = '' common['properties']['properties'] = '' common['properties']['coordinator_id'] = workflow.get_parent_job_id() common['properties']['bundle_id'] = workflow.conf_dict.get('oozie.bundle.id') common['properties']['parameters'] = self._get_variables(workflow) common['doc_url'] = common['properties'].get('doc_url') return common
def list_oozie_sla(request): oozie_api = get_oozie(request.user, api_version="v2") if request.method == 'POST': params = {} job_name = request.POST.get('job_name') if re.match('.*-oozie-oozi-[WCB]', job_name): params['id'] = job_name params['parent_id'] = job_name else: params['app_name'] = job_name if 'useDates' in request.POST: if request.POST.get('start'): params['nominal_start'] = request.POST.get('start') if request.POST.get('end'): params['nominal_end'] = request.POST.get('end') oozie_slas = oozie_api.get_oozie_slas(**params) else: oozie_slas = [] # or get latest? if request.REQUEST.get('format') == 'json': massaged_slas = [] for sla in oozie_slas: massaged_slas.append(massaged_sla_for_json(sla, request)) return HttpResponse(json.dumps({'oozie_slas': massaged_slas}), content_type="text/json") return render('dashboard/list_oozie_sla.mako', request, {'oozie_slas': oozie_slas})
def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None, local_tz=None): self.job = job self.user = user self.fs = fs self.jt = jt # Deprecated with YARN, we now use logical names only for RM self.oozie_id = oozie_id self.api = get_oozie(self.user) if properties is not None: self.properties = properties else: self.properties = {} if local_tz and isinstance(self.job.data, dict): local_tz = self.job.data.get('properties')['timezone'] # Modify start_date & end_date only when it's a coordinator from oozie.models2 import Coordinator if type(self.job) is Coordinator: if 'start_date' in self.properties: properties['start_date'] = convert_to_server_timezone(self.properties['start_date'], local_tz) if 'end_date' in self.properties: properties['end_date'] = convert_to_server_timezone(self.properties['end_date'], local_tz) self.properties['security_enabled'] = self.api.security_enabled
def app(self, appid): oozie_api = get_oozie(self.user) coordinator = oozie_api.get_coordinator(jobid=appid) request = MockDjangoRequest(self.user, get=MockGet()) response = list_oozie_coordinator(request, job_id=appid) common = { 'id': coordinator.coordJobId, 'name': coordinator.coordJobName, 'status': coordinator.status, 'apiStatus': self._api_status(coordinator.status), 'progress': coordinator.get_progress(), 'type': 'schedule', 'submitted': format_time(coordinator.startTime), 'user': coordinator.user, 'canWrite': has_job_edition_permission(coordinator, self.user), } common['properties'] = json.loads(response.content) for action in common['properties']['actions']: action['apiStatus'] = self._task_api_status(action['status']) common['properties']['tasks'] = common['properties']['actions'] common['properties']['xml'] = '' common['properties']['properties'] = '' common['properties']['bundle_id'] = coordinator.conf_dict.get('oozie.bundle.id') common['doc_url'] = common['properties'].get('doc_url') return common
def convert_to_server_timezone(date, local_tz='UTC', server_tz=None, user=DEFAULT_USER): api = get_oozie(user) if server_tz is None: oozie_conf = api.get_configuration() server_tz = oozie_conf.get('oozie.processing.timezone') or 'UTC' if date and date.startswith('$'): return date # To support previously created jobs if date.endswith('Z'): date = date[:-1] local_tz = 'UTC' try: date_local_tz = parser.parse(date) date_local_tz = date_local_tz.replace(tzinfo=tz.gettz(local_tz)) date_server_tz = date_local_tz.astimezone(tz.gettz(server_tz)) # Oozie timezone is either UTC or GMT(+/-)#### if 'UTC' == server_tz: return date_server_tz.strftime('%Y-%m-%dT%H:%M') + u'Z' else: return date_server_tz.strftime('%Y-%m-%dT%H:%M') + date_server_tz.strftime('%z') except TypeError as ValueError: LOG.error("Failed to convert Oozie timestamp: %s" % date) return None
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def list_oozie_bundles(request): kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []} if not has_dashboard_jobs_access(request.user): kwargs['filters'].append(('user', request.user.username)) oozie_api = get_oozie(request.user) if request.GET.get('format') == 'json': if request.GET.get('offset'): kwargs['offset'] = request.GET.get('offset') if request.GET.get('text') and ENABLE_OOZIE_BACKEND_FILTERING.get(): kwargs['filters'].extend([('text', request.GET.get('text'))]) json_jobs = [] total_jobs = 0 if request.GET.getlist('status'): kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')]) bundle_list = oozie_api.get_bundles(**kwargs) json_jobs = bundle_list.jobs total_jobs = bundle_list.total if request.GET.get('type') == 'progress': json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs] response = massaged_oozie_jobs_for_json(json_jobs, request.user) response['total_jobs'] = total_jobs return JsonResponse(response, encoder=JSONEncoderForHTML) return render('dashboard/list_oozie_bundles.mako', request, { 'jobs': [], 'has_job_edition_permission': has_job_edition_permission, })
def list_oozie_sla(request): oozie_api = get_oozie(request.user, api_version="v2") if request.method == 'POST': params = {} job_name = request.POST.get('job_name') if re.match('.*-oozie-oozi-[WCB]', job_name): params['id'] = job_name params['parent_id'] = job_name else: params['app_name'] = job_name if 'useDates' in request.POST: if request.POST.get('start'): params['nominal_start'] = request.POST.get('start') if request.POST.get('end'): params['nominal_end'] = request.POST.get('end') oozie_slas = oozie_api.get_oozie_slas(**params) else: oozie_slas = [] # or get latest? if request.REQUEST.get('format') == 'json': massaged_slas = [] for sla in oozie_slas: massaged_slas.append(massaged_sla_for_json(sla, request)) return HttpResponse(json.dumps({'oozie_slas': massaged_slas}), content_type="text/json") return render('dashboard/list_oozie_sla.mako', request, { 'oozie_slas': oozie_slas })
def print_while_running(oozie_jobs, job_id): while is_running(oozie_jobs): for i, id in job_id: oozie_jobs[i] = get_oozie().get_job(id) for job in oozie_jobs: print (job.appName + "," + job.status + " " + str(job.get_progress()) + " | "), sleep(1)
def app(self, appid): if '@' in appid: return WorkflowActionApi(self.user).app(appid) oozie_api = get_oozie(self.user) workflow = oozie_api.get_job(jobid=appid) common = { 'id': workflow.id, 'name': workflow.appName, 'status': workflow.status, 'apiStatus': self._api_status(workflow.status), 'progress': workflow.get_progress(), 'type': 'workflow', } request = MockDjangoRequest(self.user) response = list_oozie_workflow(request, job_id=appid) common['properties'] = json.loads(response.content) common['properties']['xml'] = '' common['properties']['properties'] = '' common['properties']['coordinator_id'] = workflow.get_parent_job_id() common['properties']['bundle_id'] = workflow.conf_dict.get('oozie.bundle.id') return common
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None workflow_uuid = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) if request.GET.get('workflow'): workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def _edit_workflow(request, doc, workflow): workflow_data = workflow.get_data() api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception as e: LOG.error(smart_str(e)) can_edit_json = doc is None or (doc.can_write(request.user) if USE_NEW_EDITOR.get() else doc.doc.get().is_editable(request.user)) return render( 'editor2/workflow_editor.mako', request, { 'layout_json': json.dumps(workflow_data['layout'], cls=JSONEncoderForHTML), 'workflow_json': json.dumps(workflow_data['workflow'], cls=JSONEncoderForHTML), 'credentials_json': json.dumps(list(credentials.credentials.keys()), cls=JSONEncoderForHTML), 'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML), 'doc_uuid': doc.uuid if doc else '', 'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML), 'can_edit_json': json.dumps(can_edit_json), 'is_embeddable': request.GET.get('is_embeddable', False), })
def list_oozie_bundles(request): kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []} if not has_dashboard_jobs_access(request.user): kwargs['filters'].append(('user', request.user.username)) oozie_api = get_oozie(request.user) if request.GET.get('format') == 'json': if request.GET.get('offset'): kwargs['offset'] = request.GET.get('offset') json_jobs = [] total_jobs = 0 if request.GET.getlist('status'): kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')]) bundle_list = oozie_api.get_bundles(**kwargs) json_jobs = bundle_list.jobs total_jobs = bundle_list.total if request.GET.get('type') == 'progress': json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs] response = massaged_oozie_jobs_for_json(json_jobs, request.user) response['total_jobs'] = total_jobs return JsonResponse(response, encoder=JSONEncoderForHTML) return render('dashboard/list_oozie_bundles.mako', request, { 'jobs': [], 'has_job_edition_permission': has_job_edition_permission, })
def get_oozie_job_log(request, job_id): oozie_api = get_oozie(request.user, api_version="v2") check_job_access_permission(request, job_id) kwargs = {'logfilter' : []} if request.GET.get('format') == 'json': if request.GET.get('recent'): kwargs['logfilter'].extend([('recent', val) for val in request.GET.get('recent').split(':')]) if request.GET.get('limit'): kwargs['logfilter'].extend([('limit', request.GET.get('limit'))]) if request.GET.get('loglevel'): kwargs['logfilter'].extend([('loglevel', request.GET.get('loglevel'))]) if request.GET.get('text'): kwargs['logfilter'].extend([('text', request.GET.get('text'))]) status_resp = oozie_api.get_job_status(job_id) log = oozie_api.get_job_log(job_id, **kwargs) return_obj = { 'id': job_id, 'status': status_resp['status'], 'log': log, } return JsonResponse(return_obj, encoder=JSONEncoderForHTML)
def check_job_access_permission(request, job_id, **kwargs): """ Decorator ensuring that the user has access to the job submitted to Oozie. Arg: Oozie 'workflow', 'coordinator' or 'bundle' ID. Return: the Oozie workflow, coordinator or bundle or raise an exception Notice: its gets an id in input and returns the full object in output (not an id). """ if job_id is not None: oozie_api = get_oozie(request.user) if job_id.endswith('W'): get_job = oozie_api.get_job elif job_id.endswith('C'): get_job = oozie_api.get_coordinator else: get_job = oozie_api.get_bundle try: if job_id.endswith('C'): oozie_job = get_job(job_id, **kwargs) else: oozie_job = get_job(job_id) except RestException, ex: msg = _("Error accessing Oozie job %s.") % (job_id,) LOG.exception(msg) raise PopupException(msg, detail=ex._headers.get('oozie-error-message'))
def list_oozie_coordinator(request, job_id): kwargs = {'cnt': 50, 'filters': []} kwargs['offset'] = request.GET.get('offset', 1) if request.GET.getlist('status'): kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')]) oozie_coordinator = check_job_access_permission(request, job_id, **kwargs) # Cross reference the submission history (if any) coordinator = get_history().get_coordinator_from_config(oozie_coordinator.conf_dict) try: if not ENABLE_V2.get(): coordinator = get_history().objects.get(oozie_job_id=job_id).job.get_full_node() except: LOG.exception("Ignoring error getting oozie job coordinator for job_id=%s", job_id) oozie_bundle = None if request.GET.get('bundle_job_id'): try: oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id')) except: LOG.exception("Ignoring error getting oozie bundle for job_id=%s", job_id) if request.GET.get('format') == 'json': actions = massaged_coordinator_actions_for_json(oozie_coordinator, oozie_bundle) return_obj = { 'id': oozie_coordinator.id, 'status': oozie_coordinator.status, 'progress': oozie_coordinator.get_progress(), 'nextTime': format_time(oozie_coordinator.nextMaterializedTime), 'endTime': format_time(oozie_coordinator.endTime), 'actions': actions, 'total_actions': oozie_coordinator.total, 'doc_url': coordinator.get_absolute_url() if coordinator else '', } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) oozie_slas = [] if oozie_coordinator.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = { 'id': oozie_coordinator.id, 'parent_id': oozie_coordinator.id } oozie_slas = oozie_api.get_oozie_slas(**params) enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get() update_coord_form = UpdateCoordinatorForm(oozie_coordinator=oozie_coordinator) return render('dashboard/list_oozie_coordinator.mako', request, { 'oozie_coordinator': oozie_coordinator, 'oozie_slas': oozie_slas, 'coordinator': coordinator, 'oozie_bundle': oozie_bundle, 'has_job_edition_permission': has_job_edition_permission, 'enable_cron_scheduling': enable_cron_scheduling, 'update_coord_form': update_coord_form, })
def rerun_bundle(self, deployment_dir, params): jobtracker = cluster.get_cluster_addr_for_job_submission() try: prev = get_oozie().setuser(self.user.username) self._update_properties(jobtracker, deployment_dir) self.properties.update( {'oozie.bundle.application.path': deployment_dir}) get_oozie().job_control(self.oozie_id, action='bundle-rerun', properties=self.properties, parameters=params) LOG.info("Rerun: %s" % (self, )) finally: get_oozie().setuser(prev) return self.oozie_id
def get_jobs(self): kwargs = { 'cnt': OozieApi.MAX_DASHBOARD_JOBS, } kwargs['user'] = self.user.username kwargs['name'] = OozieApi.WORKFLOW_NAME return get_oozie(self.user).get_workflows(**kwargs).jobs
def get_jobs(self): kwargs = { 'cnt': OozieApi.MAX_DASHBOARD_JOBS, } kwargs['filters'] = [('user', self.user.username), ('name', OozieApi.WORKFLOW_NAME)] return get_oozie(self.user).get_workflows(**kwargs).jobs
def manage_oozie_jobs(request, job_id, action): if request.method != 'POST': raise PopupException(_('Use a POST request to manage an Oozie job.')) job = check_job_access_permission(request, job_id) check_job_edition_permission(job, request.user) response = {'status': -1, 'data': ''} try: oozie_api = get_oozie(request.user) params = None if action == 'change': pause_time_val = request.POST.get('pause_time') if request.POST.get('clear_pause_time') == 'true': pause_time_val = '' end_time_val = request.POST.get('end_time') if end_time_val: end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get()) if pause_time_val: pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get()) params = {'value': 'endtime=%s' % (end_time_val) + ';' 'pausetime=%s' % (pause_time_val) + ';' 'concurrency=%s' % (request.POST.get('concurrency'))} elif action == 'ignore': oozie_api = get_oozie(request.user, api_version="v2") params = { 'type': 'action', 'scope': ','.join(job.aggreate(request.POST.get('actions').split())), } response['data'] = oozie_api.job_control(job_id, action, parameters=params) response['status'] = 0 if 'notification' in request.POST: request.info(_(request.POST.get('notification'))) except RestException, ex: ex_message = ex.message if ex._headers.get('oozie-error-message'): ex_message = ex._headers.get('oozie-error-message') msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message) LOG.exception(msg) response['data'] = _(msg)
def list_oozie_coordinator(request, job_id): oozie_coordinator = check_job_access_permission(request, job_id) # Cross reference the submission history (if any) coordinator = get_history().get_coordinator_from_config(oozie_coordinator.conf_dict) try: coordinator = get_history().objects.get(oozie_job_id=job_id).job.get_full_node() except: LOG.exception("Ignoring error getting oozie job coordinator for job_id=%s", job_id) oozie_bundle = None if request.GET.get('bundle_job_id'): try: oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id')) except: LOG.exception("Ignoring error getting oozie bundle for job_id=%s", job_id) show_all_actions = request.GET.get('show_all_actions') == 'true' if request.GET.get('format') == 'json': actions = massaged_coordinator_actions_for_json(oozie_coordinator, oozie_bundle) if not show_all_actions: actions = actions[:MAX_COORD_ACTIONS] return_obj = { 'id': oozie_coordinator.id, 'status': oozie_coordinator.status, 'progress': oozie_coordinator.get_progress(), 'nextTime': format_time(oozie_coordinator.nextMaterializedTime), 'endTime': format_time(oozie_coordinator.endTime), 'actions': actions, 'show_all_actions': show_all_actions } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) oozie_slas = [] if oozie_coordinator.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = { 'id': oozie_coordinator.id, 'parent_id': oozie_coordinator.id } oozie_slas = oozie_api.get_oozie_slas(**params) enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get() update_endtime_form = UpdateEndTimeForm() return render('dashboard/list_oozie_coordinator.mako', request, { 'oozie_coordinator': oozie_coordinator, 'oozie_slas': oozie_slas, 'coordinator': coordinator, 'oozie_bundle': oozie_bundle, 'has_job_edition_permission': has_job_edition_permission, 'show_all_actions': show_all_actions, 'MAX_COORD_ACTIONS': MAX_COORD_ACTIONS, 'enable_cron_scheduling': enable_cron_scheduling, 'update_endtime_form': update_endtime_form, })
def update_coord(self): self.api = get_oozie(self.user, api_version="v2") self.api.job_control(self.oozie_id, action='update', properties=self.properties, parameters=None) LOG.info("Update: %s" % (self, )) return self.oozie_id
def list_oozie_workflow_action(request, action): try: action = get_oozie(request.user).get_action(action) workflow = check_job_access_permission(request, action.id.split('@')[0]) except RestException, ex: msg = _("Error accessing Oozie action %s.") % (action,) LOG.exception(msg) raise PopupException(msg, detail=ex.message)