def list_oozie_workflows(request):
kwargs = {
'cnt': OOZIE_JOBS_COUNT.get(),
}
if not has_dashboard_jobs_access(request.user):
kwargs['user'] = request.user.username
workflows = get_oozie().get_workflows(**kwargs)
if request.GET.get('format') == 'json':
json_jobs = workflows.jobs
if request.GET.get('type') == 'running':
json_jobs = split_oozie_jobs(workflows.jobs)['running_jobs']
if request.GET.get('type') == 'completed':
json_jobs = split_oozie_jobs(workflows.jobs)['completed_jobs']
return HttpResponse(encode_json_for_js(
massaged_oozie_jobs_for_json(json_jobs, request.user)),
mimetype="application/json")
return render(
'dashboard/list_oozie_workflows.mako', request, {
'user': request.user,
'jobs': split_oozie_jobs(workflows.jobs),
'has_job_edition_permission': has_job_edition_permission,
})
def list_oozie_bundles(request):
kwargs = {"cnt": OOZIE_JOBS_COUNT.get(), "filters": []}
if not has_dashboard_jobs_access(request.user):
kwargs["filters"].append(("user", request.user.username))
oozie_api = get_oozie(request.user)
if request.GET.get("format") == "json":
if request.GET.get("offset"):
kwargs["offset"] = request.GET.get("offset")
json_jobs = []
total_jobs = 0
if request.GET.getlist("status"):
kwargs["filters"].extend([("status", status) for status in request.GET.getlist("status")])
bundle_list = oozie_api.get_bundles(**kwargs)
json_jobs = bundle_list.jobs
total_jobs = bundle_list.total
if request.GET.get("type") == "progress":
json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs]
response = massaged_oozie_jobs_for_json(json_jobs, request.user)
response["total_jobs"] = total_jobs
return JsonResponse(response, encoder=JSONEncoderForHTML)
return render(
"dashboard/list_oozie_bundles.mako",
request,
{"jobs": [], "has_job_edition_permission": has_job_edition_permission},
)
def list_oozie_bundles(request):
kwargs = {
'cnt': OOZIE_JOBS_COUNT.get(),
}
if not has_dashboard_jobs_access(request.user):
kwargs['user'] = request.user.username
bundles = get_oozie().get_bundles(**kwargs)
if request.GET.get('format') == 'json':
json_jobs = bundles.jobs
if request.GET.get('type') == 'running':
json_jobs = split_oozie_jobs(bundles.jobs)['running_jobs']
if request.GET.get('type') == 'completed':
json_jobs = split_oozie_jobs(bundles.jobs)['completed_jobs']
return HttpResponse(json.dumps(
massaged_oozie_jobs_for_json(json_jobs,
request.user)).replace('\\\\', '\\'),
mimetype="application/json")
return render(
'dashboard/list_oozie_bundles.mako', request, {
'jobs': split_oozie_jobs(bundles.jobs),
'has_job_edition_permission': has_job_edition_permission,
})
def list_oozie_workflows(request):
kwargs = {"cnt": OOZIE_JOBS_COUNT.get(), "filters": []}
if not has_dashboard_jobs_access(request.user):
kwargs["filters"].append(("user", request.user.username))
oozie_api = get_oozie(request.user)
if request.GET.get("format") == "json":
just_sla = request.GET.get("justsla") == "true"
if request.GET.get("startcreatedtime"):
kwargs["filters"].extend([("startcreatedtime", request.GET.get("startcreatedtime"))])
if request.GET.get("offset"):
kwargs["offset"] = request.GET.get("offset")
json_jobs = []
total_jobs = 0
if request.GET.getlist("status"):
kwargs["filters"].extend([("status", status) for status in request.GET.getlist("status")])
wf_list = oozie_api.get_workflows(**kwargs)
json_jobs = wf_list.jobs
total_jobs = wf_list.total
if request.GET.get("type") == "progress":
json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]
response = massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla)
response["total_jobs"] = total_jobs
return JsonResponse(response, encoder=JSONEncoderForHTML)
return render(
"dashboard/list_oozie_workflows.mako",
request,
{"user": request.user, "jobs": [], "has_job_edition_permission": has_job_edition_permission},
)
def list_oozie_coordinators(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
if not has_dashboard_jobs_access(request.user):
kwargs['filters'].append(('user', request.user.username))
oozie_api = get_oozie(request.user)
enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()
if request.GET.get('format') == 'json':
if request.GET.get('type') in ('running', 'progress'):
kwargs['filters'].extend([('status', status) for status in CoordinatorWorkflow.RUNNING_STATUSES])
elif request.GET.get('type') == 'completed':
kwargs['filters'].extend([('status', status) for status in CoordinatorWorkflow.FINISHED_STATUSES])
json_jobs = oozie_api.get_coordinators(**kwargs).jobs
if request.GET.get('type') == 'progress':
json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs]
return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(json_jobs, request.user)).replace('\\\\', '\\'), content_type="application/json")
return render('dashboard/list_oozie_coordinators.mako', request, {
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
'enable_cron_scheduling': enable_cron_scheduling,
})
def apps(self, filters):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get(), 'filters': []}
filters.pop('time')
_filter_oozie_jobs(self.user, filters, kwargs)
jobs = oozie_api.get_coordinators(**kwargs)
return {
'apps':[{
'id': app['id'],
'name': app['appName'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'type': 'schedule',
'user': app['user'],
'progress': app['progress'],
'queue': app['group'],
'duration': app['durationInMillis'],
'submitted': app['lastActionInMillis'] * 1000,
'canWrite': app['canEdit']
} for app in massaged_oozie_jobs_for_json(jobs.jobs, self.user)['jobs']],
'total': jobs.total
}
def list_oozie_workflows(request):
kwargs = {
'cnt': OOZIE_JOBS_COUNT.get(),
}
if not has_dashboard_jobs_access(request.user):
kwargs['user'] = request.user.username
oozie_api = get_oozie(request.user)
if request.GET.get('format') == 'json':
just_sla = request.GET.get('justsla') == 'true'
if request.GET.get('type') in ('running', 'progress'):
kwargs['filters'] = [('status', status)
for status in OozieWorkflow.RUNNING_STATUSES]
elif request.GET.get('type') == 'completed':
kwargs['filters'] = [('status', status)
for status in OozieWorkflow.FINISHED_STATUSES]
json_jobs = oozie_api.get_workflows(**kwargs).jobs
if request.GET.get('type') == 'progress':
json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]
return HttpResponse(encode_json_for_js(
massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla)),
mimetype="application/json")
return render(
'dashboard/list_oozie_workflows.mako', request, {
'user': request.user,
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
})
def _filter_oozie_jobs(user, filters, kwargs):
text_filters = _extract_query_params(filters)
if has_oozie_installed and not has_dashboard_jobs_access(user):
kwargs['filters'].append(('user', user.username))
elif 'username' in text_filters:
kwargs['filters'].append(('user', text_filters['username']))
if 'time' in filters:
kwargs['filters'].extend([
('startcreatedtime', '-%s%s' %
(filters['time']['time_value'], filters['time']['time_unit'][:1]))
])
if has_oozie_installed and ENABLE_OOZIE_BACKEND_FILTERING.get(
) and text_filters.get('text'):
kwargs['filters'].extend([('text', text_filters.get('text'))])
if filters['pagination']:
kwargs['offset'] = filters['pagination']['offset']
kwargs['cnt'] = min(
filters['pagination']['limit'],
hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get())
if filters.get('states'):
states_filters = {
'running': ['RUNNING', 'PREP', 'SUSPENDED'],
'completed': ['SUCCEEDED'],
'failed': ['FAILED', 'KILLED'],
}
for _state in filters.get('states'):
for _status in states_filters[_state]:
kwargs['filters'].extend([('status', _status)])
def list_oozie_bundles(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
if not has_dashboard_jobs_access(request.user):
kwargs['filters'].append(('user', request.user.username))
oozie_api = get_oozie(request.user)
if request.GET.get('format') == 'json':
if request.GET.get('offset'):
kwargs['offset'] = request.GET.get('offset')
json_jobs = []
total_jobs = 0
if request.GET.getlist('status'):
kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
bundle_list = oozie_api.get_bundles(**kwargs)
json_jobs = bundle_list.jobs
total_jobs = bundle_list.total
if request.GET.get('type') == 'progress':
json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs]
response = massaged_oozie_jobs_for_json(json_jobs, request.user)
response['total_jobs'] = total_jobs
return JsonResponse(response, encoder=JSONEncoderForHTML)
return render('dashboard/list_oozie_bundles.mako', request, {
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
})
def list_oozie_workflows(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
if not has_dashboard_jobs_access(request.user):
kwargs['filters'].append(('user', request.user.username))
oozie_api = get_oozie(request.user)
if request.GET.get('format') == 'json':
just_sla = request.GET.get('justsla') == 'true'
if request.GET.get('startcreatedtime'):
kwargs['filters'].extend([('startcreatedtime', request.GET.get('startcreatedtime'))])
if request.GET.get('offset'):
kwargs['offset'] = request.GET.get('offset')
json_jobs = []
total_jobs = 0
if request.GET.getlist('status'):
kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
wf_list = oozie_api.get_workflows(**kwargs)
json_jobs = wf_list.jobs
total_jobs = wf_list.total
if request.GET.get('type') == 'progress':
json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]
response = massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla)
response['total_jobs'] = total_jobs
return JsonResponse(response, encoder=JSONEncoderForHTML)
return render('dashboard/list_oozie_workflows.mako', request, {
'user': request.user,
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
})
def list_oozie_workflows(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
if not has_dashboard_jobs_access(request.user):
kwargs['filters'].append(('user', request.user.username))
oozie_api = get_oozie(request.user)
if request.GET.get('format') == 'json':
just_sla = request.GET.get('justsla') == 'true'
if request.GET.get('startcreatedtime'):
kwargs['filters'].extend([('startcreatedtime', request.GET.get('startcreatedtime'))])
if request.GET.get('offset'):
kwargs['offset'] = request.GET.get('offset')
json_jobs = []
total_jobs = 0
if request.GET.getlist('status'):
kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
wf_list = oozie_api.get_workflows(**kwargs)
json_jobs = wf_list.jobs
total_jobs = wf_list.total
if request.GET.get('type') == 'progress':
json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]
response = massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla)
response['total_jobs'] = total_jobs
return JsonResponse(response, encoder=JSONEncoderForHTML)
return render('dashboard/list_oozie_workflows.mako', request, {
'user': request.user,
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
})
def list_oozie_bundles(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
if not has_dashboard_jobs_access(request.user):
kwargs['filters'].append(('user', request.user.username))
oozie_api = get_oozie(request.user)
if request.GET.get('format') == 'json':
if request.GET.get('offset'):
kwargs['offset'] = request.GET.get('offset')
if request.GET.get('text') and ENABLE_OOZIE_BACKEND_FILTERING.get():
kwargs['filters'].extend([('text', request.GET.get('text'))])
json_jobs = []
total_jobs = 0
if request.GET.getlist('status'):
kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
bundle_list = oozie_api.get_bundles(**kwargs)
json_jobs = bundle_list.jobs
total_jobs = bundle_list.total
if request.GET.get('type') == 'progress':
json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs]
response = massaged_oozie_jobs_for_json(json_jobs, request.user)
response['total_jobs'] = total_jobs
return JsonResponse(response, encoder=JSONEncoderForHTML)
return render('dashboard/list_oozie_bundles.mako', request, {
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
})
def list_oozie_coordinators(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
if not has_dashboard_jobs_access(request.user):
kwargs['filters'].append(('user', request.user.username))
oozie_api = get_oozie(request.user)
enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()
if request.GET.get('format') == 'json':
if request.GET.get('offset'):
kwargs['offset'] = request.GET.get('offset')
if request.GET.get('text') and ENABLE_OOZIE_BACKEND_FILTERING.get():
kwargs['filters'].extend([('text', request.GET.get('text'))])
json_jobs = []
total_jobs = 0
if request.GET.getlist('status'):
kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
co_list = oozie_api.get_coordinators(**kwargs)
json_jobs = co_list.jobs
total_jobs = co_list.total
if request.GET.get('type') == 'progress':
json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs]
response = massaged_oozie_jobs_for_json(json_jobs, request.user)
response['total_jobs'] = total_jobs
return JsonResponse(response, encoder=JSONEncoderForHTML)
return render('dashboard/list_oozie_coordinators.mako', request, {
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
'enable_cron_scheduling': enable_cron_scheduling,
})
def list_oozie_bundles(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
if not has_dashboard_jobs_access(request.user):
kwargs['filters'].append(('user', request.user.username))
oozie_api = get_oozie(request.user)
if request.GET.get('format') == 'json':
if request.GET.get('type') in ('running', 'progress'):
kwargs['filters'].extend([
('status', status)
for status in BundleWorkflow.RUNNING_STATUSES
])
elif request.GET.get('type') == 'completed':
kwargs['filters'].extend([
('status', status)
for status in BundleWorkflow.FINISHED_STATUSES
])
json_jobs = oozie_api.get_bundles(**kwargs).jobs
if request.GET.get('type') == 'progress':
json_jobs = [oozie_api.get_bundle(job.id) for job in json_jobs]
return HttpResponse(json.dumps(
massaged_oozie_jobs_for_json(json_jobs,
request.user)).replace('\\\\', '\\'),
content_type="application/json")
return render('dashboard/list_oozie_bundles.mako', request, {
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
})
def list_oozie_workflows(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
if not has_dashboard_jobs_access(request.user):
kwargs['filters'].append(('user', request.user.username))
oozie_api = get_oozie(request.user)
if request.GET.get('format') == 'json':
just_sla = request.GET.get('justsla') == 'true'
if request.GET.get('type') in ('running', 'progress'):
kwargs['filters'].extend([
('status', status) for status in OozieWorkflow.RUNNING_STATUSES
])
elif request.GET.get('type') == 'completed':
kwargs['filters'].extend([
('status', status)
for status in OozieWorkflow.FINISHED_STATUSES
])
json_jobs = oozie_api.get_workflows(**kwargs).jobs
if request.GET.get('type') == 'progress':
json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]
return JsonResponse(massaged_oozie_jobs_for_json(
json_jobs, request.user, just_sla),
encoder=JSONEncoderForHTML)
return render(
'dashboard/list_oozie_workflows.mako', request, {
'user': request.user,
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
})
def apps(self, filters):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get(), 'filters': []}
filters.pop('time')
_filter_oozie_jobs(self.user, filters, kwargs)
jobs = oozie_api.get_coordinators(**kwargs)
return {
'apps':[{
'id': app['id'],
'name': app['appName'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'type': 'schedule',
'user': app['user'],
'progress': app['progress'],
'queue': app['group'],
'duration': app['durationInMillis'],
'submitted': app['lastActionInMillis'] * 1000,
'canWrite': app['canEdit']
} for app in massaged_oozie_jobs_for_json(jobs.jobs, self.user)['jobs']],
'total': jobs.total
}
def apps(self, filters):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
_filter_oozie_jobs(self.user, filters, kwargs)
wf_list = oozie_api.get_workflows(**kwargs)
return {
'apps': [{
'id': app['id'],
'name': app['appName'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'type': 'workflow',
'user': app['user'],
'progress': app['progress'],
'queue': app['group'],
'duration': app['durationInMillis'],
'submitted': app['startTimeInMillis'] * 1000,
'canWrite': app['canEdit']
}
for app in massaged_oozie_jobs_for_json(
wf_list.jobs, self.user)['jobs']],
'total':
wf_list.total
}
def apps(self, filters):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
text_filters = _extract_query_params(filters)
if not has_dashboard_jobs_access(self.user):
kwargs['filters'].append(('user', self.user.username))
elif 'user' in text_filters:
kwargs['filters'].append(('user', text_filters['username']))
if 'time' in filters:
kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))])
if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
kwargs['filters'].extend([('text', text_filters.get('text'))])
if filters['pagination']:
kwargs['offset'] = filters['pagination']['offset']
kwargs['cnt'] = min(filters['pagination']['limit'], OOZIE_JOBS_COUNT.get())
if filters.get('states'):
states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
for _state in filters.get('states'):
for _status in states_filters[_state]:
kwargs['filters'].extend([('status', _status)])
oozie_api = get_oozie(self.user)
wf_list = oozie_api.get_workflows(**kwargs)
return {
'apps':[{
'id': app['id'],
'name': app['appName'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'type': 'workflow',
'user': app['user'],
'progress': app['progress'],
'duration': app['durationInMillis'],
'submitted': app['startTimeInMillis']
} for app in massaged_oozie_jobs_for_json(wf_list.jobs, self.user)['jobs']],
'total': wf_list.total
}
def apps(self, filters):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
text_filters = _extract_query_params(filters)
if not has_dashboard_jobs_access(self.user):
kwargs['filters'].append(('user', self.user.username))
elif 'user' in text_filters:
kwargs['filters'].append(('user', text_filters['username']))
if 'time' in filters:
kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))])
if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
kwargs['filters'].extend([('text', text_filters.get('text'))])
if filters['pagination']:
kwargs['offset'] = filters['pagination']['offset']
kwargs['cnt'] = min(filters['pagination']['limit'], OOZIE_JOBS_COUNT.get())
if filters.get('states'):
states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
for _state in filters.get('states'):
for _status in states_filters[_state]:
kwargs['filters'].extend([('status', _status)])
oozie_api = get_oozie(self.user)
wf_list = oozie_api.get_workflows(**kwargs)
return {
'apps':[{
'id': app['id'],
'name': app['appName'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'type': 'workflow',
'user': app['user'],
'progress': app['progress'],
'duration': app['durationInMillis'],
'submitted': app['startTimeInMillis']
} for app in massaged_oozie_jobs_for_json(wf_list.jobs, self.user)['jobs']],
'total': wf_list.total
}
def list_oozie_coordinators(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
if not request.user.is_superuser:
kwargs['user'] = request.user.username
coordinators = get_oozie().get_coordinators(**kwargs)
return render('dashboard/list_oozie_coordinators.mako', request, {
'jobs': split_oozie_jobs(coordinators.jobs),
})
def list_oozie_workflows(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
if not request.user.is_superuser:
kwargs['user'] = request.user.username
workflows = get_oozie().get_workflows(**kwargs)
return render('dashboard/list_oozie_workflows.mako', request, {
'user': request.user,
'jobs': split_oozie_jobs(workflows.jobs),
})
def list_oozie_coordinators(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
if not has_dashboard_jobs_access(request.user):
kwargs['user'] = request.user.username
coordinators = get_oozie().get_coordinators(**kwargs)
return render('dashboard/list_oozie_coordinators.mako', request, {
'jobs': split_oozie_jobs(coordinators.jobs),
'has_job_edition_permission': has_job_edition_permission,
})
def list_oozie_coordinators(request):
kwargs = {
'cnt': OOZIE_JOBS_COUNT.get(),
}
if not has_dashboard_jobs_access(request.user):
kwargs['user'] = request.user.username
coordinators = get_oozie().get_coordinators(**kwargs)
return render(
'dashboard/list_oozie_coordinators.mako', request, {
'jobs': split_oozie_jobs(coordinators.jobs),
'has_job_edition_permission': has_job_edition_permission,
})
def apps(self):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
wf_list = oozie_api.get_coordinators(**kwargs)
return [{
'id': app.id,
'name': app.appName,
'status': app.status,
'type': 'coordinator',
'user': app.user,
'progress': 100,
'duration': 10 * 3600,
'submitted': 10 * 3600
} for app in wf_list.jobs if app.appName.startswith(OozieApi.SCHEDULE_JOB_PREFIX)]
def apps(self):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
wf_list = oozie_api.get_coordinators(**kwargs)
return [{
'id': app.id,
'name': app.appName,
'status': app.status,
'type': 'coordinator',
'user': app.user,
'progress': 100,
'duration': 10 * 3600,
'submitted': 10 * 3600
} for app in wf_list.jobs]
def apps(self, filters):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
wf_list = oozie_api.get_bundles(**kwargs)
return [{
'id': app.id,
'name': app.appName,
'status': app.status,
'type': 'bundle',
'user': app.user,
'progress': 100,
'duration': 10 * 3600,
'submitted': 10 * 3600
} for app in wf_list.jobs]
def list_oozie_workflows(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
if not has_dashboard_jobs_access(request.user):
kwargs['user'] = request.user.username
workflows = get_oozie().get_workflows(**kwargs)
if request.GET.get('format') == 'json':
return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(workflows.jobs, request.user)), mimetype="application/json")
return render('dashboard/list_oozie_workflows.mako', request, {
'user': request.user,
'jobs': split_oozie_jobs(workflows.jobs),
'has_job_edition_permission': has_job_edition_permission,
})
def apps(self, filters):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': filters}
wf_list = oozie_api.get_workflows(**kwargs)
return [{
'id': app.id,
'name': app.appName,
'status': app.status,
'apiStatus': self._api_status(app.status),
'type': 'workflow',
'user': app.user,
'progress': app.get_progress(),
'duration': 10 * 3600,
'submitted': 10 * 3600
} for app in wf_list.jobs]
def apps(self, filters):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': filters}
wf_list = oozie_api.get_workflows(**kwargs)
return [{
'id': app.id,
'name': app.appName,
'status': app.status,
'apiStatus': self._api_status(app.status),
'type': 'workflow',
'user': app.user,
'progress': app.get_progress(),
'duration': 10 * 3600,
'submitted': 10 * 3600
} for app in wf_list.jobs]
def apps(self, filters):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
text_filters = _extract_query_params(filters)
if not has_dashboard_jobs_access(self.user):
kwargs['filters'].append(('user', self.user.username))
elif 'user' in text_filters:
kwargs['filters'].append(('user', text_filters['username']))
if 'time' in filters:
kwargs['filters'].extend([
('startcreatedtime',
'-%s%s' % (filters['time']['time_value'],
filters['time']['time_unit'][:1]))
])
if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
kwargs['filters'].extend([('text', text_filters.get('text'))])
if filters.get('states'):
states_filters = {
'running': ['RUNNING', 'PREP', 'SUSPENDED'],
'completed': ['SUCCEEDED'],
'failed': ['FAILED', 'KILLED'],
}
for _state in filters.get('states'):
for _status in states_filters[_state]:
kwargs['filters'].extend([('status', _status)])
oozie_api = get_oozie(self.user)
wf_list = oozie_api.get_workflows(**kwargs)
return [{
'id': app.id,
'name': app.appName,
'status': app.status,
'apiStatus': self._api_status(app.status),
'type': 'workflow',
'user': app.user,
'progress': app.get_progress(),
'duration': 10 * 3600,
'submitted': 10 * 3600
} for app in wf_list.jobs]
def apps(self, filters):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
jobs = oozie_api.get_coordinators(**kwargs)
return {
'apps':[{
'id': app['id'],
'name': app['appName'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'type': 'schedule',
'user': app['user'],
'progress': app['progress'],
'duration': app['durationInMillis'],
'submitted': app['startTimeInMillis']
} for app in massaged_oozie_jobs_for_json(jobs.jobs, self.user)['jobs']],
'total': jobs.total
}
def apps(self):
oozie_api = get_oozie(self.user)
kwargs = {"cnt": OOZIE_JOBS_COUNT.get(), "filters": []}
wf_list = oozie_api.get_workflows(**kwargs)
return [
{
"id": app.id,
"name": app.appName,
"status": app.status,
"type": "workflow",
"user": app.user,
"progress": 100,
"duration": 10 * 3600,
"submitted": 10 * 3600,
}
for app in wf_list.jobs
if app.appName.startswith(OozieApi.BATCH_JOB_PREFIX)
]
def apps(self, filters):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
jobs = oozie_api.get_coordinators(**kwargs)
return {
'apps':[{
'id': app.id,
'name': app.appName,
'status': app.status,
'apiStatus': self._api_status(app.status),
'type': 'schedule',
'user': app.user,
'progress': app.get_progress(),
'duration': 10 * 3600,
'submitted': 10 * 3600
} for app in jobs.jobs],
'total': jobs.total
}
def list_oozie_bundles(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
if not has_dashboard_jobs_access(request.user):
kwargs['user'] = request.user.username
bundles = get_oozie().get_bundles(**kwargs)
if request.GET.get('format') == 'json':
json_jobs = bundles.jobs
if request.GET.get('type') == 'running':
json_jobs = split_oozie_jobs(bundles.jobs)['running_jobs']
if request.GET.get('type') == 'completed':
json_jobs = split_oozie_jobs(bundles.jobs)['completed_jobs']
return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(json_jobs, request.user)).replace('\\\\', '\\'), mimetype="application/json")
return render('dashboard/list_oozie_bundles.mako', request, {
'jobs': split_oozie_jobs(bundles.jobs),
'has_job_edition_permission': has_job_edition_permission,
})
def list_oozie_coordinators(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
if not has_dashboard_jobs_access(request.user):
kwargs['user'] = request.user.username
coordinators = get_oozie(request.user).get_coordinators(**kwargs)
enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()
if request.GET.get('format') == 'json':
json_jobs = coordinators.jobs
if request.GET.get('type') == 'running':
json_jobs = split_oozie_jobs(request.user, coordinators.jobs)['running_jobs']
if request.GET.get('type') == 'completed':
json_jobs = split_oozie_jobs(request.user, coordinators.jobs)['completed_jobs']
return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(json_jobs, request.user)).replace('\\\\', '\\'), mimetype="application/json")
return render('dashboard/list_oozie_coordinators.mako', request, {
'jobs': split_oozie_jobs(request.user, coordinators.jobs),
'has_job_edition_permission': has_job_edition_permission,
'enable_cron_scheduling': enable_cron_scheduling,
})
def list_oozie_workflows(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
if not has_dashboard_jobs_access(request.user):
kwargs['user'] = request.user.username
workflows = get_oozie(request.user).get_workflows(**kwargs)
if request.GET.get('format') == 'json':
json_jobs = workflows.jobs
just_sla = request.GET.get('justsla') == 'true'
if request.GET.get('type') == 'running':
json_jobs = split_oozie_jobs(request.user, workflows.jobs)['running_jobs']
if request.GET.get('type') == 'completed':
json_jobs = split_oozie_jobs(request.user, workflows.jobs)['completed_jobs']
return HttpResponse(encode_json_for_js(massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla)), mimetype="application/json")
return render('dashboard/list_oozie_workflows.mako', request, {
'user': request.user,
'jobs': split_oozie_jobs(request.user, workflows.jobs),
'has_job_edition_permission': has_job_edition_permission,
})
def apps(self, filters):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
jobs = oozie_api.get_coordinators(**kwargs)
return {
'apps': [{
'id': app['id'],
'name': app['appName'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'type': 'schedule',
'user': app['user'],
'progress': app['progress'],
'duration': app['durationInMillis'],
'submitted': app['startTimeInMillis']
}
for app in massaged_oozie_jobs_for_json(
jobs.jobs, self.user)['jobs']],
'total':
jobs.total
}
def list_oozie_coordinators(request):
kwargs = {
'cnt': OOZIE_JOBS_COUNT.get(),
}
if not has_dashboard_jobs_access(request.user):
kwargs['user'] = request.user.username
oozie_api = get_oozie(request.user)
enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()
if request.GET.get('format') == 'json':
if request.GET.get('type') in ('running', 'progress'):
kwargs['filters'] = [
('status', status)
for status in CoordinatorWorkflow.RUNNING_STATUSES
]
elif request.GET.get('type') == 'completed':
kwargs['filters'] = [
('status', status)
for status in CoordinatorWorkflow.FINISHED_STATUSES
]
json_jobs = oozie_api.get_coordinators(**kwargs).jobs
if request.GET.get('type') == 'progress':
json_jobs = [
oozie_api.get_coordinator(job.id) for job in json_jobs
]
return HttpResponse(json.dumps(
massaged_oozie_jobs_for_json(json_jobs,
request.user)).replace('\\\\', '\\'),
mimetype="application/json")
return render(
'dashboard/list_oozie_coordinators.mako', request, {
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
'enable_cron_scheduling': enable_cron_scheduling,
})
def list_oozie_bundles(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
if not has_dashboard_jobs_access(request.user):
kwargs['user'] = request.user.username
oozie_api = get_oozie(request.user)
if request.GET.get('format') == 'json':
if request.GET.get('type') in ('running', 'progress'):
kwargs['filters'] = [('status', status) for status in BundleWorkflow.RUNNING_STATUSES]
elif request.GET.get('type') == 'completed':
kwargs['filters'] = [('status', status) for status in BundleWorkflow.FINISHED_STATUSES]
json_jobs = oozie_api.get_bundles(**kwargs).jobs
if request.GET.get('type') == 'progress':
json_jobs = [oozie_api.get_bundle(job.id) for job in json_jobs]
return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(json_jobs, request.user)).replace('\\\\', '\\'), mimetype="application/json")
return render('dashboard/list_oozie_bundles.mako', request, {
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
})
def apps(self, filters):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
_filter_oozie_jobs(self.user, filters, kwargs)
wf_list = oozie_api.get_workflows(**kwargs)
return {
'apps':[{
'id': app['id'],
'name': app['appName'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'type': 'workflow',
'user': app['user'],
'progress': app['progress'],
'queue': app['group'],
'duration': app['durationInMillis'],
'submitted': app['startTimeInMillis'] * 1000,
'canWrite': app['canEdit']
} for app in massaged_oozie_jobs_for_json(wf_list.jobs, self.user)['jobs']],
'total': wf_list.total
}
def list_oozie_workflows(request):
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
if not has_dashboard_jobs_access(request.user):
kwargs['filters'].append(('user', request.user.username))
oozie_api = get_oozie(request.user)
if request.GET.get('format') == 'json':
just_sla = request.GET.get('justsla') == 'true'
if request.GET.get('type') in ('running', 'progress'):
kwargs['filters'].extend([('status', status) for status in OozieWorkflow.RUNNING_STATUSES])
elif request.GET.get('type') == 'completed':
kwargs['filters'].extend([('status', status) for status in OozieWorkflow.FINISHED_STATUSES])
json_jobs = oozie_api.get_workflows(**kwargs).jobs
if request.GET.get('type') == 'progress':
json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]
return JsonResponse(massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla), encoder=JSONEncoderForHTML)
return render('dashboard/list_oozie_workflows.mako', request, {
'user': request.user,
'jobs': [],
'has_job_edition_permission': has_job_edition_permission,
})
def _filter_oozie_jobs(user, filters, kwargs):
text_filters = _extract_query_params(filters)
if has_oozie_installed and not has_dashboard_jobs_access(user):
kwargs['filters'].append(('user', user.username))
elif 'username' in text_filters:
kwargs['filters'].append(('user', text_filters['username']))
if 'time' in filters:
kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))])
if has_oozie_installed and ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
kwargs['filters'].extend([('text', text_filters.get('text'))])
if filters['pagination']:
kwargs['offset'] = filters['pagination']['offset']
kwargs['cnt'] = min(filters['pagination']['limit'], hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get())
if filters.get('states'):
states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
for _state in filters.get('states'):
for _status in states_filters[_state]:
kwargs['filters'].extend([('status', _status)])
def apps(self):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
wf_list = oozie_api.get_coordinators(**kwargs)
return [{'id': app.id, 'status': app.status} for app in wf_list.jobs]
import json
from django.utils.translation import ugettext as _
from jobbrowser.apis.base_api import Api, MockDjangoRequest, _extract_query_params, is_linkable, hdfs_link_js
from liboozie.oozie_api import get_oozie
LOG = logging.getLogger(__name__)
try:
from oozie.forms import ParameterForm
from oozie.conf import OOZIE_JOBS_COUNT, ENABLE_OOZIE_BACKEND_FILTERING
from oozie.views.dashboard import get_oozie_job_log, list_oozie_workflow, manage_oozie_jobs, bulk_manage_oozie_jobs, \
has_dashboard_jobs_access, massaged_oozie_jobs_for_json, has_job_edition_permission
has_oozie_installed = True
OOZIE_JOBS_COUNT_LIMIT = OOZIE_JOBS_COUNT.get()
except Exception as e:
LOG.warn('Some applications are not enabled for Job Browser v2: %s' % e)
has_oozie_installed = False
OOZIE_JOBS_COUNT_LIMIT = 100
class WorkflowApi(Api):
def apps(self, filters):
oozie_api = get_oozie(self.user)
kwargs = {'cnt': OOZIE_JOBS_COUNT_LIMIT, 'filters': []}
_filter_oozie_jobs(self.user, filters, kwargs)
wf_list = oozie_api.get_workflows(**kwargs)