Esempio n. 1
0
def list_oozie_workflows(request):
    kwargs = {
        'cnt': OOZIE_JOBS_COUNT.get(),
    }
    if not has_dashboard_jobs_access(request.user):
        kwargs['user'] = request.user.username

    workflows = get_oozie().get_workflows(**kwargs)

    if request.GET.get('format') == 'json':
        json_jobs = workflows.jobs
        if request.GET.get('type') == 'running':
            json_jobs = split_oozie_jobs(workflows.jobs)['running_jobs']
        if request.GET.get('type') == 'completed':
            json_jobs = split_oozie_jobs(workflows.jobs)['completed_jobs']
        return HttpResponse(encode_json_for_js(
            massaged_oozie_jobs_for_json(json_jobs, request.user)),
                            mimetype="application/json")

    return render(
        'dashboard/list_oozie_workflows.mako', request, {
            'user': request.user,
            'jobs': split_oozie_jobs(workflows.jobs),
            'has_job_edition_permission': has_job_edition_permission,
        })
Esempio n. 2
0
def list_oozie_bundles(request):
    kwargs = {"cnt": OOZIE_JOBS_COUNT.get(), "filters": []}
    if not has_dashboard_jobs_access(request.user):
        kwargs["filters"].append(("user", request.user.username))
    oozie_api = get_oozie(request.user)

    if request.GET.get("format") == "json":
        if request.GET.get("offset"):
            kwargs["offset"] = request.GET.get("offset")

        json_jobs = []
        total_jobs = 0
        if request.GET.getlist("status"):
            kwargs["filters"].extend([("status", status) for status in request.GET.getlist("status")])
            bundle_list = oozie_api.get_bundles(**kwargs)
            json_jobs = bundle_list.jobs
            total_jobs = bundle_list.total

        if request.GET.get("type") == "progress":
            json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs]

        response = massaged_oozie_jobs_for_json(json_jobs, request.user)
        response["total_jobs"] = total_jobs
        return JsonResponse(response, encoder=JSONEncoderForHTML)

    return render(
        "dashboard/list_oozie_bundles.mako",
        request,
        {"jobs": [], "has_job_edition_permission": has_job_edition_permission},
    )
Esempio n. 3
0
def list_oozie_bundles(request):
    kwargs = {
        'cnt': OOZIE_JOBS_COUNT.get(),
    }
    if not has_dashboard_jobs_access(request.user):
        kwargs['user'] = request.user.username

    bundles = get_oozie().get_bundles(**kwargs)

    if request.GET.get('format') == 'json':
        json_jobs = bundles.jobs
        if request.GET.get('type') == 'running':
            json_jobs = split_oozie_jobs(bundles.jobs)['running_jobs']
        if request.GET.get('type') == 'completed':
            json_jobs = split_oozie_jobs(bundles.jobs)['completed_jobs']
        return HttpResponse(json.dumps(
            massaged_oozie_jobs_for_json(json_jobs,
                                         request.user)).replace('\\\\', '\\'),
                            mimetype="application/json")

    return render(
        'dashboard/list_oozie_bundles.mako', request, {
            'jobs': split_oozie_jobs(bundles.jobs),
            'has_job_edition_permission': has_job_edition_permission,
        })
Esempio n. 4
0
def list_oozie_workflows(request):
    kwargs = {"cnt": OOZIE_JOBS_COUNT.get(), "filters": []}
    if not has_dashboard_jobs_access(request.user):
        kwargs["filters"].append(("user", request.user.username))
    oozie_api = get_oozie(request.user)

    if request.GET.get("format") == "json":
        just_sla = request.GET.get("justsla") == "true"

        if request.GET.get("startcreatedtime"):
            kwargs["filters"].extend([("startcreatedtime", request.GET.get("startcreatedtime"))])

        if request.GET.get("offset"):
            kwargs["offset"] = request.GET.get("offset")

        json_jobs = []
        total_jobs = 0
        if request.GET.getlist("status"):
            kwargs["filters"].extend([("status", status) for status in request.GET.getlist("status")])
            wf_list = oozie_api.get_workflows(**kwargs)
            json_jobs = wf_list.jobs
            total_jobs = wf_list.total

        if request.GET.get("type") == "progress":
            json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]

        response = massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla)
        response["total_jobs"] = total_jobs
        return JsonResponse(response, encoder=JSONEncoderForHTML)

    return render(
        "dashboard/list_oozie_workflows.mako",
        request,
        {"user": request.user, "jobs": [], "has_job_edition_permission": has_job_edition_permission},
    )
Esempio n. 5
0
def list_oozie_coordinators(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
  if not has_dashboard_jobs_access(request.user):
    kwargs['filters'].append(('user', request.user.username))
  oozie_api = get_oozie(request.user)

  enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()

  if request.GET.get('format') == 'json':
    if request.GET.get('type') in ('running', 'progress'):
      kwargs['filters'].extend([('status', status) for status in CoordinatorWorkflow.RUNNING_STATUSES])
    elif request.GET.get('type') == 'completed':
      kwargs['filters'].extend([('status', status) for status in CoordinatorWorkflow.FINISHED_STATUSES])

    json_jobs = oozie_api.get_coordinators(**kwargs).jobs
    if request.GET.get('type') == 'progress':
      json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs]

    return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(json_jobs, request.user)).replace('\\\\', '\\'), content_type="application/json")

  return render('dashboard/list_oozie_coordinators.mako', request, {
    'jobs': [],
    'has_job_edition_permission': has_job_edition_permission,
    'enable_cron_scheduling': enable_cron_scheduling,
  })
Esempio n. 6
0
  def apps(self, filters):
    oozie_api = get_oozie(self.user)
    kwargs = {'cnt': hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get(), 'filters': []}

    filters.pop('time')

    _filter_oozie_jobs(self.user, filters, kwargs)

    jobs = oozie_api.get_coordinators(**kwargs)

    return {
      'apps':[{
        'id': app['id'],
        'name': app['appName'],
        'status': app['status'],
        'apiStatus': self._api_status(app['status']),
        'type': 'schedule',
        'user': app['user'],
        'progress': app['progress'],
        'queue': app['group'],
        'duration': app['durationInMillis'],
        'submitted': app['lastActionInMillis'] * 1000,
        'canWrite': app['canEdit']
      } for app in massaged_oozie_jobs_for_json(jobs.jobs, self.user)['jobs']],
      'total': jobs.total
    }
Esempio n. 7
0
def list_oozie_workflows(request):
    kwargs = {
        'cnt': OOZIE_JOBS_COUNT.get(),
    }
    if not has_dashboard_jobs_access(request.user):
        kwargs['user'] = request.user.username
    oozie_api = get_oozie(request.user)

    if request.GET.get('format') == 'json':
        just_sla = request.GET.get('justsla') == 'true'
        if request.GET.get('type') in ('running', 'progress'):
            kwargs['filters'] = [('status', status)
                                 for status in OozieWorkflow.RUNNING_STATUSES]
        elif request.GET.get('type') == 'completed':
            kwargs['filters'] = [('status', status)
                                 for status in OozieWorkflow.FINISHED_STATUSES]

        json_jobs = oozie_api.get_workflows(**kwargs).jobs
        if request.GET.get('type') == 'progress':
            json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]

        return HttpResponse(encode_json_for_js(
            massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla)),
                            mimetype="application/json")

    return render(
        'dashboard/list_oozie_workflows.mako', request, {
            'user': request.user,
            'jobs': [],
            'has_job_edition_permission': has_job_edition_permission,
        })
Esempio n. 8
0
def _filter_oozie_jobs(user, filters, kwargs):
    text_filters = _extract_query_params(filters)

    if has_oozie_installed and not has_dashboard_jobs_access(user):
        kwargs['filters'].append(('user', user.username))
    elif 'username' in text_filters:
        kwargs['filters'].append(('user', text_filters['username']))

    if 'time' in filters:
        kwargs['filters'].extend([
            ('startcreatedtime', '-%s%s' %
             (filters['time']['time_value'], filters['time']['time_unit'][:1]))
        ])

    if has_oozie_installed and ENABLE_OOZIE_BACKEND_FILTERING.get(
    ) and text_filters.get('text'):
        kwargs['filters'].extend([('text', text_filters.get('text'))])

    if filters['pagination']:
        kwargs['offset'] = filters['pagination']['offset']
        kwargs['cnt'] = min(
            filters['pagination']['limit'],
            hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get())

    if filters.get('states'):
        states_filters = {
            'running': ['RUNNING', 'PREP', 'SUSPENDED'],
            'completed': ['SUCCEEDED'],
            'failed': ['FAILED', 'KILLED'],
        }
        for _state in filters.get('states'):
            for _status in states_filters[_state]:
                kwargs['filters'].extend([('status', _status)])
Esempio n. 9
0
def list_oozie_bundles(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
  if not has_dashboard_jobs_access(request.user):
    kwargs['filters'].append(('user', request.user.username))
  oozie_api = get_oozie(request.user)

  if request.GET.get('format') == 'json':
    if request.GET.get('offset'):
      kwargs['offset'] = request.GET.get('offset')

    json_jobs = []
    total_jobs = 0
    if request.GET.getlist('status'):
      kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
      bundle_list = oozie_api.get_bundles(**kwargs)
      json_jobs = bundle_list.jobs
      total_jobs = bundle_list.total

    if request.GET.get('type') == 'progress':
      json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs]

    response = massaged_oozie_jobs_for_json(json_jobs, request.user)
    response['total_jobs'] = total_jobs
    return JsonResponse(response, encoder=JSONEncoderForHTML)


  return render('dashboard/list_oozie_bundles.mako', request, {
    'jobs': [],
    'has_job_edition_permission': has_job_edition_permission,
  })
Esempio n. 10
0
def list_oozie_workflows(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
  if not has_dashboard_jobs_access(request.user):
    kwargs['filters'].append(('user', request.user.username))
  oozie_api = get_oozie(request.user)

  if request.GET.get('format') == 'json':
    just_sla = request.GET.get('justsla') == 'true'

    if request.GET.get('startcreatedtime'):
      kwargs['filters'].extend([('startcreatedtime', request.GET.get('startcreatedtime'))])

    if request.GET.get('offset'):
      kwargs['offset'] = request.GET.get('offset')

    json_jobs = []
    total_jobs = 0
    if request.GET.getlist('status'):
      kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
      wf_list = oozie_api.get_workflows(**kwargs)
      json_jobs = wf_list.jobs
      total_jobs = wf_list.total

    if request.GET.get('type') == 'progress':
      json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]

    response = massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla)
    response['total_jobs'] = total_jobs
    return JsonResponse(response, encoder=JSONEncoderForHTML)

  return render('dashboard/list_oozie_workflows.mako', request, {
    'user': request.user,
    'jobs': [],
    'has_job_edition_permission':  has_job_edition_permission,
  })
Esempio n. 11
0
def list_oozie_workflows(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
  if not has_dashboard_jobs_access(request.user):
    kwargs['filters'].append(('user', request.user.username))
  oozie_api = get_oozie(request.user)

  if request.GET.get('format') == 'json':
    just_sla = request.GET.get('justsla') == 'true'

    if request.GET.get('startcreatedtime'):
      kwargs['filters'].extend([('startcreatedtime', request.GET.get('startcreatedtime'))])

    if request.GET.get('offset'):
      kwargs['offset'] = request.GET.get('offset')

    json_jobs = []
    total_jobs = 0
    if request.GET.getlist('status'):
      kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
      wf_list = oozie_api.get_workflows(**kwargs)
      json_jobs = wf_list.jobs
      total_jobs = wf_list.total

    if request.GET.get('type') == 'progress':
      json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]

    response = massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla)
    response['total_jobs'] = total_jobs
    return JsonResponse(response, encoder=JSONEncoderForHTML)

  return render('dashboard/list_oozie_workflows.mako', request, {
    'user': request.user,
    'jobs': [],
    'has_job_edition_permission':  has_job_edition_permission,
  })
Esempio n. 12
0
def list_oozie_bundles(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
  if not has_dashboard_jobs_access(request.user):
    kwargs['filters'].append(('user', request.user.username))
  oozie_api = get_oozie(request.user)

  if request.GET.get('format') == 'json':
    if request.GET.get('offset'):
      kwargs['offset'] = request.GET.get('offset')

    if request.GET.get('text') and ENABLE_OOZIE_BACKEND_FILTERING.get():
      kwargs['filters'].extend([('text', request.GET.get('text'))])

    json_jobs = []
    total_jobs = 0
    if request.GET.getlist('status'):
      kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
      bundle_list = oozie_api.get_bundles(**kwargs)
      json_jobs = bundle_list.jobs
      total_jobs = bundle_list.total

    if request.GET.get('type') == 'progress':
      json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs]

    response = massaged_oozie_jobs_for_json(json_jobs, request.user)
    response['total_jobs'] = total_jobs
    return JsonResponse(response, encoder=JSONEncoderForHTML)


  return render('dashboard/list_oozie_bundles.mako', request, {
    'jobs': [],
    'has_job_edition_permission': has_job_edition_permission,
  })
Esempio n. 13
0
def list_oozie_coordinators(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
  if not has_dashboard_jobs_access(request.user):
    kwargs['filters'].append(('user', request.user.username))
  oozie_api = get_oozie(request.user)

  enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()

  if request.GET.get('format') == 'json':
    if request.GET.get('offset'):
      kwargs['offset'] = request.GET.get('offset')

    if request.GET.get('text') and ENABLE_OOZIE_BACKEND_FILTERING.get():
      kwargs['filters'].extend([('text', request.GET.get('text'))])

    json_jobs = []
    total_jobs = 0
    if request.GET.getlist('status'):
      kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')])
      co_list = oozie_api.get_coordinators(**kwargs)
      json_jobs = co_list.jobs
      total_jobs = co_list.total

    if request.GET.get('type') == 'progress':
      json_jobs = [oozie_api.get_coordinator(job.id) for job in json_jobs]

    response = massaged_oozie_jobs_for_json(json_jobs, request.user)
    response['total_jobs'] = total_jobs
    return JsonResponse(response, encoder=JSONEncoderForHTML)

  return render('dashboard/list_oozie_coordinators.mako', request, {
    'jobs': [],
    'has_job_edition_permission': has_job_edition_permission,
    'enable_cron_scheduling': enable_cron_scheduling,
  })
Esempio n. 14
0
def list_oozie_bundles(request):
    kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
    if not has_dashboard_jobs_access(request.user):
        kwargs['filters'].append(('user', request.user.username))
    oozie_api = get_oozie(request.user)

    if request.GET.get('format') == 'json':
        if request.GET.get('type') in ('running', 'progress'):
            kwargs['filters'].extend([
                ('status', status)
                for status in BundleWorkflow.RUNNING_STATUSES
            ])
        elif request.GET.get('type') == 'completed':
            kwargs['filters'].extend([
                ('status', status)
                for status in BundleWorkflow.FINISHED_STATUSES
            ])

        json_jobs = oozie_api.get_bundles(**kwargs).jobs
        if request.GET.get('type') == 'progress':
            json_jobs = [oozie_api.get_bundle(job.id) for job in json_jobs]

        return HttpResponse(json.dumps(
            massaged_oozie_jobs_for_json(json_jobs,
                                         request.user)).replace('\\\\', '\\'),
                            content_type="application/json")

    return render('dashboard/list_oozie_bundles.mako', request, {
        'jobs': [],
        'has_job_edition_permission': has_job_edition_permission,
    })
Esempio n. 15
0
def list_oozie_workflows(request):
    kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
    if not has_dashboard_jobs_access(request.user):
        kwargs['filters'].append(('user', request.user.username))
    oozie_api = get_oozie(request.user)

    if request.GET.get('format') == 'json':
        just_sla = request.GET.get('justsla') == 'true'
        if request.GET.get('type') in ('running', 'progress'):
            kwargs['filters'].extend([
                ('status', status) for status in OozieWorkflow.RUNNING_STATUSES
            ])
        elif request.GET.get('type') == 'completed':
            kwargs['filters'].extend([
                ('status', status)
                for status in OozieWorkflow.FINISHED_STATUSES
            ])

        json_jobs = oozie_api.get_workflows(**kwargs).jobs
        if request.GET.get('type') == 'progress':
            json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]

        return JsonResponse(massaged_oozie_jobs_for_json(
            json_jobs, request.user, just_sla),
                            encoder=JSONEncoderForHTML)

    return render(
        'dashboard/list_oozie_workflows.mako', request, {
            'user': request.user,
            'jobs': [],
            'has_job_edition_permission': has_job_edition_permission,
        })
Esempio n. 16
0
  def apps(self, filters):
    oozie_api = get_oozie(self.user)
    kwargs = {'cnt': hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get(), 'filters': []}

    filters.pop('time')

    _filter_oozie_jobs(self.user, filters, kwargs)

    jobs = oozie_api.get_coordinators(**kwargs)

    return {
      'apps':[{
        'id': app['id'],
        'name': app['appName'],
        'status': app['status'],
        'apiStatus': self._api_status(app['status']),
        'type': 'schedule',
        'user': app['user'],
        'progress': app['progress'],
        'queue': app['group'],
        'duration': app['durationInMillis'],
        'submitted': app['lastActionInMillis'] * 1000,
        'canWrite': app['canEdit']
      } for app in massaged_oozie_jobs_for_json(jobs.jobs, self.user)['jobs']],
      'total': jobs.total
    }
Esempio n. 17
0
    def apps(self, filters):
        oozie_api = get_oozie(self.user)

        kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
        _filter_oozie_jobs(self.user, filters, kwargs)

        wf_list = oozie_api.get_workflows(**kwargs)

        return {
            'apps': [{
                'id': app['id'],
                'name': app['appName'],
                'status': app['status'],
                'apiStatus': self._api_status(app['status']),
                'type': 'workflow',
                'user': app['user'],
                'progress': app['progress'],
                'queue': app['group'],
                'duration': app['durationInMillis'],
                'submitted': app['startTimeInMillis'] * 1000,
                'canWrite': app['canEdit']
            }
                     for app in massaged_oozie_jobs_for_json(
                         wf_list.jobs, self.user)['jobs']],
            'total':
            wf_list.total
        }
Esempio n. 18
0
  def apps(self, filters):
    kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}

    text_filters = _extract_query_params(filters)

    if not has_dashboard_jobs_access(self.user):
      kwargs['filters'].append(('user', self.user.username))
    elif 'user' in text_filters:
      kwargs['filters'].append(('user', text_filters['username']))

    if 'time' in filters:
      kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))])

    if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
      kwargs['filters'].extend([('text', text_filters.get('text'))])

    if filters['pagination']:
      kwargs['offset'] = filters['pagination']['offset']
      kwargs['cnt'] = min(filters['pagination']['limit'], OOZIE_JOBS_COUNT.get())

    if filters.get('states'):
      states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
      for _state in filters.get('states'):
        for _status in states_filters[_state]:
          kwargs['filters'].extend([('status', _status)])

    oozie_api = get_oozie(self.user)

    wf_list = oozie_api.get_workflows(**kwargs)

    return {
      'apps':[{
        'id': app['id'],
        'name': app['appName'],
        'status': app['status'],
        'apiStatus': self._api_status(app['status']),
        'type': 'workflow',
        'user': app['user'],
        'progress': app['progress'],
        'duration': app['durationInMillis'],
        'submitted': app['startTimeInMillis']
      } for app in massaged_oozie_jobs_for_json(wf_list.jobs, self.user)['jobs']],
      'total': wf_list.total
    }
Esempio n. 19
0
  def apps(self, filters):
    kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}

    text_filters = _extract_query_params(filters)

    if not has_dashboard_jobs_access(self.user):
      kwargs['filters'].append(('user', self.user.username))
    elif 'user' in text_filters:
      kwargs['filters'].append(('user', text_filters['username']))

    if 'time' in filters:
      kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))])

    if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
      kwargs['filters'].extend([('text', text_filters.get('text'))])

    if filters['pagination']:
      kwargs['offset'] = filters['pagination']['offset']
      kwargs['cnt'] = min(filters['pagination']['limit'], OOZIE_JOBS_COUNT.get())

    if filters.get('states'):
      states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
      for _state in filters.get('states'):
        for _status in states_filters[_state]:
          kwargs['filters'].extend([('status', _status)])

    oozie_api = get_oozie(self.user)

    wf_list = oozie_api.get_workflows(**kwargs)

    return {
      'apps':[{
        'id': app['id'],
        'name': app['appName'],
        'status': app['status'],
        'apiStatus': self._api_status(app['status']),
        'type': 'workflow',
        'user': app['user'],
        'progress': app['progress'],
        'duration': app['durationInMillis'],
        'submitted': app['startTimeInMillis']
      } for app in massaged_oozie_jobs_for_json(wf_list.jobs, self.user)['jobs']],
      'total': wf_list.total
    }
Esempio n. 20
0
def list_oozie_coordinators(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
  if not request.user.is_superuser:
    kwargs['user'] = request.user.username

  coordinators = get_oozie().get_coordinators(**kwargs)

  return render('dashboard/list_oozie_coordinators.mako', request, {
    'jobs': split_oozie_jobs(coordinators.jobs),
  })
Esempio n. 21
0
def list_oozie_workflows(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
  if not request.user.is_superuser:
    kwargs['user'] = request.user.username

  workflows = get_oozie().get_workflows(**kwargs)

  return render('dashboard/list_oozie_workflows.mako', request, {
    'user': request.user,
    'jobs': split_oozie_jobs(workflows.jobs),
  })
Esempio n. 22
0
def list_oozie_coordinators(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
  if not has_dashboard_jobs_access(request.user):
    kwargs['user'] = request.user.username

  coordinators = get_oozie().get_coordinators(**kwargs)

  return render('dashboard/list_oozie_coordinators.mako', request, {
    'jobs': split_oozie_jobs(coordinators.jobs),
    'has_job_edition_permission': has_job_edition_permission,
  })
Esempio n. 23
0
def list_oozie_coordinators(request):
    kwargs = {
        'cnt': OOZIE_JOBS_COUNT.get(),
    }
    if not has_dashboard_jobs_access(request.user):
        kwargs['user'] = request.user.username

    coordinators = get_oozie().get_coordinators(**kwargs)

    return render(
        'dashboard/list_oozie_coordinators.mako', request, {
            'jobs': split_oozie_jobs(coordinators.jobs),
            'has_job_edition_permission': has_job_edition_permission,
        })
Esempio n. 24
0
  def apps(self):
    oozie_api = get_oozie(self.user)
    kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
    wf_list = oozie_api.get_coordinators(**kwargs)

    return [{
        'id': app.id,
        'name': app.appName,
        'status': app.status,
        'type': 'coordinator',
        'user': app.user,
        'progress': 100,
        'duration': 10 * 3600,
        'submitted': 10 * 3600
    } for app in wf_list.jobs if app.appName.startswith(OozieApi.SCHEDULE_JOB_PREFIX)]
Esempio n. 25
0
    def apps(self):
        oozie_api = get_oozie(self.user)
        kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
        wf_list = oozie_api.get_coordinators(**kwargs)

        return [{
            'id': app.id,
            'name': app.appName,
            'status': app.status,
            'type': 'coordinator',
            'user': app.user,
            'progress': 100,
            'duration': 10 * 3600,
            'submitted': 10 * 3600
        } for app in wf_list.jobs]
Esempio n. 26
0
  def apps(self, filters):
    oozie_api = get_oozie(self.user)
    kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
    wf_list = oozie_api.get_bundles(**kwargs)

    return [{
        'id': app.id,
        'name': app.appName,
        'status': app.status,
        'type': 'bundle',
        'user': app.user,
        'progress': 100,
        'duration': 10 * 3600,
        'submitted': 10 * 3600
    } for app in wf_list.jobs]
Esempio n. 27
0
def list_oozie_workflows(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
  if not has_dashboard_jobs_access(request.user):
    kwargs['user'] = request.user.username

  workflows = get_oozie().get_workflows(**kwargs)

  if request.GET.get('format') == 'json':
    return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(workflows.jobs, request.user)), mimetype="application/json")

  return render('dashboard/list_oozie_workflows.mako', request, {
    'user': request.user,
    'jobs': split_oozie_jobs(workflows.jobs),
    'has_job_edition_permission':  has_job_edition_permission,
  })
Esempio n. 28
0
    def apps(self, filters):
        oozie_api = get_oozie(self.user)
        kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': filters}
        wf_list = oozie_api.get_workflows(**kwargs)

        return [{
            'id': app.id,
            'name': app.appName,
            'status': app.status,
            'apiStatus': self._api_status(app.status),
            'type': 'workflow',
            'user': app.user,
            'progress': app.get_progress(),
            'duration': 10 * 3600,
            'submitted': 10 * 3600
        } for app in wf_list.jobs]
Esempio n. 29
0
  def apps(self, filters):
    oozie_api = get_oozie(self.user)
    kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': filters}
    wf_list = oozie_api.get_workflows(**kwargs)

    return [{
        'id': app.id,
        'name': app.appName,
        'status': app.status,
        'apiStatus': self._api_status(app.status),
        'type': 'workflow',
        'user': app.user,
        'progress': app.get_progress(),
        'duration': 10 * 3600,
        'submitted': 10 * 3600
    } for app in wf_list.jobs]
Esempio n. 30
0
    def apps(self, filters):
        kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}

        text_filters = _extract_query_params(filters)

        if not has_dashboard_jobs_access(self.user):
            kwargs['filters'].append(('user', self.user.username))
        elif 'user' in text_filters:
            kwargs['filters'].append(('user', text_filters['username']))

        if 'time' in filters:
            kwargs['filters'].extend([
                ('startcreatedtime',
                 '-%s%s' % (filters['time']['time_value'],
                            filters['time']['time_unit'][:1]))
            ])

        if ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
            kwargs['filters'].extend([('text', text_filters.get('text'))])

        if filters.get('states'):
            states_filters = {
                'running': ['RUNNING', 'PREP', 'SUSPENDED'],
                'completed': ['SUCCEEDED'],
                'failed': ['FAILED', 'KILLED'],
            }
            for _state in filters.get('states'):
                for _status in states_filters[_state]:
                    kwargs['filters'].extend([('status', _status)])

        oozie_api = get_oozie(self.user)

        wf_list = oozie_api.get_workflows(**kwargs)

        return [{
            'id': app.id,
            'name': app.appName,
            'status': app.status,
            'apiStatus': self._api_status(app.status),
            'type': 'workflow',
            'user': app.user,
            'progress': app.get_progress(),
            'duration': 10 * 3600,
            'submitted': 10 * 3600
        } for app in wf_list.jobs]
Esempio n. 31
0
  def apps(self, filters):
    oozie_api = get_oozie(self.user)
    kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
    jobs = oozie_api.get_coordinators(**kwargs)

    return {
      'apps':[{
        'id': app['id'],
        'name': app['appName'],
        'status': app['status'],
        'apiStatus': self._api_status(app['status']),
        'type': 'schedule',
        'user': app['user'],
        'progress': app['progress'],
        'duration': app['durationInMillis'],
        'submitted': app['startTimeInMillis']
      } for app in massaged_oozie_jobs_for_json(jobs.jobs, self.user)['jobs']],
      'total': jobs.total
    }
Esempio n. 32
0
    def apps(self):
        oozie_api = get_oozie(self.user)
        kwargs = {"cnt": OOZIE_JOBS_COUNT.get(), "filters": []}
        wf_list = oozie_api.get_workflows(**kwargs)

        return [
            {
                "id": app.id,
                "name": app.appName,
                "status": app.status,
                "type": "workflow",
                "user": app.user,
                "progress": 100,
                "duration": 10 * 3600,
                "submitted": 10 * 3600,
            }
            for app in wf_list.jobs
            if app.appName.startswith(OozieApi.BATCH_JOB_PREFIX)
        ]
Esempio n. 33
0
  def apps(self, filters):
    oozie_api = get_oozie(self.user)
    kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
    jobs = oozie_api.get_coordinators(**kwargs)

    return {
      'apps':[{
        'id': app.id,
        'name': app.appName,
        'status': app.status,
        'apiStatus': self._api_status(app.status),
        'type': 'schedule',
        'user': app.user,
        'progress': app.get_progress(),
        'duration': 10 * 3600,
        'submitted': 10 * 3600
      } for app in jobs.jobs],
      'total': jobs.total
    }
Esempio n. 34
0
def list_oozie_bundles(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
  if not has_dashboard_jobs_access(request.user):
    kwargs['user'] = request.user.username

  bundles = get_oozie().get_bundles(**kwargs)

  if request.GET.get('format') == 'json':
    json_jobs = bundles.jobs
    if request.GET.get('type') == 'running':
      json_jobs = split_oozie_jobs(bundles.jobs)['running_jobs']
    if request.GET.get('type') == 'completed':
      json_jobs = split_oozie_jobs(bundles.jobs)['completed_jobs']
    return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(json_jobs, request.user)).replace('\\\\', '\\'), mimetype="application/json")

  return render('dashboard/list_oozie_bundles.mako', request, {
    'jobs': split_oozie_jobs(bundles.jobs),
    'has_job_edition_permission': has_job_edition_permission,
  })
Esempio n. 35
0
def list_oozie_coordinators(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
  if not has_dashboard_jobs_access(request.user):
    kwargs['user'] = request.user.username

  coordinators = get_oozie(request.user).get_coordinators(**kwargs)
  enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()

  if request.GET.get('format') == 'json':
    json_jobs = coordinators.jobs
    if request.GET.get('type') == 'running':
      json_jobs = split_oozie_jobs(request.user, coordinators.jobs)['running_jobs']
    if request.GET.get('type') == 'completed':
      json_jobs = split_oozie_jobs(request.user, coordinators.jobs)['completed_jobs']
    return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(json_jobs, request.user)).replace('\\\\', '\\'), mimetype="application/json")

  return render('dashboard/list_oozie_coordinators.mako', request, {
    'jobs': split_oozie_jobs(request.user, coordinators.jobs),
    'has_job_edition_permission': has_job_edition_permission,
    'enable_cron_scheduling': enable_cron_scheduling,
  })
Esempio n. 36
0
def list_oozie_workflows(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
  if not has_dashboard_jobs_access(request.user):
    kwargs['user'] = request.user.username

  workflows = get_oozie(request.user).get_workflows(**kwargs)

  if request.GET.get('format') == 'json':
    json_jobs = workflows.jobs
    just_sla = request.GET.get('justsla') == 'true'
    if request.GET.get('type') == 'running':
      json_jobs = split_oozie_jobs(request.user, workflows.jobs)['running_jobs']
    if request.GET.get('type') == 'completed':
      json_jobs = split_oozie_jobs(request.user, workflows.jobs)['completed_jobs']
    return HttpResponse(encode_json_for_js(massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla)), mimetype="application/json")

  return render('dashboard/list_oozie_workflows.mako', request, {
    'user': request.user,
    'jobs': split_oozie_jobs(request.user, workflows.jobs),
    'has_job_edition_permission':  has_job_edition_permission,
  })
Esempio n. 37
0
    def apps(self, filters):
        oozie_api = get_oozie(self.user)
        kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
        jobs = oozie_api.get_coordinators(**kwargs)

        return {
            'apps': [{
                'id': app['id'],
                'name': app['appName'],
                'status': app['status'],
                'apiStatus': self._api_status(app['status']),
                'type': 'schedule',
                'user': app['user'],
                'progress': app['progress'],
                'duration': app['durationInMillis'],
                'submitted': app['startTimeInMillis']
            }
                     for app in massaged_oozie_jobs_for_json(
                         jobs.jobs, self.user)['jobs']],
            'total':
            jobs.total
        }
Esempio n. 38
0
def list_oozie_coordinators(request):
    kwargs = {
        'cnt': OOZIE_JOBS_COUNT.get(),
    }
    if not has_dashboard_jobs_access(request.user):
        kwargs['user'] = request.user.username
    oozie_api = get_oozie(request.user)

    enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get()

    if request.GET.get('format') == 'json':
        if request.GET.get('type') in ('running', 'progress'):
            kwargs['filters'] = [
                ('status', status)
                for status in CoordinatorWorkflow.RUNNING_STATUSES
            ]
        elif request.GET.get('type') == 'completed':
            kwargs['filters'] = [
                ('status', status)
                for status in CoordinatorWorkflow.FINISHED_STATUSES
            ]

        json_jobs = oozie_api.get_coordinators(**kwargs).jobs
        if request.GET.get('type') == 'progress':
            json_jobs = [
                oozie_api.get_coordinator(job.id) for job in json_jobs
            ]

        return HttpResponse(json.dumps(
            massaged_oozie_jobs_for_json(json_jobs,
                                         request.user)).replace('\\\\', '\\'),
                            mimetype="application/json")

    return render(
        'dashboard/list_oozie_coordinators.mako', request, {
            'jobs': [],
            'has_job_edition_permission': has_job_edition_permission,
            'enable_cron_scheduling': enable_cron_scheduling,
        })
Esempio n. 39
0
def list_oozie_bundles(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(),}
  if not has_dashboard_jobs_access(request.user):
    kwargs['user'] = request.user.username
  oozie_api = get_oozie(request.user)

  if request.GET.get('format') == 'json':
    if request.GET.get('type') in ('running', 'progress'):
      kwargs['filters'] = [('status', status) for status in BundleWorkflow.RUNNING_STATUSES]
    elif request.GET.get('type') == 'completed':
      kwargs['filters'] = [('status', status) for status in BundleWorkflow.FINISHED_STATUSES]

    json_jobs = oozie_api.get_bundles(**kwargs).jobs
    if request.GET.get('type') == 'progress':
      json_jobs = [oozie_api.get_bundle(job.id) for job in json_jobs]

    return HttpResponse(json.dumps(massaged_oozie_jobs_for_json(json_jobs, request.user)).replace('\\\\', '\\'), mimetype="application/json")

  return render('dashboard/list_oozie_bundles.mako', request, {
    'jobs': [],
    'has_job_edition_permission': has_job_edition_permission,
  })
Esempio n. 40
0
  def apps(self, filters):
    oozie_api = get_oozie(self.user)

    kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
    _filter_oozie_jobs(self.user, filters, kwargs)

    wf_list = oozie_api.get_workflows(**kwargs)

    return {
      'apps':[{
        'id': app['id'],
        'name': app['appName'],
        'status': app['status'],
        'apiStatus': self._api_status(app['status']),
        'type': 'workflow',
        'user': app['user'],
        'progress': app['progress'],
        'queue': app['group'],
        'duration': app['durationInMillis'],
        'submitted': app['startTimeInMillis'] * 1000,
        'canWrite': app['canEdit']
      } for app in massaged_oozie_jobs_for_json(wf_list.jobs, self.user)['jobs']],
      'total': wf_list.total
    }
Esempio n. 41
0
def list_oozie_workflows(request):
  kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
  if not has_dashboard_jobs_access(request.user):
    kwargs['filters'].append(('user', request.user.username))
  oozie_api = get_oozie(request.user)

  if request.GET.get('format') == 'json':
    just_sla = request.GET.get('justsla') == 'true'
    if request.GET.get('type') in ('running', 'progress'):
      kwargs['filters'].extend([('status', status) for status in OozieWorkflow.RUNNING_STATUSES])
    elif request.GET.get('type') == 'completed':
      kwargs['filters'].extend([('status', status) for status in OozieWorkflow.FINISHED_STATUSES])

    json_jobs = oozie_api.get_workflows(**kwargs).jobs
    if request.GET.get('type') == 'progress':
      json_jobs = [oozie_api.get_job(job.id) for job in json_jobs]

    return JsonResponse(massaged_oozie_jobs_for_json(json_jobs, request.user, just_sla), encoder=JSONEncoderForHTML)

  return render('dashboard/list_oozie_workflows.mako', request, {
    'user': request.user,
    'jobs': [],
    'has_job_edition_permission':  has_job_edition_permission,
  })
Esempio n. 42
0
def _filter_oozie_jobs(user, filters, kwargs):
    text_filters = _extract_query_params(filters)

    if has_oozie_installed and not has_dashboard_jobs_access(user):
      kwargs['filters'].append(('user', user.username))
    elif 'username' in text_filters:
      kwargs['filters'].append(('user', text_filters['username']))

    if 'time' in filters:
      kwargs['filters'].extend([('startcreatedtime', '-%s%s' % (filters['time']['time_value'], filters['time']['time_unit'][:1]))])

    if has_oozie_installed and ENABLE_OOZIE_BACKEND_FILTERING.get() and text_filters.get('text'):
      kwargs['filters'].extend([('text', text_filters.get('text'))])

    if filters['pagination']:
      kwargs['offset'] = filters['pagination']['offset']
      kwargs['cnt'] = min(filters['pagination']['limit'], hasattr(OOZIE_JOBS_COUNT, 'get') and OOZIE_JOBS_COUNT.get())

    if filters.get('states'):
      states_filters = {'running': ['RUNNING', 'PREP', 'SUSPENDED'], 'completed': ['SUCCEEDED'], 'failed': ['FAILED', 'KILLED'],}
      for _state in filters.get('states'):
        for _status in states_filters[_state]:
          kwargs['filters'].extend([('status', _status)])
Esempio n. 43
0
    def apps(self):
        oozie_api = get_oozie(self.user)
        kwargs = {'cnt': OOZIE_JOBS_COUNT.get(), 'filters': []}
        wf_list = oozie_api.get_coordinators(**kwargs)

        return [{'id': app.id, 'status': app.status} for app in wf_list.jobs]
Esempio n. 44
0
import json

from django.utils.translation import ugettext as _

from jobbrowser.apis.base_api import Api, MockDjangoRequest, _extract_query_params, is_linkable, hdfs_link_js
from liboozie.oozie_api import get_oozie

LOG = logging.getLogger(__name__)

try:
    from oozie.forms import ParameterForm
    from oozie.conf import OOZIE_JOBS_COUNT, ENABLE_OOZIE_BACKEND_FILTERING
    from oozie.views.dashboard import get_oozie_job_log, list_oozie_workflow, manage_oozie_jobs, bulk_manage_oozie_jobs, \
        has_dashboard_jobs_access, massaged_oozie_jobs_for_json, has_job_edition_permission
    has_oozie_installed = True
    OOZIE_JOBS_COUNT_LIMIT = OOZIE_JOBS_COUNT.get()
except Exception as e:
    LOG.warn('Some applications are not enabled for Job Browser v2: %s' % e)
    has_oozie_installed = False
    OOZIE_JOBS_COUNT_LIMIT = 100


class WorkflowApi(Api):
    def apps(self, filters):
        oozie_api = get_oozie(self.user)

        kwargs = {'cnt': OOZIE_JOBS_COUNT_LIMIT, 'filters': []}
        _filter_oozie_jobs(self.user, filters, kwargs)

        wf_list = oozie_api.get_workflows(**kwargs)