예제 #1
0
파일: editor.py 프로젝트: romainr/hue
def submit_coordinator(request, coordinator):
    ParametersFormSet = formset_factory(ParameterForm, extra=0)

    if request.method == "POST":
        params_form = ParametersFormSet(request.POST)

        if params_form.is_valid():
            mapping = dict([(param["name"], param["value"]) for param in params_form.cleaned_data])
            job_id = _submit_coordinator(request, coordinator, mapping)

            request.info(_("Coordinator submitted."))
            return redirect(reverse("oozie:list_oozie_coordinator", kwargs={"job_id": job_id}))
        else:
            request.error(_("Invalid submission form: %s" % params_form.errors))
    else:
        parameters = coordinator.find_all_parameters()
        initial_params = ParameterForm.get_initial_params(
            dict([(param["name"], param["value"]) for param in parameters])
        )
        params_form = ParametersFormSet(initial=initial_params)

    popup = render(
        "editor/submit_job_popup.mako",
        request,
        {
            "params_form": params_form,
            "action": reverse("oozie:submit_coordinator", kwargs={"coordinator": coordinator.id}),
        },
        force_template=True,
    ).content
    return HttpResponse(json.dumps(popup), mimetype="application/json")
예제 #2
0
파일: editor2.py 프로젝트: 18600597055/hue
def submit_coordinator(request, doc_id):
  coordinator = Coordinator(document=Document2.objects.get(id=doc_id))
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
      mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
      job_id = _submit_coordinator(request, coordinator, mapping)

      request.info(_('Coordinator submitted.'))
      return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = coordinator.find_all_parameters()
    initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor2/submit_job_popup.mako', request, {
                 'params_form': params_form,
                 'name': coordinator.name,
                 'action': reverse('oozie:editor_submit_coordinator',  kwargs={'doc_id': coordinator.id}),
                 'show_dryrun': True
                }, force_template=True).content
  return JsonResponse(popup, safe=False)
예제 #3
0
파일: editor.py 프로젝트: tp0101/hue
def submit_bundle(request, bundle):
    ParametersFormSet = formset_factory(ParameterForm, extra=0)

    if request.method == 'POST':
        params_form = ParametersFormSet(request.POST)

        if params_form.is_valid():
            mapping = dict([(param['name'], param['value'])
                            for param in params_form.cleaned_data])
            job_id = _submit_bundle(request, bundle, mapping)

            request.info(_('Bundle submitted.'))
            return redirect(
                reverse('oozie:list_oozie_bundle', kwargs={'job_id': job_id}))
        else:
            request.error(_('Invalid submission form: %s' %
                            params_form.errors))
    else:
        parameters = bundle.find_all_parameters()
        initial_params = ParameterForm.get_initial_params(
            dict([(param['name'], param['value']) for param in parameters]))
        params_form = ParametersFormSet(initial=initial_params)

    popup = render(
        'editor/submit_job_popup.mako',
        request, {
            'params_form': params_form,
            'action': reverse('oozie:submit_bundle',
                              kwargs={'bundle': bundle.id})
        },
        force_template=True).content
    return HttpResponse(json.dumps(popup), mimetype="application/json")
예제 #4
0
파일: editor2.py 프로젝트: cloudera/hue
def submit_bundle(request, doc_id):
  bundle = Bundle(document=Document2.objects.get(id=doc_id))
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
      job_id = _submit_bundle(request, bundle, mapping)

      jsonify = request.POST.get('format') == 'json'
      if jsonify:
        return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'bundle'}, safe=False)
      else:
        request.info(_('Bundle submitted.'))
        return redirect(reverse('oozie:list_oozie_bundle', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = bundle.find_all_parameters()
    initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor2/submit_job_popup.mako', request, {
                 'params_form': params_form,
                 'name': bundle.name,
                 'action': reverse('oozie:editor_submit_bundle',  kwargs={'doc_id': bundle.id}),
                 'return_json': request.GET.get('format') == 'json',
                 'show_dryrun': False
                }, force_template=True).content
  return JsonResponse(popup, safe=False)
예제 #5
0
def submit_workflow(request, doc_id):
  workflow = Workflow(document=Document2.objects.get(id=doc_id))
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)    

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping)

      request.info(_('Workflow submitted'))
      return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = workflow.find_all_parameters()
    initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
    params_form = ParametersFormSet(initial=initial_params)

    popup = render('editor2/submit_job_popup.mako', request, {
                     'params_form': params_form,
                     'name': workflow.name,
                     'action': reverse('oozie:editor_submit_workflow', kwargs={'doc_id': workflow.id})
                   }, force_template=True).content
    return JsonResponse(popup, safe=False)
예제 #6
0
파일: editor.py 프로젝트: yjkim/hue
def submit_workflow(request, workflow):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      job_id = _submit_workflow(request.user, request.fs, workflow, mapping)

      request.info(_('Workflow submitted'))
      return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = workflow.find_all_parameters()
    initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor/submit_job_popup.mako', request, {
                   'params_form': params_form,
                   'action': reverse('oozie:submit_workflow', kwargs={'workflow': workflow.id})
                 }, force_template=True).content
  return HttpResponse(json.dumps(popup), mimetype="application/json")
예제 #7
0
파일: editor2.py 프로젝트: jacobsy/hue
def submit_coordinator(request, doc_id):
  coordinator = Coordinator(document=Document2.objects.get(id=doc_id))
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
      mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
      job_id = _submit_coordinator(request, coordinator, mapping)

      request.info(_('Coordinator submitted.'))
      return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = coordinator.find_all_parameters()
    initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor2/submit_job_popup.mako', request, {
                 'params_form': params_form,
                 'name': coordinator.name,
                 'action': reverse('oozie:editor_submit_coordinator',  kwargs={'doc_id': coordinator.id}),
                 'show_dryrun': True
                }, force_template=True).content
  return JsonResponse(popup, safe=False)
예제 #8
0
파일: editor.py 프로젝트: bm8836900/hue
def submit_workflow(request, workflow):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping)

      request.info(_('Workflow submitted'))
      return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = workflow.find_all_parameters()
    initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor/submit_job_popup.mako', request, {
                   'params_form': params_form,
                   'action': reverse('oozie:submit_workflow', kwargs={'workflow': workflow.id})
                 }, force_template=True).content
  if not isinstance(popup, str):
    popup = popup.decode('utf-8')
  return JsonResponse(popup, safe=False)
예제 #9
0
def submit_bundle(request, doc_id):
  bundle = Bundle(document=Document2.objects.get(id=doc_id))
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
      job_id = _submit_bundle(request, bundle, mapping)

      jsonify = request.POST.get('format') == 'json'
      if jsonify:
        return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'bundle'}, safe=False)
      else:
        request.info(_('Bundle submitted.'))
        return redirect(reverse('oozie:list_oozie_bundle', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = bundle.find_all_parameters()
    initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor2/submit_job_popup.mako', request, {
                 'params_form': params_form,
                 'name': bundle.name,
                 'action': reverse('oozie:editor_submit_bundle',  kwargs={'doc_id': bundle.id}),
                 'return_json': request.GET.get('format') == 'json',
                 'show_dryrun': False
                }, force_template=True).content
  return JsonResponse(popup, safe=False)
예제 #10
0
def submit_coordinator(request, coordinator):
    ParametersFormSet = formset_factory(ParameterForm, extra=0)

    if request.method == 'POST':
        params_form = ParametersFormSet(request.POST)

        if params_form.is_valid():
            mapping = dict([(param['name'], param['value'])
                            for param in params_form.cleaned_data])
            job_id = _submit_coordinator(request, coordinator, mapping)

            request.info(_('Coordinator submitted.'))
            return redirect(
                reverse('oozie:list_oozie_coordinator',
                        kwargs={'job_id': job_id}))
        else:
            request.error(_('Invalid submission form: %s' %
                            params_form.errors))
    else:
        parameters = coordinator.find_all_parameters()
        initial_params = ParameterForm.get_initial_params(
            dict([(param['name'], param['value']) for param in parameters]))
        params_form = ParametersFormSet(initial=initial_params)

    popup = render('editor/submit_job_popup.mako',
                   request, {
                       'params_form':
                       params_form,
                       'action':
                       reverse('oozie:submit_coordinator',
                               kwargs={'coordinator': coordinator.id})
                   },
                   force_template=True).content
    return JsonResponse(popup, safe=False)
예제 #11
0
def _submit_workflow_helper(request, workflow, submit_action):
    ParametersFormSet = formset_factory(ParameterForm, extra=0)

    if request.method == 'POST':
        cluster = json.loads(request.POST.get('cluster', '{}'))
        params_form = ParametersFormSet(request.POST)

        if params_form.is_valid():
            mapping = dict([(param['name'], param['value'])
                            for param in params_form.cleaned_data])
            mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
            mapping['send_email'] = request.POST.get('email_checkbox') == 'on'
            if '/submit_single_action/' in submit_action:
                mapping['submit_single_action'] = True

            try:
                job_id = _submit_workflow(request.user, request.fs, request.jt,
                                          workflow, mapping)
            except Exception as e:
                raise PopupException(_('Workflow submission failed'),
                                     detail=smart_str(e),
                                     error_code=200)
            jsonify = request.POST.get('format') == 'json'
            if jsonify:
                return JsonResponse(
                    {
                        'status': 0,
                        'job_id': job_id,
                        'type': 'workflow'
                    },
                    safe=False)
            else:
                request.info(_('Workflow submitted'))
                return redirect(
                    reverse('oozie:list_oozie_workflow',
                            kwargs={'job_id': job_id}))
        else:
            request.error(_('Invalid submission form: %s' %
                            params_form.errors))
    else:
        cluster_json = request.GET.get('cluster', '{}')
        parameters = workflow and workflow.find_all_parameters() or []
        initial_params = ParameterForm.get_initial_params(
            dict([(param['name'], param['value']) for param in parameters]))
        params_form = ParametersFormSet(initial=initial_params)

        return render(
            '/scheduler/submit_job_popup.mako',
            request, {
                'params_form': params_form,
                'name': workflow.name,
                'action': submit_action,
                'show_dryrun': True,
                'email_id': request.user.email,
                'is_oozie_mail_enabled': _is_oozie_mail_enabled(request.user),
                'return_json': request.GET.get('format') == 'json',
                'cluster_json': cluster_json
            },
            force_template=True)
예제 #12
0
파일: dashboard.py 프로젝트: shobull/hue
def rerun_oozie_bundle(request, job_id, app_path):
    oozie_bundle = check_job_access_permission(request, job_id)
    check_job_edition_permission(oozie_bundle, request.user)
    ParametersFormSet = formset_factory(ParameterForm, extra=0)

    if request.method == "POST":
        params_form = ParametersFormSet(request.POST)
        rerun_form = RerunBundleForm(request.POST, oozie_bundle=oozie_bundle)

        if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
            args = {}
            args["deployment_dir"] = app_path

            params = {
                "coord-scope": ",".join(rerun_form.cleaned_data["coordinators"]),
                "refresh": rerun_form.cleaned_data["refresh"],
                "nocleanup": rerun_form.cleaned_data["nocleanup"],
            }

            if rerun_form.cleaned_data["start"] and rerun_form.cleaned_data["end"]:
                date = {
                    "date-scope": "%(start)s::%(end)s"
                    % {
                        "start": utc_datetime_format(rerun_form.cleaned_data["start"]),
                        "end": utc_datetime_format(rerun_form.cleaned_data["end"]),
                    }
                }
                params.update(date)

            properties = dict([(param["name"], param["value"]) for param in params_form.cleaned_data])

            _rerun_bundle(request, job_id, args, params, properties)

            request.info(_("Bundle re-running."))
            return redirect(reverse("oozie:list_oozie_bundle", kwargs={"job_id": job_id}))
        else:
            request.error(_("Invalid submission form: %s" % (rerun_form.errors,)))
            return list_oozie_bundle(request, job_id)
    else:
        rerun_form = RerunBundleForm(oozie_bundle=oozie_bundle)
        initial_params = ParameterForm.get_initial_params(oozie_bundle.conf_dict)
        params_form = ParametersFormSet(initial=initial_params)

    popup = render(
        "dashboard/rerun_bundle_popup.mako",
        request,
        {
            "rerun_form": rerun_form,
            "params_form": params_form,
            "action": reverse("oozie:rerun_oozie_bundle", kwargs={"job_id": job_id, "app_path": app_path}),
        },
        force_template=True,
    ).content

    return JsonResponse(popup, safe=False)
예제 #13
0
파일: dashboard.py 프로젝트: ranade1/hue-3
def submit_external_job(request, application_path):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if application_path.startswith('abfs:/') and not application_path.startswith('abfs://'):
    application_path = application_path.replace("abfs:/", "abfs://")
  elif application_path.startswith('s3a:/') and not application_path.startswith('s3a://'):
    application_path = application_path.replace('s3a:/', 's3a://')
  else:
    application_path = "/" + application_path

  if application_path.startswith("abfs://"):
    application_path = abfspath(application_path)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
      mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
      application_name = os.path.basename(application_path)
      application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow()
      mapping[application_class.get_application_path_key()] = os.path.dirname(application_path)

      try:
        submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
        job_id = submission.run(application_path)
      except RestException as ex:
        detail = ex._headers.get('oozie-error-message', ex)
        if 'Max retries exceeded with url' in str(detail):
          detail = '%s: %s' % (_('The Oozie server is not running'), detail)
        LOG.exception(smart_str(detail))
        raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)

      jsonify = request.POST.get('format') == 'json'
      if jsonify:
        return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'external_workflow'}, safe=False)
      else:
        request.info(_('Oozie job submitted'))
        view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
        return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = Submission(request.user, fs=request.fs, jt=request.jt).get_external_parameters(application_path)
    initial_params = ParameterForm.get_initial_params(parameters)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor/submit_job_popup.mako', request, {
                   'params_form': params_form,
                   'name': _('Job'),
                   'action': reverse('oozie:submit_external_job', kwargs={'application_path': application_path}),
                   'show_dryrun': os.path.basename(application_path) != 'bundle.xml',
                   'return_json': request.GET.get('format') == 'json'
                 }, force_template=True).content
  return JsonResponse(popup, safe=False)
예제 #14
0
def rerun_oozie_job(request, job_id, app_path):
    ParametersFormSet = formset_factory(ParameterForm, extra=0)
    oozie_workflow = check_job_access_permission(request, job_id)
    check_job_edition_permission(oozie_workflow, request.user)

    if request.method == 'POST':
        rerun_form = RerunForm(request.POST, oozie_workflow=oozie_workflow)
        params_form = ParametersFormSet(request.POST)

        if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
            args = {}

            if request.POST['rerun_form_choice'] == 'fail_nodes':
                args['fail_nodes'] = 'true'
            else:
                args['skip_nodes'] = ','.join(
                    rerun_form.cleaned_data['skip_nodes'])
            args['deployment_dir'] = app_path

            mapping = dict([(param['name'], param['value'])
                            for param in params_form.cleaned_data])

            _rerun_workflow(request, job_id, args, mapping)

            request.info(_('Workflow re-running.'))
            return redirect(
                reverse('oozie:list_oozie_workflow', kwargs={'job_id':
                                                             job_id}))
        else:
            request.error(
                _('Invalid submission form: %s %s' %
                  (rerun_form.errors, params_form.errors)))
    else:
        rerun_form = RerunForm(oozie_workflow=oozie_workflow)
        initial_params = ParameterForm.get_initial_params(
            oozie_workflow.conf_dict)
        params_form = ParametersFormSet(initial=initial_params)

    popup = render('dashboard/rerun_job_popup.mako',
                   request, {
                       'rerun_form':
                       rerun_form,
                       'params_form':
                       params_form,
                       'action':
                       reverse('oozie:rerun_oozie_job',
                               kwargs={
                                   'job_id': job_id,
                                   'app_path': app_path
                               }),
                   },
                   force_template=True).content

    return HttpResponse(json.dumps(popup), mimetype="application/json")
예제 #15
0
def sync_coord_workflow(request, job_id):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)
  job = check_job_access_permission(request, job_id)
  check_job_edition_permission(job, request.user)

  hue_coord = get_history().get_coordinator_from_config(job.conf_dict)
  hue_wf = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(job.conf_dict)
  wf_application_path = job.conf_dict.get('wf_application_path') and Hdfs.urlsplit(job.conf_dict['wf_application_path'])[2] or ''
  coord_application_path = job.conf_dict.get('oozie.coord.application.path') and Hdfs.urlsplit(job.conf_dict['oozie.coord.application.path'])[2] or ''
  properties = hue_coord and hue_coord.properties and dict([(param['name'], param['value']) for param in hue_coord.properties]) or None

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      # Update workflow params in coordinator
      hue_coord.clear_workflow_params()
      properties = dict([(param['name'], param['value']) for param in hue_coord.properties])

      # Deploy WF XML
      submission = Submission(user=request.user, job=hue_wf, fs=request.fs, jt=request.jt, properties=properties)
      submission.deploy(deployment_dir=wf_application_path)
      submission._create_file(wf_application_path, hue_wf.XML_FILE_NAME, hue_wf.to_xml(mapping=properties), do_as=True)

      # Deploy Coordinator XML
      job.conf_dict.update(mapping)
      submission = Submission(user=request.user, job=hue_coord, fs=request.fs, jt=request.jt, properties=job.conf_dict, oozie_id=job.id)
      submission._create_file(coord_application_path, hue_coord.XML_FILE_NAME, hue_coord.to_xml(mapping=job.conf_dict), do_as=True)
      # Server picks up deployed Coordinator XML changes after running 'update' action
      submission.update_coord()

      request.info(_('Successfully updated Workflow definition'))
      return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    new_params = hue_wf and hue_wf.find_all_parameters() or []
    new_params = dict([(param['name'], param['value']) for param in new_params])

    # Set previous values
    if properties:
      new_params = dict([(key, properties[key]) if key in properties.keys() else (key, new_params[key]) for key, value in new_params.iteritems()])

    initial_params = ParameterForm.get_initial_params(new_params)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor2/submit_job_popup.mako', request, {
             'params_form': params_form,
             'name': _('Job'),
             'header': _('Sync Workflow definition?'),
             'action': reverse('oozie:sync_coord_workflow', kwargs={'job_id': job_id})
           }, force_template=True).content
  return JsonResponse(popup, safe=False)
예제 #16
0
def sync_coord_workflow(request, job_id):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)
  job = check_job_access_permission(request, job_id)
  check_job_edition_permission(job, request.user)

  hue_coord = get_history().get_coordinator_from_config(job.conf_dict)
  hue_wf = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(job.conf_dict)
  wf_application_path = job.conf_dict.get('wf_application_path') and Hdfs.urlsplit(job.conf_dict['wf_application_path'])[2] or ''
  coord_application_path = job.conf_dict.get('oozie.coord.application.path') and Hdfs.urlsplit(job.conf_dict['oozie.coord.application.path'])[2] or ''
  properties = hue_coord and hue_coord.properties and dict([(param['name'], param['value']) for param in hue_coord.properties]) or None

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      # Update workflow params in coordinator
      hue_coord.clear_workflow_params()
      properties = dict([(param['name'], param['value']) for param in hue_coord.properties])

      # Deploy WF XML
      submission = Submission(user=request.user, job=hue_wf, fs=request.fs, jt=request.jt, properties=properties)
      submission.deploy(deployment_dir=wf_application_path)
      submission._create_file(wf_application_path, hue_wf.XML_FILE_NAME, hue_wf.to_xml(mapping=properties), do_as=True)

      # Deploy Coordinator XML
      job.conf_dict.update(mapping)
      submission = Submission(user=request.user, job=hue_coord, fs=request.fs, jt=request.jt, properties=job.conf_dict, oozie_id=job.id)
      submission._create_file(coord_application_path, hue_coord.XML_FILE_NAME, hue_coord.to_xml(mapping=job.conf_dict), do_as=True)
      # Server picks up deployed Coordinator XML changes after running 'update' action
      submission.update_coord()

      request.info(_('Successfully updated Workflow definition'))
      return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    new_params = hue_wf and hue_wf.find_all_parameters() or []
    new_params = dict([(param['name'], param['value']) for param in new_params])

    # Set previous values
    if properties:
      new_params = dict([(key, properties[key]) if key in properties.keys() else (key, new_params[key]) for key, value in new_params.iteritems()])

    initial_params = ParameterForm.get_initial_params(new_params)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('/scheduler/submit_job_popup.mako', request, {
             'params_form': params_form,
             'name': _('Job'),
             'header': _('Sync Workflow definition?'),
             'action': reverse('oozie:sync_coord_workflow', kwargs={'job_id': job_id})
           }, force_template=True).content
  return JsonResponse(popup, safe=False)
예제 #17
0
파일: dashboard.py 프로젝트: XOEEst/hue
def rerun_oozie_bundle(request, job_id, app_path):
  oozie_bundle = check_job_access_permission(request, job_id)
  check_job_edition_permission(oozie_bundle, request.user)
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    rerun_form = RerunBundleForm(request.POST, oozie_bundle=oozie_bundle)

    if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
      args = {}
      args['deployment_dir'] = app_path

      params = {
        'coord-scope': ','.join(rerun_form.cleaned_data['coordinators']),
        'refresh': rerun_form.cleaned_data['refresh'],
        'nocleanup': rerun_form.cleaned_data['nocleanup'],
      }

      if rerun_form.cleaned_data['start'] and rerun_form.cleaned_data['end']:
        date = {
            'date-scope':
                '%(start)s::%(end)s' % {
                    'start': utc_datetime_format(rerun_form.cleaned_data['start']),
                    'end': utc_datetime_format(rerun_form.cleaned_data['end'])
                }
        }
        params.update(date)

      properties = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      _rerun_bundle(request, job_id, args, params, properties)

      request.info(_('Bundle re-running.'))
      return redirect(reverse('oozie:list_oozie_bundle', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % (rerun_form.errors,)))
      return list_oozie_bundle(request, job_id)
  else:
    rerun_form = RerunBundleForm(oozie_bundle=oozie_bundle)
    initial_params = ParameterForm.get_initial_params(oozie_bundle.conf_dict)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('dashboard/rerun_bundle_popup.mako', request, {
                   'rerun_form': rerun_form,
                   'params_form': params_form,
                   'action': reverse('oozie:rerun_oozie_bundle', kwargs={'job_id': job_id, 'app_path': app_path}),
                 }, force_template=True).content

  return JsonResponse(popup, safe=False)
예제 #18
0
def rerun_oozie_bundle(request, job_id, app_path):
  oozie_bundle = check_job_access_permission(request, job_id)
  check_job_edition_permission(oozie_bundle, request.user)
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    rerun_form = RerunBundleForm(request.POST, oozie_bundle=oozie_bundle)

    if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
      args = {}
      args['deployment_dir'] = app_path

      params = {
        'coord-scope': ','.join(rerun_form.cleaned_data['coordinators']),
        'refresh': rerun_form.cleaned_data['refresh'],
        'nocleanup': rerun_form.cleaned_data['nocleanup'],
      }

      if rerun_form.cleaned_data['start'] and rerun_form.cleaned_data['end']:
        date = {
            'date-scope':
                '%(start)s::%(end)s' % {
                    'start': utc_datetime_format(rerun_form.cleaned_data['start']),
                    'end': utc_datetime_format(rerun_form.cleaned_data['end'])
                }
        }
        params.update(date)

      properties = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      _rerun_bundle(request, job_id, args, params, properties)

      request.info(_('Bundle re-running.'))
      return redirect(reverse('oozie:list_oozie_bundle', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % (rerun_form.errors,)))
      return list_oozie_bundle(request, job_id)
  else:
    rerun_form = RerunBundleForm(oozie_bundle=oozie_bundle)
    initial_params = ParameterForm.get_initial_params(oozie_bundle.conf_dict)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('dashboard/rerun_bundle_popup.mako', request, {
                   'rerun_form': rerun_form,
                   'params_form': params_form,
                   'action': reverse('oozie:rerun_oozie_bundle', kwargs={'job_id': job_id, 'app_path': app_path}),
                 }, force_template=True).content

  return JsonResponse(popup, safe=False)
예제 #19
0
def submit_schedule(request, doc_id):
  interface = request.GET.get('interface', request.POST.get('interface', 'beat'))

  if doc_id.isdigit():
    coordinator = Coordinator(document=Document2.objects.get(id=doc_id))
  else:
    coordinator = Coordinator(document=Document2.objects.get_by_uuid(user=request.user, uuid=doc_id))

  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
      mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
      jsonify = request.POST.get('format') == 'json'
      try:
        job_id = get_api(request, interface).submit_schedule(request, coordinator, mapping)
      except Exception as e:
        message = force_unicode(str(e))
        return JsonResponse({'status': -1, 'message': message}, safe=False)
      if jsonify:
        schedule_type = 'celery-beat' if interface == 'beat' else 'schedule'
        return JsonResponse({'status': 0, 'job_id': job_id, 'type': schedule_type}, safe=False)
      else:
        request.info(_('Schedule submitted.'))
        return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s') % params_form.errors)
  else:
    parameters = coordinator.find_all_parameters() if interface == 'oozie' else []
    initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
    params_form = ParametersFormSet(initial=initial_params)

  popup = render(
      'scheduler/submit_job_popup.mako',
      request, {
          'params_form': params_form,
          'name': coordinator.name,
          'action': '/scheduler/api/schedule/submit/%s' % coordinator.id,
          'show_dryrun': False,
          'return_json': request.GET.get('format') == 'json',
          'interface': interface
      },
      force_template=True
  ).content

  return JsonResponse(popup, safe=False)
예제 #20
0
파일: dashboard.py 프로젝트: cloudera/hue
def rerun_oozie_coordinator(request, job_id, app_path=None):
  oozie_coordinator = check_job_access_permission(request, job_id)
  check_job_edition_permission(oozie_coordinator, request.user)
  ParametersFormSet = formset_factory(ParameterForm, extra=0)
  if app_path is None:
    app_path = oozie_coordinator.coordJobPath
  return_json = request.GET.get('format') == 'json'

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    rerun_form = RerunCoordForm(request.POST, oozie_coordinator=oozie_coordinator)

    if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
      args = {}
      args['deployment_dir'] = app_path

      params = {
        'type': 'action',
        'scope': ','.join(oozie_coordinator.aggreate(rerun_form.cleaned_data['actions'])),
        'refresh': rerun_form.cleaned_data['refresh'],
        'nocleanup': rerun_form.cleaned_data['nocleanup'],
      }

      properties = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      _rerun_coordinator(request, job_id, args, params, properties)

      if rerun_form.cleaned_data['return_json']:
        return JsonResponse({'status': 0, 'job_id': job_id}, safe=False)
      else:
        request.info(_('Coordinator re-running.'))
        return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s') % smart_unicode(rerun_form.errors))
      return list_oozie_coordinator(request, job_id)
  else:
    rerun_form = RerunCoordForm(oozie_coordinator=oozie_coordinator, return_json=return_json)
    initial_params = ParameterForm.get_initial_params(oozie_coordinator.conf_dict)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('dashboard/rerun_coord_popup.mako', request, {
                   'rerun_form': rerun_form,
                   'params_form': params_form,
                   'action': reverse('oozie:rerun_oozie_coord', kwargs={'job_id': job_id, 'app_path': app_path}),
                   'return_json': return_json,
                   'is_mini': request.GET.get('is_mini', False),
                 }, force_template=True).content

  return JsonResponse(popup, safe=False)
예제 #21
0
def rerun_oozie_coordinator(request, job_id, app_path=None):
  oozie_coordinator = check_job_access_permission(request, job_id)
  check_job_edition_permission(oozie_coordinator, request.user)
  ParametersFormSet = formset_factory(ParameterForm, extra=0)
  if app_path is None:
    app_path = oozie_coordinator.coordJobPath
  else:
    app_path = urllib.unquote(app_path)
  return_json = request.GET.get('format') == 'json'

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    rerun_form = RerunCoordForm(request.POST, oozie_coordinator=oozie_coordinator)

    if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
      args = {}
      args['deployment_dir'] = app_path

      params = {
        'type': 'action',
        'scope': ','.join(oozie_coordinator.aggreate(rerun_form.cleaned_data['actions'])),
        'refresh': rerun_form.cleaned_data['refresh'],
        'nocleanup': rerun_form.cleaned_data['nocleanup'],
      }

      properties = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      _rerun_coordinator(request, job_id, args, params, properties)

      if rerun_form.cleaned_data['return_json']:
        return JsonResponse({'status': 0, 'job_id': job_id}, safe=False)
      else:
        request.info(_('Coordinator re-running.'))
        return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s') % smart_unicode(rerun_form.errors))
      return list_oozie_coordinator(request, job_id)
  else:
    rerun_form = RerunCoordForm(oozie_coordinator=oozie_coordinator, return_json=return_json)
    initial_params = ParameterForm.get_initial_params(oozie_coordinator.conf_dict)
    params_form = ParametersFormSet(initial=initial_params)

    return render('dashboard/rerun_coord_popup.mako', request, {
                   'rerun_form': rerun_form,
                   'params_form': params_form,
                   'action': reverse('oozie:rerun_oozie_coord', kwargs={'job_id': job_id, 'app_path': urllib.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)}),
                   'return_json': return_json,
                   'is_mini': request.GET.get('is_mini', False),
                 }, force_template=True)
예제 #22
0
파일: dashboard.py 프로젝트: shobull/hue
def rerun_oozie_coordinator(request, job_id, app_path):
    oozie_coordinator = check_job_access_permission(request, job_id)
    check_job_edition_permission(oozie_coordinator, request.user)
    ParametersFormSet = formset_factory(ParameterForm, extra=0)

    if request.method == "POST":
        params_form = ParametersFormSet(request.POST)
        rerun_form = RerunCoordForm(request.POST, oozie_coordinator=oozie_coordinator)

        if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
            args = {}
            args["deployment_dir"] = app_path

            params = {
                "type": "action",
                "scope": ",".join(oozie_coordinator.aggreate(rerun_form.cleaned_data["actions"])),
                "refresh": rerun_form.cleaned_data["refresh"],
                "nocleanup": rerun_form.cleaned_data["nocleanup"],
            }

            properties = dict([(param["name"], param["value"]) for param in params_form.cleaned_data])

            _rerun_coordinator(request, job_id, args, params, properties)

            request.info(_("Coordinator re-running."))
            return redirect(reverse("oozie:list_oozie_coordinator", kwargs={"job_id": job_id}))
        else:
            request.error(_("Invalid submission form: %s") % smart_unicode(rerun_form.errors))
            return list_oozie_coordinator(request, job_id)
    else:
        rerun_form = RerunCoordForm(oozie_coordinator=oozie_coordinator)
        initial_params = ParameterForm.get_initial_params(oozie_coordinator.conf_dict)
        params_form = ParametersFormSet(initial=initial_params)

    popup = render(
        "dashboard/rerun_coord_popup.mako",
        request,
        {
            "rerun_form": rerun_form,
            "params_form": params_form,
            "action": reverse("oozie:rerun_oozie_coord", kwargs={"job_id": job_id, "app_path": app_path}),
        },
        force_template=True,
    ).content

    return JsonResponse(popup, safe=False)
예제 #23
0
파일: dashboard.py 프로젝트: cloudera/hue
def rerun_oozie_job(request, job_id, app_path=None):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)
  oozie_workflow = check_job_access_permission(request, job_id)
  check_job_edition_permission(oozie_workflow, request.user)
  if app_path is None:
    app_path = oozie_workflow.appPath
  return_json = request.GET.get('format') == 'json'

  if request.method == 'POST':
    rerun_form = RerunForm(request.POST, oozie_workflow=oozie_workflow)
    params_form = ParametersFormSet(request.POST)

    if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
      args = {}

      if request.POST.get('rerun_form_choice') == 'fail_nodes':
        args['fail_nodes'] = 'true'
      else:
        args['skip_nodes'] = ','.join(rerun_form.cleaned_data['skip_nodes'])
      args['deployment_dir'] = app_path

      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      _rerun_workflow(request, job_id, args, mapping)

      if rerun_form.cleaned_data['return_json']:
        return JsonResponse({'status': 0, 'job_id': job_id}, safe=False)
      else:
        request.info(_('Workflow re-running.'))
        return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s %s' % (rerun_form.errors, params_form.errors)))
  else:
    rerun_form = RerunForm(oozie_workflow=oozie_workflow, return_json=return_json)
    initial_params = ParameterForm.get_initial_params(oozie_workflow.conf_dict)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('dashboard/rerun_workflow_popup.mako', request, {
                   'rerun_form': rerun_form,
                   'params_form': params_form,
                   'action': reverse('oozie:rerun_oozie_job', kwargs={'job_id': job_id, 'app_path': app_path}),
                   'return_json': return_json,
                   'is_mini': request.GET.get('is_mini', False),
                 }, force_template=True).content

  return JsonResponse(popup, safe=False)
예제 #24
0
def rerun_oozie_job(request, job_id, app_path=None):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)
  oozie_workflow = check_job_access_permission(request, job_id)
  check_job_edition_permission(oozie_workflow, request.user)
  if app_path is None:
    app_path = oozie_workflow.appPath
  else:
    app_path = urllib.unquote(app_path)
  return_json = request.GET.get('format') == 'json'

  if request.method == 'POST':
    rerun_form = RerunForm(request.POST, oozie_workflow=oozie_workflow)
    params_form = ParametersFormSet(request.POST)

    if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
      args = {}

      if request.POST.get('rerun_form_choice') == 'fail_nodes':
        args['fail_nodes'] = 'true'
      else:
        args['skip_nodes'] = ','.join(rerun_form.cleaned_data['skip_nodes'])
      args['deployment_dir'] = app_path

      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      _rerun_workflow(request, job_id, args, mapping)

      if rerun_form.cleaned_data['return_json']:
        return JsonResponse({'status': 0, 'job_id': job_id}, safe=False)
      else:
        request.info(_('Workflow re-running.'))
        return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s %s' % (rerun_form.errors, params_form.errors)))
  else:
    rerun_form = RerunForm(oozie_workflow=oozie_workflow, return_json=return_json)
    initial_params = ParameterForm.get_initial_params(oozie_workflow.conf_dict)
    params_form = ParametersFormSet(initial=initial_params)

    return render('dashboard/rerun_workflow_popup.mako', request, {
                   'rerun_form': rerun_form,
                   'params_form': params_form,
                   'action': reverse('oozie:rerun_oozie_job', kwargs={'job_id': job_id, 'app_path': urllib.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS) }),
                   'return_json': return_json,
                   'is_mini': request.GET.get('is_mini', False),
                 }, force_template=True)
예제 #25
0
파일: dashboard.py 프로젝트: shobull/hue
def rerun_oozie_job(request, job_id, app_path):
    ParametersFormSet = formset_factory(ParameterForm, extra=0)
    oozie_workflow = check_job_access_permission(request, job_id)
    check_job_edition_permission(oozie_workflow, request.user)

    if request.method == "POST":
        rerun_form = RerunForm(request.POST, oozie_workflow=oozie_workflow)
        params_form = ParametersFormSet(request.POST)

        if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
            args = {}

            if request.POST["rerun_form_choice"] == "fail_nodes":
                args["fail_nodes"] = "true"
            else:
                args["skip_nodes"] = ",".join(rerun_form.cleaned_data["skip_nodes"])
            args["deployment_dir"] = app_path

            mapping = dict([(param["name"], param["value"]) for param in params_form.cleaned_data])

            _rerun_workflow(request, job_id, args, mapping)

            request.info(_("Workflow re-running."))
            return redirect(reverse("oozie:list_oozie_workflow", kwargs={"job_id": job_id}))
        else:
            request.error(_("Invalid submission form: %s %s" % (rerun_form.errors, params_form.errors)))
    else:
        rerun_form = RerunForm(oozie_workflow=oozie_workflow)
        initial_params = ParameterForm.get_initial_params(oozie_workflow.conf_dict)
        params_form = ParametersFormSet(initial=initial_params)

    popup = render(
        "dashboard/rerun_job_popup.mako",
        request,
        {
            "rerun_form": rerun_form,
            "params_form": params_form,
            "action": reverse("oozie:rerun_oozie_job", kwargs={"job_id": job_id, "app_path": app_path}),
        },
        force_template=True,
    ).content

    return JsonResponse(popup, safe=False)
예제 #26
0
파일: dashboard.py 프로젝트: RunnerDu/hue
def sync_coord_workflow(request, job_id):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)
  job = check_job_access_permission(request, job_id)
  check_job_edition_permission(job, request.user)

  hue_coord = get_history().get_coordinator_from_config(job.conf_dict)
  hue_wf = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(job.conf_dict)
  wf_application_path = job.conf_dict.get('wf_application_path') and Hdfs.urlsplit(job.conf_dict['wf_application_path'])[2] or ''

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      submission = Submission(user=request.user, job=hue_wf, fs=request.fs, jt=request.jt, properties=mapping)
      submission._sync_definition(wf_application_path, mapping)

      request.info(_('Successfully updated Workflow definition'))
      return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = hue_wf and hue_wf.find_all_parameters() or []
    params_dict = dict([(param['name'], param['value']) for param in parameters])

    submission = Submission(user=request.user, job=hue_wf, fs=request.fs, jt=request.jt, properties=None)
    prev_properties = hue_wf and hue_wf.deployment_dir and \
                      submission.get_external_parameters(request.fs.join(wf_application_path, hue_wf.XML_FILE_NAME)) or {}

    for key, value in params_dict.iteritems():
      params_dict[key] = prev_properties[key] if key in prev_properties.keys() else params_dict[key]

    initial_params = ParameterForm.get_initial_params(params_dict)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor2/submit_job_popup.mako', request, {
             'params_form': params_form,
             'name': _('Job'),
             'header': _('Sync Workflow definition?'),
             'action': reverse('oozie:sync_coord_workflow', kwargs={'job_id': job_id})
           }, force_template=True).content
  return JsonResponse(popup, safe=False)
예제 #27
0
파일: dashboard.py 프로젝트: blue68/hue
def rerun_oozie_coordinator(request, job_id, app_path):
  oozie_coordinator = check_job_access_permission(request, job_id)
  check_job_edition_permission(oozie_coordinator, request.user)
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    rerun_form = RerunCoordForm(request.POST, oozie_coordinator=oozie_coordinator)

    if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
      args = {}
      args['deployment_dir'] = app_path

      params = {
        'type': 'action',
        'scope': ','.join(oozie_coordinator.aggreate(rerun_form.cleaned_data['actions'])),
        'refresh': rerun_form.cleaned_data['refresh'],
        'nocleanup': rerun_form.cleaned_data['nocleanup'],
      }

      properties = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      _rerun_coordinator(request, job_id, args, params, properties)

      request.info(_('Coordinator re-running!'))
      return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % (rerun_form.errors,)))
      return list_oozie_coordinator(request, job_id)
  else:
    rerun_form = RerunCoordForm(oozie_coordinator=oozie_coordinator)
    initial_params = ParameterForm.get_initial_params(oozie_coordinator.conf_dict)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('dashboard/rerun_coord_popup.mako', request, {
                   'rerun_form': rerun_form,
                   'params_form': params_form,
                   'action': reverse('oozie:rerun_oozie_coord', kwargs={'job_id': job_id, 'app_path': app_path}),
                 }, force_template=True).content

  return HttpResponse(json.dumps(popup), mimetype="application/json")
예제 #28
0
def rerun_oozie_coordinator(request, job_id, app_path):
  oozie_coordinator = check_job_access_permission(request, job_id)
  check_job_edition_permission(oozie_coordinator, request.user)
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    rerun_form = RerunCoordForm(request.POST, oozie_coordinator=oozie_coordinator)

    if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
      args = {}
      args['deployment_dir'] = app_path

      params = {
        'type': 'action',
        'scope': ','.join(oozie_coordinator.aggreate(rerun_form.cleaned_data['actions'])),
        'refresh': rerun_form.cleaned_data['refresh'],
        'nocleanup': rerun_form.cleaned_data['nocleanup'],
      }

      properties = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      _rerun_coordinator(request, job_id, args, params, properties)

      request.info(_('Coordinator re-running.'))
      return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s') % smart_unicode(rerun_form.errors))
      return list_oozie_coordinator(request, job_id)
  else:
    rerun_form = RerunCoordForm(oozie_coordinator=oozie_coordinator)
    initial_params = ParameterForm.get_initial_params(oozie_coordinator.conf_dict)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('dashboard/rerun_coord_popup.mako', request, {
                   'rerun_form': rerun_form,
                   'params_form': params_form,
                   'action': reverse('oozie:rerun_oozie_coord', kwargs={'job_id': job_id, 'app_path': app_path}),
                 }, force_template=True).content

  return JsonResponse(popup, safe=False)
예제 #29
0
def submit_coordinator(request, doc_id):
  # TODO: Replace URL by desktop/scheduler API
  if doc_id.isdigit():
    coordinator = Coordinator(document=Document2.objects.get(id=doc_id))
  else:
    coordinator = Coordinator(document=Document2.objects.get_by_uuid(user=request.user, uuid=doc_id))

  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
      mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
      jsonify = request.POST.get('format') == 'json'
      try:
        job_id = _submit_coordinator(request, coordinator, mapping)
      except Exception as e:
        message = force_unicode(str(e))
        return JsonResponse({'status': -1, 'message': message}, safe=False)
      if jsonify:
        return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'schedule'}, safe=False)
      else:
        request.info(_('Coordinator submitted.'))
        return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = coordinator.find_all_parameters()
    initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
    params_form = ParametersFormSet(initial=initial_params)

    return render('/scheduler/submit_job_popup.mako', request, {
                 'params_form': params_form,
                 'name': coordinator.name,
                 'action': reverse('oozie:editor_submit_coordinator',  kwargs={'doc_id': coordinator.id}),
                 'show_dryrun': True,
                 'return_json': request.GET.get('format') == 'json'
                }, force_template=True)
예제 #30
0
def submit_workflow(request, doc_id):
    workflow = Workflow(document=Document2.objects.get(id=doc_id))
    ParametersFormSet = formset_factory(ParameterForm, extra=0)

    if request.method == 'POST':
        params_form = ParametersFormSet(request.POST)

        if params_form.is_valid():
            mapping = dict([(param['name'], param['value'])
                            for param in params_form.cleaned_data])

            job_id = _submit_workflow(request.user, request.fs, request.jt,
                                      workflow, mapping)

            request.info(_('Workflow submitted'))
            return redirect(
                reverse('oozie:list_oozie_workflow', kwargs={'job_id':
                                                             job_id}))
        else:
            request.error(_('Invalid submission form: %s' %
                            params_form.errors))
    else:
        parameters = workflow.find_all_parameters()
        initial_params = ParameterForm.get_initial_params(
            dict([(param['name'], param['value']) for param in parameters]))
        params_form = ParametersFormSet(initial=initial_params)

        popup = render('editor2/submit_job_popup.mako',
                       request, {
                           'params_form':
                           params_form,
                           'name':
                           workflow.name,
                           'action':
                           reverse('oozie:editor_submit_workflow',
                                   kwargs={'doc_id': workflow.id})
                       },
                       force_template=True).content
        return HttpResponse(json.dumps(popup), mimetype="application/json")
예제 #31
0
파일: dashboard.py 프로젝트: blue68/hue
def rerun_oozie_job(request, job_id, app_path):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)
  oozie_workflow = check_job_access_permission(request, job_id)
  check_job_edition_permission(oozie_workflow, request.user)

  if request.method == 'POST':
    rerun_form = RerunForm(request.POST, oozie_workflow=oozie_workflow)
    params_form = ParametersFormSet(request.POST)

    if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
      args = {}

      if request.POST['rerun_form_choice'] == 'fail_nodes':
        args['fail_nodes'] = 'true'
      else:
        args['skip_nodes'] = ','.join(rerun_form.cleaned_data['skip_nodes'])
      args['deployment_dir'] = app_path

      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      _rerun_workflow(request, job_id, args, mapping)

      request.info(_('Workflow re-running!'))
      return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s %s' % (rerun_form.errors, params_form.errors)))
  else:
    rerun_form = RerunForm(oozie_workflow=oozie_workflow)
    initial_params = ParameterForm.get_initial_params(oozie_workflow.conf_dict)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('dashboard/rerun_job_popup.mako', request, {
                   'rerun_form': rerun_form,
                   'params_form': params_form,
                   'action': reverse('oozie:rerun_oozie_job', kwargs={'job_id': job_id, 'app_path': app_path}),
                 }, force_template=True).content

  return HttpResponse(json.dumps(popup), mimetype="application/json")
예제 #32
0
파일: editor.py 프로젝트: 18600597055/hue
def submit_bundle(request, bundle):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
      job_id = _submit_bundle(request, bundle, mapping)

      request.info(_('Bundle submitted.'))
      return redirect(reverse('oozie:list_oozie_bundle', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = bundle.find_all_parameters()
    initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor/submit_job_popup.mako', request, {
                 'params_form': params_form,
                 'action': reverse('oozie:submit_bundle',  kwargs={'bundle': bundle.id})
                }, force_template=True).content
  return JsonResponse(popup, safe=False)
예제 #33
0
파일: editor2.py 프로젝트: tempbottle/hue
            try:
                job_id = _submit_workflow(request.user, request.fs, request.jt,
                                          workflow, mapping)
            except Exception, e:
                raise PopupException(_('Workflow submission failed'),
                                     detail=smart_str(e))
            request.info(_('Workflow submitted'))
            return redirect(
                reverse('oozie:list_oozie_workflow', kwargs={'job_id':
                                                             job_id}))
        else:
            request.error(_('Invalid submission form: %s' %
                            params_form.errors))
    else:
        parameters = workflow.find_all_parameters()
        initial_params = ParameterForm.get_initial_params(
            dict([(param['name'], param['value']) for param in parameters]))
        params_form = ParametersFormSet(initial=initial_params)

        popup = render('editor2/submit_job_popup.mako',
                       request, {
                           'params_form':
                           params_form,
                           'name':
                           workflow.name,
                           'action':
                           reverse('oozie:editor_submit_workflow',
                                   kwargs={'doc_id': workflow.id}),
                           'show_dryrun':
                           True
                       },
                       force_template=True).content
예제 #34
0
파일: dashboard.py 프로젝트: shobull/hue
def sync_coord_workflow(request, job_id):
    ParametersFormSet = formset_factory(ParameterForm, extra=0)
    job = check_job_access_permission(request, job_id)
    check_job_edition_permission(job, request.user)

    hue_coord = get_history().get_coordinator_from_config(job.conf_dict)
    hue_wf = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(job.conf_dict)
    wf_application_path = (
        job.conf_dict.get("wf_application_path") and Hdfs.urlsplit(job.conf_dict["wf_application_path"])[2] or ""
    )
    coord_application_path = (
        job.conf_dict.get("oozie.coord.application.path")
        and Hdfs.urlsplit(job.conf_dict["oozie.coord.application.path"])[2]
        or ""
    )
    properties = (
        hue_coord
        and hue_coord.properties
        and dict([(param["name"], param["value"]) for param in hue_coord.properties])
        or None
    )

    if request.method == "POST":
        params_form = ParametersFormSet(request.POST)
        if params_form.is_valid():
            mapping = dict([(param["name"], param["value"]) for param in params_form.cleaned_data])

            # Update workflow params in coordinator
            hue_coord.clear_workflow_params()
            properties = dict([(param["name"], param["value"]) for param in hue_coord.properties])

            # Deploy WF XML
            submission = Submission(user=request.user, job=hue_wf, fs=request.fs, jt=request.jt, properties=properties)
            submission._create_file(
                wf_application_path, hue_wf.XML_FILE_NAME, hue_wf.to_xml(mapping=properties), do_as=True
            )

            # Deploy Coordinator XML
            job.conf_dict.update(mapping)
            submission = Submission(
                user=request.user,
                job=hue_coord,
                fs=request.fs,
                jt=request.jt,
                properties=job.conf_dict,
                oozie_id=job.id,
            )
            submission._create_file(
                coord_application_path, hue_coord.XML_FILE_NAME, hue_coord.to_xml(mapping=job.conf_dict), do_as=True
            )
            # Server picks up deployed Coordinator XML changes after running 'update' action
            submission.update_coord()

            request.info(_("Successfully updated Workflow definition"))
            return redirect(reverse("oozie:list_oozie_coordinator", kwargs={"job_id": job_id}))
        else:
            request.error(_("Invalid submission form: %s" % params_form.errors))
    else:
        new_params = hue_wf and hue_wf.find_all_parameters() or []
        new_params = dict([(param["name"], param["value"]) for param in new_params])

        # Set previous values
        if properties:
            new_params = dict(
                [
                    (key, properties[key]) if key in properties.keys() else (key, new_params[key])
                    for key, value in new_params.iteritems()
                ]
            )

        initial_params = ParameterForm.get_initial_params(new_params)
        params_form = ParametersFormSet(initial=initial_params)

    popup = render(
        "editor2/submit_job_popup.mako",
        request,
        {
            "params_form": params_form,
            "name": _("Job"),
            "header": _("Sync Workflow definition?"),
            "action": reverse("oozie:sync_coord_workflow", kwargs={"job_id": job_id}),
        },
        force_template=True,
    ).content
    return JsonResponse(popup, safe=False)
예제 #35
0
파일: editor2.py 프로젝트: cloudera/hue
      try:
        job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping)
      except Exception, e:
        raise PopupException(_('Workflow submission failed'), detail=smart_str(e), error_code=200)
      jsonify = request.POST.get('format') == 'json'
      if jsonify:
        return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'workflow'}, safe=False)
      else:
        request.info(_('Workflow submitted'))
        return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    cluster_json = request.GET.get('cluster', '{}')
    parameters = workflow and workflow.find_all_parameters() or []
    initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
    params_form = ParametersFormSet(initial=initial_params)


    popup = render('editor2/submit_job_popup.mako', request, {
                     'params_form': params_form,
                     'name': workflow.name,
                     'action': submit_action,
                     'show_dryrun': True,
                     'email_id': request.user.email,
                     'is_oozie_mail_enabled': _is_oozie_mail_enabled(request.user),
                     'return_json': request.GET.get('format') == 'json',
                     'cluster_json': cluster_json
                   }, force_template=True).content
    return JsonResponse(popup, safe=False)
예제 #36
0
파일: dashboard.py 프로젝트: XOEEst/hue
        detail = ex._headers.get('oozie-error-message', ex)
        if 'Max retries exceeded with url' in str(detail):
          detail = '%s: %s' % (_('The Oozie server is not running'), detail)

        LOG.exception(smart_str(detail))

        raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)

      request.info(_('Oozie job submitted'))
      view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
      return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = Submission(request.user, fs=request.fs, jt=request.jt).get_external_parameters(application_path)
    initial_params = ParameterForm.get_initial_params(parameters)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor/submit_job_popup.mako', request, {
                   'params_form': params_form,
                   'name': _('Job'),
                   'action': reverse('oozie:submit_external_job', kwargs={'application_path': application_path}),
                   'show_dryrun': os.path.basename(application_path) != 'bundle.xml'
                 }, force_template=True).content
  return JsonResponse(popup, safe=False)


def massaged_workflow_actions_for_json(workflow_actions, oozie_coordinator, oozie_bundle):
  actions = []

  for action in workflow_actions:
예제 #37
0
          detail = '%s: %s' % (_('The Oozie server is not running'), detail)
        LOG.exception(smart_str(detail))
        raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)

      jsonify = request.POST.get('format') == 'json'
      if jsonify:
        return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'external_workflow'}, safe=False)
      else:
        request.info(_('Oozie job submitted'))
        view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
        return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = Submission(request.user, fs=request.fs, jt=request.jt).get_external_parameters(application_path)
    initial_params = ParameterForm.get_initial_params(parameters)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor/submit_job_popup.mako', request, {
                   'params_form': params_form,
                   'name': _('Job'),
                   'action': reverse('oozie:submit_external_job', kwargs={'application_path': application_path}),
                   'show_dryrun': os.path.basename(application_path) != 'bundle.xml',
                   'return_json': request.GET.get('format') == 'json'
                 }, force_template=True).content
  return JsonResponse(popup, safe=False)


def massaged_workflow_actions_for_json(workflow_actions, oozie_coordinator, oozie_bundle):
  actions = []