Example #1
0
def sync_coord_workflow(request, job_id):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)
  job = check_job_access_permission(request, job_id)
  check_job_edition_permission(job, request.user)

  hue_coord = get_history().get_coordinator_from_config(job.conf_dict)
  hue_wf = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(job.conf_dict)
  wf_application_path = job.conf_dict.get('wf_application_path') and Hdfs.urlsplit(job.conf_dict['wf_application_path'])[2] or ''
  coord_application_path = job.conf_dict.get('oozie.coord.application.path') and Hdfs.urlsplit(job.conf_dict['oozie.coord.application.path'])[2] or ''
  properties = hue_coord and hue_coord.properties and dict([(param['name'], param['value']) for param in hue_coord.properties]) or None

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      # Update workflow params in coordinator
      hue_coord.clear_workflow_params()
      properties = dict([(param['name'], param['value']) for param in hue_coord.properties])

      # Deploy WF XML
      submission = Submission(user=request.user, job=hue_wf, fs=request.fs, jt=request.jt, properties=properties)
      submission.deploy(deployment_dir=wf_application_path)
      submission._create_file(wf_application_path, hue_wf.XML_FILE_NAME, hue_wf.to_xml(mapping=properties), do_as=True)

      # Deploy Coordinator XML
      job.conf_dict.update(mapping)
      submission = Submission(user=request.user, job=hue_coord, fs=request.fs, jt=request.jt, properties=job.conf_dict, oozie_id=job.id)
      submission._create_file(coord_application_path, hue_coord.XML_FILE_NAME, hue_coord.to_xml(mapping=job.conf_dict), do_as=True)
      # Server picks up deployed Coordinator XML changes after running 'update' action
      submission.update_coord()

      request.info(_('Successfully updated Workflow definition'))
      return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    new_params = hue_wf and hue_wf.find_all_parameters() or []
    new_params = dict([(param['name'], param['value']) for param in new_params])

    # Set previous values
    if properties:
      new_params = dict([(key, properties[key]) if key in properties.keys() else (key, new_params[key]) for key, value in new_params.iteritems()])

    initial_params = ParameterForm.get_initial_params(new_params)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor2/submit_job_popup.mako', request, {
             'params_form': params_form,
             'name': _('Job'),
             'header': _('Sync Workflow definition?'),
             'action': reverse('oozie:sync_coord_workflow', kwargs={'job_id': job_id})
           }, force_template=True).content
  return JsonResponse(popup, safe=False)
Example #2
0
def sync_coord_workflow(request, job_id):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)
  job = check_job_access_permission(request, job_id)
  check_job_edition_permission(job, request.user)

  hue_coord = get_history().get_coordinator_from_config(job.conf_dict)
  hue_wf = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(job.conf_dict)
  wf_application_path = job.conf_dict.get('wf_application_path') and Hdfs.urlsplit(job.conf_dict['wf_application_path'])[2] or ''
  coord_application_path = job.conf_dict.get('oozie.coord.application.path') and Hdfs.urlsplit(job.conf_dict['oozie.coord.application.path'])[2] or ''
  properties = hue_coord and hue_coord.properties and dict([(param['name'], param['value']) for param in hue_coord.properties]) or None

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      # Update workflow params in coordinator
      hue_coord.clear_workflow_params()
      properties = dict([(param['name'], param['value']) for param in hue_coord.properties])

      # Deploy WF XML
      submission = Submission(user=request.user, job=hue_wf, fs=request.fs, jt=request.jt, properties=properties)
      submission.deploy(deployment_dir=wf_application_path)
      submission._create_file(wf_application_path, hue_wf.XML_FILE_NAME, hue_wf.to_xml(mapping=properties), do_as=True)

      # Deploy Coordinator XML
      job.conf_dict.update(mapping)
      submission = Submission(user=request.user, job=hue_coord, fs=request.fs, jt=request.jt, properties=job.conf_dict, oozie_id=job.id)
      submission._create_file(coord_application_path, hue_coord.XML_FILE_NAME, hue_coord.to_xml(mapping=job.conf_dict), do_as=True)
      # Server picks up deployed Coordinator XML changes after running 'update' action
      submission.update_coord()

      request.info(_('Successfully updated Workflow definition'))
      return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    new_params = hue_wf and hue_wf.find_all_parameters() or []
    new_params = dict([(param['name'], param['value']) for param in new_params])

    # Set previous values
    if properties:
      new_params = dict([(key, properties[key]) if key in properties.keys() else (key, new_params[key]) for key, value in new_params.iteritems()])

    initial_params = ParameterForm.get_initial_params(new_params)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('/scheduler/submit_job_popup.mako', request, {
             'params_form': params_form,
             'name': _('Job'),
             'header': _('Sync Workflow definition?'),
             'action': reverse('oozie:sync_coord_workflow', kwargs={'job_id': job_id})
           }, force_template=True).content
  return JsonResponse(popup, safe=False)
Example #3
0
def location_to_url(location, strict=True, is_embeddable=False):
    """
    If possible, returns a file browser URL to the location.
    Prunes HDFS URI to path.
    Location is a URI, if strict is True.

    Python doesn't seem to have a readily-available URI-comparison
    library, so this is quite hacky.
    """
    if location is None:
      return None
    split_path = Hdfs.urlsplit(location)
    if strict and not split_path[1] or not split_path[2]:
      # No netloc not full url or no URL
      return None
    path = location
    if split_path[0] == 'hdfs':
      path = split_path[2]

    try:
      filebrowser_path = reverse("filebrowser.views.view", kwargs=dict(path=path))
    except Exception as e:
      LOG.warn('No table filesystem link: %s' % e)
      return None

    if is_embeddable and not filebrowser_path.startswith('/hue'):
        filebrowser_path = '/hue' + filebrowser_path

    return filebrowser_path
Example #4
0
def hdfs_link_js(url):
  link = 'javascript:void(0)'

  if url:
    path = Hdfs.urlsplit(url)[2]
    if path:
      link = ('/filebrowser/view=%s' if path.startswith(posixpath.sep) else '/filebrowser/home_relative_view=/%s') % path

  return link
Example #5
0
def _get_service_url(hdfs_config):
  override = hdfs_config.WEBHDFS_URL.get()
  if override:
    return override

  fs_defaultfs = hdfs_config.FS_DEFAULTFS.get()
  netloc = Hdfs.urlsplit(fs_defaultfs)[1]
  host = netloc.split(':')[0]
  return "{0}://{1}:{2}/webhdfs/v1".format(get_webhdfs_ssl()["protocol"], host, get_webhdfs_ssl()["port"])
Example #6
0
def hdfs_link_js(url):
  link = 'javascript:void(0)'

  if url:
    path = Hdfs.urlsplit(url)[2]
    if path:
      link = ('/filebrowser/view=%s' if path.startswith(posixpath.sep) else '/filebrowser/home_relative_view=/%s') % path

  return link
Example #7
0
def _get_service_url(hdfs_config):
  override = hdfs_config.WEBHDFS_URL.get()
  if override:
    return override

  fs_defaultfs = hdfs_config.FS_DEFAULTFS.get()
  netloc = Hdfs.urlsplit(fs_defaultfs)[1]
  host = netloc.split(':')[0]
  port = hadoop.conf.DEFAULT_NN_HTTP_PORT
  return "http://%s:%s/webhdfs/v1" % (host, port)
Example #8
0
def hdfs_link(url):
  if url:
    path = Hdfs.urlsplit(url)[2]
    if path:
      if path.startswith(posixpath.sep):
        return "/filebrowser/view=" + path
      else:
        return "/filebrowser/home_relative_view=/" + path
    else:
      return url
  else:
    return url
Example #9
0
def hdfs_link(url):
    if url:
        path = Hdfs.urlsplit(url)[2]
        if path:
            if path.startswith(posixpath.sep):
                return "/filebrowser/view" + path
            else:
                return "/filebrowser/home_relative_view/" + path
        else:
            return url
    else:
        return url
Example #10
0
def location_to_url(location, strict=True):
    """
    If possible, returns a file browser URL to the location.
    Location is a URI, if strict is True.

    Python doesn't seem to have a readily-available URI-comparison
    library, so this is quite hacky.
    """
    if location is None:
      return None
    split_path = Hdfs.urlsplit(location)
    if strict and not split_path[1] or not split_path[2]:
      # No netloc not full url or no URL
      return None
    return reverse("filebrowser.views.view", kwargs=dict(path=split_path[2]))
Example #11
0
def test_urlsplit():
  """Test Hdfs urlsplit"""
  url = 'hdfs://nn.no.port/foo/bar'
  assert_equals(('hdfs', 'nn.no.port', '/foo/bar', '', ''), Hdfs.urlsplit(url))
  url = 'hdfs://nn:8020/foo/bar'
  assert_equals(('hdfs', 'nn:8020', '/foo/bar', '', ''), Hdfs.urlsplit(url))
  url = 'hdfs://nn:8020//foo//bar'
  assert_equals(('hdfs', 'nn:8020', '/foo/bar', '', ''), Hdfs.urlsplit(url))
  url = 'hdfs://nn:8020'
  assert_equals(('hdfs', 'nn:8020', '/', '', ''), Hdfs.urlsplit(url))
  url = '/foo/bar'
  assert_equals(('hdfs', '', '/foo/bar', '', ''), Hdfs.urlsplit(url))
  url = 'foo//bar'
  assert_equals(('hdfs', '', 'foo/bar', '', ''), Hdfs.urlsplit(url))
Example #12
0
 def test_urlsplit(self):
   """Test Hdfs urlsplit"""
   url = 'hdfs://nn.no.port/foo/bar'
   assert_equals(('hdfs', 'nn.no.port', '/foo/bar', '', ''), Hdfs.urlsplit(url))
   url = 'hdfs://nn:8020/foo/bar'
   assert_equals(('hdfs', 'nn:8020', '/foo/bar', '', ''), Hdfs.urlsplit(url))
   url = 'hdfs://nn:8020//foo//bar'
   assert_equals(('hdfs', 'nn:8020', '/foo/bar', '', ''), Hdfs.urlsplit(url))
   url = 'hdfs://nn:8020'
   assert_equals(('hdfs', 'nn:8020', '/', '', ''), Hdfs.urlsplit(url))
   url = '/foo/bar'
   assert_equals(('hdfs', '', '/foo/bar', '', ''), Hdfs.urlsplit(url))
   url = 'foo//bar'
   assert_equals(('hdfs', '', 'foo/bar', '', ''), Hdfs.urlsplit(url))
Example #13
0
def sync_coord_workflow(request, job_id):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)
  job = check_job_access_permission(request, job_id)
  check_job_edition_permission(job, request.user)

  hue_coord = get_history().get_coordinator_from_config(job.conf_dict)
  hue_wf = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(job.conf_dict)
  wf_application_path = job.conf_dict.get('wf_application_path') and Hdfs.urlsplit(job.conf_dict['wf_application_path'])[2] or ''

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)
    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])

      submission = Submission(user=request.user, job=hue_wf, fs=request.fs, jt=request.jt, properties=mapping)
      submission._sync_definition(wf_application_path, mapping)

      request.info(_('Successfully updated Workflow definition'))
      return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))
  else:
    parameters = hue_wf and hue_wf.find_all_parameters() or []
    params_dict = dict([(param['name'], param['value']) for param in parameters])

    submission = Submission(user=request.user, job=hue_wf, fs=request.fs, jt=request.jt, properties=None)
    prev_properties = hue_wf and hue_wf.deployment_dir and \
                      submission.get_external_parameters(request.fs.join(wf_application_path, hue_wf.XML_FILE_NAME)) or {}

    for key, value in params_dict.iteritems():
      params_dict[key] = prev_properties[key] if key in prev_properties.keys() else params_dict[key]

    initial_params = ParameterForm.get_initial_params(params_dict)
    params_form = ParametersFormSet(initial=initial_params)

  popup = render('editor2/submit_job_popup.mako', request, {
             'params_form': params_form,
             'name': _('Job'),
             'header': _('Sync Workflow definition?'),
             'action': reverse('oozie:sync_coord_workflow', kwargs={'job_id': job_id})
           }, force_template=True).content
  return JsonResponse(popup, safe=False)
Example #14
0
def location_to_url(location, strict=True, is_embeddable=False):
    """
    If possible, returns a file browser URL to the location.
    Prunes HDFS URI to path.
    Location is a URI, if strict is True.

    Python doesn't seem to have a readily-available URI-comparison
    library, so this is quite hacky.
    """
    if location is None:
      return None
    split_path = Hdfs.urlsplit(location)
    if strict and not split_path[1] or not split_path[2]:
      # No netloc not full url or no URL
      return None
    path = location
    if split_path[0] == 'hdfs':
      path = split_path[2]

    filebrowser_path = reverse("filebrowser.views.view", kwargs=dict(path=path))
    if is_embeddable and not filebrowser_path.startswith('/hue'):
        filebrowser_path = '/hue' + filebrowser_path
    return filebrowser_path
Example #15
0
File: conf.py Project: wdai-aa/hue
def config_validator(user):
  """
  config_validator() -> [ (config_variable, error_message) ]

  Called by core check_config() view.
  """
  from hadoop.cluster import get_all_hdfs
  from hadoop.fs.hadoopfs import Hdfs
  from liboozie.oozie_api import get_oozie

  res = []

  if OOZIE_URL.get():
    status = get_oozie_status(user)
    if 'NORMAL' not in status:
      res.append((status, _('The Oozie server is not available')))

    api = get_oozie(user)
    intrumentation = api.get_instrumentation()
    sharelib_url = [param['value'] for group in intrumentation['variables'] for param in group['data'] if param['name'] == 'sharelib.system.libpath']
    if sharelib_url:
      sharelib_url = Hdfs.urlsplit(sharelib_url[0])[2]

    if not sharelib_url:
      res.append((status, _('Oozie Share Lib path is not available')))

    class ConfigMock:
      def __init__(self, value): self.value = value
      def get(self): return self.value
      def get_fully_qualifying_key(self): return self.value

    for cluster in get_all_hdfs().values():
      res.extend(validate_path(ConfigMock(sharelib_url), is_dir=True, fs=cluster,
                               message=_('Oozie Share Lib not installed in default location.')))

  return res
Example #16
0
        ]:
            metrics = api.get_metrics()
            sharelib_url = 'gauges' in metrics and 'libs.sharelib.system.libpath' in metrics[
                'gauges'] and [
                    metrics['gauges']['libs.sharelib.system.libpath']['value']
                ] or []
        else:
            intrumentation = api.get_instrumentation()
            sharelib_url = [
                param['value'] for group in intrumentation['variables']
                for param in group['data']
                if param['name'] == 'sharelib.system.libpath'
            ]

        if sharelib_url:
            sharelib_url = Hdfs.urlsplit(sharelib_url[0])[2]

        if not sharelib_url:
            res.append((status, _('Oozie Share Lib path is not available')))

        class ConfigMock:
            def __init__(self, value):
                self.value = value

            def get(self):
                return self.value

            def get_fully_qualifying_key(self):
                return self.value

        for cluster in get_all_hdfs().values():
Example #17
0
 def urlsplit(url):
   return Hdfs.urlsplit(url)
Example #18
0
 def urlsplit(url):
   return Hdfs.urlsplit(url)
Example #19
0
def config_validator(user):
    """
  config_validator() -> [ (config_variable, error_message) ]

  Called by core check_config() view.
  """
    from hadoop.cluster import get_all_hdfs
    from hadoop.fs.hadoopfs import Hdfs
    from liboozie.oozie_api import get_oozie

    res = []

    if OOZIE_URL.get():
        status = get_oozie_status(user)
        if "NORMAL" not in status:
            res.append((status, _("The Oozie server is not available")))

        api = get_oozie(user, api_version="v2")

        configuration = api.get_configuration()
        if "org.apache.oozie.service.MetricsInstrumentationService" in [
            c.strip() for c in configuration.get("oozie.services.ext", "").split(",")
        ]:
            metrics = api.get_metrics()
            sharelib_url = (
                "gauges" in metrics
                and "libs.sharelib.system.libpath" in metrics["gauges"]
                and [metrics["gauges"]["libs.sharelib.system.libpath"]["value"]]
                or []
            )
        else:
            intrumentation = api.get_instrumentation()
            sharelib_url = [
                param["value"]
                for group in intrumentation["variables"]
                for param in group["data"]
                if param["name"] == "sharelib.system.libpath"
            ]

        if sharelib_url:
            sharelib_url = Hdfs.urlsplit(sharelib_url[0])[2]

        if not sharelib_url:
            res.append((status, _("Oozie Share Lib path is not available")))

        class ConfigMock:
            def __init__(self, value):
                self.value = value

            def get(self):
                return self.value

            def get_fully_qualifying_key(self):
                return self.value

        for cluster in get_all_hdfs().values():
            res.extend(
                validate_path(
                    ConfigMock(sharelib_url),
                    is_dir=True,
                    fs=cluster,
                    message=_("Oozie Share Lib not installed in default location."),
                )
            )

    return res
Example #20
0
def config_validator(user):
    """
  config_validator() -> [ (config_variable, error_message) ]

  Called by core check_config() view.
  """
    from desktop.lib.fsmanager import get_filesystem
    from hadoop.cluster import get_all_hdfs
    from hadoop.fs.hadoopfs import Hdfs
    from liboozie.oozie_api import get_oozie

    res = []

    try:
        from oozie.conf import REMOTE_SAMPLE_DIR
    except Exception as e:
        LOG.warn('Config check failed because Oozie app not installed: %s' % e)
        return res

    if OOZIE_URL.get():
        status = get_oozie_status(user)
        if 'NORMAL' not in status:
            res.append((status, _('The Oozie server is not available')))
        fs = get_filesystem()
        NICE_NAME = 'Oozie'
        if fs.do_as_superuser(fs.exists, REMOTE_SAMPLE_DIR.get()):
            stats = fs.do_as_superuser(fs.stats, REMOTE_SAMPLE_DIR.get())
            mode = oct(stats.mode)
            # if neither group nor others have write permission
            group_has_write = int(mode[-2]) & 2
            others_has_write = int(mode[-1]) & 2

            if not group_has_write and not others_has_write:
                res.append(
                    (NICE_NAME,
                     "The permissions of workspace '%s' are too restrictive" %
                     REMOTE_SAMPLE_DIR.get()))

        api = get_oozie(user, api_version="v2")

        configuration = api.get_configuration()
        if 'org.apache.oozie.service.MetricsInstrumentationService' in [
                c.strip()
                for c in configuration.get('oozie.services.ext', '').split(',')
        ]:
            metrics = api.get_metrics()
            sharelib_url = 'gauges' in metrics and 'libs.sharelib.system.libpath' in metrics[
                'gauges'] and [
                    metrics['gauges']['libs.sharelib.system.libpath']['value']
                ] or []
        else:
            intrumentation = api.get_instrumentation()
            sharelib_url = [
                param['value'] for group in intrumentation['variables']
                for param in group['data']
                if param['name'] == 'sharelib.system.libpath'
            ]

        if sharelib_url:
            sharelib_url = Hdfs.urlsplit(sharelib_url[0])[2]

        if not sharelib_url:
            res.append((status, _('Oozie Share Lib path is not available')))

        class ConfigMock(object):
            def __init__(self, value):
                self.value = value

            def get(self):
                return self.value

            def get_fully_qualifying_key(self):
                return self.value

        for cluster in list(get_all_hdfs().values()):
            res.extend(
                validate_path(
                    ConfigMock(sharelib_url),
                    is_dir=True,
                    fs=cluster,
                    message=_(
                        'Oozie Share Lib not installed in default location.')))

    return res
Example #21
0
def sync_coord_workflow(request, job_id):
    ParametersFormSet = formset_factory(ParameterForm, extra=0)
    job = check_job_access_permission(request, job_id)
    check_job_edition_permission(job, request.user)

    hue_coord = get_history().get_coordinator_from_config(job.conf_dict)
    hue_wf = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(job.conf_dict)
    wf_application_path = (
        job.conf_dict.get("wf_application_path") and Hdfs.urlsplit(job.conf_dict["wf_application_path"])[2] or ""
    )
    coord_application_path = (
        job.conf_dict.get("oozie.coord.application.path")
        and Hdfs.urlsplit(job.conf_dict["oozie.coord.application.path"])[2]
        or ""
    )
    properties = (
        hue_coord
        and hue_coord.properties
        and dict([(param["name"], param["value"]) for param in hue_coord.properties])
        or None
    )

    if request.method == "POST":
        params_form = ParametersFormSet(request.POST)
        if params_form.is_valid():
            mapping = dict([(param["name"], param["value"]) for param in params_form.cleaned_data])

            # Update workflow params in coordinator
            hue_coord.clear_workflow_params()
            properties = dict([(param["name"], param["value"]) for param in hue_coord.properties])

            # Deploy WF XML
            submission = Submission(user=request.user, job=hue_wf, fs=request.fs, jt=request.jt, properties=properties)
            submission._create_file(
                wf_application_path, hue_wf.XML_FILE_NAME, hue_wf.to_xml(mapping=properties), do_as=True
            )

            # Deploy Coordinator XML
            job.conf_dict.update(mapping)
            submission = Submission(
                user=request.user,
                job=hue_coord,
                fs=request.fs,
                jt=request.jt,
                properties=job.conf_dict,
                oozie_id=job.id,
            )
            submission._create_file(
                coord_application_path, hue_coord.XML_FILE_NAME, hue_coord.to_xml(mapping=job.conf_dict), do_as=True
            )
            # Server picks up deployed Coordinator XML changes after running 'update' action
            submission.update_coord()

            request.info(_("Successfully updated Workflow definition"))
            return redirect(reverse("oozie:list_oozie_coordinator", kwargs={"job_id": job_id}))
        else:
            request.error(_("Invalid submission form: %s" % params_form.errors))
    else:
        new_params = hue_wf and hue_wf.find_all_parameters() or []
        new_params = dict([(param["name"], param["value"]) for param in new_params])

        # Set previous values
        if properties:
            new_params = dict(
                [
                    (key, properties[key]) if key in properties.keys() else (key, new_params[key])
                    for key, value in new_params.iteritems()
                ]
            )

        initial_params = ParameterForm.get_initial_params(new_params)
        params_form = ParametersFormSet(initial=initial_params)

    popup = render(
        "editor2/submit_job_popup.mako",
        request,
        {
            "params_form": params_form,
            "name": _("Job"),
            "header": _("Sync Workflow definition?"),
            "action": reverse("oozie:sync_coord_workflow", kwargs={"job_id": job_id}),
        },
        force_template=True,
    ).content
    return JsonResponse(popup, safe=False)