コード例 #1
0
ファイル: oozie_setup.py プロジェクト: yjkim/hue
  def handle_noargs(self, **options):
    fs = cluster.get_hdfs()    
    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    USERNAME = '******'
    try:
      sample_user = User.objects.get(username=USERNAME)
    except User.DoesNotExist:
      sample_user = User.objects.create(username=USERNAME, password='******', is_active=False, is_superuser=False, id=1100713, pk=1100713)
    management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
コード例 #2
0
ファイル: oozie_setup.py プロジェクト: zhouyan8603/HueBigSQL
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)

    self.install_examples()

    Document.objects.sync()
コード例 #3
0
ファイル: oozie_setup.py プロジェクト: shobull/hue
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)

    self.install_examples()

    Document.objects.sync()
コード例 #4
0
ファイル: oozie_setup.py プロジェクト: 277800076/hue
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(self.user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=self.user,
      name=Document2.EXAMPLES_DIR
    )

    # Share oozie examples with default group
    oozie_examples = Document2.objects.filter(
      type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
      owner=self.user,
      parent_directory=None
    )
    oozie_examples.update(parent_directory=examples_dir)
    examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])

    self.install_examples()

    Document.objects.sync()
コード例 #5
0
def create_directories(fs):
  # If needed, create the remote home, deployment and data directories
  directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get())

  for directory in directories:
    if not fs.do_as_user("hdfs", fs.exists, directory):
      remote_home_dir = Hdfs.join('/user', "hdfs")
      if directory.startswith(remote_home_dir):
        # Home is 755
        fs.do_as_user("hdfs", fs.create_home_dir, remote_home_dir)
      # Shared by all the users
      fs.do_as_user("hdfs", fs.mkdir, directory, 511)
      fs.do_as_user("hdfs", fs.chmod, directory, 511) # To remove after https://issues.apache.org/jira/browse/HDFS-3491

  return REMOTE_SAMPLE_DIR.get()
コード例 #6
0
def create_directories(fs):
  # If needed, create the remote home, deployment and data directories
  directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get())

  for directory in directories:
    if not fs.do_as_user(fs.DEFAULT_USER, fs.exists, directory):
      remote_home_dir = Hdfs.join('/user', fs.DEFAULT_USER)
      if directory.startswith(remote_home_dir):
        # Home is 755
        fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir)
      # Shared by all the users
      fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777)
      fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491

  return REMOTE_SAMPLE_DIR.get()
コード例 #7
0
ファイル: oozie_setup.py プロジェクト: kthguru/hue
def create_data_dir(fs):
  # If needed, create the remote home, deployment and data directories
  directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get())
  user = fs.user

  try:
    fs.setuser(fs.DEFAULT_USER)
    for directory in directories:
      if not fs.exists(directory):
        remote_home_dir = Hdfs.join('/user', fs.user)
        if directory.startswith(remote_home_dir):
          # Home is 755
          fs.create_home_dir(remote_home_dir)
        # Shared by all the users
        fs.mkdir(directory, 01777)
        fs.chmod(directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
  finally:
    fs.setuser(user)

  return REMOTE_SAMPLE_DIR.get()
コード例 #8
0
  def handle_noargs(self, **options):
    fs = cluster.get_hdfs()
    sample_user = CreateSandboxUserCommand().handle_noargs()
    fs.setuser(sample_user)
    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()
    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      fs.copyFromLocal(local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    fs.copyFromLocal(local_dir, remote_data_dir)

    # Load jobs

    management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
コード例 #9
0
ファイル: oozie_api_tests.py プロジェクト: ziq211/hue
    def test_workspace_has_enough_permissions(self):
        reset = REMOTE_SAMPLE_DIR.set_for_testing(
            '/tmp/oozie_test_workspace_has_enough_permissions')
        try:
            resp = self.cli.get('/desktop/debug/check_config')
            assert_false('The permissions of workspace' in resp.content, resp)

            self.cluster.fs.mkdir(REMOTE_SAMPLE_DIR.get())
            assert_equal(
                oct(0o40755),
                oct(self.cluster.fs.stats(REMOTE_SAMPLE_DIR.get())["mode"]))
            resp = self.cli.get('/desktop/debug/check_config')
            assert_true('The permissions of workspace' in resp.content, resp)

            permissions_dict = {
                'group_read': True,
                'other_execute': True,
                'user_write': True,
                'user_execute': True,
                'sticky': False,
                'user_read': True,
                'other_read': True,
                'other_write': True,
                'group_write': False,
                'group_execute': True
            }

            kwargs = {'path': [REMOTE_SAMPLE_DIR.get()]}
            kwargs.update(permissions_dict)

            # Add write permission to Others
            response = self.cli.post("/filebrowser/chmod", kwargs)
            assert_equal(
                oct(0o40757),
                oct(self.cluster.fs.stats(REMOTE_SAMPLE_DIR.get())["mode"]))

            resp = self.cli.get('/desktop/debug/check_config')
            assert_false('The permissions of workspace' in resp.content, resp)

        finally:
            self.cluster.fs.rmdir(REMOTE_SAMPLE_DIR.get(), skip_trash=True)
            reset()
コード例 #10
0
ファイル: conf.py プロジェクト: zzzharpreet/hue
    res = []

    try:
        from oozie.conf import REMOTE_SAMPLE_DIR
    except Exception, e:
        LOG.warn('Config check failed because Oozie app not installed: %s' % e)
        return res

    if OOZIE_URL.get():
        status = get_oozie_status(user)
        if 'NORMAL' not in status:
            res.append((status, _('The Oozie server is not available')))
        fs = get_filesystem()
        NICE_NAME = 'Oozie'
        if fs.do_as_superuser(fs.exists, REMOTE_SAMPLE_DIR.get()):
            stats = fs.do_as_superuser(fs.stats, REMOTE_SAMPLE_DIR.get())
            mode = oct(stats.mode)
            # if neither group nor others have write permission
            group_has_write = int(mode[-2]) & 2
            others_has_write = int(mode[-1]) & 2

            if not group_has_write and not others_has_write:
                res.append(
                    (NICE_NAME,
                     "The permissions of workspace '%s' are too restrictive" %
                     REMOTE_SAMPLE_DIR.get()))

        api = get_oozie(user, api_version="v2")

        configuration = api.get_configuration()
コード例 #11
0
ファイル: oozie_setup.py プロジェクト: igloosec/hue
    def handle_noargs(self, **options):
        self.user = install_sample_user()
        self.fs = cluster.get_hdfs()

        LOG.info(
            _("Creating sample directory '%s' in HDFS") %
            REMOTE_SAMPLE_DIR.get())
        create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = self.fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
                               local_dir, remote_data_dir)

        # Copy sample data
        local_dir = LOCAL_SAMPLE_DATA_DIR.get()
        remote_data_dir = self.fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
                           local_dir, remote_data_dir)

        # Load jobs
        LOG.info(_("Installing examples..."))

        if ENABLE_V2.get():
            management.call_command('loaddata',
                                    'initial_oozie_examples.json',
                                    verbosity=2)

        # Get or create sample user directories
        home_dir = Directory.objects.get_home_directory(self.user)
        examples_dir, created = Directory.objects.get_or_create(
            parent_directory=home_dir,
            owner=self.user,
            name=Document2.EXAMPLES_DIR)

        # Share oozie examples with default group
        oozie_examples = Document2.objects.filter(type__in=[
            'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'
        ],
                                                  owner=self.user,
                                                  parent_directory=None)
        oozie_examples.update(parent_directory=examples_dir)
        examples_dir.share(self.user,
                           Document2Permission.READ_PERM,
                           groups=[get_default_user_group()])

        self.install_examples()

        Document.objects.sync()
コード例 #12
0
ファイル: oozie_setup.py プロジェクト: cloudera/hue
  def handle(self, *args, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(self.user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=self.user,
      name=Document2.EXAMPLES_DIR
    )

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      with transaction.atomic():
        management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False)

    # Install editor oozie examples without doc1 link
    LOG.info("Using Hue 4, will install oozie editor samples.")

    example_jobs = []
    example_jobs.append(self._install_mapreduce_example())
    example_jobs.append(self._install_java_example())
    example_jobs.append(self._install_spark_example())
    example_jobs.append(self._install_pyspark_example())

    # If documents exist but have been trashed, recover from Trash
    for doc in example_jobs:
      if doc is not None and doc.parent_directory != examples_dir:
        doc.parent_directory = examples_dir
        doc.save()

    # Share oozie examples with default group
    oozie_examples = Document2.objects.filter(
      type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
      owner=self.user,
      parent_directory=None
    )
    oozie_examples.update(parent_directory=examples_dir)
    examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
コード例 #13
0
ファイル: oozie_setup.py プロジェクト: 18600597055/hue
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(self.user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=self.user,
      name=Document2.EXAMPLES_DIR
    )

    if USE_NEW_EDITOR.get():
      docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
      for doc in docs:
        if doc.content_object:
          data = doc.content_object.data_dict
          data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
          data = json.dumps(data)

          doc2 = Document2.objects.create(
            owner=self.user,
            parent_directory=examples_dir,
            name=doc.name,
            type='link-workflow',
            description=doc.description,
            data=data)

          LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))

    # Share oozie examples with default group
    oozie_examples = Document2.objects.filter(
      type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
      owner=self.user,
      parent_directory=None
    )
    oozie_examples.update(parent_directory=examples_dir)
    examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])

    self.install_examples()

    Document.objects.sync()
コード例 #14
0
def config_validator(user):
    """
  config_validator() -> [ (config_variable, error_message) ]

  Called by core check_config() view.
  """
    from desktop.lib.fsmanager import get_filesystem
    from hadoop.cluster import get_all_hdfs
    from hadoop.fs.hadoopfs import Hdfs
    from liboozie.oozie_api import get_oozie

    res = []

    try:
        from oozie.conf import REMOTE_SAMPLE_DIR
    except Exception as e:
        LOG.warn('Config check failed because Oozie app not installed: %s' % e)
        return res

    if OOZIE_URL.get():
        status = get_oozie_status(user)
        if 'NORMAL' not in status:
            res.append((status, _('The Oozie server is not available')))
        fs = get_filesystem()
        NICE_NAME = 'Oozie'
        if fs.do_as_superuser(fs.exists, REMOTE_SAMPLE_DIR.get()):
            stats = fs.do_as_superuser(fs.stats, REMOTE_SAMPLE_DIR.get())
            mode = oct(stats.mode)
            # if neither group nor others have write permission
            group_has_write = int(mode[-2]) & 2
            others_has_write = int(mode[-1]) & 2

            if not group_has_write and not others_has_write:
                res.append(
                    (NICE_NAME,
                     "The permissions of workspace '%s' are too restrictive" %
                     REMOTE_SAMPLE_DIR.get()))

        api = get_oozie(user, api_version="v2")

        configuration = api.get_configuration()
        if 'org.apache.oozie.service.MetricsInstrumentationService' in [
                c.strip()
                for c in configuration.get('oozie.services.ext', '').split(',')
        ]:
            metrics = api.get_metrics()
            sharelib_url = 'gauges' in metrics and 'libs.sharelib.system.libpath' in metrics[
                'gauges'] and [
                    metrics['gauges']['libs.sharelib.system.libpath']['value']
                ] or []
        else:
            intrumentation = api.get_instrumentation()
            sharelib_url = [
                param['value'] for group in intrumentation['variables']
                for param in group['data']
                if param['name'] == 'sharelib.system.libpath'
            ]

        if sharelib_url:
            sharelib_url = Hdfs.urlsplit(sharelib_url[0])[2]

        if not sharelib_url:
            res.append((status, _('Oozie Share Lib path is not available')))

        class ConfigMock(object):
            def __init__(self, value):
                self.value = value

            def get(self):
                return self.value

            def get_fully_qualifying_key(self):
                return self.value

        for cluster in list(get_all_hdfs().values()):
            res.extend(
                validate_path(
                    ConfigMock(sharelib_url),
                    is_dir=True,
                    fs=cluster,
                    message=_(
                        'Oozie Share Lib not installed in default location.')))

    return res
コード例 #15
0
ファイル: oozie_setup.py プロジェクト: lorelib/hue
  def handle(self, *args, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(self.user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=self.user,
      name=Document2.EXAMPLES_DIR
    )

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      with transaction.atomic():
        management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False)

    if IS_HUE_4.get():
      # Install editor oozie examples without doc1 link
      LOG.info("Using Hue 4, will install oozie editor samples.")

      example_jobs = []
      example_jobs.append(self._install_mapreduce_example())
      example_jobs.append(self._install_java_example())
      example_jobs.append(self._install_spark_example())
      example_jobs.append(self._install_pyspark_example())

      # If documents exist but have been trashed, recover from Trash
      for doc in example_jobs:
        if doc is not None and doc.parent_directory != examples_dir:
          doc.parent_directory = examples_dir
          doc.save()

    elif USE_NEW_EDITOR.get():
      # Install as link-workflow doc2 to old Job Designs
      docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
      for doc in docs:
        if doc.content_object:
          data = doc.content_object.data_dict
          data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
          data = json.dumps(data)

          # Don't overwrite
          doc2, created = Document2.objects.get_or_create(
            owner=self.user,
            parent_directory=examples_dir,
            name=doc.name,
            type='link-workflow',
            description=doc.description,
            data=data
          )

          LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))

    # Share oozie examples with default group
    oozie_examples = Document2.objects.filter(
      type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
      owner=self.user,
      parent_directory=None
    )
    oozie_examples.update(parent_directory=examples_dir)
    examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])

    if not IS_HUE_4.get():
      self.install_examples()
      Document.objects.sync()
コード例 #16
0
  res = []

  try:
    from oozie.conf import REMOTE_SAMPLE_DIR
  except Exception, e:
    LOG.warn('Config check failed because Oozie app not installed: %s' % e)
    return res

  if OOZIE_URL.get():
    status = get_oozie_status(user)
    if 'NORMAL' not in status:
      res.append((status, _('The Oozie server is not available')))
    fs = get_filesystem()
    NICE_NAME = 'Oozie'
    if fs.exists(REMOTE_SAMPLE_DIR.get()):
      stats = fs.stats(REMOTE_SAMPLE_DIR.get())
      mode = oct(stats.mode)
      # if neither group nor others have write permission
      group_has_write = int(mode[-2]) & 2
      others_has_write = int(mode[-1]) & 2

      if not group_has_write and not others_has_write:
        res.append((NICE_NAME, "The permissions of workspace '%s' are too restrictive" % REMOTE_SAMPLE_DIR.get()))

    api = get_oozie(user, api_version="v2")

    configuration = api.get_configuration()
    if 'org.apache.oozie.service.MetricsInstrumentationService' in [c.strip() for c in configuration.get('oozie.services.ext', '').split(',')]:
      metrics = api.get_metrics()
      sharelib_url = 'gauges' in metrics and 'libs.sharelib.system.libpath' in metrics['gauges'] and [metrics['gauges']['libs.sharelib.system.libpath']['value']] or []
コード例 #17
0
    def handle_noargs(self, **options):
        self.user = install_sample_user()
        self.fs = cluster.get_hdfs()

        LOG.info(
            _("Creating sample directory '%s' in HDFS") %
            REMOTE_SAMPLE_DIR.get())
        create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = self.fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
                               local_dir, remote_data_dir)

        # Copy sample data
        local_dir = LOCAL_SAMPLE_DATA_DIR.get()
        remote_data_dir = self.fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
                           local_dir, remote_data_dir)

        # Load jobs
        LOG.info(_("Installing examples..."))

        if ENABLE_V2.get():
            management.call_command('loaddata',
                                    'initial_oozie_examples.json',
                                    verbosity=2)

        # Get or create sample user directories
        home_dir = Directory.objects.get_home_directory(self.user)
        examples_dir, created = Directory.objects.get_or_create(
            parent_directory=home_dir,
            owner=self.user,
            name=Document2.EXAMPLES_DIR)

        if USE_NEW_EDITOR.get():
            docs = Document.objects.get_docs(self.user,
                                             Workflow).filter(owner=self.user)
            for doc in docs:
                if doc.content_object:
                    data = doc.content_object.data_dict
                    data.update({
                        'content_type': doc.content_type.model,
                        'object_id': doc.object_id
                    })
                    data = json.dumps(data)

                    doc2 = Document2.objects.create(
                        owner=self.user,
                        parent_directory=examples_dir,
                        name=doc.name,
                        type='link-workflow',
                        description=doc.description,
                        data=data)

                    LOG.info(
                        'Successfully installed sample link to jobsub: %s' %
                        (doc2.name, ))

        # Share oozie examples with default group
        oozie_examples = Document2.objects.filter(type__in=[
            'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'
        ],
                                                  owner=self.user,
                                                  parent_directory=None)
        oozie_examples.update(parent_directory=examples_dir)
        examples_dir.share(self.user,
                           Document2Permission.READ_PERM,
                           groups=[get_default_user_group()])

        self.install_examples()

        Document.objects.sync()
コード例 #18
0
ファイル: oozie_setup.py プロジェクト: dulems/hue-1
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(self.user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=self.user,
      name=Document2.EXAMPLES_DIR
    )

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)

    if IS_HUE_4.get():
      # Install editor oozie examples without doc1 link
      LOG.info("Using Hue 4, will install oozie editor samples.")

      example_jobs = []
      example_jobs.append(self._install_mapreduce_example())
      example_jobs.append(self._install_java_example())
      example_jobs.append(self._install_spark_example())
      example_jobs.append(self._install_pyspark_example())

      # If documents exist but have been trashed, recover from Trash
      for doc in example_jobs:
        if doc is not None and doc.parent_directory != examples_dir:
          doc.parent_directory = examples_dir
          doc.save()

    elif USE_NEW_EDITOR.get():
      # Install as link-workflow doc2 to old Job Designs
      docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
      for doc in docs:
        if doc.content_object:
          data = doc.content_object.data_dict
          data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
          data = json.dumps(data)

          # Don't overwrite
          doc2, created = Document2.objects.get_or_create(
            owner=self.user,
            parent_directory=examples_dir,
            name=doc.name,
            type='link-workflow',
            description=doc.description,
            data=data
          )

          LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))

    # Share oozie examples with default group
    oozie_examples = Document2.objects.filter(
      type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
      owner=self.user,
      parent_directory=None
    )
    oozie_examples.update(parent_directory=examples_dir)
    examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])

    if not IS_HUE_4.get():
      self.install_examples()
      Document.objects.sync()
コード例 #19
0
    def handle(self, *args, **options):
        self.user = install_sample_user()
        self.fs = cluster.get_hdfs()

        LOG.info(
            _("Creating sample directory '%s' in HDFS") %
            REMOTE_SAMPLE_DIR.get())
        create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = self.fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            self.fs.do_as_user(self.user.username, self.fs.copyFromLocal,
                               local_dir, remote_data_dir)

        # Copy sample data
        local_dir = LOCAL_SAMPLE_DATA_DIR.get()
        remote_data_dir = self.fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        self.fs.do_as_user(self.user.username, self.fs.copyFromLocal,
                           local_dir, remote_data_dir)

        # Get or create sample user directories
        home_dir = Directory.objects.get_home_directory(self.user)
        examples_dir, created = Directory.objects.get_or_create(
            parent_directory=home_dir,
            owner=self.user,
            name=Document2.EXAMPLES_DIR)

        # Load jobs
        LOG.info(_("Installing examples..."))

        if ENABLE_V2.get():
            with transaction.atomic():
                management.call_command('loaddata',
                                        'initial_oozie_examples.json',
                                        verbosity=2,
                                        commit=False)

        # Install editor oozie examples without doc1 link
        LOG.info("Using Hue 4, will install oozie editor samples.")

        example_jobs = []
        example_jobs.append(self._install_mapreduce_example())
        example_jobs.append(self._install_java_example())
        example_jobs.append(self._install_spark_example())
        example_jobs.append(self._install_pyspark_example())

        # If documents exist but have been trashed, recover from Trash
        for doc in example_jobs:
            if doc is not None and doc.parent_directory != examples_dir:
                doc.parent_directory = examples_dir
                doc.save()

        # Share oozie examples with default group
        oozie_examples = Document2.objects.filter(type__in=[
            'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'
        ],
                                                  owner=self.user,
                                                  parent_directory=None)
        oozie_examples.update(parent_directory=examples_dir)
        examples_dir.share(self.user,
                           Document2Permission.READ_PERM,
                           groups=[get_default_user_group()])