Ejemplo n.º 1
0
    def handle_noargs(self, **options):
        fs = cluster.get_hdfs()
        create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
                          remote_data_dir)

        # Copy sample data
        local_dir = paths.get_thirdparty_root("sample_data")
        remote_data_dir = fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
                      remote_data_dir)

        # Load jobs
        install_sample_user()
        management.call_command('loaddata',
                                'initial_pig_examples.json',
                                verbosity=2)
        Document.objects.sync()
Ejemplo n.º 2
0
  def handle_noargs(self, **options):

    if not Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).exists():
      install_sample_user()

      management.call_command('loaddata', 'initial_search_examples.json', verbosity=2)
      Document.objects.sync()
Ejemplo n.º 3
0
    def handle_noargs(self, **options):

        if not Document2.objects.filter(
                type='search-dashboard',
                owner__username=SAMPLE_USERNAME).exists():
            install_sample_user()

            management.call_command('loaddata',
                                    'initial_search_examples.json',
                                    verbosity=2)
            Document.objects.sync()
Ejemplo n.º 4
0
  def handle(self, *args, **options):
    if not options.get('user'):
      user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
    else:
      user = options['user']

    install_sample_user()

    management.call_command('loaddata', 'apps/spark/src/spark/fixtures/initial_spark_examples.json', verbosity=2)
    Document.objects.sync()

    from beeswax.management.commands.beeswax_install_examples import Command
    app_name = 'beeswax'
    Command().handle(app_name=app_name, user=user, tables='web_logs_table.json')
Ejemplo n.º 5
0
  def handle(self, *args, **options):
    if not options.get('user'):
      user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
    else:
      user = options['user']

    if not Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).exists():
      install_sample_user()

      management.call_command('loaddata', 'apps/spark/src/spark/fixtures/initial_spark_examples.json', verbosity=2)
      Document.objects.sync()

    from beeswax.management.commands.beeswax_install_examples import Command
    app_name = 'beeswax'
    Command().handle(app_name=app_name, user=user, tables='web_logs_table.json')
Ejemplo n.º 6
0
  def handle(self, *args, **options):
    if not options.get('user'):
      user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
    else:
      user = options['user']

    if not Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).exists():
      install_sample_user()

      management.call_command('loaddata', 'desktop/libs/notebook/src/notebook/fixtures/initial_notebook_examples.json', verbosity=2)
      Document.objects.sync()

    from beeswax.management.commands.beeswax_install_examples import Command
    app_name = 'beeswax'
    Command().handle(app_name=app_name, user=user, tables='tables.json')
Ejemplo n.º 7
0
  def handle(self, *args, **options):
    if not options.get('user'):
      user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
    else:
      user = options['user']

    # Install sample notebook from fixture if notebook with sample UUID doesn't exist
    if not Document2.objects.filter(uuid="7f2ea775-e067-4fde-8f5f-4d704ab9b002").exists():
      sample_user = install_sample_user()

      management.call_command('loaddata', 'initial_notebook_examples.json', verbosity=2)
      Document.objects.sync()

      # Get or create sample user directories
      home_dir = Directory.objects.get_home_directory(sample_user)
      examples_dir, created = Directory.objects.get_or_create(
        parent_directory=home_dir,
        owner=sample_user,
        name=Document2.EXAMPLES_DIR
      )

      Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir)

      # Share with default group
      examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
      LOG.info('Successfully installed sample notebook')

    from beeswax.management.commands.beeswax_install_examples import Command
    app_name = 'beeswax'
    Command().handle(app_name=app_name, user=user, tables='tables.json')
Ejemplo n.º 8
0
  def authenticate(self, request=None, username=None, password=None):
    username = force_username_case(username)

    if AUTH.PAM_USE_PWD_MODULE.get():
      LOG.debug('Setting username to %s using PAM pwd module for user %s' % (getpwnam(username).pw_name, username))
      username = getpwnam(username).pw_name

    if pam.authenticate(username, password, AUTH.PAM_SERVICE.get()):
      is_super = False
      if User.objects.exclude(id=install_sample_user().id).count() == 0:
        is_super = True

      try:
        if AUTH.IGNORE_USERNAME_CASE.get():
          user = User.objects.get(username__iexact=username)
        else:
          user = User.objects.get(username=username)
      except User.DoesNotExist:
        user = find_or_create_user(username, None)
        if user is not None and user.is_active:
          profile = get_profile(user)
          profile.creation_method = UserProfile.CreationMethod.EXTERNAL.name
          profile.save()
          user.is_superuser = is_super

          ensure_has_a_group(user)

          user.save()

      user = rewrite_user(user)
      return user

    return None
Ejemplo n.º 9
0
  def authenticate(self, request=None, remote_user=None):
    username = self.clean_username(remote_user)
    username = force_username_case(username)
    is_super = False
    if User.objects.exclude(id=install_sample_user().id).count() == 0:
      is_super = True

    try:
      if AUTH.IGNORE_USERNAME_CASE.get():
        user = User.objects.get(username__iexact=username)
      else:
        user = User.objects.get(username=username)
    except User.DoesNotExist:
      user = find_or_create_user(username, None)
      if user is not None and user.is_active:
        profile = get_profile(user)
        profile.creation_method = UserProfile.CreationMethod.EXTERNAL.name
        profile.save()
        user.is_superuser = is_super

        ensure_has_a_group(user)

        user.save()

    user = rewrite_user(user)
    return user
Ejemplo n.º 10
0
def create_table(user=None, query_server=None, table=None):
    if not user:
        user = install_sample_user()
    if not query_server:
        query_server = get_query_server_config('beeswax')
    if not table:
        base_dir = hive_site.get_hive_hook_proto_base_directory()
        if not base_dir:
            msg = _(
                'Error creating table query_data hive.hook.proto.base-directory is not configured'
            )
            LOG.error(msg)
            return False
        table = {'name': 'query_data', 'external_location': base_dir}

    server = dbms.get(user, query_server)
    for query in ["create_table_query_data.mako", "msck.mako"]:
        proposed_query = django_mako.render_to_string(query, {'table': table})
        query = hql_query(proposed_query)
        try:
            handle = server.execute_and_wait(query)
            if not handle:
                LOG.error(_('Error executing %s: Operation timeout.' % query))
                return False
            server.close(handle)
        except Exception as ex:
            LOG.error(
                _('Error executing %(query)s: %(error)s.') % {
                    'query': query,
                    'error': ex
                })
            return False

    LOG.info(_('Table query_data has been created successfully'))
    return True
Ejemplo n.º 11
0
  def handle(self, *args, **options):
    if not options.get('user'):
      user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
    else:
      user = options['user']
    dialect = options.get('dialect', 'hive')

    # Install sample notebook from fixture if notebook with sample UUID doesn't exist
    if not Document2.objects.filter(uuid="7f2ea775-e067-4fde-8f5f-4d704ab9b002").exists():
      sample_user = install_sample_user()

      with transaction.atomic():
        management.call_command('loaddata', 'initial_notebook_examples.json', verbosity=2, commit=False)
        Document.objects.sync()

      # Get or create sample user directories
      home_dir = Directory.objects.get_home_directory(sample_user)
      examples_dir, created = Directory.objects.get_or_create(
        parent_directory=home_dir,
        owner=sample_user,
        name=Document2.EXAMPLES_DIR
      )

      Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir)

      # Share with default group
      examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
      LOG.info('Successfully installed sample notebook')

    from beeswax.management.commands.beeswax_install_examples import Command
    Command().handle(dialect=dialect, user=user)
Ejemplo n.º 12
0
def _execute_query(proposed_query, limit):
    user = install_sample_user()
    query_server = get_query_server_config('beeswax')
    server = dbms.get(user, query_server)
    query = hql_query(proposed_query)
    try:
        handle = server.execute_and_wait(query)
        if not handle:
            LOG.error(_('Error executing %s: Operation timeout.' % query))
            return []
        results = server.fetch(handle, True, limit)
        rows = [row for row in results.rows()]
        data = {
            'data': rows,
            'columns': [column.name for column in results.data_table.cols()]
        }

        return data
    except Exception as ex:
        raise_popup_exception(_('Error fetching query history.'))
    finally:
        try:
            if server and handle:
                server.close(handle)
        except Exception as ex:
            raise_popup_exception(_('Error fetching query history.'))
Ejemplo n.º 13
0
def install_custom_examples():
    if EXAMPLES.AUTO_LOAD.get():
        from desktop.auth.backend import rewrite_user
        from beeswax.management.commands import beeswax_install_examples
        from useradmin.models import install_sample_user

        user = rewrite_user(install_sample_user())

        if has_connectors():
            interpreters = [{
                'type': connector['id'],
                'dialect': connector['dialect']
            } for connector in _get_installed_connectors(category='editor')]
        else:
            interpreters = [
                {
                    'type': interpreter['dialect'],
                    'dialect': interpreter['dialect']
                } for interpreter in get_ordered_interpreters(user)
                # Only for hive/impala currently, would also need to port to Notebook install examples.
                if interpreter['dialect'] in ('hive', 'impala')
            ]

        queries = EXAMPLES.QUERIES.get()
        tables = EXAMPLES.TABLES.get(
        )  # No-op. Only for the saved query samples, not the tables currently.

        LOG.info(
            'Installing custom examples queries: %(queries)s, tables: %(tables)s for dialects %(dialects)s '
            'belonging to user %(user)s' % {
                'queries':
                queries,
                'tables':
                tables,
                'dialects':
                [interpreter['dialect'] for interpreter in interpreters],
                'user':
                user
            })

        result = []

        for interpreter in interpreters:
            successes, errors = beeswax_install_examples.Command().handle(
                dialect=interpreter['dialect'],
                user=user,
                interpreter=interpreter,
                queries=queries,
                tables=tables,
                request=None)
            LOG.info(
                'Dialect %(dialect)s installed samples: %(successes)s, %(errors)s,'
                % {
                    'dialect': interpreter['dialect'],
                    'successes': successes,
                    'errors': errors,
                })
            result.append((successes, errors))

        return result
Ejemplo n.º 14
0
  def handle(self, *args, **options):

    sample_user = install_sample_user()

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(sample_user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=sample_user,
      name=Document2.EXAMPLES_DIR
    )

    if not Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).exists():
      with transaction.atomic():
        if sys.version_info[0] > 2:
          management.call_command('loaddata', 'initial_search_examples.json', verbosity=2)
        else:
          management.call_command('loaddata', 'initial_search_examples.json', verbosity=2, commit=False)
        Document.objects.sync()

      Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir)
    else:
      # Check if sample documents are in Trash, and if so, restore them
      for doc in Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS):
        if doc.parent_directory != examples_dir:
          doc.parent_directory = examples_dir
          doc.save()

    # Share with default group
    examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
    LOG.info('Successfully installed sample search dashboard')
Ejemplo n.º 15
0
 def is_first_login_ever(self):
     """ Return true if no one has ever logged in."""
     if ENABLE_ORGANIZATIONS.get():  # Tmp
         return False
     else:
         return User.objects.exclude(
             id=install_sample_user().id).count() == 0
Ejemplo n.º 16
0
  def handle_noargs(self, **options):
    fs = cluster.get_hdfs()
    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = paths.get_thirdparty_root("sample_data")
    remote_data_dir = fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    sample_user = install_sample_user()
    management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2)
    Document.objects.sync()

    if USE_NEW_EDITOR.get():
      # Get or create sample user directories
      home_dir = Directory.objects.get_home_directory(sample_user)
      examples_dir, created = Directory.objects.get_or_create(
        parent_directory=home_dir,
        owner=sample_user,
        name=Document2.EXAMPLES_DIR)

      try:
        # Don't overwrite
        doc = Document.objects.get(object_id=1100713)
        doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript')
        # If document exists but has been trashed, recover from Trash
        if doc2.parent_directory != examples_dir:
          doc2.parent_directory = examples_dir
          doc2.save()
      except Document.DoesNotExist:
        LOG.warn('Sample pig script document not found.')
      except Document2.DoesNotExist:
        if doc.content_object:
          data = doc.content_object.dict
          data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
          data = json.dumps(data)

          doc2 = Document2.objects.create(
            owner=sample_user,
            parent_directory=examples_dir,
            name=doc.name,
            type='link-pigscript',
            description=doc.description,
            data=data)
          LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,))

      # Share with default group
      examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
Ejemplo n.º 17
0
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)

    self.install_examples()

    Document.objects.sync()
Ejemplo n.º 18
0
  def handle_noargs(self, **options):

    sample_user = install_sample_user()

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(sample_user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=sample_user,
      name=Document2.EXAMPLES_DIR
    )

    if not Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).exists():
      management.call_command('loaddata', 'initial_search_examples.json', verbosity=2)
      Document.objects.sync()

      Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir)
    else:
      # Check if sample documents are in Trash, and if so, restore them
      for doc in Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS):
        if doc.parent_directory != examples_dir:
          doc.parent_directory = examples_dir
          doc.save()

    # Share with default group
    examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
    LOG.info('Successfully installed sample search dashboard')
Ejemplo n.º 19
0
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)

    self.install_examples()

    Document.objects.sync()
Ejemplo n.º 20
0
def _get_dialect_example(dialect):
    sample_user = install_sample_user()
    examples_dir = _get_example_directory(sample_user)

    return Document2.objects.filter(owner=sample_user,
                                    type='query-%s' % dialect,
                                    is_history=False,
                                    parent_directory=examples_dir).first()
Ejemplo n.º 21
0
  def handle_noargs(self, **options):
    exception = None

    try:
      user = install_sample_user()
      self._install_tables(user, options['app_name'])
    except Exception, ex:
      exception = ex
Ejemplo n.º 22
0
 def handle_noargs(self, **options):
   """Main entry point to install or re-install examples. May raise InstallException"""
   try:
     user = install_sample_user()
     self._install_tables(user, options['app_name'])
     self._install_queries(user, options['app_name'])
   except Exception, ex:
     LOG.exception(ex)
     raise InstallException(ex)
Ejemplo n.º 23
0
  def handle_noargs(self, **options):
    exception = None

    # Documents will belong to this user but we run the install as the current user
    try:
      sample_user = install_sample_user()
      self._install_tables(options['user'], options['app_name'])
    except Exception, ex:
      exception = ex
Ejemplo n.º 24
0
 def handle_noargs(self, **options):
     """Main entry point to install or re-install examples. May raise InstallException"""
     try:
         user = install_sample_user()
         self._install_tables(user, options['app_name'])
         self._install_queries(user, options['app_name'])
     except Exception, ex:
         LOG.exception(ex)
         raise InstallException(ex)
Ejemplo n.º 25
0
    def handle(self, *args, **options):
        fs = cluster.get_hdfs()
        create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()
        sample_user = install_sample_user()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir,
                          remote_data_dir)

        # Copy sample data
        local_dir = paths.get_thirdparty_root("sample_data")
        remote_data_dir = fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir,
                      remote_data_dir)

        # Initialize doc2, whether editor script or link
        doc2 = None

        # Install editor pig script without doc1 link
        LOG.info("Using Hue 4, will install pig editor sample.")
        doc2 = self.install_pig_script(sample_user)

        if USE_NEW_EDITOR.get():
            # Get or create sample user directories
            LOG.info("Creating sample user directories.")

            home_dir = Directory.objects.get_home_directory(sample_user)
            examples_dir, created = Directory.objects.get_or_create(
                parent_directory=home_dir,
                owner=sample_user,
                name=Document2.EXAMPLES_DIR)

            # If document exists but has been trashed, recover from Trash
            if doc2 and doc2.parent_directory != examples_dir:
                doc2.parent_directory = examples_dir
                doc2.save()

            # Share with default group
            examples_dir.share(sample_user,
                               Document2Permission.READ_PERM,
                               groups=[get_default_user_group()])
Ejemplo n.º 26
0
    def handle(self, *args, **options):
        if args:
            dialect = args[0]
            db_name = args[1] if len(args) > 1 else 'default'
            user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
            request = None
        else:
            dialect = options['dialect']
            db_name = options.get('db_name', 'default')
            interpreter = options.get(
                'interpreter'
            )  # Only when connectors are enabled. Later will deprecate `dialect`.
            user = options['user']
            request = options['request']

        tables = 'tables_standard.json' if dialect not in (
            'hive',
            'impala') else ('tables_transactional.json'
                            if has_concurrency_support() else 'tables.json')
        exception = None

        self.successes = []
        self.errors = []
        try:
            sample_user = install_sample_user(
                user
            )  # Documents will belong to the sample user but we run the SQL as the current user
            self.install_queries(sample_user, dialect, interpreter=interpreter)
            self.install_tables(user,
                                dialect,
                                db_name,
                                tables,
                                interpreter=interpreter,
                                request=request)
        except Exception as ex:
            exception = ex

        if exception is not None:
            pretty_msg = None

            if "Permission denied" in str(exception):
                pretty_msg = _(
                    "Permission denied. Please check with your system administrator."
                )

            if pretty_msg is not None:
                raise PopupException(pretty_msg)
            else:
                raise exception

        return self.successes, self.errors
Ejemplo n.º 27
0
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(self.user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=self.user,
      name=Document2.EXAMPLES_DIR
    )

    # Share oozie examples with default group
    oozie_examples = Document2.objects.filter(
      type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
      owner=self.user,
      parent_directory=None
    )
    oozie_examples.update(parent_directory=examples_dir)
    examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])

    self.install_examples()

    Document.objects.sync()
Ejemplo n.º 28
0
  def handle_noargs(self, **options):
    fs = cluster.get_hdfs()
    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = paths.get_thirdparty_root("sample_data")
    remote_data_dir = fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    install_sample_user()
    management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2)
    Document.objects.sync()
Ejemplo n.º 29
0
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()
    self.searcher = controller.CollectionManagerController(self.user)

    LOG.info(_("Installing twitter collection"))
    path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_twitter_demo/index_data.csv'))
    self._setup_collection_from_csv({
      'name': 'twitter_demo',
      'fields': self._parse_fields(path),
      'uniqueKeyField': 'id',
      'df': 'text'
    }, path)
    LOG.info(_("Twitter collection successfully installed"))

    LOG.info(_("Installing yelp collection"))
    path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_yelp_demo/index_data.csv'))
    self._setup_collection_from_csv({
      'name': 'yelp_demo',
      'fields': self._parse_fields(path),
      'uniqueKeyField': 'id',
      'df': 'text'
    }, path)
    LOG.info(_("Yelp collection successfully installed"))

    LOG.info(_("Installing jobs collection"))
    path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_jobs_demo/index_data.csv'))
    self._setup_collection_from_csv({
      'name': 'jobs_demo',
      'fields': self._parse_fields(path),
      'uniqueKeyField': 'id',
      'df': 'description'
    }, path)
    LOG.info(_("Jobs collection successfully installed"))

    LOG.info(_("Installing logs collection"))
    path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_log_analytics_demo/index_data.csv'))
    self._setup_collection_from_csv({
      'name': 'log_analytics_demo',
      'fields': self._parse_fields(path, fieldtypes={
        'region_code': 'string',
        'referer': 'string'
      }),
      'uniqueKeyField': 'id',
      'df': 'record'
    }, path)
    LOG.info(_("Logs collection successfully installed"))
  def handle(self, *args, **options):
    if args:
      app_name = args[0]
      user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
    else:
      app_name = options['app_name']
      user = options['user']

    exception = None

    # Documents will belong to this user but we run the install as the current user
    try:
      sample_user = install_sample_user()
      self._install_queries(sample_user, app_name)
      self._install_tables(user, app_name)
    except Exception, ex:
      exception = ex
Ejemplo n.º 31
0
  def handle(self, *args, **options):
    if args:
      user = args[0]
    else:
      user = install_sample_user()

    api = HbaseApi(user=user)
    cluster_name = api.getClusters()[0]['name'] # Currently pick first configured cluster

    # Check connectivity
    api.connectCluster(cluster_name)

    self.create_analytics_table(api, cluster_name)
    self.load_analytics_table(api, cluster_name)

    self.create_binary_table(api, cluster_name)
    self.load_binary_table(api, cluster_name)
Ejemplo n.º 32
0
    def handle(self, *args, **options):
        if args:
            app_name = args[0]
            user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
        else:
            app_name = options['app_name']
            user = options['user']

        exception = None

        # Documents will belong to this user but we run the install as the current user
        try:
            sample_user = install_sample_user()
            self._install_queries(sample_user, app_name)
            self._install_tables(user, app_name)
        except Exception, ex:
            exception = ex
Ejemplo n.º 33
0
  def handle(self, *args, **options):
    fs = cluster.get_hdfs()
    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()
    sample_user = install_sample_user()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = paths.get_thirdparty_root("sample_data")
    remote_data_dir = fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)

    # Initialize doc2, whether editor script or link
    doc2 = None

    # Install editor pig script without doc1 link
    LOG.info("Using Hue 4, will install pig editor sample.")
    doc2 = self.install_pig_script(sample_user)

    if USE_NEW_EDITOR.get():
      # Get or create sample user directories
      LOG.info("Creating sample user directories.")

      home_dir = Directory.objects.get_home_directory(sample_user)
      examples_dir, created = Directory.objects.get_or_create(
        parent_directory=home_dir,
        owner=sample_user,
        name=Document2.EXAMPLES_DIR)

      # If document exists but has been trashed, recover from Trash
      if doc2 and doc2.parent_directory != examples_dir:
        doc2.parent_directory = examples_dir
        doc2.save()

      # Share with default group
      examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
Ejemplo n.º 34
0
    def handle(self, *args, **options):
        if args:
            user = args[0]
        else:
            user = install_sample_user()

        api = HbaseApi(user=user)
        cluster_name = api.getClusters()[0][
            'name']  # Currently pick first configured cluster

        # Check connectivity
        api.connectCluster(cluster_name)

        self.create_analytics_table(api, cluster_name)
        self.load_analytics_table(api, cluster_name)

        self.create_binary_table(api, cluster_name)
        self.load_binary_table(api, cluster_name)
Ejemplo n.º 35
0
  def handle(self, *args, **options):
    if args:
      app_name = args[0]
      db_name = args[1] if len(args) > 1 else 'default'
      user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
    else:
      app_name = options['app_name']
      db_name = options.get('db_name', 'default')
      user = options['user']

    tables = options['tables'] if 'tables' in options else 'tables.json'

    exception = None

    # Documents will belong to this user but we run the install as the current user
    try:
      sample_user = install_sample_user()
      self._install_queries(sample_user, app_name)
      self._install_tables(user, app_name, db_name, tables)
    except Exception, ex:
      exception = ex
Ejemplo n.º 36
0
    def handle(self, *args, **options):
        if args:
            app_name = args[0]
            db_name = args[1] if len(args) > 1 else 'default'
            user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
        else:
            app_name = options['app_name']
            db_name = options.get('db_name', 'default')
            user = options['user']

        tables = options['tables'] if 'tables' in options else (
            'tables_transactional.json'
            if has_concurrency_support() else 'tables.json')

        exception = None

        # Documents will belong to this user but we run the install as the current user
        try:
            sample_user = install_sample_user()
            self._install_queries(sample_user, app_name)
            self._install_tables(user, app_name, db_name, tables)
        except Exception as ex:
            exception = ex

        Document.objects.sync()

        if exception is not None:
            pretty_msg = None

            if "AlreadyExistsException" in exception.message:
                pretty_msg = _("SQL table examples already installed.")
            if "Permission denied" in exception.message:
                pretty_msg = _(
                    "Permission denied. Please check with your system administrator."
                )

            if pretty_msg is not None:
                raise PopupException(pretty_msg)
            else:
                raise exception
Ejemplo n.º 37
0
  def handle(self, *args, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(self.user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=self.user,
      name=Document2.EXAMPLES_DIR
    )

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      with transaction.atomic():
        management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False)

    # Install editor oozie examples without doc1 link
    LOG.info("Using Hue 4, will install oozie editor samples.")

    example_jobs = []
    example_jobs.append(self._install_mapreduce_example())
    example_jobs.append(self._install_java_example())
    example_jobs.append(self._install_spark_example())
    example_jobs.append(self._install_pyspark_example())

    # If documents exist but have been trashed, recover from Trash
    for doc in example_jobs:
      if doc is not None and doc.parent_directory != examples_dir:
        doc.parent_directory = examples_dir
        doc.save()

    # Share oozie examples with default group
    oozie_examples = Document2.objects.filter(
      type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
      owner=self.user,
      parent_directory=None
    )
    oozie_examples.update(parent_directory=examples_dir)
    examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
Ejemplo n.º 38
0
    def handle(self, *args, **options):
        self.user = install_sample_user()
        self.client = SolrClient(self.user)

        collection = options['data']

        if collection == 'twitter_demo':
            LOG.info("Installing twitter collection")
            path = os.path.abspath(
                os.path.join(
                    os.path.dirname(__file__),
                    '../../../../../../../apps/search/examples/collections/solr_configs_twitter_demo/index_data.csv'
                ))
            self._setup_collection_from_csv(
                {
                    'name':
                    'twitter_demo',
                    'fields':
                    self._parse_fields(path,
                                       fieldtypes={
                                           'source': 'string',
                                           'username': '******',
                                       }),
                    'uniqueKeyField':
                    'id',
                    'df':
                    'text'
                }, path)
            LOG.info("Twitter collection successfully installed")

        if collection == 'yelp_demo':
            LOG.info("Installing yelp collection")
            path = os.path.abspath(
                os.path.join(
                    os.path.dirname(__file__),
                    '../../../../../../../apps/search/examples/collections/solr_configs_yelp_demo/index_data.csv'
                ))
            self._setup_collection_from_csv(
                {
                    'name':
                    'yelp_demo',
                    'fields':
                    self._parse_fields(path, fieldtypes={
                        'name': 'string',
                    }),
                    'uniqueKeyField':
                    'id',
                    'df':
                    'text'
                }, path)
            LOG.info("Yelp collection successfully installed")

        if collection == 'log_analytics_demo':
            LOG.info("Installing logs collection")
            path = os.path.abspath(
                os.path.join(
                    os.path.dirname(__file__),
                    '../../../../../../../apps/search/examples/collections/solr_configs_log_analytics_demo/index_data.csv'
                ))
            self._setup_collection_from_csv(
                {
                    'name':
                    'log_analytics_demo',
                    'fields':
                    self._parse_fields(path,
                                       fieldtypes={
                                           'region_code': 'string',
                                           'referer': 'string',
                                           'user_agent': 'string'
                                       }),
                    'uniqueKeyField':
                    'id',
                    'df':
                    'record'
                }, path)
            LOG.info("Logs collection successfully installed")
Ejemplo n.º 39
0
    def handle_noargs(self, **options):
        self.user = install_sample_user()
        self.fs = cluster.get_hdfs()

        LOG.info(
            _("Creating sample directory '%s' in HDFS") %
            REMOTE_SAMPLE_DIR.get())
        create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = self.fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
                               local_dir, remote_data_dir)

        # Copy sample data
        local_dir = LOCAL_SAMPLE_DATA_DIR.get()
        remote_data_dir = self.fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
                           local_dir, remote_data_dir)

        # Load jobs
        LOG.info(_("Installing examples..."))

        if ENABLE_V2.get():
            management.call_command('loaddata',
                                    'initial_oozie_examples.json',
                                    verbosity=2)

        # Get or create sample user directories
        home_dir = Directory.objects.get_home_directory(self.user)
        examples_dir, created = Directory.objects.get_or_create(
            parent_directory=home_dir,
            owner=self.user,
            name=Document2.EXAMPLES_DIR)

        if USE_NEW_EDITOR.get():
            docs = Document.objects.get_docs(self.user,
                                             Workflow).filter(owner=self.user)
            for doc in docs:
                if doc.content_object:
                    data = doc.content_object.data_dict
                    data.update({
                        'content_type': doc.content_type.model,
                        'object_id': doc.object_id
                    })
                    data = json.dumps(data)

                    doc2 = Document2.objects.create(
                        owner=self.user,
                        parent_directory=examples_dir,
                        name=doc.name,
                        type='link-workflow',
                        description=doc.description,
                        data=data)

                    LOG.info(
                        'Successfully installed sample link to jobsub: %s' %
                        (doc2.name, ))

        # Share oozie examples with default group
        oozie_examples = Document2.objects.filter(type__in=[
            'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'
        ],
                                                  owner=self.user,
                                                  parent_directory=None)
        oozie_examples.update(parent_directory=examples_dir)
        examples_dir.share(self.user,
                           Document2Permission.READ_PERM,
                           groups=[get_default_user_group()])

        self.install_examples()

        Document.objects.sync()
Ejemplo n.º 40
0
    def handle_noargs(self, **options):
        self.user = install_sample_user()
        self.fs = cluster.get_hdfs()

        LOG.info(
            _("Creating sample directory '%s' in HDFS") %
            REMOTE_SAMPLE_DIR.get())
        create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = self.fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
                               local_dir, remote_data_dir)

        # Copy sample data
        local_dir = LOCAL_SAMPLE_DATA_DIR.get()
        remote_data_dir = self.fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
                           local_dir, remote_data_dir)

        # Load jobs
        LOG.info(_("Installing examples..."))

        if ENABLE_V2.get():
            management.call_command('loaddata',
                                    'initial_oozie_examples.json',
                                    verbosity=2)

        # Get or create sample user directories
        home_dir = Directory.objects.get_home_directory(self.user)
        examples_dir, created = Directory.objects.get_or_create(
            parent_directory=home_dir,
            owner=self.user,
            name=Document2.EXAMPLES_DIR)

        # Share oozie examples with default group
        oozie_examples = Document2.objects.filter(type__in=[
            'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'
        ],
                                                  owner=self.user,
                                                  parent_directory=None)
        oozie_examples.update(parent_directory=examples_dir)
        examples_dir.share(self.user,
                           Document2Permission.READ_PERM,
                           groups=[get_default_user_group()])

        self.install_examples()

        Document.objects.sync()
Ejemplo n.º 41
0
  def handle(self, *args, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(self.user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=self.user,
      name=Document2.EXAMPLES_DIR
    )

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      with transaction.atomic():
        management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False)

    if IS_HUE_4.get():
      # Install editor oozie examples without doc1 link
      LOG.info("Using Hue 4, will install oozie editor samples.")

      example_jobs = []
      example_jobs.append(self._install_mapreduce_example())
      example_jobs.append(self._install_java_example())
      example_jobs.append(self._install_spark_example())
      example_jobs.append(self._install_pyspark_example())

      # If documents exist but have been trashed, recover from Trash
      for doc in example_jobs:
        if doc is not None and doc.parent_directory != examples_dir:
          doc.parent_directory = examples_dir
          doc.save()

    elif USE_NEW_EDITOR.get():
      # Install as link-workflow doc2 to old Job Designs
      docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
      for doc in docs:
        if doc.content_object:
          data = doc.content_object.data_dict
          data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
          data = json.dumps(data)

          # Don't overwrite
          doc2, created = Document2.objects.get_or_create(
            owner=self.user,
            parent_directory=examples_dir,
            name=doc.name,
            type='link-workflow',
            description=doc.description,
            data=data
          )

          LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))

    # Share oozie examples with default group
    oozie_examples = Document2.objects.filter(
      type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
      owner=self.user,
      parent_directory=None
    )
    oozie_examples.update(parent_directory=examples_dir)
    examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])

    if not IS_HUE_4.get():
      self.install_examples()
      Document.objects.sync()
Ejemplo n.º 42
0
    def handle(self, *args, **options):
        self.user = install_sample_user()
        self.fs = cluster.get_hdfs()

        LOG.info(
            _("Creating sample directory '%s' in HDFS") %
            REMOTE_SAMPLE_DIR.get())
        create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = self.fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            self.fs.do_as_user(self.user.username, self.fs.copyFromLocal,
                               local_dir, remote_data_dir)

        # Copy sample data
        local_dir = LOCAL_SAMPLE_DATA_DIR.get()
        remote_data_dir = self.fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        self.fs.do_as_user(self.user.username, self.fs.copyFromLocal,
                           local_dir, remote_data_dir)

        # Get or create sample user directories
        home_dir = Directory.objects.get_home_directory(self.user)
        examples_dir, created = Directory.objects.get_or_create(
            parent_directory=home_dir,
            owner=self.user,
            name=Document2.EXAMPLES_DIR)

        # Load jobs
        LOG.info(_("Installing examples..."))

        if ENABLE_V2.get():
            with transaction.atomic():
                management.call_command('loaddata',
                                        'initial_oozie_examples.json',
                                        verbosity=2,
                                        commit=False)

        # Install editor oozie examples without doc1 link
        LOG.info("Using Hue 4, will install oozie editor samples.")

        example_jobs = []
        example_jobs.append(self._install_mapreduce_example())
        example_jobs.append(self._install_java_example())
        example_jobs.append(self._install_spark_example())
        example_jobs.append(self._install_pyspark_example())

        # If documents exist but have been trashed, recover from Trash
        for doc in example_jobs:
            if doc is not None and doc.parent_directory != examples_dir:
                doc.parent_directory = examples_dir
                doc.save()

        # Share oozie examples with default group
        oozie_examples = Document2.objects.filter(type__in=[
            'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'
        ],
                                                  owner=self.user,
                                                  parent_directory=None)
        oozie_examples.update(parent_directory=examples_dir)
        examples_dir.share(self.user,
                           Document2Permission.READ_PERM,
                           groups=[get_default_user_group()])
Ejemplo n.º 43
0
Archivo: models.py Proyecto: hwl-py/hue
    except Exception, e:
      LOG.exception('error syncing pig')

    try:
      with transaction.atomic():
        from search.models import Collection

        for dashboard in Collection.objects.all():
          col_dict = dashboard.properties_dict['collection']
          if not 'uuid' in col_dict:
            _uuid = str(uuid.uuid4())
            col_dict['uuid'] = _uuid
            dashboard.update_properties({'collection': col_dict})
            if dashboard.owner is None:
              from useradmin.models import install_sample_user
              owner = install_sample_user()
            else:
              owner = dashboard.owner
            dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard', owner=owner, description=dashboard.label, data=dashboard.properties)
            Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label, extra='search-dashboard')
            dashboard.save()
    except Exception, e:
      LOG.exception('error syncing search')

    try:
      with transaction.atomic():
        for job in find_jobs_with_no_doc(Document2):
          if job.type == 'oozie-workflow2':
            extra = 'workflow2'
          elif job.type == 'oozie-coordinator2':
            extra = 'coordinator2'
Ejemplo n.º 44
0
from django.db import connection, models, transaction
from search.models import Collection
from desktop.models import Document, Document2
table_names = connection.introspection.table_names()
if Collection._meta.db_table in table_names:
    with transaction.atomic():
        for dashboard in Collection.objects.all():
            if 'collection' in dashboard.properties_dict:
                col_dict = dashboard.properties_dict['collection']
                if not 'uuid' in col_dict:
                    _uuid = str(uuid.uuid4())
                    col_dict['uuid'] = _uuid
                    dashboard.update_properties({'collection': col_dict})
                    if dashboard.owner is None:
                        from useradmin.models import install_sample_user
                        owner = install_sample_user()
                    else:
                        owner = dashboard.owner
                        dashboard_doc = Document2.objects.create(
                            name=dashboard.label,
                            uuid=_uuid,
                            type='search-dashboard',
                            owner=owner,
                            description=dashboard.label,
                            data=dashboard.properties)
                        Document.objects.link(dashboard_doc,
                                              owner=owner,
                                              name=dashboard.label,
                                              description=dashboard.label,
                                              extra='search-dashboard')
                        dashboard.save()
Ejemplo n.º 45
0
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(self.user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=self.user,
      name=Document2.EXAMPLES_DIR
    )

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)

    if IS_HUE_4.get():
      # Install editor oozie examples without doc1 link
      LOG.info("Using Hue 4, will install oozie editor samples.")

      example_jobs = []
      example_jobs.append(self._install_mapreduce_example())
      example_jobs.append(self._install_java_example())
      example_jobs.append(self._install_spark_example())
      example_jobs.append(self._install_pyspark_example())

      # If documents exist but have been trashed, recover from Trash
      for doc in example_jobs:
        if doc is not None and doc.parent_directory != examples_dir:
          doc.parent_directory = examples_dir
          doc.save()

    elif USE_NEW_EDITOR.get():
      # Install as link-workflow doc2 to old Job Designs
      docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
      for doc in docs:
        if doc.content_object:
          data = doc.content_object.data_dict
          data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
          data = json.dumps(data)

          # Don't overwrite
          doc2, created = Document2.objects.get_or_create(
            owner=self.user,
            parent_directory=examples_dir,
            name=doc.name,
            type='link-workflow',
            description=doc.description,
            data=data
          )

          LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))

    # Share oozie examples with default group
    oozie_examples = Document2.objects.filter(
      type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
      owner=self.user,
      parent_directory=None
    )
    oozie_examples.update(parent_directory=examples_dir)
    examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])

    if not IS_HUE_4.get():
      self.install_examples()
      Document.objects.sync()
Ejemplo n.º 46
0
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()

    LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
    create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = self.fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = LOCAL_SAMPLE_DATA_DIR.get()
    remote_data_dir = self.fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    LOG.info(_("Installing examples..."))

    if ENABLE_V2.get():
      management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)

    # Get or create sample user directories
    home_dir = Directory.objects.get_home_directory(self.user)
    examples_dir, created = Directory.objects.get_or_create(
      parent_directory=home_dir,
      owner=self.user,
      name=Document2.EXAMPLES_DIR
    )

    if USE_NEW_EDITOR.get():
      docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
      for doc in docs:
        if doc.content_object:
          data = doc.content_object.data_dict
          data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
          data = json.dumps(data)

          doc2 = Document2.objects.create(
            owner=self.user,
            parent_directory=examples_dir,
            name=doc.name,
            type='link-workflow',
            description=doc.description,
            data=data)

          LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))

    # Share oozie examples with default group
    oozie_examples = Document2.objects.filter(
      type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
      owner=self.user,
      parent_directory=None
    )
    oozie_examples.update(parent_directory=examples_dir)
    examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])

    self.install_examples()

    Document.objects.sync()
Ejemplo n.º 47
0
    def handle_noargs(self, **options):
        fs = cluster.get_hdfs()
        create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
                          remote_data_dir)

        # Copy sample data
        local_dir = paths.get_thirdparty_root("sample_data")
        remote_data_dir = fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
                      remote_data_dir)

        # Load jobs
        sample_user = install_sample_user()
        management.call_command('loaddata',
                                'initial_pig_examples.json',
                                verbosity=2)
        Document.objects.sync()

        if USE_NEW_EDITOR.get():
            # Get or create sample user directories
            home_dir = Directory.objects.get_home_directory(sample_user)
            examples_dir, created = Directory.objects.get_or_create(
                parent_directory=home_dir,
                owner=sample_user,
                name=Document2.EXAMPLES_DIR)

            try:
                # Don't overwrite
                doc = Document.objects.get(object_id=1100713)
                doc2 = Document2.objects.get(owner=sample_user,
                                             name=doc.name,
                                             type='link-pigscript')
                # If document exists but has been trashed, recover from Trash
                if doc2.parent_directory != examples_dir:
                    doc2.parent_directory = examples_dir
                    doc2.save()
            except Document.DoesNotExist:
                LOG.warn('Sample pig script document not found.')
            except Document2.DoesNotExist:
                if doc.content_object:
                    data = doc.content_object.dict
                    data.update({
                        'content_type': doc.content_type.model,
                        'object_id': doc.object_id
                    })
                    data = json.dumps(data)

                    doc2 = Document2.objects.create(
                        owner=sample_user,
                        parent_directory=examples_dir,
                        name=doc.name,
                        type='link-pigscript',
                        description=doc.description,
                        data=data)
                    LOG.info(
                        'Successfully installed sample link to pig script: %s'
                        % (doc2.name, ))

            # Share with default group
            examples_dir.share(sample_user,
                               Document2Permission.READ_PERM,
                               groups=[get_default_user_group()])
Ejemplo n.º 48
0
  def handle(self, *args, **options):
    fs = cluster.get_hdfs()
    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()
    sample_user = install_sample_user()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = paths.get_thirdparty_root("sample_data")
    remote_data_dir = fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)

    # Initialize doc2, whether editor script or link
    doc2 = None

    if IS_HUE_4.get():
      # Install editor pig script without doc1 link
      LOG.info("Using Hue 4, will install pig editor sample.")
      doc2 = self.install_pig_script(sample_user)
    else:
      # Install old pig script fixture
      LOG.info("Using Hue 3, will install pig script fixture.")
      with transaction.atomic():
        management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2, commit=False)
        Document.objects.sync()

    if USE_NEW_EDITOR.get():
      # Get or create sample user directories
      LOG.info("Creating sample user directories.")

      home_dir = Directory.objects.get_home_directory(sample_user)
      examples_dir, created = Directory.objects.get_or_create(
        parent_directory=home_dir,
        owner=sample_user,
        name=Document2.EXAMPLES_DIR)

      if not IS_HUE_4.get():
        try:
          # Don't overwrite
          doc = Document.objects.get(object_id=1100713)
          doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript')
        except Document.DoesNotExist:
          LOG.warn('Sample pig script document not found.')
        except Document2.DoesNotExist:
          if doc.content_object:
            data = doc.content_object.dict
            data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
            data = json.dumps(data)

            doc2 = Document2.objects.create(
              owner=sample_user,
              parent_directory=examples_dir,
              name=doc.name,
              type='link-pigscript',
              description=doc.description,
              data=data)
            LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,))

      # If document exists but has been trashed, recover from Trash
      if doc2 and doc2.parent_directory != examples_dir:
        doc2.parent_directory = examples_dir
        doc2.save()

      # Share with default group
      examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])