Пример #1
0
    def handle_noargs(self, **options):
        fs = cluster.get_hdfs()
        create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
                          remote_data_dir)

        # Copy sample data
        local_dir = paths.get_thirdparty_root("sample_data")
        remote_data_dir = fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
                      remote_data_dir)

        # Load jobs
        install_sample_user()
        management.call_command('loaddata',
                                'initial_pig_examples.json',
                                verbosity=2)
        Document.objects.sync()
Пример #2
0
  def handle_noargs(self, **options):
    fs = cluster.get_hdfs()
    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = paths.get_thirdparty_root("sample_data")
    remote_data_dir = fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    USERNAME = '******'
    try:
      sample_user = User.objects.get(username=USERNAME)
    except User.DoesNotExist:
      sample_user = User.objects.create(username=USERNAME, password='******', is_active=False, is_superuser=False, id=1100713, pk=1100713)
    management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2)
    Document.objects.sync()
Пример #3
0
  def handle_noargs(self, **options):
    fs = cluster.get_hdfs()
    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = paths.get_thirdparty_root("sample_data")
    remote_data_dir = fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    sample_user = install_sample_user()
    management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2)
    Document.objects.sync()

    if USE_NEW_EDITOR.get():
      # Get or create sample user directories
      home_dir = Directory.objects.get_home_directory(sample_user)
      examples_dir, created = Directory.objects.get_or_create(
        parent_directory=home_dir,
        owner=sample_user,
        name=Document2.EXAMPLES_DIR)

      try:
        # Don't overwrite
        doc = Document.objects.get(object_id=1100713)
        doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript')
        # If document exists but has been trashed, recover from Trash
        if doc2.parent_directory != examples_dir:
          doc2.parent_directory = examples_dir
          doc2.save()
      except Document.DoesNotExist:
        LOG.warn('Sample pig script document not found.')
      except Document2.DoesNotExist:
        if doc.content_object:
          data = doc.content_object.dict
          data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
          data = json.dumps(data)

          doc2 = Document2.objects.create(
            owner=sample_user,
            parent_directory=examples_dir,
            name=doc.name,
            type='link-pigscript',
            description=doc.description,
            data=data)
          LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,))

      # Share with default group
      examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
Пример #4
0
    def install_pig_script(self, sample_user):
        doc2 = None
        name = _('UpperText')

        if Document2.objects.filter(owner=sample_user,
                                    name=name,
                                    type='query-pig',
                                    is_history=False).exists():
            LOG.info("Sample pig editor script already installed.")
            doc2 = Document2.objects.get(owner=sample_user,
                                         name=name,
                                         type='query-pig',
                                         is_history=False)
        else:
            statement = """REGISTER hdfs://{}/piggybank.jar;

data = LOAD '{}/data/midsummer.txt' as (text:CHARARRAY);

upper_case = FOREACH data GENERATE org.apache.pig.piggybank.evaluation.string.UPPER(text);

STORE upper_case INTO '$output';
""".format(REMOTE_SAMPLE_DIR.get(), REMOTE_SAMPLE_DIR.get())
            snippet_properties = {
                'hadoopProperties': [],
                'parameters': [],
                'resources': []
            }

            notebook = make_notebook(
                name=name,
                description=_('UpperText: Example Pig script'),
                editor_type='pig',
                statement=statement,
                status='ready',
                snippet_properties=snippet_properties,
                is_saved=True)

            # Remove files, functions, settings from snippet properties
            data = notebook.get_data()
            data['snippets'][0]['properties'].pop('files')
            data['snippets'][0]['properties'].pop('functions')
            data['snippets'][0]['properties'].pop('settings')

            try:
                with transaction.atomic():
                    doc2 = Document2.objects.create(
                        owner=sample_user,
                        name=data['name'],
                        type='query-pig',
                        description=data['description'],
                        data=json.dumps(data))
            except Exception as e:
                LOG.exception(
                    "Failed to create sample pig script document: %s" % e)
                # Just to be sure we delete Doc2 object incase of exception.
                # Possible when there are mixed InnoDB and MyISAM tables
                if doc2 and Document2.objects.filter(id=doc2.id).exists():
                    doc2.delete()

        return doc2
Пример #5
0
    def handle(self, *args, **options):
        fs = cluster.get_hdfs()
        create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()
        sample_user = install_sample_user()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir,
                          remote_data_dir)

        # Copy sample data
        local_dir = paths.get_thirdparty_root("sample_data")
        remote_data_dir = fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir,
                      remote_data_dir)

        # Initialize doc2, whether editor script or link
        doc2 = None

        # Install editor pig script without doc1 link
        LOG.info("Using Hue 4, will install pig editor sample.")
        doc2 = self.install_pig_script(sample_user)

        if USE_NEW_EDITOR.get():
            # Get or create sample user directories
            LOG.info("Creating sample user directories.")

            home_dir = Directory.objects.get_home_directory(sample_user)
            examples_dir, created = Directory.objects.get_or_create(
                parent_directory=home_dir,
                owner=sample_user,
                name=Document2.EXAMPLES_DIR)

            # If document exists but has been trashed, recover from Trash
            if doc2 and doc2.parent_directory != examples_dir:
                doc2.parent_directory = examples_dir
                doc2.save()

            # Share with default group
            examples_dir.share(sample_user,
                               Document2Permission.READ_PERM,
                               groups=[get_default_user_group()])
Пример #6
0
  def install_pig_script(self, sample_user):
    doc2 = None
    name = _('UpperText')

    if Document2.objects.filter(owner=sample_user, name=name, type='query-pig', is_history=False).exists():
      LOG.info("Sample pig editor script already installed.")
      doc2 = Document2.objects.get(owner=sample_user, name=name, type='query-pig', is_history=False)
    else:
      statement = """REGISTER hdfs://{}/piggybank.jar;

data = LOAD '{}/data/midsummer.txt' as (text:CHARARRAY);

upper_case = FOREACH data GENERATE org.apache.pig.piggybank.evaluation.string.UPPER(text);

STORE upper_case INTO '$output';
""".format(REMOTE_SAMPLE_DIR.get(), REMOTE_SAMPLE_DIR.get())
      snippet_properties = {
        'hadoopProperties': [],
        'parameters': [],
        'resources': []
      }

      notebook = make_notebook(
        name=name,
        description=_('UpperText: Example Pig script'),
        editor_type='pig',
        statement=statement,
        status='ready',
        snippet_properties=snippet_properties,
        is_saved=True
      )

      # Remove files, functions, settings from snippet properties
      data = notebook.get_data()
      data['snippets'][0]['properties'].pop('files')
      data['snippets'][0]['properties'].pop('functions')
      data['snippets'][0]['properties'].pop('settings')

      try:
        with transaction.atomic():
          doc2 = Document2.objects.create(
            owner=sample_user,
            name=data['name'],
            type='query-pig',
            description=data['description'],
            data=json.dumps(data)
          )
      except Exception, e:
        LOG.exception("Failed to create sample pig script document: %s" % e)
        # Just to be sure we delete Doc2 object incase of exception.
        # Possible when there are mixed InnoDB and MyISAM tables
        if doc2 and Document2.objects.filter(id=doc2.id).exists():
          doc2.delete()
Пример #7
0
  def handle(self, *args, **options):
    fs = cluster.get_hdfs()
    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()
    sample_user = install_sample_user()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = paths.get_thirdparty_root("sample_data")
    remote_data_dir = fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)

    # Initialize doc2, whether editor script or link
    doc2 = None

    # Install editor pig script without doc1 link
    LOG.info("Using Hue 4, will install pig editor sample.")
    doc2 = self.install_pig_script(sample_user)

    if USE_NEW_EDITOR.get():
      # Get or create sample user directories
      LOG.info("Creating sample user directories.")

      home_dir = Directory.objects.get_home_directory(sample_user)
      examples_dir, created = Directory.objects.get_or_create(
        parent_directory=home_dir,
        owner=sample_user,
        name=Document2.EXAMPLES_DIR)

      # If document exists but has been trashed, recover from Trash
      if doc2 and doc2.parent_directory != examples_dir:
        doc2.parent_directory = examples_dir
        doc2.save()

      # Share with default group
      examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
Пример #8
0
    def handle_noargs(self, **options):
        fs = cluster.get_hdfs()
        create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
                          remote_data_dir)

        # Copy sample data
        local_dir = paths.get_thirdparty_root("sample_data")
        remote_data_dir = fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
                      remote_data_dir)

        # Load jobs
        USERNAME = '******'
        try:
            sample_user = User.objects.get(username=USERNAME)
        except User.DoesNotExist:
            sample_user = User.objects.create(username=USERNAME,
                                              password='******',
                                              is_active=False,
                                              is_superuser=False,
                                              id=1100713,
                                              pk=1100713)
        management.call_command('loaddata',
                                'initial_pig_examples.json',
                                verbosity=2)
Пример #9
0
  def handle_noargs(self, **options):
    fs = cluster.get_hdfs()
    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = paths.get_thirdparty_root("sample_data")
    remote_data_dir = fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)

    # Load jobs
    install_sample_user()
    management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2)
    Document.objects.sync()
Пример #10
0
    def handle_noargs(self, **options):
        fs = cluster.get_hdfs()
        create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
        remote_dir = REMOTE_SAMPLE_DIR.get()

        # Copy examples binaries
        for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
            local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
            remote_data_dir = fs.join(remote_dir, name)
            LOG.info(
                _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
                {
                    'local_dir': local_dir,
                    'remote_data_dir': remote_data_dir
                })
            fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
                          remote_data_dir)

        # Copy sample data
        local_dir = paths.get_thirdparty_root("sample_data")
        remote_data_dir = fs.join(remote_dir, 'data')
        LOG.info(
            _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir,
                'remote_data_dir': remote_data_dir
            })
        fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
                      remote_data_dir)

        # Load jobs
        sample_user = install_sample_user()
        management.call_command('loaddata',
                                'initial_pig_examples.json',
                                verbosity=2)
        Document.objects.sync()

        if USE_NEW_EDITOR.get():
            # Get or create sample user directories
            home_dir = Directory.objects.get_home_directory(sample_user)
            examples_dir, created = Directory.objects.get_or_create(
                parent_directory=home_dir,
                owner=sample_user,
                name=Document2.EXAMPLES_DIR)

            try:
                # Don't overwrite
                doc = Document.objects.get(object_id=1100713)
                doc2 = Document2.objects.get(owner=sample_user,
                                             name=doc.name,
                                             type='link-pigscript')
                # If document exists but has been trashed, recover from Trash
                if doc2.parent_directory != examples_dir:
                    doc2.parent_directory = examples_dir
                    doc2.save()
            except Document.DoesNotExist:
                LOG.warn('Sample pig script document not found.')
            except Document2.DoesNotExist:
                if doc.content_object:
                    data = doc.content_object.dict
                    data.update({
                        'content_type': doc.content_type.model,
                        'object_id': doc.object_id
                    })
                    data = json.dumps(data)

                    doc2 = Document2.objects.create(
                        owner=sample_user,
                        parent_directory=examples_dir,
                        name=doc.name,
                        type='link-pigscript',
                        description=doc.description,
                        data=data)
                    LOG.info(
                        'Successfully installed sample link to pig script: %s'
                        % (doc2.name, ))

            # Share with default group
            examples_dir.share(sample_user,
                               Document2Permission.READ_PERM,
                               groups=[get_default_user_group()])
Пример #11
0
  def handle(self, *args, **options):
    fs = cluster.get_hdfs()
    create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
    remote_dir = REMOTE_SAMPLE_DIR.get()
    sample_user = install_sample_user()

    # Copy examples binaries
    for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
      local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
      remote_data_dir = fs.join(remote_dir, name)
      LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
                  'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
      fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)

    # Copy sample data
    local_dir = paths.get_thirdparty_root("sample_data")
    remote_data_dir = fs.join(remote_dir, 'data')
    LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
                'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
    fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)

    # Initialize doc2, whether editor script or link
    doc2 = None

    if IS_HUE_4.get():
      # Install editor pig script without doc1 link
      LOG.info("Using Hue 4, will install pig editor sample.")
      doc2 = self.install_pig_script(sample_user)
    else:
      # Install old pig script fixture
      LOG.info("Using Hue 3, will install pig script fixture.")
      with transaction.atomic():
        management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2, commit=False)
        Document.objects.sync()

    if USE_NEW_EDITOR.get():
      # Get or create sample user directories
      LOG.info("Creating sample user directories.")

      home_dir = Directory.objects.get_home_directory(sample_user)
      examples_dir, created = Directory.objects.get_or_create(
        parent_directory=home_dir,
        owner=sample_user,
        name=Document2.EXAMPLES_DIR)

      if not IS_HUE_4.get():
        try:
          # Don't overwrite
          doc = Document.objects.get(object_id=1100713)
          doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript')
        except Document.DoesNotExist:
          LOG.warn('Sample pig script document not found.')
        except Document2.DoesNotExist:
          if doc.content_object:
            data = doc.content_object.dict
            data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
            data = json.dumps(data)

            doc2 = Document2.objects.create(
              owner=sample_user,
              parent_directory=examples_dir,
              name=doc.name,
              type='link-pigscript',
              description=doc.description,
              data=data)
            LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,))

      # If document exists but has been trashed, recover from Trash
      if doc2 and doc2.parent_directory != examples_dir:
        doc2.parent_directory = examples_dir
        doc2.save()

      # Share with default group
      examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])