def handle_noargs(self, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs install_sample_user() management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2) Document.objects.sync()
def handle_noargs(self, **options): if not Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).exists(): install_sample_user() management.call_command('loaddata', 'initial_search_examples.json', verbosity=2) Document.objects.sync()
def handle_noargs(self, **options): if not Document2.objects.filter( type='search-dashboard', owner__username=SAMPLE_USERNAME).exists(): install_sample_user() management.call_command('loaddata', 'initial_search_examples.json', verbosity=2) Document.objects.sync()
def handle(self, *args, **options): if not options.get('user'): user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name) else: user = options['user'] install_sample_user() management.call_command('loaddata', 'apps/spark/src/spark/fixtures/initial_spark_examples.json', verbosity=2) Document.objects.sync() from beeswax.management.commands.beeswax_install_examples import Command app_name = 'beeswax' Command().handle(app_name=app_name, user=user, tables='web_logs_table.json')
def handle(self, *args, **options): if not options.get('user'): user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name) else: user = options['user'] if not Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).exists(): install_sample_user() management.call_command('loaddata', 'apps/spark/src/spark/fixtures/initial_spark_examples.json', verbosity=2) Document.objects.sync() from beeswax.management.commands.beeswax_install_examples import Command app_name = 'beeswax' Command().handle(app_name=app_name, user=user, tables='web_logs_table.json')
def handle(self, *args, **options): if not options.get('user'): user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name) else: user = options['user'] if not Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).exists(): install_sample_user() management.call_command('loaddata', 'desktop/libs/notebook/src/notebook/fixtures/initial_notebook_examples.json', verbosity=2) Document.objects.sync() from beeswax.management.commands.beeswax_install_examples import Command app_name = 'beeswax' Command().handle(app_name=app_name, user=user, tables='tables.json')
def handle(self, *args, **options): if not options.get('user'): user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name) else: user = options['user'] # Install sample notebook from fixture if notebook with sample UUID doesn't exist if not Document2.objects.filter(uuid="7f2ea775-e067-4fde-8f5f-4d704ab9b002").exists(): sample_user = install_sample_user() management.call_command('loaddata', 'initial_notebook_examples.json', verbosity=2) Document.objects.sync() # Get or create sample user directories home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR ) Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir) # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) LOG.info('Successfully installed sample notebook') from beeswax.management.commands.beeswax_install_examples import Command app_name = 'beeswax' Command().handle(app_name=app_name, user=user, tables='tables.json')
def authenticate(self, request=None, username=None, password=None): username = force_username_case(username) if AUTH.PAM_USE_PWD_MODULE.get(): LOG.debug('Setting username to %s using PAM pwd module for user %s' % (getpwnam(username).pw_name, username)) username = getpwnam(username).pw_name if pam.authenticate(username, password, AUTH.PAM_SERVICE.get()): is_super = False if User.objects.exclude(id=install_sample_user().id).count() == 0: is_super = True try: if AUTH.IGNORE_USERNAME_CASE.get(): user = User.objects.get(username__iexact=username) else: user = User.objects.get(username=username) except User.DoesNotExist: user = find_or_create_user(username, None) if user is not None and user.is_active: profile = get_profile(user) profile.creation_method = UserProfile.CreationMethod.EXTERNAL.name profile.save() user.is_superuser = is_super ensure_has_a_group(user) user.save() user = rewrite_user(user) return user return None
def authenticate(self, request=None, remote_user=None): username = self.clean_username(remote_user) username = force_username_case(username) is_super = False if User.objects.exclude(id=install_sample_user().id).count() == 0: is_super = True try: if AUTH.IGNORE_USERNAME_CASE.get(): user = User.objects.get(username__iexact=username) else: user = User.objects.get(username=username) except User.DoesNotExist: user = find_or_create_user(username, None) if user is not None and user.is_active: profile = get_profile(user) profile.creation_method = UserProfile.CreationMethod.EXTERNAL.name profile.save() user.is_superuser = is_super ensure_has_a_group(user) user.save() user = rewrite_user(user) return user
def create_table(user=None, query_server=None, table=None): if not user: user = install_sample_user() if not query_server: query_server = get_query_server_config('beeswax') if not table: base_dir = hive_site.get_hive_hook_proto_base_directory() if not base_dir: msg = _( 'Error creating table query_data hive.hook.proto.base-directory is not configured' ) LOG.error(msg) return False table = {'name': 'query_data', 'external_location': base_dir} server = dbms.get(user, query_server) for query in ["create_table_query_data.mako", "msck.mako"]: proposed_query = django_mako.render_to_string(query, {'table': table}) query = hql_query(proposed_query) try: handle = server.execute_and_wait(query) if not handle: LOG.error(_('Error executing %s: Operation timeout.' % query)) return False server.close(handle) except Exception as ex: LOG.error( _('Error executing %(query)s: %(error)s.') % { 'query': query, 'error': ex }) return False LOG.info(_('Table query_data has been created successfully')) return True
def handle(self, *args, **options): if not options.get('user'): user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name) else: user = options['user'] dialect = options.get('dialect', 'hive') # Install sample notebook from fixture if notebook with sample UUID doesn't exist if not Document2.objects.filter(uuid="7f2ea775-e067-4fde-8f5f-4d704ab9b002").exists(): sample_user = install_sample_user() with transaction.atomic(): management.call_command('loaddata', 'initial_notebook_examples.json', verbosity=2, commit=False) Document.objects.sync() # Get or create sample user directories home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR ) Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir) # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) LOG.info('Successfully installed sample notebook') from beeswax.management.commands.beeswax_install_examples import Command Command().handle(dialect=dialect, user=user)
def _execute_query(proposed_query, limit): user = install_sample_user() query_server = get_query_server_config('beeswax') server = dbms.get(user, query_server) query = hql_query(proposed_query) try: handle = server.execute_and_wait(query) if not handle: LOG.error(_('Error executing %s: Operation timeout.' % query)) return [] results = server.fetch(handle, True, limit) rows = [row for row in results.rows()] data = { 'data': rows, 'columns': [column.name for column in results.data_table.cols()] } return data except Exception as ex: raise_popup_exception(_('Error fetching query history.')) finally: try: if server and handle: server.close(handle) except Exception as ex: raise_popup_exception(_('Error fetching query history.'))
def install_custom_examples(): if EXAMPLES.AUTO_LOAD.get(): from desktop.auth.backend import rewrite_user from beeswax.management.commands import beeswax_install_examples from useradmin.models import install_sample_user user = rewrite_user(install_sample_user()) if has_connectors(): interpreters = [{ 'type': connector['id'], 'dialect': connector['dialect'] } for connector in _get_installed_connectors(category='editor')] else: interpreters = [ { 'type': interpreter['dialect'], 'dialect': interpreter['dialect'] } for interpreter in get_ordered_interpreters(user) # Only for hive/impala currently, would also need to port to Notebook install examples. if interpreter['dialect'] in ('hive', 'impala') ] queries = EXAMPLES.QUERIES.get() tables = EXAMPLES.TABLES.get( ) # No-op. Only for the saved query samples, not the tables currently. LOG.info( 'Installing custom examples queries: %(queries)s, tables: %(tables)s for dialects %(dialects)s ' 'belonging to user %(user)s' % { 'queries': queries, 'tables': tables, 'dialects': [interpreter['dialect'] for interpreter in interpreters], 'user': user }) result = [] for interpreter in interpreters: successes, errors = beeswax_install_examples.Command().handle( dialect=interpreter['dialect'], user=user, interpreter=interpreter, queries=queries, tables=tables, request=None) LOG.info( 'Dialect %(dialect)s installed samples: %(successes)s, %(errors)s,' % { 'dialect': interpreter['dialect'], 'successes': successes, 'errors': errors, }) result.append((successes, errors)) return result
def handle(self, *args, **options): sample_user = install_sample_user() # Get or create sample user directories home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR ) if not Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).exists(): with transaction.atomic(): if sys.version_info[0] > 2: management.call_command('loaddata', 'initial_search_examples.json', verbosity=2) else: management.call_command('loaddata', 'initial_search_examples.json', verbosity=2, commit=False) Document.objects.sync() Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir) else: # Check if sample documents are in Trash, and if so, restore them for doc in Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS): if doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) LOG.info('Successfully installed sample search dashboard')
def is_first_login_ever(self): """ Return true if no one has ever logged in.""" if ENABLE_ORGANIZATIONS.get(): # Tmp return False else: return User.objects.exclude( id=install_sample_user().id).count() == 0
def handle_noargs(self, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs sample_user = install_sample_user() management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2) Document.objects.sync() if USE_NEW_EDITOR.get(): # Get or create sample user directories home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR) try: # Don't overwrite doc = Document.objects.get(object_id=1100713) doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript') # If document exists but has been trashed, recover from Trash if doc2.parent_directory != examples_dir: doc2.parent_directory = examples_dir doc2.save() except Document.DoesNotExist: LOG.warn('Sample pig script document not found.') except Document2.DoesNotExist: if doc.content_object: data = doc.content_object.dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) doc2 = Document2.objects.create( owner=sample_user, parent_directory=examples_dir, name=doc.name, type='link-pigscript', description=doc.description, data=data) LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,)) # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) self.install_examples() Document.objects.sync()
def handle_noargs(self, **options): sample_user = install_sample_user() # Get or create sample user directories home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR ) if not Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).exists(): management.call_command('loaddata', 'initial_search_examples.json', verbosity=2) Document.objects.sync() Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir) else: # Check if sample documents are in Trash, and if so, restore them for doc in Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS): if doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) LOG.info('Successfully installed sample search dashboard')
def _get_dialect_example(dialect): sample_user = install_sample_user() examples_dir = _get_example_directory(sample_user) return Document2.objects.filter(owner=sample_user, type='query-%s' % dialect, is_history=False, parent_directory=examples_dir).first()
def handle_noargs(self, **options): exception = None try: user = install_sample_user() self._install_tables(user, options['app_name']) except Exception, ex: exception = ex
def handle_noargs(self, **options): """Main entry point to install or re-install examples. May raise InstallException""" try: user = install_sample_user() self._install_tables(user, options['app_name']) self._install_queries(user, options['app_name']) except Exception, ex: LOG.exception(ex) raise InstallException(ex)
def handle_noargs(self, **options): exception = None # Documents will belong to this user but we run the install as the current user try: sample_user = install_sample_user() self._install_tables(options['user'], options['app_name']) except Exception, ex: exception = ex
def handle(self, *args, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() sample_user = install_sample_user() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Initialize doc2, whether editor script or link doc2 = None # Install editor pig script without doc1 link LOG.info("Using Hue 4, will install pig editor sample.") doc2 = self.install_pig_script(sample_user) if USE_NEW_EDITOR.get(): # Get or create sample user directories LOG.info("Creating sample user directories.") home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR) # If document exists but has been trashed, recover from Trash if doc2 and doc2.parent_directory != examples_dir: doc2.parent_directory = examples_dir doc2.save() # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle(self, *args, **options): if args: dialect = args[0] db_name = args[1] if len(args) > 1 else 'default' user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name) request = None else: dialect = options['dialect'] db_name = options.get('db_name', 'default') interpreter = options.get( 'interpreter' ) # Only when connectors are enabled. Later will deprecate `dialect`. user = options['user'] request = options['request'] tables = 'tables_standard.json' if dialect not in ( 'hive', 'impala') else ('tables_transactional.json' if has_concurrency_support() else 'tables.json') exception = None self.successes = [] self.errors = [] try: sample_user = install_sample_user( user ) # Documents will belong to the sample user but we run the SQL as the current user self.install_queries(sample_user, dialect, interpreter=interpreter) self.install_tables(user, dialect, db_name, tables, interpreter=interpreter, request=request) except Exception as ex: exception = ex if exception is not None: pretty_msg = None if "Permission denied" in str(exception): pretty_msg = _( "Permission denied. Please check with your system administrator." ) if pretty_msg is not None: raise PopupException(pretty_msg) else: raise exception return self.successes, self.errors
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def handle_noargs(self, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs install_sample_user() management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2) Document.objects.sync()
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() self.searcher = controller.CollectionManagerController(self.user) LOG.info(_("Installing twitter collection")) path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_twitter_demo/index_data.csv')) self._setup_collection_from_csv({ 'name': 'twitter_demo', 'fields': self._parse_fields(path), 'uniqueKeyField': 'id', 'df': 'text' }, path) LOG.info(_("Twitter collection successfully installed")) LOG.info(_("Installing yelp collection")) path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_yelp_demo/index_data.csv')) self._setup_collection_from_csv({ 'name': 'yelp_demo', 'fields': self._parse_fields(path), 'uniqueKeyField': 'id', 'df': 'text' }, path) LOG.info(_("Yelp collection successfully installed")) LOG.info(_("Installing jobs collection")) path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_jobs_demo/index_data.csv')) self._setup_collection_from_csv({ 'name': 'jobs_demo', 'fields': self._parse_fields(path), 'uniqueKeyField': 'id', 'df': 'description' }, path) LOG.info(_("Jobs collection successfully installed")) LOG.info(_("Installing logs collection")) path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_log_analytics_demo/index_data.csv')) self._setup_collection_from_csv({ 'name': 'log_analytics_demo', 'fields': self._parse_fields(path, fieldtypes={ 'region_code': 'string', 'referer': 'string' }), 'uniqueKeyField': 'id', 'df': 'record' }, path) LOG.info(_("Logs collection successfully installed"))
def handle(self, *args, **options): if args: app_name = args[0] user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name) else: app_name = options['app_name'] user = options['user'] exception = None # Documents will belong to this user but we run the install as the current user try: sample_user = install_sample_user() self._install_queries(sample_user, app_name) self._install_tables(user, app_name) except Exception, ex: exception = ex
def handle(self, *args, **options): if args: user = args[0] else: user = install_sample_user() api = HbaseApi(user=user) cluster_name = api.getClusters()[0]['name'] # Currently pick first configured cluster # Check connectivity api.connectCluster(cluster_name) self.create_analytics_table(api, cluster_name) self.load_analytics_table(api, cluster_name) self.create_binary_table(api, cluster_name) self.load_binary_table(api, cluster_name)
def handle(self, *args, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() sample_user = install_sample_user() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Initialize doc2, whether editor script or link doc2 = None # Install editor pig script without doc1 link LOG.info("Using Hue 4, will install pig editor sample.") doc2 = self.install_pig_script(sample_user) if USE_NEW_EDITOR.get(): # Get or create sample user directories LOG.info("Creating sample user directories.") home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR) # If document exists but has been trashed, recover from Trash if doc2 and doc2.parent_directory != examples_dir: doc2.parent_directory = examples_dir doc2.save() # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle(self, *args, **options): if args: user = args[0] else: user = install_sample_user() api = HbaseApi(user=user) cluster_name = api.getClusters()[0][ 'name'] # Currently pick first configured cluster # Check connectivity api.connectCluster(cluster_name) self.create_analytics_table(api, cluster_name) self.load_analytics_table(api, cluster_name) self.create_binary_table(api, cluster_name) self.load_binary_table(api, cluster_name)
def handle(self, *args, **options): if args: app_name = args[0] db_name = args[1] if len(args) > 1 else 'default' user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name) else: app_name = options['app_name'] db_name = options.get('db_name', 'default') user = options['user'] tables = options['tables'] if 'tables' in options else 'tables.json' exception = None # Documents will belong to this user but we run the install as the current user try: sample_user = install_sample_user() self._install_queries(sample_user, app_name) self._install_tables(user, app_name, db_name, tables) except Exception, ex: exception = ex
def handle(self, *args, **options): if args: app_name = args[0] db_name = args[1] if len(args) > 1 else 'default' user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name) else: app_name = options['app_name'] db_name = options.get('db_name', 'default') user = options['user'] tables = options['tables'] if 'tables' in options else ( 'tables_transactional.json' if has_concurrency_support() else 'tables.json') exception = None # Documents will belong to this user but we run the install as the current user try: sample_user = install_sample_user() self._install_queries(sample_user, app_name) self._install_tables(user, app_name, db_name, tables) except Exception as ex: exception = ex Document.objects.sync() if exception is not None: pretty_msg = None if "AlreadyExistsException" in exception.message: pretty_msg = _("SQL table examples already installed.") if "Permission denied" in exception.message: pretty_msg = _( "Permission denied. Please check with your system administrator." ) if pretty_msg is not None: raise PopupException(pretty_msg) else: raise exception
def handle(self, *args, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): with transaction.atomic(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False) # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle(self, *args, **options): self.user = install_sample_user() self.client = SolrClient(self.user) collection = options['data'] if collection == 'twitter_demo': LOG.info("Installing twitter collection") path = os.path.abspath( os.path.join( os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_twitter_demo/index_data.csv' )) self._setup_collection_from_csv( { 'name': 'twitter_demo', 'fields': self._parse_fields(path, fieldtypes={ 'source': 'string', 'username': '******', }), 'uniqueKeyField': 'id', 'df': 'text' }, path) LOG.info("Twitter collection successfully installed") if collection == 'yelp_demo': LOG.info("Installing yelp collection") path = os.path.abspath( os.path.join( os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_yelp_demo/index_data.csv' )) self._setup_collection_from_csv( { 'name': 'yelp_demo', 'fields': self._parse_fields(path, fieldtypes={ 'name': 'string', }), 'uniqueKeyField': 'id', 'df': 'text' }, path) LOG.info("Yelp collection successfully installed") if collection == 'log_analytics_demo': LOG.info("Installing logs collection") path = os.path.abspath( os.path.join( os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_log_analytics_demo/index_data.csv' )) self._setup_collection_from_csv( { 'name': 'log_analytics_demo', 'fields': self._parse_fields(path, fieldtypes={ 'region_code': 'string', 'referer': 'string', 'user_agent': 'string' }), 'uniqueKeyField': 'id', 'df': 'record' }, path) LOG.info("Logs collection successfully installed")
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info( _("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR) if USE_NEW_EDITOR.get(): docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({ 'content_type': doc.content_type.model, 'object_id': doc.object_id }) data = json.dumps(data) doc2 = Document2.objects.create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data) LOG.info( 'Successfully installed sample link to jobsub: %s' % (doc2.name, )) # Share oozie examples with default group oozie_examples = Document2.objects.filter(type__in=[ 'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2' ], owner=self.user, parent_directory=None) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info( _("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR) # Share oozie examples with default group oozie_examples = Document2.objects.filter(type__in=[ 'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2' ], owner=self.user, parent_directory=None) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def handle(self, *args, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): with transaction.atomic(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False) if IS_HUE_4.get(): # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() elif USE_NEW_EDITOR.get(): # Install as link-workflow doc2 to old Job Designs docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) # Don't overwrite doc2, created = Document2.objects.get_or_create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data ) LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,)) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) if not IS_HUE_4.get(): self.install_examples() Document.objects.sync()
def handle(self, *args, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info( _("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): with transaction.atomic(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False) # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() # Share oozie examples with default group oozie_examples = Document2.objects.filter(type__in=[ 'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2' ], owner=self.user, parent_directory=None) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
except Exception, e: LOG.exception('error syncing pig') try: with transaction.atomic(): from search.models import Collection for dashboard in Collection.objects.all(): col_dict = dashboard.properties_dict['collection'] if not 'uuid' in col_dict: _uuid = str(uuid.uuid4()) col_dict['uuid'] = _uuid dashboard.update_properties({'collection': col_dict}) if dashboard.owner is None: from useradmin.models import install_sample_user owner = install_sample_user() else: owner = dashboard.owner dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard', owner=owner, description=dashboard.label, data=dashboard.properties) Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label, extra='search-dashboard') dashboard.save() except Exception, e: LOG.exception('error syncing search') try: with transaction.atomic(): for job in find_jobs_with_no_doc(Document2): if job.type == 'oozie-workflow2': extra = 'workflow2' elif job.type == 'oozie-coordinator2': extra = 'coordinator2'
from django.db import connection, models, transaction from search.models import Collection from desktop.models import Document, Document2 table_names = connection.introspection.table_names() if Collection._meta.db_table in table_names: with transaction.atomic(): for dashboard in Collection.objects.all(): if 'collection' in dashboard.properties_dict: col_dict = dashboard.properties_dict['collection'] if not 'uuid' in col_dict: _uuid = str(uuid.uuid4()) col_dict['uuid'] = _uuid dashboard.update_properties({'collection': col_dict}) if dashboard.owner is None: from useradmin.models import install_sample_user owner = install_sample_user() else: owner = dashboard.owner dashboard_doc = Document2.objects.create( name=dashboard.label, uuid=_uuid, type='search-dashboard', owner=owner, description=dashboard.label, data=dashboard.properties) Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label, extra='search-dashboard') dashboard.save()
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) if IS_HUE_4.get(): # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() elif USE_NEW_EDITOR.get(): # Install as link-workflow doc2 to old Job Designs docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) # Don't overwrite doc2, created = Document2.objects.get_or_create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data ) LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,)) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) if not IS_HUE_4.get(): self.install_examples() Document.objects.sync()
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) if USE_NEW_EDITOR.get(): docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) doc2 = Document2.objects.create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data) LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,)) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def handle_noargs(self, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs sample_user = install_sample_user() management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2) Document.objects.sync() if USE_NEW_EDITOR.get(): # Get or create sample user directories home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR) try: # Don't overwrite doc = Document.objects.get(object_id=1100713) doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript') # If document exists but has been trashed, recover from Trash if doc2.parent_directory != examples_dir: doc2.parent_directory = examples_dir doc2.save() except Document.DoesNotExist: LOG.warn('Sample pig script document not found.') except Document2.DoesNotExist: if doc.content_object: data = doc.content_object.dict data.update({ 'content_type': doc.content_type.model, 'object_id': doc.object_id }) data = json.dumps(data) doc2 = Document2.objects.create( owner=sample_user, parent_directory=examples_dir, name=doc.name, type='link-pigscript', description=doc.description, data=data) LOG.info( 'Successfully installed sample link to pig script: %s' % (doc2.name, )) # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle(self, *args, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() sample_user = install_sample_user() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Initialize doc2, whether editor script or link doc2 = None if IS_HUE_4.get(): # Install editor pig script without doc1 link LOG.info("Using Hue 4, will install pig editor sample.") doc2 = self.install_pig_script(sample_user) else: # Install old pig script fixture LOG.info("Using Hue 3, will install pig script fixture.") with transaction.atomic(): management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2, commit=False) Document.objects.sync() if USE_NEW_EDITOR.get(): # Get or create sample user directories LOG.info("Creating sample user directories.") home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR) if not IS_HUE_4.get(): try: # Don't overwrite doc = Document.objects.get(object_id=1100713) doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript') except Document.DoesNotExist: LOG.warn('Sample pig script document not found.') except Document2.DoesNotExist: if doc.content_object: data = doc.content_object.dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) doc2 = Document2.objects.create( owner=sample_user, parent_directory=examples_dir, name=doc.name, type='link-pigscript', description=doc.description, data=data) LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,)) # If document exists but has been trashed, recover from Trash if doc2 and doc2.parent_directory != examples_dir: doc2.parent_directory = examples_dir doc2.save() # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])