def handle_noargs(self, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs USERNAME = '******' try: sample_user = User.objects.get(username=USERNAME) except User.DoesNotExist: sample_user = User.objects.create(username=USERNAME, password='******', is_active=False, is_superuser=False, id=1100713, pk=1100713) management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) self.install_examples() Document.objects.sync()
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def create_directories(fs): # If needed, create the remote home, deployment and data directories directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get()) for directory in directories: if not fs.do_as_user("hdfs", fs.exists, directory): remote_home_dir = Hdfs.join('/user', "hdfs") if directory.startswith(remote_home_dir): # Home is 755 fs.do_as_user("hdfs", fs.create_home_dir, remote_home_dir) # Shared by all the users fs.do_as_user("hdfs", fs.mkdir, directory, 511) fs.do_as_user("hdfs", fs.chmod, directory, 511) # To remove after https://issues.apache.org/jira/browse/HDFS-3491 return REMOTE_SAMPLE_DIR.get()
def create_directories(fs): # If needed, create the remote home, deployment and data directories directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get()) for directory in directories: if not fs.do_as_user(fs.DEFAULT_USER, fs.exists, directory): remote_home_dir = Hdfs.join('/user', fs.DEFAULT_USER) if directory.startswith(remote_home_dir): # Home is 755 fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir) # Shared by all the users fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777) fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491 return REMOTE_SAMPLE_DIR.get()
def create_data_dir(fs): # If needed, create the remote home, deployment and data directories directories = (REMOTE_DEPLOYMENT_DIR.get(), REMOTE_SAMPLE_DIR.get()) user = fs.user try: fs.setuser(fs.DEFAULT_USER) for directory in directories: if not fs.exists(directory): remote_home_dir = Hdfs.join('/user', fs.user) if directory.startswith(remote_home_dir): # Home is 755 fs.create_home_dir(remote_home_dir) # Shared by all the users fs.mkdir(directory, 01777) fs.chmod(directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491 finally: fs.setuser(user) return REMOTE_SAMPLE_DIR.get()
def handle_noargs(self, **options): fs = cluster.get_hdfs() sample_user = CreateSandboxUserCommand().handle_noargs() fs.setuser(sample_user) create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.copyFromLocal(local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.copyFromLocal(local_dir, remote_data_dir) # Load jobs management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
def test_workspace_has_enough_permissions(self): reset = REMOTE_SAMPLE_DIR.set_for_testing( '/tmp/oozie_test_workspace_has_enough_permissions') try: resp = self.cli.get('/desktop/debug/check_config') assert_false('The permissions of workspace' in resp.content, resp) self.cluster.fs.mkdir(REMOTE_SAMPLE_DIR.get()) assert_equal( oct(0o40755), oct(self.cluster.fs.stats(REMOTE_SAMPLE_DIR.get())["mode"])) resp = self.cli.get('/desktop/debug/check_config') assert_true('The permissions of workspace' in resp.content, resp) permissions_dict = { 'group_read': True, 'other_execute': True, 'user_write': True, 'user_execute': True, 'sticky': False, 'user_read': True, 'other_read': True, 'other_write': True, 'group_write': False, 'group_execute': True } kwargs = {'path': [REMOTE_SAMPLE_DIR.get()]} kwargs.update(permissions_dict) # Add write permission to Others response = self.cli.post("/filebrowser/chmod", kwargs) assert_equal( oct(0o40757), oct(self.cluster.fs.stats(REMOTE_SAMPLE_DIR.get())["mode"])) resp = self.cli.get('/desktop/debug/check_config') assert_false('The permissions of workspace' in resp.content, resp) finally: self.cluster.fs.rmdir(REMOTE_SAMPLE_DIR.get(), skip_trash=True) reset()
res = [] try: from oozie.conf import REMOTE_SAMPLE_DIR except Exception, e: LOG.warn('Config check failed because Oozie app not installed: %s' % e) return res if OOZIE_URL.get(): status = get_oozie_status(user) if 'NORMAL' not in status: res.append((status, _('The Oozie server is not available'))) fs = get_filesystem() NICE_NAME = 'Oozie' if fs.do_as_superuser(fs.exists, REMOTE_SAMPLE_DIR.get()): stats = fs.do_as_superuser(fs.stats, REMOTE_SAMPLE_DIR.get()) mode = oct(stats.mode) # if neither group nor others have write permission group_has_write = int(mode[-2]) & 2 others_has_write = int(mode[-1]) & 2 if not group_has_write and not others_has_write: res.append( (NICE_NAME, "The permissions of workspace '%s' are too restrictive" % REMOTE_SAMPLE_DIR.get())) api = get_oozie(user, api_version="v2") configuration = api.get_configuration()
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info( _("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR) # Share oozie examples with default group oozie_examples = Document2.objects.filter(type__in=[ 'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2' ], owner=self.user, parent_directory=None) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def handle(self, *args, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): with transaction.atomic(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False) # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) if USE_NEW_EDITOR.get(): docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) doc2 = Document2.objects.create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data) LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,)) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def config_validator(user): """ config_validator() -> [ (config_variable, error_message) ] Called by core check_config() view. """ from desktop.lib.fsmanager import get_filesystem from hadoop.cluster import get_all_hdfs from hadoop.fs.hadoopfs import Hdfs from liboozie.oozie_api import get_oozie res = [] try: from oozie.conf import REMOTE_SAMPLE_DIR except Exception as e: LOG.warn('Config check failed because Oozie app not installed: %s' % e) return res if OOZIE_URL.get(): status = get_oozie_status(user) if 'NORMAL' not in status: res.append((status, _('The Oozie server is not available'))) fs = get_filesystem() NICE_NAME = 'Oozie' if fs.do_as_superuser(fs.exists, REMOTE_SAMPLE_DIR.get()): stats = fs.do_as_superuser(fs.stats, REMOTE_SAMPLE_DIR.get()) mode = oct(stats.mode) # if neither group nor others have write permission group_has_write = int(mode[-2]) & 2 others_has_write = int(mode[-1]) & 2 if not group_has_write and not others_has_write: res.append( (NICE_NAME, "The permissions of workspace '%s' are too restrictive" % REMOTE_SAMPLE_DIR.get())) api = get_oozie(user, api_version="v2") configuration = api.get_configuration() if 'org.apache.oozie.service.MetricsInstrumentationService' in [ c.strip() for c in configuration.get('oozie.services.ext', '').split(',') ]: metrics = api.get_metrics() sharelib_url = 'gauges' in metrics and 'libs.sharelib.system.libpath' in metrics[ 'gauges'] and [ metrics['gauges']['libs.sharelib.system.libpath']['value'] ] or [] else: intrumentation = api.get_instrumentation() sharelib_url = [ param['value'] for group in intrumentation['variables'] for param in group['data'] if param['name'] == 'sharelib.system.libpath' ] if sharelib_url: sharelib_url = Hdfs.urlsplit(sharelib_url[0])[2] if not sharelib_url: res.append((status, _('Oozie Share Lib path is not available'))) class ConfigMock(object): def __init__(self, value): self.value = value def get(self): return self.value def get_fully_qualifying_key(self): return self.value for cluster in list(get_all_hdfs().values()): res.extend( validate_path( ConfigMock(sharelib_url), is_dir=True, fs=cluster, message=_( 'Oozie Share Lib not installed in default location.'))) return res
def handle(self, *args, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): with transaction.atomic(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False) if IS_HUE_4.get(): # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() elif USE_NEW_EDITOR.get(): # Install as link-workflow doc2 to old Job Designs docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) # Don't overwrite doc2, created = Document2.objects.get_or_create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data ) LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,)) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) if not IS_HUE_4.get(): self.install_examples() Document.objects.sync()
res = [] try: from oozie.conf import REMOTE_SAMPLE_DIR except Exception, e: LOG.warn('Config check failed because Oozie app not installed: %s' % e) return res if OOZIE_URL.get(): status = get_oozie_status(user) if 'NORMAL' not in status: res.append((status, _('The Oozie server is not available'))) fs = get_filesystem() NICE_NAME = 'Oozie' if fs.exists(REMOTE_SAMPLE_DIR.get()): stats = fs.stats(REMOTE_SAMPLE_DIR.get()) mode = oct(stats.mode) # if neither group nor others have write permission group_has_write = int(mode[-2]) & 2 others_has_write = int(mode[-1]) & 2 if not group_has_write and not others_has_write: res.append((NICE_NAME, "The permissions of workspace '%s' are too restrictive" % REMOTE_SAMPLE_DIR.get())) api = get_oozie(user, api_version="v2") configuration = api.get_configuration() if 'org.apache.oozie.service.MetricsInstrumentationService' in [c.strip() for c in configuration.get('oozie.services.ext', '').split(',')]: metrics = api.get_metrics() sharelib_url = 'gauges' in metrics and 'libs.sharelib.system.libpath' in metrics['gauges'] and [metrics['gauges']['libs.sharelib.system.libpath']['value']] or []
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info( _("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR) if USE_NEW_EDITOR.get(): docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({ 'content_type': doc.content_type.model, 'object_id': doc.object_id }) data = json.dumps(data) doc2 = Document2.objects.create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data) LOG.info( 'Successfully installed sample link to jobsub: %s' % (doc2.name, )) # Share oozie examples with default group oozie_examples = Document2.objects.filter(type__in=[ 'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2' ], owner=self.user, parent_directory=None) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) if IS_HUE_4.get(): # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() elif USE_NEW_EDITOR.get(): # Install as link-workflow doc2 to old Job Designs docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) # Don't overwrite doc2, created = Document2.objects.get_or_create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data ) LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,)) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) if not IS_HUE_4.get(): self.install_examples() Document.objects.sync()
def handle(self, *args, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info( _("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): with transaction.atomic(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False) # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() # Share oozie examples with default group oozie_examples = Document2.objects.filter(type__in=[ 'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2' ], owner=self.user, parent_directory=None) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])