def handle_noargs(self, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs install_sample_user() management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2) Document.objects.sync()
def handle_noargs(self, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs USERNAME = '******' try: sample_user = User.objects.get(username=USERNAME) except User.DoesNotExist: sample_user = User.objects.create(username=USERNAME, password='******', is_active=False, is_superuser=False, id=1100713, pk=1100713) management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2) Document.objects.sync()
def handle_noargs(self, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs sample_user = install_sample_user() management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2) Document.objects.sync() if USE_NEW_EDITOR.get(): # Get or create sample user directories home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR) try: # Don't overwrite doc = Document.objects.get(object_id=1100713) doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript') # If document exists but has been trashed, recover from Trash if doc2.parent_directory != examples_dir: doc2.parent_directory = examples_dir doc2.save() except Document.DoesNotExist: LOG.warn('Sample pig script document not found.') except Document2.DoesNotExist: if doc.content_object: data = doc.content_object.dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) doc2 = Document2.objects.create( owner=sample_user, parent_directory=examples_dir, name=doc.name, type='link-pigscript', description=doc.description, data=data) LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,)) # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle(self, *args, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() sample_user = install_sample_user() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Initialize doc2, whether editor script or link doc2 = None # Install editor pig script without doc1 link LOG.info("Using Hue 4, will install pig editor sample.") doc2 = self.install_pig_script(sample_user) if USE_NEW_EDITOR.get(): # Get or create sample user directories LOG.info("Creating sample user directories.") home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR) # If document exists but has been trashed, recover from Trash if doc2 and doc2.parent_directory != examples_dir: doc2.parent_directory = examples_dir doc2.save() # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle(self, *args, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() sample_user = install_sample_user() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Initialize doc2, whether editor script or link doc2 = None # Install editor pig script without doc1 link LOG.info("Using Hue 4, will install pig editor sample.") doc2 = self.install_pig_script(sample_user) if USE_NEW_EDITOR.get(): # Get or create sample user directories LOG.info("Creating sample user directories.") home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR) # If document exists but has been trashed, recover from Trash if doc2 and doc2.parent_directory != examples_dir: doc2.parent_directory = examples_dir doc2.save() # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle_noargs(self, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs USERNAME = '******' try: sample_user = User.objects.get(username=USERNAME) except User.DoesNotExist: sample_user = User.objects.create(username=USERNAME, password='******', is_active=False, is_superuser=False, id=1100713, pk=1100713) management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2)
def handle_noargs(self, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs install_sample_user() management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2) Document.objects.sync()
key="definition_xslt2_dir", default=os.path.join(os.path.dirname(__file__), "importlib", "xslt2"), help=_t( "Location on local FS where the xslt files are stored for workflow import." ), private=True) LOCAL_SAMPLE_DIR = Config( key="local_data_dir", default=os.path.join(os.path.dirname(__file__), "..", "..", "examples"), help=_t("Location on local filesystem where the examples are stored."), private=True) LOCAL_SAMPLE_DATA_DIR = Config( key="sample_data_dir", default=paths.get_thirdparty_root("sample_data"), help=_t( "Location on local filesystem where the data for the examples is stored." ), private=True) REMOTE_SAMPLE_DIR = Config( key="remote_data_dir", default="/user/hue/oozie/workspaces", help=_t( "Location on HDFS where the Oozie workflows are stored. Parameters are $TIME and $USER, e.g. /user/$USER/hue/workspaces/workflow-$TIME" )) def get_oozie_job_count(): '''Returns the maximum of jobs fetched by the API depending on the Hue version'''
key="definition_xslt_dir", default=os.path.join(os.path.dirname(__file__), "importlib", "xslt"), help=_t("Location on local FS where the xslt files are stored for workflow import."), private=True ) LOCAL_SAMPLE_DIR = Config( key="local_data_dir", default=os.path.join(os.path.dirname(__file__), "..", "..", "examples"), help=_t("Location on local filesystem where the examples are stored."), private=True ) LOCAL_SAMPLE_DATA_DIR = Config( key="sample_data_dir", default=paths.get_thirdparty_root("sample_data"), help=_t("Location on local filesystem where the data for the examples is stored."), private=True ) REMOTE_SAMPLE_DIR = Config( key="remote_data_dir", default="/user/hue/oozie/workspaces", help=_t("Location on HDFS where the Oozie workflows are stored.") ) OOZIE_JOBS_COUNT = Config( key='oozie_jobs_count', default=100, type=int, help=_t('Maximum number of Oozie workflows or coodinators or bundles to retrieve in one API call.')
REMOTE_DATA_DIR = Config( key="remote_data_dir", default="/user/hue/jobsub", help="Location on HDFS where the jobsub examples and templates are stored." ) # Where examples and templates are stored. LOCAL_DATA_DIR = Config( key="local_data_dir", default=os.path.join(os.path.dirname(__file__), "..", "..", "data"), help="Location on local FS where examples and template are stored", private=True) SAMPLE_DATA_DIR = Config( key="sample_data_dir", default=paths.get_thirdparty_root("sample_data"), help="Location on local FS where sample data is stored", private=True) # Location of trace jar ASPECTPATH = Config(key="aspect_path", default=os.path.join(os.path.dirname(__file__), "..", "..", "java-lib", "trace.jar"), help="Path to the built jobsub trace.jar", private=True) # Location of aspectj weaver jar ASPECTJWEAVER = Config( key="aspectj_weaver", default=paths.get_thirdparty_root("java", "aspectj-1.6.5", "aspectjweaver.jar"),
def handle_noargs(self, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs sample_user = install_sample_user() management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2) Document.objects.sync() if USE_NEW_EDITOR.get(): # Get or create sample user directories home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR) try: # Don't overwrite doc = Document.objects.get(object_id=1100713) doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript') # If document exists but has been trashed, recover from Trash if doc2.parent_directory != examples_dir: doc2.parent_directory = examples_dir doc2.save() except Document.DoesNotExist: LOG.warn('Sample pig script document not found.') except Document2.DoesNotExist: if doc.content_object: data = doc.content_object.dict data.update({ 'content_type': doc.content_type.model, 'object_id': doc.object_id }) data = json.dumps(data) doc2 = Document2.objects.create( owner=sample_user, parent_directory=examples_dir, name=doc.name, type='link-pigscript', description=doc.description, data=data) LOG.info( 'Successfully installed sample link to pig script: %s' % (doc2.name, )) # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle(self, *args, **options): fs = cluster.get_hdfs() create_directories(fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() sample_user = install_sample_user() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = paths.get_thirdparty_root("sample_data") remote_data_dir = fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir) # Initialize doc2, whether editor script or link doc2 = None if IS_HUE_4.get(): # Install editor pig script without doc1 link LOG.info("Using Hue 4, will install pig editor sample.") doc2 = self.install_pig_script(sample_user) else: # Install old pig script fixture LOG.info("Using Hue 3, will install pig script fixture.") with transaction.atomic(): management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2, commit=False) Document.objects.sync() if USE_NEW_EDITOR.get(): # Get or create sample user directories LOG.info("Creating sample user directories.") home_dir = Directory.objects.get_home_directory(sample_user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=sample_user, name=Document2.EXAMPLES_DIR) if not IS_HUE_4.get(): try: # Don't overwrite doc = Document.objects.get(object_id=1100713) doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript') except Document.DoesNotExist: LOG.warn('Sample pig script document not found.') except Document2.DoesNotExist: if doc.content_object: data = doc.content_object.dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) doc2 = Document2.objects.create( owner=sample_user, parent_directory=examples_dir, name=doc.name, type='link-pigscript', description=doc.description, data=data) LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,)) # If document exists but has been trashed, recover from Trash if doc2 and doc2.parent_directory != examples_dir: doc2.parent_directory = examples_dir doc2.save() # Share with default group examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
REMOTE_DATA_DIR = Config( key="remote_data_dir", default="/user/hue/jobsub", help="Location on HDFS where the jobsub examples and templates are stored.") # Where examples and templates are stored. LOCAL_DATA_DIR = Config( key="local_data_dir", default=os.path.join(os.path.dirname(__file__), "..", "..", "data"), help="Location on local FS where examples and template are stored", private=True ) SAMPLE_DATA_DIR = Config( key="sample_data_dir", default=paths.get_thirdparty_root("sample_data"), help="Location on local FS where sample data is stored", private=True) # Location of trace jar ASPECTPATH = Config( key="aspect_path", default=os.path.join(os.path.dirname(__file__), "..", "..", "java-lib", "trace.jar"), help="Path to the built jobsub trace.jar", private=True) # Location of aspectj weaver jar ASPECTJWEAVER = Config( key="aspectj_weaver", default=paths.get_thirdparty_root( "java", "aspectj-1.6.5", "aspectjweaver.jar"),