def handle_noargs(self, **options):
fs = cluster.get_hdfs()
create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = fs.join(remote_dir, name)
LOG.info(
_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
{
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
remote_data_dir)
# Copy sample data
local_dir = paths.get_thirdparty_root("sample_data")
remote_data_dir = fs.join(remote_dir, 'data')
LOG.info(
_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
remote_data_dir)
# Load jobs
install_sample_user()
management.call_command('loaddata',
'initial_pig_examples.json',
verbosity=2)
Document.objects.sync()
def handle_noargs(self, **options):
if not Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).exists():
install_sample_user()
management.call_command('loaddata', 'initial_search_examples.json', verbosity=2)
Document.objects.sync()
def handle_noargs(self, **options):
if not Document2.objects.filter(
type='search-dashboard',
owner__username=SAMPLE_USERNAME).exists():
install_sample_user()
management.call_command('loaddata',
'initial_search_examples.json',
verbosity=2)
Document.objects.sync()
def handle(self, *args, **options):
if not options.get('user'):
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
else:
user = options['user']
install_sample_user()
management.call_command('loaddata', 'apps/spark/src/spark/fixtures/initial_spark_examples.json', verbosity=2)
Document.objects.sync()
from beeswax.management.commands.beeswax_install_examples import Command
app_name = 'beeswax'
Command().handle(app_name=app_name, user=user, tables='web_logs_table.json')
def handle(self, *args, **options):
if not options.get('user'):
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
else:
user = options['user']
if not Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).exists():
install_sample_user()
management.call_command('loaddata', 'apps/spark/src/spark/fixtures/initial_spark_examples.json', verbosity=2)
Document.objects.sync()
from beeswax.management.commands.beeswax_install_examples import Command
app_name = 'beeswax'
Command().handle(app_name=app_name, user=user, tables='web_logs_table.json')
def handle(self, *args, **options):
if not options.get('user'):
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
else:
user = options['user']
if not Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).exists():
install_sample_user()
management.call_command('loaddata', 'desktop/libs/notebook/src/notebook/fixtures/initial_notebook_examples.json', verbosity=2)
Document.objects.sync()
from beeswax.management.commands.beeswax_install_examples import Command
app_name = 'beeswax'
Command().handle(app_name=app_name, user=user, tables='tables.json')
def handle(self, *args, **options):
if not options.get('user'):
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
else:
user = options['user']
# Install sample notebook from fixture if notebook with sample UUID doesn't exist
if not Document2.objects.filter(uuid="7f2ea775-e067-4fde-8f5f-4d704ab9b002").exists():
sample_user = install_sample_user()
management.call_command('loaddata', 'initial_notebook_examples.json', verbosity=2)
Document.objects.sync()
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR
)
Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir)
# Share with default group
examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample notebook')
from beeswax.management.commands.beeswax_install_examples import Command
app_name = 'beeswax'
Command().handle(app_name=app_name, user=user, tables='tables.json')
def authenticate(self, request=None, username=None, password=None):
username = force_username_case(username)
if AUTH.PAM_USE_PWD_MODULE.get():
LOG.debug('Setting username to %s using PAM pwd module for user %s' % (getpwnam(username).pw_name, username))
username = getpwnam(username).pw_name
if pam.authenticate(username, password, AUTH.PAM_SERVICE.get()):
is_super = False
if User.objects.exclude(id=install_sample_user().id).count() == 0:
is_super = True
try:
if AUTH.IGNORE_USERNAME_CASE.get():
user = User.objects.get(username__iexact=username)
else:
user = User.objects.get(username=username)
except User.DoesNotExist:
user = find_or_create_user(username, None)
if user is not None and user.is_active:
profile = get_profile(user)
profile.creation_method = UserProfile.CreationMethod.EXTERNAL.name
profile.save()
user.is_superuser = is_super
ensure_has_a_group(user)
user.save()
user = rewrite_user(user)
return user
return None
def authenticate(self, request=None, remote_user=None):
username = self.clean_username(remote_user)
username = force_username_case(username)
is_super = False
if User.objects.exclude(id=install_sample_user().id).count() == 0:
is_super = True
try:
if AUTH.IGNORE_USERNAME_CASE.get():
user = User.objects.get(username__iexact=username)
else:
user = User.objects.get(username=username)
except User.DoesNotExist:
user = find_or_create_user(username, None)
if user is not None and user.is_active:
profile = get_profile(user)
profile.creation_method = UserProfile.CreationMethod.EXTERNAL.name
profile.save()
user.is_superuser = is_super
ensure_has_a_group(user)
user.save()
user = rewrite_user(user)
return user
def create_table(user=None, query_server=None, table=None):
if not user:
user = install_sample_user()
if not query_server:
query_server = get_query_server_config('beeswax')
if not table:
base_dir = hive_site.get_hive_hook_proto_base_directory()
if not base_dir:
msg = _(
'Error creating table query_data hive.hook.proto.base-directory is not configured'
)
LOG.error(msg)
return False
table = {'name': 'query_data', 'external_location': base_dir}
server = dbms.get(user, query_server)
for query in ["create_table_query_data.mako", "msck.mako"]:
proposed_query = django_mako.render_to_string(query, {'table': table})
query = hql_query(proposed_query)
try:
handle = server.execute_and_wait(query)
if not handle:
LOG.error(_('Error executing %s: Operation timeout.' % query))
return False
server.close(handle)
except Exception as ex:
LOG.error(
_('Error executing %(query)s: %(error)s.') % {
'query': query,
'error': ex
})
return False
LOG.info(_('Table query_data has been created successfully'))
return True
def handle(self, *args, **options):
if not options.get('user'):
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
else:
user = options['user']
dialect = options.get('dialect', 'hive')
# Install sample notebook from fixture if notebook with sample UUID doesn't exist
if not Document2.objects.filter(uuid="7f2ea775-e067-4fde-8f5f-4d704ab9b002").exists():
sample_user = install_sample_user()
with transaction.atomic():
management.call_command('loaddata', 'initial_notebook_examples.json', verbosity=2, commit=False)
Document.objects.sync()
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR
)
Document2.objects.filter(type='notebook', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir)
# Share with default group
examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample notebook')
from beeswax.management.commands.beeswax_install_examples import Command
Command().handle(dialect=dialect, user=user)
def _execute_query(proposed_query, limit):
user = install_sample_user()
query_server = get_query_server_config('beeswax')
server = dbms.get(user, query_server)
query = hql_query(proposed_query)
try:
handle = server.execute_and_wait(query)
if not handle:
LOG.error(_('Error executing %s: Operation timeout.' % query))
return []
results = server.fetch(handle, True, limit)
rows = [row for row in results.rows()]
data = {
'data': rows,
'columns': [column.name for column in results.data_table.cols()]
}
return data
except Exception as ex:
raise_popup_exception(_('Error fetching query history.'))
finally:
try:
if server and handle:
server.close(handle)
except Exception as ex:
raise_popup_exception(_('Error fetching query history.'))
def install_custom_examples():
if EXAMPLES.AUTO_LOAD.get():
from desktop.auth.backend import rewrite_user
from beeswax.management.commands import beeswax_install_examples
from useradmin.models import install_sample_user
user = rewrite_user(install_sample_user())
if has_connectors():
interpreters = [{
'type': connector['id'],
'dialect': connector['dialect']
} for connector in _get_installed_connectors(category='editor')]
else:
interpreters = [
{
'type': interpreter['dialect'],
'dialect': interpreter['dialect']
} for interpreter in get_ordered_interpreters(user)
# Only for hive/impala currently, would also need to port to Notebook install examples.
if interpreter['dialect'] in ('hive', 'impala')
]
queries = EXAMPLES.QUERIES.get()
tables = EXAMPLES.TABLES.get(
) # No-op. Only for the saved query samples, not the tables currently.
LOG.info(
'Installing custom examples queries: %(queries)s, tables: %(tables)s for dialects %(dialects)s '
'belonging to user %(user)s' % {
'queries':
queries,
'tables':
tables,
'dialects':
[interpreter['dialect'] for interpreter in interpreters],
'user':
user
})
result = []
for interpreter in interpreters:
successes, errors = beeswax_install_examples.Command().handle(
dialect=interpreter['dialect'],
user=user,
interpreter=interpreter,
queries=queries,
tables=tables,
request=None)
LOG.info(
'Dialect %(dialect)s installed samples: %(successes)s, %(errors)s,'
% {
'dialect': interpreter['dialect'],
'successes': successes,
'errors': errors,
})
result.append((successes, errors))
return result
def handle(self, *args, **options):
sample_user = install_sample_user()
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR
)
if not Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).exists():
with transaction.atomic():
if sys.version_info[0] > 2:
management.call_command('loaddata', 'initial_search_examples.json', verbosity=2)
else:
management.call_command('loaddata', 'initial_search_examples.json', verbosity=2, commit=False)
Document.objects.sync()
Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir)
else:
# Check if sample documents are in Trash, and if so, restore them
for doc in Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS):
if doc.parent_directory != examples_dir:
doc.parent_directory = examples_dir
doc.save()
# Share with default group
examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample search dashboard')
def is_first_login_ever(self):
""" Return true if no one has ever logged in."""
if ENABLE_ORGANIZATIONS.get(): # Tmp
return False
else:
return User.objects.exclude(
id=install_sample_user().id).count() == 0
def handle_noargs(self, **options):
fs = cluster.get_hdfs()
create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = paths.get_thirdparty_root("sample_data")
remote_data_dir = fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)
# Load jobs
sample_user = install_sample_user()
management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2)
Document.objects.sync()
if USE_NEW_EDITOR.get():
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR)
try:
# Don't overwrite
doc = Document.objects.get(object_id=1100713)
doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript')
# If document exists but has been trashed, recover from Trash
if doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
except Document.DoesNotExist:
LOG.warn('Sample pig script document not found.')
except Document2.DoesNotExist:
if doc.content_object:
data = doc.content_object.dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
data = json.dumps(data)
doc2 = Document2.objects.create(
owner=sample_user,
parent_directory=examples_dir,
name=doc.name,
type='link-pigscript',
description=doc.description,
data=data)
LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,))
# Share with default group
examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle_noargs(self, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
self.install_examples()
Document.objects.sync()
def handle_noargs(self, **options):
sample_user = install_sample_user()
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR
)
if not Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).exists():
management.call_command('loaddata', 'initial_search_examples.json', verbosity=2)
Document.objects.sync()
Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS).update(parent_directory=examples_dir)
else:
# Check if sample documents are in Trash, and if so, restore them
for doc in Document2.objects.filter(type='search-dashboard', owner__username__in=SAMPLE_USER_OWNERS):
if doc.parent_directory != examples_dir:
doc.parent_directory = examples_dir
doc.save()
# Share with default group
examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample search dashboard')
def handle_noargs(self, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
self.install_examples()
Document.objects.sync()
def _get_dialect_example(dialect):
sample_user = install_sample_user()
examples_dir = _get_example_directory(sample_user)
return Document2.objects.filter(owner=sample_user,
type='query-%s' % dialect,
is_history=False,
parent_directory=examples_dir).first()
def handle_noargs(self, **options):
exception = None
try:
user = install_sample_user()
self._install_tables(user, options['app_name'])
except Exception, ex:
exception = ex
def handle_noargs(self, **options):
"""Main entry point to install or re-install examples. May raise InstallException"""
try:
user = install_sample_user()
self._install_tables(user, options['app_name'])
self._install_queries(user, options['app_name'])
except Exception, ex:
LOG.exception(ex)
raise InstallException(ex)
def handle_noargs(self, **options):
exception = None
# Documents will belong to this user but we run the install as the current user
try:
sample_user = install_sample_user()
self._install_tables(options['user'], options['app_name'])
except Exception, ex:
exception = ex
def handle_noargs(self, **options):
"""Main entry point to install or re-install examples. May raise InstallException"""
try:
user = install_sample_user()
self._install_tables(user, options['app_name'])
self._install_queries(user, options['app_name'])
except Exception, ex:
LOG.exception(ex)
raise InstallException(ex)
def handle(self, *args, **options):
fs = cluster.get_hdfs()
create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
sample_user = install_sample_user()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = fs.join(remote_dir, name)
LOG.info(
_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
{
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir,
remote_data_dir)
# Copy sample data
local_dir = paths.get_thirdparty_root("sample_data")
remote_data_dir = fs.join(remote_dir, 'data')
LOG.info(
_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir,
remote_data_dir)
# Initialize doc2, whether editor script or link
doc2 = None
# Install editor pig script without doc1 link
LOG.info("Using Hue 4, will install pig editor sample.")
doc2 = self.install_pig_script(sample_user)
if USE_NEW_EDITOR.get():
# Get or create sample user directories
LOG.info("Creating sample user directories.")
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR)
# If document exists but has been trashed, recover from Trash
if doc2 and doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
# Share with default group
examples_dir.share(sample_user,
Document2Permission.READ_PERM,
groups=[get_default_user_group()])
def handle(self, *args, **options):
if args:
dialect = args[0]
db_name = args[1] if len(args) > 1 else 'default'
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
request = None
else:
dialect = options['dialect']
db_name = options.get('db_name', 'default')
interpreter = options.get(
'interpreter'
) # Only when connectors are enabled. Later will deprecate `dialect`.
user = options['user']
request = options['request']
tables = 'tables_standard.json' if dialect not in (
'hive',
'impala') else ('tables_transactional.json'
if has_concurrency_support() else 'tables.json')
exception = None
self.successes = []
self.errors = []
try:
sample_user = install_sample_user(
user
) # Documents will belong to the sample user but we run the SQL as the current user
self.install_queries(sample_user, dialect, interpreter=interpreter)
self.install_tables(user,
dialect,
db_name,
tables,
interpreter=interpreter,
request=request)
except Exception as ex:
exception = ex
if exception is not None:
pretty_msg = None
if "Permission denied" in str(exception):
pretty_msg = _(
"Permission denied. Please check with your system administrator."
)
if pretty_msg is not None:
raise PopupException(pretty_msg)
else:
raise exception
return self.successes, self.errors
def handle_noargs(self, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(self.user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=self.user,
name=Document2.EXAMPLES_DIR
)
# Share oozie examples with default group
oozie_examples = Document2.objects.filter(
type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
owner=self.user,
parent_directory=None
)
oozie_examples.update(parent_directory=examples_dir)
examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
self.install_examples()
Document.objects.sync()
def handle_noargs(self, **options):
fs = cluster.get_hdfs()
create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = paths.get_thirdparty_root("sample_data")
remote_data_dir = fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)
# Load jobs
install_sample_user()
management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2)
Document.objects.sync()
def handle_noargs(self, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
self.searcher = controller.CollectionManagerController(self.user)
LOG.info(_("Installing twitter collection"))
path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_twitter_demo/index_data.csv'))
self._setup_collection_from_csv({
'name': 'twitter_demo',
'fields': self._parse_fields(path),
'uniqueKeyField': 'id',
'df': 'text'
}, path)
LOG.info(_("Twitter collection successfully installed"))
LOG.info(_("Installing yelp collection"))
path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_yelp_demo/index_data.csv'))
self._setup_collection_from_csv({
'name': 'yelp_demo',
'fields': self._parse_fields(path),
'uniqueKeyField': 'id',
'df': 'text'
}, path)
LOG.info(_("Yelp collection successfully installed"))
LOG.info(_("Installing jobs collection"))
path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_jobs_demo/index_data.csv'))
self._setup_collection_from_csv({
'name': 'jobs_demo',
'fields': self._parse_fields(path),
'uniqueKeyField': 'id',
'df': 'description'
}, path)
LOG.info(_("Jobs collection successfully installed"))
LOG.info(_("Installing logs collection"))
path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_log_analytics_demo/index_data.csv'))
self._setup_collection_from_csv({
'name': 'log_analytics_demo',
'fields': self._parse_fields(path, fieldtypes={
'region_code': 'string',
'referer': 'string'
}),
'uniqueKeyField': 'id',
'df': 'record'
}, path)
LOG.info(_("Logs collection successfully installed"))
def handle(self, *args, **options):
if args:
app_name = args[0]
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
else:
app_name = options['app_name']
user = options['user']
exception = None
# Documents will belong to this user but we run the install as the current user
try:
sample_user = install_sample_user()
self._install_queries(sample_user, app_name)
self._install_tables(user, app_name)
except Exception, ex:
exception = ex
def handle(self, *args, **options):
if args:
user = args[0]
else:
user = install_sample_user()
api = HbaseApi(user=user)
cluster_name = api.getClusters()[0]['name'] # Currently pick first configured cluster
# Check connectivity
api.connectCluster(cluster_name)
self.create_analytics_table(api, cluster_name)
self.load_analytics_table(api, cluster_name)
self.create_binary_table(api, cluster_name)
self.load_binary_table(api, cluster_name)
def handle(self, *args, **options):
if args:
app_name = args[0]
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
else:
app_name = options['app_name']
user = options['user']
exception = None
# Documents will belong to this user but we run the install as the current user
try:
sample_user = install_sample_user()
self._install_queries(sample_user, app_name)
self._install_tables(user, app_name)
except Exception, ex:
exception = ex
def handle(self, *args, **options):
fs = cluster.get_hdfs()
create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
sample_user = install_sample_user()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = paths.get_thirdparty_root("sample_data")
remote_data_dir = fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)
# Initialize doc2, whether editor script or link
doc2 = None
# Install editor pig script without doc1 link
LOG.info("Using Hue 4, will install pig editor sample.")
doc2 = self.install_pig_script(sample_user)
if USE_NEW_EDITOR.get():
# Get or create sample user directories
LOG.info("Creating sample user directories.")
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR)
# If document exists but has been trashed, recover from Trash
if doc2 and doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
# Share with default group
examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle(self, *args, **options):
if args:
user = args[0]
else:
user = install_sample_user()
api = HbaseApi(user=user)
cluster_name = api.getClusters()[0][
'name'] # Currently pick first configured cluster
# Check connectivity
api.connectCluster(cluster_name)
self.create_analytics_table(api, cluster_name)
self.load_analytics_table(api, cluster_name)
self.create_binary_table(api, cluster_name)
self.load_binary_table(api, cluster_name)
def handle(self, *args, **options):
if args:
app_name = args[0]
db_name = args[1] if len(args) > 1 else 'default'
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
else:
app_name = options['app_name']
db_name = options.get('db_name', 'default')
user = options['user']
tables = options['tables'] if 'tables' in options else 'tables.json'
exception = None
# Documents will belong to this user but we run the install as the current user
try:
sample_user = install_sample_user()
self._install_queries(sample_user, app_name)
self._install_tables(user, app_name, db_name, tables)
except Exception, ex:
exception = ex
def handle(self, *args, **options):
if args:
app_name = args[0]
db_name = args[1] if len(args) > 1 else 'default'
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
else:
app_name = options['app_name']
db_name = options.get('db_name', 'default')
user = options['user']
tables = options['tables'] if 'tables' in options else (
'tables_transactional.json'
if has_concurrency_support() else 'tables.json')
exception = None
# Documents will belong to this user but we run the install as the current user
try:
sample_user = install_sample_user()
self._install_queries(sample_user, app_name)
self._install_tables(user, app_name, db_name, tables)
except Exception as ex:
exception = ex
Document.objects.sync()
if exception is not None:
pretty_msg = None
if "AlreadyExistsException" in exception.message:
pretty_msg = _("SQL table examples already installed.")
if "Permission denied" in exception.message:
pretty_msg = _(
"Permission denied. Please check with your system administrator."
)
if pretty_msg is not None:
raise PopupException(pretty_msg)
else:
raise exception
def handle(self, *args, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(self.user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=self.user,
name=Document2.EXAMPLES_DIR
)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
with transaction.atomic():
management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False)
# Install editor oozie examples without doc1 link
LOG.info("Using Hue 4, will install oozie editor samples.")
example_jobs = []
example_jobs.append(self._install_mapreduce_example())
example_jobs.append(self._install_java_example())
example_jobs.append(self._install_spark_example())
example_jobs.append(self._install_pyspark_example())
# If documents exist but have been trashed, recover from Trash
for doc in example_jobs:
if doc is not None and doc.parent_directory != examples_dir:
doc.parent_directory = examples_dir
doc.save()
# Share oozie examples with default group
oozie_examples = Document2.objects.filter(
type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
owner=self.user,
parent_directory=None
)
oozie_examples.update(parent_directory=examples_dir)
examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle(self, *args, **options):
self.user = install_sample_user()
self.client = SolrClient(self.user)
collection = options['data']
if collection == 'twitter_demo':
LOG.info("Installing twitter collection")
path = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'../../../../../../../apps/search/examples/collections/solr_configs_twitter_demo/index_data.csv'
))
self._setup_collection_from_csv(
{
'name':
'twitter_demo',
'fields':
self._parse_fields(path,
fieldtypes={
'source': 'string',
'username': '******',
}),
'uniqueKeyField':
'id',
'df':
'text'
}, path)
LOG.info("Twitter collection successfully installed")
if collection == 'yelp_demo':
LOG.info("Installing yelp collection")
path = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'../../../../../../../apps/search/examples/collections/solr_configs_yelp_demo/index_data.csv'
))
self._setup_collection_from_csv(
{
'name':
'yelp_demo',
'fields':
self._parse_fields(path, fieldtypes={
'name': 'string',
}),
'uniqueKeyField':
'id',
'df':
'text'
}, path)
LOG.info("Yelp collection successfully installed")
if collection == 'log_analytics_demo':
LOG.info("Installing logs collection")
path = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'../../../../../../../apps/search/examples/collections/solr_configs_log_analytics_demo/index_data.csv'
))
self._setup_collection_from_csv(
{
'name':
'log_analytics_demo',
'fields':
self._parse_fields(path,
fieldtypes={
'region_code': 'string',
'referer': 'string',
'user_agent': 'string'
}),
'uniqueKeyField':
'id',
'df':
'record'
}, path)
LOG.info("Logs collection successfully installed")
def handle_noargs(self, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(
_("Creating sample directory '%s' in HDFS") %
REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(
_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
{
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(
_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
local_dir, remote_data_dir)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
management.call_command('loaddata',
'initial_oozie_examples.json',
verbosity=2)
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(self.user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=self.user,
name=Document2.EXAMPLES_DIR)
if USE_NEW_EDITOR.get():
docs = Document.objects.get_docs(self.user,
Workflow).filter(owner=self.user)
for doc in docs:
if doc.content_object:
data = doc.content_object.data_dict
data.update({
'content_type': doc.content_type.model,
'object_id': doc.object_id
})
data = json.dumps(data)
doc2 = Document2.objects.create(
owner=self.user,
parent_directory=examples_dir,
name=doc.name,
type='link-workflow',
description=doc.description,
data=data)
LOG.info(
'Successfully installed sample link to jobsub: %s' %
(doc2.name, ))
# Share oozie examples with default group
oozie_examples = Document2.objects.filter(type__in=[
'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'
],
owner=self.user,
parent_directory=None)
oozie_examples.update(parent_directory=examples_dir)
examples_dir.share(self.user,
Document2Permission.READ_PERM,
groups=[get_default_user_group()])
self.install_examples()
Document.objects.sync()
def handle_noargs(self, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(
_("Creating sample directory '%s' in HDFS") %
REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(
_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
{
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(
_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal,
local_dir, remote_data_dir)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
management.call_command('loaddata',
'initial_oozie_examples.json',
verbosity=2)
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(self.user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=self.user,
name=Document2.EXAMPLES_DIR)
# Share oozie examples with default group
oozie_examples = Document2.objects.filter(type__in=[
'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'
],
owner=self.user,
parent_directory=None)
oozie_examples.update(parent_directory=examples_dir)
examples_dir.share(self.user,
Document2Permission.READ_PERM,
groups=[get_default_user_group()])
self.install_examples()
Document.objects.sync()
def handle(self, *args, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(self.user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=self.user,
name=Document2.EXAMPLES_DIR
)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
with transaction.atomic():
management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False)
if IS_HUE_4.get():
# Install editor oozie examples without doc1 link
LOG.info("Using Hue 4, will install oozie editor samples.")
example_jobs = []
example_jobs.append(self._install_mapreduce_example())
example_jobs.append(self._install_java_example())
example_jobs.append(self._install_spark_example())
example_jobs.append(self._install_pyspark_example())
# If documents exist but have been trashed, recover from Trash
for doc in example_jobs:
if doc is not None and doc.parent_directory != examples_dir:
doc.parent_directory = examples_dir
doc.save()
elif USE_NEW_EDITOR.get():
# Install as link-workflow doc2 to old Job Designs
docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
for doc in docs:
if doc.content_object:
data = doc.content_object.data_dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
data = json.dumps(data)
# Don't overwrite
doc2, created = Document2.objects.get_or_create(
owner=self.user,
parent_directory=examples_dir,
name=doc.name,
type='link-workflow',
description=doc.description,
data=data
)
LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))
# Share oozie examples with default group
oozie_examples = Document2.objects.filter(
type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
owner=self.user,
parent_directory=None
)
oozie_examples.update(parent_directory=examples_dir)
examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
if not IS_HUE_4.get():
self.install_examples()
Document.objects.sync()
def handle(self, *args, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(
_("Creating sample directory '%s' in HDFS") %
REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(
_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
{
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal,
local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(
_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal,
local_dir, remote_data_dir)
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(self.user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=self.user,
name=Document2.EXAMPLES_DIR)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
with transaction.atomic():
management.call_command('loaddata',
'initial_oozie_examples.json',
verbosity=2,
commit=False)
# Install editor oozie examples without doc1 link
LOG.info("Using Hue 4, will install oozie editor samples.")
example_jobs = []
example_jobs.append(self._install_mapreduce_example())
example_jobs.append(self._install_java_example())
example_jobs.append(self._install_spark_example())
example_jobs.append(self._install_pyspark_example())
# If documents exist but have been trashed, recover from Trash
for doc in example_jobs:
if doc is not None and doc.parent_directory != examples_dir:
doc.parent_directory = examples_dir
doc.save()
# Share oozie examples with default group
oozie_examples = Document2.objects.filter(type__in=[
'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'
],
owner=self.user,
parent_directory=None)
oozie_examples.update(parent_directory=examples_dir)
examples_dir.share(self.user,
Document2Permission.READ_PERM,
groups=[get_default_user_group()])
except Exception, e:
LOG.exception('error syncing pig')
try:
with transaction.atomic():
from search.models import Collection
for dashboard in Collection.objects.all():
col_dict = dashboard.properties_dict['collection']
if not 'uuid' in col_dict:
_uuid = str(uuid.uuid4())
col_dict['uuid'] = _uuid
dashboard.update_properties({'collection': col_dict})
if dashboard.owner is None:
from useradmin.models import install_sample_user
owner = install_sample_user()
else:
owner = dashboard.owner
dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard', owner=owner, description=dashboard.label, data=dashboard.properties)
Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label, extra='search-dashboard')
dashboard.save()
except Exception, e:
LOG.exception('error syncing search')
try:
with transaction.atomic():
for job in find_jobs_with_no_doc(Document2):
if job.type == 'oozie-workflow2':
extra = 'workflow2'
elif job.type == 'oozie-coordinator2':
extra = 'coordinator2'
from django.db import connection, models, transaction
from search.models import Collection
from desktop.models import Document, Document2
table_names = connection.introspection.table_names()
if Collection._meta.db_table in table_names:
with transaction.atomic():
for dashboard in Collection.objects.all():
if 'collection' in dashboard.properties_dict:
col_dict = dashboard.properties_dict['collection']
if not 'uuid' in col_dict:
_uuid = str(uuid.uuid4())
col_dict['uuid'] = _uuid
dashboard.update_properties({'collection': col_dict})
if dashboard.owner is None:
from useradmin.models import install_sample_user
owner = install_sample_user()
else:
owner = dashboard.owner
dashboard_doc = Document2.objects.create(
name=dashboard.label,
uuid=_uuid,
type='search-dashboard',
owner=owner,
description=dashboard.label,
data=dashboard.properties)
Document.objects.link(dashboard_doc,
owner=owner,
name=dashboard.label,
description=dashboard.label,
extra='search-dashboard')
dashboard.save()
def handle_noargs(self, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(self.user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=self.user,
name=Document2.EXAMPLES_DIR
)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
if IS_HUE_4.get():
# Install editor oozie examples without doc1 link
LOG.info("Using Hue 4, will install oozie editor samples.")
example_jobs = []
example_jobs.append(self._install_mapreduce_example())
example_jobs.append(self._install_java_example())
example_jobs.append(self._install_spark_example())
example_jobs.append(self._install_pyspark_example())
# If documents exist but have been trashed, recover from Trash
for doc in example_jobs:
if doc is not None and doc.parent_directory != examples_dir:
doc.parent_directory = examples_dir
doc.save()
elif USE_NEW_EDITOR.get():
# Install as link-workflow doc2 to old Job Designs
docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
for doc in docs:
if doc.content_object:
data = doc.content_object.data_dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
data = json.dumps(data)
# Don't overwrite
doc2, created = Document2.objects.get_or_create(
owner=self.user,
parent_directory=examples_dir,
name=doc.name,
type='link-workflow',
description=doc.description,
data=data
)
LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))
# Share oozie examples with default group
oozie_examples = Document2.objects.filter(
type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
owner=self.user,
parent_directory=None
)
oozie_examples.update(parent_directory=examples_dir)
examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
if not IS_HUE_4.get():
self.install_examples()
Document.objects.sync()
def handle_noargs(self, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(self.user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=self.user,
name=Document2.EXAMPLES_DIR
)
if USE_NEW_EDITOR.get():
docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
for doc in docs:
if doc.content_object:
data = doc.content_object.data_dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
data = json.dumps(data)
doc2 = Document2.objects.create(
owner=self.user,
parent_directory=examples_dir,
name=doc.name,
type='link-workflow',
description=doc.description,
data=data)
LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))
# Share oozie examples with default group
oozie_examples = Document2.objects.filter(
type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
owner=self.user,
parent_directory=None
)
oozie_examples.update(parent_directory=examples_dir)
examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
self.install_examples()
Document.objects.sync()
def handle_noargs(self, **options):
fs = cluster.get_hdfs()
create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = fs.join(remote_dir, name)
LOG.info(
_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
{
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
remote_data_dir)
# Copy sample data
local_dir = paths.get_thirdparty_root("sample_data")
remote_data_dir = fs.join(remote_dir, 'data')
LOG.info(
_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir,
remote_data_dir)
# Load jobs
sample_user = install_sample_user()
management.call_command('loaddata',
'initial_pig_examples.json',
verbosity=2)
Document.objects.sync()
if USE_NEW_EDITOR.get():
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR)
try:
# Don't overwrite
doc = Document.objects.get(object_id=1100713)
doc2 = Document2.objects.get(owner=sample_user,
name=doc.name,
type='link-pigscript')
# If document exists but has been trashed, recover from Trash
if doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
except Document.DoesNotExist:
LOG.warn('Sample pig script document not found.')
except Document2.DoesNotExist:
if doc.content_object:
data = doc.content_object.dict
data.update({
'content_type': doc.content_type.model,
'object_id': doc.object_id
})
data = json.dumps(data)
doc2 = Document2.objects.create(
owner=sample_user,
parent_directory=examples_dir,
name=doc.name,
type='link-pigscript',
description=doc.description,
data=data)
LOG.info(
'Successfully installed sample link to pig script: %s'
% (doc2.name, ))
# Share with default group
examples_dir.share(sample_user,
Document2Permission.READ_PERM,
groups=[get_default_user_group()])
def handle(self, *args, **options):
fs = cluster.get_hdfs()
create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
sample_user = install_sample_user()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = paths.get_thirdparty_root("sample_data")
remote_data_dir = fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)
# Initialize doc2, whether editor script or link
doc2 = None
if IS_HUE_4.get():
# Install editor pig script without doc1 link
LOG.info("Using Hue 4, will install pig editor sample.")
doc2 = self.install_pig_script(sample_user)
else:
# Install old pig script fixture
LOG.info("Using Hue 3, will install pig script fixture.")
with transaction.atomic():
management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2, commit=False)
Document.objects.sync()
if USE_NEW_EDITOR.get():
# Get or create sample user directories
LOG.info("Creating sample user directories.")
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR)
if not IS_HUE_4.get():
try:
# Don't overwrite
doc = Document.objects.get(object_id=1100713)
doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript')
except Document.DoesNotExist:
LOG.warn('Sample pig script document not found.')
except Document2.DoesNotExist:
if doc.content_object:
data = doc.content_object.dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
data = json.dumps(data)
doc2 = Document2.objects.create(
owner=sample_user,
parent_directory=examples_dir,
name=doc.name,
type='link-pigscript',
description=doc.description,
data=data)
LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,))
# If document exists but has been trashed, recover from Trash
if doc2 and doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
# Share with default group
examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])