def hue(request):
apps = appmanager.get_apps_dict(request.user)
current_app, other_apps, apps_list = _get_apps(request.user, '')
default_cluster_index, default_cluster_interface = Cluster(request.user).get_list_interface_indexes()
clusters = get_clusters().values()
return render('hue.mako', request, {
'apps': apps,
'other_apps': other_apps,
'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user),
'is_adls_enabled': is_adls_enabled() and has_adls_access(request.user),
'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(),
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get(),
'map_options': json.dumps(desktop.conf.LEAFLET_MAP_OPTIONS.get()),
'layer_options': json.dumps(desktop.conf.LEAFLET_TILE_LAYER_OPTIONS.get()),
},
'is_demo': desktop.conf.DEMO_ENABLED.get(),
'banner_message': get_banner_message(request),
'user_preferences': dict((x.key, x.value) for x in UserPreferences.objects.filter(user=request.user)),
'cluster': clusters[0]['type'] if clusters else None,
'clusters_config_json': json.dumps(clusters),
'default_cluster_index': default_cluster_index,
'default_cluster_interface': default_cluster_interface
})
def config_validator(user):
'''
v2
When using the connectors, now 'hive' is seen as a dialect and only the list of connections
(instance of the 'hive' connector, e.g. pointing to a Hive server in the Cloud) should be tested.
Interpreters are now tested by the Editor in libs/notebook/conf.py.
v1
All the configuration happens in apps/beeswax.
'''
from beeswax.design import hql_query # dbms is dependent on beeswax.conf, import in method to avoid circular dependency
from beeswax.server import dbms
res = []
if has_connectors():
return res
try:
try:
if not 'test' in sys.argv: # Avoid tests hanging
server = dbms.get(user)
query = hql_query("SELECT 'Hello World!';")
handle = server.execute_and_wait(query, timeout_sec=10.0)
if handle:
server.fetch(handle, rows=100)
server.close(handle)
except StructuredThriftTransportException as e:
if 'Error validating the login' in str(e):
msg = 'Failed to authenticate to HiveServer2, check authentication configurations.'
LOG.exception(msg)
res.append((NICE_NAME, _(msg)))
else:
raise e
except Exception as e:
msg = "The application won't work without a running HiveServer2."
LOG.exception(msg)
res.append((NICE_NAME, _(msg)))
try:
from desktop.lib.fsmanager import get_filesystem
from aws.conf import is_enabled as is_s3_enabled
warehouse = beeswax.hive_site.get_metastore_warehouse_dir()
fs = get_filesystem()
fs_scheme = fs._get_scheme(warehouse)
if fs:
if fs_scheme == 's3a':
if is_s3_enabled():
fs.do_as_user(user, fs.stats, warehouse)
else:
LOG.warn("Warehouse is in S3, but no credential available.")
else:
fs.do_as_superuser(fs.stats, warehouse)
except Exception:
msg = 'Failed to access Hive warehouse: %s'
LOG.exception(msg % warehouse)
res.append((NICE_NAME, _(msg) % warehouse))
return res
def commonheader(title, section, user, request=None, padding="90px", skip_topbar=False, skip_idle_timeout=False, is_mobile=False):
"""
Returns the rendered common header
"""
current_app, other_apps, apps_list = _get_apps(user, section)
template = 'common_header.mako'
if is_mobile:
template = 'common_header_m.mako'
return django_mako.render_to_string(template, {
'current_app': current_app,
'apps': apps_list,
'other_apps': other_apps,
'title': title,
'section': section,
'padding': padding,
'user': user,
'request': request,
'skip_topbar': skip_topbar,
'skip_idle_timeout': skip_idle_timeout,
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get(),
'map_options': json.dumps(desktop.conf.LEAFLET_MAP_OPTIONS.get()),
'layer_options': json.dumps(desktop.conf.LEAFLET_TILE_LAYER_OPTIONS.get()),
},
'is_demo': desktop.conf.DEMO_ENABLED.get(),
'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(),
'is_s3_enabled': is_s3_enabled() and has_s3_access(user),
'is_adls_enabled': is_adls_enabled() and has_adls_access(request.user),
'banner_message': get_banner_message(request)
})
def hue(request):
apps = appmanager.get_apps_dict(request.user)
return render(
'hue.mako', request, {
'apps':
apps,
'interpreters':
get_ordered_interpreters(request.user),
'is_s3_enabled':
is_s3_enabled() and has_s3_access(request.user),
'is_ldap_setup':
'desktop.auth.backend.LdapBackend'
in desktop.conf.AUTH.BACKEND.get(),
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution':
desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get()
},
'is_demo':
desktop.conf.DEMO_ENABLED.get(),
'banner_message':
get_banner_message(request),
'cluster_config':
ClusterConfig(request.user),
'user_preferences':
dict((x.key, x.value)
for x in UserPreferences.objects.filter(user=request.user))
})
def hue(request):
apps = appmanager.get_apps_dict(request.user)
return render(
'hue.mako', request, {
'apps':
apps,
'tours_and_tutorials':
Settings.get_settings().tours_and_tutorials,
'interpreters':
get_ordered_interpreters(request.user),
'is_s3_enabled':
is_s3_enabled() and has_s3_access(request.user),
'is_ldap_setup':
'desktop.auth.backend.LdapBackend'
in desktop.conf.AUTH.BACKEND.get(),
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution':
desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get()
},
'is_demo':
desktop.conf.DEMO_ENABLED.get(),
'banner_message':
get_banner_message(request),
'cluster_config':
ClusterConfig(request.user)
})
def is_enabled(fs=None):
if fs == 'hdfs':
return has_hdfs_enabled()
elif fs == 'adl':
return is_adls_enabled()
elif fs == 's3a':
return is_s3_enabled()
elif fs == 'abfs':
return is_abfs_enabled()
def global_js_constants(request):
return HttpResponse(render('global_js_constants.mako', request, {
'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user),
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get(),
'map_options': json.dumps(desktop.conf.LEAFLET_MAP_OPTIONS.get()),
'layer_options': json.dumps(desktop.conf.LEAFLET_TILE_LAYER_OPTIONS.get()),
}
}), content_type="application/javascript")
def responsive(request):
apps = appmanager.get_apps_dict(request.user)
return render('responsive.mako', request, {
'apps': apps,
'tours_and_tutorials': Settings.get_settings().tours_and_tutorials,
'interpreters': get_ordered_interpreters(request.user),
'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user),
'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get()
})
def responsive(request):
apps = appmanager.get_apps_dict(request.user)
return render(
'responsive.mako', request, {
'apps': apps,
'tours_and_tutorials': Settings.get_settings().tours_and_tutorials,
'interpreters': get_interpreters(request.user),
'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user)
})
def commonheader(title, section, user, request=None, padding="90px", skip_topbar=False, skip_idle_timeout=False, is_mobile=False):
"""
Returns the rendered common header
"""
current_app = None
other_apps = []
if user.is_authenticated():
apps = appmanager.get_apps(user)
apps_list = appmanager.get_apps_dict(user)
for app in apps:
if app.display_name not in [
'beeswax', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser',
'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark', 'indexer', 'security', 'notebook'] and app.menu_index != -1:
other_apps.append(app)
if section == app.display_name:
current_app = app
else:
apps_list = []
template = 'common_header.mako'
if is_mobile:
template = 'common_header_m.mako'
return django_mako.render_to_string(template, {
'current_app': current_app,
'apps': apps_list,
'other_apps': other_apps,
'title': title,
'section': section,
'padding': padding,
'user': user,
'request': request,
'skip_topbar': skip_topbar,
'skip_idle_timeout': skip_idle_timeout,
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get()
},
'is_demo': desktop.conf.DEMO_ENABLED.get(),
'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(),
'is_s3_enabled': is_s3_enabled() and has_s3_access(user),
'banner_message': get_banner_message(request)
})
def commonheader(title, section, user, request=None, padding="90px", skip_topbar=False, skip_idle_timeout=False, is_mobile=False):
"""
Returns the rendered common header
"""
current_app = None
other_apps = []
if user.is_authenticated():
apps = appmanager.get_apps(user)
apps_list = appmanager.get_apps_dict(user)
for app in apps:
if app.display_name not in [
'beeswax', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser',
'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark', 'indexer', 'security', 'notebook'] and app.menu_index != -1:
other_apps.append(app)
if section == app.display_name:
current_app = app
else:
apps_list = []
template = 'common_header.mako'
if is_mobile:
template = 'common_header_m.mako'
return django_mako.render_to_string(template, {
'current_app': current_app,
'apps': apps_list,
'other_apps': other_apps,
'title': title,
'section': section,
'padding': padding,
'user': user,
'request': request,
'skip_topbar': skip_topbar,
'skip_idle_timeout': skip_idle_timeout,
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get()
},
'is_demo': desktop.conf.DEMO_ENABLED.get(),
'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(),
'is_s3_enabled': is_s3_enabled() and has_s3_access(user)
})
LOAD_BALANCER_COOKIE = 'ROUTEID'
################################################################
# Register file upload handlers
# This section must go after the desktop lib modules are loaded
################################################################
# Insert our custom upload handlers
file_upload_handlers = [
'hadoop.fs.upload.HDFSfileUploadHandler',
'django.core.files.uploadhandler.MemoryFileUploadHandler',
'django.core.files.uploadhandler.TemporaryFileUploadHandler',
]
if is_s3_enabled():
file_upload_handlers.insert(0, 'aws.s3.upload.S3FileUploadHandler')
FILE_UPLOAD_HANDLERS = tuple(file_upload_handlers)
############################################################
# Necessary for South to not fuzz with tests. Fixed in South 0.7.1
SKIP_SOUTH_TESTS = True
# Set up environment variable so Kerberos libraries look at our private
# ticket cache
os.environ['KRB5CCNAME'] = desktop.conf.KERBEROS.CCACHE_PATH.get()
if not os.getenv('SERVER_SOFTWARE'):
os.environ['SERVER_SOFTWARE'] = 'apache'
import sys
import logging
import aws
from aws.conf import is_enabled as is_s3_enabled
from desktop.lib.fs import ProxyFS
from hadoop import cluster
FS_CACHE = {}
DEFAULT_SCHEMA = 'hdfs'
FS_GETTERS = {
"hdfs": cluster.get_hdfs,
"s3a": aws.get_s3fs if is_s3_enabled() else None
}
def get_filesystem(name='default'):
"""
Return the filesystem with the given name.
If the filesystem is not defined, raises KeyError
"""
if name not in FS_CACHE:
FS_CACHE[name] = _make_fs(name)
return FS_CACHE[name]
def _make_fs(name):
fs_dict = {}
if 'useradmin' in [app.name for app in appmanager.DESKTOP_APPS]:
MIDDLEWARE_CLASSES.append('useradmin.middleware.LastActivityMiddleware')
################################################################
# Register file upload handlers
# This section must go after the desktop lib modules are loaded
################################################################
# Insert our custom upload handlers
file_upload_handlers = [
'hadoop.fs.upload.HDFSfileUploadHandler',
'django.core.files.uploadhandler.MemoryFileUploadHandler',
'django.core.files.uploadhandler.TemporaryFileUploadHandler',
]
if is_s3_enabled():
file_upload_handlers.insert(0, 'aws.s3.upload.S3FileUploadHandler')
FILE_UPLOAD_HANDLERS = tuple(file_upload_handlers)
############################################################
# Necessary for South to not fuzz with tests. Fixed in South 0.7.1
SKIP_SOUTH_TESTS = True
# Set up environment variable so Kerberos libraries look at our private
# ticket cache
os.environ['KRB5CCNAME'] = desktop.conf.KERBEROS.CCACHE_PATH.get()
# If Hue is configured to use a CACERTS truststore, make sure that the
# REQUESTS_CA_BUNDLE is set so that we can use it when we make external requests.
import sys
import logging
import aws
from aws.conf import is_enabled as is_s3_enabled
from desktop.lib.fs import ProxyFS
from hadoop import cluster
FS_CACHE = {}
DEFAULT_SCHEMA = 'hdfs'
FS_GETTERS = {
"hdfs": cluster.get_hdfs,
"s3a": aws.get_s3fs if is_s3_enabled() else None
}
def get_filesystem(name='default'):
"""
Return the filesystem with the given name.
If the filesystem is not defined, raises KeyError
"""
if name not in FS_CACHE:
FS_CACHE[name] = _make_fs(name)
return FS_CACHE[name]
def _make_fs(name):
fs_dict = {}
def listdir_paged(request, path):
"""
A paginated version of listdir.
Query parameters:
pagenum - The page number to show. Defaults to 1.
pagesize - How many to show on a page. Defaults to 15.
sortby=? - Specify attribute to sort by. Accepts:
(type, name, atime, mtime, size, user, group)
Defaults to name.
descending - Specify a descending sort order.
Default to false.
filter=? - Specify a substring filter to search for in
the filename field.
"""
if not request.fs.isdir(path):
raise PopupException("Not a directory: %s" % (path,))
pagenum = int(request.GET.get('pagenum', 1))
pagesize = int(request.GET.get('pagesize', 30))
do_as = None
if request.user.is_superuser or request.user.has_hue_permission(action="impersonate", app="security"):
do_as = request.GET.get('doas', request.user.username)
if hasattr(request, 'doas'):
do_as = request.doas
home_dir_path = request.user.get_home_directory()
breadcrumbs = parse_breadcrumbs(path)
if do_as:
all_stats = request.fs.do_as_user(do_as, request.fs.listdir_stats, path)
else:
all_stats = request.fs.listdir_stats(path)
# Filter first
filter_str = request.GET.get('filter', None)
if filter_str:
filtered_stats = filter(lambda sb: filter_str in sb['name'], all_stats)
all_stats = filtered_stats
# Sort next
sortby = request.GET.get('sortby', None)
descending_param = request.GET.get('descending', None)
if sortby is not None:
if sortby not in ('type', 'name', 'atime', 'mtime', 'user', 'group', 'size'):
logger.info("Invalid sort attribute '%s' for listdir." %
(sortby,))
else:
all_stats = sorted(all_stats,
key=operator.attrgetter(sortby),
reverse=coerce_bool(descending_param))
# Do pagination
page = paginator.Paginator(all_stats, pagesize).page(pagenum)
shown_stats = page.object_list
# Include parent dir always as second option, unless at filesystem root.
if not request.fs.isroot(path):
parent_path = request.fs.parent_path(path)
parent_stat = request.fs.stats(parent_path)
# The 'path' field would be absolute, but we want its basename to be
# actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
parent_stat['path'] = parent_path
parent_stat['name'] = ".."
shown_stats.insert(0, parent_stat)
# Include same dir always as first option to see stats of the current folder
current_stat = request.fs.stats(path)
# The 'path' field would be absolute, but we want its basename to be
# actually '.' for display purposes. Encode it since _massage_stats expects byte strings.
current_stat['path'] = path
current_stat['name'] = "."
shown_stats.insert(1, current_stat)
page.object_list = [ _massage_stats(request, s) for s in shown_stats ]
is_fs_superuser = _is_hdfs_superuser(request)
data = {
'path': path,
's3_path': 's3://',
'breadcrumbs': breadcrumbs,
'is_s3_enabled': is_s3_enabled(),
'current_request_path': request.path,
'files': page.object_list,
'page': _massage_page(page),
'pagesize': pagesize,
'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None,
'sortby': sortby,
'descending': descending_param,
# The following should probably be deprecated
'cwd_set': True,
'file_filter': 'any',
'current_dir_path': path,
'is_fs_superuser': is_fs_superuser,
'groups': is_fs_superuser and [str(x) for x in Group.objects.values_list('name', flat=True)] or [],
'users': is_fs_superuser and [str(x) for x in User.objects.values_list('username', flat=True)] or [],
'superuser': request.fs.superuser,
'supergroup': request.fs.supergroup,
'is_sentry_managed': request.fs.is_sentry_managed(path),
'apps': appmanager.get_apps_dict(request.user).keys(),
'show_download_button': SHOW_DOWNLOAD_BUTTON.get(),
'show_upload_button': SHOW_UPLOAD_BUTTON.get()
}
return render('listdir.mako', request, data)
