def hue(request): apps = appmanager.get_apps_dict(request.user) current_app, other_apps, apps_list = _get_apps(request.user, '') default_cluster_index, default_cluster_interface = Cluster(request.user).get_list_interface_indexes() clusters = get_clusters().values() return render('hue.mako', request, { 'apps': apps, 'other_apps': other_apps, 'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user), 'is_adls_enabled': is_adls_enabled() and has_adls_access(request.user), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(), 'leaflet': { 'layer': desktop.conf.LEAFLET_TILE_LAYER.get(), 'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get(), 'map_options': json.dumps(desktop.conf.LEAFLET_MAP_OPTIONS.get()), 'layer_options': json.dumps(desktop.conf.LEAFLET_TILE_LAYER_OPTIONS.get()), }, 'is_demo': desktop.conf.DEMO_ENABLED.get(), 'banner_message': get_banner_message(request), 'user_preferences': dict((x.key, x.value) for x in UserPreferences.objects.filter(user=request.user)), 'cluster': clusters[0]['type'] if clusters else None, 'clusters_config_json': json.dumps(clusters), 'default_cluster_index': default_cluster_index, 'default_cluster_interface': default_cluster_interface })
def config_validator(user): ''' v2 When using the connectors, now 'hive' is seen as a dialect and only the list of connections (instance of the 'hive' connector, e.g. pointing to a Hive server in the Cloud) should be tested. Interpreters are now tested by the Editor in libs/notebook/conf.py. v1 All the configuration happens in apps/beeswax. ''' from beeswax.design import hql_query # dbms is dependent on beeswax.conf, import in method to avoid circular dependency from beeswax.server import dbms res = [] if has_connectors(): return res try: try: if not 'test' in sys.argv: # Avoid tests hanging server = dbms.get(user) query = hql_query("SELECT 'Hello World!';") handle = server.execute_and_wait(query, timeout_sec=10.0) if handle: server.fetch(handle, rows=100) server.close(handle) except StructuredThriftTransportException as e: if 'Error validating the login' in str(e): msg = 'Failed to authenticate to HiveServer2, check authentication configurations.' LOG.exception(msg) res.append((NICE_NAME, _(msg))) else: raise e except Exception as e: msg = "The application won't work without a running HiveServer2." LOG.exception(msg) res.append((NICE_NAME, _(msg))) try: from desktop.lib.fsmanager import get_filesystem from aws.conf import is_enabled as is_s3_enabled warehouse = beeswax.hive_site.get_metastore_warehouse_dir() fs = get_filesystem() fs_scheme = fs._get_scheme(warehouse) if fs: if fs_scheme == 's3a': if is_s3_enabled(): fs.do_as_user(user, fs.stats, warehouse) else: LOG.warn("Warehouse is in S3, but no credential available.") else: fs.do_as_superuser(fs.stats, warehouse) except Exception: msg = 'Failed to access Hive warehouse: %s' LOG.exception(msg % warehouse) res.append((NICE_NAME, _(msg) % warehouse)) return res
def commonheader(title, section, user, request=None, padding="90px", skip_topbar=False, skip_idle_timeout=False, is_mobile=False): """ Returns the rendered common header """ current_app, other_apps, apps_list = _get_apps(user, section) template = 'common_header.mako' if is_mobile: template = 'common_header_m.mako' return django_mako.render_to_string(template, { 'current_app': current_app, 'apps': apps_list, 'other_apps': other_apps, 'title': title, 'section': section, 'padding': padding, 'user': user, 'request': request, 'skip_topbar': skip_topbar, 'skip_idle_timeout': skip_idle_timeout, 'leaflet': { 'layer': desktop.conf.LEAFLET_TILE_LAYER.get(), 'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get(), 'map_options': json.dumps(desktop.conf.LEAFLET_MAP_OPTIONS.get()), 'layer_options': json.dumps(desktop.conf.LEAFLET_TILE_LAYER_OPTIONS.get()), }, 'is_demo': desktop.conf.DEMO_ENABLED.get(), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(), 'is_s3_enabled': is_s3_enabled() and has_s3_access(user), 'is_adls_enabled': is_adls_enabled() and has_adls_access(request.user), 'banner_message': get_banner_message(request) })
def hue(request): apps = appmanager.get_apps_dict(request.user) return render( 'hue.mako', request, { 'apps': apps, 'interpreters': get_ordered_interpreters(request.user), 'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(), 'leaflet': { 'layer': desktop.conf.LEAFLET_TILE_LAYER.get(), 'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get() }, 'is_demo': desktop.conf.DEMO_ENABLED.get(), 'banner_message': get_banner_message(request), 'cluster_config': ClusterConfig(request.user), 'user_preferences': dict((x.key, x.value) for x in UserPreferences.objects.filter(user=request.user)) })
def hue(request): apps = appmanager.get_apps_dict(request.user) return render( 'hue.mako', request, { 'apps': apps, 'tours_and_tutorials': Settings.get_settings().tours_and_tutorials, 'interpreters': get_ordered_interpreters(request.user), 'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(), 'leaflet': { 'layer': desktop.conf.LEAFLET_TILE_LAYER.get(), 'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get() }, 'is_demo': desktop.conf.DEMO_ENABLED.get(), 'banner_message': get_banner_message(request), 'cluster_config': ClusterConfig(request.user) })
def is_enabled(fs=None): if fs == 'hdfs': return has_hdfs_enabled() elif fs == 'adl': return is_adls_enabled() elif fs == 's3a': return is_s3_enabled() elif fs == 'abfs': return is_abfs_enabled()
def global_js_constants(request): return HttpResponse(render('global_js_constants.mako', request, { 'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user), 'leaflet': { 'layer': desktop.conf.LEAFLET_TILE_LAYER.get(), 'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get(), 'map_options': json.dumps(desktop.conf.LEAFLET_MAP_OPTIONS.get()), 'layer_options': json.dumps(desktop.conf.LEAFLET_TILE_LAYER_OPTIONS.get()), } }), content_type="application/javascript")
def responsive(request): apps = appmanager.get_apps_dict(request.user) return render('responsive.mako', request, { 'apps': apps, 'tours_and_tutorials': Settings.get_settings().tours_and_tutorials, 'interpreters': get_ordered_interpreters(request.user), 'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get() })
def responsive(request): apps = appmanager.get_apps_dict(request.user) return render( 'responsive.mako', request, { 'apps': apps, 'tours_and_tutorials': Settings.get_settings().tours_and_tutorials, 'interpreters': get_interpreters(request.user), 'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user) })
def commonheader(title, section, user, request=None, padding="90px", skip_topbar=False, skip_idle_timeout=False, is_mobile=False): """ Returns the rendered common header """ current_app = None other_apps = [] if user.is_authenticated(): apps = appmanager.get_apps(user) apps_list = appmanager.get_apps_dict(user) for app in apps: if app.display_name not in [ 'beeswax', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser', 'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark', 'indexer', 'security', 'notebook'] and app.menu_index != -1: other_apps.append(app) if section == app.display_name: current_app = app else: apps_list = [] template = 'common_header.mako' if is_mobile: template = 'common_header_m.mako' return django_mako.render_to_string(template, { 'current_app': current_app, 'apps': apps_list, 'other_apps': other_apps, 'title': title, 'section': section, 'padding': padding, 'user': user, 'request': request, 'skip_topbar': skip_topbar, 'skip_idle_timeout': skip_idle_timeout, 'leaflet': { 'layer': desktop.conf.LEAFLET_TILE_LAYER.get(), 'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get() }, 'is_demo': desktop.conf.DEMO_ENABLED.get(), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(), 'is_s3_enabled': is_s3_enabled() and has_s3_access(user), 'banner_message': get_banner_message(request) })
def commonheader(title, section, user, request=None, padding="90px", skip_topbar=False, skip_idle_timeout=False, is_mobile=False): """ Returns the rendered common header """ current_app = None other_apps = [] if user.is_authenticated(): apps = appmanager.get_apps(user) apps_list = appmanager.get_apps_dict(user) for app in apps: if app.display_name not in [ 'beeswax', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser', 'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark', 'indexer', 'security', 'notebook'] and app.menu_index != -1: other_apps.append(app) if section == app.display_name: current_app = app else: apps_list = [] template = 'common_header.mako' if is_mobile: template = 'common_header_m.mako' return django_mako.render_to_string(template, { 'current_app': current_app, 'apps': apps_list, 'other_apps': other_apps, 'title': title, 'section': section, 'padding': padding, 'user': user, 'request': request, 'skip_topbar': skip_topbar, 'skip_idle_timeout': skip_idle_timeout, 'leaflet': { 'layer': desktop.conf.LEAFLET_TILE_LAYER.get(), 'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get() }, 'is_demo': desktop.conf.DEMO_ENABLED.get(), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(), 'is_s3_enabled': is_s3_enabled() and has_s3_access(user) })
LOAD_BALANCER_COOKIE = 'ROUTEID' ################################################################ # Register file upload handlers # This section must go after the desktop lib modules are loaded ################################################################ # Insert our custom upload handlers file_upload_handlers = [ 'hadoop.fs.upload.HDFSfileUploadHandler', 'django.core.files.uploadhandler.MemoryFileUploadHandler', 'django.core.files.uploadhandler.TemporaryFileUploadHandler', ] if is_s3_enabled(): file_upload_handlers.insert(0, 'aws.s3.upload.S3FileUploadHandler') FILE_UPLOAD_HANDLERS = tuple(file_upload_handlers) ############################################################ # Necessary for South to not fuzz with tests. Fixed in South 0.7.1 SKIP_SOUTH_TESTS = True # Set up environment variable so Kerberos libraries look at our private # ticket cache os.environ['KRB5CCNAME'] = desktop.conf.KERBEROS.CCACHE_PATH.get() if not os.getenv('SERVER_SOFTWARE'): os.environ['SERVER_SOFTWARE'] = 'apache'
import sys import logging import aws from aws.conf import is_enabled as is_s3_enabled from desktop.lib.fs import ProxyFS from hadoop import cluster FS_CACHE = {} DEFAULT_SCHEMA = 'hdfs' FS_GETTERS = { "hdfs": cluster.get_hdfs, "s3a": aws.get_s3fs if is_s3_enabled() else None } def get_filesystem(name='default'): """ Return the filesystem with the given name. If the filesystem is not defined, raises KeyError """ if name not in FS_CACHE: FS_CACHE[name] = _make_fs(name) return FS_CACHE[name] def _make_fs(name): fs_dict = {}
if 'useradmin' in [app.name for app in appmanager.DESKTOP_APPS]: MIDDLEWARE_CLASSES.append('useradmin.middleware.LastActivityMiddleware') ################################################################ # Register file upload handlers # This section must go after the desktop lib modules are loaded ################################################################ # Insert our custom upload handlers file_upload_handlers = [ 'hadoop.fs.upload.HDFSfileUploadHandler', 'django.core.files.uploadhandler.MemoryFileUploadHandler', 'django.core.files.uploadhandler.TemporaryFileUploadHandler', ] if is_s3_enabled(): file_upload_handlers.insert(0, 'aws.s3.upload.S3FileUploadHandler') FILE_UPLOAD_HANDLERS = tuple(file_upload_handlers) ############################################################ # Necessary for South to not fuzz with tests. Fixed in South 0.7.1 SKIP_SOUTH_TESTS = True # Set up environment variable so Kerberos libraries look at our private # ticket cache os.environ['KRB5CCNAME'] = desktop.conf.KERBEROS.CCACHE_PATH.get() # If Hue is configured to use a CACERTS truststore, make sure that the # REQUESTS_CA_BUNDLE is set so that we can use it when we make external requests.
def listdir_paged(request, path): """ A paginated version of listdir. Query parameters: pagenum - The page number to show. Defaults to 1. pagesize - How many to show on a page. Defaults to 15. sortby=? - Specify attribute to sort by. Accepts: (type, name, atime, mtime, size, user, group) Defaults to name. descending - Specify a descending sort order. Default to false. filter=? - Specify a substring filter to search for in the filename field. """ if not request.fs.isdir(path): raise PopupException("Not a directory: %s" % (path,)) pagenum = int(request.GET.get('pagenum', 1)) pagesize = int(request.GET.get('pagesize', 30)) do_as = None if request.user.is_superuser or request.user.has_hue_permission(action="impersonate", app="security"): do_as = request.GET.get('doas', request.user.username) if hasattr(request, 'doas'): do_as = request.doas home_dir_path = request.user.get_home_directory() breadcrumbs = parse_breadcrumbs(path) if do_as: all_stats = request.fs.do_as_user(do_as, request.fs.listdir_stats, path) else: all_stats = request.fs.listdir_stats(path) # Filter first filter_str = request.GET.get('filter', None) if filter_str: filtered_stats = filter(lambda sb: filter_str in sb['name'], all_stats) all_stats = filtered_stats # Sort next sortby = request.GET.get('sortby', None) descending_param = request.GET.get('descending', None) if sortby is not None: if sortby not in ('type', 'name', 'atime', 'mtime', 'user', 'group', 'size'): logger.info("Invalid sort attribute '%s' for listdir." % (sortby,)) else: all_stats = sorted(all_stats, key=operator.attrgetter(sortby), reverse=coerce_bool(descending_param)) # Do pagination page = paginator.Paginator(all_stats, pagesize).page(pagenum) shown_stats = page.object_list # Include parent dir always as second option, unless at filesystem root. if not request.fs.isroot(path): parent_path = request.fs.parent_path(path) parent_stat = request.fs.stats(parent_path) # The 'path' field would be absolute, but we want its basename to be # actually '..' for display purposes. Encode it since _massage_stats expects byte strings. parent_stat['path'] = parent_path parent_stat['name'] = ".." shown_stats.insert(0, parent_stat) # Include same dir always as first option to see stats of the current folder current_stat = request.fs.stats(path) # The 'path' field would be absolute, but we want its basename to be # actually '.' for display purposes. Encode it since _massage_stats expects byte strings. current_stat['path'] = path current_stat['name'] = "." shown_stats.insert(1, current_stat) page.object_list = [ _massage_stats(request, s) for s in shown_stats ] is_fs_superuser = _is_hdfs_superuser(request) data = { 'path': path, 's3_path': 's3://', 'breadcrumbs': breadcrumbs, 'is_s3_enabled': is_s3_enabled(), 'current_request_path': request.path, 'files': page.object_list, 'page': _massage_page(page), 'pagesize': pagesize, 'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None, 'sortby': sortby, 'descending': descending_param, # The following should probably be deprecated 'cwd_set': True, 'file_filter': 'any', 'current_dir_path': path, 'is_fs_superuser': is_fs_superuser, 'groups': is_fs_superuser and [str(x) for x in Group.objects.values_list('name', flat=True)] or [], 'users': is_fs_superuser and [str(x) for x in User.objects.values_list('username', flat=True)] or [], 'superuser': request.fs.superuser, 'supergroup': request.fs.supergroup, 'is_sentry_managed': request.fs.is_sentry_managed(path), 'apps': appmanager.get_apps_dict(request.user).keys(), 'show_download_button': SHOW_DOWNLOAD_BUTTON.get(), 'show_upload_button': SHOW_UPLOAD_BUTTON.get() } return render('listdir.mako', request, data)