def check_config(request): """Check config and view for the list of errors""" if not request.user.is_superuser: return HttpResponse(_("You must be a superuser.")) conf_dir = os.path.realpath(get_desktop_root('conf')) return render('check_config.mako', request, dict( error_list=_get_config_errors(cache=False), conf_dir=conf_dir))
def _upload_extract_archive_script_to_hdfs(fs): if not fs.exists('/user/' + DEFAULT_USER.get() + '/common/'): fs.do_as_user(DEFAULT_USER.get(), fs.mkdir, '/user/' + DEFAULT_USER.get() + '/common/') fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0755) if not fs.do_as_user(DEFAULT_USER.get(), fs.exists, '/user/' + DEFAULT_USER.get() + '/common/extract_archive_in_hdfs.sh'): fs.do_as_user(DEFAULT_USER.get(), fs.copyFromLocal, get_desktop_root() + '/core/src/desktop/lib/tasks/extract_archive/extract_in_hdfs.sh', '/user/' + DEFAULT_USER.get() + '/common/extract_archive_in_hdfs.sh') fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0755)
def check_config(request): """Check config and view for the list of errors""" if not request.user.is_superuser: return HttpResponse(_("You must be a superuser.")) conf_dir = os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))) return render('check_config.mako', request, { 'error_list': _get_config_errors(request, cache=False), 'conf_dir': conf_dir }, force_template=True)
def _configs_from_dir(conf_dir): """ Generator to load configurations from a directory. This will only load files that end in .ini """ for filename in sorted(os.listdir(conf_dir)): if filename.startswith(".") or not filename.endswith('.ini'): continue logging.debug("Loading configuration from: %s" % filename) conf = configobj.ConfigObj(os.path.join(conf_dir, filename)) conf['DEFAULT'] = dict(desktop_root=get_desktop_root(), build_dir=get_build_dir()) yield conf
def check_config(request): """Check config and view for the list of errors""" if not request.user.is_superuser: return HttpResponse(_("You must be a superuser.")) context = { "conf_dir": os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))), "error_list": _get_config_errors(request, cache=False), } if request.GET.get("format") == "json": return JsonResponse(context) else: return render("check_config.mako", request, context, force_template=True)
def load_confs(conf_source=None): """Loads and merges all of the configurations passed in, returning a ConfigObj for the result. @param conf_source if not specified, reads conf/ from desktop/conf/. Otherwise should be a generator of ConfigObjs """ if conf_source is None: conf_source = _configs_from_dir(get_desktop_root("conf")) conf = configobj.ConfigObj() for in_conf in conf_source: conf.merge(in_conf) return conf
def dump_config(request): # Note that this requires login (as do most apps). show_private = False conf_dir = os.path.realpath(get_desktop_root('conf')) if not request.user.is_superuser: return HttpResponse(_("You must be a superuser.")) if request.GET.get("private"): show_private = True return render("dump_config.mako", request, dict( show_private=show_private, top_level=desktop.lib.conf.GLOBAL_CONFIG, conf_dir=conf_dir, apps=appmanager.DESKTOP_MODULES))
def _configs_from_dir(conf_dir): """ Generator to load configurations from a directory. This will only load files that end in .ini """ for filename in sorted(os.listdir(conf_dir)): if filename.startswith(".") or not filename.endswith(".ini"): continue LOG.debug("Loading configuration from: %s" % filename) try: conf = configobj.ConfigObj(os.path.join(conf_dir, filename)) except configobj.ConfigObjError, ex: LOG.error("Error in configuration file '%s': %s" % (os.path.join(conf_dir, filename), ex)) raise conf["DEFAULT"] = dict(desktop_root=get_desktop_root(), build_dir=get_build_dir()) yield conf
def check_config(request): """Check config and view for the list of errors""" if not request.user.is_superuser: return HttpResponse(_("You must be a superuser.")) # double request to this view # first is page container, second is error check made with ajax show_error = False error_list = conf_dir = None if request.is_ajax(): show_error = True conf_dir = os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))) error_list = _get_config_errors(request.user, cache=False) return render('check_config.mako', request, { 'error_list': error_list, 'conf_dir': conf_dir, 'show_error': show_error },force_template=True)
def _upload_extract_archive_script_to_hdfs(fs): if not fs.exists('/user/' + DEFAULT_USER.get() + '/common/'): fs.do_as_user(DEFAULT_USER.get(), fs.mkdir, '/user/' + DEFAULT_USER.get() + '/common/') fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0o755) if not fs.do_as_user( DEFAULT_USER.get(), fs.exists, '/user/' + DEFAULT_USER.get() + '/common/extract_archive_in_hdfs.sh'): fs.do_as_user( DEFAULT_USER.get(), fs.copyFromLocal, get_desktop_root() + '/core/src/desktop/lib/tasks/extract_archive/extract_in_hdfs.sh', '/user/' + DEFAULT_USER.get() + '/common/extract_archive_in_hdfs.sh') fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0o755)
def _upload_compress_files_script_to_hdfs(fs): if not fs.exists('/user/' + DEFAULT_USER.get() + '/common/'): fs.do_as_user(DEFAULT_USER.get(), fs.mkdir, '/user/' + DEFAULT_USER.get() + '/common/') fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0755) if not fs.do_as_user( DEFAULT_USER.get(), fs.exists, '/user/' + DEFAULT_USER.get() + '/common/compress_files_in_hdfs.sh'): fs.do_as_user( DEFAULT_USER.get(), fs.copyFromLocal, get_desktop_root() + '/core/src/desktop/lib/tasks/compress_files/compress_in_hdfs.sh', '/user/' + DEFAULT_USER.get() + '/common/compress_files_in_hdfs.sh') fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0755)
def _configs_from_dir(conf_dir): """ Generator to load configurations from a directory. This will only load files that end in .ini """ for filename in sorted(os.listdir(conf_dir)): if filename.startswith(".") or not filename.endswith('.ini'): continue LOG.debug("Loading configuration from: %s" % filename) try: conf = configobj.ConfigObj(os.path.join(conf_dir, filename)) except configobj.ConfigObjError, ex: LOG.error("Error in configuration file '%s': %s" % (os.path.join(conf_dir, filename), ex)) raise conf['DEFAULT'] = dict(desktop_root=get_desktop_root(), build_dir=get_build_dir()) yield conf
def dump_config(request): # Note that this requires login (as do most apps). show_private = False conf_dir = os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))) if not request.user.is_superuser: return HttpResponse(_("You must be a superuser.")) if request.GET.get("private"): show_private = True apps = sorted(appmanager.DESKTOP_MODULES, key=lambda app: app.name) apps_names = [app.name for app in apps] top_level = sorted(GLOBAL_CONFIG.get().values(), key=lambda obj: apps_names.index(obj.config.key)) return render( "dump_config.mako", request, dict(show_private=show_private, top_level=top_level, conf_dir=conf_dir, apps=apps) )
def handle(self, *args, **options): required = ("mapping_file",) for r in required: if not options.get(r): raise CommandError(_("--%(param)s is required.") % {'param': r}) # Pull out all the mappings mapping_file = options["mapping_file"] mapping_handle = open(mapping_file, 'r') mappings = [] for mapping in mapping_handle: map_parts = mapping.strip().lstrip('s/') map_parts = map_parts.rstrip('/') map_parts = map_parts.split('/') if len(map_parts) != 2: raise CommandError(_("Invalid mapping %(mapping)s in %(file)s.") % {'mapping': mapping.strip(), 'file': mapping_file}) mappings.append(map_parts) config_dir = os.getenv("HUE_CONF_DIR", get_desktop_root("conf")) for conf_file in glob.glob(os.path.join(config_dir, '*.ini')): LOG.info("Upgrading %s" % conf_file) conf_handle = open(conf_file, 'r') data = [] for line in conf_handle: # Pull apart any variables so we don't overwrite config settings data.append(line.split('=', 1)) # Iterate over mappings to perform for line in data: for mapping in mappings: old_value = mapping[0] new_value = mapping[1] if old_value in line[0]: LOG.info("Replacing %s with %s in line %s" % (old_value, new_value, '='.join(line),)) line[0] = line[0].replace(old_value, new_value) # Rewrite file with replacements made conf_handle.close() conf_handle = open(conf_file, 'w') data_to_write = ''.join([ '='.join(split) for split in data ]) conf_handle.write(data_to_write)
def check_config(request): """Check config and view for the list of errors""" if not is_admin(request.user): return HttpResponse(_("You must be a superuser.")) context = { 'conf_dir': os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))), 'error_list': _get_config_errors(request, cache=False), } if request.GET.get('format') == 'json': return JsonResponse(context) else: return render('check_config.mako', request, context, force_template=True)
def dump_config(request): # Note that this requires login (as do most apps). show_private = False conf_dir = os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))) if not request.user.is_superuser: return HttpResponse(_("You must be a superuser.")) if request.GET.get("private"): show_private = True apps = sorted(appmanager.DESKTOP_MODULES, key=lambda app: app.name) apps_names = [app.name for app in apps] top_level = sorted(GLOBAL_CONFIG.get().values(), key=lambda obj: apps_names.index(obj.config.key)) return render("dump_config.mako", request, dict( show_private=show_private, top_level=top_level, conf_dir=conf_dir, apps=apps))
def check_config(request): """Check config and view for the list of errors""" if not request.user.is_superuser: return HttpResponse(_("You must be a superuser.")) # double request to this view # first is page container, second is error check made with ajax show_error = False error_list = conf_dir = None if request.is_ajax(): show_error = True conf_dir = os.path.realpath( os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))) error_list = _get_config_errors(request.user, cache=False) return render('check_config.mako', request, { 'error_list': error_list, 'conf_dir': conf_dir, 'show_error': show_error }, force_template=True)
def _read_log_conf(proc_name, log_dir): """ _read_log_conf(proc_name, log_dir) -> StringIO or None This method also replaces the %LOG_DIR% and %PROC_NAME% occurrences. """ def _repl(match): if match.group(0) == '%LOG_DIR%': return log_dir elif match.group(0) == '%PROC_NAME%': return proc_name log_conf = get_desktop_root('conf', 'log.conf') if not os.path.isfile(log_conf): return None try: raw = file(log_conf).read() sio = StringIO(CONF_RE.sub(_repl, raw)) return sio except IOError, ex: print >> sys.stderr, "ERROR: Failed to open %s: %s" % (log_conf, ex) return None
def dump_config(request): show_private = False conf_dir = os.path.realpath( os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))) if not is_admin(request.user): return HttpResponse(_("You must be a superuser.")) if request.GET.get("private"): show_private = True app_modules = appmanager.DESKTOP_MODULES config_modules = GLOBAL_CONFIG.get().values() if ENABLE_CONNECTORS.get(): app_modules = [ app_module for app_module in app_modules if app_module.name == 'desktop' ] config_modules = [ config_module for config_module in config_modules if config_module.config.key == 'desktop' ] apps = sorted(app_modules, key=lambda app: app.name) apps_names = [app.name for app in apps] top_level = sorted(config_modules, key=lambda obj: apps_names.index(obj.config.key)) return render( "dump_config.mako", request, { 'show_private': show_private, 'top_level': top_level, 'conf_dir': conf_dir, 'is_embeddable': request.GET.get('is_embeddable', False), 'apps': apps })
import getpass import logging import string import base64 import stat from os import path, getenv, chmod from desktop.lib import encryptor from desktop.lib.paths import get_desktop_root import hashlib from desktop.conf import KREDENTIALS_DIR LOG = logging.getLogger(__name__) HUE_MASTER_FNAME = "hue-master" HUE_CONF_DIR = config_dir = getenv("HUE_CONF_DIR", get_desktop_root("conf")) HARDCODED_SECRET = string.ascii_letters[:32] class Obfuscator(object): __HUE_MASTER_PASSWORD = None def set_master_password(self): if self.__HUE_MASTER_PASSWORD is None: hue_master_path = path.join(KREDENTIALS_DIR.get(), HUE_MASTER_FNAME) try: with open(hue_master_path, "r") as f: cyphertext = f.readline() self.__HUE_MASTER_PASSWORD = encryptor.DecryptWithAES( HARDCODED_SECRET, cyphertext, encryptor.STATIC_IV) except IOError:
type=int, default=1 * 60 * 60) CONFIG_WHITELIST = Config( key='config_whitelist', default= 'debug_action,explain_level,mem_limit,optimize_partition_key_scans,query_timeout_s,request_pool', type=coerce_csv, help=_t( 'A comma-separated list of white-listed Impala configuration properties that users are authorized to set.' )) IMPALA_CONF_DIR = Config( key='impala_conf_dir', help=_t('Impala configuration directory, where impala_flags is located.'), default=os.environ.get("HUE_CONF_DIR", get_desktop_root("conf")) + '/impala-conf') SSL = ConfigSection( key='ssl', help=_t('SSL configuration for the server.'), members=dict( ENABLED=Config(key="enabled", help=_t("SSL communication enabled for this server."), type=coerce_bool, default=False), CACERTS=Config( key="cacerts", help=_t("Path to Certificate Authority certificates."), type=str, dynamic_default=default_ssl_cacerts,
import getpass import logging import string import base64 import stat from os import path, getenv, chmod from desktop.lib import encryptor from desktop.lib.paths import get_desktop_root import hashlib from desktop.conf import KREDENTIALS_DIR LOG = logging.getLogger(__name__) HUE_MASTER_FNAME = "hue-master" HUE_CONF_DIR = config_dir = getenv("HUE_CONF_DIR", get_desktop_root("conf")) HARDCODED_SECRET = string.ascii_letters[:32] class Obfuscator(object): __HUE_MASTER_PASSWORD = None def set_master_password(self): if self.__HUE_MASTER_PASSWORD is None: hue_master_path = path.join(KREDENTIALS_DIR.get(), HUE_MASTER_FNAME) try: with open(hue_master_path, "r") as f: cyphertext = f.readline() self.__HUE_MASTER_PASSWORD = encryptor.DecryptWithAES(HARDCODED_SECRET, cyphertext, encryptor.STATIC_IV) except IOError: self.__HUE_MASTER_PASSWORD = getpass.getpass("Enter a hue master password:\n")
'django.middleware.http.ConditionalGetMiddleware', 'axes.middleware.FailedLoginMiddleware', ] # if os.environ.get(ENV_DESKTOP_DEBUG): # MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware') # logging.debug("Will try to validate generated HTML.") ROOT_URLCONF = 'desktop.urls' # Hue runs its own wsgi applications WSGI_APPLICATION = None TEMPLATE_DIRS = ( get_desktop_root("core/templates"), ) INSTALLED_APPS = [ 'django.contrib.auth', 'django_openid_auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.staticfiles', 'django.contrib.admin', 'django_extensions', # 'debug_toolbar', 'south', # database migration tool
def handle_noargs(self, **options): cdh_path = os.path.join(get_desktop_root(), '..', 'cloudera', 'cdh_version.properties') if os.path.exists(cdh_path): print open(cdh_path).read() else: print settings.HUE_DESKTOP_VERSION
# See the License for the specific language governing permissions and # limitations under the License. import os import logging import re import sys import traceback import pkg_resources import desktop import desktop.lib.apputil from desktop.lib.paths import get_desktop_root # Directories where apps and libraries are to be found APP_DIRS = [get_desktop_root('core-apps'), get_desktop_root('apps'), get_desktop_root('libs')] LOG = logging.getLogger(__name__) ###################################################################### # Global variables set after calling load_apps() ###################################################################### # List of DesktopModuleInfo that have been loaded and skipped BROKEN_APPS = None DESKTOP_LIBS = None DESKTOP_APPS = None DESKTOP_MODULES = [ ] # Sum of APPS and LIBS
For more info, see http://docs.djangoproject.com/en/1.1/ref/settings/#database-engine""" ), members=dict( ENGINE=Config( key='engine', help= _('Database engine, such as postgresql_psycopg2, mysql, or sqlite3.' ), type=coerce_database, default='django.db.backends.sqlite3', ), NAME=Config( key='name', help=_('Database name, or path to DB if using sqlite3.'), type=str, default=get_desktop_root('desktop.db'), ), USER=Config( key='user', help=_('Database username.'), type=str, default='', ), PASSWORD=Config( key='password', help=_('Database password.'), type=str, default='', ), HOST=Config( key='host',
# FIXME: we can switch back to the django version when we upgrade to 1.5+ #'django.middleware.http.ConditionalGetMiddleware', 'desktop.middleware.ConditionalGetMiddleware', ] if os.environ.get(ENV_DESKTOP_DEBUG): MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware') logging.debug("Will try to validate generated HTML.") ROOT_URLCONF = 'desktop.urls' # Hue runs its own wsgi applications WSGI_APPLICATION = None TEMPLATE_DIRS = (get_desktop_root("core/templates")) INSTALLED_APPS = [ 'django.contrib.auth', 'django_openid_auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.admin', 'django_extensions', # 'debug_toolbar', 'south', # database migration tool # i18n support 'babeldjango',
# 'debug_toolbar.middleware.DebugToolbarMiddleware' "django.middleware.csrf.CsrfViewMiddleware", "django.middleware.http.ConditionalGetMiddleware", "axes.middleware.FailedLoginMiddleware", ] if os.environ.get(ENV_DESKTOP_DEBUG): MIDDLEWARE_CLASSES.append("desktop.middleware.HtmlValidationMiddleware") logging.debug("Will try to validate generated HTML.") ROOT_URLCONF = "desktop.urls" # Hue runs its own wsgi applications WSGI_APPLICATION = None TEMPLATE_DIRS = (get_desktop_root("core/templates"),) INSTALLED_APPS = [ "django.contrib.auth", "django_openid_auth", "django.contrib.contenttypes", "django.contrib.sessions", "django.contrib.sites", "django.contrib.staticfiles", "django.contrib.admin", "django_extensions", # 'debug_toolbar', "south", # database migration tool # i18n support "babeldjango", # Desktop injects all the other installed apps into here magically.
import os import logging import re import sys import traceback import pkg_resources import desktop from desktop.lib.paths import get_desktop_root from django.utils.translation import ugettext as _ # Directories where apps and libraries are to be found APP_DIRS = [ get_desktop_root('core-apps'), get_desktop_root('apps'), get_desktop_root('libs') ] LOG = logging.getLogger(__name__) ###################################################################### # Global variables set after calling load_apps() ###################################################################### # List of DesktopModuleInfo that have been loaded DESKTOP_LIBS = None DESKTOP_APPS = None DESKTOP_MODULES = [] # Sum of APPS and LIBS
"desktop.middleware.NotificationMiddleware", "desktop.middleware.JFrameMiddleware", "desktop.middleware.ExceptionMiddleware", "desktop.middleware.ClusterMiddleware", "desktop.middleware.AppSpecificMiddleware", "django.middleware.transaction.TransactionMiddleware" # 'debug_toolbar.middleware.DebugToolbarMiddleware' ] if os.environ.get(ENV_DESKTOP_DEBUG): MIDDLEWARE_CLASSES.append("desktop.middleware.HtmlValidationMiddleware") logging.debug("Will try to validate generated HTML.") ROOT_URLCONF = "desktop.urls" TEMPLATE_DIRS = get_desktop_root("core/templates") INSTALLED_APPS = [ "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", "django.contrib.sites", "django.contrib.admin", "django_extensions", # 'debug_toolbar', "south", # database migration tool # i18n support "babeldjango", # Desktop injects all the other installed apps into here magically. "desktop", ]
'django.middleware.http.ConditionalGetMiddleware', #'axes.middleware.FailedLoginMiddleware', 'desktop.middleware.MimeTypeJSFileFixStreamingMiddleware', 'crequest.middleware.CrequestMiddleware', ] # if os.environ.get(ENV_DESKTOP_DEBUG): # MIDDLEWARE.append('desktop.middleware.HtmlValidationMiddleware') # logging.debug("Will try to validate generated HTML.") ROOT_URLCONF = 'desktop.urls' # Hue runs its own wsgi applications WSGI_APPLICATION = None GTEMPLATE_DIRS = (get_desktop_root("core/templates"), ) INSTALLED_APPS = [ 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.staticfiles', 'django_extensions', # 'debug_toolbar', #'south', # database migration tool # i18n support 'django_babel',
# FIXME: we can switch back to the django version when we upgrade to 1.5+ #'django.middleware.http.ConditionalGetMiddleware', 'desktop.middleware.ConditionalGetMiddleware', ] if os.environ.get(ENV_DESKTOP_DEBUG): MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware') logging.debug("Will try to validate generated HTML.") ROOT_URLCONF = 'desktop.urls' # Hue runs its own wsgi applications WSGI_APPLICATION = None TEMPLATE_DIRS = ( get_desktop_root("core/templates") ) INSTALLED_APPS = [ 'django.contrib.auth', 'django_openid_auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.admin', 'django_extensions', # 'debug_toolbar', 'south', # database migration tool
'django.contrib.auth.middleware.AuthenticationMiddleware', 'desktop.middleware.AjaxMiddleware', # Must be after Session, Auth, and Ajax. Before everything else. 'desktop.middleware.LoginAndPermissionMiddleware', 'desktop.middleware.FlashMessageMiddleware', 'desktop.middleware.JFrameMiddleware', 'desktop.middleware.ExceptionMiddleware', 'desktop.middleware.ClusterMiddleware', 'desktop.middleware.AppSpecificMiddleware', # 'debug_toolbar.middleware.DebugToolbarMiddleware' ] ROOT_URLCONF = 'desktop.urls' TEMPLATE_DIRS = (get_desktop_root("core/templates")) INSTALLED_APPS = [ 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.admin', 'django_extensions', # 'debug_toolbar', 'south', # database migration tool # JavaScript depency loading 'depender',
'django.middleware.csrf.CsrfViewMiddleware', 'django.middleware.http.ConditionalGetMiddleware', ] if os.environ.get(ENV_DESKTOP_DEBUG): MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware') logging.debug("Will try to validate generated HTML.") ROOT_URLCONF = 'desktop.urls' # Hue runs its own wsgi applications WSGI_APPLICATION = None TEMPLATE_DIRS = ( get_desktop_root("core/templates"), ) INSTALLED_APPS = [ 'django.contrib.auth', 'django_openid_auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.staticfiles', 'django.contrib.admin', 'django_extensions', # 'debug_toolbar', 'south', # database migration tool
), ) DATABASE = ConfigSection( key="database", help="""Configuration options for specifying the Desktop Database. For more info, see http://docs.djangoproject.com/en/1.1/ref/settings/#database-engine""", members=dict( ENGINE=Config( key="engine", help="Database engine, eg postgresql, mysql, sqlite3, or oracle", type=str, default="sqlite3" ), NAME=Config( key="name", help="Database name, or path to DB if using sqlite3", type=str, default=get_desktop_root("desktop.db"), ), USER=Config(key="user", help="Database username", type=str, default=""), PASSWORD=Config(key="password", help="Database password", type=str, default=""), HOST=Config(key="host", help="Database host", type=str, default=""), PORT=Config(key="port", help="Database port", type=int, default=0), ), ) KERBEROS = ConfigSection( key="kerberos", help="""Configuration options for specifying Hue's kerberos integration for secured Hadoop clusters.""", members=dict( HUE_KEYTAB=Config( key="hue_keytab",
def default_catalog_config_dir(): """Get from usual main Hue config directory""" return os.environ.get("HUE_CONF_DIR", get_desktop_root("conf")) + '/hive-conf'
'desktop.middleware.AjaxMiddleware', # Must be after Session, Auth, and Ajax. Before everything else. 'desktop.middleware.LoginAndPermissionMiddleware', 'desktop.middleware.FlashMessageMiddleware', 'desktop.middleware.JFrameMiddleware', 'desktop.middleware.ExceptionMiddleware', 'desktop.middleware.ClusterMiddleware', 'desktop.middleware.AppSpecificMiddleware', # 'debug_toolbar.middleware.DebugToolbarMiddleware' ] ROOT_URLCONF = 'desktop.urls' TEMPLATE_DIRS = ( get_desktop_root("core/templates") ) INSTALLED_APPS = [ 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.admin', 'django_extensions', # 'debug_toolbar', 'south', # database migration tool # JavaScript depency loading
#'axes.middleware.FailedLoginMiddleware', 'desktop.middleware.MimeTypeJSFileFixStreamingMiddleware', 'crequest.middleware.CrequestMiddleware', ] # if os.environ.get(ENV_DESKTOP_DEBUG): # MIDDLEWARE.append('desktop.middleware.HtmlValidationMiddleware') # logging.debug("Will try to validate generated HTML.") ROOT_URLCONF = 'desktop.urls' # Hue runs its own wsgi applications WSGI_APPLICATION = None GTEMPLATE_DIRS = ( get_desktop_root("core/templates"), ) INSTALLED_APPS = [ 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.staticfiles', 'django.contrib.admin', 'django_extensions', # 'debug_toolbar', #'south', # database migration tool
'desktop.middleware.AppSpecificMiddleware', 'django.middleware.transaction.TransactionMiddleware' # 'debug_toolbar.middleware.DebugToolbarMiddleware' ] if os.environ.get(ENV_DESKTOP_DEBUG): MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware') logging.debug("Will try to validate generated HTML.") ROOT_URLCONF = 'desktop.urls' # Hue runs its own wsgi applications WSGI_APPLICATION = None TEMPLATE_DIRS = ( get_desktop_root("core/templates") ) INSTALLED_APPS = [ 'django.contrib.auth', 'django_openid_auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.admin', 'django_extensions', # 'debug_toolbar', 'south', # database migration tool
#!/usr/bin/env python # Licensed to Cloudera, Inc. under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. Cloudera, Inc. licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from desktop.lib.paths import get_desktop_root DJANGO_APPS = ['filebrowser'] NICE_NAME = "File Browser" REQUIRES_HADOOP = False DEPENDER_SCRIPTS_JSON = [ ("fancyupload", get_desktop_root("libs/fancyupload/static/js/Source/scripts.json")), ] DEPENDER_PACKAGE_YMLS = [ "src/filebrowser/static/js/package.yml", ]
def handle(self, *args, **options): cdh_path = os.path.join(get_desktop_root(), '..', 'cloudera', 'cdh_version.properties') if os.path.exists(cdh_path): print open(cdh_path).read() else: print settings.HUE_DESKTOP_VERSION
'desktop.middleware.NotificationMiddleware', 'desktop.middleware.JFrameMiddleware', 'desktop.middleware.ExceptionMiddleware', 'desktop.middleware.ClusterMiddleware', 'desktop.middleware.AppSpecificMiddleware', 'django.middleware.transaction.TransactionMiddleware' # 'debug_toolbar.middleware.DebugToolbarMiddleware' ] if os.environ.get(ENV_DESKTOP_DEBUG): MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware') logging.debug("Will try to validate generated HTML.") ROOT_URLCONF = 'desktop.urls' TEMPLATE_DIRS = (get_desktop_root("core/templates")) INSTALLED_APPS = [ 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.admin', 'django_extensions', # 'debug_toolbar', 'south', # database migration tool # i18n support 'babeldjango',
# limitations under the License. import os import logging import re import sys import traceback import pkg_resources import desktop from desktop.lib.paths import get_desktop_root from django.utils.translation import ugettext as _ # Directories where apps and libraries are to be found APP_DIRS = [get_desktop_root('core-apps'), get_desktop_root('apps'), get_desktop_root('libs')] LOG = logging.getLogger(__name__) ###################################################################### # Global variables set after calling load_apps() ###################################################################### # List of DesktopModuleInfo that have been loaded DESKTOP_LIBS = None DESKTOP_APPS = None DESKTOP_MODULES = [ ] # Sum of APPS and LIBS def _import_module_or_none(module):
" (compute or send back results) for that session within SESSION_TIMEOUT_S seconds. Default: 15 min."), type=int, default=15 * 60 ) CONFIG_WHITELIST = Config( key='config_whitelist', default='debug_action,explain_level,mem_limit,optimize_partition_key_scans,query_timeout_s,request_pool', type=coerce_csv, help=_t('A comma-separated list of white-listed Impala configuration properties that users are authorized to set.') ) IMPALA_CONF_DIR = Config( key='impala_conf_dir', help=_t('Impala configuration directory, where impala_flags is located.'), default=os.environ.get("HUE_CONF_DIR", get_desktop_root("conf")) + '/impala-conf' ) SSL = ConfigSection( key='ssl', help=_t('SSL configuration for the server.'), members=dict( ENABLED = Config( key="enabled", help=_t("SSL communication enabled for this server."), type=coerce_bool, default=False ), CACERTS = Config( key="cacerts",
DATABASE = ConfigSection( key='database', help=_("""Configuration options for specifying the Desktop Database. For more info, see http://docs.djangoproject.com/en/1.1/ref/settings/#database-engine"""), members=dict( ENGINE=Config( key='engine', help=_('Database engine, such as postgresql_psycopg2, mysql, or sqlite3.'), type=str, default='sqlite3', ), NAME=Config( key='name', help=_('Database name, or path to DB if using sqlite3.'), type=str, default=get_desktop_root('desktop.db'), ), USER=Config( key='user', help=_('Database username.'), type=str, default='', ), PASSWORD=Config( key='password', help=_('Database password.'), type=str, default='', ), HOST=Config( key='host',
class Submission(object): """ Represents one unique Oozie submission. Actions are: - submit - rerun """ def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None, local_tz=None): self.job = job self.user = user self.fs = fs self.jt = jt # Deprecated with YARN, we now use logical names only for RM self.oozie_id = oozie_id self.api = get_oozie(self.user) if properties is not None: self.properties = properties else: self.properties = {} if local_tz and isinstance(self.job.data, dict): local_tz = self.job.data.get('properties')['timezone'] # Modify start_date & end_date only when it's a coordinator from oozie.models2 import Coordinator if type(self.job) is Coordinator: if 'start_date' in self.properties: properties['start_date'] = convert_to_server_timezone( self.properties['start_date'], local_tz) if 'end_date' in self.properties: properties['end_date'] = convert_to_server_timezone( self.properties['end_date'], local_tz) if 'nominal_time' in self.properties: properties['nominal_time'] = convert_to_server_timezone( self.properties['nominal_time'], local_tz) self.properties['security_enabled'] = self.api.security_enabled def __str__(self): if self.oozie_id: res = "Submission for job '%s'." % (self.oozie_id, ) else: res = "Submission for job '%s' (id %s, owner %s)." % ( self.job.name, self.job.id, self.user) if self.oozie_id: res += " -- " + self.oozie_id return res @submit_dryrun def run(self, deployment_dir=None): """ Take care of all the actions of submitting a Oozie workflow. Returns the oozie job id if all goes well. """ if self.properties and 'oozie.use.system.libpath' not in self.properties: self.properties['oozie.use.system.libpath'] = 'true' self.oozie_id = self.api.submit_job(self.properties) LOG.info("Submitted: %s" % (self, )) if self._is_workflow(): self.api.job_control(self.oozie_id, 'start') LOG.info("Started: %s" % (self, )) return self.oozie_id def rerun(self, deployment_dir, fail_nodes=None, skip_nodes=None): jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties(jt_address, deployment_dir) self.properties.update({'oozie.wf.application.path': deployment_dir}) if 'oozie.coord.application.path' in self.properties: self.properties.pop('oozie.coord.application.path') if 'oozie.bundle.application.path' in self.properties: self.properties.pop('oozie.bundle.application.path') if fail_nodes: self.properties.update({'oozie.wf.rerun.failnodes': fail_nodes}) elif not skip_nodes: self.properties.update({'oozie.wf.rerun.failnodes': 'false'}) # Case empty 'skip_nodes' list else: self.properties.update({'oozie.wf.rerun.skip.nodes': skip_nodes}) self.api.rerun(self.oozie_id, properties=self.properties) LOG.info("Rerun: %s" % (self, )) return self.oozie_id def rerun_coord(self, deployment_dir, params): jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties(jt_address, deployment_dir) self.properties.update( {'oozie.coord.application.path': deployment_dir}) self.api.job_control(self.oozie_id, action='coord-rerun', properties=self.properties, parameters=params) LOG.info("Rerun: %s" % (self, )) return self.oozie_id def update_coord(self): self.api = get_oozie(self.user, api_version="v2") self.api.job_control(self.oozie_id, action='update', properties=self.properties, parameters=None) LOG.info("Update: %s" % (self, )) return self.oozie_id def rerun_bundle(self, deployment_dir, params): jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties(jt_address, deployment_dir) self.properties.update( {'oozie.bundle.application.path': deployment_dir}) self.api.job_control(self.oozie_id, action='bundle-rerun', properties=self.properties, parameters=params) LOG.info("Rerun: %s" % (self, )) return self.oozie_id def deploy(self, deployment_dir=None): try: if not deployment_dir: deployment_dir = self._create_deployment_dir() except Exception, ex: msg = _("Failed to create deployment directory: %s" % ex) LOG.exception(msg) raise PopupException(message=msg, detail=str(ex)) if self.api.security_enabled: jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties( jt_address ) # Needed for coordinator deploying workflows with credentials if hasattr(self.job, 'nodes'): for action in self.job.nodes: # Make sure XML is there # Don't support more than one level sub-workflow if action.data['type'] == 'subworkflow': from oozie.models2 import Workflow workflow = Workflow(document=Document2.objects.get_by_uuid( user=self.user, uuid=action.data['properties']['workflow'])) sub_deploy = Submission(self.user, workflow, self.fs, self.jt, self.properties) workspace = sub_deploy.deploy() self.job.override_subworkflow_id( action, workflow.id) # For displaying the correct graph self.properties[ 'workspace_%s' % workflow. uuid] = workspace # For pointing to the correct workspace elif action.data['type'] == 'altus': service = 'dataeng' # action.data['properties'].get('script_path') auth_key_id = ALTUS.AUTH_KEY_ID.get() auth_key_secret = ALTUS.AUTH_KEY_SECRET.get().replace( '\\n', '\n') shell_script = self._generate_altus_action_script( service=service, auth_key_id=auth_key_id, auth_key_secret=auth_key_secret) self._create_file(deployment_dir, action.data['name'] + '.py', shell_script) self.fs.do_as_user( self.user, self.fs.copyFromLocal, os.path.join(get_desktop_root(), 'core', 'ext-py', 'navoptapi-0.1.0'), self.job.deployment_dir) elif action.data['type'] == 'impala' or action.data[ 'type'] == 'impala-document': from oozie.models2 import _get_impala_url from impala.impala_flags import get_ssl_server_certificate if action.data['type'] == 'impala-document': from notebook.models import Notebook if action.data['properties'].get('uuid'): notebook = Notebook( document=Document2.objects.get_by_uuid( user=self.user, uuid=action.data['properties']['uuid'])) statements = notebook.get_str() statements = Template(statements).safe_substitute( **self.properties) script_name = action.data['name'] + '.sql' self._create_file(deployment_dir, script_name, statements) else: script_name = os.path.basename( action.data['properties'].get('script_path')) if self.api.security_enabled: kinit = 'kinit -k -t *.keytab %(user_principal)s' % { 'user_principal': self.properties.get( 'user_principal', action.data['properties']. get('user_principal')) } else: kinit = '' shell_script = """#!/bin/bash # Needed to launch impala shell in oozie export PYTHON_EGG_CACHE=./myeggs %(kinit)s impala-shell %(kerberos_option)s %(ssl_option)s -i %(impalad_host)s -f %(query_file)s""" % { 'impalad_host': action.data['properties'].get('impalad_host') or _get_impala_url(), 'kerberos_option': '-k' if self.api.security_enabled else '', 'ssl_option': '--ssl' if get_ssl_server_certificate() else '', 'query_file': script_name, 'kinit': kinit } self._create_file(deployment_dir, action.data['name'] + '.sh', shell_script) elif action.data['type'] == 'hive-document': from notebook.models import Notebook if action.data['properties'].get('uuid'): notebook = Notebook( document=Document2.objects.get_by_uuid( user=self.user, uuid=action.data['properties']['uuid'])) statements = notebook.get_str() else: statements = action.data['properties'].get( 'statements') if self.properties.get('send_result_path'): statements = """ INSERT OVERWRITE DIRECTORY '%s' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' WITH SERDEPROPERTIES ( "separatorChar" = "\t", "quoteChar" = "'", "escapeChar" = "\\" ) STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.join([ snippet['statement_raw'] for snippet in notebook.get_data()['snippets'] ])) if statements is not None: self._create_file(deployment_dir, action.data['name'] + '.sql', statements) elif action.data['type'] in ('java-document', 'java', 'mapreduce-document'): if action.data['type'] == 'java-document' or action.data[ 'type'] == 'mapreduce-document': from notebook.models import Notebook notebook = Notebook( document=Document2.objects.get_by_uuid( user=self.user, uuid=action.data['properties']['uuid'])) properties = notebook.get_data( )['snippets'][0]['properties'] else: properties = action.data['properties'] if properties.get('app_jar'): LOG.debug("Adding to oozie.libpath %s" % properties['app_jar']) paths = [properties['app_jar']] if self.properties.get('oozie.libpath'): paths.append(self.properties['oozie.libpath']) self.properties['oozie.libpath'] = ','.join(paths) elif action.data['type'] == 'pig-document': from notebook.models import Notebook notebook = Notebook(document=Document2.objects.get_by_uuid( user=self.user, uuid=action.data['properties'] ['uuid'])) statements = notebook.get_data( )['snippets'][0]['statement_raw'] self._create_file(deployment_dir, action.data['name'] + '.pig', statements) elif action.data['type'] == 'spark' or action.data[ 'type'] == 'spark-document': if not [ f for f in action.data.get('properties').get( 'files', []) if f.get('value').endswith('hive-site.xml') ]: hive_site_lib = Hdfs.join(deployment_dir + '/lib/', 'hive-site.xml') hive_site_content = get_hive_site_content() if not self.fs.do_as_user( self.user, self.fs.exists, hive_site_lib) and hive_site_content: self.fs.do_as_user( self.user, self.fs.create, hive_site_lib, overwrite=True, permission=0700, data=smart_str(hive_site_content)) oozie_xml = self.job.to_xml(self.properties) self._do_as(self.user.username, self._copy_files, deployment_dir, oozie_xml, self.properties) return deployment_dir
'django.middleware.http.ConditionalGetMiddleware', #@TODO@ Prakash to check FailedLoginMiddleware working or not? #'axes.middleware.FailedLoginMiddleware', 'desktop.middleware.MimeTypeJSFileFixStreamingMiddleware', ] # if os.environ.get(ENV_DESKTOP_DEBUG): # MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware') # logging.debug("Will try to validate generated HTML.") ROOT_URLCONF = 'desktop.urls' # Hue runs its own wsgi applications WSGI_APPLICATION = None GTEMPLATE_DIRS = (get_desktop_root("core/templates"), ) INSTALLED_APPS = [ 'django.contrib.auth', 'django_openid_auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.staticfiles', 'django.contrib.admin', 'django_extensions', # 'debug_toolbar', #'south', # database migration tool # i18n support
def load_confs(configspecpath): conf_source = _configs_from_dir(get_desktop_root("conf")) conf = ConfigObj(configspec=configspecpath) for in_conf in conf_source: conf.merge(in_conf) return conf