Пример #1
0
def check_config(request):
  """Check config and view for the list of errors"""
  if not request.user.is_superuser:
    return HttpResponse(_("You must be a superuser."))
  conf_dir = os.path.realpath(get_desktop_root('conf'))
  return render('check_config.mako', request, dict(
                    error_list=_get_config_errors(cache=False),
                    conf_dir=conf_dir))
Пример #2
0
def _upload_extract_archive_script_to_hdfs(fs):
  if not fs.exists('/user/' + DEFAULT_USER.get() + '/common/'):
    fs.do_as_user(DEFAULT_USER.get(), fs.mkdir, '/user/' + DEFAULT_USER.get() + '/common/')
    fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0755)

  if not fs.do_as_user(DEFAULT_USER.get(), fs.exists, '/user/' + DEFAULT_USER.get() + '/common/extract_archive_in_hdfs.sh'):
    fs.do_as_user(DEFAULT_USER.get(), fs.copyFromLocal, get_desktop_root() + '/core/src/desktop/lib/tasks/extract_archive/extract_in_hdfs.sh',
                          '/user/' + DEFAULT_USER.get() + '/common/extract_archive_in_hdfs.sh')
    fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0755)
Пример #3
0
def check_config(request):
  """Check config and view for the list of errors"""
  if not request.user.is_superuser:
    return HttpResponse(_("You must be a superuser."))

  conf_dir = os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf")))
  return render('check_config.mako', request, {
                  'error_list': _get_config_errors(request, cache=False),
                  'conf_dir': conf_dir
              },
              force_template=True)
Пример #4
0
Файл: conf.py Проект: atm/hue
def _configs_from_dir(conf_dir):
  """
  Generator to load configurations from a directory. This will
  only load files that end in .ini
  """
  for filename in sorted(os.listdir(conf_dir)):
    if filename.startswith(".") or not filename.endswith('.ini'):
      continue
    logging.debug("Loading configuration from: %s" % filename)
    conf = configobj.ConfigObj(os.path.join(conf_dir, filename))
    conf['DEFAULT'] = dict(desktop_root=get_desktop_root(), build_dir=get_build_dir())
    yield conf
Пример #5
0
def check_config(request):
    """Check config and view for the list of errors"""
    if not request.user.is_superuser:
        return HttpResponse(_("You must be a superuser."))

    context = {
        "conf_dir": os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))),
        "error_list": _get_config_errors(request, cache=False),
    }

    if request.GET.get("format") == "json":
        return JsonResponse(context)
    else:
        return render("check_config.mako", request, context, force_template=True)
Пример #6
0
Файл: conf.py Проект: pauldb/hue
def load_confs(conf_source=None):
    """Loads and merges all of the configurations passed in,
  returning a ConfigObj for the result.

  @param conf_source if not specified, reads conf/ from
                     desktop/conf/. Otherwise should be a generator
                     of ConfigObjs
  """
    if conf_source is None:
        conf_source = _configs_from_dir(get_desktop_root("conf"))

    conf = configobj.ConfigObj()
    for in_conf in conf_source:
        conf.merge(in_conf)
    return conf
Пример #7
0
def dump_config(request):
  # Note that this requires login (as do most apps).
  show_private = False
  conf_dir = os.path.realpath(get_desktop_root('conf'))

  if not request.user.is_superuser:
    return HttpResponse(_("You must be a superuser."))

  if request.GET.get("private"):
    show_private = True

  return render("dump_config.mako", request, dict(
    show_private=show_private,
    top_level=desktop.lib.conf.GLOBAL_CONFIG,
    conf_dir=conf_dir,
    apps=appmanager.DESKTOP_MODULES))
Пример #8
0
Файл: conf.py Проект: pauldb/hue
def _configs_from_dir(conf_dir):
    """
  Generator to load configurations from a directory. This will
  only load files that end in .ini
  """
    for filename in sorted(os.listdir(conf_dir)):
        if filename.startswith(".") or not filename.endswith(".ini"):
            continue
        LOG.debug("Loading configuration from: %s" % filename)
        try:
            conf = configobj.ConfigObj(os.path.join(conf_dir, filename))
        except configobj.ConfigObjError, ex:
            LOG.error("Error in configuration file '%s': %s" % (os.path.join(conf_dir, filename), ex))
            raise
        conf["DEFAULT"] = dict(desktop_root=get_desktop_root(), build_dir=get_build_dir())
        yield conf
Пример #9
0
def check_config(request):
  """Check config and view for the list of errors"""
  if not request.user.is_superuser:
    return HttpResponse(_("You must be a superuser."))
  # double request to this view
  # first is page container, second is error check made with ajax
  show_error = False
  error_list = conf_dir = None
  if request.is_ajax():
    show_error = True
    conf_dir = os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf")))
    error_list = _get_config_errors(request.user, cache=False)
  return render('check_config.mako', request, {
                  'error_list': error_list,
                  'conf_dir': conf_dir,
                  'show_error': show_error
              },force_template=True)
Пример #10
0
def _upload_extract_archive_script_to_hdfs(fs):
    if not fs.exists('/user/' + DEFAULT_USER.get() + '/common/'):
        fs.do_as_user(DEFAULT_USER.get(), fs.mkdir,
                      '/user/' + DEFAULT_USER.get() + '/common/')
        fs.do_as_user(DEFAULT_USER.get(), fs.chmod,
                      '/user/' + DEFAULT_USER.get() + '/common/', 0o755)

    if not fs.do_as_user(
            DEFAULT_USER.get(), fs.exists, '/user/' + DEFAULT_USER.get() +
            '/common/extract_archive_in_hdfs.sh'):
        fs.do_as_user(
            DEFAULT_USER.get(), fs.copyFromLocal,
            get_desktop_root() +
            '/core/src/desktop/lib/tasks/extract_archive/extract_in_hdfs.sh',
            '/user/' + DEFAULT_USER.get() +
            '/common/extract_archive_in_hdfs.sh')
        fs.do_as_user(DEFAULT_USER.get(), fs.chmod,
                      '/user/' + DEFAULT_USER.get() + '/common/', 0o755)
Пример #11
0
def _upload_compress_files_script_to_hdfs(fs):
    if not fs.exists('/user/' + DEFAULT_USER.get() + '/common/'):
        fs.do_as_user(DEFAULT_USER.get(), fs.mkdir,
                      '/user/' + DEFAULT_USER.get() + '/common/')
        fs.do_as_user(DEFAULT_USER.get(), fs.chmod,
                      '/user/' + DEFAULT_USER.get() + '/common/', 0755)

    if not fs.do_as_user(
            DEFAULT_USER.get(), fs.exists, '/user/' + DEFAULT_USER.get() +
            '/common/compress_files_in_hdfs.sh'):
        fs.do_as_user(
            DEFAULT_USER.get(), fs.copyFromLocal,
            get_desktop_root() +
            '/core/src/desktop/lib/tasks/compress_files/compress_in_hdfs.sh',
            '/user/' + DEFAULT_USER.get() +
            '/common/compress_files_in_hdfs.sh')
        fs.do_as_user(DEFAULT_USER.get(), fs.chmod,
                      '/user/' + DEFAULT_USER.get() + '/common/', 0755)
Пример #12
0
def _configs_from_dir(conf_dir):
    """
  Generator to load configurations from a directory. This will
  only load files that end in .ini
  """
    for filename in sorted(os.listdir(conf_dir)):
        if filename.startswith(".") or not filename.endswith('.ini'):
            continue
        LOG.debug("Loading configuration from: %s" % filename)
        try:
            conf = configobj.ConfigObj(os.path.join(conf_dir, filename))
        except configobj.ConfigObjError, ex:
            LOG.error("Error in configuration file '%s': %s" %
                      (os.path.join(conf_dir, filename), ex))
            raise
        conf['DEFAULT'] = dict(desktop_root=get_desktop_root(),
                               build_dir=get_build_dir())
        yield conf
Пример #13
0
def dump_config(request):
    # Note that this requires login (as do most apps).
    show_private = False
    conf_dir = os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf")))

    if not request.user.is_superuser:
        return HttpResponse(_("You must be a superuser."))

    if request.GET.get("private"):
        show_private = True

    apps = sorted(appmanager.DESKTOP_MODULES, key=lambda app: app.name)
    apps_names = [app.name for app in apps]
    top_level = sorted(GLOBAL_CONFIG.get().values(), key=lambda obj: apps_names.index(obj.config.key))

    return render(
        "dump_config.mako", request, dict(show_private=show_private, top_level=top_level, conf_dir=conf_dir, apps=apps)
    )
Пример #14
0
  def handle(self, *args, **options):
    required = ("mapping_file",)
    for r in required:
      if not options.get(r):
        raise CommandError(_("--%(param)s is required.") % {'param': r})
  
    # Pull out all the mappings  
    mapping_file = options["mapping_file"]
    mapping_handle = open(mapping_file, 'r')
    mappings = []
    for mapping in mapping_handle:
      map_parts = mapping.strip().lstrip('s/')
      map_parts = map_parts.rstrip('/')
      map_parts = map_parts.split('/')
      if len(map_parts) != 2:
        raise CommandError(_("Invalid mapping %(mapping)s in %(file)s.") % {'mapping': mapping.strip(), 'file': mapping_file})
      mappings.append(map_parts)

    config_dir = os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))
    for conf_file in glob.glob(os.path.join(config_dir, '*.ini')):
      LOG.info("Upgrading %s" % conf_file)
      conf_handle = open(conf_file, 'r')
      data = []
      for line in conf_handle:
        # Pull apart any variables so we don't overwrite config settings
        data.append(line.split('=', 1))

      # Iterate over mappings to perform
      for line in data:
          for mapping in mappings:
            old_value = mapping[0]
            new_value = mapping[1]

            if old_value in line[0]:
              LOG.info("Replacing %s with %s in line %s" % 
                  (old_value, new_value, '='.join(line),)) 
              line[0] = line[0].replace(old_value, new_value)
            
      
      # Rewrite file with replacements made
      conf_handle.close()
      conf_handle = open(conf_file, 'w')
      data_to_write = ''.join([ '='.join(split) for split in data ])
      conf_handle.write(data_to_write)
Пример #15
0
  def handle(self, *args, **options):
    required = ("mapping_file",)
    for r in required:
      if not options.get(r):
        raise CommandError(_("--%(param)s is required.") % {'param': r})
  
    # Pull out all the mappings  
    mapping_file = options["mapping_file"]
    mapping_handle = open(mapping_file, 'r')
    mappings = []
    for mapping in mapping_handle:
      map_parts = mapping.strip().lstrip('s/')
      map_parts = map_parts.rstrip('/')
      map_parts = map_parts.split('/')
      if len(map_parts) != 2:
        raise CommandError(_("Invalid mapping %(mapping)s in %(file)s.") % {'mapping': mapping.strip(), 'file': mapping_file})
      mappings.append(map_parts)

    config_dir = os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))
    for conf_file in glob.glob(os.path.join(config_dir, '*.ini')):
      LOG.info("Upgrading %s" % conf_file)
      conf_handle = open(conf_file, 'r')
      data = []
      for line in conf_handle:
        # Pull apart any variables so we don't overwrite config settings
        data.append(line.split('=', 1))

      # Iterate over mappings to perform
      for line in data:
          for mapping in mappings:
            old_value = mapping[0]
            new_value = mapping[1]

            if old_value in line[0]:
              LOG.info("Replacing %s with %s in line %s" % 
                  (old_value, new_value, '='.join(line),)) 
              line[0] = line[0].replace(old_value, new_value)
            
      
      # Rewrite file with replacements made
      conf_handle.close()
      conf_handle = open(conf_file, 'w')
      data_to_write = ''.join([ '='.join(split) for split in data ])
      conf_handle.write(data_to_write)
Пример #16
0
def check_config(request):
    """Check config and view for the list of errors"""
    if not is_admin(request.user):
        return HttpResponse(_("You must be a superuser."))

    context = {
        'conf_dir':
        os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf"))),
        'error_list':
        _get_config_errors(request, cache=False),
    }

    if request.GET.get('format') == 'json':
        return JsonResponse(context)
    else:
        return render('check_config.mako',
                      request,
                      context,
                      force_template=True)
Пример #17
0
def dump_config(request):
  # Note that this requires login (as do most apps).
  show_private = False
  conf_dir = os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf")))

  if not request.user.is_superuser:
    return HttpResponse(_("You must be a superuser."))

  if request.GET.get("private"):
    show_private = True

  apps = sorted(appmanager.DESKTOP_MODULES, key=lambda app: app.name)
  apps_names = [app.name for app in apps]
  top_level = sorted(GLOBAL_CONFIG.get().values(), key=lambda obj: apps_names.index(obj.config.key))

  return render("dump_config.mako", request, dict(
    show_private=show_private,
    top_level=top_level,
    conf_dir=conf_dir,
    apps=apps))
Пример #18
0
def check_config(request):
    """Check config and view for the list of errors"""
    if not request.user.is_superuser:
        return HttpResponse(_("You must be a superuser."))
    # double request to this view
    # first is page container, second is error check made with ajax
    show_error = False
    error_list = conf_dir = None
    if request.is_ajax():
        show_error = True
        conf_dir = os.path.realpath(
            os.getenv("HUE_CONF_DIR", get_desktop_root("conf")))
        error_list = _get_config_errors(request.user, cache=False)
    return render('check_config.mako',
                  request, {
                      'error_list': error_list,
                      'conf_dir': conf_dir,
                      'show_error': show_error
                  },
                  force_template=True)
Пример #19
0
def _read_log_conf(proc_name, log_dir):
    """
  _read_log_conf(proc_name, log_dir) -> StringIO or None

  This method also replaces the %LOG_DIR% and %PROC_NAME% occurrences.
  """
    def _repl(match):
        if match.group(0) == '%LOG_DIR%':
            return log_dir
        elif match.group(0) == '%PROC_NAME%':
            return proc_name

    log_conf = get_desktop_root('conf', 'log.conf')
    if not os.path.isfile(log_conf):
        return None

    try:
        raw = file(log_conf).read()
        sio = StringIO(CONF_RE.sub(_repl, raw))
        return sio
    except IOError, ex:
        print >> sys.stderr, "ERROR: Failed to open %s: %s" % (log_conf, ex)
        return None
Пример #20
0
def _read_log_conf(proc_name, log_dir):
  """
  _read_log_conf(proc_name, log_dir) -> StringIO or None

  This method also replaces the %LOG_DIR% and %PROC_NAME% occurrences.
  """
  def _repl(match):
    if match.group(0) == '%LOG_DIR%':
      return log_dir
    elif match.group(0) == '%PROC_NAME%':
      return proc_name

  log_conf = get_desktop_root('conf', 'log.conf')
  if not os.path.isfile(log_conf):
    return None

  try:
    raw = file(log_conf).read()
    sio = StringIO(CONF_RE.sub(_repl, raw))
    return sio
  except IOError, ex:
    print >> sys.stderr, "ERROR: Failed to open %s: %s" % (log_conf, ex)
    return None
Пример #21
0
def dump_config(request):
    show_private = False
    conf_dir = os.path.realpath(
        os.getenv("HUE_CONF_DIR", get_desktop_root("conf")))

    if not is_admin(request.user):
        return HttpResponse(_("You must be a superuser."))

    if request.GET.get("private"):
        show_private = True

    app_modules = appmanager.DESKTOP_MODULES
    config_modules = GLOBAL_CONFIG.get().values()
    if ENABLE_CONNECTORS.get():
        app_modules = [
            app_module for app_module in app_modules
            if app_module.name == 'desktop'
        ]
        config_modules = [
            config_module for config_module in config_modules
            if config_module.config.key == 'desktop'
        ]

    apps = sorted(app_modules, key=lambda app: app.name)
    apps_names = [app.name for app in apps]
    top_level = sorted(config_modules,
                       key=lambda obj: apps_names.index(obj.config.key))

    return render(
        "dump_config.mako", request, {
            'show_private': show_private,
            'top_level': top_level,
            'conf_dir': conf_dir,
            'is_embeddable': request.GET.get('is_embeddable', False),
            'apps': apps
        })
Пример #22
0
import getpass
import logging
import string
import base64
import stat
from os import path, getenv, chmod
from desktop.lib import encryptor
from desktop.lib.paths import get_desktop_root
import hashlib
from desktop.conf import KREDENTIALS_DIR

LOG = logging.getLogger(__name__)
HUE_MASTER_FNAME = "hue-master"
HUE_CONF_DIR = config_dir = getenv("HUE_CONF_DIR", get_desktop_root("conf"))

HARDCODED_SECRET = string.ascii_letters[:32]


class Obfuscator(object):
    __HUE_MASTER_PASSWORD = None

    def set_master_password(self):
        if self.__HUE_MASTER_PASSWORD is None:
            hue_master_path = path.join(KREDENTIALS_DIR.get(),
                                        HUE_MASTER_FNAME)
            try:
                with open(hue_master_path, "r") as f:
                    cyphertext = f.readline()
                    self.__HUE_MASTER_PASSWORD = encryptor.DecryptWithAES(
                        HARDCODED_SECRET, cyphertext, encryptor.STATIC_IV)
            except IOError:
Пример #23
0
    type=int,
    default=1 * 60 * 60)

CONFIG_WHITELIST = Config(
    key='config_whitelist',
    default=
    'debug_action,explain_level,mem_limit,optimize_partition_key_scans,query_timeout_s,request_pool',
    type=coerce_csv,
    help=_t(
        'A comma-separated list of white-listed Impala configuration properties that users are authorized to set.'
    ))

IMPALA_CONF_DIR = Config(
    key='impala_conf_dir',
    help=_t('Impala configuration directory, where impala_flags is located.'),
    default=os.environ.get("HUE_CONF_DIR", get_desktop_root("conf")) +
    '/impala-conf')

SSL = ConfigSection(
    key='ssl',
    help=_t('SSL configuration for the server.'),
    members=dict(
        ENABLED=Config(key="enabled",
                       help=_t("SSL communication enabled for this server."),
                       type=coerce_bool,
                       default=False),
        CACERTS=Config(
            key="cacerts",
            help=_t("Path to Certificate Authority certificates."),
            type=str,
            dynamic_default=default_ssl_cacerts,
Пример #24
0
import getpass
import logging
import string
import base64
import stat
from os import path, getenv, chmod
from desktop.lib import encryptor
from desktop.lib.paths import get_desktop_root
import hashlib
from desktop.conf import KREDENTIALS_DIR

LOG = logging.getLogger(__name__)
HUE_MASTER_FNAME = "hue-master"
HUE_CONF_DIR = config_dir = getenv("HUE_CONF_DIR", get_desktop_root("conf"))


HARDCODED_SECRET = string.ascii_letters[:32]


class Obfuscator(object):
  __HUE_MASTER_PASSWORD = None

  def set_master_password(self):
    if self.__HUE_MASTER_PASSWORD is None:
      hue_master_path = path.join(KREDENTIALS_DIR.get(), HUE_MASTER_FNAME)
      try:
        with open(hue_master_path, "r") as f:
          cyphertext = f.readline()
          self.__HUE_MASTER_PASSWORD = encryptor.DecryptWithAES(HARDCODED_SECRET, cyphertext, encryptor.STATIC_IV)
      except IOError:
        self.__HUE_MASTER_PASSWORD = getpass.getpass("Enter a hue master password:\n")
Пример #25
0
    'django.middleware.http.ConditionalGetMiddleware',
    'axes.middleware.FailedLoginMiddleware',
]

# if os.environ.get(ENV_DESKTOP_DEBUG):
#   MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware')
#   logging.debug("Will try to validate generated HTML.")

ROOT_URLCONF = 'desktop.urls'

# Hue runs its own wsgi applications
WSGI_APPLICATION = None

TEMPLATE_DIRS = (
    get_desktop_root("core/templates"),
)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django_openid_auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',

    'django.contrib.admin',
    'django_extensions',

    # 'debug_toolbar',
    'south', # database migration tool
Пример #26
0
 def handle_noargs(self, **options):
   cdh_path = os.path.join(get_desktop_root(), '..', 'cloudera', 'cdh_version.properties')
   if os.path.exists(cdh_path):
     print open(cdh_path).read()
   else:
     print settings.HUE_DESKTOP_VERSION
Пример #27
0
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import logging
import re
import sys
import traceback
import pkg_resources

import desktop
import desktop.lib.apputil
from desktop.lib.paths import get_desktop_root

# Directories where apps and libraries are to be found
APP_DIRS = [get_desktop_root('core-apps'),
            get_desktop_root('apps'),
            get_desktop_root('libs')]

LOG = logging.getLogger(__name__)

######################################################################
# Global variables set after calling load_apps()
######################################################################

# List of DesktopModuleInfo that have been loaded and skipped
BROKEN_APPS = None
DESKTOP_LIBS = None
DESKTOP_APPS = None
DESKTOP_MODULES = [ ]           # Sum of APPS and LIBS
Пример #28
0
       For more info, see http://docs.djangoproject.com/en/1.1/ref/settings/#database-engine"""
        ),
 members=dict(
     ENGINE=Config(
         key='engine',
         help=
         _('Database engine, such as postgresql_psycopg2, mysql, or sqlite3.'
           ),
         type=coerce_database,
         default='django.db.backends.sqlite3',
     ),
     NAME=Config(
         key='name',
         help=_('Database name, or path to DB if using sqlite3.'),
         type=str,
         default=get_desktop_root('desktop.db'),
     ),
     USER=Config(
         key='user',
         help=_('Database username.'),
         type=str,
         default='',
     ),
     PASSWORD=Config(
         key='password',
         help=_('Database password.'),
         type=str,
         default='',
     ),
     HOST=Config(
         key='host',
Пример #29
0
    # FIXME: we can switch back to the django version when we upgrade to 1.5+
    #'django.middleware.http.ConditionalGetMiddleware',
    'desktop.middleware.ConditionalGetMiddleware',
]

if os.environ.get(ENV_DESKTOP_DEBUG):
    MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware')
    logging.debug("Will try to validate generated HTML.")

ROOT_URLCONF = 'desktop.urls'

# Hue runs its own wsgi applications
WSGI_APPLICATION = None

TEMPLATE_DIRS = (get_desktop_root("core/templates"))

INSTALLED_APPS = [
    'django.contrib.auth',
    'django_openid_auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.admin',
    'django_extensions',

    # 'debug_toolbar',
    'south',  # database migration tool

    # i18n support
    'babeldjango',
Пример #30
0
    # 'debug_toolbar.middleware.DebugToolbarMiddleware'
    "django.middleware.csrf.CsrfViewMiddleware",
    "django.middleware.http.ConditionalGetMiddleware",
    "axes.middleware.FailedLoginMiddleware",
]

if os.environ.get(ENV_DESKTOP_DEBUG):
    MIDDLEWARE_CLASSES.append("desktop.middleware.HtmlValidationMiddleware")
    logging.debug("Will try to validate generated HTML.")

ROOT_URLCONF = "desktop.urls"

# Hue runs its own wsgi applications
WSGI_APPLICATION = None

TEMPLATE_DIRS = (get_desktop_root("core/templates"),)

INSTALLED_APPS = [
    "django.contrib.auth",
    "django_openid_auth",
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.sites",
    "django.contrib.staticfiles",
    "django.contrib.admin",
    "django_extensions",
    # 'debug_toolbar',
    "south",  # database migration tool
    # i18n support
    "babeldjango",
    # Desktop injects all the other installed apps into here magically.
Пример #31
0
import os
import logging
import re
import sys
import traceback
import pkg_resources

import desktop
from desktop.lib.paths import get_desktop_root

from django.utils.translation import ugettext as _

# Directories where apps and libraries are to be found
APP_DIRS = [
    get_desktop_root('core-apps'),
    get_desktop_root('apps'),
    get_desktop_root('libs')
]

LOG = logging.getLogger(__name__)

######################################################################
# Global variables set after calling load_apps()
######################################################################

# List of DesktopModuleInfo that have been loaded
DESKTOP_LIBS = None
DESKTOP_APPS = None
DESKTOP_MODULES = []  # Sum of APPS and LIBS
Пример #32
0
    "desktop.middleware.NotificationMiddleware",
    "desktop.middleware.JFrameMiddleware",
    "desktop.middleware.ExceptionMiddleware",
    "desktop.middleware.ClusterMiddleware",
    "desktop.middleware.AppSpecificMiddleware",
    "django.middleware.transaction.TransactionMiddleware"
    # 'debug_toolbar.middleware.DebugToolbarMiddleware'
]

if os.environ.get(ENV_DESKTOP_DEBUG):
    MIDDLEWARE_CLASSES.append("desktop.middleware.HtmlValidationMiddleware")
    logging.debug("Will try to validate generated HTML.")

ROOT_URLCONF = "desktop.urls"

TEMPLATE_DIRS = get_desktop_root("core/templates")

INSTALLED_APPS = [
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.sites",
    "django.contrib.admin",
    "django_extensions",
    # 'debug_toolbar',
    "south",  # database migration tool
    # i18n support
    "babeldjango",
    # Desktop injects all the other installed apps into here magically.
    "desktop",
]
Пример #33
0
    'django.middleware.http.ConditionalGetMiddleware',
    #'axes.middleware.FailedLoginMiddleware',
    'desktop.middleware.MimeTypeJSFileFixStreamingMiddleware',
    'crequest.middleware.CrequestMiddleware',
]

# if os.environ.get(ENV_DESKTOP_DEBUG):
#   MIDDLEWARE.append('desktop.middleware.HtmlValidationMiddleware')
#   logging.debug("Will try to validate generated HTML.")

ROOT_URLCONF = 'desktop.urls'

# Hue runs its own wsgi applications
WSGI_APPLICATION = None

GTEMPLATE_DIRS = (get_desktop_root("core/templates"), )

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',
    'django_extensions',

    # 'debug_toolbar',
    #'south', # database migration tool

    # i18n support
    'django_babel',
Пример #34
0
    # FIXME: we can switch back to the django version when we upgrade to 1.5+
    #'django.middleware.http.ConditionalGetMiddleware',
    'desktop.middleware.ConditionalGetMiddleware',
]

if os.environ.get(ENV_DESKTOP_DEBUG):
  MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware')
  logging.debug("Will try to validate generated HTML.")

ROOT_URLCONF = 'desktop.urls'

# Hue runs its own wsgi applications
WSGI_APPLICATION = None

TEMPLATE_DIRS = (
    get_desktop_root("core/templates")
)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django_openid_auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',

    'django.contrib.admin',
    'django_extensions',

    # 'debug_toolbar',
    'south', # database migration tool
Пример #35
0
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'desktop.middleware.AjaxMiddleware',
    # Must be after Session, Auth, and Ajax.  Before everything else.
    'desktop.middleware.LoginAndPermissionMiddleware',
    'desktop.middleware.FlashMessageMiddleware',
    'desktop.middleware.JFrameMiddleware',
    'desktop.middleware.ExceptionMiddleware',
    'desktop.middleware.ClusterMiddleware',
    'desktop.middleware.AppSpecificMiddleware',

    # 'debug_toolbar.middleware.DebugToolbarMiddleware'
]

ROOT_URLCONF = 'desktop.urls'

TEMPLATE_DIRS = (get_desktop_root("core/templates"))

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.admin',
    'django_extensions',

    # 'debug_toolbar',
    'south',  # database migration tool

    # JavaScript depency loading
    'depender',
Пример #36
0
    'django.middleware.csrf.CsrfViewMiddleware',

    'django.middleware.http.ConditionalGetMiddleware',
]

if os.environ.get(ENV_DESKTOP_DEBUG):
  MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware')
  logging.debug("Will try to validate generated HTML.")

ROOT_URLCONF = 'desktop.urls'

# Hue runs its own wsgi applications
WSGI_APPLICATION = None

TEMPLATE_DIRS = (
    get_desktop_root("core/templates"),
)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django_openid_auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',

    'django.contrib.admin',
    'django_extensions',

    # 'debug_toolbar',
    'south', # database migration tool
Пример #37
0
    ),
)

DATABASE = ConfigSection(
    key="database",
    help="""Configuration options for specifying the Desktop Database.
          For more info, see http://docs.djangoproject.com/en/1.1/ref/settings/#database-engine""",
    members=dict(
        ENGINE=Config(
            key="engine", help="Database engine, eg postgresql, mysql, sqlite3, or oracle", type=str, default="sqlite3"
        ),
        NAME=Config(
            key="name",
            help="Database name, or path to DB if using sqlite3",
            type=str,
            default=get_desktop_root("desktop.db"),
        ),
        USER=Config(key="user", help="Database username", type=str, default=""),
        PASSWORD=Config(key="password", help="Database password", type=str, default=""),
        HOST=Config(key="host", help="Database host", type=str, default=""),
        PORT=Config(key="port", help="Database port", type=int, default=0),
    ),
)

KERBEROS = ConfigSection(
    key="kerberos",
    help="""Configuration options for specifying Hue's kerberos integration for
          secured Hadoop clusters.""",
    members=dict(
        HUE_KEYTAB=Config(
            key="hue_keytab",
Пример #38
0
def default_catalog_config_dir():
    """Get from usual main Hue config directory"""
    return os.environ.get("HUE_CONF_DIR",
                          get_desktop_root("conf")) + '/hive-conf'
Пример #39
0
    'desktop.middleware.AjaxMiddleware',
    # Must be after Session, Auth, and Ajax.  Before everything else.
    'desktop.middleware.LoginAndPermissionMiddleware',
    'desktop.middleware.FlashMessageMiddleware',
    'desktop.middleware.JFrameMiddleware',
    'desktop.middleware.ExceptionMiddleware',
    'desktop.middleware.ClusterMiddleware',
    'desktop.middleware.AppSpecificMiddleware',

    # 'debug_toolbar.middleware.DebugToolbarMiddleware'
]

ROOT_URLCONF = 'desktop.urls'

TEMPLATE_DIRS = (
    get_desktop_root("core/templates")
)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',

    'django.contrib.admin',
    'django_extensions',

    # 'debug_toolbar',
    'south', # database migration tool

    # JavaScript depency loading
Пример #40
0
    'desktop.middleware.AjaxMiddleware',
    # Must be after Session, Auth, and Ajax.  Before everything else.
    'desktop.middleware.LoginAndPermissionMiddleware',
    'desktop.middleware.FlashMessageMiddleware',
    'desktop.middleware.JFrameMiddleware',
    'desktop.middleware.ExceptionMiddleware',
    'desktop.middleware.ClusterMiddleware',
    'desktop.middleware.AppSpecificMiddleware',

    # 'debug_toolbar.middleware.DebugToolbarMiddleware'
]

ROOT_URLCONF = 'desktop.urls'

TEMPLATE_DIRS = (
    get_desktop_root("core/templates")
)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',

    'django.contrib.admin',
    'django_extensions',

    # 'debug_toolbar',
    'south', # database migration tool

    # JavaScript depency loading
Пример #41
0
    #'axes.middleware.FailedLoginMiddleware',
    'desktop.middleware.MimeTypeJSFileFixStreamingMiddleware',
    'crequest.middleware.CrequestMiddleware',
]

# if os.environ.get(ENV_DESKTOP_DEBUG):
#   MIDDLEWARE.append('desktop.middleware.HtmlValidationMiddleware')
#   logging.debug("Will try to validate generated HTML.")

ROOT_URLCONF = 'desktop.urls'

# Hue runs its own wsgi applications
WSGI_APPLICATION = None

GTEMPLATE_DIRS = (
    get_desktop_root("core/templates"),
)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',

    'django.contrib.admin',
    'django_extensions',

    # 'debug_toolbar',
    #'south', # database migration tool
Пример #42
0
    'desktop.middleware.AppSpecificMiddleware',
    'django.middleware.transaction.TransactionMiddleware'
    # 'debug_toolbar.middleware.DebugToolbarMiddleware'
]

if os.environ.get(ENV_DESKTOP_DEBUG):
  MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware')
  logging.debug("Will try to validate generated HTML.")

ROOT_URLCONF = 'desktop.urls'

# Hue runs its own wsgi applications
WSGI_APPLICATION = None

TEMPLATE_DIRS = (
    get_desktop_root("core/templates")
)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django_openid_auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',

    'django.contrib.admin',
    'django_extensions',

    # 'debug_toolbar',
    'south', # database migration tool
Пример #43
0
#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from desktop.lib.paths import get_desktop_root

DJANGO_APPS = ['filebrowser']
NICE_NAME = "File Browser"
REQUIRES_HADOOP = False
DEPENDER_SCRIPTS_JSON = [
    ("fancyupload", get_desktop_root("libs/fancyupload/static/js/Source/scripts.json")),
]
DEPENDER_PACKAGE_YMLS = [
    "src/filebrowser/static/js/package.yml",
]
Пример #44
0
 def handle(self, *args, **options):
   cdh_path = os.path.join(get_desktop_root(), '..', 'cloudera', 'cdh_version.properties')
   if os.path.exists(cdh_path):
     print open(cdh_path).read()
   else:
     print settings.HUE_DESKTOP_VERSION
Пример #45
0
    'desktop.middleware.NotificationMiddleware',
    'desktop.middleware.JFrameMiddleware',
    'desktop.middleware.ExceptionMiddleware',
    'desktop.middleware.ClusterMiddleware',
    'desktop.middleware.AppSpecificMiddleware',
    'django.middleware.transaction.TransactionMiddleware'
    # 'debug_toolbar.middleware.DebugToolbarMiddleware'
]

if os.environ.get(ENV_DESKTOP_DEBUG):
    MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware')
    logging.debug("Will try to validate generated HTML.")

ROOT_URLCONF = 'desktop.urls'

TEMPLATE_DIRS = (get_desktop_root("core/templates"))

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.admin',
    'django_extensions',

    # 'debug_toolbar',
    'south',  # database migration tool

    # i18n support
    'babeldjango',
Пример #46
0
# limitations under the License.

import os
import logging
import re
import sys
import traceback
import pkg_resources

import desktop
from desktop.lib.paths import get_desktop_root

from django.utils.translation import ugettext as _

# Directories where apps and libraries are to be found
APP_DIRS = [get_desktop_root('core-apps'),
            get_desktop_root('apps'),
            get_desktop_root('libs')]

LOG = logging.getLogger(__name__)

######################################################################
# Global variables set after calling load_apps()
######################################################################

# List of DesktopModuleInfo that have been loaded
DESKTOP_LIBS = None
DESKTOP_APPS = None
DESKTOP_MODULES = [ ]           # Sum of APPS and LIBS

def _import_module_or_none(module):
Пример #47
0
          " (compute or send back results) for that session within SESSION_TIMEOUT_S seconds. Default: 15 min."),
  type=int,
  default=15 * 60
)

CONFIG_WHITELIST = Config(
  key='config_whitelist',
  default='debug_action,explain_level,mem_limit,optimize_partition_key_scans,query_timeout_s,request_pool',
  type=coerce_csv,
  help=_t('A comma-separated list of white-listed Impala configuration properties that users are authorized to set.')
)

IMPALA_CONF_DIR = Config(
  key='impala_conf_dir',
  help=_t('Impala configuration directory, where impala_flags is located.'),
  default=os.environ.get("HUE_CONF_DIR", get_desktop_root("conf")) + '/impala-conf'
)

SSL = ConfigSection(
  key='ssl',
  help=_t('SSL configuration for the server.'),
  members=dict(
    ENABLED = Config(
      key="enabled",
      help=_t("SSL communication enabled for this server."),
      type=coerce_bool,
      default=False
    ),

    CACERTS = Config(
      key="cacerts",
Пример #48
0
Файл: conf.py Проект: dizhu/hue
DATABASE = ConfigSection(
  key='database',
  help=_("""Configuration options for specifying the Desktop Database.
          For more info, see http://docs.djangoproject.com/en/1.1/ref/settings/#database-engine"""),
  members=dict(
    ENGINE=Config(
      key='engine',
      help=_('Database engine, such as postgresql_psycopg2, mysql, or sqlite3.'),
      type=str,
      default='sqlite3',
    ),
    NAME=Config(
      key='name',
      help=_('Database name, or path to DB if using sqlite3.'),
      type=str,
      default=get_desktop_root('desktop.db'),
    ),
    USER=Config(
      key='user',
      help=_('Database username.'),
      type=str,
      default='',
    ),
    PASSWORD=Config(
      key='password',
      help=_('Database password.'),
      type=str,
      default='',
    ),
    HOST=Config(
      key='host',
Пример #49
0
class Submission(object):
    """
  Represents one unique Oozie submission.

  Actions are:
  - submit
  - rerun
  """
    def __init__(self,
                 user,
                 job=None,
                 fs=None,
                 jt=None,
                 properties=None,
                 oozie_id=None,
                 local_tz=None):
        self.job = job
        self.user = user
        self.fs = fs
        self.jt = jt  # Deprecated with YARN, we now use logical names only for RM
        self.oozie_id = oozie_id
        self.api = get_oozie(self.user)

        if properties is not None:
            self.properties = properties
        else:
            self.properties = {}

        if local_tz and isinstance(self.job.data, dict):
            local_tz = self.job.data.get('properties')['timezone']

        # Modify start_date & end_date only when it's a coordinator
        from oozie.models2 import Coordinator
        if type(self.job) is Coordinator:
            if 'start_date' in self.properties:
                properties['start_date'] = convert_to_server_timezone(
                    self.properties['start_date'], local_tz)
            if 'end_date' in self.properties:
                properties['end_date'] = convert_to_server_timezone(
                    self.properties['end_date'], local_tz)

        if 'nominal_time' in self.properties:
            properties['nominal_time'] = convert_to_server_timezone(
                self.properties['nominal_time'], local_tz)

        self.properties['security_enabled'] = self.api.security_enabled

    def __str__(self):
        if self.oozie_id:
            res = "Submission for job '%s'." % (self.oozie_id, )
        else:
            res = "Submission for job '%s' (id %s, owner %s)." % (
                self.job.name, self.job.id, self.user)
        if self.oozie_id:
            res += " -- " + self.oozie_id
        return res

    @submit_dryrun
    def run(self, deployment_dir=None):
        """
    Take care of all the actions of submitting a Oozie workflow.
    Returns the oozie job id if all goes well.
    """

        if self.properties and 'oozie.use.system.libpath' not in self.properties:
            self.properties['oozie.use.system.libpath'] = 'true'

        self.oozie_id = self.api.submit_job(self.properties)
        LOG.info("Submitted: %s" % (self, ))

        if self._is_workflow():
            self.api.job_control(self.oozie_id, 'start')
            LOG.info("Started: %s" % (self, ))

        return self.oozie_id

    def rerun(self, deployment_dir, fail_nodes=None, skip_nodes=None):
        jt_address = cluster.get_cluster_addr_for_job_submission()

        self._update_properties(jt_address, deployment_dir)
        self.properties.update({'oozie.wf.application.path': deployment_dir})

        if 'oozie.coord.application.path' in self.properties:
            self.properties.pop('oozie.coord.application.path')

        if 'oozie.bundle.application.path' in self.properties:
            self.properties.pop('oozie.bundle.application.path')

        if fail_nodes:
            self.properties.update({'oozie.wf.rerun.failnodes': fail_nodes})
        elif not skip_nodes:
            self.properties.update({'oozie.wf.rerun.failnodes':
                                    'false'})  # Case empty 'skip_nodes' list
        else:
            self.properties.update({'oozie.wf.rerun.skip.nodes': skip_nodes})

        self.api.rerun(self.oozie_id, properties=self.properties)

        LOG.info("Rerun: %s" % (self, ))

        return self.oozie_id

    def rerun_coord(self, deployment_dir, params):
        jt_address = cluster.get_cluster_addr_for_job_submission()

        self._update_properties(jt_address, deployment_dir)
        self.properties.update(
            {'oozie.coord.application.path': deployment_dir})

        self.api.job_control(self.oozie_id,
                             action='coord-rerun',
                             properties=self.properties,
                             parameters=params)
        LOG.info("Rerun: %s" % (self, ))

        return self.oozie_id

    def update_coord(self):
        self.api = get_oozie(self.user, api_version="v2")
        self.api.job_control(self.oozie_id,
                             action='update',
                             properties=self.properties,
                             parameters=None)
        LOG.info("Update: %s" % (self, ))

        return self.oozie_id

    def rerun_bundle(self, deployment_dir, params):
        jt_address = cluster.get_cluster_addr_for_job_submission()

        self._update_properties(jt_address, deployment_dir)
        self.properties.update(
            {'oozie.bundle.application.path': deployment_dir})
        self.api.job_control(self.oozie_id,
                             action='bundle-rerun',
                             properties=self.properties,
                             parameters=params)
        LOG.info("Rerun: %s" % (self, ))

        return self.oozie_id

    def deploy(self, deployment_dir=None):
        try:
            if not deployment_dir:
                deployment_dir = self._create_deployment_dir()
        except Exception, ex:
            msg = _("Failed to create deployment directory: %s" % ex)
            LOG.exception(msg)
            raise PopupException(message=msg, detail=str(ex))

        if self.api.security_enabled:
            jt_address = cluster.get_cluster_addr_for_job_submission()
            self._update_properties(
                jt_address
            )  # Needed for coordinator deploying workflows with credentials

        if hasattr(self.job, 'nodes'):
            for action in self.job.nodes:
                # Make sure XML is there
                # Don't support more than one level sub-workflow
                if action.data['type'] == 'subworkflow':
                    from oozie.models2 import Workflow
                    workflow = Workflow(document=Document2.objects.get_by_uuid(
                        user=self.user,
                        uuid=action.data['properties']['workflow']))
                    sub_deploy = Submission(self.user, workflow, self.fs,
                                            self.jt, self.properties)
                    workspace = sub_deploy.deploy()

                    self.job.override_subworkflow_id(
                        action,
                        workflow.id)  # For displaying the correct graph
                    self.properties[
                        'workspace_%s' % workflow.
                        uuid] = workspace  # For pointing to the correct workspace

                elif action.data['type'] == 'altus':
                    service = 'dataeng'  # action.data['properties'].get('script_path')
                    auth_key_id = ALTUS.AUTH_KEY_ID.get()
                    auth_key_secret = ALTUS.AUTH_KEY_SECRET.get().replace(
                        '\\n', '\n')
                    shell_script = self._generate_altus_action_script(
                        service=service,
                        auth_key_id=auth_key_id,
                        auth_key_secret=auth_key_secret)
                    self._create_file(deployment_dir,
                                      action.data['name'] + '.py',
                                      shell_script)
                    self.fs.do_as_user(
                        self.user, self.fs.copyFromLocal,
                        os.path.join(get_desktop_root(), 'core', 'ext-py',
                                     'navoptapi-0.1.0'),
                        self.job.deployment_dir)

                elif action.data['type'] == 'impala' or action.data[
                        'type'] == 'impala-document':
                    from oozie.models2 import _get_impala_url
                    from impala.impala_flags import get_ssl_server_certificate

                    if action.data['type'] == 'impala-document':
                        from notebook.models import Notebook
                        if action.data['properties'].get('uuid'):
                            notebook = Notebook(
                                document=Document2.objects.get_by_uuid(
                                    user=self.user,
                                    uuid=action.data['properties']['uuid']))
                            statements = notebook.get_str()
                            statements = Template(statements).safe_substitute(
                                **self.properties)
                            script_name = action.data['name'] + '.sql'
                            self._create_file(deployment_dir, script_name,
                                              statements)
                    else:
                        script_name = os.path.basename(
                            action.data['properties'].get('script_path'))

                    if self.api.security_enabled:
                        kinit = 'kinit -k -t *.keytab %(user_principal)s' % {
                            'user_principal':
                            self.properties.get(
                                'user_principal', action.data['properties'].
                                get('user_principal'))
                        }
                    else:
                        kinit = ''

                    shell_script = """#!/bin/bash

# Needed to launch impala shell in oozie
export PYTHON_EGG_CACHE=./myeggs

%(kinit)s

impala-shell %(kerberos_option)s %(ssl_option)s -i %(impalad_host)s -f %(query_file)s""" % {
                        'impalad_host':
                        action.data['properties'].get('impalad_host')
                        or _get_impala_url(),
                        'kerberos_option':
                        '-k' if self.api.security_enabled else '',
                        'ssl_option':
                        '--ssl' if get_ssl_server_certificate() else '',
                        'query_file':
                        script_name,
                        'kinit':
                        kinit
                    }

                    self._create_file(deployment_dir,
                                      action.data['name'] + '.sh',
                                      shell_script)

                elif action.data['type'] == 'hive-document':
                    from notebook.models import Notebook
                    if action.data['properties'].get('uuid'):
                        notebook = Notebook(
                            document=Document2.objects.get_by_uuid(
                                user=self.user,
                                uuid=action.data['properties']['uuid']))
                        statements = notebook.get_str()
                    else:
                        statements = action.data['properties'].get(
                            'statements')

                    if self.properties.get('send_result_path'):
                        statements = """
INSERT OVERWRITE DIRECTORY '%s'
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
WITH SERDEPROPERTIES (
   "separatorChar" = "\t",
   "quoteChar"     = "'",
   "escapeChar"    = "\\"
)
STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'),
                            '\n\n\n'.join([
                                snippet['statement_raw']
                                for snippet in notebook.get_data()['snippets']
                            ]))

                    if statements is not None:
                        self._create_file(deployment_dir,
                                          action.data['name'] + '.sql',
                                          statements)

                elif action.data['type'] in ('java-document', 'java',
                                             'mapreduce-document'):
                    if action.data['type'] == 'java-document' or action.data[
                            'type'] == 'mapreduce-document':
                        from notebook.models import Notebook
                        notebook = Notebook(
                            document=Document2.objects.get_by_uuid(
                                user=self.user,
                                uuid=action.data['properties']['uuid']))
                        properties = notebook.get_data(
                        )['snippets'][0]['properties']
                    else:
                        properties = action.data['properties']

                    if properties.get('app_jar'):
                        LOG.debug("Adding to oozie.libpath %s" %
                                  properties['app_jar'])
                        paths = [properties['app_jar']]
                        if self.properties.get('oozie.libpath'):
                            paths.append(self.properties['oozie.libpath'])
                        self.properties['oozie.libpath'] = ','.join(paths)

                elif action.data['type'] == 'pig-document':
                    from notebook.models import Notebook
                    notebook = Notebook(document=Document2.objects.get_by_uuid(
                        user=self.user, uuid=action.data['properties']
                        ['uuid']))
                    statements = notebook.get_data(
                    )['snippets'][0]['statement_raw']

                    self._create_file(deployment_dir,
                                      action.data['name'] + '.pig', statements)
                elif action.data['type'] == 'spark' or action.data[
                        'type'] == 'spark-document':
                    if not [
                            f for f in action.data.get('properties').get(
                                'files', [])
                            if f.get('value').endswith('hive-site.xml')
                    ]:
                        hive_site_lib = Hdfs.join(deployment_dir + '/lib/',
                                                  'hive-site.xml')
                        hive_site_content = get_hive_site_content()
                        if not self.fs.do_as_user(
                                self.user, self.fs.exists,
                                hive_site_lib) and hive_site_content:
                            self.fs.do_as_user(
                                self.user,
                                self.fs.create,
                                hive_site_lib,
                                overwrite=True,
                                permission=0700,
                                data=smart_str(hive_site_content))

        oozie_xml = self.job.to_xml(self.properties)
        self._do_as(self.user.username, self._copy_files, deployment_dir,
                    oozie_xml, self.properties)

        return deployment_dir
Пример #50
0
    'django.middleware.http.ConditionalGetMiddleware',
    #@TODO@ Prakash to check FailedLoginMiddleware working or not?
    #'axes.middleware.FailedLoginMiddleware',
    'desktop.middleware.MimeTypeJSFileFixStreamingMiddleware',
]

# if os.environ.get(ENV_DESKTOP_DEBUG):
#   MIDDLEWARE_CLASSES.append('desktop.middleware.HtmlValidationMiddleware')
#   logging.debug("Will try to validate generated HTML.")

ROOT_URLCONF = 'desktop.urls'

# Hue runs its own wsgi applications
WSGI_APPLICATION = None

GTEMPLATE_DIRS = (get_desktop_root("core/templates"), )

INSTALLED_APPS = [
    'django.contrib.auth',
    'django_openid_auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'django_extensions',

    # 'debug_toolbar',
    #'south', # database migration tool

    # i18n support
Пример #51
0
Файл: views.py Проект: uk0/hue
def load_confs(configspecpath):
  conf_source = _configs_from_dir(get_desktop_root("conf"))
  conf = ConfigObj(configspec=configspecpath)
  for in_conf in conf_source:
    conf.merge(in_conf)
  return conf