def setUpClass(cls):
if not ENABLE_CONNECTORS.get(): # Skip for now
raise SkipTest
cls._class_resets = [
ENABLE_CONNECTORS.set_for_testing(True),
]
class Migration(migrations.Migration):
dependencies = [
('desktop', '0010_auto_20200115_0908'),
('useradmin', '0002_userprofile_json_data'),
]
if ENABLE_CONNECTORS.get():
operations = [
migrations.AlterModelOptions(
name='huepermission',
options={
'verbose_name': 'Connector permission',
'verbose_name_plural': 'Connector permissions'
},
),
migrations.AddField(
model_name='huepermission',
name='connector',
field=models.ForeignKey(
default=1,
on_delete=django.db.models.deletion.CASCADE,
to='desktop.Connector'),
preserve_default=False,
),
migrations.AlterUniqueTogether(
name='huepermission',
unique_together=set([('connector', 'action')]),
),
]
else:
operations = []
def test_save_notebook_with_connector_on(self):
if not ENABLE_CONNECTORS.get():
raise SkipTest
notebook_cp = self.notebook.copy()
notebook_cp.pop('id')
connector = Connector.objects.create(
name='MySql',
dialect='mysql'
)
notebook_cp['snippets'][0]['connector'] = {
'name': 'MySql',
'dialect': 'mysql',
'type': str(connector.id),
'optimizer': 'api',
}
try:
response = self.client.post(reverse('notebook:save_notebook'), {'notebook': notebook_json})
data = json.loads(response.content)
finally:
connector.delete()
assert_equal(0, data['status'], data)
doc = Document2.objects.get(pk=data['id'])
assert_equal('query-mysql', doc.type)
def dump_config(request):
show_private = False
conf_dir = os.path.realpath(os.getenv("HUE_CONF_DIR", get_desktop_root("conf")))
if not is_admin(request.user):
return HttpResponse(_("You must be a superuser."))
if request.GET.get("private"):
show_private = True
app_modules = appmanager.DESKTOP_MODULES
config_modules = GLOBAL_CONFIG.get().values()
if ENABLE_CONNECTORS.get():
app_modules = [app_module for app_module in app_modules if app_module.name == 'desktop']
config_modules = [config_module for config_module in config_modules if config_module.config.key == 'desktop']
apps = sorted(app_modules, key=lambda app: app.name)
apps_names = [app.name for app in apps]
top_level = sorted(config_modules, key=lambda obj: apps_names.index(obj.config.key))
return render("dump_config.mako", request, {
'show_private': show_private,
'top_level': top_level,
'conf_dir': conf_dir,
'is_embeddable': request.GET.get('is_embeddable', False),
'apps': apps
}
)
def get_hue_config(request):
if not is_admin(request.user):
raise PopupException(_('You must be a superuser.'))
show_private = request.GET.get('private', False)
app_modules = appmanager.DESKTOP_MODULES
config_modules = GLOBAL_CONFIG.get().values()
if ENABLE_CONNECTORS.get():
app_modules = [
app_module for app_module in app_modules
if app_module.name == 'desktop'
]
config_modules = [
config_module for config_module in config_modules
if config_module.config.key == 'desktop'
]
apps = [{
'name': app.name,
'has_ui': app.menu_index != 999,
'display_name': app.display_name
} for app in sorted(app_modules, key=lambda app: app.name)]
def recurse_conf(modules):
attrs = []
for module in modules:
if not show_private and module.config.private:
continue
conf = {
'help': module.config.help or _('No help available.'),
'key': module.config.key,
'is_anonymous': is_anonymous(module.config.key)
}
if isinstance(module, BoundContainer):
conf['values'] = recurse_conf(module.get().values())
else:
conf['default'] = str(module.config.default)
if 'password' in module.config.key:
conf['value'] = '*' * 10
elif sys.version_info[0] > 2:
conf['value'] = str(module.get_raw())
else:
conf['value'] = str(module.get_raw()).decode(
'utf-8', 'replace')
attrs.append(conf)
return attrs
return JsonResponse({
'config':
sorted(recurse_conf(config_modules), key=lambda conf: conf.get('key')),
'conf_dir':
os.path.realpath(os.getenv('HUE_CONF_DIR', get_desktop_root('conf'))),
'apps':
apps
})
def _save_notebook(notebook, user):
if notebook['snippets'][0].get('connector') and notebook['snippets'][0][
'connector'].get('dialect'): # TODO Connector unification
notebook_type = 'query-%(dialect)s' % notebook['snippets'][0][
'connector']
if notebook['snippets'][0] and notebook['snippets'][0].get('executor'):
notebook['snippets'][0]['executor']['executables'] = []
else:
notebook_type = notebook.get('type', 'notebook')
save_as = False
if notebook.get(
'parentSavedQueryUuid'
): # We save into the original saved query, not into the query history
notebook_doc = Document2.objects.get_by_uuid(
user=user, uuid=notebook['parentSavedQueryUuid'])
elif notebook.get('id'):
notebook_doc = Document2.objects.get(id=notebook['id'])
else:
notebook_doc = Document2.objects.create(name=notebook['name'],
uuid=notebook['uuid'],
type=notebook_type,
owner=user)
Document.objects.link(notebook_doc,
owner=notebook_doc.owner,
name=notebook_doc.name,
description=notebook_doc.description,
extra=notebook_type)
save_as = True
if notebook.get('directoryUuid'):
notebook_doc.parent_directory = Document2.objects.get_by_uuid(
user=user,
uuid=notebook.get('directoryUuid'),
perm_type='write')
else:
notebook_doc.parent_directory = Document2.objects.get_home_directory(
user)
notebook['isSaved'] = True
notebook['isHistory'] = False
notebook['id'] = notebook_doc.id
_clear_sessions(notebook)
notebook_doc1 = notebook_doc._get_doc1(doc2_type=notebook_type)
if ENABLE_CONNECTORS.get():
notebook_doc.connector_id = int(
notebook['snippets'][0]['connector']['type'])
notebook_doc.update_data(notebook)
notebook_doc.search = _get_statement(notebook)
notebook_doc.name = notebook_doc1.name = notebook['name']
notebook_doc.description = notebook_doc1.description = notebook[
'description']
notebook_doc.save()
notebook_doc1.save()
return notebook_doc, save_as
def _historify(notebook, user):
query_type = 'query-%(dialect)s' % notebook if ENABLE_CONNECTORS.get(
) else notebook['type']
name = notebook['name'] if (
notebook['name']
and notebook['name'].strip() != '') else DEFAULT_HISTORY_NAME
is_managed = notebook.get('isManaged') == True # Prevents None
if is_managed and Document2.objects.filter(uuid=notebook['uuid']).exists():
history_doc = Document2.objects.get(uuid=notebook['uuid'])
else:
history_doc = Document2.objects.create(
name=name,
type=query_type,
owner=user,
is_history=True,
is_managed=is_managed,
)
# Link history of saved query
if notebook['isSaved']:
parent_doc = Document2.objects.get(
uuid=notebook.get('parentSavedQueryUuid') or notebook['uuid']
) # From previous history query or initial saved query
notebook['parentSavedQueryUuid'] = parent_doc.uuid
history_doc.dependencies.add(parent_doc)
if not is_managed:
Document.objects.link(history_doc,
name=history_doc.name,
owner=history_doc.owner,
description=history_doc.description,
extra=query_type)
notebook['uuid'] = history_doc.uuid
_clear_sessions(notebook)
if ENABLE_CONNECTORS.get():
history_doc.connector_id = int(notebook['type'])
history_doc.update_data(notebook)
history_doc.search = _get_statement(notebook)
history_doc.save()
return history_doc
def has_hue_permission(self, action=None, app=None, perm=None):
if perm is None:
try:
perm = self._lookup_permission(app, action)
except HuePermission.DoesNotExist:
LOG.exception("Permission object %s - %s not available. Was Django migrate command run after installation?" % (app, action))
return self.user.is_superuser
if self.user.is_superuser:
return True
if ENABLE_CONNECTORS.get() and app in ('jobbrowser', 'metastore', 'filebrowser', 'indexer'):
return True
group_ids = self.user.groups.values_list('id', flat=True)
return GroupPermission.objects.filter(group__id__in=group_ids, hue_permission=perm).exists()
def test_save_notebook_with_connector(self):
notebook_cp = self.notebook.copy()
notebook_cp.pop('id')
notebook_cp['snippets'][0]['connector'] = {"optimizer": "api"}
if not ENABLE_CONNECTORS.get(
): # At some point even v1 should set those
notebook_cp['snippets'][0]['connector']['name'] = 'MySql'
notebook_cp['snippets'][0]['connector']['dialect'] = 'mysql'
notebook_json = json.dumps(notebook_cp)
response = self.client.post(reverse('notebook:save_notebook'),
{'notebook': notebook_json})
data = json.loads(response.content)
assert_equal(0, data['status'], data)
doc = Document2.objects.get(pk=data['id'])
assert_equal('query-mysql', doc.type)
def test_save_notebook_with_connector(self):
if not ENABLE_CONNECTORS.get():
raise SkipTest
notebook_cp = self.notebook.copy()
notebook_cp.pop('id')
notebook_cp['snippets'][0]['connector'] = {
"name": "MySql",
"dialect": "mysql"
}
notebook_json = json.dumps(notebook_cp)
response = self.client.post(reverse('notebook:save_notebook'),
{'notebook': notebook_json})
data = json.loads(response.content)
assert_equal(0, data['status'], data)
doc = Document2.objects.get(pk=data['id'])
assert_equal('query-mysql', doc.type)
def test_save_notebook_with_connector_off(self):
reset = ENABLE_CONNECTORS.set_for_testing(False)
notebook_cp = self.notebook.copy()
notebook_cp.pop('id')
notebook_cp['snippets'][0]['connector'] = {
'name': 'MySql', # At some point even v1 should set those two
'dialect': 'mysql',
'optimizer': 'api',
}
notebook_json = json.dumps(notebook_cp)
try:
response = self.client.post(reverse('notebook:save_notebook'), {'notebook': notebook_json})
data = json.loads(response.content)
finally:
reset()
assert_equal(0, data['status'], data)
doc = Document2.objects.get(pk=data['id'])
assert_equal('query-mysql', doc.type)
def __init__(self, *args, **kwargs):
if not kwargs.get('organization'):
kwargs['organization'] = get_user_request_organization()
super(OrganizationConnectorPermission,
self).__init__(*args, **kwargs)
class Meta(ConnectorPermission.Meta):
abstract = True
unique_together = (
'connector',
'action',
'organization',
)
if ENABLE_CONNECTORS.get():
if ENABLE_ORGANIZATIONS.get():
class HuePermission(OrganizationConnectorPermission):
pass
else:
class HuePermission(ConnectorPermission):
pass
else:
class HuePermission(BasePermission):
pass
def setUpClass(cls):
cls._class_resets = [
ENABLE_CONNECTORS.set_for_testing(True),
ENABLE_ORGANIZATIONS.set_for_testing(False),
]
def test_get_interpreters_to_show():
default_interpreters = OrderedDict((
('hive', {
'name': 'Hive', 'displayName': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'is_sql': True,
'options': {}, 'dialect_properties': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'hive'
}),
('spark', {
'name': 'Scala', 'displayName': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {},
'dialect_properties': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'scala'
}),
('pig', {
'name': 'Pig', 'displayName': 'Pig', 'interface': 'pig', 'type': 'pig', 'is_sql': False, 'options': {},
'dialect_properties': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'pig'
}),
('java', {
'name': 'Java', 'displayName': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {},
'dialect_properties': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'java'
})
))
expected_interpreters = OrderedDict((
('java', {
'name': 'Java', 'displayName': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {},
'dialect_properties': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'java'
}),
('pig', {
'name': 'Pig', 'displayName': 'Pig', 'interface': 'pig', 'is_sql': False, 'type': 'pig', 'options': {},
'dialect_properties': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'pig'
}),
('hive', {
'name': 'Hive', 'displayName': 'Hive', 'interface': 'hiveserver2', 'is_sql': True, 'type': 'hive',
'options': {}, 'dialect_properties': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'hive'
}),
('spark', {
'name': 'Scala', 'displayName': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {},
'dialect_properties': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'scala'
})
))
try:
resets = [
INTERPRETERS.set_for_testing(default_interpreters),
APP_BLACKLIST.set_for_testing(''),
ENABLE_CONNECTORS.set_for_testing(False)
]
appmanager.DESKTOP_MODULES = []
appmanager.DESKTOP_APPS = None
appmanager.load_apps(APP_BLACKLIST.get())
notebook.conf.INTERPRETERS_CACHE = None
# 'get_interpreters_to_show should return the same as get_interpreters when interpreters_shown_on_wheel is unset'
assert_equal(
list(default_interpreters.values()), get_ordered_interpreters()
)
resets.append(INTERPRETERS_SHOWN_ON_WHEEL.set_for_testing('java,pig'))
assert_equal(
list(expected_interpreters.values()), get_ordered_interpreters(),
'get_interpreters_to_show did not return interpreters in the correct order expected'
)
finally:
for reset in resets:
reset()
appmanager.DESKTOP_MODULES = []
appmanager.DESKTOP_APPS = None
appmanager.load_apps(APP_BLACKLIST.get())
notebook.conf.INTERPRETERS_CACHE = None
def setUpClass(cls):
cls._class_resets = [
ENABLE_CONNECTORS.set_for_testing(True),
]
def update_app_permissions(**kwargs):
"""
Keep in sync apps and connectors permissions into the database table.
Map app + action to a HuePermission.
v2
Based on the connectors.
Permissions are either based on connectors instances or Hue specific actions.
Permissions can be deleted or added dynamically.
v1
This is a 'migrate' callback.
We never delete permissions automatically, because apps might come and go.
Note that signing up to the "migrate" signal is not necessarily the best thing we can do, since some apps might not
have models, but nonetheless, "migrate" is typically run when apps are installed.
"""
created_tables = connection.introspection.table_names()
if ENABLE_ORGANIZATIONS.get(
) and 'useradmin_organization' not in created_tables:
return
if u'useradmin_huepermission' in created_tables: # Check if Useradmin has been installed.
current = {}
try:
for dp in HuePermission.objects.all():
current.setdefault(dp.app, {})[dp.action] = dp
except:
LOG.exception('failed to get permissions')
return
updated = 0
uptodate = 0
added = []
if ENABLE_CONNECTORS.get():
old_apps = list(current.keys())
ConnectorPerm = collections.namedtuple('ConnectorPerm',
'name nice_name settings')
apps = [
ConnectorPerm(name=connector['name'],
nice_name=connector['nice_name'],
settings=[])
for connector in _get_installed_connectors()
]
else:
old_apps = []
apps = appmanager.DESKTOP_APPS
for app in apps:
app_name = app.name
permission_description = "Access the %s connection" % app.nice_name if ENABLE_CONNECTORS.get(
) else "Launch this application"
actions = set([("access", permission_description)])
actions.update(getattr(app.settings, "PERMISSION_ACTIONS", []))
if app_name not in current:
current[app_name] = {}
if app_name in old_apps:
old_apps.remove(app_name)
for action, description in actions:
c = current[app_name].get(action)
if c:
if c.description != description:
c.description = description
c.save()
updated += 1
else:
uptodate += 1
else:
new_dp = HuePermission(app=app_name,
action=action,
description=description)
if ENABLE_CONNECTORS.get():
new_dp.connector = Connector.objects.get(id=app_name)
new_dp.save()
added.append(new_dp)
# Only with v2
deleted, _ = HuePermission.objects.filter(app__in=old_apps).delete()
# Add all permissions to default group except some.
default_group = get_default_user_group()
if default_group:
for new_dp in added:
if not (new_dp.app == 'useradmin' and new_dp.action == 'access') and \
not (new_dp.app == 'useradmin' and new_dp.action == 'superuser') and \
not (new_dp.app == 'metastore' and new_dp.action == 'write') and \
not (new_dp.app == 'hbase' and new_dp.action == 'write') and \
not (new_dp.app == 'security' and new_dp.action == 'impersonate') and \
not (new_dp.app == 'filebrowser' and new_dp.action == 's3_access' and not is_idbroker_enabled('s3a')) and \
not (new_dp.app == 'filebrowser' and new_dp.action == 'gs_access' and not is_idbroker_enabled('gs')) and \
not (new_dp.app == 'filebrowser' and new_dp.action == 'adls_access') and \
not (new_dp.app == 'filebrowser' and new_dp.action == 'abfs_access') and \
not (new_dp.app == 'oozie' and new_dp.action == 'disable_editor_access'):
GroupPermission.objects.create(group=default_group,
hue_permission=new_dp)
available = HuePermission.objects.count()
stale = available - len(added) - updated - uptodate
if len(added) or updated or stale or deleted:
LOG.info(
"HuePermissions: %d added, %d updated, %d up to date, %d stale, %d deleted"
% (len(added), updated, uptodate, stale, deleted))
not (new_dp.app == 'filebrowser' and new_dp.action == 'adls_access') and \
not (new_dp.app == 'filebrowser' and new_dp.action == 'abfs_access') and \
not (new_dp.app == 'oozie' and new_dp.action == 'disable_editor_access'):
GroupPermission.objects.create(group=default_group,
hue_permission=new_dp)
available = HuePermission.objects.count()
stale = available - len(added) - updated - uptodate
if len(added) or updated or stale or deleted:
LOG.info(
"HuePermissions: %d added, %d updated, %d up to date, %d stale, %d deleted"
% (len(added), updated, uptodate, stale, deleted))
if not ENABLE_CONNECTORS.get():
models.signals.post_migrate.connect(update_app_permissions)
# models.signals.post_migrate.connect(get_default_user_group)
def install_sample_user(django_user=None):
"""
Setup the de-activated sample user with a certain id. Do not create a user profile.
"""
from desktop.models import SAMPLE_USER_ID, get_sample_user_install
from hadoop import cluster
user = None
django_username = get_sample_user_install(django_user)
if ENABLE_ORGANIZATIONS.get():
def process_view(self, request, view_func, view_args, view_kwargs):
"""
We also perform access logging in ``process_view()`` since we have the view function,
which tells us the log level. The downside is that we don't have the status code,
which isn't useful for status logging anyways.
"""
request.ts = time.time()
request.view_func = view_func
access_log_level = getattr(view_func, 'access_log_level', None)
# Skip loop for oidc
if request.path in [
'/oidc/authenticate/', '/oidc/callback/', '/oidc/logout/',
'/hue/oidc_failed/'
]:
return None
# Skip views not requiring login
# If the view has "opted out" of login required, skip
if hasattr(view_func, "login_notrequired"):
log_page_hit(request,
view_func,
level=access_log_level or logging.DEBUG)
return None
# There are certain django views which are also opt-out, but
# it would be evil to go add attributes to them
if view_func in DJANGO_VIEW_AUTH_WHITELIST:
log_page_hit(request,
view_func,
level=access_log_level or logging.DEBUG)
return None
# If user is logged in, check that he has permissions to access the app
if request.user.is_active and request.user.is_authenticated():
AppSpecificMiddleware.augment_request_with_app(request, view_func)
# Until Django 1.3 which resolves returning the URL name, just do a match of the name of the view
try:
access_view = 'access_view:%s:%s' % (
request._desktop_app, resolve(request.path)[0].__name__)
except Exception as e:
access_log(request,
'error checking view perm: %s' % e,
level=access_log_level)
access_view = ''
app_accessed = request._desktop_app
app_libs_whitelist = ("desktop", "home", "home2", "about", "hue",
"editor", "notebook", "indexer", "404",
"500", "403")
if not ENABLE_CONNECTORS.get():
# Accessing an app can access an underlying other app.
# e.g. impala or spark uses code from beeswax and so accessing impala shows up as beeswax here.
# Here we trust the URL to be the real app we need to check the perms.
ui_app_accessed = get_app_name(request)
if app_accessed != ui_app_accessed and ui_app_accessed not in (
'logs', 'accounts', 'login'):
app_accessed = ui_app_accessed
if app_accessed and \
app_accessed not in app_libs_whitelist and \
not (
is_admin(request.user) or
request.user.has_hue_permission(action="access", app=app_accessed) or
request.user.has_hue_permission(action=access_view, app=app_accessed)
) and \
not (app_accessed == '__debug__' and DJANGO_DEBUG_MODE.get()):
access_log(request,
'permission denied',
level=access_log_level)
return PopupException(_(
"You do not have permission to access the %(app_name)s application."
) % {
'app_name': app_accessed.capitalize()
},
error_code=401).response(request)
else:
if not hasattr(request, 'view_func'):
log_page_hit(request, view_func, level=access_log_level)
return None
logging.info("Redirecting to login page: %s", request.get_full_path())
access_log(request, 'login redirection', level=access_log_level)
no_idle_backends = ("libsaml.backend.SAML2Backend",
"desktop.auth.backend.SpnegoDjangoBackend",
"desktop.auth.backend.KnoxSpnegoDjangoBackend")
if request.ajax and all(no_idle_backend not in AUTH.BACKEND.get()
for no_idle_backend in no_idle_backends):
# Send back a magic header which causes Hue.Request to interpose itself
# in the ajax request and make the user login before resubmitting the
# request.
response = HttpResponse("/* login required */",
content_type="text/javascript")
response[MIDDLEWARE_HEADER] = 'LOGIN_REQUIRED'
return response
else:
if request.GET.get('is_embeddable'):
return JsonResponse(
{
'url':
"%s?%s=%s" %
(settings.LOGIN_URL, REDIRECT_FIELD_NAME,
urlquote('/hue' + request.get_full_path().replace(
'is_embeddable=true', '').replace('&&', '&')))
}
) # Remove embeddable so redirect from & to login works. Login page is not embeddable
else:
return HttpResponseRedirect(
"%s?%s=%s" % (settings.LOGIN_URL, REDIRECT_FIELD_NAME,
urlquote(request.get_full_path())))
