def process_view(self, request, view_func, view_args, view_kwargs):
"""
Sets request.fs and request.jt on every request to point to the
configured filesystem.
"""
has_hadoop = apputil.has_hadoop()
fs_ref = request.GET.get('fs', request.POST.get('fs', view_kwargs.get('fs')))
if "fs" in view_kwargs:
del view_kwargs["fs"]
if fs_ref is None:
request.fs_ref, request.fs = fsmanager.get_default_hdfs()
else:
try:
request.fs = fsmanager.get_filesystem(fs_ref)
request.fs_ref = fs_ref
except KeyError:
raise KeyError('Cannot find filesystem called "%s"' % (fs_ref,))
if request.user.is_authenticated() and request.fs is not None:
request.fs.setuser(request.user.username)
if request.user.is_authenticated() and has_hadoop:
request.jt = cluster.get_default_mrcluster()
if request.jt is not None:
request.jt.setuser(request.user.username)
else:
request.jt = None
def process_view(self, request, view_func, view_args, view_kwargs):
"""
Sets request.fs and request.jt on every request to point to the
configured filesystem.
"""
has_hadoop = apputil.has_hadoop()
fs_ref = request.GET.get("fs", request.POST.get("fs", view_kwargs.get("fs")))
if "fs" in view_kwargs:
del view_kwargs["fs"]
if not fs_ref:
fs_ref = "default"
try:
request.fs = fsmanager.get_filesystem(fs_ref)
request.fs_ref = fs_ref
except KeyError:
if fs_ref == "default" and not has_hadoop:
request.fs = None
else:
raise
if request.user.is_authenticated() and request.fs is not None:
request.fs.setuser(request.user.username, request.user.get_groups())
if request.user.is_authenticated() and has_hadoop:
request.jt = cluster.get_mrcluster()
if request.jt is not None:
request.jt.setuser(request.user.username, request.user.get_groups())
else:
request.jt = None
def config_validator(user):
'''
v2
When using the connectors, now 'hive' is seen as a dialect and only the list of connections
(instance of the 'hive' connector, e.g. pointing to a Hive server in the Cloud) should be tested.
Interpreters are now tested by the Editor in libs/notebook/conf.py.
v1
All the configuration happens in apps/beeswax.
'''
from beeswax.design import hql_query # dbms is dependent on beeswax.conf, import in method to avoid circular dependency
from beeswax.server import dbms
res = []
if has_connectors():
return res
try:
try:
if not 'test' in sys.argv: # Avoid tests hanging
server = dbms.get(user)
query = hql_query("SELECT 'Hello World!';")
handle = server.execute_and_wait(query, timeout_sec=10.0)
if handle:
server.fetch(handle, rows=100)
server.close(handle)
except StructuredThriftTransportException as e:
if 'Error validating the login' in str(e):
msg = 'Failed to authenticate to HiveServer2, check authentication configurations.'
LOG.exception(msg)
res.append((NICE_NAME, _(msg)))
else:
raise e
except Exception as e:
msg = "The application won't work without a running HiveServer2."
LOG.exception(msg)
res.append((NICE_NAME, _(msg)))
try:
from desktop.lib.fsmanager import get_filesystem
from aws.conf import is_enabled as is_s3_enabled
warehouse = beeswax.hive_site.get_metastore_warehouse_dir()
fs = get_filesystem()
fs_scheme = fs._get_scheme(warehouse)
if fs:
if fs_scheme == 's3a':
if is_s3_enabled():
fs.do_as_user(user, fs.stats, warehouse)
else:
LOG.warn("Warehouse is in S3, but no credential available.")
else:
fs.do_as_superuser(fs.stats, warehouse)
except Exception:
msg = 'Failed to access Hive warehouse: %s'
LOG.exception(msg % warehouse)
res.append((NICE_NAME, _(msg) % warehouse))
return res
def process_view(self, request, view_func, view_args, view_kwargs):
"""
Sets request.fs and request.jt on every request to point to the
configured filesystem.
"""
has_hadoop = apputil.has_hadoop()
fs_ref = request.GET.get('fs',
request.POST.get('fs', view_kwargs.get('fs')))
if "fs" in view_kwargs:
del view_kwargs["fs"]
if fs_ref is None:
request.fs_ref, request.fs = fsmanager.get_default_hdfs()
else:
try:
request.fs = fsmanager.get_filesystem(fs_ref)
request.fs_ref = fs_ref
except KeyError:
raise KeyError('Cannot find filesystem called "%s"' %
(fs_ref, ))
if request.user.is_authenticated() and request.fs is not None:
request.fs.setuser(request.user.username)
if request.user.is_authenticated() and has_hadoop:
request.jt = cluster.get_default_mrcluster()
if request.jt is not None:
request.jt.setuser(request.user.username)
else:
request.jt = None
def get_filesys(fs_ref):
if fs_ref is None:
fs_ref, fs = fsmanager.get_default_hdfs()
else:
try:
fs = fsmanager.get_filesystem(fs_ref)
except KeyError:
raise KeyError('Cannot find filesystem called "%s"' % (fs_ref, ))
return fs
def _get_request(postdict=None, user_id=None):
request = HttpRequest()
request.POST = postdict
request.fs_ref = 'default'
request.fs = fsmanager.get_filesystem(request.fs_ref)
request.jt = None
user = User.objects.get(id=user_id)
user = rewrite_user(user)
request.user = user
return request
def __init__(self, request):
FileUploadHandler.__init__(self, request)
self._file = None
self._starttime = 0
self._activated = False
self._destination = request.GET.get('dest', None) # GET param avoids infinite looping
self.request = request
fs = fsmanager.get_filesystem('default')
fs.setuser(request.user.username)
FileUploadHandler.chunk_size = fs.get_upload_chuck_size(self._destination) if self._destination else UPLOAD_CHUNK_SIZE.get()
LOG.debug("Chunk size = %d" % FileUploadHandler.chunk_size)
def get_django_request(request):
django_request = request._request
django_request.user = rewrite_user(django_request.user)
# Workaround ClusterMiddleware not being applied
if django_request.path.startswith('/api/') and django_request.fs is None:
django_request.fs = fsmanager.get_filesystem(django_request.fs_ref)
if django_request.user.is_authenticated and django_request.fs is not None:
django_request.fs.setuser(django_request.user.username)
return django_request
def process_view(self, request, view_func, view_args, view_kwargs):
"""
Sets request.fs and request.jt on every request to point to the configured filesystem.
"""
request.fs_ref = request.GET.get('fs', view_kwargs.get('fs', 'default'))
if "fs" in view_kwargs:
del view_kwargs["fs"]
request.fs = fsmanager.get_filesystem(request.fs_ref)
if request.user.is_authenticated():
if request.fs is not None:
request.fs.setuser(request.user.username)
# Deprecated
request.jt = None
def process_view(self, request, view_func, view_args, view_kwargs):
"""
Sets request.fs and request.jt on every request to point to the
configured filesystem.
"""
request.fs_ref = request.GET.get('fs', view_kwargs.get('fs', 'default'))
if "fs" in view_kwargs:
del view_kwargs["fs"]
request.fs = fsmanager.get_filesystem(request.fs_ref)
if request.user.is_authenticated():
if request.fs is not None:
request.fs.setuser(request.user.username)
# Deprecated
request.jt = None
def new_file(self, field_name, file_name, *args, **kwargs):
# Detect "HDFS" in the field name.
if field_name.upper().startswith('HDFS'):
LOG.info('Using HDFSfileUploadHandler to handle file upload.')
try:
fs_ref = self.request.REQUEST.get('fs', 'default')
self.request.fs = fsmanager.get_filesystem(fs_ref)
self.request.fs.setuser(self.request.user.username)
self._file = HDFStemporaryUploadedFile(self.request, file_name, self._destination)
LOG.debug('Upload attempt to %s' % (self._file.get_temp_path(),))
self._activated = True
self._starttime = time.time()
except Exception, ex:
LOG.error("Not using HDFS upload handler: %s" % (ex,))
self.request.META['upload_failed'] = ex
raise StopFutureHandlers()
def config_validator(user):
# dbms is dependent on beeswax.conf (this file)
# import in method to avoid circular dependency
from beeswax.design import hql_query
from beeswax.server import dbms
res = []
try:
try:
if not 'test' in sys.argv: # Avoid tests hanging
server = dbms.get(user)
query = hql_query("SELECT 'Hello World!';")
handle = server.execute_and_wait(query, timeout_sec=10.0)
if handle:
server.fetch(handle, rows=100)
server.close(handle)
except StructuredThriftTransportException as e:
if 'Error validating the login' in str(e):
msg = 'Failed to authenticate to HiveServer2, check authentication configurations.'
LOG.exception(msg)
res.append((NICE_NAME, _(msg)))
else:
raise e
except Exception as e:
msg = "The application won't work without a running HiveServer2."
LOG.exception(msg)
res.append((NICE_NAME, _(msg)))
try:
from desktop.lib.fsmanager import get_filesystem
warehouse = beeswax.hive_site.get_metastore_warehouse_dir()
fs = get_filesystem()
if fs:
fs.do_as_superuser(fs.stats, warehouse)
except Exception:
msg = 'Failed to access Hive warehouse: %s'
LOG.exception(msg % warehouse)
return [(NICE_NAME, _(msg) % warehouse)]
return res
def process_view(self, request, view_func, view_args, view_kwargs):
"""
Sets request.fs and request.jt on every request to point to the
configured filesystem.
"""
request.fs_ref = request.REQUEST.get("fs", view_kwargs.get("fs", "default"))
if "fs" in view_kwargs:
del view_kwargs["fs"]
request.fs = fsmanager.get_filesystem(request.fs_ref)
if request.user.is_authenticated():
if request.fs is not None:
request.fs.setuser(request.user.username)
request.jt = cluster.get_default_mrcluster() # Deprecated, only there for MR1
if request.jt is not None:
request.jt.setuser(request.user.username)
else:
request.jt = None
from hadoop.fs.hadoopfs import Hdfs
from liboozie.oozie_api import get_oozie
res = []
try:
from oozie.conf import REMOTE_SAMPLE_DIR
except Exception, e:
LOG.warn('Config check failed because Oozie app not installed: %s' % e)
return res
if OOZIE_URL.get():
status = get_oozie_status(user)
if 'NORMAL' not in status:
res.append((status, _('The Oozie server is not available')))
fs = get_filesystem()
NICE_NAME = 'Oozie'
if fs.do_as_superuser(fs.exists, REMOTE_SAMPLE_DIR.get()):
stats = fs.do_as_superuser(fs.stats, REMOTE_SAMPLE_DIR.get())
mode = oct(stats.mode)
# if neither group nor others have write permission
group_has_write = int(mode[-2]) & 2
others_has_write = int(mode[-1]) & 2
if not group_has_write and not others_has_write:
res.append(
(NICE_NAME,
"The permissions of workspace '%s' are too restrictive" %
REMOTE_SAMPLE_DIR.get()))
api = get_oozie(user, api_version="v2")
def dt_login(request, from_modal=False):
if request.method == 'GET':
redirect_to = request.GET.get('next', '/')
else:
redirect_to = request.POST.get('next', '/')
is_first_login_ever = first_login_ever()
backend_names = auth_forms.get_backend_names()
is_active_directory = auth_forms.is_active_directory()
is_ldap_option_selected = 'server' not in request.POST or request.POST.get('server') == 'LDAP' or \
request.POST.get('server') in auth_forms.get_ldap_server_keys()
if is_active_directory and is_ldap_option_selected:
UserCreationForm = auth_forms.LdapUserCreationForm
AuthenticationForm = auth_forms.LdapAuthenticationForm
else:
UserCreationForm = auth_forms.UserCreationForm
if 'ImpersonationBackend' in backend_names:
AuthenticationForm = ImpersonationAuthenticationForm
else:
AuthenticationForm = auth_forms.AuthenticationForm
if ENABLE_ORGANIZATIONS.get():
UserCreationForm = OrganizationUserCreationForm
AuthenticationForm = OrganizationAuthenticationForm
if request.method == 'POST':
request.audit = {
'operation': 'USER_LOGIN',
'username': request.POST.get('username', request.POST.get('email'))
}
# For first login, need to validate user info!
first_user_form = is_first_login_ever and UserCreationForm(data=request.POST) or None
first_user = first_user_form and first_user_form.is_valid()
if first_user or not is_first_login_ever:
auth_form = AuthenticationForm(data=request.POST)
if auth_form.is_valid():
# Must login by using the AuthenticationForm. It provides 'backend' on the User object.
user = auth_form.get_user()
userprofile = get_profile(user)
login(request, user)
# If Test cookie exists , it should be deleted
if request.session.test_cookie_worked():
request.session.delete_test_cookie()
if request.fs is None:
request.fs = fsmanager.get_filesystem(request.fs_ref)
try:
ensure_home_directory(request.fs, user)
except (IOError, WebHdfsException) as e:
LOG.error('Could not create home directory at login for %s.' % user, exc_info=e)
if require_change_password(userprofile):
return HttpResponseRedirect('/hue' + reverse('useradmin:useradmin.views.edit_user', kwargs={'username': user.username}))
userprofile.first_login = False
userprofile.last_activity = datetime.now()
if userprofile.creation_method == UserProfile.CreationMethod.EXTERNAL: # This is to fix a bug in Hue 4.3
userprofile.creation_method = UserProfile.CreationMethod.EXTERNAL.name
userprofile.update_data({'auth_backend': user.backend})
userprofile.save()
msg = 'Successful login for user: %s' % user.username
request.audit['operationText'] = msg
access_warn(request, msg)
if from_modal or request.GET.get('fromModal', 'false') == 'true':
return JsonResponse({'auth': True})
else:
return HttpResponseRedirect(redirect_to)
else:
request.audit['allowed'] = False
msg = 'Failed login for user: %s' % request.POST.get('username', request.POST.get('email'))
request.audit['operationText'] = msg
access_warn(request, msg)
if from_modal or request.GET.get('fromModal', 'false') == 'true':
return JsonResponse({'auth': False})
else:
first_user_form = None
auth_form = AuthenticationForm()
# SAML/OIDC user is already authenticated in djangosaml2.views.login
if hasattr(request, 'fs') and (
'KnoxSpnegoDjangoBackend' in backend_names or 'SpnegoDjangoBackend' in backend_names or 'OIDCBackend' in backend_names or
'SAML2Backend' in backend_names
) and request.user.is_authenticated:
if request.fs is None:
request.fs = fsmanager.get_filesystem(request.fs_ref)
try:
ensure_home_directory(request.fs, request.user)
except (IOError, WebHdfsException) as e:
LOG.error('Could not create home directory for %s user %s.' % ('OIDC' if 'OIDCBackend' in backend_names else 'SAML', request.user))
if request.user.is_authenticated and not from_modal:
return HttpResponseRedirect(redirect_to)
if is_active_directory and not is_ldap_option_selected and \
request.method == 'POST' and request.user.username != request.POST.get('username'):
# local user login failed, give the right auth_form with 'server' field
auth_form = auth_forms.LdapAuthenticationForm()
if not from_modal and SESSION.ENABLE_TEST_COOKIE.get() :
request.session.set_test_cookie()
if 'SAML2Backend' in backend_names:
request.session['samlgroup_permitted_flag'] = samlgroup_check(request)
renderable_path = 'login.mako'
if from_modal:
renderable_path = 'login_modal.mako'
response = render(renderable_path, request, {
'action': reverse('desktop_auth_views_dt_login'),
'form': first_user_form or auth_form,
'next': redirect_to,
'first_login_ever': is_first_login_ever,
'login_errors': request.method == 'POST',
'backend_names': backend_names,
'active_directory': is_active_directory,
'user': request.user
})
if not request.user.is_authenticated:
response.delete_cookie(LOAD_BALANCER_COOKIE) # Note: might be re-balanced to another Hue on login.
return response
query = hql_query("SELECT 'Hello World!';")
handle = server.execute_and_wait(query, timeout_sec=10.0)
if handle:
server.fetch(handle, rows=100)
server.close(handle)
except StructuredThriftTransportException, e:
if 'Error validating the login' in str(e):
msg = 'Failed to authenticate to HiveServer2, check authentication configurations.'
LOG.exception(msg)
res.append((NICE_NAME, _(msg)))
else:
raise e
except Exception, e:
msg = "The application won't work without a running HiveServer2."
LOG.exception(msg)
res.append((NICE_NAME, _(msg)))
try:
from desktop.lib.fsmanager import get_filesystem
warehouse = beeswax.hive_site.get_metastore_warehouse_dir()
fs = get_filesystem()
fs.stats(warehouse)
except Exception:
msg = 'Failed to access Hive warehouse: %s'
LOG.exception(msg % warehouse)
return [(NICE_NAME, _(msg) % warehouse)]
return res
def config_validator(user):
"""
config_validator() -> [ (config_variable, error_message) ]
Called by core check_config() view.
"""
from desktop.lib.fsmanager import get_filesystem
from hadoop.cluster import get_all_hdfs
from hadoop.fs.hadoopfs import Hdfs
from liboozie.oozie_api import get_oozie
res = []
try:
from oozie.conf import REMOTE_SAMPLE_DIR
except Exception as e:
LOG.warn('Config check failed because Oozie app not installed: %s' % e)
return res
if OOZIE_URL.get():
status = get_oozie_status(user)
if 'NORMAL' not in status:
res.append((status, _('The Oozie server is not available')))
fs = get_filesystem()
NICE_NAME = 'Oozie'
if fs.do_as_superuser(fs.exists, REMOTE_SAMPLE_DIR.get()):
stats = fs.do_as_superuser(fs.stats, REMOTE_SAMPLE_DIR.get())
mode = oct(stats.mode)
# if neither group nor others have write permission
group_has_write = int(mode[-2]) & 2
others_has_write = int(mode[-1]) & 2
if not group_has_write and not others_has_write:
res.append(
(NICE_NAME,
"The permissions of workspace '%s' are too restrictive" %
REMOTE_SAMPLE_DIR.get()))
api = get_oozie(user, api_version="v2")
configuration = api.get_configuration()
if 'org.apache.oozie.service.MetricsInstrumentationService' in [
c.strip()
for c in configuration.get('oozie.services.ext', '').split(',')
]:
metrics = api.get_metrics()
sharelib_url = 'gauges' in metrics and 'libs.sharelib.system.libpath' in metrics[
'gauges'] and [
metrics['gauges']['libs.sharelib.system.libpath']['value']
] or []
else:
intrumentation = api.get_instrumentation()
sharelib_url = [
param['value'] for group in intrumentation['variables']
for param in group['data']
if param['name'] == 'sharelib.system.libpath'
]
if sharelib_url:
sharelib_url = Hdfs.urlsplit(sharelib_url[0])[2]
if not sharelib_url:
res.append((status, _('Oozie Share Lib path is not available')))
class ConfigMock(object):
def __init__(self, value):
self.value = value
def get(self):
return self.value
def get_fully_qualifying_key(self):
return self.value
for cluster in list(get_all_hdfs().values()):
res.extend(
validate_path(
ConfigMock(sharelib_url),
is_dir=True,
fs=cluster,
message=_(
'Oozie Share Lib not installed in default location.')))
return res
