def process_view(self, request, view_func, view_args, view_kwargs): """ Sets request.fs and request.jt on every request to point to the configured filesystem. """ has_hadoop = apputil.has_hadoop() fs_ref = request.GET.get('fs', request.POST.get('fs', view_kwargs.get('fs'))) if "fs" in view_kwargs: del view_kwargs["fs"] if fs_ref is None: request.fs_ref, request.fs = fsmanager.get_default_hdfs() else: try: request.fs = fsmanager.get_filesystem(fs_ref) request.fs_ref = fs_ref except KeyError: raise KeyError('Cannot find filesystem called "%s"' % (fs_ref,)) if request.user.is_authenticated() and request.fs is not None: request.fs.setuser(request.user.username) if request.user.is_authenticated() and has_hadoop: request.jt = cluster.get_default_mrcluster() if request.jt is not None: request.jt.setuser(request.user.username) else: request.jt = None
def process_view(self, request, view_func, view_args, view_kwargs): """ Sets request.fs and request.jt on every request to point to the configured filesystem. """ has_hadoop = apputil.has_hadoop() fs_ref = request.GET.get("fs", request.POST.get("fs", view_kwargs.get("fs"))) if "fs" in view_kwargs: del view_kwargs["fs"] if not fs_ref: fs_ref = "default" try: request.fs = fsmanager.get_filesystem(fs_ref) request.fs_ref = fs_ref except KeyError: if fs_ref == "default" and not has_hadoop: request.fs = None else: raise if request.user.is_authenticated() and request.fs is not None: request.fs.setuser(request.user.username, request.user.get_groups()) if request.user.is_authenticated() and has_hadoop: request.jt = cluster.get_mrcluster() if request.jt is not None: request.jt.setuser(request.user.username, request.user.get_groups()) else: request.jt = None
def config_validator(user): ''' v2 When using the connectors, now 'hive' is seen as a dialect and only the list of connections (instance of the 'hive' connector, e.g. pointing to a Hive server in the Cloud) should be tested. Interpreters are now tested by the Editor in libs/notebook/conf.py. v1 All the configuration happens in apps/beeswax. ''' from beeswax.design import hql_query # dbms is dependent on beeswax.conf, import in method to avoid circular dependency from beeswax.server import dbms res = [] if has_connectors(): return res try: try: if not 'test' in sys.argv: # Avoid tests hanging server = dbms.get(user) query = hql_query("SELECT 'Hello World!';") handle = server.execute_and_wait(query, timeout_sec=10.0) if handle: server.fetch(handle, rows=100) server.close(handle) except StructuredThriftTransportException as e: if 'Error validating the login' in str(e): msg = 'Failed to authenticate to HiveServer2, check authentication configurations.' LOG.exception(msg) res.append((NICE_NAME, _(msg))) else: raise e except Exception as e: msg = "The application won't work without a running HiveServer2." LOG.exception(msg) res.append((NICE_NAME, _(msg))) try: from desktop.lib.fsmanager import get_filesystem from aws.conf import is_enabled as is_s3_enabled warehouse = beeswax.hive_site.get_metastore_warehouse_dir() fs = get_filesystem() fs_scheme = fs._get_scheme(warehouse) if fs: if fs_scheme == 's3a': if is_s3_enabled(): fs.do_as_user(user, fs.stats, warehouse) else: LOG.warn("Warehouse is in S3, but no credential available.") else: fs.do_as_superuser(fs.stats, warehouse) except Exception: msg = 'Failed to access Hive warehouse: %s' LOG.exception(msg % warehouse) res.append((NICE_NAME, _(msg) % warehouse)) return res
def process_view(self, request, view_func, view_args, view_kwargs): """ Sets request.fs and request.jt on every request to point to the configured filesystem. """ has_hadoop = apputil.has_hadoop() fs_ref = request.GET.get('fs', request.POST.get('fs', view_kwargs.get('fs'))) if "fs" in view_kwargs: del view_kwargs["fs"] if fs_ref is None: request.fs_ref, request.fs = fsmanager.get_default_hdfs() else: try: request.fs = fsmanager.get_filesystem(fs_ref) request.fs_ref = fs_ref except KeyError: raise KeyError('Cannot find filesystem called "%s"' % (fs_ref, )) if request.user.is_authenticated() and request.fs is not None: request.fs.setuser(request.user.username) if request.user.is_authenticated() and has_hadoop: request.jt = cluster.get_default_mrcluster() if request.jt is not None: request.jt.setuser(request.user.username) else: request.jt = None
def get_filesys(fs_ref): if fs_ref is None: fs_ref, fs = fsmanager.get_default_hdfs() else: try: fs = fsmanager.get_filesystem(fs_ref) except KeyError: raise KeyError('Cannot find filesystem called "%s"' % (fs_ref, )) return fs
def _get_request(postdict=None, user_id=None): request = HttpRequest() request.POST = postdict request.fs_ref = 'default' request.fs = fsmanager.get_filesystem(request.fs_ref) request.jt = None user = User.objects.get(id=user_id) user = rewrite_user(user) request.user = user return request
def __init__(self, request): FileUploadHandler.__init__(self, request) self._file = None self._starttime = 0 self._activated = False self._destination = request.GET.get('dest', None) # GET param avoids infinite looping self.request = request fs = fsmanager.get_filesystem('default') fs.setuser(request.user.username) FileUploadHandler.chunk_size = fs.get_upload_chuck_size(self._destination) if self._destination else UPLOAD_CHUNK_SIZE.get() LOG.debug("Chunk size = %d" % FileUploadHandler.chunk_size)
def get_django_request(request): django_request = request._request django_request.user = rewrite_user(django_request.user) # Workaround ClusterMiddleware not being applied if django_request.path.startswith('/api/') and django_request.fs is None: django_request.fs = fsmanager.get_filesystem(django_request.fs_ref) if django_request.user.is_authenticated and django_request.fs is not None: django_request.fs.setuser(django_request.user.username) return django_request
def process_view(self, request, view_func, view_args, view_kwargs): """ Sets request.fs and request.jt on every request to point to the configured filesystem. """ request.fs_ref = request.GET.get('fs', view_kwargs.get('fs', 'default')) if "fs" in view_kwargs: del view_kwargs["fs"] request.fs = fsmanager.get_filesystem(request.fs_ref) if request.user.is_authenticated(): if request.fs is not None: request.fs.setuser(request.user.username) # Deprecated request.jt = None
def new_file(self, field_name, file_name, *args, **kwargs): # Detect "HDFS" in the field name. if field_name.upper().startswith('HDFS'): LOG.info('Using HDFSfileUploadHandler to handle file upload.') try: fs_ref = self.request.REQUEST.get('fs', 'default') self.request.fs = fsmanager.get_filesystem(fs_ref) self.request.fs.setuser(self.request.user.username) self._file = HDFStemporaryUploadedFile(self.request, file_name, self._destination) LOG.debug('Upload attempt to %s' % (self._file.get_temp_path(),)) self._activated = True self._starttime = time.time() except Exception, ex: LOG.error("Not using HDFS upload handler: %s" % (ex,)) self.request.META['upload_failed'] = ex raise StopFutureHandlers()
def config_validator(user): # dbms is dependent on beeswax.conf (this file) # import in method to avoid circular dependency from beeswax.design import hql_query from beeswax.server import dbms res = [] try: try: if not 'test' in sys.argv: # Avoid tests hanging server = dbms.get(user) query = hql_query("SELECT 'Hello World!';") handle = server.execute_and_wait(query, timeout_sec=10.0) if handle: server.fetch(handle, rows=100) server.close(handle) except StructuredThriftTransportException as e: if 'Error validating the login' in str(e): msg = 'Failed to authenticate to HiveServer2, check authentication configurations.' LOG.exception(msg) res.append((NICE_NAME, _(msg))) else: raise e except Exception as e: msg = "The application won't work without a running HiveServer2." LOG.exception(msg) res.append((NICE_NAME, _(msg))) try: from desktop.lib.fsmanager import get_filesystem warehouse = beeswax.hive_site.get_metastore_warehouse_dir() fs = get_filesystem() if fs: fs.do_as_superuser(fs.stats, warehouse) except Exception: msg = 'Failed to access Hive warehouse: %s' LOG.exception(msg % warehouse) return [(NICE_NAME, _(msg) % warehouse)] return res
def process_view(self, request, view_func, view_args, view_kwargs): """ Sets request.fs and request.jt on every request to point to the configured filesystem. """ request.fs_ref = request.REQUEST.get("fs", view_kwargs.get("fs", "default")) if "fs" in view_kwargs: del view_kwargs["fs"] request.fs = fsmanager.get_filesystem(request.fs_ref) if request.user.is_authenticated(): if request.fs is not None: request.fs.setuser(request.user.username) request.jt = cluster.get_default_mrcluster() # Deprecated, only there for MR1 if request.jt is not None: request.jt.setuser(request.user.username) else: request.jt = None
from hadoop.fs.hadoopfs import Hdfs from liboozie.oozie_api import get_oozie res = [] try: from oozie.conf import REMOTE_SAMPLE_DIR except Exception, e: LOG.warn('Config check failed because Oozie app not installed: %s' % e) return res if OOZIE_URL.get(): status = get_oozie_status(user) if 'NORMAL' not in status: res.append((status, _('The Oozie server is not available'))) fs = get_filesystem() NICE_NAME = 'Oozie' if fs.do_as_superuser(fs.exists, REMOTE_SAMPLE_DIR.get()): stats = fs.do_as_superuser(fs.stats, REMOTE_SAMPLE_DIR.get()) mode = oct(stats.mode) # if neither group nor others have write permission group_has_write = int(mode[-2]) & 2 others_has_write = int(mode[-1]) & 2 if not group_has_write and not others_has_write: res.append( (NICE_NAME, "The permissions of workspace '%s' are too restrictive" % REMOTE_SAMPLE_DIR.get())) api = get_oozie(user, api_version="v2")
def dt_login(request, from_modal=False): if request.method == 'GET': redirect_to = request.GET.get('next', '/') else: redirect_to = request.POST.get('next', '/') is_first_login_ever = first_login_ever() backend_names = auth_forms.get_backend_names() is_active_directory = auth_forms.is_active_directory() is_ldap_option_selected = 'server' not in request.POST or request.POST.get('server') == 'LDAP' or \ request.POST.get('server') in auth_forms.get_ldap_server_keys() if is_active_directory and is_ldap_option_selected: UserCreationForm = auth_forms.LdapUserCreationForm AuthenticationForm = auth_forms.LdapAuthenticationForm else: UserCreationForm = auth_forms.UserCreationForm if 'ImpersonationBackend' in backend_names: AuthenticationForm = ImpersonationAuthenticationForm else: AuthenticationForm = auth_forms.AuthenticationForm if ENABLE_ORGANIZATIONS.get(): UserCreationForm = OrganizationUserCreationForm AuthenticationForm = OrganizationAuthenticationForm if request.method == 'POST': request.audit = { 'operation': 'USER_LOGIN', 'username': request.POST.get('username', request.POST.get('email')) } # For first login, need to validate user info! first_user_form = is_first_login_ever and UserCreationForm(data=request.POST) or None first_user = first_user_form and first_user_form.is_valid() if first_user or not is_first_login_ever: auth_form = AuthenticationForm(data=request.POST) if auth_form.is_valid(): # Must login by using the AuthenticationForm. It provides 'backend' on the User object. user = auth_form.get_user() userprofile = get_profile(user) login(request, user) # If Test cookie exists , it should be deleted if request.session.test_cookie_worked(): request.session.delete_test_cookie() if request.fs is None: request.fs = fsmanager.get_filesystem(request.fs_ref) try: ensure_home_directory(request.fs, user) except (IOError, WebHdfsException) as e: LOG.error('Could not create home directory at login for %s.' % user, exc_info=e) if require_change_password(userprofile): return HttpResponseRedirect('/hue' + reverse('useradmin:useradmin.views.edit_user', kwargs={'username': user.username})) userprofile.first_login = False userprofile.last_activity = datetime.now() if userprofile.creation_method == UserProfile.CreationMethod.EXTERNAL: # This is to fix a bug in Hue 4.3 userprofile.creation_method = UserProfile.CreationMethod.EXTERNAL.name userprofile.update_data({'auth_backend': user.backend}) userprofile.save() msg = 'Successful login for user: %s' % user.username request.audit['operationText'] = msg access_warn(request, msg) if from_modal or request.GET.get('fromModal', 'false') == 'true': return JsonResponse({'auth': True}) else: return HttpResponseRedirect(redirect_to) else: request.audit['allowed'] = False msg = 'Failed login for user: %s' % request.POST.get('username', request.POST.get('email')) request.audit['operationText'] = msg access_warn(request, msg) if from_modal or request.GET.get('fromModal', 'false') == 'true': return JsonResponse({'auth': False}) else: first_user_form = None auth_form = AuthenticationForm() # SAML/OIDC user is already authenticated in djangosaml2.views.login if hasattr(request, 'fs') and ( 'KnoxSpnegoDjangoBackend' in backend_names or 'SpnegoDjangoBackend' in backend_names or 'OIDCBackend' in backend_names or 'SAML2Backend' in backend_names ) and request.user.is_authenticated: if request.fs is None: request.fs = fsmanager.get_filesystem(request.fs_ref) try: ensure_home_directory(request.fs, request.user) except (IOError, WebHdfsException) as e: LOG.error('Could not create home directory for %s user %s.' % ('OIDC' if 'OIDCBackend' in backend_names else 'SAML', request.user)) if request.user.is_authenticated and not from_modal: return HttpResponseRedirect(redirect_to) if is_active_directory and not is_ldap_option_selected and \ request.method == 'POST' and request.user.username != request.POST.get('username'): # local user login failed, give the right auth_form with 'server' field auth_form = auth_forms.LdapAuthenticationForm() if not from_modal and SESSION.ENABLE_TEST_COOKIE.get() : request.session.set_test_cookie() if 'SAML2Backend' in backend_names: request.session['samlgroup_permitted_flag'] = samlgroup_check(request) renderable_path = 'login.mako' if from_modal: renderable_path = 'login_modal.mako' response = render(renderable_path, request, { 'action': reverse('desktop_auth_views_dt_login'), 'form': first_user_form or auth_form, 'next': redirect_to, 'first_login_ever': is_first_login_ever, 'login_errors': request.method == 'POST', 'backend_names': backend_names, 'active_directory': is_active_directory, 'user': request.user }) if not request.user.is_authenticated: response.delete_cookie(LOAD_BALANCER_COOKIE) # Note: might be re-balanced to another Hue on login. return response
query = hql_query("SELECT 'Hello World!';") handle = server.execute_and_wait(query, timeout_sec=10.0) if handle: server.fetch(handle, rows=100) server.close(handle) except StructuredThriftTransportException, e: if 'Error validating the login' in str(e): msg = 'Failed to authenticate to HiveServer2, check authentication configurations.' LOG.exception(msg) res.append((NICE_NAME, _(msg))) else: raise e except Exception, e: msg = "The application won't work without a running HiveServer2." LOG.exception(msg) res.append((NICE_NAME, _(msg))) try: from desktop.lib.fsmanager import get_filesystem warehouse = beeswax.hive_site.get_metastore_warehouse_dir() fs = get_filesystem() fs.stats(warehouse) except Exception: msg = 'Failed to access Hive warehouse: %s' LOG.exception(msg % warehouse) return [(NICE_NAME, _(msg) % warehouse)] return res
def config_validator(user): """ config_validator() -> [ (config_variable, error_message) ] Called by core check_config() view. """ from desktop.lib.fsmanager import get_filesystem from hadoop.cluster import get_all_hdfs from hadoop.fs.hadoopfs import Hdfs from liboozie.oozie_api import get_oozie res = [] try: from oozie.conf import REMOTE_SAMPLE_DIR except Exception as e: LOG.warn('Config check failed because Oozie app not installed: %s' % e) return res if OOZIE_URL.get(): status = get_oozie_status(user) if 'NORMAL' not in status: res.append((status, _('The Oozie server is not available'))) fs = get_filesystem() NICE_NAME = 'Oozie' if fs.do_as_superuser(fs.exists, REMOTE_SAMPLE_DIR.get()): stats = fs.do_as_superuser(fs.stats, REMOTE_SAMPLE_DIR.get()) mode = oct(stats.mode) # if neither group nor others have write permission group_has_write = int(mode[-2]) & 2 others_has_write = int(mode[-1]) & 2 if not group_has_write and not others_has_write: res.append( (NICE_NAME, "The permissions of workspace '%s' are too restrictive" % REMOTE_SAMPLE_DIR.get())) api = get_oozie(user, api_version="v2") configuration = api.get_configuration() if 'org.apache.oozie.service.MetricsInstrumentationService' in [ c.strip() for c in configuration.get('oozie.services.ext', '').split(',') ]: metrics = api.get_metrics() sharelib_url = 'gauges' in metrics and 'libs.sharelib.system.libpath' in metrics[ 'gauges'] and [ metrics['gauges']['libs.sharelib.system.libpath']['value'] ] or [] else: intrumentation = api.get_instrumentation() sharelib_url = [ param['value'] for group in intrumentation['variables'] for param in group['data'] if param['name'] == 'sharelib.system.libpath' ] if sharelib_url: sharelib_url = Hdfs.urlsplit(sharelib_url[0])[2] if not sharelib_url: res.append((status, _('Oozie Share Lib path is not available'))) class ConfigMock(object): def __init__(self, value): self.value = value def get(self): return self.value def get_fully_qualifying_key(self): return self.value for cluster in list(get_all_hdfs().values()): res.extend( validate_path( ConfigMock(sharelib_url), is_dir=True, fs=cluster, message=_( 'Oozie Share Lib not installed in default location.'))) return res