def sync_ldap_users_groups(request):
"""
Handler for syncing the Hue database with LDAP users and groups.
This will not import any users or groups that don't already exist in Hue. All
user information and group memberships will be updated based on the LDAP
server's current state.
"""
if not request.user.is_superuser:
raise PopupException(_("You must be a superuser to sync the LDAP users/groups."))
if request.method == 'POST':
form = SyncLdapUsersGroupsForm(request.POST)
if form.is_valid():
users = sync_ldap_users()
groups = sync_ldap_groups()
# Create home dirs for every user sync'd
if form.cleaned_data['ensure_home_directory']:
for user in users:
try:
ensure_home_directory(request.fs, user.username)
except (IOError, WebHdfsException), e:
raise PopupException(_("The import may not be complete, sync again"), detail=e)
return redirect(reverse(list_users))
def delete_group(request, name):
if not request.user.is_superuser:
raise PopupException(_("You must be a superuser to delete groups."))
if request.method == 'POST':
try:
global groups_lock
__groups_lock.acquire()
try:
# Get the default group before getting the group, because we may be
# trying to delete the default group, and it may not have been created
# yet
default_group = get_default_user_group()
group = Group.objects.get(name=name)
if default_group is not None and default_group.name == name:
raise PopupException(_("The default user group may not be deleted."))
group.delete()
finally:
__groups_lock.release()
request.info(_('The group was deleted !'))
return redirect(reverse(list_groups))
except Group.DoesNotExist:
raise PopupException(_("Group not found."))
else:
return render("delete_group.mako", request, dict(path=request.path, groupname=name))
def submit_design(request, design_id):
"""
Submit a workflow to Oozie.
The POST data should contain parameter values.
"""
if request.method != 'POST':
raise PopupException(_('Use a POST request to submit a design.'))
design_obj = _get_design(design_id)
_check_permission(
request, design_obj.owner.username,
_("Access denied: submit design %(id)s.") % {'id': design_id})
# Expect the parameter mapping in the POST data
design_obj.bind_parameters(request.POST)
try:
submission = submit.Submission(design_obj, request.fs)
jobid = submission.run()
except RestException, ex:
detail = ex.message
if 'urlopen error' in ex.message:
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
raise PopupException(_("Error submitting design %(id)s.") %
{'id': design_id},
detail=detail)
def setup_app(request):
if request.method != 'POST':
raise PopupException(_('A POST request is required.'))
try:
oozie_setup.Command().handle_noargs()
request.info(_('Workspaces and examples installed.'))
except WebHdfsException, e:
raise PopupException(_('The app setup could complete.'), detail=e)
def _upload_archive(request):
"""
Handles archive upload.
The uploaded file is stored in memory.
We need to extract it and rename it.
"""
form = UploadArchiveForm(request.POST, request.FILES)
if form.is_valid():
uploaded_file = request.FILES['archive']
# Always a dir
if request.fs.isdir(form.cleaned_data['dest']
) and posixpath.sep in uploaded_file.name:
raise PopupException(
_('Sorry, no "%(sep)s" in the filename %(name)s.' % {
'sep': posixpath.sep,
'name': uploaded_file.name
}))
dest = request.fs.join(form.cleaned_data['dest'], uploaded_file.name)
try:
# Extract if necessary
# Make sure dest path is without '.zip' extension
if dest.endswith('.zip'):
temp_path = archive_factory(uploaded_file).extract()
if not temp_path:
raise PopupException(
_('Could not extract contents of file.'))
# Move the file to where it belongs
dest = dest[:-4]
request.fs.copyFromLocal(temp_path, dest)
shutil.rmtree(temp_path)
else:
raise PopupException(_('Could not interpret archive type.'))
except IOError, ex:
already_exists = False
try:
already_exists = request.fs.exists(dest)
except Exception:
pass
if already_exists:
msg = _('Destination %(name)s already exists.' %
{'name': dest})
else:
msg = _('Copy to "%(name)s failed: %(error)s') % {
'name': dest,
'error': ex
}
raise PopupException(msg)
return {
'status': 0,
'path': dest,
'result': _massage_stats(request, request.fs.stats(dest)),
'next': request.GET.get("next")
}
def load_table(request, table):
"""
Loads data into a table.
"""
try:
table_desc_extended = hcat_client().describe_table_extended(table)
is_table_partitioned = table_desc_extended['partitioned']
partitionColumns = []
if is_table_partitioned:
partitionColumns = table_desc_extended['partitionColumns']
table_obj = {
'tableName': table,
'columns': table_desc_extended['columns'],
'partitionKeys': partitionColumns
}
except Exception:
import traceback
error = traceback.format_exc()
raise PopupException('Error getting table description',
title="Error getting table description",
detail=error)
if request.method == "POST":
form = hcatalog.forms.LoadDataForm(table_obj, request.POST)
hql = ''
if form.is_valid():
hql += "LOAD DATA INPATH"
hql += " '%s'" % form.cleaned_data['path']
if form.cleaned_data['overwrite']:
hql += " OVERWRITE"
hql += " INTO TABLE "
hql += "`%s`" % (table, )
if len(form.partition_columns) > 0:
hql += " PARTITION ("
vals = []
for key, column_name in form.partition_columns.iteritems():
vals.append("%s='%s'" %
(column_name, form.cleaned_data[key]))
hql += ", ".join(vals)
hql += ")"
hql += ";"
try:
do_load_table(request, hql)
except Exception:
import traceback
error = traceback.format_exc()
raise PopupException('Error loading data into the table',
title="Error loading data into the table",
detail=error)
on_success_url = urlresolvers.reverse(describe_table,
kwargs=dict(table=table))
result = {'on_success_url': on_success_url}
return HttpResponse(json.dumps(result))
else:
form = hcatalog.forms.LoadDataForm(table_obj)
return render(
"load_table.mako", request,
dict(form=form, table=table, action=request.get_full_path()))
def setup(request):
"""Installs jobsub examples."""
if request.method != "POST":
raise PopupException(_('Use a POST request to install the examples.'))
try:
# Warning: below will modify fs.user
jobsub_setup.Command().handle_noargs()
except WebHdfsException, e:
raise PopupException(_('The examples could not be installed.'),
detail=e)
def _upload_file(request):
"""
Handles file uploaded by HDFSfileUploadHandler.
The uploaded file is stored in HDFS. We just need to rename it to the destination path.
"""
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
uploaded_file = request.FILES['hdfs_file']
dest = form.cleaned_data['dest']
if request.fs.isdir(dest) and posixpath.sep in uploaded_file.name:
raise PopupException(
_('Sorry, no "%(sep)s" in the filename %(name)s.' % {
'sep': posixpath.sep,
'name': uploaded_file.name
}))
dest = request.fs.join(dest, uploaded_file.name)
tmp_file = uploaded_file.get_temp_path()
username = request.user.username
try:
# Temp file is created by superuser. Chown the file.
request.fs.do_as_superuser(request.fs.chmod, tmp_file, 0644)
request.fs.do_as_superuser(request.fs.chown, tmp_file, username,
username)
# Move the file to where it belongs
request.fs.rename(tmp_file, dest)
except IOError, ex:
already_exists = False
try:
already_exists = request.fs.exists(dest)
except Exception:
pass
if already_exists:
msg = _('Destination %(name)s already exists.' %
{'name': dest})
else:
msg = _('Copy to "%(name)s failed: %(error)s') % {
'name': dest,
'error': ex
}
raise PopupException(msg)
return {
'status': 0,
'path': dest,
'result': _massage_stats(request, request.fs.stats(dest)),
'next': request.GET.get("next")
}
def get_help_fs(app_name):
"""
Creates a local file system for a given app's help directory.
"""
app = appmanager.get_desktop_module(app_name)
if app is not None:
if app.help_dir is None:
raise PopupException("No help available for app '%s'." % app_name)
return LocalSubFileSystem(app.help_dir)
else:
raise PopupException(
"App '%s' is not loaded, so no help is available for it!" %
app_name)
def _read_gzip(fs, path, offset, length):
contents = ''
if offset and offset != 0:
raise PopupException(_("Offsets are not supported with Gzip compression."))
try:
fhandle = fs.open(path)
try:
contents = GzipFile('', 'r', 0, StringIO(fhandle.read())).read(length)
except:
logging.warn("Could not decompress file at %s" % path, exc_info=True)
raise PopupException(_("Failed to decompress file."))
finally:
fhandle.close()
return contents
def view(request, path):
"""Dispatches viewing of a path to either index() or fileview(), depending on type."""
# default_to_home is set in bootstrap.js
if 'default_to_home' in request.GET:
home_dir_path = request.user.get_home_directory()
if request.fs.isdir(home_dir_path):
return format_preserving_redirect(
request,
urlresolvers.reverse(view, kwargs=dict(path=home_dir_path)))
try:
stats = request.fs.stats(path)
if stats.isDir:
return listdir_paged(request, path)
else:
return display(request, path)
except (IOError, WebHdfsException), e:
msg = _("Cannot access: %(path)s.") % {'path': escape(path)}
if request.user.is_superuser and not request.user == request.fs.superuser:
msg += _(
' Note: you are a Hue admin but not a HDFS superuser (which is "%(superuser)s").'
) % {
'superuser': request.fs.superuser
}
raise PopupException(msg, detail=e)
def check_job_edition_permission(oozie_job, user):
if has_job_edition_permission(oozie_job, user):
return oozie_job
else:
message = _("Permission denied. %(username)s don't have the permissions to modify job %(id)s") % \
{'username': user.username, 'id': oozie_job.id}
raise PopupException(message)
def decorate(request, *args, **kwargs):
try:
return view_func(request, *args, **kwargs)
except RestException, ex:
raise PopupException(_('Sorry, an error with Oozie happened.'),
detail=ex._headers.get(
'oozie-error-message', ex))
def deploy(self):
try:
deployment_dir = self._create_deployment_dir()
except Exception, ex:
msg = _("Failed to access deployment directory.")
LOG.exception(msg)
raise PopupException(message=msg, detail=str(ex))
def check_request_permission(self, request):
"""Raise PopupException if request user doesn't have permission to modify workflow"""
if not request.user.is_superuser and request.user.username != self.user:
access_warn(request, _('Insufficient permission.'))
raise PopupException(
_("Permission denied. User %(username)s cannot modify user %(user)s's job."
) % dict(username=request.user.username, user=self.user))
def create_table(request):
"""Create a table by specifying its attributes manually"""
form = MultiForm(
table=hcatalog.forms.CreateTableForm,
columns=hcatalog.forms.ColumnTypeFormSet,
partitions=hcatalog.forms.PartitionTypeFormSet)
if request.method == "POST":
form.bind(request.POST)
if form.is_valid() and 'createTable' in request.POST:
columns = [f.cleaned_data for f in form.columns.forms]
partition_columns = [f.cleaned_data for f in form.partitions.forms]
proposed_query = django_mako.render_to_string("create_table_statement.mako",
{
'table': form.table.cleaned_data,
'columns': columns,
'partition_columns': partition_columns
}
)
# Mako outputs bytestring in utf8
proposed_query = proposed_query.decode('utf-8')
tablename = form.table.cleaned_data['name']
tables = []
try:
hcat_client().create_table("default", proposed_query)
tables = hcat_client().get_tables()
except Exception, ex:
raise PopupException('Error on creating table', title="Error on creating table", detail=str(ex))
return render("show_tables.mako", request, dict(tables=tables,))
def _parse_fields(path, file_obj, encoding, filetypes, delimiters):
"""
_parse_fields(path, file_obj, encoding, filetypes, delimiters)
-> (delimiter, filetype, fields_list)
Go through the list of ``filetypes`` (gzip, text) and stop at the first one
that works for the data. Then apply the list of ``delimiters`` and pick the
most appropriate one.
``path`` is used for debugging only.
Return the best delimiter, filetype and the data broken down into rows of fields.
"""
file_readers = [
reader for reader in FILE_READERS if reader.TYPE in filetypes
]
for reader in file_readers:
LOG.debug("Trying %s for file: %s" % (reader.TYPE, path))
file_obj.seek(0, hadoopfs.SEEK_SET)
lines = reader.readlines(file_obj, encoding)
if lines is not None:
delim, fields_list = _readfields(lines, delimiters)
return delim, reader.TYPE, fields_list
else:
# Even TextFileReader doesn't work
msg = _(
"Failed to decode file '%(path)s' into printable characters under %(encoding)s"
) % {
'path': path,
'encoding': encoding
}
LOG.error(msg)
raise PopupException(msg)
def _read_avro(fs, path, offset, length):
contents = ''
try:
fhandle = fs.open(path)
try:
fhandle.seek(offset)
data_file_reader = datafile.DataFileReader(fhandle,
io.DatumReader())
contents_list = []
read_start = fhandle.tell()
# Iterate over the entire sought file.
for datum in data_file_reader:
read_length = fhandle.tell() - read_start
if read_length > length and len(contents_list) > 0:
break
else:
datum_str = str(datum) + "\n"
contents_list.append(datum_str)
data_file_reader.close()
contents = "".join(contents_list)
except:
logging.warn("Could not read avro file at %s" % path,
exc_info=True)
raise PopupException(_("Failed to read Avro file."))
finally:
fhandle.close()
return contents
def save_file(request):
"""
The POST endpoint to save a file in the file editor.
Does the save and then redirects back to the edit page.
"""
form = EditorForm(request.POST)
is_valid = form.is_valid()
path = form.cleaned_data.get('path')
if request.POST.get('save') == "Save As":
if not is_valid:
return edit(request, path, form=form)
else:
data = dict(form=form)
return render("saveas.mako", request, data)
if not path:
raise PopupException("No path specified")
if not is_valid:
return edit(request, path, form=form)
if request.fs.exists(path):
_do_overwrite_save(request.fs, path, form.cleaned_data['contents'],
form.cleaned_data['encoding'])
else:
_do_newfile_save(request.fs, path, form.cleaned_data['contents'],
form.cleaned_data['encoding'])
messages.info(request,
_('Saved %(path)s.') % {'path': os.path.basename(path)})
""" Changing path to reflect the request path of the JFrame that will actually be returned."""
request.path = urlresolvers.reverse("filebrowser.views.edit",
kwargs=dict(path=path))
return edit(request, path, form)
def generic_op(form_class, request, op, parameter_names, piggyback=None, template="fileop.mako", data_extractor=default_data_extractor, arg_extractor=default_arg_extractor, initial_value_extractor=default_initial_value_extractor, extra_params=None):
"""
Generic implementation for several operations.
@param form_class form to instantiate
@param request incoming request, used for parameters
@param op callable with the filesystem operation
@param parameter_names list of form parameters that are extracted and then passed to op
@param piggyback list of form parameters whose file stats to look up after the operation
@param data_extractor function that extracts POST data to be used by op
@param arg_extractor function that extracts args from a given form or formset
@param initial_value_extractor function that extracts the initial values of a form or formset
@param extra_params dictionary of extra parameters to send to the template for rendering
"""
# Use next for non-ajax requests, when available.
next = request.GET.get("next", request.POST.get("next", None))
ret = dict({
'next': next
})
if extra_params is not None:
ret['extra_params'] = extra_params
for p in parameter_names:
val = request.REQUEST.get(p)
if val:
ret[p] = val
if request.method == 'POST':
form = form_class(**data_extractor(request))
ret['form'] = form
if form.is_valid():
args = arg_extractor(request, form, parameter_names)
try:
op(*args)
except (IOError, WebHdfsException), e:
msg = _("Cannot perform operation.")
if request.user.is_superuser and not request.user == request.fs.superuser:
msg += _(' Note: you are a Hue admin but not a HDFS superuser (which is "%(superuser)s").') \
% {'superuser': request.fs.superuser}
raise PopupException(msg, detail=e)
if next:
logging.debug("Next: %s" % next)
# Doesn't need to be quoted: quoting is done by HttpResponseRedirect.
return format_preserving_redirect(request, next)
ret["success"] = True
try:
if piggyback:
piggy_path = form.cleaned_data[piggyback]
ret["result"] = _massage_stats(request, request.fs.stats(piggy_path))
except Exception, e:
# Hard to report these more naturally here. These happen either
# because of a bug in the piggy-back code or because of a
# race condition.
logger.exception("Exception while processing piggyback data")
ret["result_error"] = True
ret['user'] = request.user
return render(template, request, ret)
def describe_table(request, table):
try:
table_desc_extended = hcat_client().describe_table_extended(table)
is_table_partitioned = table_desc_extended['partitioned']
partitions = []
partitionColumns = []
if is_table_partitioned:
partitions = hcat_client().get_partitions(table)
partitionColumns = table_desc_extended['partitionColumns']
table_obj = {
'tableName': table,
'columns': table_desc_extended['columns'],
'partitionKeys': partitionColumns,
'partitions': partitions
}
except Exception:
import traceback
error = traceback.format_exc()
raise PopupException('Error getting table description',
title="Error getting table description",
detail=error)
load_form = hcatalog.forms.LoadDataForm(table_obj)
return render(
"describe_table.mako", request,
dict(
table=table_obj,
table_name=table,
load_form=load_form,
is_table_partitioned=is_table_partitioned,
))
def _check_permission(request, owner_name, error_msg, allow_root=False):
"""Raise PopupException if user doesn't have permission to modify the design"""
if request.user.username != owner_name:
if allow_root and request.user.is_superuser:
return
access_warn(request, error_msg)
raise PopupException(_("Permission denied. You are not the owner."))
def download(request, path):
"""
Downloads a file.
This is inspired by django.views.static.serve.
"""
if not request.fs.exists(path):
raise Http404(_("File not found: %(path)s") % {'path': escape(path)})
if not request.fs.isfile(path):
raise PopupException(_("'%(path)s' is not a file") % {'path': path})
mimetype = mimetypes.guess_type(path)[0] or 'application/octet-stream'
stats = request.fs.stats(path)
mtime = stats['mtime']
size = stats['size']
if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'),
mtime, size):
return HttpResponseNotModified()
# TODO(philip): Ideally a with statement would protect from leaks,
# but tricky to do here.
fh = request.fs.open(path)
response = HttpResponse(_file_reader(fh), mimetype=mimetype)
response["Last-Modified"] = http_date(stats['mtime'])
response["Content-Length"] = stats['size']
response["Content-Disposition"] = "attachment"
return response
def add_ldap_group(request):
"""
add_ldap_group(request) -> reply
Handler for importing LDAP groups into the Hue database.
If a group has been previously imported, this will sync membership within the
group with the LDAP server. If --import-members is specified, it will import
all unimported users.
"""
if not request.user.is_superuser:
raise PopupException(_("You must be a superuser to add another group."))
if request.method == 'POST':
form = AddLdapGroupForm(request.POST)
if form.is_valid():
groupname = form.cleaned_data['name']
import_by_dn = form.cleaned_data['dn']
import_members = form.cleaned_data['import_members']
group = import_ldap_group(groupname, import_members, import_by_dn)
if group is None:
errors = form._errors.setdefault('name', ErrorList())
errors.append(_('Could not get LDAP details for group %(groupname)s') % dict(groupname=(groupname,)))
else:
return redirect(reverse(list_groups))
else:
form = AddLdapGroupForm()
return render('edit_group.mako', request, dict(form=form, action=request.path, ldap=True))
def index(request, obj_id=None):
result = {}
result['scripts'] = PigScript.objects.filter(saved=True, user=request.user)
result['udfs'] = UDF.objects.all()
if request.method == 'POST':
form = PigScriptForm(request.POST)
if not form.is_valid():
raise PopupException("".join([
"%s: %s" % (field, error)
for field, error in form.errors.iteritems()
]))
if "autosave" in request.session:
del request.session['autosave']
if request.POST.get("script_id"):
instance = PigScript.objects.get(pk=request.POST['script_id'])
form = PigScriptForm(request.POST, instance=instance)
form.save()
else:
instance = PigScript(**form.cleaned_data)
instance.user = request.user
instance.save()
return redirect(reverse("view_script", args=[instance.pk]))
if not request.GET.get("new"):
result.update(request.session.get("autosave", {}))
#If we have obj_id, we renew or get instance to send it into form.
if obj_id:
instance = get_object_or_404(PigScript, pk=obj_id)
for field in instance._meta.fields:
result[field.name] = getattr(instance, field.name)
return render('edit_script.mako', request, dict(result=result))
def add_ldap_user(request):
"""
add_ldap_user(request) -> reply
Handler for importing LDAP users into the Hue database.
If a user has been previously imported, this will sync their user information.
If the LDAP request failed, the error message is generic right now.
"""
if not request.user.is_superuser:
raise PopupException(_("You must be a superuser to add another user."))
if request.method == 'POST':
form = AddLdapUserForm(request.POST)
if form.is_valid():
username = form.cleaned_data['username']
import_by_dn = form.cleaned_data['dn']
user = import_ldap_user(username, import_by_dn)
if form.cleaned_data['ensure_home_directory']:
try:
ensure_home_directory(request.fs, user.username)
except (IOError, WebHdfsException), e:
request.error(_("Cannot make home directory for user %s" % user.username))
if user is None:
errors = form._errors.setdefault('username', ErrorList())
errors.append(_('Could not get LDAP details for user %(username)s') % dict(username=(username,)))
else:
return redirect(reverse(list_users))
def edit_permission(request, app=None, priv=None):
"""
edit_permission(request, app = None, priv = None) -> reply
@type request: HttpRequest
@param request: The request object
@type app: string
@param app: Default to None, specifies the app of the privilege
@type priv: string
@param priv Default to None, the action of the privilege
Only superusers may modify permissions
"""
if not request.user.is_superuser:
raise PopupException(_("You must be a superuser to change permissions."))
instance = HuePermission.objects.get(app=app, action=priv)
if request.method == 'POST':
form = PermissionsEditForm(request.POST, instance=instance)
if form.is_valid():
form.save()
request.info(_('Permission information updated'))
return render("list_permissions.mako", request, dict(permissions=HuePermission.objects.all()))
else:
form = PermissionsEditForm(instance=instance)
return render('edit_permissions.mako', request,
dict(form=form, action=request.path, app=app, priv=priv))
def edit_group(request, name=None):
"""
edit_group(request, name = None) -> reply
@type request: HttpRequest
@param request: The request object
@type name: string
@param name: Default to None, when creating a new group
Only superusers may create a group
"""
if not request.user.is_superuser:
raise PopupException(_("You must be a superuser to add or edit a group."))
if name is not None:
instance = Group.objects.get(name=name)
else:
instance = None
if request.method == 'POST':
form = GroupEditForm(request.POST, instance=instance)
if form.is_valid():
form.save()
request.info(_('Group information updated'))
return list_groups(request)
else:
form = GroupEditForm(instance=instance)
return render('edit_group.mako', request, dict(form=form, action=request.path, name=name))
def move_down_action(request, action):
if request.method == 'POST':
action.workflow.move_action_down(action)
return redirect(
reverse('oozie:edit_workflow',
kwargs={'workflow': action.workflow.id}))
else:
raise PopupException(_('A POST request is required.'))
def decorated(request, *args, **kwargs):
if not request.user.has_hue_permission(action, app):
raise PopupException(
_("Permission denied (%(action)s/%(app)s).") % {
'action': action,
'app': app
})
return view_func(request, *args, **kwargs)