def save_file(request):
"""
The POST endpoint to save a file in the file editor.
Does the save and then redirects back to the edit page.
"""
form = EditorForm(request.POST)
is_valid = form.is_valid()
path = form.cleaned_data.get('path')
if request.POST.get('save') == "saveAs":
if not is_valid:
return edit(request, path, form=form)
else:
data = dict(form=form)
return render_with_toolbars("saveas.mako", request, data)
if not path:
raise PopupException("No path specified")
if not is_valid:
return edit(request, path, form=form)
if request.fs.exists(path):
_do_overwrite_save(request.fs, path, form.cleaned_data['contents'],
form.cleaned_data['encoding'])
else:
_do_newfile_save(request.fs, path, form.cleaned_data['contents'],
form.cleaned_data['encoding'])
request.flash.put('Saved %s.' % os.path.basename(path))
""" Changing path to reflect the request path of the JFrame that will actually be returned."""
request.path = urlresolvers.reverse("filebrowser.views.edit",
kwargs=dict(path=path))
return edit(request, path, form)
def save_file(request):
"""
The POST endpoint to save a file in the file editor.
Does the save and then redirects back to the edit page.
"""
form = EditorForm(request.POST)
is_valid = form.is_valid()
path = form.cleaned_data.get('path')
if request.POST.get('save') == "saveAs":
if not is_valid:
return edit(request, path, form=form)
else:
data = dict(form = form)
return render_with_toolbars("saveas.mako", request, data)
if not path:
raise PopupException("No path specified")
if not is_valid:
return edit(request, path, form=form)
if request.fs.exists(path):
_do_overwrite_save(request.fs, path,
form.cleaned_data['contents'],
form.cleaned_data['encoding'])
else:
_do_newfile_save(request.fs, path,
form.cleaned_data['contents'],
form.cleaned_data['encoding'])
request.flash.put('Saved %s.' % os.path.basename(path))
""" Changing path to reflect the request path of the JFrame that will actually be returned."""
request.path = urlresolvers.reverse("filebrowser.views.edit", kwargs=dict(path=path))
return edit(request, path, form)
def generic_op(form_class,
request,
op,
parameter_names,
piggyback=None,
template="fileop.mako",
extra_params=None):
"""
Generic implementation for several operations.
@param form_class form to instantiate
@param request incoming request, used for parameters
@param op callable with the filesystem operation
@param parameter_names list of form parameters that are extracted and then passed to op
@param piggyback list of form parameters whose file stats to look up after the operation
@param extra_params dictionary of extra parameters to send to the template for rendering
"""
# Use next for non-ajax requests, when available.
next = request.GET.get("next")
if next is None:
next = request.POST.get("next")
ret = dict({'next': next})
if extra_params is not None:
ret['extra_params'] = extra_params
for p in parameter_names:
val = request.REQUEST.get(p)
if val:
ret[p] = val
if request.method == 'POST':
form = form_class(request.POST)
# TODO(philip): How best to do error handling? fs will throw
# an arbitrary-ish exception (typically file not found or maybe permission
# denied), and this needs to be coaxed into an HTTP error.
ret['form'] = form
if form.is_valid():
args = [form.cleaned_data[p] for p in parameter_names]
op(*args)
if next:
logging.debug("Next: %s" % next)
# Doesn't need to be quoted: quoting is done by HttpResponseRedirect.
return format_preserving_redirect(request, next)
ret["success"] = True
try:
if piggyback:
piggy_path = form.cleaned_data[piggyback]
ret["result"] = _massage_stats(
request, request.fs.stats(piggy_path))
except Exception, e:
# Hard to report these more naturally here. These happen either
# because of a bug in the piggy-back code or because of a
# race condition.
logger.exception("Exception while processing piggyback data")
ret["result_error"] = True
return render_with_toolbars(template, request, ret)
def upload(request):
"""
Handles file uploads.
Django has a hook for "upload handlers" that could make this more
efficient. Currently Django will store files greater than 2.5MB in
a temp directory. We could implement an upload handler to
send data right through to the destination.
http://docs.djangoproject.com/en/1.0/topics/http/file-uploads/#upload-handlers
"""
if request.method == 'POST':
form = UploadForm(request.POST, request.FILES)
if form.is_valid():
# Bit of a wart that form.file doesn't give you the file,
# and you have to do form.files.get("file").
file = form.files.get("file")
dest = form.cleaned_data["dest"]
if request.fs.isdir(dest):
assert posixpath.sep not in file.name
dest = posixpath.join(dest, file.name)
output = request.fs.open(dest, "w")
try:
for chunk in file.chunks():
output.write(chunk)
finally:
output.close()
dest_stats = request.fs.stats(dest)
return render_with_toolbars('upload_done.mako', request, {
# status is used by "fancy uploader"
'status': 1,
'path': dest,
'result': _massage_stats(request, dest_stats),
'next': request.GET.get("next")
})
else:
dest = request.GET.get("dest")
initial_values = {}
if dest:
initial_values["dest"] = dest
form = UploadForm(initial=initial_values)
return render_with_toolbars('upload.mako', request,
{'form': form, 'next': request.REQUEST.get("dest")})
def status(request):
status = request.fs.status()
data = {
# Beware: "messages" is special in the context browser.
'msgs': status.get_messages(),
'health': status.get_health(),
'datanode_report': status.get_datanode_report(),
'name': request.fs.name
}
return render_with_toolbars("status.mako", request, data)
def status(request):
status = request.fs.status()
data = {
# Beware: "messages" is special in the context browser.
'msgs': status.get_messages(),
'health': status.get_health(),
'datanode_report': status.get_datanode_report(),
'name': request.fs.name
}
return render_with_toolbars("status.mako", request, data)
def generic_op(form_class, request, op, parameter_names, piggyback=None, template="fileop.mako", extra_params=None):
"""
Generic implementation for several operations.
@param form_class form to instantiate
@param request incoming request, used for parameters
@param op callable with the filesystem operation
@param parameter_names list of form parameters that are extracted and then passed to op
@param piggyback list of form parameters whose file stats to look up after the operation
@param extra_params dictionary of extra parameters to send to the template for rendering
"""
# Use next for non-ajax requests, when available.
next = request.GET.get("next")
if next is None:
next = request.POST.get("next")
ret = dict({
'next':next
})
if extra_params is not None:
ret['extra_params'] = extra_params
for p in parameter_names:
val = request.REQUEST.get(p)
if val:
ret[p] = val
if request.method == 'POST':
form = form_class(request.POST)
# TODO(philip): How best to do error handling? fs will throw
# an arbitrary-ish exception (typically file not found or maybe permission
# denied), and this needs to be coaxed into an HTTP error.
ret['form'] = form
if form.is_valid():
args = [ form.cleaned_data[p] for p in parameter_names ]
op(*args)
if next:
logging.debug("Next: %s" % next)
# Doesn't need to be quoted: quoting is done by HttpResponseRedirect.
return format_preserving_redirect(request, next)
ret["success"] = True
try:
if piggyback:
piggy_path = form.cleaned_data[piggyback]
ret["result"] = _massage_stats(request, request.fs.stats(piggy_path))
except Exception, e:
# Hard to report these more naturally here. These happen either
# because of a bug in the piggy-back code or because of a
# race condition.
logger.exception("Exception while processing piggyback data")
ret["result_error"] = True
return render_with_toolbars(template, request, ret)
def _upload(request):
"""
Handles file uploads. The uploaded file is stored in HDFS. We just
need to rename it to the right path.
"""
if request.method == 'POST':
form = UploadForm(request.POST, request.FILES)
if not form.is_valid():
logger.error("Error in upload form: %s" % (form.errors, ))
else:
uploaded_file = request.FILES['hdfs_file']
dest = form.cleaned_data["dest"]
if request.fs.isdir(dest):
assert posixpath.sep not in uploaded_file.name
dest = request.fs.join(dest, uploaded_file.name)
# Temp file is created by superuser. Chown the file.
tmp_file = uploaded_file.get_temp_path()
username = request.user.username
try:
try:
request.fs.setuser(request.fs.superuser)
request.fs.chmod(tmp_file, 0644)
request.fs.chown(tmp_file, username, username)
except IOError, ex:
msg = 'Failed to chown uploaded file ("%s") as superuser %s' % \
(tmp_file, request.fs.superuser)
logger.exception(msg)
raise PopupException(msg, detail=str(ex))
finally:
request.fs.setuser(username)
# Move the file to where it belongs
try:
request.fs.rename(uploaded_file.get_temp_path(), dest)
except IOError, ex:
raise PopupException(
'Failed to rename uploaded temporary file ("%s") to "%s": %s'
% (tmp_file, dest, ex))
dest_stats = request.fs.stats(dest)
return render_with_toolbars(
'upload_done.mako',
request,
{
# status is used by "fancy uploader"
'status': 1,
'path': dest,
'result': _massage_stats(request, dest_stats),
'next': request.GET.get("next")
})
def _upload(request):
"""
Handles file uploads. The uploaded file is stored in HDFS. We just
need to rename it to the right path.
"""
if request.method == 'POST':
form = UploadForm(request.POST, request.FILES)
if not form.is_valid():
logger.error("Error in upload form: %s" % (form.errors,))
else:
uploaded_file = request.FILES['hdfs_file']
dest = form.cleaned_data["dest"]
if request.fs.isdir(dest):
assert posixpath.sep not in uploaded_file.name
dest = request.fs.join(dest, uploaded_file.name)
# Temp file is created by superuser. Chown the file.
tmp_file = uploaded_file.get_temp_path()
username = request.user.username
try:
try:
request.fs.setuser(request.fs.superuser)
request.fs.chmod(tmp_file, 0644)
request.fs.chown(tmp_file, username, username)
except IOError, ex:
msg = 'Failed to chown uploaded file ("%s") as superuser %s' % \
(tmp_file, request.fs.superuser)
logger.exception(msg)
raise PopupException(msg, detail=str(ex))
finally:
request.fs.setuser(username)
# Move the file to where it belongs
try:
request.fs.rename(uploaded_file.get_temp_path(), dest)
except IOError, ex:
raise PopupException(
'Failed to rename uploaded temporary file ("%s") to "%s": %s' %
(tmp_file, dest, ex))
dest_stats = request.fs.stats(dest)
return render_with_toolbars('upload_done.mako', request, {
# status is used by "fancy uploader"
'status': 1,
'path': dest,
'result': _massage_stats(request, dest_stats),
'next': request.GET.get("next")
})
def listdir(request, path):
"""
Implements directory listing (or index).
Intended to be called via view().
"""
path = _unquote_path(path)
if not request.fs.isdir(path):
raise PopupException("Not a directory: %s" % (path, ))
file_filter = request.REQUEST.get('file_filter', 'any')
assert file_filter in ['any', 'file', 'dir']
home_dir_path = request.user.get_home_directory()
data = {
'path':
path,
'file_filter':
file_filter,
# These could also be put in automatically via
# http://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-request,
# but manually seems cleaner, since we only need it here.
'current_request_path':
request.path,
'home_directory':
request.fs.isdir(home_dir_path) and home_dir_path or None,
'cwd_set':
True,
'show_upload':
(request.REQUEST.get('show_upload') == 'false' and (False, )
or (True, ))[0]
}
stats = request.fs.listdir_stats(path)
# Include parent dir, unless at filesystem root.
if normpath(path) != posixpath.sep:
parent_stat = request.fs.stats(posixpath.join(path, ".."))
# The 'path' field would be absolute, but we want its basename to be
# actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
parent_stat['path'] = posixpath.join(path, "..")
stats.insert(0, parent_stat)
data['files'] = [_massage_stats(request, stat) for stat in stats]
return render_with_toolbars('listdir.mako', request, data)
def listdir(request, path):
"""
Implements directory listing (or index).
Intended to be called via view().
"""
path = _unquote_path(path)
if not request.fs.isdir(path):
raise PopupException("Not a directory: %s" % (path,))
file_filter = request.REQUEST.get('file_filter', 'any')
assert file_filter in ['any', 'file', 'dir']
home_dir_path = request.user.get_home_directory()
data = {
'path': path,
'file_filter': file_filter,
# These could also be put in automatically via
# http://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-request,
# but manually seems cleaner, since we only need it here.
'current_request_path': request.path,
'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None,
'cwd_set': True,
'show_upload': (request.REQUEST.get('show_upload') == 'false' and (False,) or (True,))[0]
}
stats = request.fs.listdir_stats(path)
# Include parent dir, unless at filesystem root.
if normpath(path) != posixpath.sep:
parent_stat = request.fs.stats(posixpath.join(path, ".."))
# The 'path' field would be absolute, but we want its basename to be
# actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
parent_stat['path'] = posixpath.join(path, "..")
stats.insert(0, parent_stat)
data['files'] = [_massage_stats(request, stat) for stat in stats]
return render_with_toolbars('listdir.mako', request, data)
def display(request, path):
"""
Implements displaying part of a file.
GET arguments are length, offset, mode, compression and encoding
with reasonable defaults chosen.
Note that display by length and offset are on bytes, not on characters.
TODO(philip): Could easily built-in file type detection
(perhaps using something similar to file(1)), as well
as more advanced binary-file viewing capability (de-serialize
sequence files, decompress gzipped text files, etc.).
There exists a python-magic package to interface with libmagic.
"""
path = _unquote_path(path)
if not request.fs.isfile(path):
raise PopupException("Not a file: '%s'" % (path,))
stats = request.fs.stats(path)
encoding = request.GET.get('encoding') or i18n.get_site_encoding()
# I'm mixing URL-based parameters and traditional
# HTTP GET parameters, since URL-based parameters
# can't naturally be optional.
# Need to deal with possibility that length is not present
# because the offset came in via the toolbar manual byte entry.
end = request.GET.get("end")
if end:
end = int(end)
begin = request.GET.get("begin", 1)
if begin:
# Subtract one to zero index for file read
begin = int(begin) - 1
if end:
offset = begin
length = end - begin
if begin >= end:
raise PopupException("First byte to display must be before last byte to display.")
else:
length = int(request.GET.get("length", DEFAULT_CHUNK_SIZE_BYTES))
# Display first block by default.
offset = int(request.GET.get("offset", 0))
mode = request.GET.get("mode")
compression = request.GET.get("compression")
if mode and mode not in ["binary", "text"]:
raise PopupException("Mode must be one of 'binary' or 'text'.")
if offset < 0:
raise PopupException("Offset may not be less than zero.")
if length < 0:
raise PopupException("Length may not be less than zero.")
if length > MAX_CHUNK_SIZE_BYTES:
raise PopupException("Cannot request chunks greater than %d bytes" % MAX_CHUNK_SIZE_BYTES)
# Auto gzip detection, unless we are explicitly told to view binary
if not compression and mode != 'binary':
if path.endswith('.gz') and detect_gzip(request.fs.open(path).read(2)):
compression = 'gzip'
offset = 0
else:
compression = 'none'
f = request.fs.open(path)
if compression == 'gzip':
if offset and offset != 0:
raise PopupException("We don't support offset and gzip Compression")
try:
try:
contents = GzipFile('', 'r', 0, StringIO(f.read())).read(length)
except:
logging.warn("Could not decompress file at %s" % path, exc_info=True)
contents = ''
raise PopupException("Failed to decompress file")
finally:
f.close()
else:
try:
f.seek(offset)
contents = f.read(length)
finally:
f.close()
# Get contents as string for text mode, or at least try
uni_contents = None
if not mode or mode == 'text':
uni_contents = unicode(contents, encoding, errors='replace')
is_binary = uni_contents.find(i18n.REPLACEMENT_CHAR) != -1
# Auto-detect mode
if not mode:
mode = is_binary and 'binary' or 'text'
# Get contents as bytes
if mode == "binary":
xxd_out = list(xxd.xxd(offset, contents, BYTES_PER_LINE, BYTES_PER_SENTENCE))
dirname = posixpath.dirname(path)
# Start with index-like data:
data = _massage_stats(request, request.fs.stats(path))
# And add a view structure:
data["success"] = True
data["view"] = {
'offset': offset,
'length': length,
'end': offset + len(contents),
'dirname': dirname,
'mode': mode,
'compression': compression,
'size': stats['size']
}
data["filename"] = os.path.basename(path)
data["editable"] = stats['size'] < MAX_FILEEDITOR_SIZE
if mode == "binary":
# This might be the wrong thing for ?format=json; doing the
# xxd'ing in javascript might be more compact, or sending a less
# intermediate representation...
logger.debug("xxd: " + str(xxd_out))
data['view']['xxd'] = xxd_out
data['view']['masked_binary_data'] = False
else:
data['view']['contents'] = uni_contents
data['view']['masked_binary_data'] = is_binary
return render_with_toolbars("display.mako", request, data)
except UnicodeDecodeError:
raise PopupException("File is not encoded in %s; cannot be edited: %s" % (encoding, path))
finally:
f.close()
else:
current_contents = u""
form = EditorForm(dict(path=path, contents=current_contents, encoding=encoding))
data = dict(
exists=(stats is not None),
form=form,
path=path,
filename=os.path.basename(path),
dirname=os.path.dirname(path))
return render_with_toolbars("edit.mako", request, data)
def save_file(request):
"""
The POST endpoint to save a file in the file editor.
Does the save and then redirects back to the edit page.
"""
form = EditorForm(request.POST)
is_valid = form.is_valid()
path = form.cleaned_data.get('path')
if request.POST.get('save') == "saveAs":
if not is_valid:
return edit(request, path, form=form)
else:
except UnicodeDecodeError:
raise PopupException("File is not encoded in %s; cannot be edited: %s" % (encoding, path))
finally:
f.close()
else:
current_contents = u""
form = EditorForm(dict(path=path, contents=current_contents, encoding=encoding))
data = dict(
exists=(stats is not None),
form=form,
path=path,
filename=os.path.basename(path),
dirname=os.path.dirname(path))
return render_with_toolbars("edit.mako", request, data)
def save_file(request):
"""
The POST endpoint to save a file in the file editor.
Does the save and then redirects back to the edit page.
"""
form = EditorForm(request.POST)
is_valid = form.is_valid()
path = form.cleaned_data.get('path')
if request.POST.get('save') == "saveAs":
if not is_valid:
return edit(request, path, form=form)
else:
def display(request, path):
"""
Implements displaying part of a file.
GET arguments are length, offset, mode, compression and encoding
with reasonable defaults chosen.
Note that display by length and offset are on bytes, not on characters.
TODO(philip): Could easily built-in file type detection
(perhaps using something similar to file(1)), as well
as more advanced binary-file viewing capability (de-serialize
sequence files, decompress gzipped text files, etc.).
There exists a python-magic package to interface with libmagic.
"""
path = _unquote_path(path)
if not request.fs.isfile(path):
raise PopupException("Not a file: '%s'" % (path, ))
stats = request.fs.stats(path)
encoding = request.GET.get('encoding') or i18n.get_site_encoding()
# I'm mixing URL-based parameters and traditional
# HTTP GET parameters, since URL-based parameters
# can't naturally be optional.
# Need to deal with possibility that length is not present
# because the offset came in via the toolbar manual byte entry.
end = request.GET.get("end")
if end:
end = int(end)
begin = request.GET.get("begin", 1)
if begin:
# Subtract one to zero index for file read
begin = int(begin) - 1
if end:
offset = begin
length = end - begin
if begin >= end:
raise PopupException(
"First byte to display must be before last byte to display.")
else:
length = int(request.GET.get("length", DEFAULT_CHUNK_SIZE_BYTES))
# Display first block by default.
offset = int(request.GET.get("offset", 0))
mode = request.GET.get("mode")
compression = request.GET.get("compression")
if mode and mode not in ["binary", "text"]:
raise PopupException("Mode must be one of 'binary' or 'text'.")
if offset < 0:
raise PopupException("Offset may not be less than zero.")
if length < 0:
raise PopupException("Length may not be less than zero.")
if length > MAX_CHUNK_SIZE_BYTES:
raise PopupException("Cannot request chunks greater than %d bytes" %
MAX_CHUNK_SIZE_BYTES)
# Auto gzip detection, unless we are explicitly told to view binary
if not compression and mode != 'binary':
if path.endswith('.gz') and detect_gzip(request.fs.open(path).read(2)):
compression = 'gzip'
offset = 0
else:
compression = 'none'
f = request.fs.open(path)
if compression == 'gzip':
if offset and offset != 0:
raise PopupException(
"We don't support offset and gzip Compression")
try:
try:
contents = GzipFile('', 'r', 0,
StringIO(f.read())).read(length)
except:
logging.warn("Could not decompress file at %s" % path,
exc_info=True)
contents = ''
raise PopupException("Failed to decompress file")
finally:
f.close()
else:
try:
f.seek(offset)
contents = f.read(length)
finally:
f.close()
# Get contents as string for text mode, or at least try
uni_contents = None
if not mode or mode == 'text':
uni_contents = unicode(contents, encoding, errors='replace')
is_binary = uni_contents.find(i18n.REPLACEMENT_CHAR) != -1
# Auto-detect mode
if not mode:
mode = is_binary and 'binary' or 'text'
# Get contents as bytes
if mode == "binary":
xxd_out = list(
xxd.xxd(offset, contents, BYTES_PER_LINE, BYTES_PER_SENTENCE))
dirname = posixpath.dirname(path)
# Start with index-like data:
data = _massage_stats(request, request.fs.stats(path))
# And add a view structure:
data["success"] = True
data["view"] = {
'offset': offset,
'length': length,
'end': offset + len(contents),
'dirname': dirname,
'mode': mode,
'compression': compression,
'size': stats['size']
}
data["filename"] = os.path.basename(path)
data["editable"] = stats['size'] < MAX_FILEEDITOR_SIZE
if mode == "binary":
# This might be the wrong thing for ?format=json; doing the
# xxd'ing in javascript might be more compact, or sending a less
# intermediate representation...
logger.debug("xxd: " + str(xxd_out))
data['view']['xxd'] = xxd_out
data['view']['masked_binary_data'] = False
else:
data['view']['contents'] = uni_contents
data['view']['masked_binary_data'] = is_binary
return render_with_toolbars("display.mako", request, data)
"File is not encoded in %s; cannot be edited: %s" %
(encoding, path))
finally:
f.close()
else:
current_contents = u""
form = EditorForm(
dict(path=path, contents=current_contents, encoding=encoding))
data = dict(exists=(stats is not None),
form=form,
path=path,
filename=os.path.basename(path),
dirname=os.path.dirname(path))
return render_with_toolbars("edit.mako", request, data)
def save_file(request):
"""
The POST endpoint to save a file in the file editor.
Does the save and then redirects back to the edit page.
"""
form = EditorForm(request.POST)
is_valid = form.is_valid()
path = form.cleaned_data.get('path')
if request.POST.get('save') == "saveAs":
if not is_valid:
return edit(request, path, form=form)
