def rename(self, old, new):
"""rename(old, new)"""
old = Hdfs.normpath(old)
if not new.startswith("/"):
new = Hdfs.join(Hdfs.dirname(old), new)
new = Hdfs.normpath(new)
params = self._getparams()
params["op"] = "RENAME"
# Encode `new' because it's in the params
params["destination"] = smart_str(new)
result = self._root.put(old, params)
if not result["boolean"]:
raise IOError("Rename failed: %s -> %s" % (smart_str(old), smart_str(new)))
def rename(self, old, new):
"""rename(old, new)"""
old = Hdfs.normpath(old)
if not new.startswith('/'):
new = Hdfs.join(Hdfs.dirname(old), new)
new = Hdfs.normpath(new)
params = self._getparams()
params['op'] = 'RENAME'
# Encode `new' because it's in the params
params['destination'] = smart_str(new)
result = self._root.put(old, params)
if not result['boolean']:
raise IOError(_("Rename failed: %s -> %s") %
(str(smart_str(old)), str(smart_str(new))))
def get_content_summary(self, path): """ get_content_summary(path) -> WebHdfsContentSummary """ path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'GETCONTENTSUMMARY' json = self._root.get(path, params) return WebHdfsContentSummary(json['ContentSummary'])
def get_content_summary(self, path):
"""
get_content_summary(path) -> WebHdfsContentSummary
"""
path = Hdfs.normpath(path)
params = self._getparams()
params["op"] = "GETCONTENTSUMMARY"
json = self._root.get(path, params)
return WebHdfsContentSummary(json["ContentSummary"])
def append(self, path, data):
"""
append(path, data)
Append data to a given file.
"""
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'APPEND'
self._invoke_with_redirect('POST', path, params, data)
def parse_breadcrumbs(path):
breadcrumbs_parts = Hdfs.normpath(path).split("/")
i = 1
breadcrumbs = [{"url": "", "label": "/"}]
while i < len(breadcrumbs_parts):
breadcrumb_url = breadcrumbs[i - 1]["url"] + "/" + breadcrumbs_parts[i]
if breadcrumb_url != "/":
breadcrumbs.append({"url": breadcrumb_url, "label": breadcrumbs_parts[i]})
i = i + 1
return breadcrumbs
def chown(self, path, user=None, group=None):
"""chown(path, user=None, group=None)"""
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'SETOWNER'
if user is not None:
params['owner'] = user
if group is not None:
params['group'] = group
self._root.put(path, params)
def append(self, path, data):
"""
append(path, data)
Append data to a given file.
"""
path = Hdfs.normpath(path)
params = self._getparams()
params["op"] = "APPEND"
self._invoke_with_redirect("POST", path, params, data)
def parse_breadcrumbs(path):
breadcrumbs_parts = Hdfs.normpath(path).split('/')
i = 1
breadcrumbs = [{'url': '', 'label': '/'}]
while (i < len(breadcrumbs_parts)):
breadcrumb_url = breadcrumbs[i - 1]['url'] + '/' + breadcrumbs_parts[i]
if breadcrumb_url != '/':
breadcrumbs.append({'url': breadcrumb_url, 'label': breadcrumbs_parts[i]})
i = i + 1
return breadcrumbs
def create_directory(request):
parent_path = json.loads(request.POST.get("parent_path"))
name = json.loads(request.POST.get("name"))
parent_dir = Directory.objects.get(owner=request.user, name=parent_path)
path = Hdfs.normpath(parent_path + "/" + name)
file_doc = Directory.objects.create(name=path, type="directory", owner=request.user)
parent_dir.dependencies.add(file_doc)
return JsonResponse({"status": 0, "file": file_doc.to_dict()})
def chmod(self, path, mode):
"""
chmod(path, mode)
`mode' should be an octal integer or string.
"""
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'SETPERMISSION'
params['permission'] = safe_octal(mode)
self._root.put(path, params)
def _stats(self, path):
"""This version of stats returns None if the entry is not found"""
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'GETFILESTATUS'
try:
json = self._root.get(path, params)
return WebHdfsStat(json['FileStatus'], path)
except WebHdfsException, ex:
if ex.server_exc == 'FileNotFoundException' or ex.code == 404:
return None
raise ex
def listdir(request, path, chooser):
"""
Implements directory listing (or index).
Intended to be called via view().
TODO: Remove?
"""
if not request.fs.isdir(path):
raise PopupException(_("Not a directory: %(path)s") % {'path': path})
file_filter = request.REQUEST.get('file_filter', 'any')
assert file_filter in ['any', 'file', 'dir']
home_dir_path = request.user.get_home_directory()
breadcrumbs = parse_breadcrumbs(path)
data = {
'path': path,
'file_filter': file_filter,
'breadcrumbs': breadcrumbs,
'current_dir_path': path,
# These could also be put in automatically via
# http://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-request,
# but manually seems cleaner, since we only need it here.
'current_request_path': request.path,
'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None,
'cwd_set': True,
'is_superuser': request.user.username == request.fs.superuser,
'groups': request.user.username == request.fs.superuser and [str(x) for x in Group.objects.values_list('name', flat=True)] or [],
'users': request.user.username == request.fs.superuser and [str(x) for x in User.objects.values_list('username', flat=True)] or [],
'superuser': request.fs.superuser,
'show_upload': (request.REQUEST.get('show_upload') == 'false' and (False,) or (True,))[0]
}
stats = request.fs.listdir_stats(path)
# Include parent dir, unless at filesystem root.
if Hdfs.normpath(path) != posixpath.sep:
parent_path = request.fs.join(path, "..")
parent_stat = request.fs.stats(parent_path)
# The 'path' field would be absolute, but we want its basename to be
# actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
parent_stat['path'] = parent_path
stats.insert(0, parent_stat)
data['files'] = [_massage_stats(request, stat) for stat in stats]
if chooser:
return render('chooser.mako', request, data)
else:
return render('listdir.mako', request, data)
def mkdir(self, path, mode=None):
"""
mkdir(path, mode=None)
Creates a directory and any parent directory if necessary.
"""
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'MKDIRS'
if mode is not None:
params['permission'] = safe_octal(mode)
success = self._root.put(path, params)
if not success:
raise IOError(_("Mkdir failed: %s") % path)
def create_directory(request):
parent_path = json.loads(request.POST.get('parent_path'))
name = json.loads(request.POST.get('name'))
parent_dir = Directory.objects.get(owner=request.user, name=parent_path)
path = Hdfs.normpath(parent_path + '/' + name)
file_doc = Directory.objects.create(name=path, owner=request.user)
parent_dir.dependencies.add(file_doc)
return JsonResponse({
'status': 0,
'file': file_doc.to_dict()
})
def check_access(self, path, aclspec='rw-'):
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'CHECKACCESS'
params['fsaction'] = aclspec
try:
return self._root.get(path, params)
except WebHdfsException, ex:
if ex.code == 500 or ex.code == 400:
LOG.warn('Failed to check access to path %s, CHECKACCESS operation may not be supported.' % path)
return None
else:
raise ex
def chown(self, path, user=None, group=None, recursive=False):
"""chown(path, user=None, group=None, recursive=False)"""
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'SETOWNER'
if user is not None:
params['owner'] = user
if group is not None:
params['group'] = group
if recursive:
for xpath in self.listdir_recursive(path):
self._root.put(xpath, params)
else:
self._root.put(path, params)
def listdir_stats(self, path, glob=None):
"""
listdir_stats(path, glob=None) -> [ WebHdfsStat ]
Get directory listing with stats.
"""
path = Hdfs.normpath(path)
params = self._getparams()
if glob is not None:
params["filter"] = glob
params["op"] = "LISTSTATUS"
json = self._root.get(path, params)
filestatus_list = json["FileStatuses"]["FileStatus"]
return [WebHdfsStat(st, path) for st in filestatus_list]
def chown(self, path, user=None, group=None, recursive=False):
"""chown(path, user=None, group=None, recursive=False)"""
path = Hdfs.normpath(path)
params = self._getparams()
params["op"] = "SETOWNER"
if user is not None:
params["owner"] = user
if group is not None:
params["group"] = group
if recursive:
for xpath in self._listdir_r(path):
self._root.put(xpath, params)
else:
self._root.put(path, params)
def listdir_stats(self, path, glob=None):
"""
listdir_stats(path, glob=None) -> [ WebHdfsStat ]
Get directory listing with stats.
"""
path = Hdfs.normpath(path)
params = self._getparams()
if glob is not None:
params['filter'] = glob
params['op'] = 'LISTSTATUS'
json = self._root.get(path, params)
filestatus_list = json['FileStatuses']['FileStatus']
return [ WebHdfsStat(st, path) for st in filestatus_list ]
def mkdir(self, path, mode=None):
"""
mkdir(path, mode=None)
Creates a directory and any parent directory if necessary.
"""
path = Hdfs.normpath(path)
params = self._getparams()
params["op"] = "MKDIRS"
if mode is not None:
params["permission"] = safe_octal(mode)
success = self._root.put(path, params)
if not success:
raise IOError("Mkdir failed: %s" % (smart_str(path),))
def chmod(self, path, mode, recursive=False):
"""
chmod(path, mode, recursive=False)
`mode' should be an octal integer or string.
"""
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'SETPERMISSION'
params['permission'] = safe_octal(mode)
if recursive:
for xpath in self.listdir_recursive(path):
self._root.put(xpath, params)
else:
self._root.put(path, params)
def _massage_stats(request, stats):
"""
Massage a stats record as returned by the filesystem implementation
into the format that the views would like it in.
"""
path = stats["path"]
normalized = Hdfs.normpath(path)
return {
"path": normalized,
"name": posixpath.basename(path),
"stats": stats.to_json_dict(),
"type": filetype(stats["mode"]),
"rwx": rwx(stats["mode"]),
"url": make_absolute(request, "view", dict(path=urlquote(normalized))),
}
def _massage_stats(request, stats):
"""
Massage a stats record as returned by the filesystem implementation
into the format that the views would like it in.
"""
path = stats['path']
normalized = Hdfs.normpath(path)
return {
'path': normalized,
'name': stats['name'],
'stats': stats.to_json_dict(),
'type': filetype(stats['mode']),
'rwx': rwx(stats['mode']),
'url': make_absolute(request, "view", dict(path=urlquote(normalized))),
}
def _delete(self, path, recursive=False):
"""
_delete(path, recursive=False)
Delete a file or directory.
"""
path = Hdfs.normpath(path)
params = self._getparams()
params["op"] = "DELETE"
params["recursive"] = recursive and "true" or "false"
result = self._root.delete(path, params)
# This part of the API is nonsense.
# The lack of exception should indicate success.
if not result["boolean"]:
raise IOError("Delete failed: %s" % (smart_str(path),))
def _delete(self, path, recursive=False):
"""
_delete(path, recursive=False)
Delete a file or directory.
"""
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'DELETE'
params['recursive'] = recursive and 'true' or 'false'
result = self._root.delete(path, params)
# This part of the API is nonsense.
# The lack of exception should indicate success.
if not result['boolean']:
raise IOError(_('Delete failed: %s') % path)
def listdir(request, path, chooser):
"""
Implements directory listing (or index).
Intended to be called via view().
"""
if not request.fs.isdir(path):
raise PopupException(_("Not a directory: %(path)s") % {"path": path})
file_filter = request.REQUEST.get("file_filter", "any")
assert file_filter in ["any", "file", "dir"]
home_dir_path = request.user.get_home_directory()
breadcrumbs = parse_breadcrumbs(path)
data = {
"path": path,
"file_filter": file_filter,
"breadcrumbs": breadcrumbs,
"current_dir_path": path,
# These could also be put in automatically via
# http://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-request,
# but manually seems cleaner, since we only need it here.
"current_request_path": request.path,
"home_directory": request.fs.isdir(home_dir_path) and home_dir_path or None,
"cwd_set": True,
"show_upload": (request.REQUEST.get("show_upload") == "false" and (False,) or (True,))[0],
}
stats = request.fs.listdir_stats(path)
# Include parent dir, unless at filesystem root.
if Hdfs.normpath(path) != posixpath.sep:
parent_path = request.fs.join(path, "..")
parent_stat = request.fs.stats(parent_path)
# The 'path' field would be absolute, but we want its basename to be
# actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
parent_stat["path"] = parent_path
stats.insert(0, parent_stat)
data["files"] = [_massage_stats(request, stat) for stat in stats]
if chooser:
return render("chooser.mako", request, data)
else:
return render("listdir.mako", request, data)
def _massage_stats(request, stats):
"""
Massage a stats record as returned by the filesystem implementation
into the format that the views would like it in.
"""
path = stats['path']
normalized = Hdfs.normpath(path)
return {
'path': normalized,
'name': stats['name'],
'stats': stats.to_json_dict(),
'mtime': datetime.fromtimestamp(stats['mtime']).strftime('%B %d, %Y %I:%M %p'),
'humansize': filesizeformat(stats['size']),
'type': filetype(stats['mode']),
'rwx': rwx(stats['mode']),
'mode': stringformat(stats['mode'], "o"),
'url': make_absolute(request, "view", dict(path=urlquote(normalized))),
}
def read(self, path, offset, length, bufsize=None):
"""
read(path, offset, length[, bufsize]) -> data
Read data from a file.
"""
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'OPEN'
params['offset'] = long(offset)
params['length'] = long(length)
if bufsize is not None:
params['bufsize'] = bufsize
try:
return self._root.get(path, params)
except WebHdfsException, ex:
if "out of the range" in ex.message:
return ""
raise ex
def create(self, path, overwrite=False, blocksize=None, replication=None, permission=None, data=None):
"""
create(path, overwrite=False, blocksize=None, replication=None, permission=None)
Creates a file with the specified parameters.
`permission' should be an octal integer or string.
"""
path = Hdfs.normpath(path)
params = self._getparams()
params["op"] = "CREATE"
params["overwrite"] = overwrite and "true" or "false"
if blocksize is not None:
params["blocksize"] = long(blocksize)
if replication is not None:
params["replication"] = int(replication)
if permission is not None:
params["permission"] = safe_octal(permission)
self._invoke_with_redirect("PUT", path, params, data)
def strip_normpath(self, path):
split = urlparse(path)
path = split._replace(scheme="", netloc="").geturl()
return Hdfs.normpath(path)
def remove_acl_entries(self, path, aclspec):
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'REMOVEACLENTRIES'
params['aclspec'] = aclspec
return self._root.put(path, params)
def remove_acl(self, path):
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'REMOVEACL'
return self._root.put(path, params)
def set_acl(self, path, aclspec):
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'SETACL'
params['aclspec'] = aclspec
return self._root.put(path, params)
def get_acl_status(self, path):
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'GETACLSTATUS'
return self._root.get(path, params)
def listdir_paged(request, path):
"""
A paginated version of listdir.
Query parameters:
pagenum - The page number to show. Defaults to 1.
pagesize - How many to show on a page. Defaults to 15.
sortby=? - Specify attribute to sort by. Accepts:
(type, name, atime, mtime, size, user, group)
Defaults to name.
descending - Specify a descending sort order.
Default to false.
filter=? - Specify a substring filter to search for in
the filename field.
"""
if not request.fs.isdir(path):
raise PopupException("Not a directory: %s" % (path,))
pagenum = int(request.GET.get('pagenum', 1))
pagesize = int(request.GET.get('pagesize', 30))
do_as = None
if request.user.is_superuser or request.user.has_hue_permission(action="impersonate", app="security"):
do_as = request.GET.get('doas', request.user.username)
if hasattr(request, 'doas'):
do_as = request.doas
home_dir_path = request.user.get_home_directory()
breadcrumbs = parse_breadcrumbs(path)
if do_as:
all_stats = request.fs.do_as_user(do_as, request.fs.listdir_stats, path)
else:
all_stats = request.fs.listdir_stats(path)
# Filter first
filter_str = request.GET.get('filter', None)
if filter_str:
filtered_stats = filter(lambda sb: filter_str in sb['name'], all_stats)
all_stats = filtered_stats
# Sort next
sortby = request.GET.get('sortby', None)
descending_param = request.GET.get('descending', None)
if sortby is not None:
if sortby not in ('type', 'name', 'atime', 'mtime', 'user', 'group', 'size'):
logger.info("Invalid sort attribute '%s' for listdir." %
(sortby,))
else:
all_stats = sorted(all_stats,
key=operator.attrgetter(sortby),
reverse=coerce_bool(descending_param))
# Do pagination
page = paginator.Paginator(all_stats, pagesize).page(pagenum)
shown_stats = page.object_list
# Include parent dir always as second option, unless at filesystem root.
if Hdfs.normpath(path) != posixpath.sep:
parent_path = request.fs.join(path, "..")
parent_stat = request.fs.stats(parent_path)
# The 'path' field would be absolute, but we want its basename to be
# actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
parent_stat['path'] = parent_path
parent_stat['name'] = ".."
shown_stats.insert(0, parent_stat)
# Include same dir always as first option to see stats of the current folder
current_stat = request.fs.stats(path)
# The 'path' field would be absolute, but we want its basename to be
# actually '.' for display purposes. Encode it since _massage_stats expects byte strings.
current_stat['path'] = path
current_stat['name'] = "."
shown_stats.insert(1, current_stat)
page.object_list = [ _massage_stats(request, s) for s in shown_stats ]
is_fs_superuser = _is_hdfs_superuser(request)
data = {
'path': path,
'breadcrumbs': breadcrumbs,
'current_request_path': request.path,
'files': page.object_list,
'page': _massage_page(page),
'pagesize': pagesize,
'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None,
'sortby': sortby,
'descending': descending_param,
# The following should probably be deprecated
'cwd_set': True,
'file_filter': 'any',
'current_dir_path': path,
'is_fs_superuser': is_fs_superuser,
'groups': is_fs_superuser and [str(x) for x in Group.objects.values_list('name', flat=True)] or [],
'users': is_fs_superuser and [str(x) for x in User.objects.values_list('username', flat=True)] or [],
'superuser': request.fs.superuser,
'supergroup': request.fs.supergroup,
'is_sentry_managed': request.fs.is_sentry_managed(path)
}
return render('listdir.mako', request, data)
def modify_acl_entries(self, path, aclspec):
path = Hdfs.normpath(path)
params = self._getparams()
params['op'] = 'MODIFYACLENTRIES'
params['aclspec'] = aclspec
return self._root.put(path, params)
