def rename(self, old, new): """rename(old, new)""" old = Hdfs.normpath(old) if not new.startswith("/"): new = Hdfs.join(Hdfs.dirname(old), new) new = Hdfs.normpath(new) params = self._getparams() params["op"] = "RENAME" # Encode `new' because it's in the params params["destination"] = smart_str(new) result = self._root.put(old, params) if not result["boolean"]: raise IOError("Rename failed: %s -> %s" % (smart_str(old), smart_str(new)))
def rename(self, old, new): """rename(old, new)""" old = Hdfs.normpath(old) if not new.startswith('/'): new = Hdfs.join(Hdfs.dirname(old), new) new = Hdfs.normpath(new) params = self._getparams() params['op'] = 'RENAME' # Encode `new' because it's in the params params['destination'] = smart_str(new) result = self._root.put(old, params) if not result['boolean']: raise IOError(_("Rename failed: %s -> %s") % (str(smart_str(old)), str(smart_str(new))))
def get_content_summary(self, path): """ get_content_summary(path) -> WebHdfsContentSummary """ path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'GETCONTENTSUMMARY' json = self._root.get(path, params) return WebHdfsContentSummary(json['ContentSummary'])
def get_content_summary(self, path): """ get_content_summary(path) -> WebHdfsContentSummary """ path = Hdfs.normpath(path) params = self._getparams() params["op"] = "GETCONTENTSUMMARY" json = self._root.get(path, params) return WebHdfsContentSummary(json["ContentSummary"])
def append(self, path, data): """ append(path, data) Append data to a given file. """ path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'APPEND' self._invoke_with_redirect('POST', path, params, data)
def parse_breadcrumbs(path): breadcrumbs_parts = Hdfs.normpath(path).split("/") i = 1 breadcrumbs = [{"url": "", "label": "/"}] while i < len(breadcrumbs_parts): breadcrumb_url = breadcrumbs[i - 1]["url"] + "/" + breadcrumbs_parts[i] if breadcrumb_url != "/": breadcrumbs.append({"url": breadcrumb_url, "label": breadcrumbs_parts[i]}) i = i + 1 return breadcrumbs
def chown(self, path, user=None, group=None): """chown(path, user=None, group=None)""" path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'SETOWNER' if user is not None: params['owner'] = user if group is not None: params['group'] = group self._root.put(path, params)
def append(self, path, data): """ append(path, data) Append data to a given file. """ path = Hdfs.normpath(path) params = self._getparams() params["op"] = "APPEND" self._invoke_with_redirect("POST", path, params, data)
def parse_breadcrumbs(path): breadcrumbs_parts = Hdfs.normpath(path).split('/') i = 1 breadcrumbs = [{'url': '', 'label': '/'}] while (i < len(breadcrumbs_parts)): breadcrumb_url = breadcrumbs[i - 1]['url'] + '/' + breadcrumbs_parts[i] if breadcrumb_url != '/': breadcrumbs.append({'url': breadcrumb_url, 'label': breadcrumbs_parts[i]}) i = i + 1 return breadcrumbs
def create_directory(request): parent_path = json.loads(request.POST.get("parent_path")) name = json.loads(request.POST.get("name")) parent_dir = Directory.objects.get(owner=request.user, name=parent_path) path = Hdfs.normpath(parent_path + "/" + name) file_doc = Directory.objects.create(name=path, type="directory", owner=request.user) parent_dir.dependencies.add(file_doc) return JsonResponse({"status": 0, "file": file_doc.to_dict()})
def chmod(self, path, mode): """ chmod(path, mode) `mode' should be an octal integer or string. """ path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'SETPERMISSION' params['permission'] = safe_octal(mode) self._root.put(path, params)
def _stats(self, path): """This version of stats returns None if the entry is not found""" path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'GETFILESTATUS' try: json = self._root.get(path, params) return WebHdfsStat(json['FileStatus'], path) except WebHdfsException, ex: if ex.server_exc == 'FileNotFoundException' or ex.code == 404: return None raise ex
def listdir(request, path, chooser): """ Implements directory listing (or index). Intended to be called via view(). TODO: Remove? """ if not request.fs.isdir(path): raise PopupException(_("Not a directory: %(path)s") % {'path': path}) file_filter = request.REQUEST.get('file_filter', 'any') assert file_filter in ['any', 'file', 'dir'] home_dir_path = request.user.get_home_directory() breadcrumbs = parse_breadcrumbs(path) data = { 'path': path, 'file_filter': file_filter, 'breadcrumbs': breadcrumbs, 'current_dir_path': path, # These could also be put in automatically via # http://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-request, # but manually seems cleaner, since we only need it here. 'current_request_path': request.path, 'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None, 'cwd_set': True, 'is_superuser': request.user.username == request.fs.superuser, 'groups': request.user.username == request.fs.superuser and [str(x) for x in Group.objects.values_list('name', flat=True)] or [], 'users': request.user.username == request.fs.superuser and [str(x) for x in User.objects.values_list('username', flat=True)] or [], 'superuser': request.fs.superuser, 'show_upload': (request.REQUEST.get('show_upload') == 'false' and (False,) or (True,))[0] } stats = request.fs.listdir_stats(path) # Include parent dir, unless at filesystem root. if Hdfs.normpath(path) != posixpath.sep: parent_path = request.fs.join(path, "..") parent_stat = request.fs.stats(parent_path) # The 'path' field would be absolute, but we want its basename to be # actually '..' for display purposes. Encode it since _massage_stats expects byte strings. parent_stat['path'] = parent_path stats.insert(0, parent_stat) data['files'] = [_massage_stats(request, stat) for stat in stats] if chooser: return render('chooser.mako', request, data) else: return render('listdir.mako', request, data)
def mkdir(self, path, mode=None): """ mkdir(path, mode=None) Creates a directory and any parent directory if necessary. """ path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'MKDIRS' if mode is not None: params['permission'] = safe_octal(mode) success = self._root.put(path, params) if not success: raise IOError(_("Mkdir failed: %s") % path)
def create_directory(request): parent_path = json.loads(request.POST.get('parent_path')) name = json.loads(request.POST.get('name')) parent_dir = Directory.objects.get(owner=request.user, name=parent_path) path = Hdfs.normpath(parent_path + '/' + name) file_doc = Directory.objects.create(name=path, owner=request.user) parent_dir.dependencies.add(file_doc) return JsonResponse({ 'status': 0, 'file': file_doc.to_dict() })
def check_access(self, path, aclspec='rw-'): path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'CHECKACCESS' params['fsaction'] = aclspec try: return self._root.get(path, params) except WebHdfsException, ex: if ex.code == 500 or ex.code == 400: LOG.warn('Failed to check access to path %s, CHECKACCESS operation may not be supported.' % path) return None else: raise ex
def chown(self, path, user=None, group=None, recursive=False): """chown(path, user=None, group=None, recursive=False)""" path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'SETOWNER' if user is not None: params['owner'] = user if group is not None: params['group'] = group if recursive: for xpath in self.listdir_recursive(path): self._root.put(xpath, params) else: self._root.put(path, params)
def listdir_stats(self, path, glob=None): """ listdir_stats(path, glob=None) -> [ WebHdfsStat ] Get directory listing with stats. """ path = Hdfs.normpath(path) params = self._getparams() if glob is not None: params["filter"] = glob params["op"] = "LISTSTATUS" json = self._root.get(path, params) filestatus_list = json["FileStatuses"]["FileStatus"] return [WebHdfsStat(st, path) for st in filestatus_list]
def chown(self, path, user=None, group=None, recursive=False): """chown(path, user=None, group=None, recursive=False)""" path = Hdfs.normpath(path) params = self._getparams() params["op"] = "SETOWNER" if user is not None: params["owner"] = user if group is not None: params["group"] = group if recursive: for xpath in self._listdir_r(path): self._root.put(xpath, params) else: self._root.put(path, params)
def listdir_stats(self, path, glob=None): """ listdir_stats(path, glob=None) -> [ WebHdfsStat ] Get directory listing with stats. """ path = Hdfs.normpath(path) params = self._getparams() if glob is not None: params['filter'] = glob params['op'] = 'LISTSTATUS' json = self._root.get(path, params) filestatus_list = json['FileStatuses']['FileStatus'] return [ WebHdfsStat(st, path) for st in filestatus_list ]
def mkdir(self, path, mode=None): """ mkdir(path, mode=None) Creates a directory and any parent directory if necessary. """ path = Hdfs.normpath(path) params = self._getparams() params["op"] = "MKDIRS" if mode is not None: params["permission"] = safe_octal(mode) success = self._root.put(path, params) if not success: raise IOError("Mkdir failed: %s" % (smart_str(path),))
def chmod(self, path, mode, recursive=False): """ chmod(path, mode, recursive=False) `mode' should be an octal integer or string. """ path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'SETPERMISSION' params['permission'] = safe_octal(mode) if recursive: for xpath in self.listdir_recursive(path): self._root.put(xpath, params) else: self._root.put(path, params)
def _massage_stats(request, stats): """ Massage a stats record as returned by the filesystem implementation into the format that the views would like it in. """ path = stats["path"] normalized = Hdfs.normpath(path) return { "path": normalized, "name": posixpath.basename(path), "stats": stats.to_json_dict(), "type": filetype(stats["mode"]), "rwx": rwx(stats["mode"]), "url": make_absolute(request, "view", dict(path=urlquote(normalized))), }
def _massage_stats(request, stats): """ Massage a stats record as returned by the filesystem implementation into the format that the views would like it in. """ path = stats['path'] normalized = Hdfs.normpath(path) return { 'path': normalized, 'name': stats['name'], 'stats': stats.to_json_dict(), 'type': filetype(stats['mode']), 'rwx': rwx(stats['mode']), 'url': make_absolute(request, "view", dict(path=urlquote(normalized))), }
def _delete(self, path, recursive=False): """ _delete(path, recursive=False) Delete a file or directory. """ path = Hdfs.normpath(path) params = self._getparams() params["op"] = "DELETE" params["recursive"] = recursive and "true" or "false" result = self._root.delete(path, params) # This part of the API is nonsense. # The lack of exception should indicate success. if not result["boolean"]: raise IOError("Delete failed: %s" % (smart_str(path),))
def _delete(self, path, recursive=False): """ _delete(path, recursive=False) Delete a file or directory. """ path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'DELETE' params['recursive'] = recursive and 'true' or 'false' result = self._root.delete(path, params) # This part of the API is nonsense. # The lack of exception should indicate success. if not result['boolean']: raise IOError(_('Delete failed: %s') % path)
def listdir(request, path, chooser): """ Implements directory listing (or index). Intended to be called via view(). """ if not request.fs.isdir(path): raise PopupException(_("Not a directory: %(path)s") % {"path": path}) file_filter = request.REQUEST.get("file_filter", "any") assert file_filter in ["any", "file", "dir"] home_dir_path = request.user.get_home_directory() breadcrumbs = parse_breadcrumbs(path) data = { "path": path, "file_filter": file_filter, "breadcrumbs": breadcrumbs, "current_dir_path": path, # These could also be put in automatically via # http://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-request, # but manually seems cleaner, since we only need it here. "current_request_path": request.path, "home_directory": request.fs.isdir(home_dir_path) and home_dir_path or None, "cwd_set": True, "show_upload": (request.REQUEST.get("show_upload") == "false" and (False,) or (True,))[0], } stats = request.fs.listdir_stats(path) # Include parent dir, unless at filesystem root. if Hdfs.normpath(path) != posixpath.sep: parent_path = request.fs.join(path, "..") parent_stat = request.fs.stats(parent_path) # The 'path' field would be absolute, but we want its basename to be # actually '..' for display purposes. Encode it since _massage_stats expects byte strings. parent_stat["path"] = parent_path stats.insert(0, parent_stat) data["files"] = [_massage_stats(request, stat) for stat in stats] if chooser: return render("chooser.mako", request, data) else: return render("listdir.mako", request, data)
def _massage_stats(request, stats): """ Massage a stats record as returned by the filesystem implementation into the format that the views would like it in. """ path = stats['path'] normalized = Hdfs.normpath(path) return { 'path': normalized, 'name': stats['name'], 'stats': stats.to_json_dict(), 'mtime': datetime.fromtimestamp(stats['mtime']).strftime('%B %d, %Y %I:%M %p'), 'humansize': filesizeformat(stats['size']), 'type': filetype(stats['mode']), 'rwx': rwx(stats['mode']), 'mode': stringformat(stats['mode'], "o"), 'url': make_absolute(request, "view", dict(path=urlquote(normalized))), }
def read(self, path, offset, length, bufsize=None): """ read(path, offset, length[, bufsize]) -> data Read data from a file. """ path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'OPEN' params['offset'] = long(offset) params['length'] = long(length) if bufsize is not None: params['bufsize'] = bufsize try: return self._root.get(path, params) except WebHdfsException, ex: if "out of the range" in ex.message: return "" raise ex
def create(self, path, overwrite=False, blocksize=None, replication=None, permission=None, data=None): """ create(path, overwrite=False, blocksize=None, replication=None, permission=None) Creates a file with the specified parameters. `permission' should be an octal integer or string. """ path = Hdfs.normpath(path) params = self._getparams() params["op"] = "CREATE" params["overwrite"] = overwrite and "true" or "false" if blocksize is not None: params["blocksize"] = long(blocksize) if replication is not None: params["replication"] = int(replication) if permission is not None: params["permission"] = safe_octal(permission) self._invoke_with_redirect("PUT", path, params, data)
def strip_normpath(self, path): split = urlparse(path) path = split._replace(scheme="", netloc="").geturl() return Hdfs.normpath(path)
def remove_acl_entries(self, path, aclspec): path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'REMOVEACLENTRIES' params['aclspec'] = aclspec return self._root.put(path, params)
def remove_acl(self, path): path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'REMOVEACL' return self._root.put(path, params)
def set_acl(self, path, aclspec): path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'SETACL' params['aclspec'] = aclspec return self._root.put(path, params)
def get_acl_status(self, path): path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'GETACLSTATUS' return self._root.get(path, params)
def listdir_paged(request, path): """ A paginated version of listdir. Query parameters: pagenum - The page number to show. Defaults to 1. pagesize - How many to show on a page. Defaults to 15. sortby=? - Specify attribute to sort by. Accepts: (type, name, atime, mtime, size, user, group) Defaults to name. descending - Specify a descending sort order. Default to false. filter=? - Specify a substring filter to search for in the filename field. """ if not request.fs.isdir(path): raise PopupException("Not a directory: %s" % (path,)) pagenum = int(request.GET.get('pagenum', 1)) pagesize = int(request.GET.get('pagesize', 30)) do_as = None if request.user.is_superuser or request.user.has_hue_permission(action="impersonate", app="security"): do_as = request.GET.get('doas', request.user.username) if hasattr(request, 'doas'): do_as = request.doas home_dir_path = request.user.get_home_directory() breadcrumbs = parse_breadcrumbs(path) if do_as: all_stats = request.fs.do_as_user(do_as, request.fs.listdir_stats, path) else: all_stats = request.fs.listdir_stats(path) # Filter first filter_str = request.GET.get('filter', None) if filter_str: filtered_stats = filter(lambda sb: filter_str in sb['name'], all_stats) all_stats = filtered_stats # Sort next sortby = request.GET.get('sortby', None) descending_param = request.GET.get('descending', None) if sortby is not None: if sortby not in ('type', 'name', 'atime', 'mtime', 'user', 'group', 'size'): logger.info("Invalid sort attribute '%s' for listdir." % (sortby,)) else: all_stats = sorted(all_stats, key=operator.attrgetter(sortby), reverse=coerce_bool(descending_param)) # Do pagination page = paginator.Paginator(all_stats, pagesize).page(pagenum) shown_stats = page.object_list # Include parent dir always as second option, unless at filesystem root. if Hdfs.normpath(path) != posixpath.sep: parent_path = request.fs.join(path, "..") parent_stat = request.fs.stats(parent_path) # The 'path' field would be absolute, but we want its basename to be # actually '..' for display purposes. Encode it since _massage_stats expects byte strings. parent_stat['path'] = parent_path parent_stat['name'] = ".." shown_stats.insert(0, parent_stat) # Include same dir always as first option to see stats of the current folder current_stat = request.fs.stats(path) # The 'path' field would be absolute, but we want its basename to be # actually '.' for display purposes. Encode it since _massage_stats expects byte strings. current_stat['path'] = path current_stat['name'] = "." shown_stats.insert(1, current_stat) page.object_list = [ _massage_stats(request, s) for s in shown_stats ] is_fs_superuser = _is_hdfs_superuser(request) data = { 'path': path, 'breadcrumbs': breadcrumbs, 'current_request_path': request.path, 'files': page.object_list, 'page': _massage_page(page), 'pagesize': pagesize, 'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None, 'sortby': sortby, 'descending': descending_param, # The following should probably be deprecated 'cwd_set': True, 'file_filter': 'any', 'current_dir_path': path, 'is_fs_superuser': is_fs_superuser, 'groups': is_fs_superuser and [str(x) for x in Group.objects.values_list('name', flat=True)] or [], 'users': is_fs_superuser and [str(x) for x in User.objects.values_list('username', flat=True)] or [], 'superuser': request.fs.superuser, 'supergroup': request.fs.supergroup, 'is_sentry_managed': request.fs.is_sentry_managed(path) } return render('listdir.mako', request, data)
def modify_acl_entries(self, path, aclspec): path = Hdfs.normpath(path) params = self._getparams() params['op'] = 'MODIFYACLENTRIES' params['aclspec'] = aclspec return self._root.put(path, params)