def rename(self, old, new): old = encode_fs_path(old) new = encode_fs_path(new) success = self.nn_client.rename( self.request_context, normpath(old), normpath(new)) if not success: #TODO(todd) these functions should just throw if failed raise IOError("Rename failed")
def rename(self, old, new): old = encode_fs_path(old) new = encode_fs_path(new) success = self.nn_client.rename(self.request_context, normpath(old), normpath(new)) if not success: #TODO(todd) these functions should just throw if failed raise IOError("Rename failed")
def set_diskspace_quota(self, path, size): """ Set the diskspace quota of a given path. @param path The path to the given hdfs resource @param size The amount of bytes that a given subtree of files can grow to. """ path = encode_fs_path(path) if normpath(path) == '/': raise ValueError('Cannot set quota for "/"') if size < 0: raise ValueError("The size quota should be 0 or positive or unset") self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_DONT_SET, size)
def set_namespace_quota(self, path, num_files): """ Set the maximum number of files of a given path. @param path The path to the given hdfs resource @param num_files The amount of files that can exist within that subtree. """ path = encode_fs_path(path) if normpath(path) == '/': raise ValueError('Cannot set quota for "/"') if num_files < 0: raise ValueError("The number of files quota should be 0 or positive or unset") self.nn_client.setQuota(self.request_context, normpath(path), num_files, QUOTA_DONT_SET)
def get_content_summaries(self, paths): paths = [ normpath(encode_fs_path(path)) for path in paths ] summaries = self.nn_client.multiGetContentSummary(self.request_context, paths) def _fix_summary(summary): summary.path = decode_fs_path(summary.path) return summary return [_fix_summary(s) for s in summaries]
def mkdir(self, path, mode=0755): # TODO(todd) there should be a mkdir that isn't mkdirHIER # (this is mkdir -p I think) path = encode_fs_path(path) success = self.nn_client.mkdirhier(self.request_context, normpath(path), mode) if not success: raise IOError("mkdir failed")
def clean(self, value): cleaned_path = CharField.clean(self, value) if value.lower().startswith(S3_ROOT): cleaned_path = s3_normpath(cleaned_path) else: cleaned_path = normpath(cleaned_path) return cleaned_path
def clear_namespace_quota(self, path): """ Remove the namespace quota at a given path """ path = encode_fs_path(path) self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_RESET, QUOTA_DONT_SET)
def clean(self, value): cleaned_path = CharField.clean(self, value) if value.lower().startswith(S3A_ROOT): cleaned_path = s3_normpath(cleaned_path) else: cleaned_path = normpath(cleaned_path) return cleaned_path
def get_namespace_quota(self, path): """ Get the current quota in number of files. None if it is unset """ file_count_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).quota if file_count_quota == QUOTA_RESET or file_count_quota == QUOTA_DONT_SET: return None else: return file_count_quota
def get_diskspace_quota(self, path): """ Get the current space quota in bytes for disk space. None if it is unset """ space_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).spaceQuota if space_quota == QUOTA_RESET or space_quota == QUOTA_DONT_SET: return None else: return space_quota
def _hadoop_stat(self, path): """Returns None if file does not exist.""" try: stat = self.nn_client.stat(self.request_context, normpath(path)) return stat except IOException, ioe: if ioe.clazz == 'java.io.FileNotFoundException': return None raise
def get_namespace_quota(self, path): """ Get the current quota in number of files. None if it is unset """ path = encode_fs_path(path) file_count_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).quota if file_count_quota == QUOTA_RESET or file_count_quota == QUOTA_DONT_SET: return None else: return file_count_quota
def abspath(cd, uri): """ Returns absolute URI, examples: abspath('s3://bucket/key', key2') == 's3://bucket/key/key2' abspath('s3://bucket/key', 's3://bucket2/key2') == 's3://bucket2/key2' """ if not uri.lower().startswith(S3_ROOT): uri = normpath(join(cd, '..', uri)) return uri
def get_diskspace_quota(self, path): """ Get the current space quota in bytes for disk space. None if it is unset """ path = encode_fs_path(path) space_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).spaceQuota if space_quota == QUOTA_RESET or space_quota == QUOTA_DONT_SET: return None else: return space_quota
def _get_blocks(self, path, offset, length): """ Get block locations from the Name Node. Returns an array of Block instances that might look like: [ Block(path='/user/todd/motd', genStamp=1001, blockId=5564389078175231298, nodes=[DatanodeInfo(xceiverCount=1, capacity=37265149952, name='127.0.0.1:50010', thriftPort=53417, state=1, remaining=18987925504, host='127.0.0.1', storageID='DS-1238582576-127.0.1.1-50010-1240968238474', dfsUsed=36864)], numBytes=424)] """ return self.nn_client.getBlocks(self.request_context, normpath(path), offset, length)
def remove(self, path): stat = self._hadoop_stat(path) if not stat: raise IOError("File not found: %s" % path) if stat.isDir: raise IOError("Is a directory: %s" % path) success = self.nn_client.unlink( self.request_context, normpath(path), recursive=False) if not success: raise IOError("Unlink failed")
def _rmdir(self, path, recursive=False): stat = self._hadoop_stat(path) if not stat: raise IOError("Directory not found: %s" % (path,)) if not stat.isDir: raise IOError("Is not a directory: %s" % (path,)) success = self.nn_client.unlink( self.request_context, normpath(path), recursive=recursive) if not success: raise IOError("Unlink failed")
def urlsplit(url): """ Take an HDFS path (hdfs://nn:port/foo) or just (/foo) and split it into the standard urlsplit's 5-tuple. """ i = url.find('://') if i == -1: # Not found. Treat the entire argument as an HDFS path return ('hdfs', '', normpath(url), '', '') if url[:i] != 'hdfs': # Default to standard for non-hdfs return urlparse.urlsplit(url) url = url[i+3:] i = url.find('/') if i == -1: # Everything is netloc. Assume path is root. return ('hdfs', url, '/', '', '') netloc = url[:i] path = url[i:] return ('hdfs', netloc, normpath(path), '', '')
def clean(self, value): cleaned_path = CharField.clean(self, value) if not value: value = '' elif value.lower().startswith(S3A_ROOT): cleaned_path = s3_normpath(cleaned_path) elif value.lower().startswith(ABFS_ROOT): cleaned_path = abfs_normpath(cleaned_path) else: cleaned_path = normpath(cleaned_path) return cleaned_path
def urlsplit(url): """ Take an HDFS path (hdfs://nn:port/foo) or just (/foo) and split it into the standard urlsplit's 5-tuple. """ i = url.find('://') if i == -1: # Not found. Treat the entire argument as an HDFS path return ('hdfs', '', normpath(url), '', '') if url[:i] != 'hdfs': # Default to standard for non-hdfs return urlparse.urlsplit(url) url = url[i + 3:] i = url.find('/') if i == -1: # Everything is netloc. Assume path is root. return ('hdfs', url, '/', '', '') netloc = url[:i] path = url[i:] return ('hdfs', netloc, normpath(path), '', '')
def _hadoop_stat(self, path): """Returns None if file does not exist.""" path = encode_fs_path(path) try: stat = self.nn_client.stat(self.request_context, normpath(path)) stat.path = decode_fs_path(stat.path) return stat except IOException, ioe: if ioe.clazz == 'java.io.FileNotFoundException': return None raise
def _rmdir(self, path, recursive=False): path = encode_fs_path(path) stat = self._hadoop_stat(path) if not stat: raise IOError(errno.ENOENT, "Directory not found: %s" % (path,)) if not stat.isDir: raise IOError(errno.EISDIR, "Is not a directory: %s" % (path,)) success = self.nn_client.unlink( self.request_context, normpath(path), recursive=recursive) if not success: raise IOError("Unlink failed")
def _massage_stats(request, stats): """ Massage a stats record as returned by the filesystem implementation into the format that the views would like it in. """ normalized = normpath(stats['path']) return { 'path': normalized, 'name': posixpath.basename(stats['path']), 'stats': stats, 'type': filetype(stats['mode']), 'rwx': rwx(stats['mode']), 'url': make_absolute(request, "view", dict(path=urlquote(normalized))), }
def _get_blocks(self, path, offset, length): """ Get block locations from the Name Node. Returns an array of Block instances that might look like: [ Block(path='/user/todd/motd', genStamp=1001, blockId=5564389078175231298, nodes=[DatanodeInfo(xceiverCount=1, capacity=37265149952, name='127.0.0.1:50010', thriftPort=53417, state=1, remaining=18987925504, host='127.0.0.1', storageID='DS-1238582576-127.0.1.1-50010-1240968238474', dfsUsed=36864)], numBytes=424)] """ path = encode_fs_path(path) blocks = self.nn_client.getBlocks(self.request_context, normpath(path), offset, length) def _fix_block(blk): blk.path = decode_fs_path(blk.path) return blk return [_fix_block(blk) for blk in blocks]
def _massage_stats(request, stats): """ Massage a stats record as returned by the filesystem implementation into the format that the views would like it in. """ path = stats['path'] normalized = normpath(path) return { 'path': normalized, 'name': posixpath.basename(path), 'stats': stats, 'type': filetype(stats['mode']), 'rwx': rwx(stats['mode']), 'url': make_absolute(request, "view", dict(path=urlquote(normalized))), }
def __init__(self, fs, path, mode="r", buffering=False): self.fs = fs self.path = normpath(path) self.pos = 0 self.closed = False self._block_cache = BlockCache() if buffering or mode != "r": raise Exception("buffering and write support not yet implemented") # NYI stat = self._stat() if stat is None: raise IOError(errno.ENOENT, "No such file or directory: '%s'" % path) if stat.isDir: raise IOError(errno.EISDIR, "Is a directory: '%s'" % path)
def __init__(self, fs, path, mode='r'): self._fs = fs self._path = normpath(path) self._pos = 0 self._mode = mode try: self._stat = fs.stats(path) if self._stat.isDir: raise IOError(errno.EISDIR, _("Is a directory: '%s'") % path) except IOError, ex: if ex.errno == errno.ENOENT and 'w' in self._mode: self._fs.create(self._path) self.stat() else: raise ex
def __init__(self, fs, path, mode="r"): self._fs = fs self._path = normpath(path) self._pos = 0 self._mode = mode try: self._stat = fs.stats(path) if self._stat.isDir: raise IOError(errno.EISDIR, "Is a directory: '%s'" % (smart_str(path),)) except IOError, ex: if ex.errno == errno.ENOENT and "w" in self._mode: self._fs.create(self._path) self.stat() else: raise ex
def listdir(request, path): """ Implements directory listing (or index). Intended to be called via view(). """ path = _unquote_path(path) if not request.fs.isdir(path): raise PopupException("Not a directory: %s" % (path, )) file_filter = request.REQUEST.get('file_filter', 'any') assert file_filter in ['any', 'file', 'dir'] home_dir_path = request.user.get_home_directory() data = { 'path': path, 'file_filter': file_filter, # These could also be put in automatically via # http://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-request, # but manually seems cleaner, since we only need it here. 'current_request_path': request.path, 'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None, 'cwd_set': True, 'show_upload': (request.REQUEST.get('show_upload') == 'false' and (False, ) or (True, ))[0] } stats = request.fs.listdir_stats(path) # Include parent dir, unless at filesystem root. if normpath(path) != posixpath.sep: parent_stat = request.fs.stats(posixpath.join(path, "..")) # The 'path' field would be absolute, but we want its basename to be # actually '..' for display purposes. Encode it since _massage_stats expects byte strings. parent_stat['path'] = posixpath.join(path, "..") stats.insert(0, parent_stat) data['files'] = [_massage_stats(request, stat) for stat in stats] return render_with_toolbars('listdir.mako', request, data)
def get_usage_and_quota(self, path): """ Returns a dictionary with "file_count", "file_quota", "space_used", and "space_quota". The quotas may be None. """ summary = self.nn_client.getContentSummary(self.request_context, normpath(path)) ret = dict() ret["file_count"] = summary.fileCount ret["space_used"] = summary.spaceConsumed if summary.quota in (QUOTA_RESET, QUOTA_DONT_SET): ret["file_quota"] = None else: ret["file_quota"] = summary.quota if summary.spaceQuota in (QUOTA_RESET, QUOTA_DONT_SET): ret["space_quota"] = None else: ret["space_quota"] = summary.spaceQuota return ret
def get_usage_and_quota(self, path): """ Returns a dictionary with "file_count", "file_quota", "space_used", and "space_quota". The quotas may be None. """ path = encode_fs_path(path) summary = self.nn_client.getContentSummary(self.request_context, normpath(path)) ret = dict() ret["file_count"] = summary.fileCount ret["space_used"] = summary.spaceConsumed if summary.quota in (QUOTA_RESET, QUOTA_DONT_SET): ret["file_quota"] = None else: ret["file_quota"] = summary.quota if summary.spaceQuota in (QUOTA_RESET, QUOTA_DONT_SET): ret["space_quota"] = None else: ret["space_quota"] = summary.spaceQuota return ret
def listdir(request, path): """ Implements directory listing (or index). Intended to be called via view(). """ path = _unquote_path(path) if not request.fs.isdir(path): raise PopupException("Not a directory: %s" % (path,)) file_filter = request.REQUEST.get('file_filter', 'any') assert file_filter in ['any', 'file', 'dir'] home_dir_path = request.user.get_home_directory() data = { 'path': path, 'file_filter': file_filter, # These could also be put in automatically via # http://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-request, # but manually seems cleaner, since we only need it here. 'current_request_path': request.path, 'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None, 'cwd_set': True, 'show_upload': (request.REQUEST.get('show_upload') == 'false' and (False,) or (True,))[0] } stats = request.fs.listdir_stats(path) # Include parent dir, unless at filesystem root. if normpath(path) != posixpath.sep: parent_stat = request.fs.stats(posixpath.join(path, "..")) # The 'path' field would be absolute, but we want its basename to be # actually '..' for display purposes. Encode it since _massage_stats expects byte strings. parent_stat['path'] = posixpath.join(path, "..") stats.insert(0, parent_stat) data['files'] = [_massage_stats(request, stat) for stat in stats] return render_with_toolbars('listdir.mako', request, data)
def get_content_summaries(self, paths): return self.nn_client.multiGetContentSummary(self.request_context, [normpath(path) for path in paths])
def listdir_stats(self, path): path = encode_fs_path(path) stats = self.nn_client.ls(self.request_context, normpath(path)) return [self._unpack_stat(s) for s in stats]
def listdir(self, path): path = encode_fs_path(path) stats = self.nn_client.ls(self.request_context, normpath(path)) return [self.basename(decode_fs_path(stat.path)) for stat in stats]
def chmod(self, path, mode): self.nn_client.chmod(self.request_context, normpath(path), mode)
def clear_diskspace_quota(self, path): """ Remove the diskspace quota at a given path """ self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_DONT_SET, QUOTA_RESET)
def chown(self, path, user, group): path = encode_fs_path(path) self.nn_client.chown(self.request_context, normpath(path), user, group)
def chmod(self, path, mode): path = encode_fs_path(path) self.nn_client.chmod(self.request_context, normpath(path), mode)
def normpath(path): return normpath(path)
def stats(self, path): path = normpath(path) stats = self._stats(path) if stats: return stats raise IOError(errno.ENOENT, "No such file or directory: '%s'" % path)
def clean(self, value): return normpath(CharField.clean(self, value))
def chown(self, path, user, group): self.nn_client.chown(self.request_context, normpath(path), user, group)