def rename(self, old, new):
old = encode_fs_path(old)
new = encode_fs_path(new)
success = self.nn_client.rename(
self.request_context, normpath(old), normpath(new))
if not success: #TODO(todd) these functions should just throw if failed
raise IOError("Rename failed")
def rename(self, old, new):
old = encode_fs_path(old)
new = encode_fs_path(new)
success = self.nn_client.rename(self.request_context, normpath(old),
normpath(new))
if not success: #TODO(todd) these functions should just throw if failed
raise IOError("Rename failed")
def set_diskspace_quota(self, path, size):
"""
Set the diskspace quota of a given path.
@param path The path to the given hdfs resource
@param size The amount of bytes that a given subtree of files can grow to.
"""
path = encode_fs_path(path)
if normpath(path) == '/':
raise ValueError('Cannot set quota for "/"')
if size < 0:
raise ValueError("The size quota should be 0 or positive or unset")
self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_DONT_SET, size)
def set_namespace_quota(self, path, num_files):
"""
Set the maximum number of files of a given path.
@param path The path to the given hdfs resource
@param num_files The amount of files that can exist within that subtree.
"""
path = encode_fs_path(path)
if normpath(path) == '/':
raise ValueError('Cannot set quota for "/"')
if num_files < 0:
raise ValueError("The number of files quota should be 0 or positive or unset")
self.nn_client.setQuota(self.request_context, normpath(path), num_files, QUOTA_DONT_SET)
def set_diskspace_quota(self, path, size):
"""
Set the diskspace quota of a given path.
@param path The path to the given hdfs resource
@param size The amount of bytes that a given subtree of files can grow to.
"""
path = encode_fs_path(path)
if normpath(path) == '/':
raise ValueError('Cannot set quota for "/"')
if size < 0:
raise ValueError("The size quota should be 0 or positive or unset")
self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_DONT_SET, size)
def set_namespace_quota(self, path, num_files):
"""
Set the maximum number of files of a given path.
@param path The path to the given hdfs resource
@param num_files The amount of files that can exist within that subtree.
"""
path = encode_fs_path(path)
if normpath(path) == '/':
raise ValueError('Cannot set quota for "/"')
if num_files < 0:
raise ValueError("The number of files quota should be 0 or positive or unset")
self.nn_client.setQuota(self.request_context, normpath(path), num_files, QUOTA_DONT_SET)
def get_content_summaries(self, paths):
paths = [ normpath(encode_fs_path(path)) for path in paths ]
summaries = self.nn_client.multiGetContentSummary(self.request_context, paths)
def _fix_summary(summary):
summary.path = decode_fs_path(summary.path)
return summary
return [_fix_summary(s) for s in summaries]
def get_content_summaries(self, paths):
paths = [ normpath(encode_fs_path(path)) for path in paths ]
summaries = self.nn_client.multiGetContentSummary(self.request_context, paths)
def _fix_summary(summary):
summary.path = decode_fs_path(summary.path)
return summary
return [_fix_summary(s) for s in summaries]
def mkdir(self, path, mode=0755):
# TODO(todd) there should be a mkdir that isn't mkdirHIER
# (this is mkdir -p I think)
path = encode_fs_path(path)
success = self.nn_client.mkdirhier(self.request_context, normpath(path), mode)
if not success:
raise IOError("mkdir failed")
def clean(self, value):
cleaned_path = CharField.clean(self, value)
if value.lower().startswith(S3_ROOT):
cleaned_path = s3_normpath(cleaned_path)
else:
cleaned_path = normpath(cleaned_path)
return cleaned_path
def mkdir(self, path, mode=0755):
# TODO(todd) there should be a mkdir that isn't mkdirHIER
# (this is mkdir -p I think)
path = encode_fs_path(path)
success = self.nn_client.mkdirhier(self.request_context, normpath(path), mode)
if not success:
raise IOError("mkdir failed")
def clear_namespace_quota(self, path):
"""
Remove the namespace quota at a given path
"""
path = encode_fs_path(path)
self.nn_client.setQuota(self.request_context, normpath(path),
QUOTA_RESET, QUOTA_DONT_SET)
def clean(self, value):
cleaned_path = CharField.clean(self, value)
if value.lower().startswith(S3A_ROOT):
cleaned_path = s3_normpath(cleaned_path)
else:
cleaned_path = normpath(cleaned_path)
return cleaned_path
def get_namespace_quota(self, path):
"""
Get the current quota in number of files. None if it is unset
"""
file_count_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).quota
if file_count_quota == QUOTA_RESET or file_count_quota == QUOTA_DONT_SET:
return None
else:
return file_count_quota
def get_diskspace_quota(self, path):
"""
Get the current space quota in bytes for disk space. None if it is unset
"""
space_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).spaceQuota
if space_quota == QUOTA_RESET or space_quota == QUOTA_DONT_SET:
return None
else:
return space_quota
def _hadoop_stat(self, path):
"""Returns None if file does not exist."""
try:
stat = self.nn_client.stat(self.request_context, normpath(path))
return stat
except IOException, ioe:
if ioe.clazz == 'java.io.FileNotFoundException':
return None
raise
def get_namespace_quota(self, path):
"""
Get the current quota in number of files. None if it is unset
"""
path = encode_fs_path(path)
file_count_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).quota
if file_count_quota == QUOTA_RESET or file_count_quota == QUOTA_DONT_SET:
return None
else:
return file_count_quota
def abspath(cd, uri):
"""
Returns absolute URI, examples:
abspath('s3://bucket/key', key2') == 's3://bucket/key/key2'
abspath('s3://bucket/key', 's3://bucket2/key2') == 's3://bucket2/key2'
"""
if not uri.lower().startswith(S3_ROOT):
uri = normpath(join(cd, '..', uri))
return uri
def get_diskspace_quota(self, path):
"""
Get the current space quota in bytes for disk space. None if it is unset
"""
path = encode_fs_path(path)
space_quota = self.nn_client.getContentSummary(self.request_context, normpath(path)).spaceQuota
if space_quota == QUOTA_RESET or space_quota == QUOTA_DONT_SET:
return None
else:
return space_quota
def _get_blocks(self, path, offset, length):
"""
Get block locations from the Name Node. Returns an array of Block
instances that might look like:
[ Block(path='/user/todd/motd', genStamp=1001, blockId=5564389078175231298,
nodes=[DatanodeInfo(xceiverCount=1, capacity=37265149952, name='127.0.0.1:50010',
thriftPort=53417, state=1, remaining=18987925504, host='127.0.0.1',
storageID='DS-1238582576-127.0.1.1-50010-1240968238474', dfsUsed=36864)], numBytes=424)]
"""
return self.nn_client.getBlocks(self.request_context, normpath(path), offset, length)
def abspath(cd, uri):
"""
Returns absolute URI, examples:
abspath('s3://bucket/key', key2') == 's3://bucket/key/key2'
abspath('s3://bucket/key', 's3://bucket2/key2') == 's3://bucket2/key2'
"""
if not uri.lower().startswith(S3_ROOT):
uri = normpath(join(cd, '..', uri))
return uri
def remove(self, path):
stat = self._hadoop_stat(path)
if not stat:
raise IOError("File not found: %s" % path)
if stat.isDir:
raise IOError("Is a directory: %s" % path)
success = self.nn_client.unlink(
self.request_context, normpath(path), recursive=False)
if not success:
raise IOError("Unlink failed")
def _rmdir(self, path, recursive=False):
stat = self._hadoop_stat(path)
if not stat:
raise IOError("Directory not found: %s" % (path,))
if not stat.isDir:
raise IOError("Is not a directory: %s" % (path,))
success = self.nn_client.unlink(
self.request_context, normpath(path), recursive=recursive)
if not success:
raise IOError("Unlink failed")
def urlsplit(url):
"""
Take an HDFS path (hdfs://nn:port/foo) or just (/foo) and split it into
the standard urlsplit's 5-tuple.
"""
i = url.find('://')
if i == -1:
# Not found. Treat the entire argument as an HDFS path
return ('hdfs', '', normpath(url), '', '')
if url[:i] != 'hdfs':
# Default to standard for non-hdfs
return urlparse.urlsplit(url)
url = url[i+3:]
i = url.find('/')
if i == -1:
# Everything is netloc. Assume path is root.
return ('hdfs', url, '/', '', '')
netloc = url[:i]
path = url[i:]
return ('hdfs', netloc, normpath(path), '', '')
def clean(self, value):
cleaned_path = CharField.clean(self, value)
if not value:
value = ''
elif value.lower().startswith(S3A_ROOT):
cleaned_path = s3_normpath(cleaned_path)
elif value.lower().startswith(ABFS_ROOT):
cleaned_path = abfs_normpath(cleaned_path)
else:
cleaned_path = normpath(cleaned_path)
return cleaned_path
def urlsplit(url):
"""
Take an HDFS path (hdfs://nn:port/foo) or just (/foo) and split it into
the standard urlsplit's 5-tuple.
"""
i = url.find('://')
if i == -1:
# Not found. Treat the entire argument as an HDFS path
return ('hdfs', '', normpath(url), '', '')
if url[:i] != 'hdfs':
# Default to standard for non-hdfs
return urlparse.urlsplit(url)
url = url[i + 3:]
i = url.find('/')
if i == -1:
# Everything is netloc. Assume path is root.
return ('hdfs', url, '/', '', '')
netloc = url[:i]
path = url[i:]
return ('hdfs', netloc, normpath(path), '', '')
def _hadoop_stat(self, path):
"""Returns None if file does not exist."""
path = encode_fs_path(path)
try:
stat = self.nn_client.stat(self.request_context, normpath(path))
stat.path = decode_fs_path(stat.path)
return stat
except IOException, ioe:
if ioe.clazz == 'java.io.FileNotFoundException':
return None
raise
def _rmdir(self, path, recursive=False):
path = encode_fs_path(path)
stat = self._hadoop_stat(path)
if not stat:
raise IOError(errno.ENOENT, "Directory not found: %s" % (path,))
if not stat.isDir:
raise IOError(errno.EISDIR, "Is not a directory: %s" % (path,))
success = self.nn_client.unlink(
self.request_context, normpath(path), recursive=recursive)
if not success:
raise IOError("Unlink failed")
def _massage_stats(request, stats):
"""
Massage a stats record as returned by the filesystem implementation
into the format that the views would like it in.
"""
normalized = normpath(stats['path'])
return {
'path': normalized,
'name': posixpath.basename(stats['path']),
'stats': stats,
'type': filetype(stats['mode']),
'rwx': rwx(stats['mode']),
'url': make_absolute(request, "view", dict(path=urlquote(normalized))),
}
def _get_blocks(self, path, offset, length):
"""
Get block locations from the Name Node. Returns an array of Block
instances that might look like:
[ Block(path='/user/todd/motd', genStamp=1001, blockId=5564389078175231298,
nodes=[DatanodeInfo(xceiverCount=1, capacity=37265149952, name='127.0.0.1:50010',
thriftPort=53417, state=1, remaining=18987925504, host='127.0.0.1',
storageID='DS-1238582576-127.0.1.1-50010-1240968238474', dfsUsed=36864)], numBytes=424)]
"""
path = encode_fs_path(path)
blocks = self.nn_client.getBlocks(self.request_context, normpath(path), offset, length)
def _fix_block(blk):
blk.path = decode_fs_path(blk.path)
return blk
return [_fix_block(blk) for blk in blocks]
def _massage_stats(request, stats):
"""
Massage a stats record as returned by the filesystem implementation
into the format that the views would like it in.
"""
path = stats['path']
normalized = normpath(path)
return {
'path': normalized,
'name': posixpath.basename(path),
'stats': stats,
'type': filetype(stats['mode']),
'rwx': rwx(stats['mode']),
'url': make_absolute(request, "view", dict(path=urlquote(normalized))),
}
def __init__(self, fs, path, mode="r", buffering=False):
self.fs = fs
self.path = normpath(path)
self.pos = 0
self.closed = False
self._block_cache = BlockCache()
if buffering or mode != "r":
raise Exception("buffering and write support not yet implemented") # NYI
stat = self._stat()
if stat is None:
raise IOError(errno.ENOENT, "No such file or directory: '%s'" % path)
if stat.isDir:
raise IOError(errno.EISDIR, "Is a directory: '%s'" % path)
def __init__(self, fs, path, mode="r", buffering=False):
self.fs = fs
self.path = normpath(path)
self.pos = 0
self.closed = False
self._block_cache = BlockCache()
if buffering or mode != "r":
raise Exception("buffering and write support not yet implemented") # NYI
stat = self._stat()
if stat is None:
raise IOError(errno.ENOENT, "No such file or directory: '%s'" % path)
if stat.isDir:
raise IOError(errno.EISDIR, "Is a directory: '%s'" % path)
def __init__(self, fs, path, mode='r'):
self._fs = fs
self._path = normpath(path)
self._pos = 0
self._mode = mode
try:
self._stat = fs.stats(path)
if self._stat.isDir:
raise IOError(errno.EISDIR, _("Is a directory: '%s'") % path)
except IOError, ex:
if ex.errno == errno.ENOENT and 'w' in self._mode:
self._fs.create(self._path)
self.stat()
else:
raise ex
def __init__(self, fs, path, mode="r"):
self._fs = fs
self._path = normpath(path)
self._pos = 0
self._mode = mode
try:
self._stat = fs.stats(path)
if self._stat.isDir:
raise IOError(errno.EISDIR, "Is a directory: '%s'" % (smart_str(path),))
except IOError, ex:
if ex.errno == errno.ENOENT and "w" in self._mode:
self._fs.create(self._path)
self.stat()
else:
raise ex
def listdir(request, path):
"""
Implements directory listing (or index).
Intended to be called via view().
"""
path = _unquote_path(path)
if not request.fs.isdir(path):
raise PopupException("Not a directory: %s" % (path, ))
file_filter = request.REQUEST.get('file_filter', 'any')
assert file_filter in ['any', 'file', 'dir']
home_dir_path = request.user.get_home_directory()
data = {
'path':
path,
'file_filter':
file_filter,
# These could also be put in automatically via
# http://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-request,
# but manually seems cleaner, since we only need it here.
'current_request_path':
request.path,
'home_directory':
request.fs.isdir(home_dir_path) and home_dir_path or None,
'cwd_set':
True,
'show_upload':
(request.REQUEST.get('show_upload') == 'false' and (False, )
or (True, ))[0]
}
stats = request.fs.listdir_stats(path)
# Include parent dir, unless at filesystem root.
if normpath(path) != posixpath.sep:
parent_stat = request.fs.stats(posixpath.join(path, ".."))
# The 'path' field would be absolute, but we want its basename to be
# actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
parent_stat['path'] = posixpath.join(path, "..")
stats.insert(0, parent_stat)
data['files'] = [_massage_stats(request, stat) for stat in stats]
return render_with_toolbars('listdir.mako', request, data)
def get_usage_and_quota(self, path):
"""
Returns a dictionary with "file_count", "file_quota",
"space_used", and "space_quota". The quotas
may be None.
"""
summary = self.nn_client.getContentSummary(self.request_context, normpath(path))
ret = dict()
ret["file_count"] = summary.fileCount
ret["space_used"] = summary.spaceConsumed
if summary.quota in (QUOTA_RESET, QUOTA_DONT_SET):
ret["file_quota"] = None
else:
ret["file_quota"] = summary.quota
if summary.spaceQuota in (QUOTA_RESET, QUOTA_DONT_SET):
ret["space_quota"] = None
else:
ret["space_quota"] = summary.spaceQuota
return ret
def get_usage_and_quota(self, path):
"""
Returns a dictionary with "file_count", "file_quota",
"space_used", and "space_quota". The quotas
may be None.
"""
path = encode_fs_path(path)
summary = self.nn_client.getContentSummary(self.request_context, normpath(path))
ret = dict()
ret["file_count"] = summary.fileCount
ret["space_used"] = summary.spaceConsumed
if summary.quota in (QUOTA_RESET, QUOTA_DONT_SET):
ret["file_quota"] = None
else:
ret["file_quota"] = summary.quota
if summary.spaceQuota in (QUOTA_RESET, QUOTA_DONT_SET):
ret["space_quota"] = None
else:
ret["space_quota"] = summary.spaceQuota
return ret
def listdir(request, path):
"""
Implements directory listing (or index).
Intended to be called via view().
"""
path = _unquote_path(path)
if not request.fs.isdir(path):
raise PopupException("Not a directory: %s" % (path,))
file_filter = request.REQUEST.get('file_filter', 'any')
assert file_filter in ['any', 'file', 'dir']
home_dir_path = request.user.get_home_directory()
data = {
'path': path,
'file_filter': file_filter,
# These could also be put in automatically via
# http://docs.djangoproject.com/en/dev/ref/templates/api/#django-core-context-processors-request,
# but manually seems cleaner, since we only need it here.
'current_request_path': request.path,
'home_directory': request.fs.isdir(home_dir_path) and home_dir_path or None,
'cwd_set': True,
'show_upload': (request.REQUEST.get('show_upload') == 'false' and (False,) or (True,))[0]
}
stats = request.fs.listdir_stats(path)
# Include parent dir, unless at filesystem root.
if normpath(path) != posixpath.sep:
parent_stat = request.fs.stats(posixpath.join(path, ".."))
# The 'path' field would be absolute, but we want its basename to be
# actually '..' for display purposes. Encode it since _massage_stats expects byte strings.
parent_stat['path'] = posixpath.join(path, "..")
stats.insert(0, parent_stat)
data['files'] = [_massage_stats(request, stat) for stat in stats]
return render_with_toolbars('listdir.mako', request, data)
def get_content_summaries(self, paths): return self.nn_client.multiGetContentSummary(self.request_context, [normpath(path) for path in paths])
def listdir_stats(self, path): path = encode_fs_path(path) stats = self.nn_client.ls(self.request_context, normpath(path)) return [self._unpack_stat(s) for s in stats]
def listdir(self, path): path = encode_fs_path(path) stats = self.nn_client.ls(self.request_context, normpath(path)) return [self.basename(decode_fs_path(stat.path)) for stat in stats]
def chmod(self, path, mode): self.nn_client.chmod(self.request_context, normpath(path), mode)
def listdir_stats(self, path):
path = encode_fs_path(path)
stats = self.nn_client.ls(self.request_context, normpath(path))
return [self._unpack_stat(s) for s in stats]
def clear_diskspace_quota(self, path): """ Remove the diskspace quota at a given path """ self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_DONT_SET, QUOTA_RESET)
def chown(self, path, user, group):
path = encode_fs_path(path)
self.nn_client.chown(self.request_context, normpath(path), user, group)
def chmod(self, path, mode):
path = encode_fs_path(path)
self.nn_client.chmod(self.request_context, normpath(path), mode)
def normpath(path):
return normpath(path)
def chmod(self, path, mode): path = encode_fs_path(path) self.nn_client.chmod(self.request_context, normpath(path), mode)
def clear_namespace_quota(self, path): """ Remove the namespace quota at a given path """ path = encode_fs_path(path) self.nn_client.setQuota(self.request_context, normpath(path), QUOTA_RESET, QUOTA_DONT_SET)
def chown(self, path, user, group): path = encode_fs_path(path) self.nn_client.chown(self.request_context, normpath(path), user, group)
def listdir(self, path):
path = encode_fs_path(path)
stats = self.nn_client.ls(self.request_context, normpath(path))
return [self.basename(decode_fs_path(stat.path)) for stat in stats]
def stats(self, path):
path = normpath(path)
stats = self._stats(path)
if stats:
return stats
raise IOError(errno.ENOENT, "No such file or directory: '%s'" % path)
def clean(self, value): return normpath(CharField.clean(self, value))
def chown(self, path, user, group): self.nn_client.chown(self.request_context, normpath(path), user, group)
def clean(self, value):
return normpath(CharField.clean(self, value))
