def getMemberNames(self):
"""Return list of direct collection member names (utf-8 encoded).
See DAVCollection.getMemberNames()
"""
return [i for i in super(MiGFolderResource, self).getMemberNames() if \
not invisible_path(i)]
def handle_file(
configuration,
listing,
filename,
file_with_dir,
actual_file,
flags='',
dest='',
show_dest=False,
):
"""handle a file"""
# Build entire line before printing to avoid newlines
# Recursion can get here when called without explicit invisible files
if invisible_path(file_with_dir):
return
file_obj = {
'object_type': 'direntry',
'type': 'file',
'name': filename,
'rel_path': file_with_dir,
'rel_path_enc': quote(file_with_dir),
'rel_dir_enc': quote(os.path.dirname(file_with_dir)),
# NOTE: file_with_dir is kept for backwards compliance
'file_with_dir': file_with_dir,
'flags': flags,
}
if show_dest:
file_obj['file_dest'] = dest
listing.append(file_obj)
def handle_file(
listing,
filename,
file_with_dir,
actual_file,
flags='',
dest='',
show_dest=False,
):
"""handle a file"""
# Build entire line before printing to avoid newlines
# Recursion can get here when called without explicit invisible files
if invisible_path(file_with_dir):
return
file_obj = {
'object_type': 'direntry',
'type': 'file',
'name': filename,
'file_with_dir': file_with_dir,
'flags': flags,
}
if show_dest:
file_obj['file_dest'] = dest
listing.append(file_obj)
def _check_access(configuration, action, target_list):
"""Verify access to action on paths in target_list"""
_logger = configuration.logger
for path in target_list:
if invisible_path(path):
_logger.warning("%s rejected on invisible path: %s" %
(action, path))
raise ValueError('contains protected files/folders')
elif os.path.islink(path):
_logger.warning("%s rejected on link %s" % (action, path))
raise ValueError('contains special files/folders')
def __init__(self, path, environ, filePath):
super(MiGFolderResource, self).__init__(path, environ, filePath)
if invisible_path(path):
raise DAVError(HTTP_FORBIDDEN)
self._user_chroot = _user_chroot_path(environ)
# Replace native _locToFilePath method with our chrooted version
def wrapLocToFilePath(path):
"""Wrap native _locToFilePathmethod in chrooted version"""
return self.provider._chroot_locToFilePath(self._user_chroot, path)
self.provider._locToFilePath = wrapLocToFilePath
def acceptable_chmod(path, mode, chmod_exceptions):
"""Internal helper to check that a chmod request is safe. That is, it
only changes permissions that does not lock out user. Furthermore anything
we deem invisible_path or inside the dirs in chmod_exceptions should be
left alone to avoid users touching read-only files or enabling execution
of custom cgi scripts with potentially arbitrary code.
We require preservation of user read+write on files and user
read+write+execute on dirs.
Limitation to sane group/other access perms is left to the caller.
"""
if invisible_path(path) or \
True in [os.path.realpath(path).startswith(i) for i in
chmod_exceptions]:
return False
# Never touch special leading bits (suid, sgid, etc.)
if mode & 07000 != 00000:
return False
def get_childs(self, uri, filter=None):
"""return the child objects as self.baseuris for the given URI.
We override the listing to hide invisible_path hits.
"""
fileloc = self.uri2local(uri)
filelist = []
if os.path.exists(fileloc):
if os.path.isdir(fileloc):
try:
files = os.listdir(fileloc)
except:
logger.warning("could not listfiles in %s" % uri)
raise DAV_NotFound
for filename in files:
if invisible_path(filename):
continue
newloc = os.path.join(fileloc, filename)
filelist.append(self.local2uri(newloc))
logger.info('get_childs: Childs %s' % filelist)
return filelist
def handle_file(
configuration,
listing,
filename,
file_with_dir,
actual_file,
flags='',
):
"""handle a file"""
# Build entire line before printing to avoid newlines
# Recursion can get here when called without explicit invisible files
if invisible_path(file_with_dir):
return
special = ''
file_obj = {
'object_type': 'direntry',
'type': 'file',
'name': filename,
'rel_path': file_with_dir,
'rel_path_enc': quote(file_with_dir),
'rel_dir_enc': quote(os.path.dirname(file_with_dir)),
# NOTE: file_with_dir is kept for backwards compliance
'file_with_dir': file_with_dir,
'flags': flags,
'special': special,
}
if long_list(flags):
file_obj['long_format'] = long_format(actual_file)
if file_info(flags):
file_obj['file_info'] = fileinfo_stat(actual_file)
listing.append(file_obj)
logger.error("create zip failed: %s" % exc)
msg += 'Could not create zipfile: %s! ' % exc
return (False, msg)
if os.path.isdir(real_src):
walker = os.walk(real_src)
else:
(root, filename) = os.path.split(real_src)
walker = ((root + os.sep, [], [filename]), )
for (root, _, files) in walker:
relative_root = root.replace(real_src_dir + os.sep, '')
for entry in files:
real_target = os.path.join(root, entry)
relative_target = os.path.join(relative_root,
entry)
if invisible_path(real_target):
logger.warning('skipping hidden file: %s' \
% real_target)
continue
elif real_dst == real_target:
msg += 'Skipping destination file %s . ' % dst
continue
logger.debug("pack file %s" % relative_target)
try:
pack_file.write(real_target, relative_target)
except Exception, exc:
logger.error('write of %s failed: %s' % \
(real_target, exc))
msg += 'Failed to write file %s . ' % relative_target
status = False
continue
def valid_user_path(configuration,
path,
home_dir,
allow_equal=False,
chroot_exceptions=keyword_auto,
apache_scripts=False):
"""This is a convenience function for making sure that users do
not access restricted files including files outside their own file
tree(s): Check that path is a valid path inside user home directory,
home_dir and it does not map to an invisible file or dir.
In a few situations it may be relevant to not allow an exact
match, e.g. to prevent users from deleting or sharing the base of their
home directory.
IMPORTANT: it is still essential to always ONLY operate on explicitly
abs-expanded paths in backends to avoid MYVGRID/../bla silently
mapping to vgrid_files_home/bla rather than bla in user home.
We include a check to make sure that path is already abspath expanded to
help make sure this essential step is always done in backend.
This check also rejects all 'invisible' files like htaccess files.
NB: This check relies on the home_dir already verified from
certificate data.
Thus this function should *only* be used in relation to
checking user home related paths. Other paths should be
validated with the valid_dir_input from shared.base instead.
IMPORTANT: additionally uses a chroot_exceptions list to follow symlinks
e.g. into vgrid shared folders and verify their validity. This should be
the case based on the symlink availability, but we check to avoid the
attack vector. Otherwise it would be possible for users to access out of
bounds data if they could somehow sneak in a symlink pointing to such
locations. In particular this may be possible for users setting up their
own storage resource where they have unrestricted symlink control. Thus,
we explicitly check any such links and refuse them if they point outside
the mount in question.
If left to keyword_auto the list of chroot_exceptions is automatically
extracted based on the configuration.
The optional apache_scripts argument can be used to exclude the vgrid
collaboration scripts when checking for invisible files. In that way we
can allow the apache chroot checker exclusively to accept access to those
scripts as needed for Xgi execution of them.
"""
# We allow None value to support the few call points without one
if configuration is None:
configuration = get_configuration_object()
_logger = configuration.logger
#_logger.debug("valid_user_path on %s %s" % (path, home_dir))
# Make sure caller has explicitly forced abs path
if path != os.path.abspath(path):
return False
if invisible_path(path, apache_scripts):
return False
abs_home = os.path.abspath(home_dir)
if chroot_exceptions == keyword_auto:
chroot_exceptions = user_chroot_exceptions(configuration)
# IMPORTANT: verify proper chrooting inside home_dir or chroot_exceptions
real_path = os.path.realpath(path)
real_home = os.path.realpath(abs_home)
accept_roots = [real_home] + chroot_exceptions
#_logger.debug("check that path %s (%s) is inside %s" % (path, real_path, accept_roots))
accepted = False
for accept_path in accept_roots:
if real_path == accept_path or \
real_path.startswith(accept_path + os.sep):
accepted = True
#_logger.debug("path %s is inside chroot %s" % (real_path, accept_path))
break
if not accepted:
_logger.error("%s is outside chroot boundaries!" % path)
return False
# IMPORTANT: make sure path is not a symlink on a storage res mount
# We cannot prevent users from creating arbitrary symlinks on resources
# they have direct access to, so *don't ever* trust such symlinks unless
# they point inside the storage resource mount itself.
#_logger.debug("check that path %s is not inside store res" % path)
if path != real_path and untrusted_store_res_symlink(configuration, path):
_logger.error("untrusted symlink on a storage resource: %s" % path)
return False
# NOTE: abs_home may be e.g. email alias for real home so we test that
# path either starts with abs_home or real_home to make sure it is really
# a path in user home in addition to being in home or (general) chroots.
inside = (path.startswith(abs_home + os.sep)
or path.startswith(real_home + os.sep))
#_logger.debug("path %s is inside " % path)
if not allow_equal:
# path must be abs_home/X
return inside
else:
# path must be either abs_home/X or abs_home
try:
same = os.path.samefile(abs_home, path)
except Exception:
# At least one of the paths doesn't exist
same = False
return inside or same
logger.error("create zip failed: %s" % exc)
msg += 'Could not create zipfile: %s! ' % exc
return (False, msg)
if os.path.isdir(real_src):
walker = os.walk(real_src)
else:
(root, filename) = os.path.split(real_src)
walker = ((root + os.sep, [], [filename]), )
for (root, _, files) in walker:
relative_root = root.replace(real_src_dir + os.sep, '')
for entry in files:
real_target = os.path.join(root, entry)
relative_target = os.path.join(relative_root,
entry)
if invisible_path(real_target):
logger.warning('skipping hidden file: %s' \
% real_target)
continue
elif real_dst == real_target:
msg += 'Skipping destination file %s . ' % dst
continue
logger.info("pack file %s" % relative_target)
try:
pack_file.write(real_target, relative_target)
except Exception, exc:
logger.error('write of %s failed: %s' % \
(real_target, exc))
msg += 'Failed to write file %s . ' % relative_target
status = False
continue
def listdir(self, path):
"""List the content of a directory with MiG restrictions"""
return [
i for i in AbstractedFS.listdir(self, path)
if not invisible_path(i)
]
def listdir(self, path):
"""List the content of a directory with MiG restrictions"""
return [i for i in AbstractedFS.listdir(self, path) if not \
invisible_path(i)]
# Backwards compatibility
if fileinfo_dict.has_key('filename'):
fileinfo_dict['path'] = fileinfo_dict['filename']
if not fileinfo_dict.has_key('path'):
# Check if path was in querystring.
o.out('No path provided, unable to process file!')
o.reply_and_exit(o.ERROR)
# Reject modification of invisible files, even if apache generally allows it
if invisible_path(fileinfo_dict['path']):
o.out('Access to %s is prohibited!' % fileinfo_dict['path'])
o.reply_and_exit(o.CLIENT_ERROR)
logger.info('rangefileaccess on %s (%s)' % (fileinfo_dict['path'],
fileinfo_dict))
if client_id:
# logger.debug("Certificate found as a user cert: " + client_id)
fileinfo_dict['base_path'] = \
os.path.normpath(os.path.join(configuration.user_home,
client_dir)) + os.sep
elif fileinfo_dict.has_key('iosessionid'):
self.logger.warning('list_folder %s: %s' % (path, err))
return paramiko.SFTP_PERMISSION_DENIED
self.logger.debug("list_folder %s :: %s" % (path, real_path))
reply = []
if not os.path.exists(real_path):
self.logger.error("list_folder on missing path %s :: %s" % \
(path, real_path))
return paramiko.SFTP_NO_SUCH_FILE
try:
files = os.listdir(real_path)
except Exception, err:
self.logger.error("list_folder on %s :: %s failed: %s" % \
(path, real_path, err))
return paramiko.SFTP_FAILURE
for filename in files:
if invisible_path(filename):
continue
full_name = ("%s/%s" % (real_path, filename)).replace("//", "/")
# stat may fail e.g. if filename is a stale storage mount point
try:
reply.append(paramiko.SFTPAttributes.from_stat(
os.stat(full_name), self._strip_root(filename)))
except Exception, err:
self.logger.warning("list_folder %s: stat on %s failed: %s" % \
(path, full_name, err))
self.logger.debug("list_folder %s reply %s" % (path, reply))
return reply
def stat(self, path):
"""Handle operations of same name"""
path = force_utf8(path)
def handle_expand(
configuration,
output_objects,
listing,
base_dir,
real_path,
flags='',
dest='',
depth=0,
show_dest=False,
):
"""Recursive function to expand paths in a way not unlike ls, but only
files are interesting in this context. The order of recursively expanded
paths is different from that in ls since it simplifies the code and
doesn't really matter to the clients.
"""
# Sanity check
if depth > 255:
output_objects.append({
'object_type': 'error_text',
'text': 'Error: file recursion maximum exceeded!'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# references to '.' or similar are stripped by abspath
if real_path + os.sep == base_dir:
base_name = relative_path = '.'
else:
base_name = os.path.basename(real_path)
relative_path = real_path.replace(base_dir, '')
# Recursion can get here when called without explicit invisible files
if invisible_path(relative_path):
return
if os.path.isfile(real_path):
handle_file(configuration, listing, relative_path, relative_path,
real_path, flags, dest, show_dest)
else:
try:
contents = os.listdir(real_path)
except Exception, exc:
output_objects.append({
'object_type':
'error_text',
'text':
'Failed to list contents of %s: %s' % (base_name, exc)
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Filter out dot files unless '-a' is used
if not all(flags):
contents = [i for i in contents if not i.startswith('.')]
contents.sort()
if not recursive(flags) or depth < 0:
for name in contents:
path = real_path + os.sep + name
rel_path = path.replace(base_dir, '')
if os.path.isfile(path):
handle_file(configuration, listing, rel_path, rel_path,
path, flags,
os.path.join(dest, os.path.basename(rel_path)),
show_dest)
else:
# Force pure content listing first by passing a negative depth
handle_expand(
configuration,
output_objects,
listing,
base_dir,
real_path,
flags,
dest,
-1,
show_dest,
)
for name in contents:
path = real_path + os.sep + name
rel_path = path.replace(base_dir, '')
if os.path.isdir(path):
handle_expand(
configuration,
output_objects,
listing,
base_dir,
path,
flags,
os.path.join(dest, name),
depth + 1,
show_dest,
)
def handle_dir(
configuration,
listing,
dirname,
dirname_with_dir,
actual_dir,
flags='',
):
"""handle a dir"""
# Recursion can get here when called without explicit invisible files
if invisible_path(dirname_with_dir):
return
special = ''
extra_class = ''
real_dir = os.path.realpath(actual_dir)
abs_dir = os.path.abspath(actual_dir)
# If we followed a symlink it is not a plain dir
if real_dir != abs_dir:
access_type = configuration.site_vgrid_label
parent_dir = os.path.basename(os.path.dirname(actual_dir))
# configuration.logger.debug("checking link %s type (%s)" % \
# (dirname_with_dir, real_dir))
# Separate vgrid special dirs from plain ones
if in_vgrid_share(configuration, actual_dir) == dirname_with_dir:
dir_type = 'shared files'
extra_class = 'vgridshared'
elif in_vgrid_readonly(configuration, actual_dir) == dirname_with_dir:
access_type = 'read-only'
dir_type = 'shared files'
extra_class = 'vgridshared readonly'
elif in_vgrid_pub_web(configuration, actual_dir) == \
dirname_with_dir[len('public_base/'):]:
dir_type = 'public web page'
extra_class = 'vgridpublicweb'
elif in_vgrid_priv_web(configuration, actual_dir) == \
dirname_with_dir[len('private_base/'):]:
dir_type = 'private web page'
extra_class = 'vgridprivateweb'
# NOTE: in_vgrid_X returns None on miss, so don't use replace directly
elif (in_vgrid_store_res(configuration, actual_dir) or '').replace(
os.sep, '_') == os.path.basename(dirname_with_dir):
dir_type = 'storage resource files'
if os.path.islink(actual_dir):
extra_class = 'vgridstoreres'
elif real_dir.startswith(configuration.seafile_mount):
access_type = 'read-only'
dir_type = 'Seafile library access'
if os.path.islink(actual_dir):
extra_class = 'seafilereadonly'
elif real_dir.find(trash_destdir) != -1:
access_type = 'recently deleted data'
dir_type = 'sub'
else:
dir_type = 'sub'
# TODO: improve this greedy matching here?
# configuration.logger.debug("check real_dir %s vs %s" % (real_dir,
# trash_destdir))
if real_dir.endswith(trash_destdir):
dir_type = ''
extra_class = 'trashbin'
special = ' - %s %s directory' % (access_type, dir_type)
dir_obj = {
'object_type': 'direntry',
'type': 'directory',
'name': dirname,
'rel_path': dirname_with_dir,
'rel_path_enc': quote(dirname_with_dir),
'rel_dir_enc': quote(dirname_with_dir),
# NOTE: dirname_with_dir is kept for backwards compliance
'dirname_with_dir': dirname_with_dir,
'flags': flags,
'special': special,
'extra_class': extra_class,
}
if long_list(flags):
dir_obj['actual_dir'] = long_format(actual_dir)
if file_info(flags):
dir_obj['file_info'] = fileinfo_stat(actual_dir)
listing.append(dir_obj)
def handle_expand(
output_objects,
listing,
base_dir,
real_path,
flags='',
dest='',
depth=0,
show_dest=False,
):
"""Recursive function to expand paths in a way not unlike ls, but only
files are interesting in this context. The order of recursively expanded
paths is different from that in ls since it simplifies the code and
doesn't really matter to the clients.
"""
# Sanity check
if depth > 255:
output_objects.append({'object_type': 'error_text', 'text'
: 'Error: file recursion maximum exceeded!'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# references to '.' or similar are stripped by abspath
if real_path + os.sep == base_dir:
base_name = relative_path = '.'
else:
base_name = os.path.basename(real_path)
relative_path = real_path.replace(base_dir, '')
if invisible_path(relative_path):
return
if os.path.isfile(real_path):
handle_file(
listing,
relative_path,
relative_path,
real_path,
flags,
dest,
show_dest,
)
else:
try:
contents = os.listdir(real_path)
except Exception, exc:
output_objects.append({'object_type': 'error_text', 'text'
: 'Failed to list contents of %s: %s'
% (base_name, exc)})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Filter out dot files unless '-a' is used
if not all(flags):
contents = [i for i in contents if not i.startswith('.')]
contents.sort()
if not recursive(flags) or depth < 0:
for name in contents:
path = real_path + os.sep + name
rel_path = path.replace(base_dir, '')
if os.path.isfile(path):
handle_file(
listing,
rel_path,
rel_path,
path,
flags,
os.path.join(dest, os.path.basename(rel_path)),
show_dest,
)
else:
# Force pure content listing first by passing a negative depth
handle_expand(
output_objects,
listing,
base_dir,
real_path,
flags,
dest,
-1,
show_dest,
)
for name in contents:
path = real_path + os.sep + name
rel_path = path.replace(base_dir, '')
if os.path.isdir(path):
handle_expand(
output_objects,
listing,
base_dir,
path,
flags,
os.path.join(dest, name),
depth + 1,
show_dest,
)
# Backwards compatibility
if fileinfo_dict.has_key('filename'):
fileinfo_dict['path'] = fileinfo_dict['filename']
if not fileinfo_dict.has_key('path'):
# Check if path was in querystring.
o.out('No path provided, unable to process file!')
o.reply_and_exit(o.ERROR)
# Reject modification of invisible files, even if apache generally allows it
if invisible_path(fileinfo_dict['path']):
o.out('Access to %s is prohibited!' % fileinfo_dict['path'])
o.reply_and_exit(o.CLIENT_ERROR)
logger.info('rangefileaccess on %s (%s)' %
(fileinfo_dict['path'], fileinfo_dict))
if client_id:
# logger.debug("Certificate found as a user cert: " + client_id)
fileinfo_dict['base_path'] = \
os.path.normpath(os.path.join(configuration.user_home,
client_dir)) + os.sep
elif fileinfo_dict.has_key('iosessionid'):
def handle_ls(
configuration,
output_objects,
listing,
base_dir,
real_path,
flags='',
depth=0,
):
"""Recursive function to emulate GNU ls (-R)"""
# Sanity check
if depth > 255:
output_objects.append({'object_type': 'error_text', 'text':
'Error: file recursion maximum exceeded!'
})
return (output_objects, returnvalues.SYSTEM_ERROR)
# references to '.' or similar are stripped by abspath
if real_path + os.sep == base_dir:
base_name = relative_path = '.'
else:
base_name = os.path.basename(real_path)
relative_path = real_path.replace(base_dir, '')
# Recursion can get here when called without explicit invisible files
if invisible_path(relative_path):
return
if os.path.isfile(real_path):
handle_file(configuration, listing, base_name, relative_path,
real_path, flags)
else:
try:
contents = os.listdir(real_path)
except Exception, exc:
output_objects.append({'object_type': 'error_text', 'text':
'Failed to list contents of %s: %s'
% (base_name, exc)})
return (output_objects, returnvalues.SYSTEM_ERROR)
# Filter out dot files unless '-a' is used
if not all(flags):
contents = [i for i in contents if not i.startswith('.')]
contents.sort()
if not recursive(flags) or depth < 0:
# listdir does not include '.' and '..' - add manually
# to ease navigation
if all(flags):
handle_dir(configuration, listing, '.', relative_path,
real_path, flags)
handle_dir(configuration, listing, '..',
os.path.dirname(relative_path),
os.path.dirname(real_path), flags)
for name in contents:
path = real_path + os.sep + name
rel_path = path.replace(base_dir, '')
if os.path.isfile(path):
handle_file(configuration, listing, name, rel_path, path,
flags)
else:
handle_dir(configuration, listing, name, rel_path, path,
flags)
else:
# Force pure content listing first by passing a negative depth
handle_ls(
configuration,
output_objects,
listing,
base_dir,
real_path,
flags,
-1,
)
for name in contents:
path = real_path + os.sep + name
rel_path = path.replace(base_dir, '')
if os.path.isdir(path):
handle_ls(
configuration,
output_objects,
listing,
base_dir,
path,
flags,
depth + 1,
)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
return (accepted, returnvalues.CLIENT_ERROR)
flags = ''.join(accepted['flags'])
pattern_list = accepted['path']
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
status = returnvalues.OK
if verbose(flags):
for flag in flags:
output_objects.append({'object_type': 'text', 'text':
'%s using flag: %s' % (op_name, flag)})
for pattern in pattern_list:
# Check directory traversal attempts before actual handling to avoid
# leaking information about file system layout while allowing
# consistent error messages
unfiltered_match = glob.glob(base_dir + pattern)
match = []
for server_path in unfiltered_match:
# IMPORTANT: path must be expanded to abs for proper chrooting
abs_path = os.path.abspath(server_path)
if not valid_user_path(configuration, abs_path, base_dir, True):
# out of bounds - save user warning for later to allow
# partial match:
# ../*/* is technically allowed to match own files.
logger.warning('%s tried to %s restricted path %s ! (%s)'
% (client_id, op_name, abs_path, pattern))
continue
match.append(abs_path)
# Now actually treat list of allowed matchings and notify if no
# (allowed) match
if not match:
output_objects.append({'object_type': 'file_not_found',
'name': pattern})
status = returnvalues.FILE_NOT_FOUND
# NOTE: we produce output matching an invocation of:
# du -aL --apparent-size --block-size=1 PATH [PATH ...]
filedus = []
summarize_output = summarize(flags)
for abs_path in match:
if invisible_path(abs_path):
continue
relative_path = abs_path.replace(base_dir, '')
# cache accumulated sub dir sizes - du sums into parent dir size
dir_sizes = {}
try:
# Assume a directory to walk
for (root, dirs, files) in walk(abs_path, topdown=False,
followlinks=True):
if invisible_path(root):
continue
dir_bytes = 0
for name in files:
real_file = os.path.join(root, name)
if invisible_path(real_file):
continue
relative_file = real_file.replace(base_dir, '')
size = os.path.getsize(real_file)
dir_bytes += size
if not summarize_output:
filedus.append({'object_type': 'filedu',
'name': relative_file,
'bytes': size})
for name in dirs:
real_dir = os.path.join(root, name)
if invisible_path(real_dir):
continue
dir_bytes += dir_sizes[real_dir]
relative_root = root.replace(base_dir, '')
dir_bytes += os.path.getsize(root)
dir_sizes[root] = dir_bytes
if root == abs_path or not summarize_output:
filedus.append({'object_type': 'filedu',
'name': relative_root,
'bytes': dir_bytes})
if os.path.isfile(abs_path):
# Fall back to plain file where walk is empty
size = os.path.getsize(abs_path)
filedus.append({'object_type': 'filedu',
'name': relative_path,
'bytes': size})
except Exception, exc:
output_objects.append({'object_type': 'error_text',
'text': "%s: '%s': %s" % (op_name,
relative_path,
exc)})
logger.error("%s: failed on '%s': %s" % (op_name,
relative_path, exc))
status = returnvalues.SYSTEM_ERROR
continue
output_objects.append({'object_type': 'filedus', 'filedus': filedus})
