def Walk(self, root):
'''Recursively walk the directories in a file system, starting with root.
Behaviour is very similar to os.walk from the standard os module, yielding
(base, dirs, files) recursively, where |base| is the base path of |files|,
|dirs| relative to |root|, and |files| and |dirs| the list of files/dirs in
|base| respectively.
Note that directories will always end with a '/', files never will.
If |root| cannot be found, raises a FileNotFoundError.
For any other failure, raises a FileSystemError.
'''
AssertIsDirectory(root)
basepath = root
def walk(root):
AssertIsDirectory(root)
dirs, files = [], []
for f in self.ReadSingle(root).Get():
if IsDirectory(f):
dirs.append(f)
else:
files.append(f)
yield root[len(basepath):].rstrip('/'), dirs, files
for d in dirs:
for walkinfo in walk(root + d):
yield walkinfo
for walkinfo in walk(root):
yield walkinfo
def __init__(self, server_instance, _, partial_dir=PRIVATE_TEMPLATES):
AssertIsDirectory(partial_dir)
self._template_cache = server_instance.compiled_fs_factory.ForTemplates(
server_instance.host_file_system_provider.GetTrunk())
self._partial_dir = partial_dir
self._file_system = server_instance.host_file_system_provider.GetTrunk(
)
def __init__(self, server_instance, request=None):
self._dir = type(self)._BASE
AssertIsDirectory(self._dir)
self._request = request
self._template_cache = server_instance.compiled_fs_factory.ForTemplates(
server_instance.host_file_system_provider.GetTrunk())
self._file_system = server_instance.host_file_system_provider.GetTrunk()
def _ListDir(self, path, recursive=False):
AssertIsDirectory(path)
# The listbucket method uses a prefix approach to simulate hierarchy.
# Calling it with the "delimiter" argument set to '/' gets only files
# directly inside the directory, not all recursive content.
# Subdirectories are returned in the 'prefixes' property, but they are
# full paths from the root. This plucks off the name of the leaf with a
# trailing slash.
def path_from_prefix(prefix):
return posixpath.split(posixpath.split(prefix)[0])[1] + '/'
query = {'prefix': path}
if not recursive:
query['delimiter'] = '/'
root_object = json.loads(self._FetchObject('', query=query))
files = [
posixpath.basename(o['name'])
for o in root_object.get('items', [])
]
dirs = [
path_from_prefix(prefix)
for prefix in root_object.get('prefixes', [])
]
return files + dirs
def List(self, path):
'''Returns all files within a directory at |path|. Not recursive. Paths
are returned relative to |path|.
'''
AssertIsDirectory(path)
return [
p[len(path):] for p in self._paths if p != path
and p.startswith(path) and '/' not in p[len(path):].rstrip('/')
]
def MoveTo(base, obj):
'''Returns an object as |obj| moved to |base|. That is,
MoveTo('foo/bar', {'a': 'b'}) -> {'foo': {'bar': {'a': 'b'}}}
'''
AssertIsDirectory(base)
result = {}
leaf = result
for k in base.rstrip('/').split('/'):
leaf[k] = {}
leaf = leaf[k]
leaf.update(obj)
return result
def _ListDir(dir_name, recursive=False):
AssertIsDirectory(dir_name)
try:
# The listbucket method uses a prefix approach to simulate hierarchy.
# Calling it with the "delimiter" argument set to '/' gets only files
# directly inside the directory, not all recursive content.
delimiter = None if recursive else '/'
files = cloudstorage_api.listbucket('/' + dir_name, delimiter=delimiter)
return [os_path.filename.lstrip('/')[len(dir_name):] for os_path in files]
except errors.Error:
raise FileNotFoundError('cloudstorage.listbucket failed for %s: %s' %
(dir_name, traceback.format_exc()))
def Walk(self, root, depth=-1, file_lister=None):
'''Recursively walk the directories in a file system, starting with root.
Behaviour is very similar to os.walk from the standard os module, yielding
(base, dirs, files) recursively, where |base| is the base path of |files|,
|dirs| relative to |root|, and |files| and |dirs| the list of files/dirs in
|base| respectively. If |depth| is specified and greater than 0, Walk will
only recurse |depth| times.
|file_lister|, if specified, should be a callback of signature
def my_file_lister(root):,
which returns a tuple (dirs, files), where |dirs| is a list of directory
names under |root|, and |files| is a list of file names under |root|. Note
that the listing of files and directories should be for a *single* level
only, i.e. it should not recursively list anything.
Note that directories will always end with a '/', files never will.
If |root| cannot be found, raises a FileNotFoundError.
For any other failure, raises a FileSystemError.
'''
AssertIsDirectory(root)
basepath = root
def walk(root, depth):
if depth == 0:
return
AssertIsDirectory(root)
if file_lister:
dirs, files = file_lister(root)
else:
dirs, files = [], []
for f in self.ReadSingle(root).Get():
if IsDirectory(f):
dirs.append(f)
else:
files.append(f)
yield root[len(basepath):].rstrip('/'), dirs, files
for d in dirs:
for walkinfo in walk(root + d, depth - 1):
yield walkinfo
for walkinfo in walk(root, depth):
yield walkinfo
def walk(root):
AssertIsDirectory(root)
dirs, files = [], []
for f in self.ReadSingle(root).Get():
if IsDirectory(f):
dirs.append(f)
else:
files.append(f)
yield root[len(basepath):].rstrip('/'), dirs, files
for d in dirs:
for walkinfo in walk(root + d):
yield walkinfo
def __init__(self, file_system, renderer, public_path, root_pages):
'''Creates a new broken link detector. |renderer| is a callable that takes
a path and returns a full html page. |public_path| is the path to public
template files. All URLs in |root_pages| are used as the starting nodes for
the orphaned page search.
'''
AssertIsDirectory(public_path)
self._file_system = file_system
self._renderer = renderer
self._public_path = public_path
self._pages = defaultdict(lambda: Page(404, (), (), ()))
self._root_pages = frozenset(root_pages)
self._always_detached = frozenset(
('apps/404.html', 'extensions/404.html', 'apps/private_apis.html',
'extensions/private_apis.html'))
self._redirection_whitelist = frozenset(('extensions/', 'apps/'))
self._RenderAllPages()
def walk(root, depth):
if depth == 0:
return
AssertIsDirectory(root)
if file_lister:
dirs, files = file_lister(root)
else:
dirs, files = [], []
for f in self.ReadSingle(root).Get():
if IsDirectory(f):
dirs.append(f)
else:
files.append(f)
yield root[len(basepath):].rstrip('/'), dirs, files
for d in dirs:
for walkinfo in walk(root + d, depth - 1):
yield walkinfo
def GetFromFileListing(self, path):
'''Calls |compilation_function| on the listing of the files at |path|.
Assumes that the path given is to a directory.
'''
AssertIsDirectory(path)
try:
version = self._file_system.Stat(path).version
except FileNotFoundError:
return Future(exc_info=sys.exc_info())
cache_entry = self._list_object_store.Get(path).Get()
if (cache_entry is not None) and (version == cache_entry.version):
return Future(value=cache_entry._cache_data)
def next(files):
cache_data = self._compilation_function(path, files)
self._list_object_store.Set(path, _CacheEntry(cache_data, version))
return cache_data
return self._RecursiveList(path).Then(next)
def __init__(self, base_path):
# Enforce POSIX path, so path validity checks pass for Windows.
base_path = base_path.replace(os.sep, '/')
AssertIsDirectory(base_path)
self._base_path = _ConvertToFilepath(base_path)
def __init__(self, file_system, base_path):
AssertIsDirectory(base_path)
self._base_path = base_path
self._file_system = file_system
def __init__(self, base_path):
AssertIsDirectory(base_path)
self._base_path = _ConvertToFilepath(base_path)
