def GetContentAndType(self, path):
'''Returns the ContentAndType of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
# Check for a zip file first, if zip is enabled.
if self._directory_zipper and ext == '.zip':
zip_future = self._directory_zipper.Zip(ToDirectory(base))
return Future(callback=lambda: ContentAndType(
zip_future.Get(), 'application/zip', None))
# If there is no file extension, look for a file with one of the default
# extensions. If one cannot be found, check if the path is a directory.
# If it is, then check for an index file with one of the default
# extensions.
if not ext:
new_path = self._AddExt(path)
# Add a trailing / to check if it is a directory and not a file with
# no extension.
if new_path is None and self.file_system.Exists(
ToDirectory(path)).Get():
new_path = self._AddExt(Join(path, 'index'))
# If an index file wasn't found in this directly then we're never going
# to find a file.
if new_path is None:
return FileNotFoundError.RaiseInFuture(
'"%s" is a directory' % path)
if new_path is not None:
path = new_path
return self._content_cache.GetFromFile(path)
def GetContentAndType(self, path):
'''Returns the ContentAndType of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
# Check for a zip file first, if zip is enabled.
if self._directory_zipper and ext == '.zip':
zip_future = self._directory_zipper.Zip(ToDirectory(base))
return Future(callback=lambda: ContentAndType(
zip_future.Get(), 'application/zip', None))
# If there is no file extension, look for a file with one of the default
# extensions.
#
# Note that it would make sense to guard this on Exists(path), since a file
# without an extension may actually exist, but it's such an uncommon case
# it hardly seems worth the potential performance hit.
if not ext:
for default_ext in self._default_extensions:
if self.file_system.Exists(path + default_ext).Get():
path += default_ext
break
return self._content_cache.GetFromFile(path)
def StatAsync(self, path):
'''Stats the directory given, or if a file is given, stats the file's parent
directory to get info about the file.
'''
# Always stat the parent directory, since it will have the stat of the child
# anyway, and this gives us an entire directory's stat info at once.
dir_path, file_path = posixpath.split(path)
dir_path = ToDirectory(dir_path)
def make_stat_info(dir_stat):
'''Converts a dir stat into the correct resulting StatInfo; if the Stat
was for a file, the StatInfo should just contain that file.
'''
if path == dir_path:
return dir_stat
# Was a file stat. Extract that file.
file_version = dir_stat.child_versions.get(file_path)
if file_version is None:
raise FileNotFoundError(
'No stat found for %s in %s (found %s)' %
(path, dir_path, dir_stat.child_versions))
return StatInfo(file_version)
dir_stat = self._stat_object_store.Get(dir_path).Get()
if dir_stat is not None:
return Future(value=make_stat_info(dir_stat))
def next(dir_stat):
assert dir_stat is not None # should have raised a FileNotFoundError
# We only ever need to cache the dir stat.
self._stat_object_store.Set(dir_path, dir_stat)
return make_stat_info(dir_stat)
return self._MemoizedStatAsyncFromFileSystem(dir_path).Then(next)
def _GetFileListingVersionFromCache(self, path):
path = ToDirectory(path)
cache_entry = self._Get(self._list_object_store, path).Get()
if cache_entry is not None:
return Future(value=cache_entry.version)
stat_future = self._file_system.StatAsync(path)
return Future(callback=lambda: stat_future.Get().version)
def find_index_file():
'''Tries to find an index file in |path|, if |path| is a directory.
If not, or if there is no index file, returns None.
'''
def get_index_if_directory_exists(directory_exists):
if not directory_exists:
return None
return find_file_with_name(Join(path, 'index'))
return (self.file_system.Exists(ToDirectory(path))
.Then(get_index_if_directory_exists))
def GetVersion(self, path):
'''Returns a Future to the version of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
stat_future = self.file_system.StatAsync(ToDirectory(base))
else:
stat_future = self._FindFileForPath(path).Then(self.file_system.StatAsync)
return stat_future.Then(lambda stat: stat.version)
def GetContentAndType(self, path):
'''Returns a Future to the ContentAndType of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
return (self._directory_zipper.Zip(ToDirectory(base))
.Then(lambda zipped: ContentAndType(zipped,
'application/zip',
None)))
return self._FindFileForPath(path).Then(self._content_cache.GetFromFile)
def StatAsync(self, path):
dir_, filename = posixpath.split(path)
def stat(content):
stat_info = _CreateStatInfo(content)
if stat_info.version is None:
raise FileSystemError('Failed to find version of dir %s' % dir_)
if IsDirectory(path):
return stat_info
if filename not in stat_info.child_versions:
raise FileNotFoundError(
'%s from %s was not in child versions for Stat' % (filename, path))
return StatInfo(stat_info.child_versions[filename])
fetch_future = self._FetchAsync(ToDirectory(dir_) + _JSON_FORMAT)
return self._ResolveFetchContent(path, fetch_future).Then(stat)
def StatAsync(self, path):
'''Stats the directory given, or if a file is given, stats the file's parent
directory to get info about the file.
'''
# Always stat the parent directory, since it will have the stat of the child
# anyway, and this gives us an entire directory's stat info at once.
dir_path, file_path = posixpath.split(path)
dir_path = ToDirectory(dir_path)
def make_stat_info(dir_stat):
'''Converts a dir stat into the correct resulting StatInfo; if the Stat
was for a file, the StatInfo should just contain that file.
'''
if path == dir_path:
return dir_stat
# Was a file stat. Extract that file.
file_version = dir_stat.child_versions.get(file_path)
if file_version is None:
raise FileNotFoundError('No stat found for %s in %s (found %s)' %
(path, dir_path, dir_stat.child_versions))
return StatInfo(file_version)
def raise_cache_miss(path):
raise FileNotFoundError('Got cache miss when trying to stat %s' % path)
dir_stat = self._stat_cache.Get(dir_path).Get()
if dir_stat is not None:
return Future(callback=lambda: make_stat_info(dir_stat))
if self._fail_on_miss:
logging.warning('Bailing on stat cache miss for %s on %s' %
(dir_path, self.GetIdentity()))
return Future(callback=lambda: raise_cache_miss(dir_path))
def next(dir_stat):
assert dir_stat is not None # should have raised a FileNotFoundError
# We only ever need to cache the dir stat.
self._stat_cache.Set(dir_path, dir_stat)
return make_stat_info(dir_stat)
return self._MemoizedStatAsyncFromFileSystem(dir_path).Then(next)
def Exists(self, path):
'''Returns a Future to the existence of |path|; True if |path| exists,
False if not. This method will not throw a FileNotFoundError unlike
the Read* methods, however it may still throw a FileSystemError.
There are several ways to implement this method via the interface but this
method exists to do so in a canonical and most efficient way for caching.
'''
AssertIsValid(path)
if path == '':
# There is always a root directory.
return Future(value=True)
parent, base = SplitParent(path)
def handle(error):
if isinstance(error, FileNotFoundError):
return False
raise error
return self.ReadSingle(ToDirectory(parent)).Then(
lambda l: base in l, handle)
def Exists(self, path):
'''Returns a Future to the existence of |path|; True if |path| exists,
False if not. This method will not throw a FileNotFoundError unlike
the Read* methods, however it may still throw a FileSystemError.
There are several ways to implement this method via the interface but this
method exists to do so in a canonical and most efficient way for caching.
'''
AssertIsValid(path)
if path == '':
# There is always a root directory.
return Future(value=True)
parent, base = SplitParent(path)
list_future = self.ReadSingle(ToDirectory(parent))
def resolve():
try:
return base in list_future.Get()
except FileNotFoundError:
return False
return Future(callback=resolve)
def Create(self, bucket):
'''Creates a CloudStorageFileSystemProvider.
|bucket| is the name of GCS bucket, eg devtools-docs. It is expected
that this bucket has Read permission for this app in its ACLs.
Optional configuration can be set in a local_debug/gcs_debug.conf file:
use_local_fs=True|False
remote_bucket_prefix=<prefix>
If running in Preview mode or in Development mode with use_local_fs set to
True, buckets and files are looked for inside the local_debug folder instead
of in the real GCS server.
'''
if IsTest():
return EmptyDirFileSystem()
debug_bucket_prefix = None
use_local_fs = False
if os.path.exists(LOCAL_GCS_DEBUG_CONF):
with open(LOCAL_GCS_DEBUG_CONF, "r") as token_file:
properties = dict(line.strip().split('=', 1)
for line in token_file)
use_local_fs = properties.get('use_local_fs', 'False') == 'True'
debug_bucket_prefix = properties.get('remote_bucket_prefix', None)
logging.debug('gcs: prefixing all bucket names with %s' %
debug_bucket_prefix)
if use_local_fs:
return LocalFileSystem(
ToDirectory(os.path.join(LOCAL_GCS_DIR, bucket)))
if debug_bucket_prefix:
bucket = debug_bucket_prefix + bucket
return CachingFileSystem(CloudStorageFileSystem(bucket),
self._object_store_creator)
def GetFromFileListing(self, path):
path = ToDirectory(path)
return self._GetImpl(
path, lambda compiled_fs: compiled_fs.GetFromFileListing(path),
lambda compiled_fs: compiled_fs.GetFileListingVersion(path))
def GetFileListingVersion(self, path):
path = ToDirectory(path)
cache_entry = self._list_object_store.Get(path).Get()
if cache_entry is not None:
return cache_entry.version
return self._file_system.Stat(path).version
def Create(self, bucket):
'''Creates a CloudStorageFileSystemProvider.
|bucket| is the name of GCS bucket, eg devtools-docs. It is expected
that this bucket has Read permission for this app in its ACLs.
Optional configuration can be set in a local_debug/gcs_debug.conf file:
use_local_fs=True|False
access_token=<token>
remote_bucket_prefix=<prefix>
If running in Preview mode or in Development mode with use_local_fs set to
True, buckets and files are looked inside the local_debug folder instead
of in the real GCS server. Preview server does not support direct GCS
access, so it is always forced to use a LocalFileSystem.
For real GCS access in the Development mode (dev_appserver.py),
access_token and remote_bucket_prefix options can be
used to change the way GCS files are accessed. Both are ignored in a real
appengine instance.
"access_token" is always REQUIRED on dev_appengine, otherwise you will
get 404 (auth) errors. You can get one access_token valid for a few minutes
by typing:
gsutil -d ls 2>&1 | grep "Bearer" |
sed "s/.*Bearer \(.*\).r.nUser-Agent.*/access_token=\1/" )"
A sample output would be:
access_token=ya29.1.AADtN_VW5ibbfLHV5cMIK5ss4bHtVzBXpa4byjd
Now add this line to the local_debug/gcs_debug.conf file and restart the
appengine development server.
Remember that you will need a new access_token every ten minutes or
so. If you get 404 errors on log, update it. Access token is not
used for a deployed appengine app, only if you use dev_appengine.py.
remote_bucket_prefix is useful if you want to test on your own GCS buckets
before using the real GCS buckets.
'''
if not environment.IsReleaseServer() and not environment.IsDevServer():
bucket_local_path = os.path.join(LOCAL_GCS_DIR, bucket)
if IsDirectory(bucket_local_path):
return LocalFileSystem(bucket_local_path)
else:
return EmptyDirFileSystem()
debug_access_token = None
debug_bucket_prefix = None
use_local_fs = False
if environment.IsDevServer() and os.path.exists(LOCAL_GCS_DEBUG_CONF):
with open(LOCAL_GCS_DEBUG_CONF, "r") as token_file:
properties = dict(line.strip().split('=', 1) for line in token_file)
use_local_fs = properties.get('use_local_fs', 'False')=='True'
debug_access_token = properties.get('access_token', None)
debug_bucket_prefix = properties.get('remote_bucket_prefix', None)
if environment.IsDevServer() and use_local_fs:
return LocalFileSystem(ToDirectory(os.path.join(LOCAL_GCS_DIR, bucket)))
# gcs_file_system has strong dependencies on runtime appengine APIs,
# so we only import it when we are sure we are not on preview.py or tests.
from gcs_file_system import CloudStorageFileSystem
return CachingFileSystem(CloudStorageFileSystem(bucket,
debug_access_token, debug_bucket_prefix),
self._object_store_creator)