def Stat(self, path):
'''Stats |path| returning its version as as StatInfo object. If |path| ends
with a '/', it is assumed to be a directory and the StatInfo object returned
includes child_versions for all paths in the directory.
File paths do not include the name of the zip file, which is arbitrary and
useless to consumers.
Because the repository will only be downloaded once per server version, all
stat versions are always 0.
'''
self._EnsureRepoZip()
repo_zip = self._repo_zip.Get()
if path not in repo_zip.Paths():
raise FileNotFoundError('"%s" does not contain file "%s"' %
(self._repo_key, path))
version = self._stat_cache.Get(self._repo_key).Get()
assert version is not None, ('There was a zipball in datastore; there '
'should be a version cached for it')
stat_info = StatInfo(version)
if IsDirectory(path):
stat_info.child_versions = dict(
(p, StatInfo(version)) for p in repo_zip.List(path))
return stat_info
def _MemoizedStatAsyncFromFileSystem(self, dir_path):
'''This is a simple wrapper to memoize Futures to directory stats, since
StatAsync makes heavy use of it. Only cache directories so that the
memoized cache doesn't blow up.
'''
assert IsDirectory(dir_path)
return self._file_system.StatAsync(dir_path)
def Update(self, update):
self._updates.append(TestFileSystem(update))
for path in _List(update).iterkeys():
# Any files (not directories) which changed are now at the version
# derived from |_updates|.
if not IsDirectory(path):
self._stat_tracker.SetVersion(path, len(self._updates))
def StatAsync(self, path):
def get_child_versions(path):
return dict((e['name'], e['id'])
for e in local_git_util.ListDir(path, self._commit))
def get_file_version(dir, filename):
try:
return next(e['id'] for e in local_git_util.ListDir(dir, self._commit)
if e['name'] == filename)
except StopIteration:
raise FileNotFoundError('%s not found in revision %s' %
(path, self._commit))
dir, filename = posixpath.split(path)
if path == '':
version = local_git_util.GetRootTree(self._commit)
child_versions = get_child_versions('')
elif IsDirectory(path):
parent_dir, stat_dir = posixpath.split(dir)
version = get_file_version(parent_dir, stat_dir)
child_versions = get_child_versions(dir)
else:
version = get_file_version(dir, filename)
child_versions = None
#print 'Accessing local git for stat on %s (%s)' % (path, version)
return Future(value=StatInfo(version, child_versions))
def resolve():
assert '?' in url
if url == _BASE_URL + '?format=JSON':
return _Response(json.dumps({'commit': 'a_commit'}))
path, fmt = url.split('?')
# Fetch urls are of the form <base_url>/<path>. We only want <path>.
path = path.split('/', 1)[1]
if path == _REAL_DATA_DIR:
return _Response(ReadFile(*_TEST_DATA))
# ALWAYS skip not found here.
content = self._fs.Read((path, ),
skip_not_found=True).Get().get(path, None)
if content is None:
# GitilesFS expects a DownloadError if the file wasn't found.
raise DownloadError
# GitilesFS expects directory content as a JSON string.
if 'JSON' in fmt:
content = json.dumps({
'entries': [
{
# GitilesFS expects directory names to not have a trailing '/'.
'name': name.rstrip('/'),
'type': 'tree' if IsDirectory(name) else 'blob'
} for name in content
]
})
return _Response(content)
def resolve():
result = {}
for path in paths:
if IsDirectory(path):
result[path] = self._ListDir(path)
else:
result[path] = self._ReadFile(path)
return result
def Read(self, paths):
result = {}
for path in paths:
if not IsDirectory(path):
raise FileNotFoundError('EmptyDirFileSystem cannot read %s' %
path)
result[path] = []
return Future(value=result)
def _DoFetch(self, url):
url = url.rsplit('?', 1)[0]
result = _Response()
if IsDirectory(url):
result.content = self._ListDir(url)
else:
result.content = self._ReadFile(url)
return result
def _CreateStatInfo(self, path):
if not self._last_commit_hash:
self._last_commit_hash = self._ReadFile(_LAST_COMMIT_HASH_FILENAME)
if IsDirectory(path):
child_versions = dict((filename, self._last_commit_hash)
for filename in self._ListDir(path))
else:
child_versions = None
return StatInfo(self._last_commit_hash, child_versions)
def parse_contents(results):
value = {}
for path, content in izip(paths, results):
if content is None:
continue
# Gitiles encodes text content in base64 (see
# http://tools.ietf.org/html/rfc4648 for info about base64).
value[path] = (list_dir if IsDirectory(path) else b64decode)(content)
return value
def Read(self, paths, skip_not_found=False):
result = {}
for path in paths:
if not IsDirectory(path):
if skip_not_found: continue
raise FileNotFoundError('EmptyDirFileSystem cannot read %s' %
path)
result[path] = []
return Future(value=result)
def read(repo_zip):
reads = {}
for path in paths:
if path not in repo_zip.Paths():
raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path))
if IsDirectory(path):
reads[path] = repo_zip.List(path)
else:
reads[path] = repo_zip.Read(path)
return reads
def stat(content):
stat_info = _CreateStatInfo(content)
if stat_info.version is None:
raise FileSystemError('Failed to find version of dir %s' % dir_)
if IsDirectory(path):
return stat_info
if filename not in stat_info.child_versions:
raise FileNotFoundError(
'%s from %s was not in child versions for Stat' % (filename, path))
return StatInfo(stat_info.child_versions[filename])
def read_path(path):
try:
if IsDirectory(path):
return [get_entry_name(e)
for e in local_git_util.ListDir(path, self._commit)]
else:
return local_git_util.ReadFile(path, self._commit)
except FileNotFoundError as e:
if skip_not_found:
return None
raise e
def Read(self, paths, skip_not_found=False):
version = self.Stat(ZIP_KEY).version
if version != self._version:
self._GetZip(version)
result = {}
for path in paths:
if IsDirectory(path):
result[path] = self._ListDir(path)
else:
result[path] = self._ReadFile(path)
return Future(value=result)
def _CreateStatInfo(bucket, path):
full_path = Join(bucket, path)
last_commit_file = Join(bucket, LAST_COMMIT_HASH_FILENAME)
try:
last_commit = _ReadFile(last_commit_file)
if IsDirectory(full_path):
child_versions = dict(
(filename, last_commit) for filename in _ListDir(full_path))
else:
child_versions = None
return StatInfo(last_commit, child_versions)
except (TypeError, errors.Error):
raise FileNotFoundError('cloudstorage.stat failed for %s: %s' %
(path, traceback.format_exc()))
def resolve():
try:
result = {}
for path in paths:
full_path = Join(self._bucket, path)
logging.debug('gcs: requested path "%s", reading "%s"' %
(path, full_path))
if IsDirectory(path):
result[path] = _ListDir(full_path)
else:
result[path] = _ReadFile(full_path)
return result
except errors.AuthorizationError:
self._warnAboutAuthError()
raise
def walk(root):
AssertIsDirectory(root)
dirs, files = [], []
for f in self.ReadSingle(root).Get():
if IsDirectory(f):
dirs.append(f)
else:
files.append(f)
yield root[len(basepath):].rstrip('/'), dirs, files
for d in dirs:
for walkinfo in walk(root + d):
yield walkinfo
def file_lister(root):
res, root_stat = All(
(self._walk_cache.Get(root), self.StatAsync(root))).Get()
if res and res[2] == root_stat.version:
dirs, files = res[0], res[1]
else:
# Wasn't cached, or not up to date.
dirs, files = [], []
for f in self.ReadSingle(root).Get():
if IsDirectory(f):
dirs.append(f)
else:
files.append(f)
# Update the cache. This is a root -> (dirs, files, version) mapping.
self._walk_cache.Set(root, (dirs, files, root_stat.version))
return dirs, files
def walk(root, depth):
if depth == 0:
return
AssertIsDirectory(root)
if file_lister:
dirs, files = file_lister(root)
else:
dirs, files = [], []
for f in self.ReadSingle(root).Get():
if IsDirectory(f):
dirs.append(f)
else:
files.append(f)
yield root[len(basepath):].rstrip('/'), dirs, files
for d in dirs:
for walkinfo in walk(root + d, depth - 1):
yield walkinfo
def fetch(self, url):
path = _ExtractPathFromSvnUrl(url)
if IsDirectory(path):
html = ['<html>Revision 000000']
try:
for f in self._ListDir(ChromiumPath(path)):
if f.startswith('.'):
continue
if self._IsDir(ChromiumPath(path, f)):
html.append('<a>' + f + '/</a>')
else:
html.append('<a>' + f + '</a>')
html.append('</html>')
return '\n'.join(html)
except OSError as e:
return None
try:
return ReadFile(path)
except IOError:
return None
def SetVersion(self, path, new_version):
if IsDirectory(path):
raise ValueError('Only files have an incrementable stat, '
'but "%s" is a directory' % path)
# Update version of that file.
self._path_stats[path] = new_version
# Update all parent directory versions as well.
slash_index = 0 # (deliberately including '' in the dir paths)
while slash_index != -1:
dir_path = path[:slash_index] + '/'
self._path_stats[dir_path] = max(self._path_stats.get(dir_path, 0),
new_version)
if dir_path == '/':
# Legacy support for '/' being the root of the file system rather
# than ''. Eventually when the path normalisation logic is complete
# this will be impossible and this logic will change slightly.
self._path_stats[''] = self._path_stats['/']
slash_index = path.find('/', slash_index + 1)
def Stat(self, path):
if not IsDirectory(path):
raise FileNotFoundError('EmptyDirFileSystem cannot stat %s' % path)
return StatInfo(0, child_versions=[])
def Create(self, bucket):
'''Creates a CloudStorageFileSystemProvider.
|bucket| is the name of GCS bucket, eg devtools-docs. It is expected
that this bucket has Read permission for this app in its ACLs.
Optional configuration can be set in a local_debug/gcs_debug.conf file:
use_local_fs=True|False
access_token=<token>
remote_bucket_prefix=<prefix>
If running in Preview mode or in Development mode with use_local_fs set to
True, buckets and files are looked inside the local_debug folder instead
of in the real GCS server. Preview server does not support direct GCS
access, so it is always forced to use a LocalFileSystem.
For real GCS access in the Development mode (dev_appserver.py),
access_token and remote_bucket_prefix options can be
used to change the way GCS files are accessed. Both are ignored in a real
appengine instance.
"access_token" is always REQUIRED on dev_appengine, otherwise you will
get 404 (auth) errors. You can get one access_token valid for a few minutes
by typing:
gsutil -d ls 2>&1 | grep "Bearer" |
sed "s/.*Bearer \(.*\).r.nUser-Agent.*/access_token=\1/" )"
A sample output would be:
access_token=ya29.1.AADtN_VW5ibbfLHV5cMIK5ss4bHtVzBXpa4byjd
Now add this line to the local_debug/gcs_debug.conf file and restart the
appengine development server.
Remember that you will need a new access_token every ten minutes or
so. If you get 404 errors on log, update it. Access token is not
used for a deployed appengine app, only if you use dev_appengine.py.
remote_bucket_prefix is useful if you want to test on your own GCS buckets
before using the real GCS buckets.
'''
if not environment.IsReleaseServer() and not environment.IsDevServer():
bucket_local_path = os.path.join(LOCAL_GCS_DIR, bucket)
if IsDirectory(bucket_local_path):
return LocalFileSystem(bucket_local_path)
else:
return EmptyDirFileSystem()
debug_access_token = None
debug_bucket_prefix = None
use_local_fs = False
if environment.IsDevServer() and os.path.exists(LOCAL_GCS_DEBUG_CONF):
with open(LOCAL_GCS_DEBUG_CONF, "r") as token_file:
properties = dict(line.strip().split('=', 1)
for line in token_file)
use_local_fs = properties.get('use_local_fs', 'False') == 'True'
debug_access_token = properties.get('access_token', None)
debug_bucket_prefix = properties.get('remote_bucket_prefix', None)
if environment.IsDevServer() and use_local_fs:
return LocalFileSystem(os.path.join(LOCAL_GCS_DIR, bucket))
# gcs_file_system has strong dependencies on runtime appengine APIs,
# so we only import it when we are sure we are not on preview.py or tests.
from gcs_file_system import CloudStorageFileSystem
return CachingFileSystem(
CloudStorageFileSystem(bucket, debug_access_token,
debug_bucket_prefix),
self._object_store_creator)
def fixup_url_format(path): # By default, Gitiles URLs display resources in HTML. To get resources # suitable for our consumption, a '?format=' string must be appended to # the URL. The format may be one of 'JSON' or 'TEXT' for directory or # text resources, respectively. return path + (_JSON_FORMAT if IsDirectory(path) else _TEXT_FORMAT)