def Stat(self, path): '''Stats |path| returning its version as as StatInfo object. If |path| ends with a '/', it is assumed to be a directory and the StatInfo object returned includes child_versions for all paths in the directory. File paths do not include the name of the zip file, which is arbitrary and useless to consumers. Because the repository will only be downloaded once per server version, all stat versions are always 0. ''' self._EnsureRepoZip() repo_zip = self._repo_zip.Get() if path not in repo_zip.Paths(): raise FileNotFoundError('"%s" does not contain file "%s"' % (self._repo_key, path)) version = self._stat_cache.Get(self._repo_key).Get() assert version is not None, ('There was a zipball in datastore; there ' 'should be a version cached for it') stat_info = StatInfo(version) if IsDirectory(path): stat_info.child_versions = dict( (p, StatInfo(version)) for p in repo_zip.List(path)) return stat_info
def _MemoizedStatAsyncFromFileSystem(self, dir_path): '''This is a simple wrapper to memoize Futures to directory stats, since StatAsync makes heavy use of it. Only cache directories so that the memoized cache doesn't blow up. ''' assert IsDirectory(dir_path) return self._file_system.StatAsync(dir_path)
def Update(self, update): self._updates.append(TestFileSystem(update)) for path in _List(update).iterkeys(): # Any files (not directories) which changed are now at the version # derived from |_updates|. if not IsDirectory(path): self._stat_tracker.SetVersion(path, len(self._updates))
def StatAsync(self, path): def get_child_versions(path): return dict((e['name'], e['id']) for e in local_git_util.ListDir(path, self._commit)) def get_file_version(dir, filename): try: return next(e['id'] for e in local_git_util.ListDir(dir, self._commit) if e['name'] == filename) except StopIteration: raise FileNotFoundError('%s not found in revision %s' % (path, self._commit)) dir, filename = posixpath.split(path) if path == '': version = local_git_util.GetRootTree(self._commit) child_versions = get_child_versions('') elif IsDirectory(path): parent_dir, stat_dir = posixpath.split(dir) version = get_file_version(parent_dir, stat_dir) child_versions = get_child_versions(dir) else: version = get_file_version(dir, filename) child_versions = None #print 'Accessing local git for stat on %s (%s)' % (path, version) return Future(value=StatInfo(version, child_versions))
def resolve(): assert '?' in url if url == _BASE_URL + '?format=JSON': return _Response(json.dumps({'commit': 'a_commit'})) path, fmt = url.split('?') # Fetch urls are of the form <base_url>/<path>. We only want <path>. path = path.split('/', 1)[1] if path == _REAL_DATA_DIR: return _Response(ReadFile(*_TEST_DATA)) # ALWAYS skip not found here. content = self._fs.Read((path, ), skip_not_found=True).Get().get(path, None) if content is None: # GitilesFS expects a DownloadError if the file wasn't found. raise DownloadError # GitilesFS expects directory content as a JSON string. if 'JSON' in fmt: content = json.dumps({ 'entries': [ { # GitilesFS expects directory names to not have a trailing '/'. 'name': name.rstrip('/'), 'type': 'tree' if IsDirectory(name) else 'blob' } for name in content ] }) return _Response(content)
def resolve(): result = {} for path in paths: if IsDirectory(path): result[path] = self._ListDir(path) else: result[path] = self._ReadFile(path) return result
def Read(self, paths): result = {} for path in paths: if not IsDirectory(path): raise FileNotFoundError('EmptyDirFileSystem cannot read %s' % path) result[path] = [] return Future(value=result)
def _DoFetch(self, url): url = url.rsplit('?', 1)[0] result = _Response() if IsDirectory(url): result.content = self._ListDir(url) else: result.content = self._ReadFile(url) return result
def _CreateStatInfo(self, path): if not self._last_commit_hash: self._last_commit_hash = self._ReadFile(_LAST_COMMIT_HASH_FILENAME) if IsDirectory(path): child_versions = dict((filename, self._last_commit_hash) for filename in self._ListDir(path)) else: child_versions = None return StatInfo(self._last_commit_hash, child_versions)
def parse_contents(results): value = {} for path, content in izip(paths, results): if content is None: continue # Gitiles encodes text content in base64 (see # http://tools.ietf.org/html/rfc4648 for info about base64). value[path] = (list_dir if IsDirectory(path) else b64decode)(content) return value
def Read(self, paths, skip_not_found=False): result = {} for path in paths: if not IsDirectory(path): if skip_not_found: continue raise FileNotFoundError('EmptyDirFileSystem cannot read %s' % path) result[path] = [] return Future(value=result)
def read(repo_zip): reads = {} for path in paths: if path not in repo_zip.Paths(): raise FileNotFoundError('"%s": %s not found' % (self._repo_key, path)) if IsDirectory(path): reads[path] = repo_zip.List(path) else: reads[path] = repo_zip.Read(path) return reads
def stat(content): stat_info = _CreateStatInfo(content) if stat_info.version is None: raise FileSystemError('Failed to find version of dir %s' % dir_) if IsDirectory(path): return stat_info if filename not in stat_info.child_versions: raise FileNotFoundError( '%s from %s was not in child versions for Stat' % (filename, path)) return StatInfo(stat_info.child_versions[filename])
def read_path(path): try: if IsDirectory(path): return [get_entry_name(e) for e in local_git_util.ListDir(path, self._commit)] else: return local_git_util.ReadFile(path, self._commit) except FileNotFoundError as e: if skip_not_found: return None raise e
def Read(self, paths, skip_not_found=False): version = self.Stat(ZIP_KEY).version if version != self._version: self._GetZip(version) result = {} for path in paths: if IsDirectory(path): result[path] = self._ListDir(path) else: result[path] = self._ReadFile(path) return Future(value=result)
def _CreateStatInfo(bucket, path): full_path = Join(bucket, path) last_commit_file = Join(bucket, LAST_COMMIT_HASH_FILENAME) try: last_commit = _ReadFile(last_commit_file) if IsDirectory(full_path): child_versions = dict( (filename, last_commit) for filename in _ListDir(full_path)) else: child_versions = None return StatInfo(last_commit, child_versions) except (TypeError, errors.Error): raise FileNotFoundError('cloudstorage.stat failed for %s: %s' % (path, traceback.format_exc()))
def resolve(): try: result = {} for path in paths: full_path = Join(self._bucket, path) logging.debug('gcs: requested path "%s", reading "%s"' % (path, full_path)) if IsDirectory(path): result[path] = _ListDir(full_path) else: result[path] = _ReadFile(full_path) return result except errors.AuthorizationError: self._warnAboutAuthError() raise
def walk(root): AssertIsDirectory(root) dirs, files = [], [] for f in self.ReadSingle(root).Get(): if IsDirectory(f): dirs.append(f) else: files.append(f) yield root[len(basepath):].rstrip('/'), dirs, files for d in dirs: for walkinfo in walk(root + d): yield walkinfo
def file_lister(root): res, root_stat = All( (self._walk_cache.Get(root), self.StatAsync(root))).Get() if res and res[2] == root_stat.version: dirs, files = res[0], res[1] else: # Wasn't cached, or not up to date. dirs, files = [], [] for f in self.ReadSingle(root).Get(): if IsDirectory(f): dirs.append(f) else: files.append(f) # Update the cache. This is a root -> (dirs, files, version) mapping. self._walk_cache.Set(root, (dirs, files, root_stat.version)) return dirs, files
def walk(root, depth): if depth == 0: return AssertIsDirectory(root) if file_lister: dirs, files = file_lister(root) else: dirs, files = [], [] for f in self.ReadSingle(root).Get(): if IsDirectory(f): dirs.append(f) else: files.append(f) yield root[len(basepath):].rstrip('/'), dirs, files for d in dirs: for walkinfo in walk(root + d, depth - 1): yield walkinfo
def fetch(self, url): path = _ExtractPathFromSvnUrl(url) if IsDirectory(path): html = ['<html>Revision 000000'] try: for f in self._ListDir(ChromiumPath(path)): if f.startswith('.'): continue if self._IsDir(ChromiumPath(path, f)): html.append('<a>' + f + '/</a>') else: html.append('<a>' + f + '</a>') html.append('</html>') return '\n'.join(html) except OSError as e: return None try: return ReadFile(path) except IOError: return None
def SetVersion(self, path, new_version): if IsDirectory(path): raise ValueError('Only files have an incrementable stat, ' 'but "%s" is a directory' % path) # Update version of that file. self._path_stats[path] = new_version # Update all parent directory versions as well. slash_index = 0 # (deliberately including '' in the dir paths) while slash_index != -1: dir_path = path[:slash_index] + '/' self._path_stats[dir_path] = max(self._path_stats.get(dir_path, 0), new_version) if dir_path == '/': # Legacy support for '/' being the root of the file system rather # than ''. Eventually when the path normalisation logic is complete # this will be impossible and this logic will change slightly. self._path_stats[''] = self._path_stats['/'] slash_index = path.find('/', slash_index + 1)
def Stat(self, path): if not IsDirectory(path): raise FileNotFoundError('EmptyDirFileSystem cannot stat %s' % path) return StatInfo(0, child_versions=[])
def Create(self, bucket): '''Creates a CloudStorageFileSystemProvider. |bucket| is the name of GCS bucket, eg devtools-docs. It is expected that this bucket has Read permission for this app in its ACLs. Optional configuration can be set in a local_debug/gcs_debug.conf file: use_local_fs=True|False access_token=<token> remote_bucket_prefix=<prefix> If running in Preview mode or in Development mode with use_local_fs set to True, buckets and files are looked inside the local_debug folder instead of in the real GCS server. Preview server does not support direct GCS access, so it is always forced to use a LocalFileSystem. For real GCS access in the Development mode (dev_appserver.py), access_token and remote_bucket_prefix options can be used to change the way GCS files are accessed. Both are ignored in a real appengine instance. "access_token" is always REQUIRED on dev_appengine, otherwise you will get 404 (auth) errors. You can get one access_token valid for a few minutes by typing: gsutil -d ls 2>&1 | grep "Bearer" | sed "s/.*Bearer \(.*\).r.nUser-Agent.*/access_token=\1/" )" A sample output would be: access_token=ya29.1.AADtN_VW5ibbfLHV5cMIK5ss4bHtVzBXpa4byjd Now add this line to the local_debug/gcs_debug.conf file and restart the appengine development server. Remember that you will need a new access_token every ten minutes or so. If you get 404 errors on log, update it. Access token is not used for a deployed appengine app, only if you use dev_appengine.py. remote_bucket_prefix is useful if you want to test on your own GCS buckets before using the real GCS buckets. ''' if not environment.IsReleaseServer() and not environment.IsDevServer(): bucket_local_path = os.path.join(LOCAL_GCS_DIR, bucket) if IsDirectory(bucket_local_path): return LocalFileSystem(bucket_local_path) else: return EmptyDirFileSystem() debug_access_token = None debug_bucket_prefix = None use_local_fs = False if environment.IsDevServer() and os.path.exists(LOCAL_GCS_DEBUG_CONF): with open(LOCAL_GCS_DEBUG_CONF, "r") as token_file: properties = dict(line.strip().split('=', 1) for line in token_file) use_local_fs = properties.get('use_local_fs', 'False') == 'True' debug_access_token = properties.get('access_token', None) debug_bucket_prefix = properties.get('remote_bucket_prefix', None) if environment.IsDevServer() and use_local_fs: return LocalFileSystem(os.path.join(LOCAL_GCS_DIR, bucket)) # gcs_file_system has strong dependencies on runtime appengine APIs, # so we only import it when we are sure we are not on preview.py or tests. from gcs_file_system import CloudStorageFileSystem return CachingFileSystem( CloudStorageFileSystem(bucket, debug_access_token, debug_bucket_prefix), self._object_store_creator)
def fixup_url_format(path): # By default, Gitiles URLs display resources in HTML. To get resources # suitable for our consumption, a '?format=' string must be appended to # the URL. The format may be one of 'JSON' or 'TEXT' for directory or # text resources, respectively. return path + (_JSON_FORMAT if IsDirectory(path) else _TEXT_FORMAT)