def __init__(self,
name,
compiled_fs_factory,
file_system,
supports_templates=False,
supports_zip=False):
# Public.
self.name = name
self.file_system = file_system
# Private.
self._content_cache = compiled_fs_factory.Create(file_system,
self._CompileContent,
ContentProvider)
self._supports_templates = supports_templates
if supports_zip:
self._directory_zipper = DirectoryZipper(compiled_fs_factory, file_system)
else:
self._directory_zipper = None
def __init__(self,
name,
compiled_fs_factory,
file_system,
object_store_creator,
default_extensions=(),
supports_templates=False,
supports_zip=False):
# Public.
self.name = name
self.file_system = file_system
# Private.
self._content_cache = compiled_fs_factory.Create(
file_system, self._CompileContent, ContentProvider)
self._path_canonicalizer = PathCanonicalizer(file_system,
object_store_creator,
default_extensions)
self._default_extensions = default_extensions
self._supports_templates = supports_templates
if supports_zip:
self._directory_zipper = DirectoryZipper(compiled_fs_factory,
file_system)
else:
self._directory_zipper = None
class DirectoryZipperTest(unittest.TestCase):
def setUp(self):
self._directory_zipper = DirectoryZipper(
CompiledFileSystem.Factory(ObjectStoreCreator.ForTest()),
TestFileSystem(_TEST_DATA))
def testTopZip(self):
top_zip = ZipFile(StringIO(self._directory_zipper.Zip('top').Get()))
self.assertEqual(['top/one.txt', 'top/two/four.txt', 'top/two/three.txt'],
sorted(top_zip.namelist()))
self.assertEqual('one.txt contents', top_zip.read('top/one.txt'))
self.assertEqual('three.txt contents', top_zip.read('top/two/three.txt'))
self.assertEqual('four.txt contents', top_zip.read('top/two/four.txt'))
def testTwoZip(self):
two_zip = ZipFile(StringIO(self._directory_zipper.Zip('top/two').Get()))
self.assertEqual(['two/four.txt', 'two/three.txt'],
sorted(two_zip.namelist()))
self.assertEqual('three.txt contents', two_zip.read('two/three.txt'))
self.assertEqual('four.txt contents', two_zip.read('two/four.txt'))
def testNotFound(self):
self.assertRaises(FileNotFoundError,
self._directory_zipper.Zip('notfound').Get)
class ContentProvider(object):
'''Returns file contents correctly typed for their content-types (in the HTTP
sense). Content-type is determined from Python's mimetype library which
guesses based on the file extension.
Typically the file contents will be either str (for binary content) or
unicode (for text content). However, HTML files *may* be returned as
Handlebar templates (if supports_templates is True on construction), in which
case the caller will presumably want to Render them.
'''
def __init__(self,
name,
compiled_fs_factory,
file_system,
supports_templates=False,
supports_zip=False):
# Public.
self.name = name
self.file_system = file_system
# Private.
self._content_cache = compiled_fs_factory.Create(file_system,
self._CompileContent,
ContentProvider)
self._supports_templates = supports_templates
if supports_zip:
self._directory_zipper = DirectoryZipper(compiled_fs_factory, file_system)
else:
self._directory_zipper = None
@SingleFile
def _CompileContent(self, path, text):
assert text is not None, path
mimetype = mimetypes.guess_type(path)[0]
if mimetype is None:
content = text
mimetype = 'text/plain'
elif mimetype == 'text/html':
content = ToUnicode(text)
if self._supports_templates:
content = Handlebar(content)
elif (mimetype.startswith('text/') or
mimetype in ('application/javascript', 'application/json')):
content = ToUnicode(text)
else:
content = text
return ContentAndType(content, mimetype)
def GetContentAndType(self, host, path):
path = path.lstrip('/')
base, ext = os.path.splitext(path)
# Check for a zip file first, if zip is enabled.
if self._directory_zipper and ext == '.zip':
zip_future = self._directory_zipper.Zip(base)
return Future(delegate=Gettable(
lambda: ContentAndType(zip_future.Get(), 'application/zip')))
return self._content_cache.GetFromFile(path, binary=True)
def Cron(self):
# Running Refresh() on the file system is enough to pull GitHub content,
# which is all we need for now while the full render-every-page cron step
# is in effect.
# TODO(kalman): Walk over the whole filesystem and compile the content.
return self.file_system.Refresh()
class ContentProvider(object):
'''Returns file contents correctly typed for their content-types (in the HTTP
sense). Content-type is determined from Python's mimetype library which
guesses based on the file extension.
Typically the file contents will be either str (for binary content) or
unicode (for text content). However, HTML files *may* be returned as
Handlebar templates (if |supports_templates| is True on construction), in
which case the caller will presumably want to Render them.
Zip file are automatically created and returned for .zip file extensions if
|supports_zip| is True.
|default_extensions| is a list of file extensions which are queried when no
file extension is given to GetCanonicalPath/GetContentAndType. Typically
this will include .html.
'''
def __init__(self,
name,
compiled_fs_factory,
file_system,
object_store_creator,
default_extensions=(),
supports_templates=False,
supports_zip=False):
# Public.
self.name = name
self.file_system = file_system
# Private.
self._content_cache = compiled_fs_factory.Create(
file_system, self._CompileContent, ContentProvider)
self._path_canonicalizer = PathCanonicalizer(file_system,
object_store_creator,
default_extensions)
self._default_extensions = default_extensions
self._supports_templates = supports_templates
if supports_zip:
self._directory_zipper = DirectoryZipper(compiled_fs_factory,
file_system)
else:
self._directory_zipper = None
@SingleFile
def _CompileContent(self, path, text):
assert text is not None, path
_, ext = posixpath.splitext(path)
mimetype = _MIMETYPE_OVERRIDES.get(ext, mimetypes.guess_type(path)[0])
if ext == '.md':
# See http://pythonhosted.org/Markdown/extensions
# for details on "extensions=".
content = markdown(ToUnicode(text),
extensions=('extra', 'headerid', 'sane_lists'))
if self._supports_templates:
content = Handlebar(content, name=path)
mimetype = 'text/html'
elif mimetype is None:
content = text
mimetype = 'text/plain'
elif mimetype == 'text/html':
content = ToUnicode(text)
if self._supports_templates:
content = Handlebar(content, name=path)
elif (mimetype.startswith('text/')
or mimetype in ('application/javascript', 'application/json')):
content = ToUnicode(text)
else:
content = text
return ContentAndType(content, mimetype,
self.file_system.Stat(path).version)
def GetCanonicalPath(self, path):
'''Gets the canonical location of |path|. This class is tolerant of
spelling errors and missing files that are in other directories, and this
returns the correct/canonical path for those.
For example, the canonical path of "browseraction" is probably
"extensions/browserAction.html".
Note that the canonical path is relative to this content provider i.e.
given relative to |path|. It does not add the "serveFrom" prefix which
would have been pulled out in ContentProviders, callers must do that
themselves.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
# The canonical location of zip files is the canonical location of the
# directory to zip + '.zip'.
return self._path_canonicalizer.Canonicalize(base +
'/').rstrip('/') + ext
return self._path_canonicalizer.Canonicalize(path)
def GetContentAndType(self, path):
'''Returns the ContentAndType of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
# Check for a zip file first, if zip is enabled.
if self._directory_zipper and ext == '.zip':
zip_future = self._directory_zipper.Zip(ToDirectory(base))
return Future(callback=lambda: ContentAndType(
zip_future.Get(), 'application/zip', None))
# If there is no file extension, look for a file with one of the default
# extensions.
#
# Note that it would make sense to guard this on Exists(path), since a file
# without an extension may actually exist, but it's such an uncommon case
# it hardly seems worth the potential performance hit.
if not ext:
for default_ext in self._default_extensions:
if self.file_system.Exists(path + default_ext).Get():
path += default_ext
break
return self._content_cache.GetFromFile(path)
def Cron(self):
futures = [self._path_canonicalizer.Cron()]
for root, _, files in self.file_system.Walk(''):
for f in files:
futures.append(self.GetContentAndType(Join(root, f)))
# Also cache the extension-less version of the file if needed.
base, ext = posixpath.splitext(f)
if f != SITE_VERIFICATION_FILE and ext in self._default_extensions:
futures.append(self.GetContentAndType(Join(root, base)))
# TODO(kalman): Cache .zip files for each directory (if supported).
return Future(callback=lambda: [f.Get() for f in futures])
def __repr__(self):
return 'ContentProvider of <%s>' % repr(self.file_system)
class ContentProvider(object):
'''Returns file contents correctly typed for their content-types (in the HTTP
sense). Content-type is determined from Python's mimetype library which
guesses based on the file extension.
Typically the file contents will be either str (for binary content) or
unicode (for text content). However, HTML files *may* be returned as
Motemplate templates (if |supports_templates| is True on construction), in
which case the caller will presumably want to Render them.
Zip file are automatically created and returned for .zip file extensions if
|supports_zip| is True.
|default_extensions| is a list of file extensions which are queried when no
file extension is given to GetCanonicalPath/GetContentAndType. Typically
this will include .html.
'''
def __init__(self,
name,
compiled_fs_factory,
file_system,
object_store_creator,
default_extensions=(),
supports_templates=False,
supports_zip=False):
# Public.
self.name = name
self.file_system = file_system
# Private.
self._content_cache = compiled_fs_factory.Create(file_system,
self._CompileContent,
ContentProvider)
self._path_canonicalizer = PathCanonicalizer(file_system,
object_store_creator,
default_extensions)
self._default_extensions = default_extensions
self._supports_templates = supports_templates
if supports_zip:
self._directory_zipper = DirectoryZipper(compiled_fs_factory, file_system)
else:
self._directory_zipper = None
@SingleFile
def _CompileContent(self, path, text):
assert text is not None, path
try:
_, ext = posixpath.splitext(path)
mimetype = _MIMETYPE_OVERRIDES.get(ext, mimetypes.guess_type(path)[0])
if ext == '.md':
# See http://pythonhosted.org/Markdown/extensions
# for details on "extensions=".
content = markdown(ToUnicode(text),
extensions=('extra', 'headerid', 'sane_lists'))
mimetype = 'text/html'
if self._supports_templates:
content = Motemplate(content, name=path)
elif mimetype is None:
content = text
mimetype = 'text/plain'
elif mimetype == 'text/html':
content = ToUnicode(text)
if self._supports_templates:
content = Motemplate(content, name=path)
elif (mimetype.startswith('text/') or
mimetype in ('application/javascript', 'application/json')):
content = ToUnicode(text)
else:
content = text
return ContentAndType(content,
mimetype,
self.file_system.Stat(path).version)
except Exception as e:
logging.warn('In file %s: %s' % (path, e.message))
return ContentAndType('', mimetype, self.file_system.Stat(path).version)
def GetCanonicalPath(self, path):
'''Gets the canonical location of |path|. This class is tolerant of
spelling errors and missing files that are in other directories, and this
returns the correct/canonical path for those.
For example, the canonical path of "browseraction" is probably
"extensions/browserAction.html".
Note that the canonical path is relative to this content provider i.e.
given relative to |path|. It does not add the "serveFrom" prefix which
would have been pulled out in ContentProviders, callers must do that
themselves.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
# The canonical location of zip files is the canonical location of the
# directory to zip + '.zip'.
return self._path_canonicalizer.Canonicalize(base + '/').rstrip('/') + ext
return self._path_canonicalizer.Canonicalize(path)
def GetContentAndType(self, path):
'''Returns a Future to the ContentAndType of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
return (self._directory_zipper.Zip(ToDirectory(base))
.Then(lambda zipped: ContentAndType(zipped,
'application/zip',
None)))
return self._FindFileForPath(path).Then(self._content_cache.GetFromFile)
def GetVersion(self, path):
'''Returns a Future to the version of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
stat_future = self.file_system.StatAsync(ToDirectory(base))
else:
stat_future = self._FindFileForPath(path).Then(self.file_system.StatAsync)
return stat_future.Then(lambda stat: stat.version)
def _FindFileForPath(self, path):
'''Finds the real file backing |path|. This may require looking for the
correct file extension, or looking for an 'index' file if it's a directory.
Returns None if no path is found.
'''
AssertIsValid(path)
_, ext = posixpath.splitext(path)
if ext:
# There was already an extension, trust that it's a path. Elsewhere
# up the stack this will be caught if it's not.
return Future(value=path)
def find_file_with_name(name):
'''Tries to find a file in the file system called |name| with one of the
default extensions of this content provider.
If none is found, returns None.
'''
paths = [name + ext for ext in self._default_extensions]
def get_first_path_which_exists(existence):
for exists, path in zip(existence, paths):
if exists:
return path
return None
return (All(self.file_system.Exists(path) for path in paths)
.Then(get_first_path_which_exists))
def find_index_file():
'''Tries to find an index file in |path|, if |path| is a directory.
If not, or if there is no index file, returns None.
'''
def get_index_if_directory_exists(directory_exists):
if not directory_exists:
return None
return find_file_with_name(Join(path, 'index'))
return (self.file_system.Exists(ToDirectory(path))
.Then(get_index_if_directory_exists))
# Try to find a file with the right name. If not, and it's a directory,
# look for an index file in that directory. If nothing at all is found,
# return the original |path| - its nonexistence will be caught up the stack.
return (find_file_with_name(path)
.Then(lambda found: found or find_index_file())
.Then(lambda found: found or path))
def Refresh(self):
futures = [self._path_canonicalizer.Refresh()]
for root, _, files in self.file_system.Walk(''):
for f in files:
futures.append(self.GetContentAndType(Join(root, f)))
# Also cache the extension-less version of the file if needed.
base, ext = posixpath.splitext(f)
if f != SITE_VERIFICATION_FILE and ext in self._default_extensions:
futures.append(self.GetContentAndType(Join(root, base)))
# TODO(kalman): Cache .zip files for each directory (if supported).
return All(futures, except_pass=Exception, except_pass_log=True)
def __repr__(self):
return 'ContentProvider of <%s>' % repr(self.file_system)
def setUp(self):
self._directory_zipper = DirectoryZipper(
CompiledFileSystem.Factory(ObjectStoreCreator.ForTest()),
TestFileSystem(_TEST_DATA))
class ContentProvider(object):
'''Returns file contents correctly typed for their content-types (in the HTTP
sense). Content-type is determined from Python's mimetype library which
guesses based on the file extension.
Typically the file contents will be either str (for binary content) or
unicode (for text content). However, HTML files *may* be returned as
Handlebar templates (if |supports_templates| is True on construction), in
which case the caller will presumably want to Render them.
Zip file are automatically created and returned for .zip file extensions if
|supports_zip| is True.
|default_extensions| is a list of file extensions which are queried when no
file extension is given to GetCanonicalPath/GetContentAndType. Typically
this will include .html.
'''
def __init__(self,
name,
compiled_fs_factory,
file_system,
object_store_creator,
default_extensions=(),
supports_templates=False,
supports_zip=False):
# Public.
self.name = name
self.file_system = file_system
# Private.
self._content_cache = compiled_fs_factory.Create(
file_system, self._CompileContent, ContentProvider)
self._path_canonicalizer = PathCanonicalizer(file_system,
object_store_creator,
default_extensions)
self._default_extensions = default_extensions
self._supports_templates = supports_templates
if supports_zip:
self._directory_zipper = DirectoryZipper(compiled_fs_factory,
file_system)
else:
self._directory_zipper = None
@SingleFile
def _CompileContent(self, path, text):
assert text is not None, path
_, ext = posixpath.splitext(path)
mimetype = _MIMETYPE_OVERRIDES.get(ext, mimetypes.guess_type(path)[0])
if ext == '.md':
# See http://pythonhosted.org/Markdown/extensions
# for details on "extensions=".
content = markdown(ToUnicode(text),
extensions=('extra', 'headerid', 'sane_lists'))
if self._supports_templates:
content = Handlebar(content, name=path)
mimetype = 'text/html'
elif mimetype is None:
content = text
mimetype = 'text/plain'
elif mimetype == 'text/html':
content = ToUnicode(text)
if self._supports_templates:
content = Handlebar(content, name=path)
elif (mimetype.startswith('text/')
or mimetype in ('application/javascript', 'application/json')):
content = ToUnicode(text)
else:
content = text
return ContentAndType(content, mimetype,
self.file_system.Stat(path).version)
def GetCanonicalPath(self, path):
'''Gets the canonical location of |path|. This class is tolerant of
spelling errors and missing files that are in other directories, and this
returns the correct/canonical path for those.
For example, the canonical path of "browseraction" is probably
"extensions/browserAction.html".
Note that the canonical path is relative to this content provider i.e.
given relative to |path|. It does not add the "serveFrom" prefix which
would have been pulled out in ContentProviders, callers must do that
themselves.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
# The canonical location of zip files is the canonical location of the
# directory to zip + '.zip'.
return self._path_canonicalizer.Canonicalize(base +
'/').rstrip('/') + ext
return self._path_canonicalizer.Canonicalize(path)
def GetContentAndType(self, path):
'''Returns the ContentAndType of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
# Check for a zip file first, if zip is enabled.
if self._directory_zipper and ext == '.zip':
zip_future = self._directory_zipper.Zip(ToDirectory(base))
return Future(callback=lambda: ContentAndType(
zip_future.Get(), 'application/zip', None))
# If there is no file extension, look for a file with one of the default
# extensions. If one cannot be found, check if the path is a directory.
# If it is, then check for an index file with one of the default
# extensions.
if not ext:
new_path = self._AddExt(path)
# Add a trailing / to check if it is a directory and not a file with
# no extension.
if new_path is None and self.file_system.Exists(
ToDirectory(path)).Get():
new_path = self._AddExt(Join(path, 'index'))
# If an index file wasn't found in this directly then we're never going
# to find a file.
if new_path is None:
return FileNotFoundError.RaiseInFuture(
'"%s" is a directory' % path)
if new_path is not None:
path = new_path
return self._content_cache.GetFromFile(path)
def _AddExt(self, path):
'''Tries to append each of the default file extensions to path and returns
the first one that is an existing file.
'''
for default_ext in self._default_extensions:
if self.file_system.Exists(path + default_ext).Get():
return path + default_ext
return None
def Cron(self):
futures = [(
'<path_canonicalizer>', # semi-arbitrary string since there is
# no path associated with this Future.
self._path_canonicalizer.Cron())]
for root, _, files in self.file_system.Walk(''):
for f in files:
futures.append(
(Join(root, f), self.GetContentAndType(Join(root, f))))
# Also cache the extension-less version of the file if needed.
base, ext = posixpath.splitext(f)
if f != SITE_VERIFICATION_FILE and ext in self._default_extensions:
futures.append(
(Join(root,
base), self.GetContentAndType(Join(root, base))))
# TODO(kalman): Cache .zip files for each directory (if supported).
def resolve():
for label, future in futures:
try:
future.Get()
except:
logging.error('%s: %s' % (label, traceback.format_exc()))
return Future(callback=resolve)
def __repr__(self):
return 'ContentProvider of <%s>' % repr(self.file_system)