def __init__(self, channel, object_store_creator_factory, svn_file_system,
github_file_system):
self.svn_file_system = svn_file_system
self.github_file_system = github_file_system
self.compiled_fs_factory = CompiledFileSystem.Factory(
svn_file_system, object_store_creator_factory)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH,
svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator_factory)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them. If you actually want to test samples, then
# good luck, and modify _IsSamplesDisabled at the top.
if _IsSamplesDisabled():
svn_fs_for_samples = EmptyDirFileSystem()
else:
svn_fs_for_samples = self.svn_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
channel, svn_fs_for_samples, self.github_file_system,
self.ref_resolver_factory, object_store_creator_factory,
svn_constants.EXAMPLES_PATH)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_fs_factory, svn_constants.JSON_PATH)
self.template_data_source_factory = TemplateDataSource.Factory(
channel, self.api_data_source_factory,
self.api_list_data_source_factory, self.intro_data_source_factory,
self.samples_data_source_factory, self.sidenav_data_source_factory,
self.compiled_fs_factory, self.ref_resolver_factory,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH)
self.example_zipper = ExampleZipper(self.compiled_fs_factory,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(channel,
self.compiled_fs_factory)
self.content_cache = self.compiled_fs_factory.GetOrCreateIdentity()
def testUnderscoreSeparated(self):
file_system = TestFileSystem({
'pepper_dev': {
'c': {
'index.html': ''
}
},
'pepper_stable': {
'c': {
'index.html': ''
}
}
})
self._path_canonicalizer = PathCanonicalizer(
file_system, ObjectStoreCreator.ForTest(), ('.html', '.md'))
self._AssertIdentity('pepper_stable/c/index')
self._AssertRedirect('pepper_stable/c/index',
'pepper_stable/c/index.html')
def __init__(self, channel, object_store_creator_factory, svn_file_system,
github_file_system):
self.svn_file_system = svn_file_system
self.github_file_system = github_file_system
self.compiled_fs_factory = CompiledFileSystem.Factory(
svn_file_system, object_store_creator_factory)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH,
svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory, svn_constants.API_PATH)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator_factory)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
self.samples_data_source_factory = SamplesDataSource.Factory(
channel, self.svn_file_system, ServerInstance.github_file_system,
self.ref_resolver_factory, object_store_creator_factory,
svn_constants.EXAMPLES_PATH)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_fs_factory, svn_constants.JSON_PATH)
self.template_data_source_factory = TemplateDataSource.Factory(
channel, self.api_data_source_factory,
self.api_list_data_source_factory, self.intro_data_source_factory,
self.samples_data_source_factory, self.sidenav_data_source_factory,
self.compiled_fs_factory, self.ref_resolver_factory,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH)
self.example_zipper = ExampleZipper(self.compiled_fs_factory,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(channel,
self.compiled_fs_factory)
self.content_cache = self.compiled_fs_factory.GetOrCreateIdentity()
def __init__(self,
name,
compiled_fs_factory,
file_system,
object_store_creator,
default_extensions=(),
supports_templates=False,
supports_zip=False):
# Public.
self.name = name
self.file_system = file_system
# Private.
self._content_cache = compiled_fs_factory.Create(
file_system, self._CompileContent, ContentProvider)
self._path_canonicalizer = PathCanonicalizer(file_system,
object_store_creator,
default_extensions)
self._default_extensions = default_extensions
self._supports_templates = supports_templates
if supports_zip:
self._directory_zipper = DirectoryZipper(compiled_fs_factory,
file_system)
else:
self._directory_zipper = None
class ContentProvider(object):
'''Returns file contents correctly typed for their content-types (in the HTTP
sense). Content-type is determined from Python's mimetype library which
guesses based on the file extension.
Typically the file contents will be either str (for binary content) or
unicode (for text content). However, HTML files *may* be returned as
Handlebar templates (if |supports_templates| is True on construction), in
which case the caller will presumably want to Render them.
Zip file are automatically created and returned for .zip file extensions if
|supports_zip| is True.
|default_extensions| is a list of file extensions which are queried when no
file extension is given to GetCanonicalPath/GetContentAndType. Typically
this will include .html.
'''
def __init__(self,
name,
compiled_fs_factory,
file_system,
object_store_creator,
default_extensions=(),
supports_templates=False,
supports_zip=False):
# Public.
self.name = name
self.file_system = file_system
# Private.
self._content_cache = compiled_fs_factory.Create(
file_system, self._CompileContent, ContentProvider)
self._path_canonicalizer = PathCanonicalizer(file_system,
object_store_creator,
default_extensions)
self._default_extensions = default_extensions
self._supports_templates = supports_templates
if supports_zip:
self._directory_zipper = DirectoryZipper(compiled_fs_factory,
file_system)
else:
self._directory_zipper = None
@SingleFile
def _CompileContent(self, path, text):
assert text is not None, path
_, ext = posixpath.splitext(path)
mimetype = _MIMETYPE_OVERRIDES.get(ext, mimetypes.guess_type(path)[0])
if ext == '.md':
# See http://pythonhosted.org/Markdown/extensions
# for details on "extensions=".
content = markdown(ToUnicode(text),
extensions=('extra', 'headerid', 'sane_lists'))
if self._supports_templates:
content = Handlebar(content, name=path)
mimetype = 'text/html'
elif mimetype is None:
content = text
mimetype = 'text/plain'
elif mimetype == 'text/html':
content = ToUnicode(text)
if self._supports_templates:
content = Handlebar(content, name=path)
elif (mimetype.startswith('text/')
or mimetype in ('application/javascript', 'application/json')):
content = ToUnicode(text)
else:
content = text
return ContentAndType(content, mimetype,
self.file_system.Stat(path).version)
def GetCanonicalPath(self, path):
'''Gets the canonical location of |path|. This class is tolerant of
spelling errors and missing files that are in other directories, and this
returns the correct/canonical path for those.
For example, the canonical path of "browseraction" is probably
"extensions/browserAction.html".
Note that the canonical path is relative to this content provider i.e.
given relative to |path|. It does not add the "serveFrom" prefix which
would have been pulled out in ContentProviders, callers must do that
themselves.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
# The canonical location of zip files is the canonical location of the
# directory to zip + '.zip'.
return self._path_canonicalizer.Canonicalize(base +
'/').rstrip('/') + ext
return self._path_canonicalizer.Canonicalize(path)
def GetContentAndType(self, path):
'''Returns the ContentAndType of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
# Check for a zip file first, if zip is enabled.
if self._directory_zipper and ext == '.zip':
zip_future = self._directory_zipper.Zip(ToDirectory(base))
return Future(callback=lambda: ContentAndType(
zip_future.Get(), 'application/zip', None))
# If there is no file extension, look for a file with one of the default
# extensions.
#
# Note that it would make sense to guard this on Exists(path), since a file
# without an extension may actually exist, but it's such an uncommon case
# it hardly seems worth the potential performance hit.
if not ext:
for default_ext in self._default_extensions:
if self.file_system.Exists(path + default_ext).Get():
path += default_ext
break
return self._content_cache.GetFromFile(path)
def Cron(self):
futures = [self._path_canonicalizer.Cron()]
for root, _, files in self.file_system.Walk(''):
for f in files:
futures.append(self.GetContentAndType(Join(root, f)))
# Also cache the extension-less version of the file if needed.
base, ext = posixpath.splitext(f)
if f != SITE_VERIFICATION_FILE and ext in self._default_extensions:
futures.append(self.GetContentAndType(Join(root, base)))
# TODO(kalman): Cache .zip files for each directory (if supported).
return Future(callback=lambda: [f.Get() for f in futures])
def __repr__(self):
return 'ContentProvider of <%s>' % repr(self.file_system)
def __init__(self, channel, object_store_creator, host_file_system,
app_samples_file_system, base_path, compiled_fs_factory):
self.channel = channel
self.object_store_creator = object_store_creator
self.host_file_system = host_file_system
self.app_samples_file_system = app_samples_file_system
self.compiled_host_fs_factory = compiled_fs_factory
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH,
svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = self.host_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
channel, extension_samples_fs,
CompiledFileSystem.Factory(extension_samples_fs,
object_store_creator),
self.app_samples_file_system,
CompiledFileSystem.Factory(self.app_samples_file_system,
object_store_creator),
self.ref_resolver_factory, svn_constants.EXAMPLES_PATH)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_host_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.JSON_PATH, base_path)
self.template_data_source_factory = TemplateDataSource.Factory(
channel, self.api_data_source_factory,
self.api_list_data_source_factory, self.intro_data_source_factory,
self.samples_data_source_factory, self.sidenav_data_source_factory,
self.compiled_host_fs_factory, self.ref_resolver_factory,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH, base_path)
self.example_zipper = ExampleZipper(self.compiled_host_fs_factory,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(
channel, self.compiled_host_fs_factory)
self.content_cache = self.compiled_host_fs_factory.CreateIdentity(
ServerInstance)
def setUp(self):
self._path_canonicalizer = PathCanonicalizer(
LocalFileSystem.Create(PUBLIC_TEMPLATES),
ObjectStoreCreator.ForTest(), ('.html', '.md'))
class PathCanonicalizerTest(unittest.TestCase):
def setUp(self):
self._path_canonicalizer = PathCanonicalizer(
LocalFileSystem.Create(PUBLIC_TEMPLATES),
ObjectStoreCreator.ForTest(), ('.html', '.md'))
def testSpecifyCorrectly(self):
self._AssertIdentity('extensions/browserAction')
self._AssertIdentity('extensions/storage')
self._AssertIdentity('extensions/blah')
self._AssertIdentity('extensions/index')
self._AssertIdentity('extensions/whats_new')
self._AssertIdentity('apps/storage')
self._AssertIdentity('apps/bluetooth')
self._AssertIdentity('apps/blah')
self._AssertIdentity('apps/tags/webview')
def testSpecifyIncorrectly(self):
self._AssertRedirectWithDefaultExtensions('extensions/browserAction',
'apps/browserAction')
self._AssertRedirectWithDefaultExtensions(
'extensions/browserAction', 'apps/extensions/browserAction')
self._AssertRedirectWithDefaultExtensions('apps/bluetooth',
'extensions/bluetooth')
self._AssertRedirectWithDefaultExtensions('apps/bluetooth',
'extensions/apps/bluetooth')
self._AssertRedirectWithDefaultExtensions('extensions/index',
'apps/index')
self._AssertRedirectWithDefaultExtensions('extensions/browserAction',
'static/browserAction')
self._AssertRedirectWithDefaultExtensions('apps/tags/webview',
'apps/webview')
self._AssertRedirectWithDefaultExtensions('apps/tags/webview',
'extensions/webview')
self._AssertRedirectWithDefaultExtensions('apps/tags/webview',
'extensions/tags/webview')
# These are a little trickier because storage.html is in both directories.
# They must canonicalize to the closest match.
self._AssertRedirectWithDefaultExtensions('extensions/storage',
'extensions/apps/storage')
self._AssertRedirectWithDefaultExtensions('apps/storage',
'apps/extensions/storage')
def testUnspecified(self):
self._AssertRedirectWithDefaultExtensions('extensions/browserAction',
'browserAction')
self._AssertRedirectWithDefaultExtensions('apps/bluetooth',
'bluetooth')
# Default happens to be apps because it's first alphabetically.
self._AssertRedirectWithDefaultExtensions('apps/storage', 'storage')
# Nonexistent APIs should be left alone.
self._AssertIdentity('blah.html')
def testDirectories(self):
# Directories can be canonicalized too!
self._AssertIdentity('apps/')
self._AssertIdentity('apps/tags/')
self._AssertIdentity('extensions/')
# No trailing slash should be treated as files not directories, at least
# at least according to PathCanonicalizer.
self._AssertRedirect('extensions/apps', 'apps')
self._AssertRedirect('extensions', 'extensions')
# Just as tolerant of spelling mistakes.
self._AssertRedirect('apps/', 'Apps/')
self._AssertRedirect('apps/tags/', 'Apps/TAGS/')
self._AssertRedirect('extensions/', 'Extensions/')
# Find directories in the correct place.
self._AssertRedirect('apps/tags/', 'tags/')
self._AssertRedirect('apps/tags/', 'extensions/tags/')
def testSpellingErrors(self):
for spelme in ('browseraction', 'browseraction.htm', 'BrowserAction',
'BrowserAction.html', 'browseraction.html',
'Browseraction', 'browser-action',
'Browser.action.html', 'browser_action',
'browser-action.html', 'Browser_Action.html'):
self._AssertRedirect('extensions/browserAction', spelme)
self._AssertRedirect('extensions/browserAction',
'extensions/%s' % spelme)
self._AssertRedirect('extensions/browserAction',
'apps/%s' % spelme)
def testNonDefaultExtensions(self):
# The only example currently of a file with a non-default extension is
# the redirects.json file. That shouldn't have its extension stripped since
# it's not in the default extensions.
self._AssertIdentity('redirects.json')
self._AssertRedirect('redirects.json', 'redirects')
self._AssertRedirect('redirects.json', 'redirects.html')
self._AssertRedirect('redirects.json', 'redirects.js')
self._AssertRedirect('redirects.json', 'redirects.md')
def testSiteVerificationFile(self):
# The site verification file should not redirect.
self._AssertIdentity(SITE_VERIFICATION_FILE)
self._AssertRedirect(SITE_VERIFICATION_FILE,
posixpath.splitext(SITE_VERIFICATION_FILE)[0])
def testDotSeparated(self):
self._AssertIdentity('extensions/devtools_inspectedWindow')
self._AssertRedirect('extensions/devtools_inspectedWindow',
'extensions/devtools.inspectedWindow')
def testUnderscoreSeparated(self):
file_system = TestFileSystem({
'pepper_dev': {
'c': {
'index.html': ''
}
},
'pepper_stable': {
'c': {
'index.html': ''
}
}
})
self._path_canonicalizer = PathCanonicalizer(
file_system, ObjectStoreCreator.ForTest(), ('.html', '.md'))
self._AssertIdentity('pepper_stable/c/index')
self._AssertRedirect('pepper_stable/c/index',
'pepper_stable/c/index.html')
def _AssertIdentity(self, path):
self._AssertRedirect(path, path)
def _AssertRedirect(self, to, from_):
self.assertEqual(to, self._path_canonicalizer.Canonicalize(from_))
def _AssertRedirectWithDefaultExtensions(self, to, from_):
for ext in ('', '.html', '.md'):
self._AssertRedirect(
to, self._path_canonicalizer.Canonicalize(from_ + ext))
def __init__(self, object_store_creator, host_file_system,
app_samples_file_system, base_path, compiled_fs_factory,
branch_utility, host_file_system_creator):
self.object_store_creator = object_store_creator
self.host_file_system = host_file_system
self.app_samples_file_system = app_samples_file_system
self.compiled_host_fs_factory = compiled_fs_factory
self.host_file_system_creator = host_file_system_creator
self.availability_finder_factory = AvailabilityFinder.Factory(
object_store_creator, self.compiled_host_fs_factory,
branch_utility, host_file_system_creator)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_host_fs_factory, self.host_file_system,
svn_constants.API_PATH, svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH,
self.availability_finder_factory)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = self.host_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
extension_samples_fs,
CompiledFileSystem.Factory(extension_samples_fs,
object_store_creator),
self.app_samples_file_system,
CompiledFileSystem.Factory(self.app_samples_file_system,
object_store_creator),
self.ref_resolver_factory, svn_constants.EXAMPLES_PATH, base_path)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_host_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.JSON_PATH)
self.manifest_data_source = ManifestDataSource(
self.compiled_host_fs_factory, host_file_system, '/'.join(
(svn_constants.JSON_PATH, 'manifest.json')), '/'.join(
(svn_constants.API_PATH, '_manifest_features.json')))
self.template_data_source_factory = TemplateDataSource.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
self.intro_data_source_factory, self.samples_data_source_factory,
self.sidenav_data_source_factory, self.compiled_host_fs_factory,
self.ref_resolver_factory, self.manifest_data_source,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH, base_path)
self.api_data_source_factory.SetTemplateDataSource(
self.template_data_source_factory)
self.example_zipper = ExampleZipper(self.compiled_host_fs_factory,
self.host_file_system,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(
self.compiled_host_fs_factory)
self.redirector = Redirector(self.compiled_host_fs_factory,
self.host_file_system,
svn_constants.PUBLIC_TEMPLATE_PATH)
class ContentProvider(object):
'''Returns file contents correctly typed for their content-types (in the HTTP
sense). Content-type is determined from Python's mimetype library which
guesses based on the file extension.
Typically the file contents will be either str (for binary content) or
unicode (for text content). However, HTML files *may* be returned as
Motemplate templates (if |supports_templates| is True on construction), in
which case the caller will presumably want to Render them.
Zip file are automatically created and returned for .zip file extensions if
|supports_zip| is True.
|default_extensions| is a list of file extensions which are queried when no
file extension is given to GetCanonicalPath/GetContentAndType. Typically
this will include .html.
'''
def __init__(self,
name,
compiled_fs_factory,
file_system,
object_store_creator,
default_extensions=(),
supports_templates=False,
supports_zip=False):
# Public.
self.name = name
self.file_system = file_system
# Private.
self._content_cache = compiled_fs_factory.Create(file_system,
self._CompileContent,
ContentProvider)
self._path_canonicalizer = PathCanonicalizer(file_system,
object_store_creator,
default_extensions)
self._default_extensions = default_extensions
self._supports_templates = supports_templates
if supports_zip:
self._directory_zipper = DirectoryZipper(compiled_fs_factory, file_system)
else:
self._directory_zipper = None
@SingleFile
def _CompileContent(self, path, text):
assert text is not None, path
try:
_, ext = posixpath.splitext(path)
mimetype = _MIMETYPE_OVERRIDES.get(ext, mimetypes.guess_type(path)[0])
if ext == '.md':
# See http://pythonhosted.org/Markdown/extensions
# for details on "extensions=".
content = markdown(ToUnicode(text),
extensions=('extra', 'headerid', 'sane_lists'))
mimetype = 'text/html'
if self._supports_templates:
content = Motemplate(content, name=path)
elif mimetype is None:
content = text
mimetype = 'text/plain'
elif mimetype == 'text/html':
content = ToUnicode(text)
if self._supports_templates:
content = Motemplate(content, name=path)
elif (mimetype.startswith('text/') or
mimetype in ('application/javascript', 'application/json')):
content = ToUnicode(text)
else:
content = text
return ContentAndType(content,
mimetype,
self.file_system.Stat(path).version)
except Exception as e:
logging.warn('In file %s: %s' % (path, e.message))
return ContentAndType('', mimetype, self.file_system.Stat(path).version)
def GetCanonicalPath(self, path):
'''Gets the canonical location of |path|. This class is tolerant of
spelling errors and missing files that are in other directories, and this
returns the correct/canonical path for those.
For example, the canonical path of "browseraction" is probably
"extensions/browserAction.html".
Note that the canonical path is relative to this content provider i.e.
given relative to |path|. It does not add the "serveFrom" prefix which
would have been pulled out in ContentProviders, callers must do that
themselves.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
# The canonical location of zip files is the canonical location of the
# directory to zip + '.zip'.
return self._path_canonicalizer.Canonicalize(base + '/').rstrip('/') + ext
return self._path_canonicalizer.Canonicalize(path)
def GetContentAndType(self, path):
'''Returns a Future to the ContentAndType of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
return (self._directory_zipper.Zip(ToDirectory(base))
.Then(lambda zipped: ContentAndType(zipped,
'application/zip',
None)))
return self._FindFileForPath(path).Then(self._content_cache.GetFromFile)
def GetVersion(self, path):
'''Returns a Future to the version of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
stat_future = self.file_system.StatAsync(ToDirectory(base))
else:
stat_future = self._FindFileForPath(path).Then(self.file_system.StatAsync)
return stat_future.Then(lambda stat: stat.version)
def _FindFileForPath(self, path):
'''Finds the real file backing |path|. This may require looking for the
correct file extension, or looking for an 'index' file if it's a directory.
Returns None if no path is found.
'''
AssertIsValid(path)
_, ext = posixpath.splitext(path)
if ext:
# There was already an extension, trust that it's a path. Elsewhere
# up the stack this will be caught if it's not.
return Future(value=path)
def find_file_with_name(name):
'''Tries to find a file in the file system called |name| with one of the
default extensions of this content provider.
If none is found, returns None.
'''
paths = [name + ext for ext in self._default_extensions]
def get_first_path_which_exists(existence):
for exists, path in zip(existence, paths):
if exists:
return path
return None
return (All(self.file_system.Exists(path) for path in paths)
.Then(get_first_path_which_exists))
def find_index_file():
'''Tries to find an index file in |path|, if |path| is a directory.
If not, or if there is no index file, returns None.
'''
def get_index_if_directory_exists(directory_exists):
if not directory_exists:
return None
return find_file_with_name(Join(path, 'index'))
return (self.file_system.Exists(ToDirectory(path))
.Then(get_index_if_directory_exists))
# Try to find a file with the right name. If not, and it's a directory,
# look for an index file in that directory. If nothing at all is found,
# return the original |path| - its nonexistence will be caught up the stack.
return (find_file_with_name(path)
.Then(lambda found: found or find_index_file())
.Then(lambda found: found or path))
def Refresh(self):
futures = [self._path_canonicalizer.Refresh()]
for root, _, files in self.file_system.Walk(''):
for f in files:
futures.append(self.GetContentAndType(Join(root, f)))
# Also cache the extension-less version of the file if needed.
base, ext = posixpath.splitext(f)
if f != SITE_VERIFICATION_FILE and ext in self._default_extensions:
futures.append(self.GetContentAndType(Join(root, base)))
# TODO(kalman): Cache .zip files for each directory (if supported).
return All(futures, except_pass=Exception, except_pass_log=True)
def __repr__(self):
return 'ContentProvider of <%s>' % repr(self.file_system)
def __init__(self,
object_store_creator,
compiled_fs_factory,
branch_utility,
host_file_system_provider,
github_file_system_provider,
base_path='/'):
'''
|object_store_creator|
The ObjectStoreCreator used to create almost all caches.
|compiled_fs_factory|
Factory used to create CompiledFileSystems, a higher-level cache type
than ObjectStores. This can usually be derived from just
|object_store_creator| but under special circumstances a different
implementation needs to be passed in.
|branch_utility|
Has knowledge of Chrome branches, channels, and versions.
|host_file_system_provider|
Creates FileSystem instances which host the server at alternative
revisions.
|github_file_system_provider|
Creates FileSystem instances backed by GitHub.
|base_path|
The path which all HTML is generated relative to. Usually this is /
but some servlets need to override this.
'''
self.object_store_creator = object_store_creator
self.compiled_fs_factory = compiled_fs_factory
self.host_file_system_provider = host_file_system_provider
host_fs_at_trunk = host_file_system_provider.GetTrunk()
self.github_file_system_provider = github_file_system_provider
assert base_path.startswith('/') and base_path.endswith('/')
self.base_path = base_path
self.host_file_system_iterator = HostFileSystemIterator(
host_file_system_provider, branch_utility)
self.features_bundle = FeaturesBundle(host_fs_at_trunk,
self.compiled_fs_factory,
self.object_store_creator)
self.api_models = APIModels(self.features_bundle,
self.compiled_fs_factory, host_fs_at_trunk)
self.availability_finder = AvailabilityFinder(
branch_utility, compiled_fs_factory,
self.host_file_system_iterator, host_fs_at_trunk,
object_store_creator)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_fs_factory, host_fs_at_trunk, self.features_bundle,
self.object_store_creator)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory, host_fs_at_trunk, svn_constants.API_PATH,
self.availability_finder, branch_utility)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_models,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
app_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = host_fs_at_trunk
app_samples_fs = github_file_system_provider.Create(
'GoogleChrome', 'chrome-app-samples')
self.samples_data_source_factory = SamplesDataSource.Factory(
extension_samples_fs, app_samples_fs,
CompiledFileSystem.Factory(object_store_creator),
self.ref_resolver_factory, svn_constants.EXAMPLES_PATH, base_path)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_fs_factory, host_fs_at_trunk,
self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.path_canonicalizer = PathCanonicalizer(self.compiled_fs_factory,
host_fs_at_trunk)
self.content_providers = ContentProviders(
self.compiled_fs_factory, host_fs_at_trunk,
self.github_file_system_provider)
# TODO(kalman): Move all the remaining DataSources into DataSourceRegistry,
# then factor out the DataSource creation into a factory method, so that
# the entire ServerInstance doesn't need to be passed in here.
self.template_renderer = TemplateRenderer(self)
self.strings_json_path = svn_constants.STRINGS_JSON_PATH
self.manifest_json_path = svn_constants.MANIFEST_JSON_PATH
self.manifest_features_path = svn_constants.MANIFEST_FEATURES_PATH
def setUp(self):
test_fs = TestFileSystem(_TEST_DATA)
compiled_fs_factory = CompiledFileSystem.Factory(
test_fs, ObjectStoreCreator.Factory())
self._path_canonicalizer = PathCanonicalizer('stable',
compiled_fs_factory)
class PathCanonicalizerTest(unittest.TestCase):
def setUp(self):
test_fs = TestFileSystem(_TEST_DATA)
compiled_fs_factory = CompiledFileSystem.Factory(
test_fs, ObjectStoreCreator.Factory())
self._path_canonicalizer = PathCanonicalizer('stable',
compiled_fs_factory)
def _assertIdentity(self, path):
self.assertEqual(path, self._path_canonicalizer.Canonicalize(path))
def testExtensions(self):
self._assertIdentity('extensions/browserAction.html')
self._assertIdentity('extensions/storage.html')
self._assertIdentity('extensions/bluetooth.html')
self._assertIdentity('extensions/blah.html')
self._assertIdentity('stable/extensions/browserAction.html')
self._assertIdentity('stable/extensions/storage.html')
self._assertIdentity('stable/extensions/bluetooth.html')
self._assertIdentity('stable/extensions/blah.html')
def testApps(self):
self._assertIdentity('apps/browserAction.html')
self._assertIdentity('apps/storage.html')
self._assertIdentity('apps/bluetooth.html')
self._assertIdentity('apps/blah.html')
self._assertIdentity('stable/apps/browserAction.html')
self._assertIdentity('stable/apps/storage.html')
self._assertIdentity('stable/apps/bluetooth.html')
self._assertIdentity('stable/apps/blah.html')
def testStatic(self):
self._assertIdentity('static/browserAction.html')
self._assertIdentity('static/storage.html')
self._assertIdentity('static/bluetooth.html')
self._assertIdentity('static/blah.html')
self._assertIdentity('stable/static/browserAction.html')
self._assertIdentity('stable/static/storage.html')
self._assertIdentity('stable/static/bluetooth.html')
self._assertIdentity('stable/static/blah.html')
def testNeither(self):
self.assertEqual(
'extensions/browserAction.html',
self._path_canonicalizer.Canonicalize('browserAction.html'))
self.assertEqual(
'stable/extensions/browserAction.html',
self._path_canonicalizer.Canonicalize('stable/browserAction.html'))
self.assertEqual('extensions/storage.html',
self._path_canonicalizer.Canonicalize('storage.html'))
self.assertEqual(
'stable/extensions/storage.html',
self._path_canonicalizer.Canonicalize('stable/storage.html'))
self.assertEqual(
'apps/bluetooth.html',
self._path_canonicalizer.Canonicalize('bluetooth.html'))
self.assertEqual(
'stable/apps/bluetooth.html',
self._path_canonicalizer.Canonicalize('stable/bluetooth.html'))
# Assign non-existent paths to extensions because they came first, so such
# paths are more likely to be for extensions.
self.assertEqual('extensions/blah.html',
self._path_canonicalizer.Canonicalize('blah.html'))
self.assertEqual(
'stable/extensions/blah.html',
self._path_canonicalizer.Canonicalize('stable/blah.html'))
class ContentProvider(object):
'''Returns file contents correctly typed for their content-types (in the HTTP
sense). Content-type is determined from Python's mimetype library which
guesses based on the file extension.
Typically the file contents will be either str (for binary content) or
unicode (for text content). However, HTML files *may* be returned as
Handlebar templates (if |supports_templates| is True on construction), in
which case the caller will presumably want to Render them.
Zip file are automatically created and returned for .zip file extensions if
|supports_zip| is True.
|default_extensions| is a list of file extensions which are queried when no
file extension is given to GetCanonicalPath/GetContentAndType. Typically
this will include .html.
'''
def __init__(self,
name,
compiled_fs_factory,
file_system,
object_store_creator,
default_extensions=(),
supports_templates=False,
supports_zip=False):
# Public.
self.name = name
self.file_system = file_system
# Private.
self._content_cache = compiled_fs_factory.Create(
file_system, self._CompileContent, ContentProvider)
self._path_canonicalizer = PathCanonicalizer(file_system,
object_store_creator,
default_extensions)
self._default_extensions = default_extensions
self._supports_templates = supports_templates
if supports_zip:
self._directory_zipper = DirectoryZipper(compiled_fs_factory,
file_system)
else:
self._directory_zipper = None
@SingleFile
def _CompileContent(self, path, text):
assert text is not None, path
_, ext = posixpath.splitext(path)
mimetype = _MIMETYPE_OVERRIDES.get(ext, mimetypes.guess_type(path)[0])
if ext == '.md':
# See http://pythonhosted.org/Markdown/extensions
# for details on "extensions=".
content = markdown(ToUnicode(text),
extensions=('extra', 'headerid', 'sane_lists'))
if self._supports_templates:
content = Handlebar(content, name=path)
mimetype = 'text/html'
elif mimetype is None:
content = text
mimetype = 'text/plain'
elif mimetype == 'text/html':
content = ToUnicode(text)
if self._supports_templates:
content = Handlebar(content, name=path)
elif (mimetype.startswith('text/')
or mimetype in ('application/javascript', 'application/json')):
content = ToUnicode(text)
else:
content = text
return ContentAndType(content, mimetype,
self.file_system.Stat(path).version)
def GetCanonicalPath(self, path):
'''Gets the canonical location of |path|. This class is tolerant of
spelling errors and missing files that are in other directories, and this
returns the correct/canonical path for those.
For example, the canonical path of "browseraction" is probably
"extensions/browserAction.html".
Note that the canonical path is relative to this content provider i.e.
given relative to |path|. It does not add the "serveFrom" prefix which
would have been pulled out in ContentProviders, callers must do that
themselves.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
if self._directory_zipper and ext == '.zip':
# The canonical location of zip files is the canonical location of the
# directory to zip + '.zip'.
return self._path_canonicalizer.Canonicalize(base +
'/').rstrip('/') + ext
return self._path_canonicalizer.Canonicalize(path)
def GetContentAndType(self, path):
'''Returns the ContentAndType of the file at |path|.
'''
AssertIsValid(path)
base, ext = posixpath.splitext(path)
# Check for a zip file first, if zip is enabled.
if self._directory_zipper and ext == '.zip':
zip_future = self._directory_zipper.Zip(ToDirectory(base))
return Future(callback=lambda: ContentAndType(
zip_future.Get(), 'application/zip', None))
# If there is no file extension, look for a file with one of the default
# extensions. If one cannot be found, check if the path is a directory.
# If it is, then check for an index file with one of the default
# extensions.
if not ext:
new_path = self._AddExt(path)
# Add a trailing / to check if it is a directory and not a file with
# no extension.
if new_path is None and self.file_system.Exists(
ToDirectory(path)).Get():
new_path = self._AddExt(Join(path, 'index'))
# If an index file wasn't found in this directly then we're never going
# to find a file.
if new_path is None:
return FileNotFoundError.RaiseInFuture(
'"%s" is a directory' % path)
if new_path is not None:
path = new_path
return self._content_cache.GetFromFile(path)
def _AddExt(self, path):
'''Tries to append each of the default file extensions to path and returns
the first one that is an existing file.
'''
for default_ext in self._default_extensions:
if self.file_system.Exists(path + default_ext).Get():
return path + default_ext
return None
def Cron(self):
futures = [(
'<path_canonicalizer>', # semi-arbitrary string since there is
# no path associated with this Future.
self._path_canonicalizer.Cron())]
for root, _, files in self.file_system.Walk(''):
for f in files:
futures.append(
(Join(root, f), self.GetContentAndType(Join(root, f))))
# Also cache the extension-less version of the file if needed.
base, ext = posixpath.splitext(f)
if f != SITE_VERIFICATION_FILE and ext in self._default_extensions:
futures.append(
(Join(root,
base), self.GetContentAndType(Join(root, base))))
# TODO(kalman): Cache .zip files for each directory (if supported).
def resolve():
for label, future in futures:
try:
future.Get()
except:
logging.error('%s: %s' % (label, traceback.format_exc()))
return Future(callback=resolve)
def __repr__(self):
return 'ContentProvider of <%s>' % repr(self.file_system)