def write_control_file(self, path, content):
path = os.path.join(self.control_dir, path.lstrip('/'))
if self.config.keep_control_dir:
return self.pod.write_file(path, content)
if self._has_custom_control_dir:
return self.storage.write(path, content)
if self.batch_writes:
return self.write_files(
{path: rendered_document.RenderedDocument(path, content)})
return self.write_file(
rendered_document.RenderedDocument(path, content))
def render(self, jinja_env=None):
"""Read the static file."""
timer = self.pod.profile.timer('RenderStaticDocumentController.render',
label=self.serving_path,
meta={
'path': self.serving_path
}).start_timer()
pod_path = None
if 'pod_path' in self.route_info.meta:
pod_path = self.route_info.meta['pod_path']
else:
pod_path = self.serving_path[len(self.route_info.
meta['path_format']):]
pod_path = os.path.join(self.route_info.meta['source_format'],
pod_path)
if not self.pod.file_exists(pod_path):
text = '{} was not found in static files.'
raise errors.RouteNotFoundError(text.format(self.serving_path))
# Validate the path with the static config specific filter.
self.validate_path(self.route_info.meta['path_filter'])
rendered_content = self.pod.read_file(pod_path)
rendered_content = self.pod.extensions_controller.trigger(
'post_render', self.static_doc, rendered_content)
rendered_doc = rendered_document.RenderedDocument(
self.serving_path, rendered_content)
timer.stop_timer()
return rendered_doc
def load(self, source_dir):
"""Load the pod content from file system."""
timer = self.pod.profile.timer(
'RenderSitemapController.load',
label='{}'.format(self.serving_path),
meta=self.route_info.meta,
).start_timer()
source_dir = self.clean_source_dir(source_dir)
# Validate the path with the config filters.
self.validate_path()
try:
rendered_path = '{}{}'.format(source_dir, self.serving_path)
rendered_content = self.pod.read_file(rendered_path)
rendered_doc = rendered_document.RenderedDocument(
self.serving_path, rendered_content)
timer.stop_timer()
return rendered_doc
except Exception as err:
text = 'Error building {}: {}'
if self.pod:
self.pod.logger.exception(text.format(self, err))
exception = errors.BuildError(text.format(self, err))
exception.traceback = sys.exc_info()[2]
exception.controller = self
exception.exception = err
raise exception
def load(self, source_dir):
"""Load the pod content from file system."""
timer = self.pod.profile.timer(
'RenderDocumentController.load',
label='{} ({})'.format(self.doc.pod_path, self.doc.locale),
meta={
'path': self.doc.pod_path,
'locale': str(self.doc.locale)}
).start_timer()
source_dir = self.clean_source_dir(source_dir)
# Validate the path with the config filters.
self.validate_path()
try:
doc = self.doc
serving_path = doc.get_serving_path()
if serving_path.endswith('/'):
serving_path = '{}{}'.format(serving_path, self.suffix)
rendered_path = '{}{}'.format(source_dir, serving_path)
rendered_content = self.pod.read_file(rendered_path)
rendered_doc = rendered_document.RenderedDocument(
serving_path, rendered_content)
timer.stop_timer()
return rendered_doc
except Exception as err:
exception = errors.BuildError(str(err))
exception.traceback = sys.exc_info()[2]
exception.controller = self
exception.exception = err
raise exception
def render(self, jinja_env=None):
"""Render the document using the render pool."""
timer = self.pod.profile.timer('RenderDocumentController.render',
label='{} ({})'.format(
self.doc.pod_path, self.doc.locale),
meta={
'path': self.doc.pod_path,
'locale': str(self.doc.locale)
}).start_timer()
# Validate the path with the config filters.
self.validate_path()
doc = self.doc
template = jinja_env['env'].get_template(doc.view.lstrip('/'))
track_dependency = doc_dependency.DocDependency(doc)
local_tags = tags.create_builtin_tags(
self.pod, doc, track_dependency=track_dependency)
# NOTE: This should be done using get_template(... globals=...)
# or passed as an argument into render but
# it is not available included inside macros???
# See: https://github.com/pallets/jinja/issues/688
template.globals['g'] = local_tags
# Track the message stats, including untranslated strings.
if self.pod.is_enabled(self.pod.FEATURE_TRANSLATION_STATS):
template.globals['_'] = tags.make_doc_gettext(doc)
try:
serving_path = doc.get_serving_path()
if serving_path.endswith('/'):
serving_path = '{}{}'.format(serving_path, self.suffix)
rendered_content = template.render({
'doc':
doc,
'env':
self.pod.env,
'podspec':
self.pod.podspec,
'_track_dependency':
track_dependency,
}).lstrip()
rendered_content = self.pod.extensions_controller.trigger(
'post_render', doc, rendered_content)
rendered_doc = rendered_document.RenderedDocument(
serving_path, rendered_content)
timer.stop_timer()
return rendered_doc
except Exception as err:
text = 'Error building {}: {}'
if self.pod:
self.pod.logger.exception(text.format(self, err))
exception = errors.BuildError(text.format(self, err))
exception.traceback = sys.exc_info()[2]
exception.controller = self
exception.exception = err
raise exception
def export(self, suffix=None, append_slashes=False, pod_paths=None):
"""Builds the pod, returning a mapping of paths to content based on pod routes."""
paths, routes = self.determine_paths_to_build(pod_paths=pod_paths)
for rendered_doc in self.render_paths(
paths, routes, suffix=suffix, append_slashes=append_slashes):
yield rendered_doc
if not pod_paths:
error_controller = routes.match_error('/404.html')
if error_controller:
yield rendered_document.RenderedDocument(
'/404.html', error_controller.render({}), tmp_dir=self.tmp_dir)
def export_ui(self):
"""Builds the grow ui tools, returning a mapping of paths to content."""
paths = []
source_prefix = 'node_modules/'
destination_root = '_grow/ui/'
tools_dir = 'tools/'
tool_prefix = 'grow-tool-'
# Add the base ui files.
source_root = os.path.join(utils.get_grow_dir(), 'ui', 'dist')
for path in ['css/ui.min.css', 'js/ui.min.js']:
source_path = os.path.join(source_root, path)
output_path = os.sep + os.path.join(destination_root, path)
yield rendered_document.RenderedDocument(
output_path, self.storage.read(source_path))
# Add the files from each of the tools.
for tool in self.ui.get('tools', []):
tool_path = '{}{}{}'.format(source_prefix, tool_prefix,
tool['kind'])
for root, dirs, files in self.walk(tool_path):
for directory in dirs:
if directory.startswith('.'):
dirs.remove(directory)
pod_dir = root.replace(self.root, '')
for file_name in files:
paths.append(os.path.join(pod_dir, file_name))
text = 'Building UI Tools: %(value)d/{} (in %(time_elapsed).9s)'
widgets = [progressbar.FormatLabel(text.format(len(paths)))]
progress = progressbar_non.create_progressbar("Building UI Tools...",
widgets=widgets,
max_value=len(paths))
progress.start()
for path in paths:
output_path = path.replace(
source_prefix, '{}{}'.format(destination_root, tools_dir))
yield rendered_document.RenderedDocument(output_path,
self.read_file(path))
progress.update(progress.value + 1)
progress.finish()
def test_diff(self):
my_index = indexes.Index.create({
'/file.txt':
rendered_document.RenderedDocument('/file.txt', 'test'),
'/file2.txt':
rendered_document.RenderedDocument('/file2.txt', 'test'),
'/foo/file.txt':
rendered_document.RenderedDocument('/foo/file.txt', 'test'),
})
their_index = indexes.Index.create({
'/file2.txt':
rendered_document.RenderedDocument('/file2.txt', 'change'),
'/foo/file.txt':
rendered_document.RenderedDocument('/foo/file.txt', 'test'),
'/bar/new.txt':
rendered_document.RenderedDocument('/bar/new.txt', 'test'),
})
expected = messages.DiffMessage(
adds=[messages.FileMessage(path='/file.txt')],
edits=[messages.FileMessage(path='/file2.txt')],
deletes=[messages.FileMessage(path='/bar/new.txt')],
nochanges=[messages.FileMessage(path='/foo/file.txt')],
)
if utils.is_appengine():
self.assertRaises(utils.UnavailableError, indexes.Diff.create,
my_index, their_index)
else:
diff = indexes.Diff.create(my_index, their_index)
self.assertFilePathsEqual(expected.adds, diff.adds)
def test_diff(self):
my_index = indexes.Index.create({
'/file.txt':
rendered_document.RenderedDocument('/file.txt', 'test'),
'/file2.txt':
rendered_document.RenderedDocument('/file2.txt', 'test'),
'/foo/file.txt':
rendered_document.RenderedDocument('/foo/file.txt', 'test'),
})
their_index = indexes.Index.create({
'/file2.txt':
rendered_document.RenderedDocument('/file2.txt', 'change'),
'/foo/file.txt':
rendered_document.RenderedDocument('/foo/file.txt', 'test'),
'/bar/new.txt':
rendered_document.RenderedDocument('/bar/new.txt', 'test'),
})
expected = messages.DiffMessage(
adds=[messages.FileMessage(path='/file.txt')],
edits=[messages.FileMessage(path='/file2.txt')],
deletes=[messages.FileMessage(path='/bar/new.txt')],
nochanges=[messages.FileMessage(path='/foo/file.txt')],
)
diff = indexes.Diff.create(my_index, their_index)
self.assertFilePathsEqual(expected.adds, diff.adds)
def load(self, source_dir):
"""Load the pod content from file system."""
timer = self.pod.profile.timer(
'RenderStaticDocumentController.load', label=self.serving_path,
meta={'path': self.serving_path}).start_timer()
source_dir = self.clean_source_dir(source_dir)
# Validate the path with the static config specific filter.
self.validate_path(self.route_info.meta['path_filter'])
rendered_path = '{}{}'.format(source_dir, self.serving_path)
rendered_content = self.pod.read_file(rendered_path)
rendered_doc = rendered_document.RenderedDocument(
self.serving_path, rendered_content)
timer.stop_timer()
return rendered_doc
def render(self, jinja_env=None):
"""Render the document using the render pool."""
timer = self.pod.profile.timer(
'RenderSitemapController.render',
label='{}'.format(self.serving_path),
meta=self.route_info.meta,
).start_timer()
# Validate the path with the config filters.
self.validate_path()
# Need a custom root for rendering sitemap.
root = os.path.join(utils.get_grow_dir(), 'pods', 'templates')
jinja_env = self.pod.render_pool.custom_jinja_env(root=root)
with jinja_env['lock']:
if self.route_info.meta.get('template'):
content = self.pod.read_file(self.route_info.meta['template'])
template = jinja_env['env'].from_string(content)
else:
template = jinja_env['env'].get_template('sitemap.xml')
try:
docs = []
locales = self.route_info.meta.get('locales')
collections = self.route_info.meta.get('collections')
for col in list(self.pod.list_collections(collections)):
docs += col.list_servable_documents(locales=locales)
rendered_doc = rendered_document.RenderedDocument(
self.serving_path,
template.render({
'pod': self.pod,
'docs': docs,
}).lstrip())
timer.stop_timer()
return rendered_doc
except Exception as err:
text = 'Error building {}: {}'
if self.pod:
self.pod.logger.exception(text.format(self, err))
exception = errors.BuildError(text.format(self, err))
exception.traceback = sys.exc_info()[2]
exception.controller = self
exception.exception = err
raise exception
def render(self, jinja_env=None):
"""Render the document using the render pool."""
timer = self.pod.profile.timer('RenderErrorController.render',
label='{} ({})'.format(
self.route_info.meta['key'],
self.route_info.meta['view']),
meta={
'key': self.route_info.meta['key'],
'view':
self.route_info.meta['view'],
}).start_timer()
# Validate the path with the config filters.
self.validate_path()
with jinja_env['lock']:
template = jinja_env['env'].get_template(
self.route_info.meta['view'].lstrip('/'))
local_tags = tags.create_builtin_tags(self.pod, doc=None)
# NOTE: This should be done using get_template(... globals=...)
# or passed as an argument into render but
# it is not available included inside macros???
# See: https://github.com/pallets/jinja/issues/688
template.globals['g'] = local_tags
try:
serving_path = '/{}.html'.format(self.route_info.meta['key'])
rendered_doc = rendered_document.RenderedDocument(
serving_path,
template.render({
'doc': None,
'env': self.pod.env,
'podspec': self.pod.podspec,
}).lstrip())
timer.stop_timer()
return rendered_doc
except Exception as err:
text = 'Error building {}: {}'
if self.pod:
self.pod.logger.exception(text.format(self, err))
exception = errors.BuildError(text.format(self, err))
exception.traceback = sys.exc_info()[2]
exception.controller = self
exception.exception = err
raise exception
def render_paths(self, paths, routes, suffix=None, append_slashes=False):
"""Renders the given paths and yields each path and content."""
text = 'Building: %(value)d/{} (in %(time_elapsed).9s)'
widgets = [progressbar.FormatLabel(text.format(len(paths)))]
bar = progressbar_non.create_progressbar("Building pod...",
widgets=widgets,
max_value=len(paths))
bar.start()
for path in paths:
output_path = path
controller, params = routes.match(path, env=self.env.to_wsgi_env())
# Append a suffix onto rendered routes only. This supports dumping
# paths that would serve at URLs that terminate in "/" or without
# an extension to an HTML file suitable for writing to a
# filesystem. Static routes and other routes that may export to
# paths without extensions should remain unmodified.
if suffix and controller.KIND == messages.Kind.RENDERED:
if (append_slashes and not output_path.endswith('/')
and not os.path.splitext(output_path)[-1]):
output_path = output_path.rstrip('/') + '/'
if append_slashes and output_path.endswith('/') and suffix:
output_path += suffix
try:
key = 'Pod.render_paths.render'
if isinstance(controller, grow_static.StaticController):
key = 'Pod.render_paths.render.static'
with self.profile.timer(key,
label=output_path,
meta={'path': output_path}):
yield rendered_document.RenderedDocument(
output_path,
controller.render(params, inject=False),
tmp_dir=self.tmp_dir)
except:
self.logger.error('Error building: {}'.format(controller))
raise
bar.update(bar.value + 1)
bar.finish()
def export(self,
suffix=None,
append_slashes=False,
pod_paths=None,
use_threading=True):
"""Builds the pod, yielding rendered_doc based on pod routes."""
if self.use_reroute:
for rendered_doc in renderer.Renderer.rendered_docs(
self, self.router.routes, use_threading=use_threading):
yield rendered_doc
else:
paths, routes = self.determine_paths_to_build(pod_paths=pod_paths)
for rendered_doc in self.render_paths(
paths, routes, suffix=suffix,
append_slashes=append_slashes):
yield rendered_doc
if not pod_paths:
error_controller = routes.match_error('/404.html')
if error_controller:
yield rendered_document.RenderedDocument(
'/404.html',
error_controller.render({}),
tmp_dir=self.tmp_dir)
def render(self, jinja_env=None):
"""Read the static file."""
timer = self.pod.profile.timer(
'RenderStaticDocumentController.render', label=self.serving_path,
meta={'path': self.serving_path}).start_timer()
pod_path = None
if 'pod_path' in self.route_info.meta:
pod_path = self.route_info.meta['pod_path']
else:
pod_path = self.serving_path[
len(self.route_info.meta['path_format']):]
pod_path = os.path.join(
self.route_info.meta['source_format'], pod_path)
if not self.pod.file_exists(pod_path):
text = '{} was not found in static files.'
raise errors.RouteNotFoundError(text.format(self.serving_path))
rendered_doc = rendered_document.RenderedDocument(
self.serving_path, self.pod.read_file(pod_path))
# rendered_doc.render_timer = timer.stop_timer()
timer.stop_timer()
return rendered_doc
def _create_doc(path, content=None):
return rendered_document.RenderedDocument(path, content)
def render(self, jinja_env=None, request=None):
"""Render the document using the render pool."""
timer = self.pod.profile.timer(
'RenderSitemapController.render',
label='{}'.format(self.serving_path),
meta=self.route_info.meta,
).start_timer()
# Validate the path with the config filters.
self.validate_path()
# Duplicate the routes to use the filters without messing up routing.
temp_router = self.pod.router.__class__(self.pod)
temp_router.add_all()
# Sitemaps only show documents...?
temp_router.filter('whitelist', kinds=['doc'])
for sitemap_filter in self.route_info.meta.get('filters') or []:
temp_router.filter(
sitemap_filter['type'],
collection_paths=sitemap_filter.get('collections'),
paths=sitemap_filter.get('paths'),
locales=sitemap_filter.get('locales'))
# Need a custom root for rendering sitemap.
root = os.path.join(utils.get_grow_dir(), 'pods', 'templates')
jinja_env = self.pod.render_pool.custom_jinja_env(root=root)
with jinja_env['lock']:
if self.route_info.meta.get('template'):
content = self.pod.read_file(self.route_info.meta['template'])
template = jinja_env['env'].from_string(content)
else:
template = jinja_env['env'].get_template('sitemap.xml')
try:
docs = []
for _, value, _ in temp_router.routes.nodes:
docs.append(
self.pod.get_doc(value.meta['pod_path'],
locale=value.meta['locale']))
rendered_doc = rendered_document.RenderedDocument(
self.serving_path,
template.render({
'pod': self.pod,
'env': self.pod.env,
'docs': docs,
'podspec': self.pod.podspec,
}).lstrip())
timer.stop_timer()
return rendered_doc
except Exception as err:
text = 'Error building {}: {}'
if self.pod:
self.pod.logger.exception(text.format(self, err))
exception = errors.BuildError(text.format(self, err))
exception.traceback = sys.exc_info()[2]
exception.controller = self
exception.exception = err
raise exception
def write_control_file(self, path, content):
path = os.path.join(self.control_dir, path.lstrip('/'))
return self.write_file(rendered_document.RenderedDocument(
path, content),
policy='private')
