def run(host, port, https, debug, browser, update_check, preprocess, ui,
pod_path, deployment):
"""Starts a development server for a single pod."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
scheme = 'https' if https else 'http'
config = env.EnvConfig(host=host,
port=port,
name=env.Name.DEV,
scheme=scheme,
cached=False,
dev=True)
environment = env.Env(config)
pod = pods.Pod(root, storage=storage.FileStorage, env=environment)
if deployment:
deployment_obj = pod.get_deployment(deployment)
pod.set_env(deployment_obj.config.env)
if not ui:
pod.disable(pod.FEATURE_UI)
try:
manager.start(pod,
host=host,
port=port,
open_browser=browser,
debug=debug,
preprocess=preprocess,
update_check=update_check)
except pods.Error as e:
raise click.ClickException(str(e))
return pod
def test_get_edit_url(self):
pod = testing.create_pod()
path = '/content/pages/home.yaml'
pod.write_yaml(path, {})
# Verify Google Sheets URLs.
fields = {
'preprocessors': [{
'kind': 'google_sheets',
'path': path,
'id': '012345',
'gid': '987654',
}],
}
pod.write_yaml('/podspec.yaml', fields)
doc = pod.get_doc(path)
preprocessor = pod.list_preprocessors()[0]
self.assertEqual(
'https://docs.google.com/spreadsheets/d/012345/edit#gid=987654',
preprocessor.get_edit_url(doc))
# Verify Google Docs URLs.
fields = {
'preprocessors': [{
'kind': 'google_docs',
'path': path,
'id': '012345',
}],
}
pod.write_yaml('/podspec.yaml', fields)
pod = pods.Pod(pod.root) # Reset pod.list_preprocessors.
preprocessor = pod.list_preprocessors()[0]
self.assertEqual('https://docs.google.com/document/d/012345/edit',
preprocessor.get_edit_url(doc))
def install(pod_path, gerrit):
"""Checks whether the pod depends on npm, bower, and gulp and installs them
if necessary. Then, runs install commands. Also optionally installs the
Gerrit Code Review commit hook."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
sdk_utils.install(pod, gerrit=gerrit)
def deploy(deployment_name, pod_path, build, confirm, test, test_only, auth):
"""Deploys a pod to a destination."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
try:
pod = pods.Pod(root, storage=storage.FileStorage)
deployment = pod.get_deployment(deployment_name)
if auth:
deployment.login(auth)
if build:
pod.preprocess()
# Set the environment information for the pod based on the deployment.
pod.env = deployment.get_env()
if test_only:
deployment.test()
return
paths_to_contents = pod.dump()
repo = utils.get_git_repo(pod.root)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
deployment.deploy(paths_to_contents,
stats=stats_obj,
repo=repo,
confirm=confirm,
test=test)
except base.Error as e:
raise click.ClickException(str(e))
except pods.Error as e:
raise click.ClickException(str(e))
def extract(pod_path, init, update, include_obsolete, localized,
include_header, locale, fuzzy_matching):
"""Extracts tagged messages from source files into a template catalog."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
catalogs = pod.get_catalogs()
catalogs.extract(include_obsolete=include_obsolete,
localized=localized,
include_header=include_header,
use_fuzzy_matching=fuzzy_matching)
if localized:
return
if init:
locales = validate_locales(pod.list_locales(), locale)
text = 'Initializing {} empty translation catalogs.'
pod.logger.info(text.format(len(locales)))
catalogs.init(locales=locales, include_header=include_header)
return
if update:
locales = validate_locales(catalogs.list_locales(), locale)
text = 'Updating {} catalogs with extracted messages.'
pod.logger.info(text.format(len(locales)))
catalogs.update(locales=locales,
include_header=include_header,
use_fuzzy_matching=fuzzy_matching)
def inspect_routes(pod_path, routes_file, deployment):
"""Lists routes handled by a pod."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
if deployment:
deployment_obj = pod.get_deployment(deployment)
pod.set_env(deployment_obj.config.env)
with pod.profile.timer('grow_inspect_routes'):
out = []
pod.router.use_simple()
pod.router.add_all()
if routes_file:
# print pod.router.routes.export()
pod.write_file(
routes_file,
json.dumps(
pod.router.routes.export(), sort_keys=True, indent=2,
separators=(',', ': ')))
print 'Exported routes to file: {}'.format(routes_file)
return pod
for path, node, _ in pod.router.routes.nodes:
out.append(u'{} [{}]\n {}\n'.format(path, node.kind, node.meta))
click.echo_via_pager('\n'.join(out))
return pod
def upload_translations(pod_path, locale, force, service, update_acl,
download, extract, prune, update_meta):
"""Uploads translations to a translation service."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
translator = pod.get_translator(service)
if not translator:
raise click.ClickException('No translators specified in podspec.yaml.')
if update_acl:
translator.update_acl(locales=locale)
return
if update_meta:
translator.update_meta(locales=locale)
return
if download:
translator.download(locales=locale)
if extract:
include_obsolete, localized, include_header, use_fuzzy_matching, = \
pod.catalogs.get_extract_config()
catalogs = pod.get_catalogs()
locales = validate_locales(pod.list_locales(), locale)
catalogs.extract(include_obsolete=include_obsolete, localized=localized,
include_header=include_header,
use_fuzzy_matching=use_fuzzy_matching, locales=locales)
if not localized:
catalogs.update(locales=locales, include_header=include_header,
use_fuzzy_matching=use_fuzzy_matching)
translator.upload(locales=locale, force=force, verbose=True, prune=prune)
def init(theme, pod_path, force):
"""Initializes a pod with a theme."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
with pod.profile.timer('grow_init'):
themes.init(pod, theme, force=force)
return pod
def inspect_untranslated(pod_path, locale, localized, threaded):
"""Displays statistics about the pod."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
with pod.profile.timer('grow_inspect_untranslated'):
# Find all of the untagged strings.
catalogs = pod.get_catalogs(template_path=None)
untagged_strings, _ = catalogs.extract(localized=localized,
locales=locale,
audit=True,
out_path=None)
# Find all the untranslated strings in use.
pod.enable(pod.FEATURE_TRANSLATION_STATS)
pod.router.use_simple()
pod.router.add_all()
if locale:
pod.router.filter(locales=list(locale))
content_generator = renderer.Renderer.rendered_docs(
pod, pod.router.routes, use_threading=threaded)
# Render each document to get the untranslated strings.
for _ in content_generator:
pass
pod.translation_stats.add_untagged(untagged_strings)
pod.translation_stats.pretty_print()
return pod
def stage(context, pod_path, remote, preprocess, subdomain, api_key,
force_untranslated):
"""Stages a build on a WebReview server."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
auth = context.parent.params.get('auth')
try:
pod = pods.Pod(root, storage=storage.FileStorage)
with pod.profile.timer('grow_stage'):
deployment = _get_deployment(pod, remote, subdomain, api_key)
# use the deployment's environment for preprocessing and later
# steps.
pod.set_env(deployment.config.env)
require_translations = pod.podspec.localization.get(
'require_translations', False)
require_translations = require_translations and not force_untranslated
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
content_generator = deployment.dump(pod)
repo = utils.get_git_repo(pod.root)
paths, _ = pod.determine_paths_to_build()
stats_obj = stats.Stats(pod, paths=paths)
deployment.deploy(content_generator,
stats=stats_obj,
repo=repo,
confirm=False,
test=False,
require_translations=require_translations)
except base.Error as err:
raise click.ClickException(str(err))
except pods.Error as err:
raise click.ClickException(str(err))
return pod
def setUp(self):
self.dir_path = testing.create_test_pod_dir()
self.pod = pods.Pod(self.dir_path,
storage=storage.FileStorage,
use_reroute=True)
self.batches = render_batch.RenderBatches(self.pod.render_pool,
self.pod.profile)
def setUp(self):
self.config = base_config.BaseConfig()
self.config.set('gerrit', True)
self.dir_path = testing.create_test_pod_dir()
self.pod = pods.Pod(self.dir_path, storage=storage.FileStorage)
self.installer = gerrit_installer.GerritInstaller(
self.pod, self.config)
def deploy(context, deployment_name, pod_path, preprocess, confirm, test,
test_only, auth):
"""Deploys a pod to a destination."""
if auth:
text = ('--auth must now be specified before deploy. Usage:'
' grow [email protected] deploy')
raise click.ClickException(text)
auth = context.parent.params.get('auth')
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
try:
pod = pods.Pod(root, storage=storage.FileStorage)
deployment = pod.get_deployment(deployment_name)
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
if test_only:
deployment.test()
return
paths_to_contents = deployment.dump(pod)
repo = utils.get_git_repo(pod.root)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
deployment.deploy(paths_to_contents, stats=stats_obj, repo=repo,
confirm=confirm, test=test)
except base.Error as e:
raise click.ClickException(str(e))
except pods.Error as e:
raise click.ClickException(str(e))
def test_toc(self):
pod = testing.create_pod()
pod.write_yaml('/podspec.yaml', {
'markdown': {
'extensions': [{
'kind': 'toc',
}],
}
})
pod.write_yaml('/content/pages/_blueprint.yaml', {
'$view': '/views/base.html',
'$path': '/{base}/'
})
pod.write_file('/content/pages/test.md', textwrap.dedent(
"""\
[TOC]
# H1
## H2
### H3
## H2 A
"""))
pod.write_file('/views/base.html', '{{doc.html|safe}}')
pod.router.add_all()
result = testing.render_path(pod, '/test/')
toc_sentinel = '<div class="toc">'
toclink_sentinel = '<a class="toclink"'
title_sentinel = '<span class="toctitle">toc-title</span>'
para_sentinel = '¶'
sep_sentinel = 'h2_a'
h2_sentinel = '<h2'
self.assertIn(toc_sentinel, result)
self.assertNotIn(title_sentinel, result)
self.assertNotIn(toclink_sentinel, result)
self.assertNotIn(sep_sentinel, result)
self.assertNotIn(para_sentinel, result)
self.assertIn(h2_sentinel, result)
pod.write_yaml('/podspec.yaml', {
'markdown': {
'extensions': [{
'kind': 'toc',
'title': 'toc-title',
'baselevel': 3,
'anchorlink': True,
'permalink': True,
'separator': '_'
}],
}
})
pod = pods.Pod(pod.root)
pod.router.add_all()
result = testing.render_path(pod, '/test/')
self.assertIn(title_sentinel, result)
self.assertNotIn(h2_sentinel, result)
self.assertIn(toclink_sentinel, result)
self.assertIn(sep_sentinel, result)
self.assertIn(para_sentinel, result)
def inspect_routes(pod_path, use_reroute):
"""Lists routes handled by a pod."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage, use_reroute=use_reroute)
with pod.profile.timer('grow_inspect_routes'):
out = []
if use_reroute:
pod.router.use_simple()
pod.router.add_all()
for path, node in pod.router.routes.nodes:
out.append(u'{}\n > {}'.format(path, node))
else:
pod_routes = pod.get_routes()
controllers_to_paths = collections.defaultdict(set)
for route in pod_routes:
name = str(route.endpoint)
paths = route.endpoint.list_concrete_paths()
controllers_to_paths[name].update(paths)
for controller, paths in controllers_to_paths.iteritems():
paths = sorted(paths)
if len(paths) > 1 or True:
out.append(controller)
for path in paths:
out.append(' {}'.format(path))
else:
out.append(paths[0])
out.append('')
click.echo_via_pager('\n'.join(out))
return pod
def preprocess(pod_path, preprocessor, run_all, tag, ratelimit):
"""Runs preprocessors."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
pod.preprocess(preprocessor,
run_all=run_all,
tags=tag,
ratelimit=ratelimit)
def download_translations(pod_path, locale, service):
"""Downloads translations from a translation service."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
with pod.profile.timer('grow_download_translations'):
translator = pod.get_translator(service)
translator.download(locales=locale)
return pod
def translations_diff(pod_path, translations, out_dir):
"""Diffs a translations directory with another."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
translations = os.path.join(translations, 'messages.pot')
other_catalogs = pod.get_catalogs(translations)
pod.catalogs.diff(other_catalogs, out_dir)
return pod
def test(pod_path):
"""Runs tests."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
try:
click.echo(pod.test())
except pods.Error as e:
raise click.ClickException(str(e))
def translations_download(pod_path, locale, service, include_obsolete, use_reroute):
"""Downloads translations from a translation service."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage, use_reroute=use_reroute)
with pod.profile.timer('grow_translations_download'):
translator = pod.get_translator(service)
translator.download(locales=locale, include_obsolete=include_obsolete)
return pod
def stage(context, pod_path, remote, preprocess, subdomain, api_key,
force_untranslated, threaded, work_dir, routes_file):
"""Stages a build on a WebReview server."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
auth = context.parent.params.get('auth')
try:
pod = pods.Pod(root, storage=storage.FileStorage)
with pod.profile.timer('grow_stage'):
deployment = _get_deployment(pod, remote, subdomain, api_key)
# use the deployment's environment for preprocessing and later
# steps.
pod.set_env(deployment.get_env())
# Always clear the cache when building.
pod.podcache.reset()
require_translations = \
pod.podspec.localization \
and pod.podspec.localization.get('require_translations', False)
require_translations = require_translations \
and not force_untranslated
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
repo = utils.get_git_repo(pod.root)
pod.router.use_simple()
if routes_file:
pod.router.from_data(pod.read_json(routes_file))
else:
pod.router.add_all()
if not work_dir:
# Preload the documents used by the paths after filtering.
docs_loader.DocsLoader.load_from_routes(pod, pod.router.routes)
paths = pod.router.routes.paths
stats_obj = stats.Stats(pod, paths=paths)
content_generator = deployment.dump(pod,
source_dir=work_dir,
use_threading=threaded)
content_generator = hooks.generator_wrapper(
pod, 'pre_deploy', content_generator, 'stage')
deployment.deploy(content_generator,
stats=stats_obj,
repo=repo,
confirm=False,
test=False,
require_translations=require_translations)
pod.podcache.write()
except bulk_errors.BulkErrors as err:
# Write the podcache files even when there are rendering errors.
pod.podcache.write()
bulk_errors.display_bulk_errors(err)
raise click.Abort()
# except base.Error as err:
# raise click.ClickException(str(err))
except pods.Error as err:
raise click.ClickException(str(err))
return pod
def test_request(self):
dir_path = testing.create_test_pod_dir()
pod = pods.Pod(dir_path)
pod.router.add_all(use_cache=False)
# When serving a pod, should 200.
app = main.create_wsgi_app(pod, 'localhost', 8080)
request = webob.Request.blank('/')
response = request.get_response(app)
self.assertEqual(200, response.status_int)
# Verify 404 is sent for page not found.
request = webob.Request.blank('/dummy/page/')
response = request.get_response(app)
self.assertEqual(404, response.status_int)
# Verify 206 for partial content.
headers = {'Range': 'bytes=0-4'}
request = webob.Request.blank('/public/file.txt', headers=headers)
response = request.get_response(app)
self.assertEqual(206, response.status_int)
self.assertEqual('bytes 0-4/13', response.headers['Content-Range'])
self.assertEqual(b'Hello', response.body)
headers = {'Range': 'bytes=5-13'}
request = webob.Request.blank('/public/file.txt', headers=headers)
response = request.get_response(app)
self.assertEqual('bytes 5-12/13', response.headers['Content-Range'])
self.assertEqual(b' World!\n', response.body)
# Verify full response when headers omitted.
request = webob.Request.blank('/public/file.txt')
response = request.get_response(app)
self.assertEqual(b'Hello World!\n', response.body)
# Verify 304.
url_path = '/public/file.txt'
matched = pod.router.routes.match(url_path)
controller = pod.router.get_render_controller(url_path,
matched.value,
params=matched.params)
response_headers = controller.get_http_headers()
headers = {'If-None-Match': response_headers['Last-Modified']}
request = webob.Request.blank(url_path, headers=headers)
response = request.get_response(app)
self.assertEqual(304, response.status_int)
self.assertEqual(b'', response.body)
response = request.get_response(app)
self.assertEqual(304, response.status_int)
self.assertEqual(b'', response.body)
# Verify sitemap on server.
path = '/root/sitemap.xml'
request = webob.Request.blank(path)
response = request.get_response(app)
self.assertEqual(200, response.status_int)
self.assertEqual('application/xml', response.headers['Content-Type'])
def test_noclasses(self):
pod = testing.create_pod()
fields = {
'markdown': {
'extensions': [{
'kind': 'sourcecode',
}],
}
}
pod.write_yaml('/podspec.yaml', fields)
fields = {'$view': '/views/base.html', '$path': '/{base}/'}
pod.write_yaml('/content/pages/_blueprint.yaml', fields)
content = """
[sourcecode:html]
<div class="test">
Hello World
</div>
[/sourcecode]
"""
pod.write_file('/content/pages/test.md', content)
content = '{{doc.html|safe}}'
pod.write_file('/views/base.html', content)
pod.router.add_all(use_cache=False)
result = testing.render_path(pod, '/test/')
style_sentinel = 'style="background: #f8f8f8"'
self.assertIn(style_sentinel, result)
# Verify no language.
content = """
[sourcecode]
<div class="test">
Hello World
</div>
[/sourcecode]
"""
pod.write_file('/content/pages/test.md', content)
content = '{{doc.html|safe}}'
pod.write_file('/views/base.html', content)
pod.router.add_all(use_cache=False)
result = testing.render_path(pod, '/test/')
style_sentinel = 'style="background: #f8f8f8"'
self.assertIn(style_sentinel, result)
fields = {
'markdown': {
'extensions': [{
'kind': 'sourcecode',
'highlighter': 'plain',
'classes': True,
}],
}
}
pod.write_yaml('/podspec.yaml', fields)
pod = pods.Pod(pod.root)
pod.router.add_all(use_cache=False)
result = testing.render_path(pod, '/test/')
code_sentinel = '<div class="code"><pre>'
self.assertIn(code_sentinel, result)
def stats(pod_path, full):
"""Displays statistics about the pod."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
try:
stats = stats_lib.Stats(pod, full=full)
click.echo_via_pager('\n\n'.join(stats.to_tables()))
except pods.Error as e:
raise click.ClickException(str(e))
def install(pod_path, gerrit):
"""Checks whether the pod depends on npm, bower, and gulp and installs them
if necessary. Then, runs install commands. Also optionally installs the
Gerrit Code Review commit hook."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage, load_extensions=False)
with pod.profile.timer('grow_install'):
sdk_utils.install(pod, gerrit=gerrit)
return pod
def build(pod_path, out_dir, preprocess, clear_cache, pod_paths,
locate_untranslated, deployment, threaded, locale):
"""Generates static files and dumps them to a local destination."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
out_dir = out_dir or os.path.join(root, 'build')
pod = pods.Pod(root, storage=storage.FileStorage)
if not pod_paths or clear_cache:
# Clear the cache when building all, only force if the flag is used.
pod.podcache.reset(force=clear_cache)
deployment_obj = None
if deployment:
deployment_obj = pod.get_deployment(deployment)
pod.set_env(deployment_obj.config.env)
if preprocess:
with pod.profile.timer('grow_preprocess'):
pod.preprocess()
if locate_untranslated:
pod.enable(pod.FEATURE_TRANSLATION_STATS)
try:
with pod.profile.timer('grow_build'):
config = local_destination.Config(out_dir=out_dir)
# When using a specific deployment env need to also copy over.
if deployment_obj:
config.env = deployment_obj.config.env
destination = local_destination.LocalDestination(config)
destination.pod = pod
repo = utils.get_git_repo(pod.root)
pod.router.use_simple()
if pod_paths:
pod.router.add_pod_paths(pod_paths)
else:
pod.router.add_all()
if locale:
pod.router.filter(locales=list(locale))
paths = pod.router.routes.paths
content_generator = renderer.Renderer.rendered_docs(
pod, pod.router.routes, use_threading=threaded)
stats_obj = stats.Stats(pod, paths=paths)
is_partial = bool(pod_paths) or bool(locale)
destination.deploy(content_generator,
stats=stats_obj,
repo=repo,
confirm=False,
test=False,
is_partial=is_partial)
pod.podcache.write()
except renderer.RenderErrors as err:
# Ignore the build error since it outputs the errors.
raise click.ClickException(str(err))
except pods.Error as err:
raise click.ClickException(str(err))
if locate_untranslated:
pod.translation_stats.pretty_print()
destination.export_untranslated_catalogs()
return pod
def preprocess(pod_path, preprocessor, run_all, tag, ratelimit, deployment):
"""Runs preprocessors."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
if deployment:
deployment_obj = pod.get_deployment(deployment)
pod.set_env(deployment_obj.config.env)
with pod.profile.timer('grow_preprocess'):
pod.preprocess(preprocessor, run_all=run_all, tags=tag, ratelimit=ratelimit)
return pod
def convert(pod_path, type):
"""Converts pod files from an earlier version of Grow."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
if type == 'content_locale_split':
content_locale_split.Converter.convert(pod)
else:
raise click.UsageError(
'Unable to convert files without a --type option.\n'
'Run `grow convert --help` to see valid --type values.')
def machine_translate(pod_path, locale):
"""Translates the pod message catalog using machine translation."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
pod.catalogs.extract()
for locale in locale:
catalog = pod.catalogs.get(locale)
catalog.update()
catalog.machine_translate()
print colorize('WARNING! Use machine translations with caution.', ansi=197)
print colorize('Machine translations are not intended for use in production.', ansi=197)
def inspect_stats(pod_path, full):
"""Displays statistics about the pod."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
pod = pods.Pod(root, storage=storage.FileStorage)
try:
with pod.profile.timer('grow_inspect_stats'):
pod_stats = stats_lib.Stats(pod, full=full)
click.echo_via_pager('\n\n'.join(pod_stats.to_tables()))
except pods.Error as err:
raise click.ClickException(str(err))
return pod
