def run(host, port, https, debug, browser, update_check, preprocess, ui, pod_path, deployment): """Starts a development server for a single pod.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) scheme = 'https' if https else 'http' config = env.EnvConfig(host=host, port=port, name=env.Name.DEV, scheme=scheme, cached=False, dev=True) environment = env.Env(config) pod = pods.Pod(root, storage=storage.FileStorage, env=environment) if deployment: deployment_obj = pod.get_deployment(deployment) pod.set_env(deployment_obj.config.env) if not ui: pod.disable(pod.FEATURE_UI) try: manager.start(pod, host=host, port=port, open_browser=browser, debug=debug, preprocess=preprocess, update_check=update_check) except pods.Error as e: raise click.ClickException(str(e)) return pod
def test_get_edit_url(self): pod = testing.create_pod() path = '/content/pages/home.yaml' pod.write_yaml(path, {}) # Verify Google Sheets URLs. fields = { 'preprocessors': [{ 'kind': 'google_sheets', 'path': path, 'id': '012345', 'gid': '987654', }], } pod.write_yaml('/podspec.yaml', fields) doc = pod.get_doc(path) preprocessor = pod.list_preprocessors()[0] self.assertEqual( 'https://docs.google.com/spreadsheets/d/012345/edit#gid=987654', preprocessor.get_edit_url(doc)) # Verify Google Docs URLs. fields = { 'preprocessors': [{ 'kind': 'google_docs', 'path': path, 'id': '012345', }], } pod.write_yaml('/podspec.yaml', fields) pod = pods.Pod(pod.root) # Reset pod.list_preprocessors. preprocessor = pod.list_preprocessors()[0] self.assertEqual('https://docs.google.com/document/d/012345/edit', preprocessor.get_edit_url(doc))
def install(pod_path, gerrit): """Checks whether the pod depends on npm, bower, and gulp and installs them if necessary. Then, runs install commands. Also optionally installs the Gerrit Code Review commit hook.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) sdk_utils.install(pod, gerrit=gerrit)
def deploy(deployment_name, pod_path, build, confirm, test, test_only, auth): """Deploys a pod to a destination.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) try: pod = pods.Pod(root, storage=storage.FileStorage) deployment = pod.get_deployment(deployment_name) if auth: deployment.login(auth) if build: pod.preprocess() # Set the environment information for the pod based on the deployment. pod.env = deployment.get_env() if test_only: deployment.test() return paths_to_contents = pod.dump() repo = utils.get_git_repo(pod.root) stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents) deployment.deploy(paths_to_contents, stats=stats_obj, repo=repo, confirm=confirm, test=test) except base.Error as e: raise click.ClickException(str(e)) except pods.Error as e: raise click.ClickException(str(e))
def extract(pod_path, init, update, include_obsolete, localized, include_header, locale, fuzzy_matching): """Extracts tagged messages from source files into a template catalog.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) catalogs = pod.get_catalogs() catalogs.extract(include_obsolete=include_obsolete, localized=localized, include_header=include_header, use_fuzzy_matching=fuzzy_matching) if localized: return if init: locales = validate_locales(pod.list_locales(), locale) text = 'Initializing {} empty translation catalogs.' pod.logger.info(text.format(len(locales))) catalogs.init(locales=locales, include_header=include_header) return if update: locales = validate_locales(catalogs.list_locales(), locale) text = 'Updating {} catalogs with extracted messages.' pod.logger.info(text.format(len(locales))) catalogs.update(locales=locales, include_header=include_header, use_fuzzy_matching=fuzzy_matching)
def inspect_routes(pod_path, routes_file, deployment): """Lists routes handled by a pod.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) if deployment: deployment_obj = pod.get_deployment(deployment) pod.set_env(deployment_obj.config.env) with pod.profile.timer('grow_inspect_routes'): out = [] pod.router.use_simple() pod.router.add_all() if routes_file: # print pod.router.routes.export() pod.write_file( routes_file, json.dumps( pod.router.routes.export(), sort_keys=True, indent=2, separators=(',', ': '))) print 'Exported routes to file: {}'.format(routes_file) return pod for path, node, _ in pod.router.routes.nodes: out.append(u'{} [{}]\n {}\n'.format(path, node.kind, node.meta)) click.echo_via_pager('\n'.join(out)) return pod
def upload_translations(pod_path, locale, force, service, update_acl, download, extract, prune, update_meta): """Uploads translations to a translation service.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) translator = pod.get_translator(service) if not translator: raise click.ClickException('No translators specified in podspec.yaml.') if update_acl: translator.update_acl(locales=locale) return if update_meta: translator.update_meta(locales=locale) return if download: translator.download(locales=locale) if extract: include_obsolete, localized, include_header, use_fuzzy_matching, = \ pod.catalogs.get_extract_config() catalogs = pod.get_catalogs() locales = validate_locales(pod.list_locales(), locale) catalogs.extract(include_obsolete=include_obsolete, localized=localized, include_header=include_header, use_fuzzy_matching=use_fuzzy_matching, locales=locales) if not localized: catalogs.update(locales=locales, include_header=include_header, use_fuzzy_matching=use_fuzzy_matching) translator.upload(locales=locale, force=force, verbose=True, prune=prune)
def init(theme, pod_path, force): """Initializes a pod with a theme.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) with pod.profile.timer('grow_init'): themes.init(pod, theme, force=force) return pod
def inspect_untranslated(pod_path, locale, localized, threaded): """Displays statistics about the pod.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) with pod.profile.timer('grow_inspect_untranslated'): # Find all of the untagged strings. catalogs = pod.get_catalogs(template_path=None) untagged_strings, _ = catalogs.extract(localized=localized, locales=locale, audit=True, out_path=None) # Find all the untranslated strings in use. pod.enable(pod.FEATURE_TRANSLATION_STATS) pod.router.use_simple() pod.router.add_all() if locale: pod.router.filter(locales=list(locale)) content_generator = renderer.Renderer.rendered_docs( pod, pod.router.routes, use_threading=threaded) # Render each document to get the untranslated strings. for _ in content_generator: pass pod.translation_stats.add_untagged(untagged_strings) pod.translation_stats.pretty_print() return pod
def stage(context, pod_path, remote, preprocess, subdomain, api_key, force_untranslated): """Stages a build on a WebReview server.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) auth = context.parent.params.get('auth') try: pod = pods.Pod(root, storage=storage.FileStorage) with pod.profile.timer('grow_stage'): deployment = _get_deployment(pod, remote, subdomain, api_key) # use the deployment's environment for preprocessing and later # steps. pod.set_env(deployment.config.env) require_translations = pod.podspec.localization.get( 'require_translations', False) require_translations = require_translations and not force_untranslated if auth: deployment.login(auth) if preprocess: pod.preprocess() content_generator = deployment.dump(pod) repo = utils.get_git_repo(pod.root) paths, _ = pod.determine_paths_to_build() stats_obj = stats.Stats(pod, paths=paths) deployment.deploy(content_generator, stats=stats_obj, repo=repo, confirm=False, test=False, require_translations=require_translations) except base.Error as err: raise click.ClickException(str(err)) except pods.Error as err: raise click.ClickException(str(err)) return pod
def setUp(self): self.dir_path = testing.create_test_pod_dir() self.pod = pods.Pod(self.dir_path, storage=storage.FileStorage, use_reroute=True) self.batches = render_batch.RenderBatches(self.pod.render_pool, self.pod.profile)
def setUp(self): self.config = base_config.BaseConfig() self.config.set('gerrit', True) self.dir_path = testing.create_test_pod_dir() self.pod = pods.Pod(self.dir_path, storage=storage.FileStorage) self.installer = gerrit_installer.GerritInstaller( self.pod, self.config)
def deploy(context, deployment_name, pod_path, preprocess, confirm, test, test_only, auth): """Deploys a pod to a destination.""" if auth: text = ('--auth must now be specified before deploy. Usage:' ' grow [email protected] deploy') raise click.ClickException(text) auth = context.parent.params.get('auth') root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) try: pod = pods.Pod(root, storage=storage.FileStorage) deployment = pod.get_deployment(deployment_name) if auth: deployment.login(auth) if preprocess: pod.preprocess() if test_only: deployment.test() return paths_to_contents = deployment.dump(pod) repo = utils.get_git_repo(pod.root) stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents) deployment.deploy(paths_to_contents, stats=stats_obj, repo=repo, confirm=confirm, test=test) except base.Error as e: raise click.ClickException(str(e)) except pods.Error as e: raise click.ClickException(str(e))
def test_toc(self): pod = testing.create_pod() pod.write_yaml('/podspec.yaml', { 'markdown': { 'extensions': [{ 'kind': 'toc', }], } }) pod.write_yaml('/content/pages/_blueprint.yaml', { '$view': '/views/base.html', '$path': '/{base}/' }) pod.write_file('/content/pages/test.md', textwrap.dedent( """\ [TOC] # H1 ## H2 ### H3 ## H2 A """)) pod.write_file('/views/base.html', '{{doc.html|safe}}') pod.router.add_all() result = testing.render_path(pod, '/test/') toc_sentinel = '<div class="toc">' toclink_sentinel = '<a class="toclink"' title_sentinel = '<span class="toctitle">toc-title</span>' para_sentinel = '¶' sep_sentinel = 'h2_a' h2_sentinel = '<h2' self.assertIn(toc_sentinel, result) self.assertNotIn(title_sentinel, result) self.assertNotIn(toclink_sentinel, result) self.assertNotIn(sep_sentinel, result) self.assertNotIn(para_sentinel, result) self.assertIn(h2_sentinel, result) pod.write_yaml('/podspec.yaml', { 'markdown': { 'extensions': [{ 'kind': 'toc', 'title': 'toc-title', 'baselevel': 3, 'anchorlink': True, 'permalink': True, 'separator': '_' }], } }) pod = pods.Pod(pod.root) pod.router.add_all() result = testing.render_path(pod, '/test/') self.assertIn(title_sentinel, result) self.assertNotIn(h2_sentinel, result) self.assertIn(toclink_sentinel, result) self.assertIn(sep_sentinel, result) self.assertIn(para_sentinel, result)
def inspect_routes(pod_path, use_reroute): """Lists routes handled by a pod.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage, use_reroute=use_reroute) with pod.profile.timer('grow_inspect_routes'): out = [] if use_reroute: pod.router.use_simple() pod.router.add_all() for path, node in pod.router.routes.nodes: out.append(u'{}\n > {}'.format(path, node)) else: pod_routes = pod.get_routes() controllers_to_paths = collections.defaultdict(set) for route in pod_routes: name = str(route.endpoint) paths = route.endpoint.list_concrete_paths() controllers_to_paths[name].update(paths) for controller, paths in controllers_to_paths.iteritems(): paths = sorted(paths) if len(paths) > 1 or True: out.append(controller) for path in paths: out.append(' {}'.format(path)) else: out.append(paths[0]) out.append('') click.echo_via_pager('\n'.join(out)) return pod
def preprocess(pod_path, preprocessor, run_all, tag, ratelimit): """Runs preprocessors.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) pod.preprocess(preprocessor, run_all=run_all, tags=tag, ratelimit=ratelimit)
def download_translations(pod_path, locale, service): """Downloads translations from a translation service.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) with pod.profile.timer('grow_download_translations'): translator = pod.get_translator(service) translator.download(locales=locale) return pod
def translations_diff(pod_path, translations, out_dir): """Diffs a translations directory with another.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) translations = os.path.join(translations, 'messages.pot') other_catalogs = pod.get_catalogs(translations) pod.catalogs.diff(other_catalogs, out_dir) return pod
def test(pod_path): """Runs tests.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) try: click.echo(pod.test()) except pods.Error as e: raise click.ClickException(str(e))
def translations_download(pod_path, locale, service, include_obsolete, use_reroute): """Downloads translations from a translation service.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage, use_reroute=use_reroute) with pod.profile.timer('grow_translations_download'): translator = pod.get_translator(service) translator.download(locales=locale, include_obsolete=include_obsolete) return pod
def stage(context, pod_path, remote, preprocess, subdomain, api_key, force_untranslated, threaded, work_dir, routes_file): """Stages a build on a WebReview server.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) auth = context.parent.params.get('auth') try: pod = pods.Pod(root, storage=storage.FileStorage) with pod.profile.timer('grow_stage'): deployment = _get_deployment(pod, remote, subdomain, api_key) # use the deployment's environment for preprocessing and later # steps. pod.set_env(deployment.get_env()) # Always clear the cache when building. pod.podcache.reset() require_translations = \ pod.podspec.localization \ and pod.podspec.localization.get('require_translations', False) require_translations = require_translations \ and not force_untranslated if auth: deployment.login(auth) if preprocess: pod.preprocess() repo = utils.get_git_repo(pod.root) pod.router.use_simple() if routes_file: pod.router.from_data(pod.read_json(routes_file)) else: pod.router.add_all() if not work_dir: # Preload the documents used by the paths after filtering. docs_loader.DocsLoader.load_from_routes(pod, pod.router.routes) paths = pod.router.routes.paths stats_obj = stats.Stats(pod, paths=paths) content_generator = deployment.dump(pod, source_dir=work_dir, use_threading=threaded) content_generator = hooks.generator_wrapper( pod, 'pre_deploy', content_generator, 'stage') deployment.deploy(content_generator, stats=stats_obj, repo=repo, confirm=False, test=False, require_translations=require_translations) pod.podcache.write() except bulk_errors.BulkErrors as err: # Write the podcache files even when there are rendering errors. pod.podcache.write() bulk_errors.display_bulk_errors(err) raise click.Abort() # except base.Error as err: # raise click.ClickException(str(err)) except pods.Error as err: raise click.ClickException(str(err)) return pod
def test_request(self): dir_path = testing.create_test_pod_dir() pod = pods.Pod(dir_path) pod.router.add_all(use_cache=False) # When serving a pod, should 200. app = main.create_wsgi_app(pod, 'localhost', 8080) request = webob.Request.blank('/') response = request.get_response(app) self.assertEqual(200, response.status_int) # Verify 404 is sent for page not found. request = webob.Request.blank('/dummy/page/') response = request.get_response(app) self.assertEqual(404, response.status_int) # Verify 206 for partial content. headers = {'Range': 'bytes=0-4'} request = webob.Request.blank('/public/file.txt', headers=headers) response = request.get_response(app) self.assertEqual(206, response.status_int) self.assertEqual('bytes 0-4/13', response.headers['Content-Range']) self.assertEqual(b'Hello', response.body) headers = {'Range': 'bytes=5-13'} request = webob.Request.blank('/public/file.txt', headers=headers) response = request.get_response(app) self.assertEqual('bytes 5-12/13', response.headers['Content-Range']) self.assertEqual(b' World!\n', response.body) # Verify full response when headers omitted. request = webob.Request.blank('/public/file.txt') response = request.get_response(app) self.assertEqual(b'Hello World!\n', response.body) # Verify 304. url_path = '/public/file.txt' matched = pod.router.routes.match(url_path) controller = pod.router.get_render_controller(url_path, matched.value, params=matched.params) response_headers = controller.get_http_headers() headers = {'If-None-Match': response_headers['Last-Modified']} request = webob.Request.blank(url_path, headers=headers) response = request.get_response(app) self.assertEqual(304, response.status_int) self.assertEqual(b'', response.body) response = request.get_response(app) self.assertEqual(304, response.status_int) self.assertEqual(b'', response.body) # Verify sitemap on server. path = '/root/sitemap.xml' request = webob.Request.blank(path) response = request.get_response(app) self.assertEqual(200, response.status_int) self.assertEqual('application/xml', response.headers['Content-Type'])
def test_noclasses(self): pod = testing.create_pod() fields = { 'markdown': { 'extensions': [{ 'kind': 'sourcecode', }], } } pod.write_yaml('/podspec.yaml', fields) fields = {'$view': '/views/base.html', '$path': '/{base}/'} pod.write_yaml('/content/pages/_blueprint.yaml', fields) content = """ [sourcecode:html] <div class="test"> Hello World </div> [/sourcecode] """ pod.write_file('/content/pages/test.md', content) content = '{{doc.html|safe}}' pod.write_file('/views/base.html', content) pod.router.add_all(use_cache=False) result = testing.render_path(pod, '/test/') style_sentinel = 'style="background: #f8f8f8"' self.assertIn(style_sentinel, result) # Verify no language. content = """ [sourcecode] <div class="test"> Hello World </div> [/sourcecode] """ pod.write_file('/content/pages/test.md', content) content = '{{doc.html|safe}}' pod.write_file('/views/base.html', content) pod.router.add_all(use_cache=False) result = testing.render_path(pod, '/test/') style_sentinel = 'style="background: #f8f8f8"' self.assertIn(style_sentinel, result) fields = { 'markdown': { 'extensions': [{ 'kind': 'sourcecode', 'highlighter': 'plain', 'classes': True, }], } } pod.write_yaml('/podspec.yaml', fields) pod = pods.Pod(pod.root) pod.router.add_all(use_cache=False) result = testing.render_path(pod, '/test/') code_sentinel = '<div class="code"><pre>' self.assertIn(code_sentinel, result)
def stats(pod_path, full): """Displays statistics about the pod.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) try: stats = stats_lib.Stats(pod, full=full) click.echo_via_pager('\n\n'.join(stats.to_tables())) except pods.Error as e: raise click.ClickException(str(e))
def install(pod_path, gerrit): """Checks whether the pod depends on npm, bower, and gulp and installs them if necessary. Then, runs install commands. Also optionally installs the Gerrit Code Review commit hook.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage, load_extensions=False) with pod.profile.timer('grow_install'): sdk_utils.install(pod, gerrit=gerrit) return pod
def build(pod_path, out_dir, preprocess, clear_cache, pod_paths, locate_untranslated, deployment, threaded, locale): """Generates static files and dumps them to a local destination.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) out_dir = out_dir or os.path.join(root, 'build') pod = pods.Pod(root, storage=storage.FileStorage) if not pod_paths or clear_cache: # Clear the cache when building all, only force if the flag is used. pod.podcache.reset(force=clear_cache) deployment_obj = None if deployment: deployment_obj = pod.get_deployment(deployment) pod.set_env(deployment_obj.config.env) if preprocess: with pod.profile.timer('grow_preprocess'): pod.preprocess() if locate_untranslated: pod.enable(pod.FEATURE_TRANSLATION_STATS) try: with pod.profile.timer('grow_build'): config = local_destination.Config(out_dir=out_dir) # When using a specific deployment env need to also copy over. if deployment_obj: config.env = deployment_obj.config.env destination = local_destination.LocalDestination(config) destination.pod = pod repo = utils.get_git_repo(pod.root) pod.router.use_simple() if pod_paths: pod.router.add_pod_paths(pod_paths) else: pod.router.add_all() if locale: pod.router.filter(locales=list(locale)) paths = pod.router.routes.paths content_generator = renderer.Renderer.rendered_docs( pod, pod.router.routes, use_threading=threaded) stats_obj = stats.Stats(pod, paths=paths) is_partial = bool(pod_paths) or bool(locale) destination.deploy(content_generator, stats=stats_obj, repo=repo, confirm=False, test=False, is_partial=is_partial) pod.podcache.write() except renderer.RenderErrors as err: # Ignore the build error since it outputs the errors. raise click.ClickException(str(err)) except pods.Error as err: raise click.ClickException(str(err)) if locate_untranslated: pod.translation_stats.pretty_print() destination.export_untranslated_catalogs() return pod
def preprocess(pod_path, preprocessor, run_all, tag, ratelimit, deployment): """Runs preprocessors.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) if deployment: deployment_obj = pod.get_deployment(deployment) pod.set_env(deployment_obj.config.env) with pod.profile.timer('grow_preprocess'): pod.preprocess(preprocessor, run_all=run_all, tags=tag, ratelimit=ratelimit) return pod
def convert(pod_path, type): """Converts pod files from an earlier version of Grow.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) if type == 'content_locale_split': content_locale_split.Converter.convert(pod) else: raise click.UsageError( 'Unable to convert files without a --type option.\n' 'Run `grow convert --help` to see valid --type values.')
def machine_translate(pod_path, locale): """Translates the pod message catalog using machine translation.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) pod.catalogs.extract() for locale in locale: catalog = pod.catalogs.get(locale) catalog.update() catalog.machine_translate() print colorize('WARNING! Use machine translations with caution.', ansi=197) print colorize('Machine translations are not intended for use in production.', ansi=197)
def inspect_stats(pod_path, full): """Displays statistics about the pod.""" root = os.path.abspath(os.path.join(os.getcwd(), pod_path)) pod = pods.Pod(root, storage=storage.FileStorage) try: with pod.profile.timer('grow_inspect_stats'): pod_stats = stats_lib.Stats(pod, full=full) click.echo_via_pager('\n\n'.join(pod_stats.to_tables())) except pods.Error as err: raise click.ClickException(str(err)) return pod