def deploy(deployment_name, pod_path, build, confirm, test, test_only, auth):
"""Deploys a pod to a destination."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
try:
pod = pods.Pod(root, storage=storage.FileStorage)
deployment = pod.get_deployment(deployment_name)
if auth:
deployment.login(auth)
if build:
pod.preprocess()
# Set the environment information for the pod based on the deployment.
pod.env = deployment.get_env()
if test_only:
deployment.test()
return
paths_to_contents = pod.dump()
repo = utils.get_git_repo(pod.root)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
deployment.deploy(paths_to_contents,
stats=stats_obj,
repo=repo,
confirm=confirm,
test=test)
except base.Error as e:
raise click.ClickException(str(e))
except pods.Error as e:
raise click.ClickException(str(e))
def _test_deploy(self, repo_url):
pod = testing.create_pod()
pod.write_yaml('/podspec.yaml', {
'deployments': {
'git': {
'destination': 'git',
'repo': repo_url,
'branch': 'gh-pages',
},
},
})
pod.write_yaml('/content/pages/_blueprint.yaml', {})
pod.write_yaml('/content/pages/page.yaml', {
'$path': '/{base}/',
'$view': '/views/base.html',
})
pod.write_file('/views/base.html', str(random.randint(0, 999)))
deployment = pod.get_deployment('git')
paths = []
for rendered_doc in deployment.dump(pod):
paths.append(rendered_doc.path)
repo = utils.get_git_repo(pod.root)
stats_obj = stats.Stats(pod, paths=paths)
deployment.deploy(deployment.dump(pod), stats=stats_obj, repo=repo,
confirm=False, test=False)
def get_branch(self):
if self.config.branch and not self.config.branch == 'auto':
return self.config.branch
# Always use "master" on localhost.
if self.config.server.startswith('localhost'):
return 'master'
if self.config.debug:
self.pod.logger.info('environ: {}'.format(os.environ))
if os.environ.get('FILESET_BRANCH_NAME'):
branch = os.environ['FILESET_BRANCH_NAME']
elif os.environ.get('BRANCH_NAME'):
# Google Cloud Build uses "BRANCH_NAME" environ variable.
# https://cloud.google.com/cloud-build/docs/configuring-builds/substitute-variable-values
branch = os.environ['BRANCH_NAME']
elif os.environ.get('CI_COMMIT_REF_NAME'):
# Gitlab uses a detached git reference, so use the
# "CI_COMMIT_REF_NAME" environ variable instead.
branch = os.environ['CI_COMMIT_REF_NAME']
else:
repo = utils.get_git_repo(self.pod.root)
branch = repo.active_branch.name
if branch.startswith('feature/'):
branch = branch[8:]
branch = branch.replace('/', '-').lower()
# Append branch prefix.
branch_prefix = self.config.branch_prefix or ''
return branch_prefix + branch
def deploy(context, deployment_name, pod_path, preprocess, confirm, test,
test_only, auth):
"""Deploys a pod to a destination."""
if auth:
text = ('--auth must now be specified before deploy. Usage:'
' grow [email protected] deploy')
raise click.ClickException(text)
auth = context.parent.params.get('auth')
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
try:
pod = pods.Pod(root, storage=storage.FileStorage)
deployment = pod.get_deployment(deployment_name)
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
if test_only:
deployment.test()
return
paths_to_contents = deployment.dump(pod)
repo = utils.get_git_repo(pod.root)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
deployment.deploy(paths_to_contents, stats=stats_obj, repo=repo,
confirm=confirm, test=test)
except base.Error as e:
raise click.ClickException(str(e))
except pods.Error as e:
raise click.ClickException(str(e))
def stage(context, pod_path, remote, preprocess, subdomain, api_key,
force_untranslated):
"""Stages a build on a WebReview server."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
auth = context.parent.params.get('auth')
try:
pod = pods.Pod(root, storage=storage.FileStorage)
with pod.profile.timer('grow_stage'):
deployment = _get_deployment(pod, remote, subdomain, api_key)
# use the deployment's environment for preprocessing and later
# steps.
pod.set_env(deployment.config.env)
require_translations = pod.podspec.localization.get(
'require_translations', False)
require_translations = require_translations and not force_untranslated
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
content_generator = deployment.dump(pod)
repo = utils.get_git_repo(pod.root)
paths, _ = pod.determine_paths_to_build()
stats_obj = stats.Stats(pod, paths=paths)
deployment.deploy(content_generator,
stats=stats_obj,
repo=repo,
confirm=False,
test=False,
require_translations=require_translations)
except base.Error as err:
raise click.ClickException(str(err))
except pods.Error as err:
raise click.ClickException(str(err))
return pod
def _test_deploy(self, repo_url):
pod = testing.create_pod()
pod.write_yaml('/podspec.yaml', {
'deployments': {
'git': {
'destination': 'git',
'repo': repo_url,
'branch': 'gh-pages',
},
},
})
pod.write_yaml('/content/pages/_blueprint.yaml', {})
pod.write_yaml('/content/pages/page.yaml', {
'$path': '/{base}/',
'$view': '/views/base.html',
})
pod.write_file('/views/base.html', str(random.randint(0, 999)))
deployment = pod.get_deployment('git')
paths = []
for rendered_doc in deployment.dump(pod):
paths.append(rendered_doc.path)
repo = utils.get_git_repo(pod.root)
stats_obj = stats.Stats(pod, paths=paths)
deployment.deploy(deployment.dump(pod), stats=stats_obj, repo=repo,
confirm=False, test=False)
def deploy(context, deployment_name, pod_path, preprocess, confirm, test, test_only, auth):
"""Deploys a pod to a destination."""
if auth:
text = "--auth must now be specified before deploy. Usage:" " grow [email protected] deploy"
raise click.ClickException(text)
auth = context.parent.params.get("auth")
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
try:
pod = pods.Pod(root, storage=storage.FileStorage)
deployment = pod.get_deployment(deployment_name)
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
if test_only:
deployment.test()
return
paths_to_contents = deployment.dump(pod)
repo = utils.get_git_repo(pod.root)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
deployment.deploy(paths_to_contents, stats=stats_obj, repo=repo, confirm=confirm, test=test)
except base.Error as e:
raise click.ClickException(str(e))
except pods.Error as e:
raise click.ClickException(str(e))
def get_commit(self):
if self.config.debug:
self.pod.logger.info('environ: {}'.format(os.environ))
if os.environ.get('FILESET_COMMIT_SHA'):
sha = os.environ['FILESET_COMMIT_SHA']
message = os.environ.get('FILESET_COMMIT_TITLE', '')
elif os.environ.get('COMMIT_SHA'):
# Google Cloud Build uses "BRANCH_NAME" environ variable.
# https://cloud.google.com/cloud-build/docs/configuring-builds/substitute-variable-values
sha = os.environ['COMMIT_SHA']
message = ''
elif os.environ.get('CI_COMMIT_SHA'):
# Gitlab uses a detached git reference, so use the
# "CI_COMMIT_SHA" environ variable instead.
sha = os.environ['CI_COMMIT_SHA']
message = os.environ.get('CI_COMMIT_TITLE', '')
else:
repo = utils.get_git_repo(self.pod.root)
if repo and repo.active_branch:
commit = repo.active_branch.commit
sha = commit.hexsha
message = commit.message.split('\n', 1)[0]
else:
sha = ''
message = ''
return {
'sha': sha,
'message': message,
}
def build(pod_path, out_dir, preprocess, clear_cache, pod_paths,
locate_untranslated, deployment, threaded, locale, shards, shard):
"""Generates static files and dumps them to a local destination."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
out_dir = out_dir or os.path.join(root, 'build')
pod = pods.Pod(root, storage=storage.FileStorage)
if not pod_paths or clear_cache:
# Clear the cache when building all, only force if the flag is used.
pod.podcache.reset(force=clear_cache)
deployment_obj = None
if deployment:
deployment_obj = pod.get_deployment(deployment)
pod.set_env(deployment_obj.config.env)
if preprocess:
with pod.profile.timer('grow_preprocess'):
pod.preprocess()
if locate_untranslated:
pod.enable(pod.FEATURE_TRANSLATION_STATS)
try:
with pod.profile.timer('grow_build'):
config = local_destination.Config(out_dir=out_dir)
# When using a specific deployment env need to also copy over.
if deployment_obj:
config.env = deployment_obj.config.env
destination = local_destination.LocalDestination(config)
destination.pod = pod
repo = utils.get_git_repo(pod.root)
pod.router.use_simple()
is_partial = bool(pod_paths) or bool(locale)
if pod_paths:
pod.router.add_pod_paths(pod_paths)
else:
pod.router.add_all()
if locale:
pod.router.filter('whitelist', locales=list(locale))
# Shard the routes when using sharding.
if shards and shard:
is_partial = True
pod.router.shard(shards, shard)
paths = pod.router.routes.paths
content_generator = renderer.Renderer.rendered_docs(
pod, pod.router.routes, use_threading=threaded)
stats_obj = stats.Stats(pod, paths=paths)
destination.deploy(
content_generator, stats=stats_obj, repo=repo, confirm=False,
test=False, is_partial=is_partial)
pod.podcache.write()
except renderer.RenderErrors as err:
# Write the podcache files even when there are rendering errors.
pod.podcache.write()
# Ignore the build error since it outputs the errors.
raise click.ClickException(str(err))
except pods.Error as err:
raise click.ClickException(str(err))
if locate_untranslated:
pod.translation_stats.pretty_print()
destination.export_untranslated_catalogs()
return pod
def stage(context, pod_path, remote, preprocess, subdomain, api_key,
force_untranslated, threaded, work_dir, routes_file):
"""Stages a build on a WebReview server."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
auth = context.parent.params.get('auth')
try:
pod = pods.Pod(root, storage=storage.FileStorage)
with pod.profile.timer('grow_stage'):
deployment = _get_deployment(pod, remote, subdomain, api_key)
# use the deployment's environment for preprocessing and later
# steps.
pod.set_env(deployment.get_env())
# Always clear the cache when building.
pod.podcache.reset()
require_translations = \
pod.podspec.localization \
and pod.podspec.localization.get('require_translations', False)
require_translations = require_translations \
and not force_untranslated
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
repo = utils.get_git_repo(pod.root)
pod.router.use_simple()
if routes_file:
pod.router.from_data(pod.read_json(routes_file))
else:
pod.router.add_all()
if not work_dir:
# Preload the documents used by the paths after filtering.
docs_loader.DocsLoader.load_from_routes(pod, pod.router.routes)
paths = pod.router.routes.paths
stats_obj = stats.Stats(pod, paths=paths)
content_generator = deployment.dump(pod,
source_dir=work_dir,
use_threading=threaded)
content_generator = hooks.generator_wrapper(
pod, 'pre_deploy', content_generator, 'stage')
deployment.deploy(content_generator,
stats=stats_obj,
repo=repo,
confirm=False,
test=False,
require_translations=require_translations)
pod.podcache.write()
except bulk_errors.BulkErrors as err:
# Write the podcache files even when there are rendering errors.
pod.podcache.write()
bulk_errors.display_bulk_errors(err)
raise click.Abort()
# except base.Error as err:
# raise click.ClickException(str(err))
except pods.Error as err:
raise click.ClickException(str(err))
return pod
def _get_subdomain(self):
repo = common_utils.get_git_repo(self.pod.root)
if self.config.subdomain_prefix and not self.config.subdomain:
token = repo.active_branch.name.split('/')[-1]
if token == 'master':
return self.config.subdomain_prefix
else:
return self.config.subdomain_prefix + '-{}'.format(token)
return self.config.subdomain
def build(pod_path, out_dir, preprocess, clear_cache, pod_paths,
locate_untranslated, deployment, threaded, locale):
"""Generates static files and dumps them to a local destination."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
out_dir = out_dir or os.path.join(root, 'build')
pod = pods.Pod(root, storage=storage.FileStorage)
if not pod_paths or clear_cache:
# Clear the cache when building all, only force if the flag is used.
pod.podcache.reset(force=clear_cache)
deployment_obj = None
if deployment:
deployment_obj = pod.get_deployment(deployment)
pod.set_env(deployment_obj.config.env)
if preprocess:
with pod.profile.timer('grow_preprocess'):
pod.preprocess()
if locate_untranslated:
pod.enable(pod.FEATURE_TRANSLATION_STATS)
try:
with pod.profile.timer('grow_build'):
config = local_destination.Config(out_dir=out_dir)
# When using a specific deployment env need to also copy over.
if deployment_obj:
config.env = deployment_obj.config.env
destination = local_destination.LocalDestination(config)
destination.pod = pod
repo = utils.get_git_repo(pod.root)
pod.router.use_simple()
if pod_paths:
pod.router.add_pod_paths(pod_paths)
else:
pod.router.add_all()
if locale:
pod.router.filter(locales=list(locale))
paths = pod.router.routes.paths
content_generator = renderer.Renderer.rendered_docs(
pod, pod.router.routes, use_threading=threaded)
stats_obj = stats.Stats(pod, paths=paths)
is_partial = bool(pod_paths) or bool(locale)
destination.deploy(content_generator,
stats=stats_obj,
repo=repo,
confirm=False,
test=False,
is_partial=is_partial)
pod.podcache.write()
except renderer.RenderErrors as err:
# Ignore the build error since it outputs the errors.
raise click.ClickException(str(err))
except pods.Error as err:
raise click.ClickException(str(err))
if locate_untranslated:
pod.translation_stats.pretty_print()
destination.export_untranslated_catalogs()
return pod
def has_gerrit_remote(pod):
KNOWN_GERRIT_HOSTS = ('googlesource.com', )
repo = utils.get_git_repo(pod.root)
if repo is None:
return False
for remote in repo.remotes:
url = remote.config_reader.get('url')
result = urlparse.urlparse(url)
if result.netloc.endswith(KNOWN_GERRIT_HOSTS):
return True
def has_gerrit_remote(pod):
KNOWN_GERRIT_HOSTS = (
'googlesource.com',
)
repo = utils.get_git_repo(pod.root)
if repo is None:
return False
for remote in repo.remotes:
url = remote.config_reader.get('url')
result = urlparse.urlparse(url)
if result.netloc.endswith(KNOWN_GERRIT_HOSTS):
return True
def build(pod_path, out_dir, preprocess, clear_cache, pod_paths,
locate_untranslated, deployment, use_reroute):
"""Generates static files and dumps them to a local destination."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
out_dir = out_dir or os.path.join(root, 'build')
pod = pods.Pod(root, storage=storage.FileStorage, use_reroute=use_reroute)
if not pod_paths or clear_cache:
# Clear the cache when building all, only force if the flag is used.
pod.podcache.reset(force=clear_cache)
if deployment:
deployment_obj = pod.get_deployment(deployment)
pod.set_env(deployment_obj.config.env)
if preprocess:
with pod.profile.timer('grow_preprocess'):
pod.preprocess()
if locate_untranslated:
pod.enable(pod.FEATURE_TRANSLATION_STATS)
try:
with pod.profile.timer('grow_build'):
config = local_destination.Config(out_dir=out_dir)
destination = local_destination.LocalDestination(config)
destination.pod = pod
repo = utils.get_git_repo(pod.root)
if use_reroute:
pod.router.use_simple()
if pod_paths:
pod.router.add_pod_paths(pod_paths)
else:
pod.router.add_all()
routes = pod.router.routes
stats_obj = stats.Stats(pod, paths=routes.paths)
rendered_docs = renderer.Renderer.rendered_docs(pod, routes)
destination.deploy(
rendered_docs, stats=stats_obj, repo=repo,
confirm=False, test=False, is_partial=bool(pod_paths))
else:
paths, _ = pod.determine_paths_to_build(pod_paths=pod_paths)
stats_obj = stats.Stats(pod, paths=paths)
content_generator = destination.dump(pod, pod_paths=pod_paths)
destination.deploy(
content_generator, stats=stats_obj, repo=repo, confirm=False,
test=False, is_partial=bool(pod_paths))
pod.podcache.write()
except pods.Error as err:
raise click.ClickException(str(err))
if locate_untranslated:
pod.translation_stats.pretty_print()
destination.export_untranslated_catalogs()
return pod
def should_run(self):
"""Should the installer run?"""
gerrit_setting = self.config.get('gerrit', None)
if gerrit_setting is not None:
return gerrit_setting
repo = utils.get_git_repo(self.pod.root)
if repo is None:
return False
for remote in repo.remotes:
url = remote.config_reader.get('url')
result = urllib.parse.urlparse(url)
if result.netloc.endswith(KNOWN_GERRIT_HOSTS):
return True
return False
def should_run(self):
"""Should the installer run?"""
gerrit_setting = self.config.get('gerrit', None)
if gerrit_setting is not None:
return gerrit_setting
repo = utils.get_git_repo(self.pod.root)
if repo is None:
return False
for remote in repo.remotes:
url = remote.config_reader.get('url')
result = urlparse.urlparse(url)
if result.netloc.endswith(KNOWN_GERRIT_HOSTS):
return True
return False
def build(pod_path, out_dir):
"""Generates static files and dumps them to a local destination."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
out_dir = out_dir or os.path.join(root, 'build')
pod = pods.Pod(root, storage=storage.FileStorage)
pod.preprocess()
try:
config = local_destination.Config(out_dir=out_dir)
destination = local_destination.LocalDestination(config)
paths_to_contents = destination.dump(pod)
repo = utils.get_git_repo(pod.root)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
destination.deploy(paths_to_contents, stats=stats_obj, repo=repo, confirm=False,
test=False)
except pods.Error as e:
raise click.ClickException(str(e))
def deploy(context, deployment_name, pod_path, preprocess, confirm, test,
test_only, auth, force_untranslated, use_reroute, threaded):
"""Deploys a pod to a destination."""
if auth:
text = ('--auth must now be specified before deploy. Usage:'
' grow [email protected] deploy')
raise click.ClickException(text)
auth = context.parent.params.get('auth')
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
try:
pod = pods.Pod(root, storage=storage.FileStorage, use_reroute=use_reroute)
with pod.profile.timer('grow_deploy'):
# Always clear the cache when building.
pod.podcache.reset()
deployment = pod.get_deployment(deployment_name)
# use the deployment's environment for preprocessing and later
# steps.
if deployment.config.env:
pod.set_env(deployment.config.env)
require_translations = pod.podspec.localization.get(
'require_translations', False)
require_translations = require_translations and not force_untranslated
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
if test_only:
deployment.test()
return
content_generator = deployment.dump(pod, use_threading=threaded)
repo = utils.get_git_repo(pod.root)
if use_reroute:
pod.router.use_simple()
pod.router.add_all()
paths = pod.router.routes.paths
else:
paths, _ = pod.determine_paths_to_build()
stats_obj = stats.Stats(pod, paths=paths)
deployment.deploy(
content_generator, stats=stats_obj, repo=repo, confirm=confirm,
test=test, require_translations=require_translations)
pod.podcache.write()
except base.Error as err:
raise click.ClickException(str(err))
except pods.Error as err:
raise click.ClickException(str(err))
return pod
def _get_branch_token(self):
# Try getting the branch name from CI environments.
# https://github.com/grow/grow/issues/903
branch_name = os.getenv('BRANCH_NAME') or os.getenv('CIRCLE_BRANCH')
if branch_name:
return branch_name.split('/')[-1]
repo = common_utils.get_git_repo(self.pod.root)
try:
return repo.active_branch.name.split('/')[-1]
except TypeError as e:
# Permit staging from detached branches. Note this will clobber
# other stages originating from detaching the same branch.
if 'is a detached symbolic reference' not in str(e):
raise
# Extract the sha from the error message.
sha = str(e).split(' ')[-1].replace("'", '')
return '{}-d'.format(sha[:10])
def _get_branch_token(self):
# Try getting the branch name from CI environments.
# https://github.com/grow/grow/issues/903
branch_name = os.getenv('BRANCH_NAME') or os.getenv('CIRCLE_BRANCH')
if branch_name:
return branch_name.split('/')[-1]
repo = common_utils.get_git_repo(self.pod.root)
try:
return repo.active_branch.name.split('/')[-1]
except TypeError as e:
# Permit staging from detached branches. Note this will clobber
# other stages originating from detaching the same branch.
if 'is a detached symbolic reference' not in str(e):
raise
# Extract the sha from the error message.
sha = str(e).split(' ')[-1].replace("'", '')
return '{}-d'.format(sha[:10])
def _get_subdomain(self):
if self.config.subdomain_prefix and not self.config.subdomain:
repo = common_utils.get_git_repo(self.pod.root)
try:
token = repo.active_branch.name.split('/')[-1]
except TypeError as e:
# Permit staging from detached branches. Note this will clobber
# other stages originating from detaching the same branch.
if 'is a detached symbolic reference' not in str(e):
raise
# Extract the sha from the error message.
sha = str(e).split(' ')[-1].replace("'", '')
token = '{}-d'.format(sha[:10])
if token == 'master':
return self.config.subdomain_prefix
else:
return self.config.subdomain_prefix + '-{}'.format(token)
return self.config.subdomain
def _get_subdomain(self):
if self.config.subdomain_prefix and not self.config.subdomain:
repo = common_utils.get_git_repo(self.pod.root)
try:
token = repo.active_branch.name.split('/')[-1]
except TypeError as e:
# Permit staging from detached branches. Note this will clobber
# other stages originating from detaching the same branch.
if 'is a detached symbolic reference' not in str(e):
raise
# Extract the sha from the error message.
sha = str(e).split(' ')[-1].replace("'", '')
token = '{}-d'.format(sha[:10])
if token == 'master':
return self.config.subdomain_prefix
else:
return self.config.subdomain_prefix + '-{}'.format(token)
return self.config.subdomain
def build(pod_path, out_dir):
"""Generates static files and dumps them to a local destination."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
out_dir = out_dir or os.path.join(root, 'build')
pod = pods.Pod(root, storage=storage.FileStorage)
pod.preprocess()
try:
paths_to_contents = pod.dump()
repo = utils.get_git_repo(pod.root)
config = local_destination.Config(out_dir=out_dir)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
destination = local_destination.LocalDestination(config)
destination.deploy(paths_to_contents,
stats=stats_obj,
repo=repo,
confirm=False,
test=False)
except pods.Error as e:
raise click.ClickException(str(e))
def stage(pod_path, remote, auth, preprocess):
"""Stages a build on a WebReview server."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
try:
pod = pods.Pod(root, storage=storage.FileStorage)
dest_class = webreview_destination.WebReviewDestination
deployment = dest_class(dest_class.Config(remote=remote))
if auth:
deployment.login(auth)
pod.preprocess()
repo = utils.get_git_repo(pod.root)
paths_to_contents = deployment.dump(pod)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
deployment.deploy(paths_to_contents, stats=stats_obj, repo=repo,
confirm=False, test=False)
except base.Error as e:
raise click.ClickException(str(e))
except pods.Error as e:
raise click.ClickException(str(e))
def stage(context, pod_path, remote, preprocess, subdomain, api_key):
"""Stages a build on a WebReview server."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
auth = context.parent.params.get('auth')
try:
pod = pods.Pod(root, storage=storage.FileStorage)
deployment = _get_deployment(pod, remote, subdomain, api_key)
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
repo = utils.get_git_repo(pod.root)
paths_to_contents = deployment.dump(pod)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
deployment.deploy(paths_to_contents, stats=stats_obj, repo=repo,
confirm=False, test=False)
except base.Error as e:
raise click.ClickException(str(e))
except pods.Error as e:
raise click.ClickException(str(e))
def stage(context, pod_path, remote, preprocess, subdomain):
"""Stages a build on a WebReview server."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
auth = context.parent.params.get('auth')
try:
pod = pods.Pod(root, storage=storage.FileStorage)
deployment = _get_deployment(pod, remote, subdomain)
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
repo = utils.get_git_repo(pod.root)
paths_to_contents = deployment.dump(pod)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
deployment.deploy(paths_to_contents, stats=stats_obj, repo=repo,
confirm=False, test=False)
except base.Error as e:
raise click.ClickException(str(e))
except pods.Error as e:
raise click.ClickException(str(e))
def stage(context, pod_path, remote, preprocess, subdomain, api_key,
force_untranslated, threaded):
"""Stages a build on a WebReview server."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
auth = context.parent.params.get('auth')
try:
pod = pods.Pod(root, storage=storage.FileStorage)
with pod.profile.timer('grow_stage'):
deployment = _get_deployment(pod, remote, subdomain, api_key)
# use the deployment's environment for preprocessing and later
# steps.
pod.set_env(deployment.get_env())
# Always clear the cache when building.
pod.podcache.reset()
require_translations = \
pod.podspec.localization \
and pod.podspec.localization.get('require_translations', False)
require_translations = require_translations \
and not force_untranslated
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
content_generator = deployment.dump(pod, use_threading=threaded)
repo = utils.get_git_repo(pod.root)
pod.router.use_simple()
pod.router.add_all()
paths = pod.router.routes.paths
stats_obj = stats.Stats(pod, paths=paths)
deployment.deploy(content_generator,
stats=stats_obj,
repo=repo,
confirm=False,
test=False,
require_translations=require_translations)
pod.podcache.write()
except base.Error as err:
raise click.ClickException(str(err))
except pods.Error as err:
raise click.ClickException(str(err))
return pod
def deploy(context, deployment_name, pod_path, preprocess, confirm, test,
test_only, auth, force_untranslated):
"""Deploys a pod to a destination."""
if auth:
text = ('--auth must now be specified before deploy. Usage:'
' grow [email protected] deploy')
raise click.ClickException(text)
auth = context.parent.params.get('auth')
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
try:
pod = pods.Pod(root, storage=storage.FileStorage)
deployment = pod.get_deployment(deployment_name)
# use the deployment's environment for preprocessing and later steps.
pod.set_env(deployment.config.env)
require_translations = pod.podspec.localization.get(
'require_translations', False)
require_translations = require_translations and not force_untranslated
if require_translations:
pod.enable(pod.FEATURE_TRANSLATION_STATS)
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
if test_only:
deployment.test()
return
paths_to_contents = deployment.dump(pod)
if require_translations and pod.translation_stats.untranslated:
pod.translation_stats.pretty_print()
raise pods.Error('Aborted deploy due to untranslated strings. '
'Use the --force-untranslated flag to force deployment.')
repo = utils.get_git_repo(pod.root)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
deployment.deploy(paths_to_contents, stats=stats_obj, repo=repo,
confirm=confirm, test=test)
except base.Error as e:
raise click.ClickException(str(e))
except pods.Error as e:
raise click.ClickException(str(e))
def deploy(deployment_name, pod_path, preprocess, confirm, test, test_only, auth):
"""Deploys a pod to a destination."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
try:
pod = pods.Pod(root, storage=storage.FileStorage)
deployment = pod.get_deployment(deployment_name)
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
if test_only:
deployment.test()
return
paths_to_contents = deployment.dump(pod)
repo = utils.get_git_repo(pod.root)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
deployment.deploy(paths_to_contents, stats=stats_obj, repo=repo,
confirm=confirm, test=test)
except base.Error as e:
raise click.ClickException(str(e))
except pods.Error as e:
raise click.ClickException(str(e))
def build(pod_path, out_dir, preprocess, clear_cache, pod_paths, locate_untranslated):
"""Generates static files and dumps them to a local destination."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
out_dir = out_dir or os.path.join(root, 'build')
pod = pods.Pod(root, storage=storage.FileStorage)
if clear_cache:
pod.podcache.reset(force=True)
if preprocess:
pod.preprocess()
if locate_untranslated:
pod.enable(pod.FEATURE_TRANSLATION_STATS)
try:
config = local_destination.Config(out_dir=out_dir)
destination = local_destination.LocalDestination(config)
paths_to_contents = destination.dump(pod, pod_paths=pod_paths)
repo = utils.get_git_repo(pod.root)
stats_obj = stats.Stats(pod, paths_to_contents=paths_to_contents)
destination.deploy(paths_to_contents, stats=stats_obj, repo=repo, confirm=False,
test=False, is_partial=bool(pod_paths))
pod.podcache.write()
except pods.Error as e:
raise click.ClickException(str(e))
if locate_untranslated:
pod.translation_stats.pretty_print()
def build(pod_path, out_dir, preprocess, clear_cache, pod_paths,
locate_untranslated, deployment, threaded, locale, shards, shard,
work_dir, routes_file):
"""Generates static files and dumps them to a local destination."""
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
out_dir = out_dir or os.path.join(root, 'build')
pod = pods.Pod(root, storage=storage.FileStorage)
if not pod_paths or clear_cache:
# Clear the cache when building all, only force if the flag is used.
pod.podcache.reset(force=clear_cache)
deployment_obj = None
if deployment:
deployment_obj = pod.get_deployment(deployment)
pod.set_env(deployment_obj.config.env)
if preprocess:
with pod.profile.timer('grow_preprocess'):
pod.preprocess()
if locate_untranslated:
pod.enable(pod.FEATURE_TRANSLATION_STATS)
try:
with pod.profile.timer('grow_build'):
config = local_destination.Config(out_dir=out_dir)
# When using a specific deployment env need to also copy over.
if deployment_obj:
config.env = deployment_obj.config.env
destination = local_destination.LocalDestination(config)
destination.pod = pod
repo = utils.get_git_repo(pod.root)
pod.router.use_simple()
is_partial = bool(pod_paths) or bool(locale)
if pod_paths:
pod_paths = [pod.clean_pod_path(path) for path in pod_paths]
pod.router.add_pod_paths(pod_paths)
elif routes_file:
pod.router.from_data(pod.read_json(routes_file))
else:
pod.router.add_all()
if locale:
pod.router.filter('whitelist', locales=list(locale))
# Shard the routes when using sharding.
if shards and shard:
is_partial = True
pod.router.shard(shards, shard)
if not work_dir:
# Preload the documents used by the paths after filtering.
docs_loader.DocsLoader.load_from_routes(pod, pod.router.routes)
paths = pod.router.routes.paths
content_generator = renderer.Renderer.rendered_docs(
pod,
pod.router.routes,
source_dir=work_dir,
use_threading=threaded)
content_generator = hooks.generator_wrapper(
pod, 'pre_deploy', content_generator, 'build')
stats_obj = stats.Stats(pod, paths=paths)
destination.deploy(content_generator,
stats=stats_obj,
repo=repo,
confirm=False,
test=False,
is_partial=is_partial)
pod.podcache.write()
except bulk_errors.BulkErrors as err:
# Write the podcache files even when there are rendering errors.
pod.podcache.write()
bulk_errors.display_bulk_errors(err)
raise click.Abort()
except pods.Error as err:
raise click.ClickException(str(err))
if locate_untranslated:
pod.translation_stats.pretty_print()
destination.export_untranslated_catalogs()
return pod
def get_repo(self):
remote_name = self._get_param('remote') or 'origin'
repo = utils.get_git_repo(self.pod.root)
if not repo:
return {}
branch = str(repo.active_branch)
revision = repo.git.rev_list('--count', 'HEAD')
remote_url = None
web_url = None
remote = None
for repo_remote in repo.remotes:
if repo_remote.name == remote_name:
remote = repo_remote
break
if remote:
urls = list(remote.urls)
remote_url = urls and urls[0]
web_url = remote_url
# Github
web_url = web_url.replace('[email protected]:',
'https://www.github.com/')
if web_url and web_url.endswith('.git'):
web_url = web_url[:-4]
# Google Cloud Source
web_url = web_url.replace(
'https://source.developers.google.com/p/',
'https://source.cloud.google.com/')
web_url = web_url.replace('/r/', '/')
if not remote:
remote = repo.remote()
# Remove the remote name from branch name (ex: origin/master => master)
branch_names = [
'/'.join(ref.name.split('/')[1:]) for ref in remote.refs
]
branches = [
name for name in branch_names if name not in self.IGNORED_BRANCHES
]
commits = []
if repo.head.ref: # Handle repo with no commits.
for commit in repo.iter_commits(branch, max_count=10):
committed_date = datetime.datetime.utcfromtimestamp(
commit.committed_date)
commits.append({
'message': commit.message,
'commit_date': committed_date.isoformat(),
'sha': commit.hexsha,
'author': {
'name': commit.author.name,
'email': commit.author.email,
},
})
self.data = {
'repo': {
'web_url': web_url,
'remote_url': remote_url,
'revision': revision,
'branch': branch,
'branches': branches,
'commits': commits,
},
}
def deploy(context, deployment_name, pod_path, preprocess, confirm, test,
test_only, auth, force_untranslated, threaded, shards, shard,
work_dir):
"""Deploys a pod to a destination."""
if auth:
text = ('--auth must now be specified before deploy. Usage:'
' grow [email protected] deploy')
raise click.ClickException(text)
auth = context.parent.params.get('auth')
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
try:
pod = pods.Pod(root, storage=storage.FileStorage)
with pod.profile.timer('grow_deploy'):
# Always clear the cache when building.
pod.podcache.reset()
deployment = pod.get_deployment(deployment_name)
# use the deployment's environment for preprocessing and later
# steps.
if deployment.config.env:
pod.set_env(deployment.config.env)
require_translations = pod.podspec.localization.get(
'require_translations', False)
require_translations = require_translations and not force_untranslated
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
if test_only:
deployment.test()
return
content_generator = deployment.dump(
pod, source_dir=work_dir, use_threading=threaded)
repo = utils.get_git_repo(pod.root)
pod.router.use_simple()
pod.router.add_all()
is_partial = False
# Filter routes based on deployment config.
for build_filter in deployment.filters:
is_partial = True
pod.router.filter(
build_filter.type, collection_paths=build_filter.collections,
paths=build_filter.paths, locales=build_filter.locales)
# Shard the routes when using sharding.
if shards and shard:
is_partial = True
pod.router.shard(shards, shard)
paths = pod.router.routes.paths
stats_obj = stats.Stats(pod, paths=paths)
deployment.deploy(
content_generator, stats=stats_obj, repo=repo, confirm=confirm,
test=test, require_translations=require_translations,
is_partial=is_partial)
pod.podcache.write()
except renderer.RenderErrors as err:
# Write the podcache files even when there are rendering errors.
pod.podcache.write()
# Ignore the build error since it outputs the errors.
raise click.ClickException(str(err))
except base.Error as err:
raise click.ClickException(str(err))
except pods.Error as err:
raise click.ClickException(str(err))
return pod
def deploy(context, deployment_name, pod_path, preprocess, confirm, test,
test_only, auth, force_untranslated, threaded, shards, shard,
work_dir, routes_file):
"""Deploys a pod to a destination."""
if auth:
text = ('--auth must now be specified before deploy. Usage:'
' grow [email protected] deploy')
raise click.ClickException(text)
auth = context.parent.params.get('auth')
root = os.path.abspath(os.path.join(os.getcwd(), pod_path))
try:
pod = pods.Pod(root, storage=storage.FileStorage)
with pod.profile.timer('grow_deploy'):
# Always clear the cache when building.
pod.podcache.reset()
deployment = pod.get_deployment(deployment_name)
# use the deployment's environment for preprocessing and later
# steps.
if deployment.config.env:
pod.set_env(deployment.config.env)
require_translations = pod.podspec.localization.get(
'require_translations', False)
require_translations = require_translations and not force_untranslated
if auth:
deployment.login(auth)
if preprocess:
pod.preprocess()
if test_only:
deployment.test()
return
repo = utils.get_git_repo(pod.root)
pod.router.use_simple()
if routes_file:
pod.router.from_data(pod.read_json(routes_file))
else:
pod.router.add_all()
is_partial = False
# Filter routes based on deployment config.
for build_filter in deployment.filters:
is_partial = True
pod.router.filter(
build_filter.type, collection_paths=build_filter.collections,
paths=build_filter.paths, locales=build_filter.locales)
# Shard the routes when using sharding.
if shards and shard:
is_partial = True
pod.router.shard(shards, shard)
if not work_dir:
# Preload the documents used by the paths after filtering.
docs_loader.DocsLoader.load_from_routes(pod, pod.router.routes)
paths = pod.router.routes.paths
stats_obj = stats.Stats(pod, paths=paths)
content_generator = deployment.dump(
pod, source_dir=work_dir, use_threading=threaded)
content_generator = hooks.generator_wrapper(
pod, 'pre_deploy', content_generator, 'deploy')
deployment.deploy(
content_generator, stats=stats_obj, repo=repo, confirm=confirm,
test=test, require_translations=require_translations,
is_partial=is_partial)
pod.podcache.write()
except bulk_errors.BulkErrors as err:
# Write the podcache files even when there are rendering errors.
pod.podcache.write()
bulk_errors.display_bulk_errors(err)
raise click.Abort()
except base.Error as err:
raise click.ClickException(str(err))
except pods.Error as err:
raise click.ClickException(str(err))
return pod
