def BuildAndPushDockerImage(
project,
service,
source_dir,
version_id,
code_bucket_ref,
gcr_domain,
runtime_builder_strategy=runtime_builders.RuntimeBuilderStrategy.NEVER,
parallel_build=False):
"""Builds and pushes a set of docker images.
Args:
project: str, The project being deployed to.
service: ServiceYamlInfo, The parsed service config.
source_dir: str, path to the service's source directory
version_id: The version id to deploy these services under.
code_bucket_ref: The reference to the GCS bucket where the source will be
uploaded.
gcr_domain: str, Cloud Registry domain, determines the physical location
of the image. E.g. `us.gcr.io`.
runtime_builder_strategy: runtime_builders.RuntimeBuilderStrategy, whether
to use the new CloudBuild-based runtime builders (alternative is old
externalized runtimes).
parallel_build: bool, if True, enable parallel build and deploy.
Returns:
BuildArtifact, Representing the pushed container image or in-progress build.
Raises:
DockerfileError: if a Dockerfile is present, but the runtime is not
"custom".
NoDockerfileError: Raised if a user didn't supply a Dockerfile and chose a
custom runtime.
UnsatisfiedRequirementsError: Raised if the code in the directory doesn't
satisfy the requirements of the specified runtime type.
ValueError: if an unrecognized runtime_builder_strategy is given
"""
needs_dockerfile = _NeedsDockerfile(service, source_dir)
use_runtime_builders = ShouldUseRuntimeBuilders(service,
runtime_builder_strategy,
needs_dockerfile)
# Nothing to do if this is not an image-based deployment.
if not service.RequiresImage():
return None
log.status.Print(
'Building and pushing image for service [{service}]'.format(
service=service.module))
gen_files = dict(_GetSourceContextsForUpload(source_dir))
if needs_dockerfile and not use_runtime_builders:
# The runtime builders will generate a Dockerfile in the Cloud, so we only
# need to do this if use_runtime_builders is True
gen_files.update(_GetDockerfiles(service, source_dir))
image = docker_image.Image(dockerfile_dir=source_dir,
repo=_GetImageName(project, service.module,
version_id, gcr_domain),
nocache=False,
tag=config.DOCKER_IMAGE_TAG)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD_START)
object_ref = storage_util.ObjectReference(code_bucket_ref,
image.tagged_repo)
relative_yaml_path = _GetYamlPath(source_dir, service.file,
service.parsed.skip_files, gen_files)
try:
cloud_build.UploadSource(image.dockerfile_dir,
object_ref,
gen_files=gen_files,
skip_files=service.parsed.skip_files.regex)
except (OSError, IOError) as err:
if platforms.OperatingSystem.IsWindows():
if err.filename and len(err.filename) > _WINDOWS_MAX_PATH:
raise WindowMaxPathError(err.filename)
raise
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD)
if use_runtime_builders:
builder_reference = runtime_builders.FromServiceInfo(
service, source_dir)
log.info('Using runtime builder [%s]',
builder_reference.build_file_uri)
builder_reference.WarnIfDeprecated()
yaml_path = util.ConvertToPosixPath(relative_yaml_path)
build = builder_reference.LoadCloudBuild({
'_OUTPUT_IMAGE':
image.tagged_repo,
'_GAE_APPLICATION_YAML_PATH':
yaml_path
})
else:
build = cloud_build.GetDefaultBuild(image.tagged_repo)
build = cloud_build.FixUpBuild(build, object_ref)
return _SubmitBuild(build, image, project, parallel_build)
def CopyFilesToCodeBucket(service, source_dir, bucket_ref,
upload_strategy):
"""Copies application files to the Google Cloud Storage code bucket.
Uses either gsutil, the Cloud Storage API using processes, or the Cloud
Storage API using threads.
Consider the following original structure:
app/
main.py
tools/
foo.py
Assume main.py has SHA1 hash 123 and foo.py has SHA1 hash 456. The resultant
GCS bucket will look like this:
gs://$BUCKET/
123
456
The resulting App Engine API manifest will be:
{
"app/main.py": {
"sourceUrl": "https://storage.googleapis.com/staging-bucket/123",
"sha1Sum": "123"
},
"app/tools/foo.py": {
"sourceUrl": "https://storage.googleapis.com/staging-bucket/456",
"sha1Sum": "456"
}
}
A 'list' call of the bucket is made at the start, and files that hash to
values already present in the bucket will not be uploaded again.
Args:
service: ServiceYamlInfo, The service being deployed.
source_dir: str, path to the service's source directory
bucket_ref: The reference to the bucket files will be placed in.
upload_strategy: The UploadStrategy to use
Returns:
A dictionary representing the manifest.
Raises:
ValueError: if an invalid upload strategy or None is given
"""
if upload_strategy is UploadStrategy.GSUTIL:
manifest = CopyFilesToCodeBucketGsutil(service, source_dir, bucket_ref)
metrics.CustomTimedEvent(metric_names.COPY_APP_FILES)
else:
# Collect a list of files to upload, indexed by the SHA so uploads are
# deduplicated.
with file_utils.TemporaryDirectory() as tmp_dir:
manifest = _BuildDeploymentManifest(service, source_dir, bucket_ref,
tmp_dir)
files_to_upload = _BuildFileUploadMap(
manifest, source_dir, bucket_ref, tmp_dir)
if upload_strategy is UploadStrategy.THREADS:
_UploadFilesThreads(files_to_upload, bucket_ref)
elif upload_strategy is UploadStrategy.PROCESSES:
_UploadFilesProcesses(files_to_upload, bucket_ref)
else:
raise ValueError('Invalid upload strategy ' + str(upload_strategy))
log.status.Print('File upload done.')
log.info('Manifest: [{0}]'.format(manifest))
metrics.CustomTimedEvent(metric_names.COPY_APP_FILES)
return manifest
def BuildAndPushDockerImages(module_configs, version_id, cloudbuild_client,
storage_client, http, code_bucket_ref, cli,
remote, source_contexts, config_cleanup):
"""Builds and pushes a set of docker images.
Args:
module_configs: A map of module name to parsed config.
version_id: The version id to deploy these modules under.
cloudbuild_client: An instance of the cloudbuild.CloudBuildV1 api client.
storage_client: An instance of the storage_v1.StorageV1 client.
http: a http provider that can be used to create API clients
code_bucket_ref: The reference to the GCS bucket where the source will be
uploaded.
cli: calliope.cli.CLI, The CLI object representing this command line tool.
remote: Whether the user specified a remote build.
source_contexts: A list of json-serializable source contexts to place in
the application directory for each config.
config_cleanup: (callable() or None) If a temporary Dockerfile has already
been created during the course of the deployment, this should be a
callable that deletes it.
Returns:
A dictionary mapping modules to the name of the pushed container image.
"""
project = properties.VALUES.core.project.Get(required=True)
use_cloud_build = properties.VALUES.app.use_cloud_build.GetBool()
# Prepare temporary dockerfile creators for all modules that need them
# before doing the heavy lifting so we can fail fast if there are errors.
modules = []
for (name, info) in module_configs.iteritems():
if info.RequiresImage():
context_creator = context_util.GetSourceContextFilesCreator(
os.path.dirname(info.file), source_contexts)
modules.append(
(name, info, _GetDockerfileCreator(info, config_cleanup),
context_creator))
if not modules:
# No images need to be built.
return {}
log.status.Print('Verifying that Managed VMs are enabled and ready.')
if use_cloud_build:
return _BuildImagesWithCloudBuild(project, modules, version_id,
code_bucket_ref, cloudbuild_client,
storage_client, http)
# Update docker client's credentials.
for registry_host in constants.ALL_SUPPORTED_REGISTRIES:
docker.UpdateDockerCredentials(registry_host)
metrics.CustomTimedEvent(metric_names.DOCKER_UPDATE_CREDENTIALS)
# Build docker images.
images = {}
with docker_util.DockerHost(cli, version_id, remote,
project) as docker_client:
# Build and push all images.
for module, info, ensure_dockerfile, ensure_context in modules:
log.status.Print(
'Building and pushing image for module [{module}]'.format(
module=module))
cleanup_dockerfile = ensure_dockerfile()
cleanup_context = ensure_context()
try:
image_name = _GetImageName(project, module, version_id)
images[module] = BuildAndPushDockerImage(
info.file, docker_client, image_name)
finally:
cleanup_dockerfile()
cleanup_context()
metric_name = (metric_names.DOCKER_REMOTE_BUILD
if remote else metric_names.DOCKER_BUILD)
metrics.CustomTimedEvent(metric_name)
return images
def Run(self, args):
project = properties.VALUES.core.project.Get(required=True)
version = args.version or util.GenerateVersionId()
use_cloud_build = properties.VALUES.app.use_cloud_build.GetBool()
config_cleanup = None
if args.deployables:
app_config = yaml_parsing.AppConfigSet(args.deployables)
else:
if not os.path.exists(DEFAULT_DEPLOYABLE):
console_io.PromptContinue(
'Deployment to Google App Engine requires an app.yaml file. '
'This command will run `gcloud preview app gen-config` to generate '
'an app.yaml file for you in the current directory (if the current '
'directory does not contain an App Engine service, please answer '
'"no").',
cancel_on_no=True)
# This generates the app.yaml AND the Dockerfile (and related files).
params = ext_runtime.Params(deploy=True)
configurator = fingerprinter.IdentifyDirectory(os.getcwd(),
params=params)
if configurator is None:
raise NoAppIdentifiedError(
'Could not identify an app in the current directory.\n\n'
'Please prepare an app.yaml file for your application manually '
'and deploy again.')
config_cleanup = configurator.GenerateConfigs()
log.status.Print(
'\nCreated [{0}] in the current directory.\n'.format(
DEFAULT_DEPLOYABLE))
app_config = yaml_parsing.AppConfigSet([DEFAULT_DEPLOYABLE])
# If the app has enabled Endpoints API Management features, pass
# control to the cloud_endpoints handler.
service_management_util.ProcessEndpointsServices(
app_config.Services().items(), project)
remote_build = True
docker_build_property = properties.VALUES.app.docker_build.Get()
if args.docker_build:
remote_build = args.docker_build == 'remote'
elif docker_build_property:
remote_build = docker_build_property == 'remote'
clients = _AppEngineClients(
appengine_client.AppengineClient(args.server,
args.ignore_bad_certs),
appengine_api_client.GetApiClient())
# pylint: disable=protected-access
log.debug(
'API endpoint: [{endpoint}], API version: [{version}]'.format(
endpoint=clients.api.client.url,
version=clients.api.client._VERSION))
cloudbuild_client = apis.GetClientInstance('cloudbuild', 'v1')
storage_client = apis.GetClientInstance('storage', 'v1')
promote = properties.VALUES.app.promote_by_default.GetBool()
deployed_urls = _DisplayProposedDeployment(project, app_config,
version, promote)
if args.version or promote:
# Prompt if there's a chance that you're overwriting something important:
# If the version is set manually, you could be deploying over something.
# If you're setting the new deployment to be the default version, you're
# changing the target of the default URL.
# Otherwise, all existing URLs will continue to work, so need to prompt.
console_io.PromptContinue(default=True,
throw_if_unattended=False,
cancel_on_no=True)
log.status.Print('Beginning deployment...')
source_contexts = []
if args.repo_info_file:
if args.image_url:
raise NoRepoInfoWithImageUrlError()
try:
with open(args.repo_info_file, 'r') as f:
source_contexts = json.load(f)
except (ValueError, IOError) as ex:
raise RepoInfoLoadError(args.repo_info_file, ex)
if isinstance(source_contexts, dict):
# This is an old-style source-context.json file. Convert to a new-
# style array of extended contexts.
source_contexts = [
context_util.ExtendContextDict(source_contexts)
]
services = app_config.Services()
code_bucket_ref = None
if services and (use_cloud_build or app_config.NonHermeticServices()):
# If using Argo CloudBuild, we'll need to upload source to a GCS bucket.
code_bucket_ref = self._GetCodeBucket(clients.api, args)
metrics.CustomTimedEvent(metric_names.GET_CODE_BUCKET)
log.debug('Using bucket [{b}].'.format(b=code_bucket_ref))
if any([m.RequiresImage() for m in services.values()]):
deploy_command_util.DoPrepareManagedVms(clients.gae)
if args.image_url:
if len(services) != 1:
raise MultiDeployError()
for registry in constants.ALL_SUPPORTED_REGISTRIES:
if args.image_url.startswith(registry):
break
else:
raise UnsupportedRegistryError(args.image_url)
service = services.keys()[0]
images = {service: args.image_url}
else:
images = deploy_command_util.BuildAndPushDockerImages(
services, version, cloudbuild_client, storage_client,
code_bucket_ref, self.cli, remote_build, source_contexts,
config_cleanup)
deployment_manifests = {}
if app_config.NonHermeticServices():
if properties.VALUES.app.use_gsutil.GetBool():
copy_func = deploy_app_command_util.CopyFilesToCodeBucket
metric_name = metric_names.COPY_APP_FILES
else:
copy_func = deploy_app_command_util.CopyFilesToCodeBucketNoGsUtil
metric_name = metric_names.COPY_APP_FILES_NO_GSUTIL
deployment_manifests = copy_func(
app_config.NonHermeticServices().items(), code_bucket_ref,
source_contexts, storage_client)
metrics.CustomTimedEvent(metric_name)
new_versions = []
if services:
# We don't necessarily have permission to list services if we only want to
# update config files (below).
all_services = clients.api.ListServices()
# Now do deployment.
for (service, info) in services.iteritems():
message = 'Updating service [{service}]'.format(
service=service)
new_version = version_util.Version(project, service, version)
with console_io.ProgressTracker(message):
if args.force:
log.warning(
'The --force argument is deprecated and no longer '
'required. It will be removed in a future release.'
)
clients.api.DeployService(
service, version, info,
deployment_manifests.get(service), images.get(service))
metrics.CustomTimedEvent(metric_names.DEPLOY_API)
stop_previous_version = (
deploy_command_util.GetStopPreviousVersionFromArgs(
args))
if promote:
_Promote(all_services, new_version, clients,
stop_previous_version)
elif stop_previous_version:
log.info(
'Not stopping previous version because new version was '
'not promoted.')
log.status.Print('Deployed service [{0}] to [{1}]'.format(
service, deployed_urls[service]))
new_versions.append(new_version)
# Config files.
for (c, info) in app_config.Configs().iteritems():
message = 'Updating config [{config}]'.format(config=c)
with console_io.ProgressTracker(message):
clients.gae.UpdateConfig(c, info.parsed)
return {
'versions': new_versions,
'configs': app_config.Configs().keys()
}
def Run(self, args):
if args.env_vars:
log.warn('The env-vars flag is deprecated, and will soon be removed.')
# Do this up-front to print applicable warnings early
promote = deploy_command_util.GetPromoteFromArgs(args)
project = properties.VALUES.core.project.Get(required=True)
use_cloud_build = properties.VALUES.app.use_cloud_build.GetBool()
app_config = yaml_parsing.AppConfigSet(
args.deployables, project, args.version or util.GenerateVersionId())
remote_build = True
docker_build_property = properties.VALUES.app.docker_build.Get()
if args.docker_build:
remote_build = args.docker_build == 'remote'
elif docker_build_property:
remote_build = docker_build_property == 'remote'
# This will either be args.version or a generated version. Either way, if
# any yaml file has a version in it, it must match that version.
version = app_config.Version()
gae_client = appengine_client.AppengineClient(args.server)
api_client = appengine_api_client.GetApiClient(self.Http(timeout=None))
log.debug('API endpoint: [{endpoint}], API version: [{version}]'.format(
endpoint=api_client.client.url,
version=api_client.api_version))
cloudbuild_client = cloudbuild_v1.CloudbuildV1(http=self.Http(),
get_credentials=False)
deployed_urls = _DisplayProposedDeployment(project, app_config, version,
promote)
if args.version or promote:
# Prompt if there's a chance that you're overwriting something important:
# If the version is set manually, you could be deploying over something.
# If you're setting the new deployment to be the default version, you're
# changing the target of the default URL.
# Otherwise, all existing URLs will continue to work, so need to prompt.
console_io.PromptContinue(default=True, throw_if_unattended=False,
cancel_on_no=True)
log.status.Print('Beginning deployment...')
code_bucket = None
if use_cloud_build:
# If using Argo CloudBuild, we'll need to upload source to a GCS bucket.
code_bucket = self._GetCodeBucket(api_client, args)
modules = app_config.Modules()
if args.image_url:
if len(modules) != 1:
raise exceptions.ToolException(
'No more than one module may be deployed when using the '
'image-url flag')
for registry in constants.ALL_SUPPORTED_REGISTRIES:
if args.image_url.startswith(registry):
break
else:
raise exceptions.ToolException(
'%s is not in a supported registry. Supported registries are %s' %
(args.image_url,
constants.ALL_SUPPORTED_REGISTRIES))
module = modules.keys()[0]
images = {module: args.image_url}
else:
images = deploy_command_util.BuildAndPushDockerImages(modules,
version,
gae_client,
cloudbuild_client,
code_bucket,
self.cli,
remote_build)
deployment_manifests = {}
if app_config.NonHermeticModules() and self.use_admin_api:
# TODO(user): Consider doing this in parallel with
# BuildAndPushDockerImage.
code_bucket = self._GetCodeBucket(api_client, args)
metrics.CustomTimedEvent(metric_names.GET_CODE_BUCKET)
log.debug('Using bucket [{b}].'.format(b=code_bucket))
if not code_bucket:
raise exceptions.ToolException(('Could not retrieve the default Google '
'Cloud Storage bucket for [{a}]. '
'Please try again or use the [bucket] '
'argument.').format(a=project))
deployment_manifests = deploy_app_command_util.CopyFilesToCodeBucket(
app_config.NonHermeticModules().items(), code_bucket)
metrics.CustomTimedEvent(metric_names.COPY_APP_FILES)
# Now do deployment.
for (module, info) in app_config.Modules().iteritems():
message = 'Updating module [{module}]'.format(module=module)
with console_io.ProgressTracker(message):
if args.force:
gae_client.CancelDeployment(module=module, version=version)
metrics.CustomTimedEvent(metric_names.CANCEL_DEPLOYMENT)
if info.is_hermetic or self.use_admin_api:
api_client.DeployModule(module, version, info,
deployment_manifests.get(module),
images.get(module))
metrics.CustomTimedEvent(metric_names.DEPLOY_API)
else:
gae_client.DeployModule(module, version, info.parsed, info.file)
metrics.CustomTimedEvent(metric_names.DEPLOY_ADMIN_CONSOLE)
if promote:
if info.is_hermetic or self.use_admin_api:
api_client.SetDefaultVersion(module, version)
metrics.CustomTimedEvent(metric_names.SET_DEFAULT_VERSION_API)
else:
gae_client.SetDefaultVersion(modules=[module], version=version)
metrics.CustomTimedEvent(
metric_names.SET_DEFAULT_VERSION_ADMIN_CONSOLE)
# Config files.
for (c, info) in app_config.Configs().iteritems():
message = 'Updating config [{config}]'.format(config=c)
with console_io.ProgressTracker(message):
gae_client.UpdateConfig(c, info.parsed)
return deployed_urls
def RunDeploy(
args,
api_client,
use_beta_stager=False,
runtime_builder_strategy=runtime_builders.RuntimeBuilderStrategy.NEVER,
parallel_build=True,
flex_image_build_option=FlexImageBuildOptions.ON_CLIENT,
disable_build_cache=False,
dispatch_admin_api=False):
"""Perform a deployment based on the given args.
Args:
args: argparse.Namespace, An object that contains the values for the
arguments specified in the ArgsDeploy() function.
api_client: api_lib.app.appengine_api_client.AppengineClient, App Engine
Admin API client.
use_beta_stager: Use the stager registry defined for the beta track rather
than the default stager registry.
runtime_builder_strategy: runtime_builders.RuntimeBuilderStrategy, when to
use the new CloudBuild-based runtime builders (alternative is old
externalized runtimes).
parallel_build: bool, whether to use parallel build and deployment path.
Only supported in v1beta and v1alpha App Engine Admin API.
flex_image_build_option: FlexImageBuildOptions, whether a flex deployment
should upload files so that the server can build the image or build the
image on client.
disable_build_cache: bool, disable the build cache.
dispatch_admin_api: bool, speak to the (new) Admin API rather than the (old)
Admin Console for config push of dispatch.yaml.
Returns:
A dict on the form `{'versions': new_versions, 'configs': updated_configs}`
where new_versions is a list of version_util.Version, and updated_configs
is a list of config file identifiers, see yaml_parsing.ConfigYamlInfo.
"""
project = properties.VALUES.core.project.Get(required=True)
deploy_options = DeployOptions.FromProperties(
runtime_builder_strategy=runtime_builder_strategy,
parallel_build=parallel_build,
flex_image_build_option=flex_image_build_option)
with files.TemporaryDirectory() as staging_area:
stager = _MakeStager(args.skip_staging, use_beta_stager,
args.staging_command, staging_area)
services, configs = deployables.GetDeployables(
args.deployables, stager, deployables.GetPathMatchers())
service_infos = [d.service_info for d in services]
flags.ValidateImageUrl(args.image_url, service_infos)
# pylint: disable=protected-access
log.debug(
'API endpoint: [{endpoint}], API version: [{version}]'.format(
endpoint=api_client.client.url,
version=api_client.client._VERSION))
# The legacy admin console API client.
# The Admin Console API existed long before the App Engine Admin API, and
# isn't being improved. We're in the process of migrating all of the calls
# over to the Admin API, but a few things (notably config deployments)
# haven't been ported over yet.
# Import only when necessary, to decrease startup time.
# pylint: disable=g-import-not-at-top
from googlecloudsdk.api_lib.app import appengine_client
# pylint: enable=g-import-not-at-top
ac_client = appengine_client.AppengineClient(args.server,
args.ignore_bad_certs)
app = _PossiblyCreateApp(api_client, project)
_RaiseIfStopped(api_client, app)
# Call _PossiblyRepairApp when --bucket param is unspecified
if not args.bucket:
app = _PossiblyRepairApp(api_client, app)
# Tell the user what is going to happen, and ask them to confirm.
version_id = args.version or util.GenerateVersionId()
deployed_urls = output_helpers.DisplayProposedDeployment(
app, project, services, configs, version_id,
deploy_options.promote)
console_io.PromptContinue(cancel_on_no=True)
if service_infos:
# Do generic app setup if deploying any services.
# All deployment paths for a service involve uploading source to GCS.
metrics.CustomTimedEvent(metric_names.GET_CODE_BUCKET_START)
code_bucket_ref = args.bucket or flags.GetCodeBucket(app, project)
metrics.CustomTimedEvent(metric_names.GET_CODE_BUCKET)
log.debug('Using bucket [{b}].'.format(b=code_bucket_ref.ToUrl()))
# Prepare Flex if any service is going to deploy an image.
if any([s.RequiresImage() for s in service_infos]):
deploy_command_util.PossiblyEnableFlex(project)
all_services = dict([(s.id, s) for s in api_client.ListServices()])
else:
code_bucket_ref = None
all_services = {}
new_versions = []
deployer = ServiceDeployer(api_client, deploy_options)
# Track whether a service has been deployed yet, for metrics.
service_deployed = False
for service in services:
if not service_deployed:
metrics.CustomTimedEvent(
metric_names.FIRST_SERVICE_DEPLOY_START)
new_version = version_util.Version(project, service.service_id,
version_id)
deployer.Deploy(service,
new_version,
code_bucket_ref,
args.image_url,
all_services,
app.gcrDomain,
disable_build_cache=disable_build_cache,
flex_image_build_option=flex_image_build_option)
new_versions.append(new_version)
log.status.Print('Deployed service [{0}] to [{1}]'.format(
service.service_id, deployed_urls[service.service_id]))
if not service_deployed:
metrics.CustomTimedEvent(metric_names.FIRST_SERVICE_DEPLOY)
service_deployed = True
# Deploy config files.
if configs:
metrics.CustomTimedEvent(metric_names.UPDATE_CONFIG_START)
for config in configs:
message = 'Updating config [{config}]'.format(config=config.name)
with progress_tracker.ProgressTracker(message):
if config.name == 'dispatch' and dispatch_admin_api:
api_client.UpdateDispatchRules(config.GetRules())
else:
ac_client.UpdateConfig(config.name, config.parsed)
metrics.CustomTimedEvent(metric_names.UPDATE_CONFIG)
updated_configs = [c.name for c in configs]
PrintPostDeployHints(new_versions, updated_configs)
# Return all the things that were deployed.
return {'versions': new_versions, 'configs': updated_configs}
def RunDeploy(args, enable_endpoints=False, use_beta_stager=False,
upload_strategy=None, use_runtime_builders=False):
"""Perform a deployment based on the given args.
Args:
args: argparse.Namespace, An object that contains the values for the
arguments specified in the ArgsDeploy() function.
enable_endpoints: Enable Cloud Endpoints for the deployed app.
use_beta_stager: Use the stager registry defined for the beta track rather
than the default stager registry.
upload_strategy: deploy_app_command_util.UploadStrategy, the parallelism
straetgy to use for uploading files, or None to use the default.
use_runtime_builders: bool, whether to use the new CloudBuild-based
runtime builders (alternative is old externalized runtimes).
Returns:
A dict on the form `{'versions': new_versions, 'configs': updated_configs}`
where new_versions is a list of version_util.Version, and updated_configs
is a list of config file identifiers, see yaml_parsing.ConfigYamlInfo.
"""
project = properties.VALUES.core.project.Get(required=True)
deploy_options = DeployOptions.FromProperties(
enable_endpoints, upload_strategy=upload_strategy,
use_runtime_builders=use_runtime_builders)
# Parse existing app.yamls or try to generate a new one if the directory is
# empty.
if not args.deployables:
yaml_path = deploy_command_util.DEFAULT_DEPLOYABLE
if not os.path.exists(deploy_command_util.DEFAULT_DEPLOYABLE):
log.warning('Automatic app detection is currently in Beta')
yaml_path = deploy_command_util.CreateAppYamlForAppDirectory(os.getcwd())
app_config = yaml_parsing.AppConfigSet([yaml_path])
else:
app_config = yaml_parsing.AppConfigSet(args.deployables)
# If applicable, sort services by order they were passed to the command.
services = app_config.Services()
if not args.skip_image_url_validation:
flags.ValidateImageUrl(args.image_url, services)
# The new API client.
api_client = appengine_api_client.GetApiClient()
# pylint: disable=protected-access
log.debug('API endpoint: [{endpoint}], API version: [{version}]'.format(
endpoint=api_client.client.url,
version=api_client.client._VERSION))
# The legacy admin console API client.
# The Admin Console API existed long before the App Engine Admin API, and
# isn't being improved. We're in the process of migrating all of the calls
# over to the Admin API, but a few things (notably config deployments) haven't
# been ported over yet.
ac_client = appengine_client.AppengineClient(
args.server, args.ignore_bad_certs)
app = _PossiblyCreateApp(api_client, project)
app = _PossiblyRepairApp(api_client, app)
# Tell the user what is going to happen, and ask them to confirm.
version_id = args.version or util.GenerateVersionId()
deployed_urls = output_helpers.DisplayProposedDeployment(
app, project, app_config, version_id, deploy_options.promote)
console_io.PromptContinue(cancel_on_no=True)
if services:
# Do generic app setup if deploying any services.
# All deployment paths for a service involve uploading source to GCS.
code_bucket_ref = args.bucket or flags.GetCodeBucket(app, project)
metrics.CustomTimedEvent(metric_names.GET_CODE_BUCKET)
log.debug('Using bucket [{b}].'.format(b=code_bucket_ref.ToBucketUrl()))
# Prepare Flex if any service is going to deploy an image.
if any([m.RequiresImage() for m in services.values()]):
deploy_command_util.DoPrepareManagedVms(ac_client)
all_services = dict([(s.id, s) for s in api_client.ListServices()])
else:
code_bucket_ref = None
all_services = {}
new_versions = []
if args.skip_staging:
stager = staging.GetNoopStager()
elif use_beta_stager:
stager = staging.GetBetaStager()
else:
stager = staging.GetStager()
deployer = ServiceDeployer(api_client, stager, deploy_options)
for name, service in services.iteritems():
new_version = version_util.Version(project, name, version_id)
deployer.Deploy(service, new_version, code_bucket_ref, args.image_url,
all_services)
new_versions.append(new_version)
log.status.Print('Deployed service [{0}] to [{1}]'.format(
name, deployed_urls[name]))
# Deploy config files.
for (name, config) in app_config.Configs().iteritems():
message = 'Updating config [{config}]'.format(config=name)
with progress_tracker.ProgressTracker(message):
ac_client.UpdateConfig(name, config.parsed)
updated_configs = app_config.Configs().keys()
PrintPostDeployHints(new_versions, updated_configs)
# Return all the things that were deployed.
return {
'versions': new_versions,
'configs': updated_configs
}
def Deploy(self,
service,
new_version,
code_bucket_ref,
image,
all_services,
gcr_domain,
disable_build_cache,
flex_image_build_option=FlexImageBuildOptions.ON_CLIENT):
"""Deploy the given service.
Performs all deployment steps for the given service (if applicable):
* Enable endpoints (for beta deployments)
* Build and push the Docker image (Flex only, if image_url not provided)
* Upload files (non-hermetic deployments and flex deployments with
flex_image_build_option=FlexImageBuildOptions.ON_SERVER)
* Create the new version
* Promote the version to receive all traffic (if --promote given (default))
* Stop the previous version (if new version promoted and
--stop-previous-version given (default))
Args:
service: deployables.Service, service to be deployed.
new_version: version_util.Version describing where to deploy the service
code_bucket_ref: cloud_storage.BucketReference where the service's files
will be uploaded
image: str or None, the URL for the Docker image to be deployed (if image
already exists).
all_services: dict of service ID to service_util.Service objects
corresponding to all pre-existing services (used to determine how to
promote this version to receive all traffic, if applicable).
gcr_domain: str, Cloud Registry domain, determines the physical location
of the image. E.g. `us.gcr.io`.
disable_build_cache: bool, disable the build cache.
flex_image_build_option: FlexImageBuildOptions, whether a flex deployment
should upload files so that the server can build the image or build the
image on client.
"""
log.status.Print(
'Beginning deployment of service [{service}]...'.format(
service=new_version.service))
if (service.service_info.env == env.MANAGED_VMS and
flex_image_build_option == FlexImageBuildOptions.ON_SERVER):
# Server-side builds are not supported for Managed VMs.
flex_image_build_option = FlexImageBuildOptions.ON_CLIENT
service_info = service.service_info
self._ValidateRuntime(service_info)
source_files = source_files_util.GetSourceFiles(
service.upload_dir, service_info.parsed.skip_files.regex,
service_info.HasExplicitSkipFiles(), service_info.runtime,
service_info.env, service.source)
# Tar-based upload for flex
build = self._PossiblyBuildAndPush(new_version, service_info,
service.upload_dir, source_files,
image, code_bucket_ref, gcr_domain,
flex_image_build_option)
# Manifest-based incremental source upload for all envs
manifest = self._PossiblyUploadFiles(image, service_info,
service.upload_dir, source_files,
code_bucket_ref,
flex_image_build_option)
del source_files # Free some memory
extra_config_settings = {}
if disable_build_cache:
extra_config_settings['no-cache'] = 'true'
# Actually create the new version of the service.
metrics.CustomTimedEvent(metric_names.DEPLOY_API_START)
self.api_client.DeployService(new_version.service, new_version.id,
service_info, manifest, build,
extra_config_settings)
metrics.CustomTimedEvent(metric_names.DEPLOY_API)
self._PossiblyPromote(all_services, new_version)
def BuildAndPushDockerImage(
project,
service,
source_dir,
version_id,
code_bucket_ref,
gcr_domain,
runtime_builder_strategy=runtime_builders.RuntimeBuilderStrategy.NEVER
):
"""Builds and pushes a set of docker images.
Args:
project: str, The project being deployed to.
service: ServiceYamlInfo, The parsed service config.
source_dir: str, path to the service's source directory
version_id: The version id to deploy these services under.
code_bucket_ref: The reference to the GCS bucket where the source will be
uploaded.
gcr_domain: str, Cloud Registry domain, determines the physical location
of the image. E.g. `us.gcr.io`.
runtime_builder_strategy: runtime_builders.RuntimeBuilderStrategy, whether
to use the new CloudBuild-based runtime builders (alternative is old
externalized runtimes).
Returns:
str, The name of the pushed container image.
Raises:
DockerfileError: if a Dockerfile is present, but the runtime is not
"custom".
NoDockerfileError: Raised if a user didn't supply a Dockerfile and chose a
custom runtime.
UnsatisfiedRequirementsError: Raised if the code in the directory doesn't
satisfy the requirements of the specified runtime type.
"""
needs_dockerfile = _NeedsDockerfile(service, source_dir)
use_runtime_builders = runtime_builder_strategy.ShouldUseRuntimeBuilders(
service.runtime, needs_dockerfile)
# Nothing to do if this is not an image-based deployment.
if not service.RequiresImage():
return None
log.status.Print(
'Building and pushing image for service [{service}]'.format(
service=service.module))
gen_files = dict(_GetSourceContextsForUpload(source_dir))
if needs_dockerfile and not use_runtime_builders:
# The runtime builders will generate a Dockerfile in the Cloud, so we only
# need to do this if use_runtime_builders is True
gen_files.update(_GetDockerfiles(service, source_dir))
image = docker_image.Image(dockerfile_dir=source_dir,
repo=_GetImageName(project, service.module,
version_id, gcr_domain),
nocache=False,
tag=config.DOCKER_IMAGE_TAG)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD_START)
object_ref = storage_util.ObjectReference(code_bucket_ref,
image.tagged_repo)
try:
cloud_build.UploadSource(image.dockerfile_dir,
object_ref,
gen_files=gen_files,
skip_files=service.parsed.skip_files.regex)
except (OSError, IOError) as err:
if platforms.OperatingSystem.IsWindows():
if err.filename and len(err.filename) > _WINDOWS_MAX_PATH:
raise WindowMaxPathError(err.filename)
raise
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD)
if use_runtime_builders:
builder_reference = runtime_builders.FromServiceInfo(
service, source_dir)
log.info('Using runtime builder [%s]',
builder_reference.build_file_uri)
builder_reference.WarnIfDeprecated()
build = builder_reference.LoadCloudBuild(
{'_OUTPUT_IMAGE': image.tagged_repo})
# TODO(b/37542869) Remove this hack once the API can take the gs:// path
# as a runtime name.
service.runtime = builder_reference.runtime
service.parsed.SetEffectiveRuntime(builder_reference.runtime)
else:
build = cloud_build.GetDefaultBuild(image.tagged_repo)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE_START)
cloudbuild_build.CloudBuildClient().ExecuteCloudBuild(
cloud_build.FixUpBuild(build, object_ref), project=project)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE)
return image.tagged_repo
def BuildAndPushDockerImage(project,
service,
source_dir,
version_id,
code_bucket_ref,
use_runtime_builders=False):
"""Builds and pushes a set of docker images.
Args:
project: str, The project being deployed to.
service: ServiceYamlInfo, The parsed service config.
source_dir: str, path to the service's source directory
version_id: The version id to deploy these services under.
code_bucket_ref: The reference to the GCS bucket where the source will be
uploaded.
use_runtime_builders: bool, whether to use the new CloudBuild-based runtime
builders (alternative is old externalized runtimes).
Returns:
str, The name of the pushed container image.
"""
# Nothing to do if this is not an image-based deployment.
if not service.RequiresImage():
return None
log.status.Print(
'Building and pushing image for service [{service}]'.format(
service=service.module))
gen_files = dict(_GetSourceContextsForUpload(source_dir))
if not use_runtime_builders:
gen_files.update(_GetDockerfiles(service, source_dir))
image = docker_image.Image(dockerfile_dir=source_dir,
repo=_GetImageName(project, service.module,
version_id),
nocache=False,
tag=config.DOCKER_IMAGE_TAG)
object_ref = storage_util.ObjectReference(code_bucket_ref,
image.tagged_repo)
try:
cloud_build.UploadSource(image.dockerfile_dir,
object_ref,
gen_files=gen_files,
skip_files=service.parsed.skip_files.regex)
except (OSError, IOError) as err:
if platforms.OperatingSystem.IsWindows():
if err.filename and len(err.filename) > _WINDOWS_MAX_PATH:
raise WindowMaxPathError(err.filename)
raise
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD)
if use_runtime_builders:
builder_version = runtime_builders.RuntimeBuilderVersion.FromServiceInfo(
service)
build = builder_version.LoadCloudBuild(
{'_OUTPUT_IMAGE': image.tagged_repo})
else:
build = cloud_build.GetDefaultBuild(image.tagged_repo)
cloudbuild_build.CloudBuildClient().ExecuteCloudBuild(
cloud_build.FixUpBuild(build, object_ref), project=project)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE)
return image.tagged_repo
def RunDeploy(
args,
enable_endpoints=False,
use_beta_stager=False,
runtime_builder_strategy=runtime_builders.RuntimeBuilderStrategy.NEVER,
use_service_management=False,
check_for_stopped=False):
"""Perform a deployment based on the given args.
Args:
args: argparse.Namespace, An object that contains the values for the
arguments specified in the ArgsDeploy() function.
enable_endpoints: Enable Cloud Endpoints for the deployed app.
use_beta_stager: Use the stager registry defined for the beta track rather
than the default stager registry.
runtime_builder_strategy: runtime_builders.RuntimeBuilderStrategy, when to
use the new CloudBuild-based runtime builders (alternative is old
externalized runtimes).
use_service_management: bool, whether to use servicemanagement API to
enable the Appengine Flexible API for a Flexible deployment.
check_for_stopped: bool, whether to check if the app is stopped before
deploying.
Returns:
A dict on the form `{'versions': new_versions, 'configs': updated_configs}`
where new_versions is a list of version_util.Version, and updated_configs
is a list of config file identifiers, see yaml_parsing.ConfigYamlInfo.
"""
project = properties.VALUES.core.project.Get(required=True)
deploy_options = DeployOptions.FromProperties(
enable_endpoints, runtime_builder_strategy=runtime_builder_strategy)
with files.TemporaryDirectory() as staging_area:
if args.skip_staging:
stager = staging.GetNoopStager(staging_area)
elif use_beta_stager:
stager = staging.GetBetaStager(staging_area)
else:
stager = staging.GetStager(staging_area)
services, configs = deployables.GetDeployables(
args.deployables, stager, deployables.GetPathMatchers())
service_infos = [d.service_info for d in services]
if not args.skip_image_url_validation:
flags.ValidateImageUrl(args.image_url, service_infos)
# The new API client.
api_client = appengine_api_client.GetApiClient()
# pylint: disable=protected-access
log.debug(
'API endpoint: [{endpoint}], API version: [{version}]'.format(
endpoint=api_client.client.url,
version=api_client.client._VERSION))
# The legacy admin console API client.
# The Admin Console API existed long before the App Engine Admin API, and
# isn't being improved. We're in the process of migrating all of the calls
# over to the Admin API, but a few things (notably config deployments)
# haven't been ported over yet.
ac_client = appengine_client.AppengineClient(args.server,
args.ignore_bad_certs)
app = _PossiblyCreateApp(api_client, project)
if check_for_stopped:
_RaiseIfStopped(api_client, app)
app = _PossiblyRepairApp(api_client, app)
# Tell the user what is going to happen, and ask them to confirm.
version_id = args.version or util.GenerateVersionId()
deployed_urls = output_helpers.DisplayProposedDeployment(
app, project, services, configs, version_id,
deploy_options.promote)
console_io.PromptContinue(cancel_on_no=True)
if service_infos:
# Do generic app setup if deploying any services.
# All deployment paths for a service involve uploading source to GCS.
metrics.CustomTimedEvent(metric_names.GET_CODE_BUCKET_START)
code_bucket_ref = args.bucket or flags.GetCodeBucket(app, project)
metrics.CustomTimedEvent(metric_names.GET_CODE_BUCKET)
log.debug(
'Using bucket [{b}].'.format(b=code_bucket_ref.ToBucketUrl()))
# Prepare Flex if any service is going to deploy an image.
if any([s.RequiresImage() for s in service_infos]):
if use_service_management:
deploy_command_util.PossiblyEnableFlex(project)
else:
deploy_command_util.DoPrepareManagedVms(ac_client)
all_services = dict([(s.id, s) for s in api_client.ListServices()])
else:
code_bucket_ref = None
all_services = {}
new_versions = []
deployer = ServiceDeployer(api_client, deploy_options)
# Track whether a service has been deployed yet, for metrics.
service_deployed = False
for service in services:
if not service_deployed:
metrics.CustomTimedEvent(
metric_names.FIRST_SERVICE_DEPLOY_START)
new_version = version_util.Version(project, service.service_id,
version_id)
deployer.Deploy(service, new_version, code_bucket_ref,
args.image_url, all_services, app.gcrDomain)
new_versions.append(new_version)
log.status.Print('Deployed service [{0}] to [{1}]'.format(
service.service_id, deployed_urls[service.service_id]))
if not service_deployed:
metrics.CustomTimedEvent(metric_names.FIRST_SERVICE_DEPLOY)
service_deployed = True
# Deploy config files.
if configs:
metrics.CustomTimedEvent(metric_names.UPDATE_CONFIG_START)
for config in configs:
message = 'Updating config [{config}]'.format(config=config.name)
with progress_tracker.ProgressTracker(message):
ac_client.UpdateConfig(config.name, config.parsed)
metrics.CustomTimedEvent(metric_names.UPDATE_CONFIG)
updated_configs = [c.name for c in configs]
PrintPostDeployHints(new_versions, updated_configs)
# Return all the things that were deployed.
return {'versions': new_versions, 'configs': updated_configs}
def Deploy(self,
service,
new_version,
code_bucket_ref,
image,
all_services,
gcr_domain,
flex_image_build_option=False):
"""Deploy the given service.
Performs all deployment steps for the given service (if applicable):
* Enable endpoints (for beta deployments)
* Build and push the Docker image (Flex only, if image_url not provided)
* Upload files (non-hermetic deployments and flex deployments with
flex_image_build_option=FlexImageBuildOptions.ON_SERVER)
* Create the new version
* Promote the version to receieve all traffic (if --promote given (default))
* Stop the previous version (if new version promoted and
--stop-previous-version given (default))
Args:
service: deployables.Service, service to be deployed.
new_version: version_util.Version describing where to deploy the service
code_bucket_ref: cloud_storage.BucketReference where the service's files
have been uploaded
image: str or None, the URL for the Docker image to be deployed (if image
already exists).
all_services: dict of service ID to service_util.Service objects
corresponding to all pre-existing services (used to determine how to
promote this version to receive all traffic, if applicable).
gcr_domain: str, Cloud Registry domain, determines the physical location
of the image. E.g. `us.gcr.io`.
flex_image_build_option: FlexImageBuildOptions, whether a flex deployment
should upload files so that the server can build the image or build the
image on client.
"""
log.status.Print(
'Beginning deployment of service [{service}]...'.format(
service=new_version.service))
source_dir = service.upload_dir
service_info = service.service_info
endpoints_info = self._PossiblyConfigureEndpoints(
service_info, source_dir, new_version)
self._PossiblyRewriteRuntime(service_info)
build = self._PossiblyBuildAndPush(new_version, service_info,
source_dir, image, code_bucket_ref,
gcr_domain, flex_image_build_option)
manifest = self._PossiblyUploadFiles(image, service_info, source_dir,
code_bucket_ref,
flex_image_build_option)
extra_config_settings = None
if flex_image_build_option == FlexImageBuildOptions.ON_SERVER:
extra_config_settings = {
'cloud_build_timeout':
properties.VALUES.app.cloud_build_timeout.Get(),
'runtime_root':
properties.VALUES.app.runtime_root.Get(),
}
# Actually create the new version of the service.
metrics.CustomTimedEvent(metric_names.DEPLOY_API_START)
self.api_client.DeployService(new_version.service, new_version.id,
service_info, manifest, build,
endpoints_info, extra_config_settings)
metrics.CustomTimedEvent(metric_names.DEPLOY_API)
message = 'Updating service [{service}]'.format(
service=new_version.service)
with progress_tracker.ProgressTracker(message):
self._PossiblyPromote(all_services, new_version)
