def _BuildImagesWithCloudBuild(project, services, version_id, code_bucket_ref,
cloudbuild_client, storage_client):
"""Build multiple services with Cloud Build."""
images = {}
for service, info, ensure_dockerfile, ensure_context in services:
log.status.Print(
'Building and pushing image for service [{service}]'.format(
service=service))
cleanup_dockerfile = ensure_dockerfile()
cleanup_context = ensure_context()
try:
image = docker_image.Image(
dockerfile_dir=os.path.dirname(info.file),
tag=_GetImageName(project, service, version_id),
nocache=False)
cloud_build.UploadSource(image.dockerfile_dir, code_bucket_ref,
image.tag, storage_client)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD)
cloud_build.ExecuteCloudBuild(project, code_bucket_ref, image.tag,
image.tag, cloudbuild_client)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE)
images[service] = image.tag
finally:
cleanup_dockerfile()
cleanup_context()
return images
def _BuildImagesWithCloudBuild(project, modules, version_id, code_bucket,
cloudbuild_client):
"""Build multiple modules with Cloud Build."""
images = {}
for module, info, ensure_dockerfile, ensure_context in modules:
log.status.Print(
'Building and pushing image for module [{module}]'.format(
module=module))
cleanup_dockerfile = ensure_dockerfile()
cleanup_context = ensure_context()
try:
image = docker_image.Image(
dockerfile_dir=os.path.dirname(info.file),
tag=_GetImageName(project, module, version_id),
nocache=False)
source_gcs_uri = '/'.join([code_bucket.rstrip('/'), image.tag])
cloud_build.UploadSource(image.dockerfile_dir, source_gcs_uri)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD)
cloud_build.ExecuteCloudBuild(project, source_gcs_uri,
image.repo_tag, cloudbuild_client)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE)
images[module] = image.repo_tag
finally:
cleanup_dockerfile()
cleanup_context()
return images
def testUpload(self):
"""Test basic upload with single file."""
tmp = self.CreateTempDir()
self.WriteFile(os.path.join(tmp, 'Dockerfile'), 'empty')
self._ExpectUpload()
cloud_build.UploadSource(tmp, util.FileIterator(tmp, self.re),
self.object_ref)
def testFailure(self):
"""Test HttpError raises to user."""
tmp = self.CreateTempDir()
self.WriteFile(os.path.join(tmp, 'Dockerfile'), 'empty')
self._ExpectUpload(exception=http_error.MakeHttpError())
with self.assertRaises(storage_api.UploadError):
cloud_build.UploadSource(tmp, util.FileIterator(tmp, self.re),
self.object_ref)
def testUploadWithGenFiles(self):
"""Test that generated files passed to UploadSource don't raise error."""
tmp = self.CreateTempDir()
self.WriteFile(os.path.join(tmp, 'main.py'), 'empty')
gen_files = {'Dockerfile': 'empty'}
self._ExpectUpload()
cloud_build.UploadSource(tmp,
util.FileIterator(tmp, self.re),
self.object_ref,
gen_files=gen_files)
def BuildAndPushDockerImage(project, service, version_id, code_bucket_ref):
"""Builds and pushes a set of docker images.
Args:
project: str, The project being deployed to.
service: ServiceYamlInfo, The parsed service config.
version_id: The version id to deploy these services under.
code_bucket_ref: The reference to the GCS bucket where the source will be
uploaded.
Returns:
str, The name of the pushed container image.
"""
# Nothing to do if this is not an image based deployment.
if not service.RequiresImage():
return None
dockerfile_creator = _GetDockerfileCreator(service)
context_creator = context_util.GetSourceContextFilesCreator(
os.path.dirname(service.file), None)
log.status.Print(
'Building and pushing image for service [{service}]'.format(
service=service.module))
cleanup_dockerfile = dockerfile_creator()
cleanup_context = context_creator()
try:
image = docker_image.Image(
dockerfile_dir=os.path.dirname(service.file),
repo=_GetImageName(project, service.module, version_id),
nocache=False,
tag=config.DOCKER_IMAGE_TAG)
try:
cloud_build.UploadSource(image.dockerfile_dir, code_bucket_ref,
image.tagged_repo)
except (OSError, IOError) as err:
if platforms.OperatingSystem.IsWindows():
if len(err.filename) > _WINDOWS_MAX_PATH:
raise WindowMaxPathError(err.filename)
raise
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD)
cloud_build.ExecuteCloudBuild(project, code_bucket_ref,
image.tagged_repo, image.tagged_repo)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE)
return image.tagged_repo
finally:
cleanup_dockerfile()
cleanup_context()
def BuildAndPushDockerImage(project, service, source_dir, version_id,
code_bucket_ref):
"""Builds and pushes a set of docker images.
Args:
project: str, The project being deployed to.
service: ServiceYamlInfo, The parsed service config.
source_dir: str, path to the service's source directory
version_id: The version id to deploy these services under.
code_bucket_ref: The reference to the GCS bucket where the source will be
uploaded.
Returns:
str, The name of the pushed container image.
"""
# Nothing to do if this is not an image-based deployment.
if not service.RequiresImage():
return None
gen_files = {}
gen_files.update(_GetDockerfiles(service, source_dir))
gen_files.update(_GetSourceContextsForUpload(source_dir))
log.status.Print(
'Building and pushing image for service [{service}]'.format(
service=service.module))
image = docker_image.Image(dockerfile_dir=source_dir,
repo=_GetImageName(project, service.module,
version_id),
nocache=False,
tag=config.DOCKER_IMAGE_TAG)
try:
cloud_build.UploadSource(image.dockerfile_dir,
code_bucket_ref,
image.tagged_repo,
gen_files=gen_files,
skip_files=service.parsed.skip_files.regex)
except (OSError, IOError) as err:
if platforms.OperatingSystem.IsWindows():
if err.filename and len(err.filename) > _WINDOWS_MAX_PATH:
raise WindowMaxPathError(err.filename)
raise
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD)
cloud_build.ExecuteCloudBuild(project, code_bucket_ref, image.tagged_repo,
image.tagged_repo)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE)
return image.tagged_repo
def testUploadWithGeneratedDockerignore(self):
"""Test that UploadSource correctly interprets generated .dockerignore."""
tmp = self.CreateTempDir()
create_tar_mock = self.StartObjectPatch(cloud_build, '_CreateTar')
create_tar_mock.return_value = 1
self.WriteFile(os.path.join(tmp, 'Dockerfile'), 'empty')
self.WriteFile(os.path.join(tmp, 'main.py'), 'empty')
self.WriteFile(os.path.join(tmp, 'fake.zip'), 'Dummy')
os.mkdir(os.path.join(tmp, 'tmpsubdir'))
self.WriteFile(os.path.join(tmp, 'tmpsubdir', 'fake2.zip'), 'Dummy')
gen_files = {'.dockerignore': 'main.py'}
self._ExpectUpload()
cloud_build.UploadSource(tmp, util.FileIterator(tmp, self.re),
self.object_ref, gen_files)
# Test that _CreateTar was called with the correct directory, files, and
# exclusions
create_tar_mock.assert_called_once_with(
tmp, gen_files,
{'Dockerfile', 'fake.zip',
os.path.join('tmpsubdir', 'fake2.zip')}, mock.ANY)
def BuildAndPushDockerImage(
project,
service,
source_dir,
version_id,
code_bucket_ref,
gcr_domain,
runtime_builder_strategy=runtime_builders.RuntimeBuilderStrategy.NEVER,
parallel_build=False):
"""Builds and pushes a set of docker images.
Args:
project: str, The project being deployed to.
service: ServiceYamlInfo, The parsed service config.
source_dir: str, path to the service's source directory
version_id: The version id to deploy these services under.
code_bucket_ref: The reference to the GCS bucket where the source will be
uploaded.
gcr_domain: str, Cloud Registry domain, determines the physical location
of the image. E.g. `us.gcr.io`.
runtime_builder_strategy: runtime_builders.RuntimeBuilderStrategy, whether
to use the new CloudBuild-based runtime builders (alternative is old
externalized runtimes).
parallel_build: bool, if True, enable parallel build and deploy.
Returns:
BuildArtifact, Representing the pushed container image or in-progress build.
Raises:
DockerfileError: if a Dockerfile is present, but the runtime is not
"custom".
NoDockerfileError: Raised if a user didn't supply a Dockerfile and chose a
custom runtime.
UnsatisfiedRequirementsError: Raised if the code in the directory doesn't
satisfy the requirements of the specified runtime type.
ValueError: if an unrecognized runtime_builder_strategy is given
"""
needs_dockerfile = _NeedsDockerfile(service, source_dir)
use_runtime_builders = ShouldUseRuntimeBuilders(service,
runtime_builder_strategy,
needs_dockerfile)
# Nothing to do if this is not an image-based deployment.
if not service.RequiresImage():
return None
log.status.Print(
'Building and pushing image for service [{service}]'
.format(service=service.module))
gen_files = dict(_GetSourceContextsForUpload(source_dir))
if needs_dockerfile and not use_runtime_builders:
# The runtime builders will generate a Dockerfile in the Cloud, so we only
# need to do this if use_runtime_builders is True
gen_files.update(_GetDockerfiles(service, source_dir))
image = docker_image.Image(
dockerfile_dir=source_dir,
repo=_GetImageName(project, service.module, version_id, gcr_domain),
nocache=False,
tag=config.DOCKER_IMAGE_TAG)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD_START)
object_ref = storage_util.ObjectReference(code_bucket_ref, image.tagged_repo)
relative_yaml_path = _GetYamlPath(source_dir, service.file,
service.parsed.skip_files, gen_files)
try:
cloud_build.UploadSource(image.dockerfile_dir, object_ref,
gen_files=gen_files,
skip_files=service.parsed.skip_files.regex)
except (OSError, IOError) as err:
if platforms.OperatingSystem.IsWindows():
if err.filename and len(err.filename) > _WINDOWS_MAX_PATH:
raise WindowMaxPathError(err.filename)
raise
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD)
if use_runtime_builders:
builder_reference = runtime_builders.FromServiceInfo(service, source_dir)
log.info('Using runtime builder [%s]', builder_reference.build_file_uri)
builder_reference.WarnIfDeprecated()
yaml_path = util.ConvertToPosixPath(relative_yaml_path)
build = builder_reference.LoadCloudBuild(
{'_OUTPUT_IMAGE': image.tagged_repo,
'_GAE_APPLICATION_YAML_PATH': yaml_path})
else:
build = cloud_build.GetDefaultBuild(image.tagged_repo)
build = cloud_build.FixUpBuild(build, object_ref)
return _SubmitBuild(build, image, project, parallel_build)
def BuildAndPushDockerImage(project,
service,
source_dir,
version_id,
code_bucket_ref,
use_runtime_builders=False):
"""Builds and pushes a set of docker images.
Args:
project: str, The project being deployed to.
service: ServiceYamlInfo, The parsed service config.
source_dir: str, path to the service's source directory
version_id: The version id to deploy these services under.
code_bucket_ref: The reference to the GCS bucket where the source will be
uploaded.
use_runtime_builders: bool, whether to use the new CloudBuild-based runtime
builders (alternative is old externalized runtimes).
Returns:
str, The name of the pushed container image.
"""
# Nothing to do if this is not an image-based deployment.
if not service.RequiresImage():
return None
log.status.Print(
'Building and pushing image for service [{service}]'.format(
service=service.module))
gen_files = dict(_GetSourceContextsForUpload(source_dir))
if not use_runtime_builders:
gen_files.update(_GetDockerfiles(service, source_dir))
image = docker_image.Image(dockerfile_dir=source_dir,
repo=_GetImageName(project, service.module,
version_id),
nocache=False,
tag=config.DOCKER_IMAGE_TAG)
object_ref = storage_util.ObjectReference(code_bucket_ref,
image.tagged_repo)
try:
cloud_build.UploadSource(image.dockerfile_dir,
object_ref,
gen_files=gen_files,
skip_files=service.parsed.skip_files.regex)
except (OSError, IOError) as err:
if platforms.OperatingSystem.IsWindows():
if err.filename and len(err.filename) > _WINDOWS_MAX_PATH:
raise WindowMaxPathError(err.filename)
raise
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD)
if use_runtime_builders:
builder_version = runtime_builders.RuntimeBuilderVersion.FromServiceInfo(
service)
build = builder_version.LoadCloudBuild(
{'_OUTPUT_IMAGE': image.tagged_repo})
else:
build = cloud_build.GetDefaultBuild(image.tagged_repo)
cloudbuild_build.CloudBuildClient().ExecuteCloudBuild(
cloud_build.FixUpBuild(build, object_ref), project=project)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE)
return image.tagged_repo
def BuildAndPushDockerImage(
project,
service,
source_dir,
version_id,
code_bucket_ref,
gcr_domain,
runtime_builder_strategy=runtime_builders.RuntimeBuilderStrategy.NEVER
):
"""Builds and pushes a set of docker images.
Args:
project: str, The project being deployed to.
service: ServiceYamlInfo, The parsed service config.
source_dir: str, path to the service's source directory
version_id: The version id to deploy these services under.
code_bucket_ref: The reference to the GCS bucket where the source will be
uploaded.
gcr_domain: str, Cloud Registry domain, determines the physical location
of the image. E.g. `us.gcr.io`.
runtime_builder_strategy: runtime_builders.RuntimeBuilderStrategy, whether
to use the new CloudBuild-based runtime builders (alternative is old
externalized runtimes).
Returns:
str, The name of the pushed container image.
Raises:
DockerfileError: if a Dockerfile is present, but the runtime is not
"custom".
NoDockerfileError: Raised if a user didn't supply a Dockerfile and chose a
custom runtime.
UnsatisfiedRequirementsError: Raised if the code in the directory doesn't
satisfy the requirements of the specified runtime type.
"""
needs_dockerfile = _NeedsDockerfile(service, source_dir)
use_runtime_builders = runtime_builder_strategy.ShouldUseRuntimeBuilders(
service.runtime, needs_dockerfile)
# Nothing to do if this is not an image-based deployment.
if not service.RequiresImage():
return None
log.status.Print(
'Building and pushing image for service [{service}]'.format(
service=service.module))
gen_files = dict(_GetSourceContextsForUpload(source_dir))
if needs_dockerfile and not use_runtime_builders:
# The runtime builders will generate a Dockerfile in the Cloud, so we only
# need to do this if use_runtime_builders is True
gen_files.update(_GetDockerfiles(service, source_dir))
image = docker_image.Image(dockerfile_dir=source_dir,
repo=_GetImageName(project, service.module,
version_id, gcr_domain),
nocache=False,
tag=config.DOCKER_IMAGE_TAG)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD_START)
object_ref = storage_util.ObjectReference(code_bucket_ref,
image.tagged_repo)
if files.IsDirAncestorOf(source_dir, service.file):
relative_yaml_path = os.path.relpath(service.file, source_dir)
else:
yaml_contents = files.GetFileContents(service.file)
checksum = files.Checksum().AddContents(yaml_contents).HexDigest()
relative_yaml_path = checksum + '.yaml'
gen_files[relative_yaml_path] = yaml_contents
try:
cloud_build.UploadSource(image.dockerfile_dir,
object_ref,
gen_files=gen_files,
skip_files=service.parsed.skip_files.regex)
except (OSError, IOError) as err:
if platforms.OperatingSystem.IsWindows():
if err.filename and len(err.filename) > _WINDOWS_MAX_PATH:
raise WindowMaxPathError(err.filename)
raise
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD)
if use_runtime_builders:
builder_reference = runtime_builders.FromServiceInfo(
service, source_dir)
log.info('Using runtime builder [%s]',
builder_reference.build_file_uri)
builder_reference.WarnIfDeprecated()
yaml_path = posixpath.join(*relative_yaml_path.split(os.sep))
build = builder_reference.LoadCloudBuild({
'_OUTPUT_IMAGE':
image.tagged_repo,
'_GAE_APPLICATION_YAML_PATH':
yaml_path
})
# TODO(b/37542869) Remove this hack once the API can take the gs:// path
# as a runtime name.
service.runtime = builder_reference.runtime
service.parsed.SetEffectiveRuntime(builder_reference.runtime)
else:
build = cloud_build.GetDefaultBuild(image.tagged_repo)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE_START)
cloudbuild_build.CloudBuildClient().ExecuteCloudBuild(
cloud_build.FixUpBuild(build, object_ref), project=project)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE)
return image.tagged_repo
def BuildAndPushDockerImage(
project,
service,
source_dir,
version_id,
code_bucket_ref,
gcr_domain,
runtime_builder_strategy=runtime_builders.RuntimeBuilderStrategy.NEVER
):
"""Builds and pushes a set of docker images.
Args:
project: str, The project being deployed to.
service: ServiceYamlInfo, The parsed service config.
source_dir: str, path to the service's source directory
version_id: The version id to deploy these services under.
code_bucket_ref: The reference to the GCS bucket where the source will be
uploaded.
gcr_domain: str, Cloud Registry domain, determines the physical location
of the image. E.g. `us.gcr.io`.
runtime_builder_strategy: runtime_builders.RuntimeBuilderStrategy, whether
to use the new CloudBuild-based runtime builders (alternative is old
externalized runtimes).
Returns:
str, The name of the pushed container image.
Raises:
DockerfileError: if a Dockerfile is present, but the runtime is not
"custom".
NoDockerfileError: Raised if a user didn't supply a Dockerfile and chose a
custom runtime.
UnsatisfiedRequirementsError: Raised if the code in the directory doesn't
satisfy the requirements of the specified runtime type.
"""
use_runtime_builders = runtime_builder_strategy.ShouldUseRuntimeBuilders(
service.runtime)
# Nothing to do if this is not an image-based deployment.
if not service.RequiresImage():
return None
log.status.Print(
'Building and pushing image for service [{service}]'.format(
service=service.module))
gen_files = dict(_GetSourceContextsForUpload(source_dir))
needs_dockerfile = _NeedsDockerfile(service, source_dir)
if needs_dockerfile and not use_runtime_builders:
# The runtime builders will generate a Dockerfile in the Cloud, so we only
# need to do this if use_runtime_builders is True
gen_files.update(_GetDockerfiles(service, source_dir))
image = docker_image.Image(dockerfile_dir=source_dir,
repo=_GetImageName(project, service.module,
version_id, gcr_domain),
nocache=False,
tag=config.DOCKER_IMAGE_TAG)
object_ref = storage_util.ObjectReference(code_bucket_ref,
image.tagged_repo)
try:
cloud_build.UploadSource(image.dockerfile_dir,
object_ref,
gen_files=gen_files,
skip_files=service.parsed.skip_files.regex)
except (OSError, IOError) as err:
if platforms.OperatingSystem.IsWindows():
if err.filename and len(err.filename) > _WINDOWS_MAX_PATH:
raise WindowMaxPathError(err.filename)
raise
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD)
if use_runtime_builders:
builder_version = runtime_builders.RuntimeBuilderVersion.FromServiceInfo(
service)
build = builder_version.LoadCloudBuild(
{'_OUTPUT_IMAGE': image.tagged_repo})
else:
build = cloud_build.GetDefaultBuild(image.tagged_repo)
cloudbuild_build.CloudBuildClient().ExecuteCloudBuild(
cloud_build.FixUpBuild(build, object_ref), project=project)
metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE)
return image.tagged_repo
