def _SubmitBuild(build, image, project, parallel_build): """Builds and pushes a set of docker images. Args: build: A fixed up Build object. image: docker_image.Image, A docker image. project: str, The project being deployed to. parallel_build: bool, if True, enable parallel build and deploy. Returns: BuildArtifact, Representing the pushed container image or in-progress build. """ build_timeout = properties.VALUES.app.cloud_build_timeout.Get() if build_timeout and build_timeout > MAX_PARALLEL_BUILD_TIME: parallel_build = False log.info( 'Property cloud_build_timeout configured to [{0}], which exceeds ' 'the maximum build time for parallelized beta deployments of [{1}] ' 'seconds. Performing serial deployment.'.format( build_timeout, MAX_PARALLEL_BUILD_TIME)) if parallel_build: metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE_ASYNC_START) build_op = cloudbuild_build.CloudBuildClient().ExecuteCloudBuildAsync( build, project=project) return app_build.BuildArtifact.MakeBuildIdArtifactFromOp(build_op) else: metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE_START) cloudbuild_build.CloudBuildClient().ExecuteCloudBuild( build, project=project) metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE) return app_build.BuildArtifact.MakeImageArtifact(image.tagged_repo)
def DeployService(self, service_name, version_id, service_config, manifest, build, endpoints_info=None, extra_config_settings=None): """Updates and deploys new app versions based on given config. If the build operation has not yet completed, streams the Google Cloud Builder logs as well. Args: service_name: str, The service to deploy. version_id: str, The version of the service to deploy. service_config: AppInfoExternal, Service info parsed from a service yaml file. manifest: Dictionary mapping source files to Google Cloud Storage locations. build: BuildArtifact, a wrapper which contains either the build ID for an in-progress parallel build, or the name of the container image for a serial build. endpoints_info: EndpointsServiceInfo, Endpoints service info to be added to the AppInfoExternal configuration. Only provided when Endpoints API Management feature is enabled. extra_config_settings: dict, client config settings to pass to the server as beta settings. Returns: A Version resource representing the deployed version. """ version_resource = self._CreateVersionResource(service_config, manifest, version_id, build, endpoints_info, extra_config_settings) create_request = self.messages.AppengineAppsServicesVersionsCreateRequest( parent=self._GetServiceRelativeName(service_name=service_name), version=version_resource) operation = self.client.apps_services_versions.Create(create_request) # If build operation is still in progress, stream build logs and wait for # completion before polling the service deployment operation for completion. # Service deployment can never complete before the build has finished, as it # is dependent on the built image. if build and build.IsBuildId() and build.build_op: cloud_build.CloudBuildClient().WaitAndStreamLogs(build.build_op) metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE_ASYNC) log.debug('Received operation: [{operation}]'.format( operation=operation.name)) message = 'Updating service [{service}]'.format(service=service_name) return operations_util.WaitForOperation(self.client.apps_operations, operation, message=message)
def SetUp(self): self.mock_client = api_mock.Client( core_apis.GetClientClass('cloudbuild', 'v1')) self.mock_client.Mock() self.addCleanup(self.mock_client.Unmock) self.messages = core_apis.GetMessagesModule('cloudbuild', 'v1') # Intercept logging output. self.mock_log_content = '' mock_log_print = self.StartPatch( 'googlecloudsdk.api_lib.cloudbuild.logs.LogTailer._PrintLogLine') mock_log_print.side_effect = self._MockPrintLogLine self.StartPatch( 'googlecloudsdk.core.console.console_attr_os.GetTermSize', return_value=(40, 100)) self.client = build.CloudBuildClient(self.mock_client, self.messages) self.build = self.messages.Build( images=['gcr.io/my-project/output-tag']) self.StartObjectPatch(time, 'sleep')
def ExecuteCloudBuild(project, bucket_ref, object_name, output_image): """Execute a CloudBuild to build an app and wait for it to finish. Args: project: the cloud project ID. bucket_ref: Reference to GCS bucket containing source to build. The same bucket will be used for streaming logs. object_name: GCS object name containing source to build. output_image: GCR location for the output docker image; eg, gcr.io/test-gae/hardcoded-output-tag. Raises: BuildFailedError: when the build fails. """ builder = properties.VALUES.app.container_builder_image.Get() log.debug('Using builder image: [{0}]'.format(builder)) logs_bucket = bucket_ref.bucket build_timeout = properties.VALUES.app.cloud_build_timeout.Get() timeout_str = GetServiceTimeoutString(build_timeout) cloudbuild_client = cloudbuild_build.CloudBuildClient() build = cloudbuild_client.messages.Build( timeout=timeout_str, source=cloudbuild_client.messages.Source( storageSource=cloudbuild_client.messages.StorageSource( bucket=bucket_ref.bucket, object=object_name, ), ), steps=[ cloudbuild_client.messages.BuildStep( name=builder, args=['build', '-t', output_image, '.']) ], images=[output_image], logsBucket=logs_bucket) cloudbuild_client.ExecuteCloudBuild(build, project=project)
def BuildAndPushDockerImage(project, service, source_dir, version_id, code_bucket_ref, use_runtime_builders=False): """Builds and pushes a set of docker images. Args: project: str, The project being deployed to. service: ServiceYamlInfo, The parsed service config. source_dir: str, path to the service's source directory version_id: The version id to deploy these services under. code_bucket_ref: The reference to the GCS bucket where the source will be uploaded. use_runtime_builders: bool, whether to use the new CloudBuild-based runtime builders (alternative is old externalized runtimes). Returns: str, The name of the pushed container image. """ # Nothing to do if this is not an image-based deployment. if not service.RequiresImage(): return None log.status.Print( 'Building and pushing image for service [{service}]'.format( service=service.module)) gen_files = dict(_GetSourceContextsForUpload(source_dir)) if not use_runtime_builders: gen_files.update(_GetDockerfiles(service, source_dir)) image = docker_image.Image(dockerfile_dir=source_dir, repo=_GetImageName(project, service.module, version_id), nocache=False, tag=config.DOCKER_IMAGE_TAG) object_ref = storage_util.ObjectReference(code_bucket_ref, image.tagged_repo) try: cloud_build.UploadSource(image.dockerfile_dir, object_ref, gen_files=gen_files, skip_files=service.parsed.skip_files.regex) except (OSError, IOError) as err: if platforms.OperatingSystem.IsWindows(): if err.filename and len(err.filename) > _WINDOWS_MAX_PATH: raise WindowMaxPathError(err.filename) raise metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD) if use_runtime_builders: builder_version = runtime_builders.RuntimeBuilderVersion.FromServiceInfo( service) build = builder_version.LoadCloudBuild( {'_OUTPUT_IMAGE': image.tagged_repo}) else: build = cloud_build.GetDefaultBuild(image.tagged_repo) cloudbuild_build.CloudBuildClient().ExecuteCloudBuild( cloud_build.FixUpBuild(build, object_ref), project=project) metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE) return image.tagged_repo
def BuildAndPushDockerImage( project, service, source_dir, version_id, code_bucket_ref, gcr_domain, runtime_builder_strategy=runtime_builders.RuntimeBuilderStrategy.NEVER ): """Builds and pushes a set of docker images. Args: project: str, The project being deployed to. service: ServiceYamlInfo, The parsed service config. source_dir: str, path to the service's source directory version_id: The version id to deploy these services under. code_bucket_ref: The reference to the GCS bucket where the source will be uploaded. gcr_domain: str, Cloud Registry domain, determines the physical location of the image. E.g. `us.gcr.io`. runtime_builder_strategy: runtime_builders.RuntimeBuilderStrategy, whether to use the new CloudBuild-based runtime builders (alternative is old externalized runtimes). Returns: str, The name of the pushed container image. Raises: DockerfileError: if a Dockerfile is present, but the runtime is not "custom". NoDockerfileError: Raised if a user didn't supply a Dockerfile and chose a custom runtime. UnsatisfiedRequirementsError: Raised if the code in the directory doesn't satisfy the requirements of the specified runtime type. """ needs_dockerfile = _NeedsDockerfile(service, source_dir) use_runtime_builders = runtime_builder_strategy.ShouldUseRuntimeBuilders( service.runtime, needs_dockerfile) # Nothing to do if this is not an image-based deployment. if not service.RequiresImage(): return None log.status.Print( 'Building and pushing image for service [{service}]'.format( service=service.module)) gen_files = dict(_GetSourceContextsForUpload(source_dir)) if needs_dockerfile and not use_runtime_builders: # The runtime builders will generate a Dockerfile in the Cloud, so we only # need to do this if use_runtime_builders is True gen_files.update(_GetDockerfiles(service, source_dir)) image = docker_image.Image(dockerfile_dir=source_dir, repo=_GetImageName(project, service.module, version_id, gcr_domain), nocache=False, tag=config.DOCKER_IMAGE_TAG) metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD_START) object_ref = storage_util.ObjectReference(code_bucket_ref, image.tagged_repo) if files.IsDirAncestorOf(source_dir, service.file): relative_yaml_path = os.path.relpath(service.file, source_dir) else: yaml_contents = files.GetFileContents(service.file) checksum = files.Checksum().AddContents(yaml_contents).HexDigest() relative_yaml_path = checksum + '.yaml' gen_files[relative_yaml_path] = yaml_contents try: cloud_build.UploadSource(image.dockerfile_dir, object_ref, gen_files=gen_files, skip_files=service.parsed.skip_files.regex) except (OSError, IOError) as err: if platforms.OperatingSystem.IsWindows(): if err.filename and len(err.filename) > _WINDOWS_MAX_PATH: raise WindowMaxPathError(err.filename) raise metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD) if use_runtime_builders: builder_reference = runtime_builders.FromServiceInfo( service, source_dir) log.info('Using runtime builder [%s]', builder_reference.build_file_uri) builder_reference.WarnIfDeprecated() yaml_path = posixpath.join(*relative_yaml_path.split(os.sep)) build = builder_reference.LoadCloudBuild({ '_OUTPUT_IMAGE': image.tagged_repo, '_GAE_APPLICATION_YAML_PATH': yaml_path }) # TODO(b/37542869) Remove this hack once the API can take the gs:// path # as a runtime name. service.runtime = builder_reference.runtime service.parsed.SetEffectiveRuntime(builder_reference.runtime) else: build = cloud_build.GetDefaultBuild(image.tagged_repo) metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE_START) cloudbuild_build.CloudBuildClient().ExecuteCloudBuild( cloud_build.FixUpBuild(build, object_ref), project=project) metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE) return image.tagged_repo
def BuildAndPushDockerImage( project, service, source_dir, version_id, code_bucket_ref, gcr_domain, runtime_builder_strategy=runtime_builders.RuntimeBuilderStrategy.NEVER ): """Builds and pushes a set of docker images. Args: project: str, The project being deployed to. service: ServiceYamlInfo, The parsed service config. source_dir: str, path to the service's source directory version_id: The version id to deploy these services under. code_bucket_ref: The reference to the GCS bucket where the source will be uploaded. gcr_domain: str, Cloud Registry domain, determines the physical location of the image. E.g. `us.gcr.io`. runtime_builder_strategy: runtime_builders.RuntimeBuilderStrategy, whether to use the new CloudBuild-based runtime builders (alternative is old externalized runtimes). Returns: str, The name of the pushed container image. Raises: DockerfileError: if a Dockerfile is present, but the runtime is not "custom". NoDockerfileError: Raised if a user didn't supply a Dockerfile and chose a custom runtime. UnsatisfiedRequirementsError: Raised if the code in the directory doesn't satisfy the requirements of the specified runtime type. """ use_runtime_builders = runtime_builder_strategy.ShouldUseRuntimeBuilders( service.runtime) # Nothing to do if this is not an image-based deployment. if not service.RequiresImage(): return None log.status.Print( 'Building and pushing image for service [{service}]'.format( service=service.module)) gen_files = dict(_GetSourceContextsForUpload(source_dir)) needs_dockerfile = _NeedsDockerfile(service, source_dir) if needs_dockerfile and not use_runtime_builders: # The runtime builders will generate a Dockerfile in the Cloud, so we only # need to do this if use_runtime_builders is True gen_files.update(_GetDockerfiles(service, source_dir)) image = docker_image.Image(dockerfile_dir=source_dir, repo=_GetImageName(project, service.module, version_id, gcr_domain), nocache=False, tag=config.DOCKER_IMAGE_TAG) object_ref = storage_util.ObjectReference(code_bucket_ref, image.tagged_repo) try: cloud_build.UploadSource(image.dockerfile_dir, object_ref, gen_files=gen_files, skip_files=service.parsed.skip_files.regex) except (OSError, IOError) as err: if platforms.OperatingSystem.IsWindows(): if err.filename and len(err.filename) > _WINDOWS_MAX_PATH: raise WindowMaxPathError(err.filename) raise metrics.CustomTimedEvent(metric_names.CLOUDBUILD_UPLOAD) if use_runtime_builders: builder_version = runtime_builders.RuntimeBuilderVersion.FromServiceInfo( service) build = builder_version.LoadCloudBuild( {'_OUTPUT_IMAGE': image.tagged_repo}) else: build = cloud_build.GetDefaultBuild(image.tagged_repo) cloudbuild_build.CloudBuildClient().ExecuteCloudBuild( cloud_build.FixUpBuild(build, object_ref), project=project) metrics.CustomTimedEvent(metric_names.CLOUDBUILD_EXECUTE) return image.tagged_repo