def Build(messages, async_, build_config, hide_logs=False, build_region=cloudbuild_util.DEFAULT_REGION): """Starts the build.""" log.debug('submitting build: ' + repr(build_config)) client = cloudbuild_util.GetClientInstance() parent_resource = resources.REGISTRY.Create( collection='cloudbuild.projects.locations', projectsId=properties.VALUES.core.project.GetOrFail(), locationsId=build_region) op = client.projects_locations_builds.Create( messages.CloudbuildProjectsLocationsBuildsCreateRequest( parent=parent_resource.RelativeName(), build=build_config)) json = encoding.MessageToJson(op.metadata) build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build # Need to set the default version to 'v1' build_ref = resources.REGISTRY.Parse( None, collection='cloudbuild.projects.locations.builds', api_version='v1', params={ 'projectsId': build.projectId, 'locationsId': build_region, 'buildsId': build.id, }) if not hide_logs: log.CreatedResource(build_ref) if build.logUrl: log.status.Print('Logs are available at [{log_url}].'.format( log_url=build.logUrl)) else: log.status.Print('Logs are available in the Cloud Console.') # If the command is run --async, we just print out a reference to the build. if async_: return build, op mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) # Otherwise, logs are streamed from GCS. with execution_utils.CtrlCSection(mash_handler): build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref) if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT: log.status.Print( 'Your build timed out. Use the [--timeout=DURATION] flag to change ' 'the timeout threshold.') if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: raise FailedBuildException(build) return build, op
def _RunBuild(name, arch, function_type, source): """Builds the Edge function image with Cloud Build. Args: name: str, name of the Edge Function arch: str, target architecture, should be one of 'x86-64', 'armhf', 'aarch64' function_type: str, type of function, should be one of 'on-demand', 'stream-processing' source: str, GCS URI to source archive object or local path to source directory Returns: Finished cloudbuild.Build message. build.results.images contains built image's name and digest Raises: FailedBuildException: If the build is completed and not 'SUCCESS' FunctionBuilderError: For invalid arguments """ client = cloudbuild_util.GetClientInstance() messages = cloudbuild_util.GetMessagesModule() build_config = _EdgeFunctionBuildMessage(name, arch, function_type, source) log.debug('submitting build: ' + repr(build_config)) # Start the build. op = client.projects_builds.Create( messages.CloudbuildProjectsBuildsCreateRequest( build=build_config, projectId=properties.VALUES.core.project.Get())) json = encoding.MessageToJson(op.metadata) build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build build_ref = resources.REGISTRY.Create( collection='cloudbuild.projects.builds', projectId=build.projectId, id=build.id) log.CreatedResource(build_ref) mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) # Stream logs from GCS. with execution_utils.CtrlCSection(mash_handler): build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref) if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: raise FailedBuildException(build) return build
def testCancelBuildHandler(self): self.mocked_cloudbuild_v1.projects_builds.Cancel.Expect( self.cloudbuild_v1_messages.CloudbuildProjectsBuildsCancelRequest( cancelBuildRequest=None, id='id', projectId='proj', ), response='ignored', enable_type_checking=False) build_ref = resources.REGISTRY.Create( collection='cloudbuild.projects.builds', projectId='proj', id='id') handler = execution.GetCancelBuildHandler(self.mocked_cloudbuild_v1, self.cloudbuild_v1_messages, build_ref) handler(None, None)
def Build(messages, async_, build_config): """Starts the build.""" log.debug('submitting build: ' + repr(build_config)) client = cloudbuild_util.GetClientInstance() op = client.projects_builds.Create( messages.CloudbuildProjectsBuildsCreateRequest( build=build_config, projectId=properties.VALUES.core.project.Get())) json = encoding.MessageToJson(op.metadata) build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build build_ref = resources.REGISTRY.Create( collection='cloudbuild.projects.builds', projectId=build.projectId, id=build.id) log.CreatedResource(build_ref) if build.logUrl: log.status.Print( 'Logs are available at [{log_url}].'.format(log_url=build.logUrl)) else: log.status.Print('Logs are available in the Cloud Console.') # If the command is run --async, we just print out a reference to the build. if async_: return build mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) # Otherwise, logs are streamed from GCS. with execution_utils.CtrlCSection(mash_handler): build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref) if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT: log.status.Print( 'Your build timed out. Use the [--timeout=DURATION] flag to change ' 'the timeout threshold.') if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: raise FailedBuildException(build) return build
def _RunCloudBuild(args, builder, build_args, build_tags=None, output_filter=None, log_location=None, backoff=lambda elapsed: 1): """Run a build with a specific builder on Google Cloud Builder. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. builder: path to builder image build_args: args to be sent to builder build_tags: tags to be attached to the build output_filter: A list of strings indicating what lines from the log should be output. Only lines that start with one of the strings in output_filter will be displayed. log_location: GCS path to directory where logs will be stored. backoff: A function that takes the current elapsed time and returns the next sleep length. Both are in seconds. Returns: A build object that either streams the output or is displayed as a link to the build. Raises: FailedBuildException: If the build is completed and not 'SUCCESS'. """ client = cloudbuild_util.GetClientInstance() messages = cloudbuild_util.GetMessagesModule() # Create the build request. build_config = messages.Build( steps=[ messages.BuildStep( name=builder, args=build_args, ), ], tags=build_tags, timeout='{0}s'.format(args.timeout), ) if log_location: gcs_log_dir = resources.REGISTRY.Parse(args.log_location, collection='storage.objects') build_config.logsBucket = ('gs://{0}/{1}'.format( gcs_log_dir.bucket, gcs_log_dir.object)) # Start the build. build, build_ref = _CreateCloudBuild(build_config, client, messages) # If the command is run --async, we just print out a reference to the build. if args.async_: return build mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) # Otherwise, logs are streamed from GCS. with execution_utils.CtrlCSection(mash_handler): build = CloudBuildClientWithFiltering( client, messages).StreamWithFilter(build_ref, backoff, output_filter=output_filter) if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT: log.status.Print( 'Your build timed out. Use the [--timeout=DURATION] flag to change ' 'the timeout threshold.') if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: raise FailedBuildException(build) return build
def _SubmitBuild(self, client, messages, build_config, gcs_config_staging_path, suggest_configs, async_): """Submits the build. Args: client: Client used to make calls to Cloud Build API. messages: Cloud Build messages module. This is the value returned from cloudbuild_util.GetMessagesModule(). build_config: Build to submit. gcs_config_staging_path: A path to a GCS subdirectory where deployed configs will be saved to. This value will be printed to the user. suggest_configs: If True, suggest YAML configs for the user to add to their repo. async_: If true, exit immediately after submitting Build, rather than waiting for it to complete or fail. Raises: FailedDeployException: If the build is completed and not 'SUCCESS'. """ project = properties.VALUES.core.project.Get(required=True) op = client.projects_builds.Create( messages.CloudbuildProjectsBuildsCreateRequest(build=build_config, projectId=project)) log.debug('submitting build: ' + six.text_type(build_config)) json = encoding.MessageToJson(op.metadata) build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build build_ref = resources.REGISTRY.Create( collection='cloudbuild.projects.builds', projectId=build.projectId, id=build.id) log.status.Print( 'Starting Cloud Build to build and deploy to the target ' 'Google Kubernetes Engine cluster...\n') log.CreatedResource(build_ref) if build.logUrl: log.status.Print('Logs are available at [{log_url}].'.format( log_url=build.logUrl)) else: log.status.Print('Logs are available in the Cloud Console.') suggested_configs_path = build_util.SuggestedConfigsPath( gcs_config_staging_path, build.id) expanded_configs_path = build_util.ExpandedConfigsPath( gcs_config_staging_path, build.id) if async_: log.status.Print( '\nIf successful, you can find the configuration files of the deployed ' 'Kubernetes objects stored at gs://{expanded} or by visiting ' 'https://console.cloud.google.com/storage/browser/{expanded}/.' .format(expanded=expanded_configs_path)) if suggest_configs: log.status.Print( '\nYou will also be able to find the suggested base Kubernetes ' 'configuration files at gs://{suggested} or by visiting ' 'https://console.cloud.google.com/storage/browser/{suggested}/.' .format(suggested=suggested_configs_path)) # Return here, otherwise, logs are streamed from GCS. return mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) with execution_utils.CtrlCSection(mash_handler): build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref) if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT: log.status.Print( 'Your build and deploy timed out. Use the [--timeout=DURATION] flag ' 'to change the timeout threshold.') if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: if build_util.SaveConfigsBuildStepIsSuccessful(messages, build): log.status.Print( 'You can find the configuration files for this attempt at gs://{}.' .format(expanded_configs_path)) raise FailedDeployException(build) log.status.Print( 'Successfully deployed to your Google Kubernetes Engine cluster.\n\n' 'You can find the configuration files of the deployed Kubernetes ' 'objects stored at gs://{expanded} or by visiting ' 'https://console.cloud.google.com/storage/browser/{expanded}/.'. format(expanded=expanded_configs_path)) if suggest_configs: log.status.Print( '\nYou can also find suggested base Kubernetes configuration files at ' 'gs://{suggested} or by visiting ' 'https://console.cloud.google.com/storage/browser/{suggested}/.' .format(suggested=suggested_configs_path))
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. Raises: FailedBuildException: If the build is completed and not 'SUCCESS'. """ project = properties.VALUES.core.project.Get(required=True) safe_project = project.replace(':', '_') safe_project = safe_project.replace('.', '_') # The string 'google' is not allowed in bucket names. safe_project = safe_project.replace('google', 'elgoog') default_bucket_name = '{}_cloudbuild'.format(safe_project) default_gcs_source = False if args.gcs_source_staging_dir is None: default_gcs_source = True args.gcs_source_staging_dir = 'gs://{}/source'.format(default_bucket_name) client = cloudbuild_util.GetClientInstance() messages = cloudbuild_util.GetMessagesModule() gcs_client = storage_api.StorageClient() # First, create the build request. build_timeout = properties.VALUES.builds.timeout.Get() if build_timeout is not None: try: # A bare number is interpreted as seconds. build_timeout_secs = int(build_timeout) except ValueError: build_timeout_duration = times.ParseDuration(build_timeout) build_timeout_secs = int(build_timeout_duration.total_seconds) timeout_str = six.text_type(build_timeout_secs) + 's' else: timeout_str = None if args.tag is not None: if (properties.VALUES.builds.check_tag.GetBool() and 'gcr.io/' not in args.tag): raise c_exceptions.InvalidArgumentException( '--tag', 'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.') if properties.VALUES.builds.use_kaniko.GetBool(): if args.no_cache: ttl = '0h' else: ttl = '{}h'.format(properties.VALUES.builds.kaniko_cache_ttl.Get()) build_config = messages.Build( steps=[ messages.BuildStep( name=properties.VALUES.builds.kaniko_image.Get(), args=[ '--destination', args.tag, '--cache', 'true', '--cache-ttl', ttl ], ), ], timeout=timeout_str, substitutions=cloudbuild_util.EncodeSubstitutions( args.substitutions, messages)) else: if args.no_cache: raise c_exceptions.InvalidArgumentException( 'no-cache', 'Cannot specify --no-cache if builds/use_kaniko property is ' 'False') build_config = messages.Build( images=[args.tag], steps=[ messages.BuildStep( name='gcr.io/cloud-builders/docker', args=[ 'build', '--network', 'cloudbuild', '--no-cache', '-t', args.tag, '.' ], ), ], timeout=timeout_str, substitutions=cloudbuild_util.EncodeSubstitutions( args.substitutions, messages)) elif args.config is not None: if args.no_cache: raise c_exceptions.ConflictingArgumentsException( '--config', '--no-cache') if not args.config: raise c_exceptions.InvalidArgumentException( '--config', 'Config file path must not be empty.') build_config = config.LoadCloudbuildConfigFromPath( args.config, messages, params=args.substitutions) else: raise c_exceptions.OneOfArgumentsRequiredException( ['--tag', '--config'], 'Requires either a docker tag or a config file.') # If timeout was set by flag, overwrite the config file. if timeout_str: build_config.timeout = timeout_str # --no-source overrides the default --source. if not args.IsSpecified('source') and args.no_source: args.source = None gcs_source_staging = None if args.source: suffix = '.tgz' if args.source.startswith('gs://') or os.path.isfile(args.source): _, suffix = os.path.splitext(args.source) # Next, stage the source to Cloud Storage. staged_object = '{stamp}-{uuid}{suffix}'.format( stamp=times.GetTimeStampFromDateTime(times.Now()), uuid=uuid.uuid4().hex, suffix=suffix, ) gcs_source_staging_dir = resources.REGISTRY.Parse( args.gcs_source_staging_dir, collection='storage.objects') # We create the bucket (if it does not exist) first. If we do an existence # check and then create the bucket ourselves, it would be possible for an # attacker to get lucky and beat us to creating the bucket. Block on this # creation to avoid this race condition. gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket) # If no bucket is specified (for the source `default_gcs_source`), check # that the default bucket is also owned by the project (b/33046325). if default_gcs_source: # This request returns only the buckets owned by the project. bucket_list_req = gcs_client.messages.StorageBucketsListRequest( project=project, prefix=default_bucket_name) bucket_list = gcs_client.client.buckets.List(bucket_list_req) found_bucket = False for bucket in bucket_list.items: if bucket.id == default_bucket_name: found_bucket = True break if not found_bucket: if default_gcs_source: raise c_exceptions.RequiredArgumentException( 'gcs_source_staging_dir', 'A bucket with name {} already exists and is owned by ' 'another project. Specify a bucket using ' '--gcs_source_staging_dir.'.format(default_bucket_name)) if gcs_source_staging_dir.object: staged_object = gcs_source_staging_dir.object + '/' + staged_object gcs_source_staging = resources.REGISTRY.Create( collection='storage.objects', bucket=gcs_source_staging_dir.bucket, object=staged_object) if args.source.startswith('gs://'): gcs_source = resources.REGISTRY.Parse( args.source, collection='storage.objects') staged_source_obj = gcs_client.Rewrite(gcs_source, gcs_source_staging) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) else: if not os.path.exists(args.source): raise c_exceptions.BadFileException( 'could not find source [{src}]'.format(src=args.source)) if os.path.isdir(args.source): source_snapshot = snapshot.Snapshot(args.source, ignore_file=args.ignore_file) size_str = resource_transform.TransformSize( source_snapshot.uncompressed_size) log.status.Print( 'Creating temporary tarball archive of {num_files} file(s)' ' totalling {size} before compression.'.format( num_files=len(source_snapshot.files), size=size_str)) staged_source_obj = source_snapshot.CopyTarballToGCS( gcs_client, gcs_source_staging, ignore_file=args.ignore_file) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) elif os.path.isfile(args.source): unused_root, ext = os.path.splitext(args.source) if ext not in _ALLOWED_SOURCE_EXT: raise c_exceptions.BadFileException( 'Local file [{src}] is none of ' + ', '.join(_ALLOWED_SOURCE_EXT)) log.status.Print('Uploading local file [{src}] to ' '[gs://{bucket}/{object}].'.format( src=args.source, bucket=gcs_source_staging.bucket, object=gcs_source_staging.object, )) staged_source_obj = gcs_client.CopyFileToGCS(args.source, gcs_source_staging) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) else: # No source if not args.no_source: raise c_exceptions.InvalidArgumentException( '--no-source', 'To omit source, use the --no-source flag.') if args.gcs_log_dir: gcs_log_dir = resources.REGISTRY.Parse( args.gcs_log_dir, collection='storage.objects') build_config.logsBucket = ('gs://' + gcs_log_dir.bucket + '/' + gcs_log_dir.object) # Machine type. if args.machine_type is not None: machine_type = Submit._machine_type_flag_map.GetEnumForChoice( args.machine_type) if not build_config.options: build_config.options = messages.BuildOptions() build_config.options.machineType = machine_type # Disk size. if args.disk_size is not None: disk_size = compute_utils.BytesToGb(args.disk_size) if not build_config.options: build_config.options = messages.BuildOptions() build_config.options.diskSizeGb = int(disk_size) log.debug('submitting build: ' + repr(build_config)) # Start the build. op = client.projects_builds.Create( messages.CloudbuildProjectsBuildsCreateRequest( build=build_config, projectId=properties.VALUES.core.project.Get())) json = encoding.MessageToJson(op.metadata) build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build build_ref = resources.REGISTRY.Create( collection='cloudbuild.projects.builds', projectId=build.projectId, id=build.id) log.CreatedResource(build_ref) if build.logUrl: log.status.Print( 'Logs are available at [{log_url}].'.format(log_url=build.logUrl)) else: log.status.Print('Logs are available in the Cloud Console.') # If the command is run --async, we just print out a reference to the build. if args.async: return build mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) # Otherwise, logs are streamed from GCS. with execution_utils.CtrlCSection(mash_handler): build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref) if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT: log.status.Print( 'Your build timed out. Use the [--timeout=DURATION] flag to change ' 'the timeout threshold.') if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: raise FailedBuildException(build) return build
def RunDaisyBuild(args, workflow, variables, daisy_bucket=None, tags=None, user_zone=None, output_filter=None): """Run a build with Daisy on Google Cloud Builder. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. workflow: The path to the Daisy workflow to run. variables: A string of key-value pairs to pass to Daisy. daisy_bucket: A string containing the name of the GCS bucket that daisy should use. tags: A list of strings for adding tags to the Argo build. user_zone: The GCP zone to tell Daisy to do work in. If unspecified, defaults to wherever the Argo runner happens to be. output_filter: A list of strings indicating what lines from the log should be output. Only lines that start with one of the strings in output_filter will be displayed. Returns: A build object that either streams the output or is displayed as a link to the build. Raises: FailedBuildException: If the build is completed and not 'SUCCESS'. """ client = cloudbuild_util.GetClientInstance() messages = cloudbuild_util.GetMessagesModule() project_id = projects_util.ParseProject( properties.VALUES.core.project.GetOrFail()) CheckIamPermissions(project_id) # Make Daisy time out before gcloud by shaving off 2% from the timeout time, # up to a max of 5m (300s). two_percent = int(args.timeout * 0.02) daisy_timeout = args.timeout - min(two_percent, 300) daisy_bucket = daisy_bucket or GetAndCreateDaisyBucket() daisy_args = [ '-gcs_path=gs://{0}/'.format(daisy_bucket), '-default_timeout={0}s'.format(daisy_timeout), '-variables={0}'.format(variables), workflow, ] if user_zone is not None: daisy_args = ['-zone={0}'.format(user_zone)] + daisy_args build_tags = ['gce-daisy'] if tags: build_tags.extend(tags) # First, create the build request. build_config = messages.Build( steps=[ messages.BuildStep( name=_BUILDER, args=daisy_args, ), ], tags=build_tags, timeout='{0}s'.format(args.timeout), ) if args.log_location: gcs_log_dir = resources.REGISTRY.Parse(args.log_location, collection='storage.objects') build_config.logsBucket = ('gs://{0}/{1}'.format( gcs_log_dir.bucket, gcs_log_dir.object)) # Start the build. build, build_ref = _CreateCloudBuild(build_config, client, messages) # If the command is run --async, we just print out a reference to the build. if args. async: return build mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) # Otherwise, logs are streamed from GCS. with execution_utils.CtrlCSection(mash_handler): build = DaisyCloudBuildClient(client, messages).StreamWithFilter( build_ref, output_filter=output_filter) if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT: log.status.Print( 'Your build timed out. Use the [--timeout=DURATION] flag to change ' 'the timeout threshold.') if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: raise FailedBuildException(build) return build
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. Raises: FailedBuildException: If the build is completed and not 'SUCCESS'. """ project = properties.VALUES.core.project.Get() safe_project = project.replace(':', '_') safe_project = safe_project.replace('.', '_') # The string 'google' is not allowed in bucket names. safe_project = safe_project.replace('google', 'elgoog') default_bucket_name = '{}_cloudbuild'.format(safe_project) default_gcs_source = False if args.gcs_source_staging_dir is None: default_gcs_source = True args.gcs_source_staging_dir = 'gs://{}/source'.format(default_bucket_name) default_gcs_log_dir = False if args.gcs_log_dir is None: default_gcs_log_dir = True args.gcs_log_dir = 'gs://{}/logs'.format(default_bucket_name) client = cloudbuild_util.GetClientInstance() messages = cloudbuild_util.GetMessagesModule() gcs_client = storage_api.StorageClient() # First, create the build request. build_timeout = properties.VALUES.container.build_timeout.Get() if build_timeout is not None: try: # A bare number is interpreted as seconds. build_timeout_secs = int(build_timeout) except ValueError: build_timeout_duration = times.ParseDuration(build_timeout) build_timeout_secs = int(build_timeout_duration.total_seconds) timeout_str = str(build_timeout_secs) + 's' else: timeout_str = None if args.tag: if 'gcr.io/' not in args.tag: raise c_exceptions.InvalidArgumentException( '--tag', 'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.') build_config = messages.Build( images=[args.tag], steps=[ messages.BuildStep( name='gcr.io/cloud-builders/docker', args=['build', '--no-cache', '-t', args.tag, '.'], ), ], timeout=timeout_str, substitutions=cloudbuild_util.EncodeSubstitutions(args.substitutions, messages) ) elif args.config: build_config = config.LoadCloudbuildConfigFromPath( args.config, messages, params=args.substitutions) # If timeout was set by flag, overwrite the config file. if timeout_str: build_config.timeout = timeout_str suffix = '.tgz' if args.source.startswith('gs://') or os.path.isfile(args.source): _, suffix = os.path.splitext(args.source) # Next, stage the source to Cloud Storage. staged_object = '{stamp}{suffix}'.format( stamp=times.GetTimeStampFromDateTime(times.Now()), suffix=suffix, ) gcs_source_staging_dir = resources.REGISTRY.Parse( args.gcs_source_staging_dir, collection='storage.objects') # We first try to create the bucket, before doing all the checks, in order # to avoid a race condition. If we do the check first, an attacker could # be lucky enough to create the bucket after the check and before this # bucket creation. gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket) # If no bucket is specified (for the source `default_gcs_source` or for the # logs `default_gcs_log_dir`), check that the default bucket is also owned # by the project (b/33046325). if default_gcs_source or default_gcs_log_dir: # This request returns only the buckets owned by the project. bucket_list_req = gcs_client.messages.StorageBucketsListRequest( project=project, prefix=default_bucket_name) bucket_list = gcs_client.client.buckets.List(bucket_list_req) found_bucket = False for bucket in bucket_list.items: if bucket.id == default_bucket_name: found_bucket = True break if not found_bucket: if default_gcs_source: raise c_exceptions.RequiredArgumentException( 'gcs_source_staging_dir', 'A bucket with name {} already exists and is owned by ' 'another project. Specify a bucket using ' '--gcs_source_staging_dir.'.format(default_bucket_name)) elif default_gcs_log_dir: raise c_exceptions.RequiredArgumentException( 'gcs-log-dir', 'A bucket with name {} already exists and is owned by ' 'another project. Specify a bucket to hold build logs ' 'using --gcs-log-dir.'.format(default_bucket_name)) if gcs_source_staging_dir.object: staged_object = gcs_source_staging_dir.object + '/' + staged_object gcs_source_staging = resources.REGISTRY.Create( collection='storage.objects', bucket=gcs_source_staging_dir.bucket, object=staged_object) if args.source.startswith('gs://'): gcs_source = resources.REGISTRY.Parse( args.source, collection='storage.objects') staged_source_obj = gcs_client.Rewrite(gcs_source, gcs_source_staging) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) else: if not os.path.exists(args.source): raise c_exceptions.BadFileException( 'could not find source [{src}]'.format(src=args.source)) if os.path.isdir(args.source): source_snapshot = snapshot.Snapshot(args.source) size_str = resource_transform.TransformSize( source_snapshot.uncompressed_size) log.status.Print( 'Creating temporary tarball archive of {num_files} file(s)' ' totalling {size} before compression.'.format( num_files=len(source_snapshot.files), size=size_str)) staged_source_obj = source_snapshot.CopyTarballToGCS( gcs_client, gcs_source_staging) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) elif os.path.isfile(args.source): unused_root, ext = os.path.splitext(args.source) if ext not in _ALLOWED_SOURCE_EXT: raise c_exceptions.BadFileException( 'Local file [{src}] is none of '+', '.join(_ALLOWED_SOURCE_EXT)) log.status.Print( 'Uploading local file [{src}] to ' '[gs://{bucket}/{object}].'.format( src=args.source, bucket=gcs_source_staging.bucket, object=gcs_source_staging.object, )) staged_source_obj = gcs_client.CopyFileToGCS( storage_util.BucketReference.FromBucketUrl( gcs_source_staging.bucket), args.source, gcs_source_staging.object) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) gcs_log_dir = resources.REGISTRY.Parse( args.gcs_log_dir, collection='storage.objects') if gcs_log_dir.bucket != gcs_source_staging.bucket: # Create the logs bucket if it does not yet exist. gcs_client.CreateBucketIfNotExists(gcs_log_dir.bucket) build_config.logsBucket = 'gs://'+gcs_log_dir.bucket+'/'+gcs_log_dir.object log.debug('submitting build: '+repr(build_config)) # Start the build. op = client.projects_builds.Create( messages.CloudbuildProjectsBuildsCreateRequest( build=build_config, projectId=properties.VALUES.core.project.Get())) json = encoding.MessageToJson(op.metadata) build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build build_ref = resources.REGISTRY.Create( collection='cloudbuild.projects.builds', projectId=build.projectId, id=build.id) log.CreatedResource(build_ref) if build.logUrl: log.status.Print('Logs are available at [{log_url}].'.format( log_url=build.logUrl)) else: log.status.Print('Logs are available in the Cloud Console.') # If the command is run --async, we just print out a reference to the build. if args.async: return build mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) # Otherwise, logs are streamed from GCS. with execution_utils.CtrlCSection(mash_handler): build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref) if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT: log.status.Print( 'Your build timed out. Use the [--timeout=DURATION] flag to change ' 'the timeout threshold.') if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: raise FailedBuildException(build) return build
def RunDaisyBuild(args, workflow, variables): """Run a build with Daisy on Google Cloud Builder. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. workflow: The path to the Daisy workflow to run. variables: A string of key-value pairs to pass to Daisy. Returns: A build object that either streams the output or is displayed as a link to the build. Raises: FailedBuildException: If the build is completed and not 'SUCCESS'. """ client = cloudbuild_util.GetClientInstance() messages = cloudbuild_util.GetMessagesModule() project_id = projects_util.ParseProject( properties.VALUES.core.project.GetOrFail()) CheckIamPermissions(project_id) timeout_str = '{0}s'.format(args.timeout) # First, create the build request. build_config = messages.Build( steps=[ messages.BuildStep( name=_BUILDER, args=[ '-variables={0}'.format(variables), workflow, ], ), ], timeout=timeout_str, ) if args.log_location: gcs_log_dir = resources.REGISTRY.Parse(args.log_location, collection='storage.objects') build_config.logsBucket = ('gs://{0}/{1}'.format( gcs_log_dir.bucket, gcs_log_dir.object)) # Start the build. build, build_ref = _CreateCloudBuild(build_config, client, messages) # If the command is run --async, we just print out a reference to the build. if args. async: return build mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) # Otherwise, logs are streamed from GCS. with execution_utils.CtrlCSection(mash_handler): build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref) if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT: log.status.Print( 'Your build timed out. Use the [--timeout=DURATION] flag to change ' 'the timeout threshold.') if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: raise FailedBuildException(build) return build
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. Raises: FailedDeployException: If the build is completed and not 'SUCCESS'. """ client = cloudbuild_util.GetClientInstance() messages = cloudbuild_util.GetMessagesModule() build_config = self._CreateBuildFromArgs(args, messages) # Start the build project = properties.VALUES.core.project.Get(required=True) op = client.projects_builds.Create( messages.CloudbuildProjectsBuildsCreateRequest(build=build_config, projectId=project)) log.debug('submitting build: ' + repr(build_config)) json = encoding.MessageToJson(op.metadata) build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build build_ref = resources.REGISTRY.Create( collection='cloudbuild.projects.builds', projectId=build.projectId, id=build.id) log.status.Print( 'Starting Cloud Build to build and deploy to the target ' 'Google Kubernetes Engine cluster...\n') log.CreatedResource(build_ref) if build.logUrl: log.status.Print('Logs are available at [{log_url}].'.format( log_url=build.logUrl)) else: log.status.Print('Logs are available in the Cloud Console.') if args.async_: return mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) # Otherwise, logs are streamed from GCS. with execution_utils.CtrlCSection(mash_handler): build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref) if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT: log.status.Print( 'Your build and deploy timed out. Use the [--timeout=DURATION] flag ' 'to change the timeout threshold.') if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: raise FailedDeployException(build) # Get location of suggested config bucket from gsutil build step suggested_configs = build.steps[-1].args[-1] log.status.Print( 'Successfully deployed to your Google Kubernetes Engine cluster.\n\n' 'You can find the configuration files of the deployed Kubernetes ' 'objects stored at {expanded} These configurations are expanded by the ' 'deployer with additional labels like app name and version.\n\n' 'You can also find suggested base Kubernetes configuration files ' 'created by the deployer at {suggested}'.format( expanded=build.artifacts.objects.location, suggested=suggested_configs)) return
def Build(messages, async_, build_config, hide_logs=False, build_region=cloudbuild_util.DEFAULT_REGION, support_gcl=False, suppress_logs=False): """Starts the build.""" log.debug('submitting build: ' + repr(build_config)) client = cloudbuild_util.GetClientInstance() parent_resource = resources.REGISTRY.Create( collection='cloudbuild.projects.locations', projectsId=properties.VALUES.core.project.GetOrFail(), locationsId=build_region) op = client.projects_locations_builds.Create( messages.CloudbuildProjectsLocationsBuildsCreateRequest( parent=parent_resource.RelativeName(), build=build_config)) json = encoding.MessageToJson(op.metadata) build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build # Need to set the default version to 'v1' build_ref = resources.REGISTRY.Parse( None, collection='cloudbuild.projects.locations.builds', api_version='v1', params={ 'projectsId': build.projectId, 'locationsId': build_region, 'buildsId': build.id, }) if not hide_logs: log.CreatedResource(build_ref) if build.logUrl: log.status.Print( 'Logs are available at [{log_url}].'.format(log_url=build.logUrl)) else: log.status.Print('Logs are available in the Cloud Console.') # If the command is run --async, we just print out a reference to the build. if async_: return build, op if not support_gcl and build.options and build.options.logging in [ messages.BuildOptions.LoggingValueValuesEnum.STACKDRIVER_ONLY, messages.BuildOptions.LoggingValueValuesEnum.CLOUD_LOGGING_ONLY, ]: log.status.Print('\ngcloud builds submit only displays logs from Cloud' ' Storage. To view logs from Cloud Logging, run:\ngcloud' ' beta builds submit\n') mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) out = log.out if not suppress_logs else None # Otherwise, logs are streamed from the chosen logging service # (defaulted to GCS). with execution_utils.CtrlCSection(mash_handler): build = cb_logs.CloudBuildClient(client, messages, support_gcl).Stream(build_ref, out) if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT: log.status.Print( 'Your build timed out. Use the [--timeout=DURATION] flag to change ' 'the timeout threshold.') if build.warnings: for warn in build.warnings: log.status.Print('\n{priority}: {text}'.format( text=warn.text, priority=warn.priority)) log.status.Print( '\n{count} message(s) issued.'.format(count=len(build.warnings))) if build.failureInfo: log.status.Print( '\nBUILD FAILURE: {detail}'.format(detail=build.failureInfo.detail)) if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: raise FailedBuildException(build) return build, op