def Build(messages,
async_,
build_config,
hide_logs=False,
build_region=cloudbuild_util.DEFAULT_REGION):
"""Starts the build."""
log.debug('submitting build: ' + repr(build_config))
client = cloudbuild_util.GetClientInstance()
parent_resource = resources.REGISTRY.Create(
collection='cloudbuild.projects.locations',
projectsId=properties.VALUES.core.project.GetOrFail(),
locationsId=build_region)
op = client.projects_locations_builds.Create(
messages.CloudbuildProjectsLocationsBuildsCreateRequest(
parent=parent_resource.RelativeName(), build=build_config))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build
# Need to set the default version to 'v1'
build_ref = resources.REGISTRY.Parse(
None,
collection='cloudbuild.projects.locations.builds',
api_version='v1',
params={
'projectsId': build.projectId,
'locationsId': build_region,
'buildsId': build.id,
})
if not hide_logs:
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print('Logs are available at [{log_url}].'.format(
log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
# If the command is run --async, we just print out a reference to the build.
if async_:
return build, op
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build, op
def WaitFor(poller, operation_ref, message):
"""Waits with retrues for operation to be done given poller.
Args:
poller: OperationPoller, poller to use during retrials.
operation_ref: object, passed to operation poller poll method.
message: str, string to display for progrss_tracker.
Returns:
poller.GetResult(operation).
Raises:
AbortWaitError: if ctrl-c was pressed.
TimeoutError: if retryer has finished wihout being done.
"""
def _CtrlCHandler(unused_signal, unused_frame):
raise AbortWaitError('Ctrl-C aborted wait.')
try:
with execution_utils.CtrlCSection(_CtrlCHandler):
try:
with progress_tracker.ProgressTracker(message) as tracker:
if poller.PRE_START_SLEEP_MS:
_SleepMs(poller.PRE_START_SLEEP_MS)
def _StatusUpdate(unused_result, unused_status):
tracker.Tick()
retryer = retry.Retryer(
max_retrials=poller.MAX_RETRIALS,
max_wait_ms=poller.MAX_WAIT_MS,
exponential_sleep_multiplier=poller.
EXPONENTIAL_SLEEP_MULTIPLIER,
jitter_ms=poller.JITTER_MS,
status_update_func=_StatusUpdate)
def _IsNotDone(operation, unused_state):
return not poller.IsDone(operation)
operation = retryer.RetryOnResult(
func=poller.Poll,
args=(operation_ref, ),
should_retry_if=_IsNotDone,
sleep_ms=poller.SLEEP_MS)
except retry.RetryException:
raise TimeoutError(
'Operation {0} has not finished in {1} seconds'.format(
operation_ref, int(poller.MAX_WAIT_MS / 1000)))
except AbortWaitError:
# Write this out now that progress tracker is done.
sys.stderr.write(
'Aborting wait for operation {0}.\n'.format(operation_ref))
raise
return poller.GetResult(operation)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
# TODO(b/33234717): remove this after deprecation period
flags.ProcessPackages(args)
region = properties.VALUES.compute.region.Get(required=True)
uris = jobs_prep.RunSetupAndUpload(args.packages, args.staging_bucket,
args.package_path, args.job)
log.debug('Using {0} as trainer uris'.format(uris))
job = jobs.BuildTrainingJob(path=args.config,
module_name=args.module_name,
job_name=args.job,
trainer_uri=uris,
region=region,
user_args=args.user_args)
job = jobs.Create(job)
if args. async:
log.status.Print('Job [{}] submitted successfully.'.format(
job.jobId))
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
return job
log_fetcher = jobs.LogFetcher(job_id=job.jobId,
polling_interval=_POLLING_INTERVAL,
allow_multiline_logs=False)
printer = resource_printer.Printer(jobs.LogFetcher.LOG_FORMAT,
out=log.err)
def _CtrlCHandler(signal, frame):
del signal, frame # Unused
raise KeyboardInterrupt
with execution_utils.CtrlCSection(_CtrlCHandler):
try:
printer.Print(log_fetcher.YieldLogs())
except KeyboardInterrupt:
log.status.Print('Received keyboard interrupt.')
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
job = jobs.Get(job.jobId)
# If the job itself failed, we will return a failure status.
if job.state is not job.StateValueValuesEnum.SUCCEEDED:
self.exit_code = 1
return job
def _RunBuild(name, arch, function_type, source):
"""Builds the Edge function image with Cloud Build.
Args:
name: str, name of the Edge Function
arch: str, target architecture,
should be one of 'x86-64', 'armhf', 'aarch64'
function_type: str, type of function,
should be one of 'on-demand', 'stream-processing'
source: str, GCS URI to source archive object or
local path to source directory
Returns:
Finished cloudbuild.Build message. build.results.images contains
built image's name and digest
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'
FunctionBuilderError: For invalid arguments
"""
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
build_config = _EdgeFunctionBuildMessage(name, arch, function_type, source)
log.debug('submitting build: ' + repr(build_config))
# Start the build.
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(
build=build_config,
projectId=properties.VALUES.core.project.Get()))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build
build_ref = resources.REGISTRY.Create(
collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.CreatedResource(build_ref)
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Stream logs from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def Build(messages, async_, build_config):
"""Starts the build."""
log.debug('submitting build: ' + repr(build_config))
client = cloudbuild_util.GetClientInstance()
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(
build=build_config,
projectId=properties.VALUES.core.project.Get()))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build
build_ref = resources.REGISTRY.Create(
collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print(
'Logs are available at [{log_url}].'.format(log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
# If the command is run --async, we just print out a reference to the build.
if async_:
return build
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def _RunCloudBuild(args,
builder,
build_args,
build_tags=None,
output_filter=None,
log_location=None,
backoff=lambda elapsed: 1):
"""Run a build with a specific builder on Google Cloud Builder.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
builder: path to builder image
build_args: args to be sent to builder
build_tags: tags to be attached to the build
output_filter: A list of strings indicating what lines from the log should
be output. Only lines that start with one of the strings in output_filter
will be displayed.
log_location: GCS path to directory where logs will be stored.
backoff: A function that takes the current elapsed time and returns
the next sleep length. Both are in seconds.
Returns:
A build object that either streams the output or is displayed as a
link to the build.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
# Create the build request.
build_config = messages.Build(
steps=[
messages.BuildStep(
name=builder,
args=build_args,
),
],
tags=build_tags,
timeout='{0}s'.format(args.timeout),
)
if log_location:
gcs_log_dir = resources.REGISTRY.Parse(args.log_location,
collection='storage.objects')
build_config.logsBucket = ('gs://{0}/{1}'.format(
gcs_log_dir.bucket, gcs_log_dir.object))
# Start the build.
build, build_ref = _CreateCloudBuild(build_config, client, messages)
# If the command is run --async, we just print out a reference to the build.
if args.async_:
return build
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = CloudBuildClientWithFiltering(
client, messages).StreamWithFilter(build_ref,
backoff,
output_filter=output_filter)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def _SubmitBuild(self, client, messages, build_config,
gcs_config_staging_path, suggest_configs, async_):
"""Submits the build.
Args:
client: Client used to make calls to Cloud Build API.
messages: Cloud Build messages module. This is the value returned from
cloudbuild_util.GetMessagesModule().
build_config: Build to submit.
gcs_config_staging_path: A path to a GCS subdirectory where deployed
configs will be saved to. This value will be printed to the user.
suggest_configs: If True, suggest YAML configs for the user to add to
their repo.
async_: If true, exit immediately after submitting Build, rather than
waiting for it to complete or fail.
Raises:
FailedDeployException: If the build is completed and not 'SUCCESS'.
"""
project = properties.VALUES.core.project.Get(required=True)
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(build=build_config,
projectId=project))
log.debug('submitting build: ' + six.text_type(build_config))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata,
json).build
build_ref = resources.REGISTRY.Create(
collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.status.Print(
'Starting Cloud Build to build and deploy to the target '
'Google Kubernetes Engine cluster...\n')
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print('Logs are available at [{log_url}].'.format(
log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
suggested_configs_path = build_util.SuggestedConfigsPath(
gcs_config_staging_path, build.id)
expanded_configs_path = build_util.ExpandedConfigsPath(
gcs_config_staging_path, build.id)
if async_:
log.status.Print(
'\nIf successful, you can find the configuration files of the deployed '
'Kubernetes objects stored at gs://{expanded} or by visiting '
'https://console.cloud.google.com/storage/browser/{expanded}/.'
.format(expanded=expanded_configs_path))
if suggest_configs:
log.status.Print(
'\nYou will also be able to find the suggested base Kubernetes '
'configuration files at gs://{suggested} or by visiting '
'https://console.cloud.google.com/storage/browser/{suggested}/.'
.format(suggested=suggested_configs_path))
# Return here, otherwise, logs are streamed from GCS.
return
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client,
messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build and deploy timed out. Use the [--timeout=DURATION] flag '
'to change the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
if build_util.SaveConfigsBuildStepIsSuccessful(messages, build):
log.status.Print(
'You can find the configuration files for this attempt at gs://{}.'
.format(expanded_configs_path))
raise FailedDeployException(build)
log.status.Print(
'Successfully deployed to your Google Kubernetes Engine cluster.\n\n'
'You can find the configuration files of the deployed Kubernetes '
'objects stored at gs://{expanded} or by visiting '
'https://console.cloud.google.com/storage/browser/{expanded}/.'.
format(expanded=expanded_configs_path))
if suggest_configs:
log.status.Print(
'\nYou can also find suggested base Kubernetes configuration files at '
'gs://{suggested} or by visiting '
'https://console.cloud.google.com/storage/browser/{suggested}/.'
.format(suggested=suggested_configs_path))
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
project = properties.VALUES.core.project.Get(required=True)
safe_project = project.replace(':', '_')
safe_project = safe_project.replace('.', '_')
# The string 'google' is not allowed in bucket names.
safe_project = safe_project.replace('google', 'elgoog')
default_bucket_name = '{}_cloudbuild'.format(safe_project)
default_gcs_source = False
if args.gcs_source_staging_dir is None:
default_gcs_source = True
args.gcs_source_staging_dir = 'gs://{}/source'.format(default_bucket_name)
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
gcs_client = storage_api.StorageClient()
# First, create the build request.
build_timeout = properties.VALUES.builds.timeout.Get()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
timeout_str = six.text_type(build_timeout_secs) + 's'
else:
timeout_str = None
if args.tag is not None:
if (properties.VALUES.builds.check_tag.GetBool() and
'gcr.io/' not in args.tag):
raise c_exceptions.InvalidArgumentException(
'--tag',
'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.')
if properties.VALUES.builds.use_kaniko.GetBool():
if args.no_cache:
ttl = '0h'
else:
ttl = '{}h'.format(properties.VALUES.builds.kaniko_cache_ttl.Get())
build_config = messages.Build(
steps=[
messages.BuildStep(
name=properties.VALUES.builds.kaniko_image.Get(),
args=[
'--destination', args.tag, '--cache', 'true',
'--cache-ttl', ttl
],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(
args.substitutions, messages))
else:
if args.no_cache:
raise c_exceptions.InvalidArgumentException(
'no-cache',
'Cannot specify --no-cache if builds/use_kaniko property is '
'False')
build_config = messages.Build(
images=[args.tag],
steps=[
messages.BuildStep(
name='gcr.io/cloud-builders/docker',
args=[
'build', '--network', 'cloudbuild', '--no-cache', '-t',
args.tag, '.'
],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(
args.substitutions, messages))
elif args.config is not None:
if args.no_cache:
raise c_exceptions.ConflictingArgumentsException(
'--config', '--no-cache')
if not args.config:
raise c_exceptions.InvalidArgumentException(
'--config', 'Config file path must not be empty.')
build_config = config.LoadCloudbuildConfigFromPath(
args.config, messages, params=args.substitutions)
else:
raise c_exceptions.OneOfArgumentsRequiredException(
['--tag', '--config'],
'Requires either a docker tag or a config file.')
# If timeout was set by flag, overwrite the config file.
if timeout_str:
build_config.timeout = timeout_str
# --no-source overrides the default --source.
if not args.IsSpecified('source') and args.no_source:
args.source = None
gcs_source_staging = None
if args.source:
suffix = '.tgz'
if args.source.startswith('gs://') or os.path.isfile(args.source):
_, suffix = os.path.splitext(args.source)
# Next, stage the source to Cloud Storage.
staged_object = '{stamp}-{uuid}{suffix}'.format(
stamp=times.GetTimeStampFromDateTime(times.Now()),
uuid=uuid.uuid4().hex,
suffix=suffix,
)
gcs_source_staging_dir = resources.REGISTRY.Parse(
args.gcs_source_staging_dir, collection='storage.objects')
# We create the bucket (if it does not exist) first. If we do an existence
# check and then create the bucket ourselves, it would be possible for an
# attacker to get lucky and beat us to creating the bucket. Block on this
# creation to avoid this race condition.
gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket)
# If no bucket is specified (for the source `default_gcs_source`), check
# that the default bucket is also owned by the project (b/33046325).
if default_gcs_source:
# This request returns only the buckets owned by the project.
bucket_list_req = gcs_client.messages.StorageBucketsListRequest(
project=project, prefix=default_bucket_name)
bucket_list = gcs_client.client.buckets.List(bucket_list_req)
found_bucket = False
for bucket in bucket_list.items:
if bucket.id == default_bucket_name:
found_bucket = True
break
if not found_bucket:
if default_gcs_source:
raise c_exceptions.RequiredArgumentException(
'gcs_source_staging_dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket using '
'--gcs_source_staging_dir.'.format(default_bucket_name))
if gcs_source_staging_dir.object:
staged_object = gcs_source_staging_dir.object + '/' + staged_object
gcs_source_staging = resources.REGISTRY.Create(
collection='storage.objects',
bucket=gcs_source_staging_dir.bucket,
object=staged_object)
if args.source.startswith('gs://'):
gcs_source = resources.REGISTRY.Parse(
args.source, collection='storage.objects')
staged_source_obj = gcs_client.Rewrite(gcs_source, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
if not os.path.exists(args.source):
raise c_exceptions.BadFileException(
'could not find source [{src}]'.format(src=args.source))
if os.path.isdir(args.source):
source_snapshot = snapshot.Snapshot(args.source,
ignore_file=args.ignore_file)
size_str = resource_transform.TransformSize(
source_snapshot.uncompressed_size)
log.status.Print(
'Creating temporary tarball archive of {num_files} file(s)'
' totalling {size} before compression.'.format(
num_files=len(source_snapshot.files), size=size_str))
staged_source_obj = source_snapshot.CopyTarballToGCS(
gcs_client, gcs_source_staging, ignore_file=args.ignore_file)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
elif os.path.isfile(args.source):
unused_root, ext = os.path.splitext(args.source)
if ext not in _ALLOWED_SOURCE_EXT:
raise c_exceptions.BadFileException(
'Local file [{src}] is none of ' +
', '.join(_ALLOWED_SOURCE_EXT))
log.status.Print('Uploading local file [{src}] to '
'[gs://{bucket}/{object}].'.format(
src=args.source,
bucket=gcs_source_staging.bucket,
object=gcs_source_staging.object,
))
staged_source_obj = gcs_client.CopyFileToGCS(args.source,
gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
# No source
if not args.no_source:
raise c_exceptions.InvalidArgumentException(
'--no-source', 'To omit source, use the --no-source flag.')
if args.gcs_log_dir:
gcs_log_dir = resources.REGISTRY.Parse(
args.gcs_log_dir, collection='storage.objects')
build_config.logsBucket = ('gs://' + gcs_log_dir.bucket + '/' +
gcs_log_dir.object)
# Machine type.
if args.machine_type is not None:
machine_type = Submit._machine_type_flag_map.GetEnumForChoice(
args.machine_type)
if not build_config.options:
build_config.options = messages.BuildOptions()
build_config.options.machineType = machine_type
# Disk size.
if args.disk_size is not None:
disk_size = compute_utils.BytesToGb(args.disk_size)
if not build_config.options:
build_config.options = messages.BuildOptions()
build_config.options.diskSizeGb = int(disk_size)
log.debug('submitting build: ' + repr(build_config))
# Start the build.
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(
build=build_config, projectId=properties.VALUES.core.project.Get()))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build
build_ref = resources.REGISTRY.Create(
collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print(
'Logs are available at [{log_url}].'.format(log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
# If the command is run --async, we just print out a reference to the build.
if args.async:
return build
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def WaitFor(poller, operation_ref, message,
pre_start_sleep_ms=1000,
max_retrials=None,
max_wait_ms=300000,
exponential_sleep_multiplier=1.4,
jitter_ms=1000,
wait_ceiling_ms=180000,
sleep_ms=2000):
"""Waits with retrues for operation to be done given poller.
Args:
poller: OperationPoller, poller to use during retrials.
operation_ref: object, passed to operation poller poll method.
message: str, string to display for progrss_tracker.
pre_start_sleep_ms: int, Time to wait before making first poll request.
max_retrials: int, max number of retrials before raising RetryException.
max_wait_ms: int, number of ms to wait before raising WaitException.
exponential_sleep_multiplier: float, factor to use on subsequent retries.
jitter_ms: int, random (up to the value) additional sleep between retries.
wait_ceiling_ms: int, Maximum wait between retries.
sleep_ms: int or iterable: for how long to wait between trials.
Returns:
poller.GetResult(operation).
Raises:
AbortWaitError: if ctrl-c was pressed.
TimeoutError: if retryer has finished wihout being done.
"""
def _CtrlCHandler(unused_signal, unused_frame):
raise AbortWaitError('Ctrl-C aborted wait.')
try:
with execution_utils.CtrlCSection(_CtrlCHandler):
try:
with progress_tracker.ProgressTracker(message) as tracker:
if pre_start_sleep_ms:
_SleepMs(pre_start_sleep_ms)
def _StatusUpdate(unused_result, unused_status):
tracker.Tick()
retryer = retry.Retryer(
max_retrials=max_retrials,
max_wait_ms=max_wait_ms,
exponential_sleep_multiplier=exponential_sleep_multiplier,
jitter_ms=jitter_ms,
wait_ceiling_ms=wait_ceiling_ms,
status_update_func=_StatusUpdate)
def _IsNotDone(operation, unused_state):
return not poller.IsDone(operation)
operation = retryer.RetryOnResult(
func=poller.Poll,
args=(operation_ref,),
should_retry_if=_IsNotDone,
sleep_ms=sleep_ms)
except retry.WaitException:
raise TimeoutError(
'Operation {0} has not finished in {1} seconds'
.format(operation_ref, int(max_wait_ms / 1000)))
except retry.MaxRetrialsException as e:
raise TimeoutError(
'Operation {0} has not finished in {1} seconds '
'after max {2} retrials'
.format(operation_ref,
int(e.state.time_passed_ms / 1000), e.state.retrial))
except AbortWaitError:
# Write this out now that progress tracker is done.
sys.stderr.write('Aborting wait for operation {0}.\n'.format(operation_ref))
raise
return poller.GetResult(operation)
def RunDaisyBuild(args,
workflow,
variables,
daisy_bucket=None,
tags=None,
user_zone=None,
output_filter=None):
"""Run a build with Daisy on Google Cloud Builder.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
workflow: The path to the Daisy workflow to run.
variables: A string of key-value pairs to pass to Daisy.
daisy_bucket: A string containing the name of the GCS bucket that daisy
should use.
tags: A list of strings for adding tags to the Argo build.
user_zone: The GCP zone to tell Daisy to do work in. If unspecified,
defaults to wherever the Argo runner happens to be.
output_filter: A list of strings indicating what lines from the log should
be output. Only lines that start with one of the strings in output_filter
will be displayed.
Returns:
A build object that either streams the output or is displayed as a
link to the build.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
project_id = projects_util.ParseProject(
properties.VALUES.core.project.GetOrFail())
CheckIamPermissions(project_id)
# Make Daisy time out before gcloud by shaving off 2% from the timeout time,
# up to a max of 5m (300s).
two_percent = int(args.timeout * 0.02)
daisy_timeout = args.timeout - min(two_percent, 300)
daisy_bucket = daisy_bucket or GetAndCreateDaisyBucket()
daisy_args = [
'-gcs_path=gs://{0}/'.format(daisy_bucket),
'-default_timeout={0}s'.format(daisy_timeout),
'-variables={0}'.format(variables),
workflow,
]
if user_zone is not None:
daisy_args = ['-zone={0}'.format(user_zone)] + daisy_args
build_tags = ['gce-daisy']
if tags:
build_tags.extend(tags)
# First, create the build request.
build_config = messages.Build(
steps=[
messages.BuildStep(
name=_BUILDER,
args=daisy_args,
),
],
tags=build_tags,
timeout='{0}s'.format(args.timeout),
)
if args.log_location:
gcs_log_dir = resources.REGISTRY.Parse(args.log_location,
collection='storage.objects')
build_config.logsBucket = ('gs://{0}/{1}'.format(
gcs_log_dir.bucket, gcs_log_dir.object))
# Start the build.
build, build_ref = _CreateCloudBuild(build_config, client, messages)
# If the command is run --async, we just print out a reference to the build.
if args. async:
return build
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = DaisyCloudBuildClient(client, messages).StreamWithFilter(
build_ref, output_filter=output_filter)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
project = properties.VALUES.core.project.Get()
safe_project = project.replace(':', '_')
safe_project = safe_project.replace('.', '_')
# The string 'google' is not allowed in bucket names.
safe_project = safe_project.replace('google', 'elgoog')
default_bucket_name = '{}_cloudbuild'.format(safe_project)
default_gcs_source = False
if args.gcs_source_staging_dir is None:
default_gcs_source = True
args.gcs_source_staging_dir = 'gs://{}/source'.format(default_bucket_name)
default_gcs_log_dir = False
if args.gcs_log_dir is None:
default_gcs_log_dir = True
args.gcs_log_dir = 'gs://{}/logs'.format(default_bucket_name)
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
gcs_client = storage_api.StorageClient()
# First, create the build request.
build_timeout = properties.VALUES.container.build_timeout.Get()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
timeout_str = str(build_timeout_secs) + 's'
else:
timeout_str = None
if args.tag:
if 'gcr.io/' not in args.tag:
raise c_exceptions.InvalidArgumentException(
'--tag',
'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.')
build_config = messages.Build(
images=[args.tag],
steps=[
messages.BuildStep(
name='gcr.io/cloud-builders/docker',
args=['build', '--no-cache', '-t', args.tag, '.'],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(args.substitutions,
messages)
)
elif args.config:
build_config = config.LoadCloudbuildConfigFromPath(
args.config, messages, params=args.substitutions)
# If timeout was set by flag, overwrite the config file.
if timeout_str:
build_config.timeout = timeout_str
suffix = '.tgz'
if args.source.startswith('gs://') or os.path.isfile(args.source):
_, suffix = os.path.splitext(args.source)
# Next, stage the source to Cloud Storage.
staged_object = '{stamp}{suffix}'.format(
stamp=times.GetTimeStampFromDateTime(times.Now()),
suffix=suffix,
)
gcs_source_staging_dir = resources.REGISTRY.Parse(
args.gcs_source_staging_dir, collection='storage.objects')
# We first try to create the bucket, before doing all the checks, in order
# to avoid a race condition. If we do the check first, an attacker could
# be lucky enough to create the bucket after the check and before this
# bucket creation.
gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket)
# If no bucket is specified (for the source `default_gcs_source` or for the
# logs `default_gcs_log_dir`), check that the default bucket is also owned
# by the project (b/33046325).
if default_gcs_source or default_gcs_log_dir:
# This request returns only the buckets owned by the project.
bucket_list_req = gcs_client.messages.StorageBucketsListRequest(
project=project,
prefix=default_bucket_name)
bucket_list = gcs_client.client.buckets.List(bucket_list_req)
found_bucket = False
for bucket in bucket_list.items:
if bucket.id == default_bucket_name:
found_bucket = True
break
if not found_bucket:
if default_gcs_source:
raise c_exceptions.RequiredArgumentException(
'gcs_source_staging_dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket using '
'--gcs_source_staging_dir.'.format(default_bucket_name))
elif default_gcs_log_dir:
raise c_exceptions.RequiredArgumentException(
'gcs-log-dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket to hold build logs '
'using --gcs-log-dir.'.format(default_bucket_name))
if gcs_source_staging_dir.object:
staged_object = gcs_source_staging_dir.object + '/' + staged_object
gcs_source_staging = resources.REGISTRY.Create(
collection='storage.objects',
bucket=gcs_source_staging_dir.bucket,
object=staged_object)
if args.source.startswith('gs://'):
gcs_source = resources.REGISTRY.Parse(
args.source, collection='storage.objects')
staged_source_obj = gcs_client.Rewrite(gcs_source, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
if not os.path.exists(args.source):
raise c_exceptions.BadFileException(
'could not find source [{src}]'.format(src=args.source))
if os.path.isdir(args.source):
source_snapshot = snapshot.Snapshot(args.source)
size_str = resource_transform.TransformSize(
source_snapshot.uncompressed_size)
log.status.Print(
'Creating temporary tarball archive of {num_files} file(s)'
' totalling {size} before compression.'.format(
num_files=len(source_snapshot.files),
size=size_str))
staged_source_obj = source_snapshot.CopyTarballToGCS(
gcs_client, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
elif os.path.isfile(args.source):
unused_root, ext = os.path.splitext(args.source)
if ext not in _ALLOWED_SOURCE_EXT:
raise c_exceptions.BadFileException(
'Local file [{src}] is none of '+', '.join(_ALLOWED_SOURCE_EXT))
log.status.Print(
'Uploading local file [{src}] to '
'[gs://{bucket}/{object}].'.format(
src=args.source,
bucket=gcs_source_staging.bucket,
object=gcs_source_staging.object,
))
staged_source_obj = gcs_client.CopyFileToGCS(
storage_util.BucketReference.FromBucketUrl(
gcs_source_staging.bucket),
args.source, gcs_source_staging.object)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
gcs_log_dir = resources.REGISTRY.Parse(
args.gcs_log_dir, collection='storage.objects')
if gcs_log_dir.bucket != gcs_source_staging.bucket:
# Create the logs bucket if it does not yet exist.
gcs_client.CreateBucketIfNotExists(gcs_log_dir.bucket)
build_config.logsBucket = 'gs://'+gcs_log_dir.bucket+'/'+gcs_log_dir.object
log.debug('submitting build: '+repr(build_config))
# Start the build.
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(
build=build_config,
projectId=properties.VALUES.core.project.Get()))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build
build_ref = resources.REGISTRY.Create(
collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print('Logs are available at [{log_url}].'.format(
log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
# If the command is run --async, we just print out a reference to the build.
if args.async:
return build
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def RunDaisyBuild(args, workflow, variables):
"""Run a build with Daisy on Google Cloud Builder.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
workflow: The path to the Daisy workflow to run.
variables: A string of key-value pairs to pass to Daisy.
Returns:
A build object that either streams the output or is displayed as a
link to the build.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
project_id = projects_util.ParseProject(
properties.VALUES.core.project.GetOrFail())
CheckIamPermissions(project_id)
timeout_str = '{0}s'.format(args.timeout)
# First, create the build request.
build_config = messages.Build(
steps=[
messages.BuildStep(
name=_BUILDER,
args=[
'-variables={0}'.format(variables),
workflow,
],
),
],
timeout=timeout_str,
)
if args.log_location:
gcs_log_dir = resources.REGISTRY.Parse(args.log_location,
collection='storage.objects')
build_config.logsBucket = ('gs://{0}/{1}'.format(
gcs_log_dir.bucket, gcs_log_dir.object))
# Start the build.
build, build_ref = _CreateCloudBuild(build_config, client, messages)
# If the command is run --async, we just print out a reference to the build.
if args. async:
return build
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
region = properties.VALUES.compute.region.Get(required=True)
staging_location = jobs_prep.GetStagingLocation(
staging_bucket=args.staging_bucket,
job_id=args.job,
job_dir=args.job_dir)
try:
uris = jobs_prep.UploadPythonPackages(
packages=args.packages,
package_path=args.package_path,
staging_location=staging_location)
except jobs_prep.NoStagingLocationError:
raise flags.ArgumentError(
'If local packages are provided, the `--staging-bucket` or '
'`--job-dir` flag must be given.')
log.debug('Using {0} as trainer uris'.format(uris))
scale_tier_enum = (jobs.GetMessagesModule(
).GoogleCloudMlV1beta1TrainingInput.ScaleTierValueValuesEnum)
scale_tier = scale_tier_enum(
args.scale_tier) if args.scale_tier else None
job = jobs.BuildTrainingJob(
path=args.config,
module_name=args.module_name,
job_name=args.job,
trainer_uri=uris,
region=region,
job_dir=args.job_dir.ToUrl() if args.job_dir else None,
scale_tier=scale_tier,
user_args=args.user_args,
runtime_version=args.runtime_version)
jobs_client = jobs.JobsClient()
project_ref = resources.REGISTRY.Parse(
properties.VALUES.core.project.Get(required=True),
collection='ml.projects')
job = jobs_client.Create(project_ref, job)
log.status.Print('Job [{}] submitted successfully.'.format(job.jobId))
if args. async:
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
return job
log_fetcher = stream.LogFetcher(
filters=log_utils.LogFilters(job.jobId),
polling_interval=_POLLING_INTERVAL,
continue_func=log_utils.MakeContinueFunction(job.jobId))
printer = resource_printer.Printer(log_utils.LOG_FORMAT, out=log.err)
def _CtrlCHandler(signal, frame):
del signal, frame # Unused
raise KeyboardInterrupt
with execution_utils.CtrlCSection(_CtrlCHandler):
try:
printer.Print(log_utils.SplitMultiline(
log_fetcher.YieldLogs()))
except KeyboardInterrupt:
log.status.Print('Received keyboard interrupt.')
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
job_ref = resources.REGISTRY.Parse(job.jobId,
collection='ml.projects.jobs')
job = jobs_client.Get(job_ref)
# If the job itself failed, we will return a failure status.
if job.state is not job.StateValueValuesEnum.SUCCEEDED:
self.exit_code = 1
return job
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
Raises:
FailedDeployException: If the build is completed and not 'SUCCESS'.
"""
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
build_config = self._CreateBuildFromArgs(args, messages)
# Start the build
project = properties.VALUES.core.project.Get(required=True)
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(build=build_config,
projectId=project))
log.debug('submitting build: ' + repr(build_config))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata,
json).build
build_ref = resources.REGISTRY.Create(
collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.status.Print(
'Starting Cloud Build to build and deploy to the target '
'Google Kubernetes Engine cluster...\n')
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print('Logs are available at [{log_url}].'.format(
log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
if args.async_:
return
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client,
messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build and deploy timed out. Use the [--timeout=DURATION] flag '
'to change the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedDeployException(build)
# Get location of suggested config bucket from gsutil build step
suggested_configs = build.steps[-1].args[-1]
log.status.Print(
'Successfully deployed to your Google Kubernetes Engine cluster.\n\n'
'You can find the configuration files of the deployed Kubernetes '
'objects stored at {expanded} These configurations are expanded by the '
'deployer with additional labels like app name and version.\n\n'
'You can also find suggested base Kubernetes configuration files '
'created by the deployer at {suggested}'.format(
expanded=build.artifacts.objects.location,
suggested=suggested_configs))
return
def Build(messages,
async_,
build_config,
hide_logs=False,
build_region=cloudbuild_util.DEFAULT_REGION,
support_gcl=False,
suppress_logs=False):
"""Starts the build."""
log.debug('submitting build: ' + repr(build_config))
client = cloudbuild_util.GetClientInstance()
parent_resource = resources.REGISTRY.Create(
collection='cloudbuild.projects.locations',
projectsId=properties.VALUES.core.project.GetOrFail(),
locationsId=build_region)
op = client.projects_locations_builds.Create(
messages.CloudbuildProjectsLocationsBuildsCreateRequest(
parent=parent_resource.RelativeName(), build=build_config))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build
# Need to set the default version to 'v1'
build_ref = resources.REGISTRY.Parse(
None,
collection='cloudbuild.projects.locations.builds',
api_version='v1',
params={
'projectsId': build.projectId,
'locationsId': build_region,
'buildsId': build.id,
})
if not hide_logs:
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print(
'Logs are available at [{log_url}].'.format(log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
# If the command is run --async, we just print out a reference to the build.
if async_:
return build, op
if not support_gcl and build.options and build.options.logging in [
messages.BuildOptions.LoggingValueValuesEnum.STACKDRIVER_ONLY,
messages.BuildOptions.LoggingValueValuesEnum.CLOUD_LOGGING_ONLY,
]:
log.status.Print('\ngcloud builds submit only displays logs from Cloud'
' Storage. To view logs from Cloud Logging, run:\ngcloud'
' beta builds submit\n')
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
out = log.out if not suppress_logs else None
# Otherwise, logs are streamed from the chosen logging service
# (defaulted to GCS).
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client, messages,
support_gcl).Stream(build_ref, out)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.warnings:
for warn in build.warnings:
log.status.Print('\n{priority}: {text}'.format(
text=warn.text, priority=warn.priority))
log.status.Print(
'\n{count} message(s) issued.'.format(count=len(build.warnings)))
if build.failureInfo:
log.status.Print(
'\nBUILD FAILURE: {detail}'.format(detail=build.failureInfo.detail))
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build, op