def GetConfigurationChanges(args):
"""Returns a list of changes to Configuration, based on the flags set."""
changes = []
if _HasEnvChanges(args):
changes.append(_GetEnvChanges(args))
if 'memory' in args and args.memory:
changes.append(config_changes.ResourceChanges(memory=args.memory))
if 'concurrency' in args and args.concurrency:
try:
c = int(args.concurrency)
except ValueError:
c = args.concurrency
if c != 'default':
log.warning(
'Specifying concurrency as Single or Multi is deprecated; '
'an integer is preferred.')
changes.append(config_changes.ConcurrencyChanges(concurrency=c))
if 'timeout' in args and args.timeout:
try:
# A bare number is interpreted as seconds.
timeout_secs = int(args.timeout)
except ValueError:
timeout_duration = times.ParseDuration(args.timeout)
timeout_secs = int(timeout_duration.total_seconds)
if timeout_secs <= 0:
raise ArgumentError(
'The --timeout argument must be a positive time duration.')
changes.append(config_changes.TimeoutChanges(timeout=timeout_secs))
return changes
def Display(self, value):
"""Returns the display string for a duration value, leading PT dropped."""
d = times.ParseDuration('{}s'.format(value))
s = times.FormatDuration(d)
if s.startswith('PT'):
s = s[2:].lower()
return s
def GetServiceTimeoutSeconds(timeout_property_str):
"""Returns the service timeout in seconds given the duration string."""
if timeout_property_str is None:
return None
build_timeout_duration = times.ParseDuration(timeout_property_str,
default_suffix='s')
return int(build_timeout_duration.total_seconds)
def _PossiblyBuildAndPush(
self, new_version, service, upload_dir, source_files, image,
code_bucket_ref, gcr_domain, flex_image_build_option):
"""Builds and Pushes the Docker image if necessary for this service.
Args:
new_version: version_util.Version describing where to deploy the service
service: yaml_parsing.ServiceYamlInfo, service configuration to be
deployed
upload_dir: str, path to the service's upload directory
source_files: [str], relative paths to upload.
image: str or None, the URL for the Docker image to be deployed (if image
already exists).
code_bucket_ref: cloud_storage.BucketReference where the service's files
have been uploaded
gcr_domain: str, Cloud Registry domain, determines the physical location
of the image. E.g. `us.gcr.io`.
flex_image_build_option: FlexImageBuildOptions, whether a flex deployment
should upload files so that the server can build the image or build the
image on client.
Returns:
BuildArtifact, a wrapper which contains either the build ID for
an in-progress build, or the name of the container image for a serial
build. Possibly None if the service does not require an image.
Raises:
RequiredFileMissingError: if a required file is not uploaded.
"""
build = None
if image:
if service.RequiresImage() and service.parsed.skip_files.regex:
log.warning('Deployment of service [{0}] will ignore the skip_files '
'field in the configuration file, because the image has '
'already been built.'.format(new_version.service))
return app_cloud_build.BuildArtifact.MakeImageArtifact(image)
elif service.RequiresImage():
if not _AppYamlInSourceFiles(source_files, service.GetAppYamlBasename()):
raise RequiredFileMissingError(service.GetAppYamlBasename())
if flex_image_build_option == FlexImageBuildOptions.ON_SERVER:
cloud_build_options = {
'appYamlPath': service.GetAppYamlBasename(),
}
timeout = properties.VALUES.app.cloud_build_timeout.Get()
if timeout:
build_timeout = int(
times.ParseDuration(timeout, default_suffix='s').total_seconds)
cloud_build_options['cloudBuildTimeout'] = six.text_type(
build_timeout) + 's'
build = app_cloud_build.BuildArtifact.MakeBuildOptionsArtifact(
cloud_build_options)
else:
build = deploy_command_util.BuildAndPushDockerImage(
new_version.project, service, upload_dir, source_files,
new_version.id, code_bucket_ref, gcr_domain,
self.deploy_options.runtime_builder_strategy,
self.deploy_options.parallel_build)
return build
def ParseExpireTime(expiration_value):
"""Parse flag value into Datetime format for expireTime."""
# expiration_value could be in Datetime format or Duration format.
datetime = (
times.ParseDuration(expiration_value).GetRelativeDateTime(
times.Now(times.UTC)))
parsed_datetime = times.FormatDateTime(
datetime, '%Y-%m-%dT%H:%M:%S.%6f%Ez', tzinfo=times.UTC)
return parsed_datetime
def Parse(value):
"""Parses a duration from value and returns integer seconds."""
try:
return int(
times.ParseDuration(value, default_suffix=default_unit).total_seconds)
except times.Error as e:
message = six.text_type(e).rstrip('.')
raise ArgumentTypeError(_GenerateErrorMessage(
'Failed to parse duration: {0}'.format(message, user_input=value)))
def GetServiceTimeoutString(timeout_property_str):
if timeout_property_str is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(timeout_property_str)
except ValueError:
build_timeout_duration = times.ParseDuration(timeout_property_str)
build_timeout_secs = int(build_timeout_duration.total_seconds)
return str(build_timeout_secs) + 's'
return None
def GetConfigurationChanges(args):
"""Returns a list of changes to Configuration, based on the flags set."""
changes = []
if _HasEnvChanges(args):
changes.append(_GetEnvChanges(args))
if _HasCloudSQLChanges(args):
region = GetRegion(args)
project = (getattr(args, 'project', None)
or properties.VALUES.core.project.Get(required=True))
_CheckCloudSQLApiEnablement()
changes.append(config_changes.CloudSQLChanges(project, region, args))
if 'cpu' in args and args.cpu:
changes.append(config_changes.ResourceChanges(cpu=args.cpu))
if 'memory' in args and args.memory:
changes.append(config_changes.ResourceChanges(memory=args.memory))
if 'concurrency' in args and args.concurrency:
try:
c = int(args.concurrency)
except ValueError:
c = args.concurrency
if c != 'default':
log.warning(
'Specifying concurrency as Single or Multi is deprecated; '
'an integer is preferred.')
changes.append(config_changes.ConcurrencyChanges(concurrency=c))
if 'timeout' in args and args.timeout:
try:
# A bare number is interpreted as seconds.
timeout_secs = int(args.timeout)
except ValueError:
timeout_duration = times.ParseDuration(args.timeout)
timeout_secs = int(timeout_duration.total_seconds)
if timeout_secs <= 0:
raise ArgumentError(
'The --timeout argument must be a positive time duration.')
changes.append(config_changes.TimeoutChanges(timeout=timeout_secs))
if 'service_account' in args and args.service_account:
changes.append(
config_changes.ServiceAccountChanges(
service_account=args.service_account))
if _HasLabelChanges(args):
diff = labels_util.Diff.FromUpdateArgs(args)
if diff.MayHaveUpdates():
changes.append(config_changes.LabelChanges(diff))
if 'revision_suffix' in args and args.revision_suffix:
changes.append(config_changes.RevisionNameChanges(
args.revision_suffix))
if 'vpc_connector' in args and args.vpc_connector:
changes.append(config_changes.VpcConnectorChange(args.vpc_connector))
if 'clear_vpc_connector' in args and args.clear_vpc_connector:
changes.append(config_changes.ClearVpcConnectorChange())
return changes
def Convert(self, string):
"""Converts a duration from string returns it."""
if not string:
return None
try:
d = times.ParseDuration(
string, default_suffix=self._default_suffix).total_seconds
return d if self._subsecond else int(d)
except times.Error as e:
raise exceptions.ParseError(
self.GetPresentationName(),
'Failed to parse duration [{}]: {}.'.format(
string, _SubException(e)))
def _GetBuildTimeout():
"""Get the build timeout."""
build_timeout = properties.VALUES.builds.timeout.Get()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
timeout_str = six.text_type(build_timeout_secs) + 's'
else:
timeout_str = None
return timeout_str
def Parse(value):
"""Parses a duration from value and returns integer of the parsed_unit."""
if parsed_unit == 'ms':
multiplier = 1000
elif parsed_unit == 'us':
multiplier = 1000000
elif parsed_unit == 's':
multiplier = 1
else:
raise ArgumentTypeError(
_GenerateErrorMessage('parsed_unit must be one of s, ms, us.'))
try:
duration = times.ParseDuration(value, default_suffix=default_unit)
return int(duration.total_seconds * multiplier)
except times.Error as e:
message = six.text_type(e).rstrip('.')
raise ArgumentTypeError(_GenerateErrorMessage(
'Failed to parse duration: {0}'.format(message, user_input=value)))
def _ParseSegmentTimestamp(timestamp_string):
"""Parse duration formatted segment timestamp into a Duration object.
Assumes string with no duration unit specified (e.g. 's' or 'm' etc.) is
an int representing microseconds.
Args:
timestamp_string: str, string to convert
Raises:
ValueError: timestamp_string is not a properly formatted duration, not a
int or int value is <0
Returns:
Duration object represented by timestamp_string
"""
# Assume timestamp_string passed as int number of microseconds if no unit
# e.g. 4566, 100, etc.
try:
microseconds = int(timestamp_string)
except ValueError:
try:
duration = times.ParseDuration(timestamp_string)
if duration.total_seconds < 0:
raise times.DurationValueError()
return duration
except (times.DurationSyntaxError, times.DurationValueError):
raise ValueError(
'Could not parse timestamp string [{}]. Timestamp must '
'be a properly formatted duration string with time '
'amount and units (e.g. 1m3.456s, 2m, 14.4353s)'.format(
timestamp_string))
else:
log.warning(
"Time unit missing ('s', 'm','h') for segment timestamp [{}], "
"parsed as microseconds.".format(timestamp_string))
if microseconds < 0:
raise ValueError(
'Could not parse duration string [{}]. Timestamp must be'
'greater than >= 0)'.format(timestamp_string))
return iso_duration.Duration(microseconds=microseconds)
def _StartMinikubeCluster(cluster_name, vm_driver, debug=False):
"""Starts a minikube cluster."""
# pylint: disable=broad-except
try:
if not _IsMinikubeClusterUp(cluster_name):
cmd = [
_FindMinikube(),
'start',
'-p',
cluster_name,
'--keep-context',
'--interactive=false',
'--delete-on-failure',
'--install-addons=false',
'--output=json',
]
if vm_driver:
cmd.append('--vm-driver=' + vm_driver)
if vm_driver == 'docker':
cmd.append('--container-runtime=docker')
if debug:
cmd.extend(['--alsologtostderr', '-v8'])
start_msg = "Starting development environment '%s' ..." % cluster_name
event_timeout = times.ParseDuration(
properties.VALUES.code.minikube_event_timeout.Get(
required=True)).total_seconds
with console_io.ProgressBar(start_msg) as progress_bar:
for json_obj in run_subprocess.StreamOutputJson(
cmd, event_timeout_sec=event_timeout,
show_stderr=debug):
if debug:
print('minikube', json_obj)
_HandleMinikubeStatusEvent(progress_bar, json_obj)
except Exception as e:
six.reraise(MinikubeStartError, e, sys.exc_info()[2])
def _TokenExpiresWithinWindow(expiry_window,
token_expiry_time,
max_window_seconds=3600):
"""Determines if token_expiry_time is within expiry_window_duration.
Calculates the amount of time between utcnow() and token_expiry_time and
returns true, if that amount is less thank the provided duration window. All
calculations are done in number of seconds for consistency.
Args:
expiry_window: string, Duration representing the amount of time between
now and token_expiry_time to compare against.
token_expiry_time: datetime, The time when token expires.
max_window_seconds: int, Maximum size of expiry window, in seconds.
Raises:
ValueError: If expiry_window is invalid or can not be parsed.
Returns:
True if token is expired or will expire with in the provided window,
False otherwise.
"""
try:
min_expiry = times.ParseDuration(expiry_window, default_suffix='s')
if min_expiry.total_seconds > max_window_seconds:
raise ValueError('Invalid expiry window duration [{}]: '
'Must be between 0s and 1h'.format(expiry_window))
except times.Error as e:
message = six.text_type(e).rstrip('.')
raise ValueError('Error Parsing expiry window duration '
'[{}]: {}'.format(expiry_window, message))
token_expiry_time = times.LocalizeDateTime(token_expiry_time,
tzinfo=dateutil.tz.tzutc())
window_end = times.GetDateTimePlusDuration(
times.Now(tzinfo=dateutil.tz.tzutc()), min_expiry)
return token_expiry_time <= window_end
def _ExpectCreateCertificate(self,
parent_name,
cert_id,
request_id,
public_key,
lifetime='P30D',
reusable_config=None,
subject_config=None,
include_name=False,
labels=None):
reusable_config = reusable_config or self.messages.ReusableConfigWrapper(
)
subject_config = subject_config or self.messsages.SubjectConfig(
subject=self.messages.Subject,
subjectAltName=self.messages.SubjectAltNames())
lifetime = times.FormatDurationForJson(times.ParseDuration(lifetime))
response = self.messages.Certificate(
pemCertificate=self.test_cert,
pemCertificateChain=[self.parent_cert])
if include_name:
response.name = '{}/certificate/{}'.format(parent_name, cert_id)
request = self.messages.PrivatecaProjectsLocationsCertificateAuthoritiesCertificatesCreateRequest(
certificateId=cert_id,
requestId=request_id,
parent=parent_name,
certificate=self.messages.Certificate(
lifetime=lifetime,
config=self.messages.CertificateConfig(
reusableConfig=reusable_config,
subjectConfig=subject_config,
publicKey=self.messages.PublicKey(
type=self.messages.PublicKey.TypeValueValuesEnum.
PEM_RSA_KEY,
key=public_key)),
labels=labels))
self.mock_client.projects_locations_certificateAuthorities_certificates.Create.Expect(
request=request, response=response)
def TransformDuration(r,
start='',
end='',
parts=3,
precision=3,
calendar=True,
unit=1,
undefined=''):
"""Formats the resource as an ISO 8601 duration string.
The [ISO 8601 Duration](https://en.wikipedia.org/wiki/ISO_8601#Durations)
format is: "[-]P[nY][nM][nD][T[nH][nM][n[.m]S]]". The 0 duration is "P0".
Otherwise at least one part will always be displayed. Negative durations are
prefixed by "-". "T" disambiguates months "P2M" to the left of "T" and minutes
"PT5M" to the right.
If the resource is a datetime then the duration of `resource - current_time`
is returned.
Args:
r: A JSON-serializable object.
start: The name of a start time attribute in the resource. The duration of
the `end - start` time attributes in resource is returned. If `end` is
not specified then the current time is used.
end: The name of an end time attribute in the resource. Defaults to
the current time if omitted. Ignored if `start` is not specified.
parts: Format at most this many duration parts starting with largest
non-zero part.
precision: Format the last duration part with precision digits after the
decimal point. Trailing "0" and "." are always stripped.
calendar: Allow time units larger than hours in formated durations if true.
Durations specifying hours or smaller units are exact across daylight
savings time boundaries. On by default. Use calendar=false to disable.
For example, if `calendar=true` then at the daylight savings boundary
2016-03-13T01:00:00 + P1D => 2016-03-14T01:00:00 but 2016-03-13T01:00:00 +
PT24H => 2016-03-14T03:00:00. Similarly, a +P1Y duration will be inexact
but "calendar correct", yielding the same month and day number next year,
even in leap years.
unit: Divide the resource numeric value by _unit_ to yield seconds.
undefined: Returns this value if the resource is not a valid timestamp.
Returns:
The ISO 8601 duration string for r or undefined if r is not a duration.
"""
try:
parts = int(parts)
precision = int(precision)
except ValueError:
return undefined
calendar = GetBooleanArgValue(calendar)
if start:
# Duration of ((end or Now()) - start).
# Get the datetime of both.
try:
start_datetime = times.ParseDateTime(_GetKeyValue(r, start))
end_value = _GetKeyValue(r, end) if end else None
if end_value:
end_datetime = times.ParseDateTime(end_value)
else:
end_datetime = times.Now(tzinfo=start_datetime.tzinfo)
except (AttributeError, ImportError, TypeError, ValueError):
return undefined
# Finally format the duration of the delta.
delta = end_datetime - start_datetime
return times.GetDurationFromTimeDelta(
delta=delta, calendar=calendar).Format(parts=parts,
precision=precision)
# Check if the resource is a float duration.
try:
duration = times.ParseDuration('PT{0}S'.format(float(r) / unit),
calendar=calendar)
return duration.Format(parts=parts, precision=precision)
except (TypeError, ValueError):
pass
# Check if the resource is an ISO 8601 duration.
try:
duration = times.ParseDuration(r)
return duration.Format(parts=parts, precision=precision)
except (AttributeError, TypeError, ValueError):
pass
# Check if the resource is a datetime.
try:
start_datetime = times.ParseDateTime(r)
except (AttributeError, ImportError, TypeError, ValueError):
return undefined
# Format the duration of (now - r).
end_datetime = times.Now(tzinfo=start_datetime.tzinfo)
delta = end_datetime - start_datetime
return times.GetDurationFromTimeDelta(
delta=delta, calendar=calendar).Format(parts=parts,
precision=precision)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
project = properties.VALUES.core.project.Get(required=True)
safe_project = project.replace(':', '_')
safe_project = safe_project.replace('.', '_')
# The string 'google' is not allowed in bucket names.
safe_project = safe_project.replace('google', 'elgoog')
default_bucket_name = '{}_cloudbuild'.format(safe_project)
default_gcs_source = False
if args.gcs_source_staging_dir is None:
default_gcs_source = True
args.gcs_source_staging_dir = 'gs://{}/source'.format(default_bucket_name)
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
gcs_client = storage_api.StorageClient()
# First, create the build request.
build_timeout = properties.VALUES.builds.timeout.Get()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
timeout_str = six.text_type(build_timeout_secs) + 's'
else:
timeout_str = None
if args.tag is not None:
if (properties.VALUES.builds.check_tag.GetBool() and
'gcr.io/' not in args.tag):
raise c_exceptions.InvalidArgumentException(
'--tag',
'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.')
if properties.VALUES.builds.use_kaniko.GetBool():
if args.no_cache:
ttl = '0h'
else:
ttl = '{}h'.format(properties.VALUES.builds.kaniko_cache_ttl.Get())
build_config = messages.Build(
steps=[
messages.BuildStep(
name=properties.VALUES.builds.kaniko_image.Get(),
args=[
'--destination', args.tag, '--cache', 'true',
'--cache-ttl', ttl
],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(
args.substitutions, messages))
else:
if args.no_cache:
raise c_exceptions.InvalidArgumentException(
'no-cache',
'Cannot specify --no-cache if builds/use_kaniko property is '
'False')
build_config = messages.Build(
images=[args.tag],
steps=[
messages.BuildStep(
name='gcr.io/cloud-builders/docker',
args=[
'build', '--network', 'cloudbuild', '--no-cache', '-t',
args.tag, '.'
],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(
args.substitutions, messages))
elif args.config is not None:
if args.no_cache:
raise c_exceptions.ConflictingArgumentsException(
'--config', '--no-cache')
if not args.config:
raise c_exceptions.InvalidArgumentException(
'--config', 'Config file path must not be empty.')
build_config = config.LoadCloudbuildConfigFromPath(
args.config, messages, params=args.substitutions)
else:
raise c_exceptions.OneOfArgumentsRequiredException(
['--tag', '--config'],
'Requires either a docker tag or a config file.')
# If timeout was set by flag, overwrite the config file.
if timeout_str:
build_config.timeout = timeout_str
# --no-source overrides the default --source.
if not args.IsSpecified('source') and args.no_source:
args.source = None
gcs_source_staging = None
if args.source:
suffix = '.tgz'
if args.source.startswith('gs://') or os.path.isfile(args.source):
_, suffix = os.path.splitext(args.source)
# Next, stage the source to Cloud Storage.
staged_object = '{stamp}-{uuid}{suffix}'.format(
stamp=times.GetTimeStampFromDateTime(times.Now()),
uuid=uuid.uuid4().hex,
suffix=suffix,
)
gcs_source_staging_dir = resources.REGISTRY.Parse(
args.gcs_source_staging_dir, collection='storage.objects')
# We create the bucket (if it does not exist) first. If we do an existence
# check and then create the bucket ourselves, it would be possible for an
# attacker to get lucky and beat us to creating the bucket. Block on this
# creation to avoid this race condition.
gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket)
# If no bucket is specified (for the source `default_gcs_source`), check
# that the default bucket is also owned by the project (b/33046325).
if default_gcs_source:
# This request returns only the buckets owned by the project.
bucket_list_req = gcs_client.messages.StorageBucketsListRequest(
project=project, prefix=default_bucket_name)
bucket_list = gcs_client.client.buckets.List(bucket_list_req)
found_bucket = False
for bucket in bucket_list.items:
if bucket.id == default_bucket_name:
found_bucket = True
break
if not found_bucket:
if default_gcs_source:
raise c_exceptions.RequiredArgumentException(
'gcs_source_staging_dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket using '
'--gcs_source_staging_dir.'.format(default_bucket_name))
if gcs_source_staging_dir.object:
staged_object = gcs_source_staging_dir.object + '/' + staged_object
gcs_source_staging = resources.REGISTRY.Create(
collection='storage.objects',
bucket=gcs_source_staging_dir.bucket,
object=staged_object)
if args.source.startswith('gs://'):
gcs_source = resources.REGISTRY.Parse(
args.source, collection='storage.objects')
staged_source_obj = gcs_client.Rewrite(gcs_source, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
if not os.path.exists(args.source):
raise c_exceptions.BadFileException(
'could not find source [{src}]'.format(src=args.source))
if os.path.isdir(args.source):
source_snapshot = snapshot.Snapshot(args.source,
ignore_file=args.ignore_file)
size_str = resource_transform.TransformSize(
source_snapshot.uncompressed_size)
log.status.Print(
'Creating temporary tarball archive of {num_files} file(s)'
' totalling {size} before compression.'.format(
num_files=len(source_snapshot.files), size=size_str))
staged_source_obj = source_snapshot.CopyTarballToGCS(
gcs_client, gcs_source_staging, ignore_file=args.ignore_file)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
elif os.path.isfile(args.source):
unused_root, ext = os.path.splitext(args.source)
if ext not in _ALLOWED_SOURCE_EXT:
raise c_exceptions.BadFileException(
'Local file [{src}] is none of ' +
', '.join(_ALLOWED_SOURCE_EXT))
log.status.Print('Uploading local file [{src}] to '
'[gs://{bucket}/{object}].'.format(
src=args.source,
bucket=gcs_source_staging.bucket,
object=gcs_source_staging.object,
))
staged_source_obj = gcs_client.CopyFileToGCS(args.source,
gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
# No source
if not args.no_source:
raise c_exceptions.InvalidArgumentException(
'--no-source', 'To omit source, use the --no-source flag.')
if args.gcs_log_dir:
gcs_log_dir = resources.REGISTRY.Parse(
args.gcs_log_dir, collection='storage.objects')
build_config.logsBucket = ('gs://' + gcs_log_dir.bucket + '/' +
gcs_log_dir.object)
# Machine type.
if args.machine_type is not None:
machine_type = Submit._machine_type_flag_map.GetEnumForChoice(
args.machine_type)
if not build_config.options:
build_config.options = messages.BuildOptions()
build_config.options.machineType = machine_type
# Disk size.
if args.disk_size is not None:
disk_size = compute_utils.BytesToGb(args.disk_size)
if not build_config.options:
build_config.options = messages.BuildOptions()
build_config.options.diskSizeGb = int(disk_size)
log.debug('submitting build: ' + repr(build_config))
# Start the build.
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(
build=build_config, projectId=properties.VALUES.core.project.Get()))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build
build_ref = resources.REGISTRY.Create(
collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print(
'Logs are available at [{log_url}].'.format(log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
# If the command is run --async, we just print out a reference to the build.
if args.async:
return build
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def ParseValidityFlag(args):
return times.FormatDurationForJson(times.ParseDuration(args.validity))
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
project = properties.VALUES.core.project.Get()
safe_project = project.replace(':', '_')
safe_project = safe_project.replace('.', '_')
# The string 'google' is not allowed in bucket names.
safe_project = safe_project.replace('google', 'elgoog')
default_bucket_name = '{}_cloudbuild'.format(safe_project)
default_gcs_source = False
if args.gcs_source_staging_dir is None:
default_gcs_source = True
args.gcs_source_staging_dir = 'gs://{}/source'.format(default_bucket_name)
default_gcs_log_dir = False
if args.gcs_log_dir is None:
default_gcs_log_dir = True
args.gcs_log_dir = 'gs://{}/logs'.format(default_bucket_name)
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
gcs_client = storage_api.StorageClient()
# First, create the build request.
build_timeout = properties.VALUES.container.build_timeout.Get()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
timeout_str = str(build_timeout_secs) + 's'
else:
timeout_str = None
if args.tag:
if 'gcr.io/' not in args.tag:
raise c_exceptions.InvalidArgumentException(
'--tag',
'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.')
build_config = messages.Build(
images=[args.tag],
steps=[
messages.BuildStep(
name='gcr.io/cloud-builders/docker',
args=['build', '--no-cache', '-t', args.tag, '.'],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(args.substitutions,
messages)
)
elif args.config:
build_config = config.LoadCloudbuildConfigFromPath(
args.config, messages, params=args.substitutions)
# If timeout was set by flag, overwrite the config file.
if timeout_str:
build_config.timeout = timeout_str
suffix = '.tgz'
if args.source.startswith('gs://') or os.path.isfile(args.source):
_, suffix = os.path.splitext(args.source)
# Next, stage the source to Cloud Storage.
staged_object = '{stamp}{suffix}'.format(
stamp=times.GetTimeStampFromDateTime(times.Now()),
suffix=suffix,
)
gcs_source_staging_dir = resources.REGISTRY.Parse(
args.gcs_source_staging_dir, collection='storage.objects')
# We first try to create the bucket, before doing all the checks, in order
# to avoid a race condition. If we do the check first, an attacker could
# be lucky enough to create the bucket after the check and before this
# bucket creation.
gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket)
# If no bucket is specified (for the source `default_gcs_source` or for the
# logs `default_gcs_log_dir`), check that the default bucket is also owned
# by the project (b/33046325).
if default_gcs_source or default_gcs_log_dir:
# This request returns only the buckets owned by the project.
bucket_list_req = gcs_client.messages.StorageBucketsListRequest(
project=project,
prefix=default_bucket_name)
bucket_list = gcs_client.client.buckets.List(bucket_list_req)
found_bucket = False
for bucket in bucket_list.items:
if bucket.id == default_bucket_name:
found_bucket = True
break
if not found_bucket:
if default_gcs_source:
raise c_exceptions.RequiredArgumentException(
'gcs_source_staging_dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket using '
'--gcs_source_staging_dir.'.format(default_bucket_name))
elif default_gcs_log_dir:
raise c_exceptions.RequiredArgumentException(
'gcs-log-dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket to hold build logs '
'using --gcs-log-dir.'.format(default_bucket_name))
if gcs_source_staging_dir.object:
staged_object = gcs_source_staging_dir.object + '/' + staged_object
gcs_source_staging = resources.REGISTRY.Create(
collection='storage.objects',
bucket=gcs_source_staging_dir.bucket,
object=staged_object)
if args.source.startswith('gs://'):
gcs_source = resources.REGISTRY.Parse(
args.source, collection='storage.objects')
staged_source_obj = gcs_client.Rewrite(gcs_source, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
if not os.path.exists(args.source):
raise c_exceptions.BadFileException(
'could not find source [{src}]'.format(src=args.source))
if os.path.isdir(args.source):
source_snapshot = snapshot.Snapshot(args.source)
size_str = resource_transform.TransformSize(
source_snapshot.uncompressed_size)
log.status.Print(
'Creating temporary tarball archive of {num_files} file(s)'
' totalling {size} before compression.'.format(
num_files=len(source_snapshot.files),
size=size_str))
staged_source_obj = source_snapshot.CopyTarballToGCS(
gcs_client, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
elif os.path.isfile(args.source):
unused_root, ext = os.path.splitext(args.source)
if ext not in _ALLOWED_SOURCE_EXT:
raise c_exceptions.BadFileException(
'Local file [{src}] is none of '+', '.join(_ALLOWED_SOURCE_EXT))
log.status.Print(
'Uploading local file [{src}] to '
'[gs://{bucket}/{object}].'.format(
src=args.source,
bucket=gcs_source_staging.bucket,
object=gcs_source_staging.object,
))
staged_source_obj = gcs_client.CopyFileToGCS(
storage_util.BucketReference.FromBucketUrl(
gcs_source_staging.bucket),
args.source, gcs_source_staging.object)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
gcs_log_dir = resources.REGISTRY.Parse(
args.gcs_log_dir, collection='storage.objects')
if gcs_log_dir.bucket != gcs_source_staging.bucket:
# Create the logs bucket if it does not yet exist.
gcs_client.CreateBucketIfNotExists(gcs_log_dir.bucket)
build_config.logsBucket = 'gs://'+gcs_log_dir.bucket+'/'+gcs_log_dir.object
log.debug('submitting build: '+repr(build_config))
# Start the build.
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(
build=build_config,
projectId=properties.VALUES.core.project.Get()))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build
build_ref = resources.REGISTRY.Create(
collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print('Logs are available at [{log_url}].'.format(
log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
# If the command is run --async, we just print out a reference to the build.
if args.async:
return build
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def CreateBuild(messages, build_timeout, build_and_push, staged_source, image,
dockerfile_path, app_name, app_version, config_path, namespace,
expose_port, gcs_config_staging_path, cluster, location,
build_tags):
"""Creates the Cloud Build config to run.
Args:
messages: Cloud Build messages module. i.e., the return value of
cloudbuild_util.GetMessagesModule().
build_timeout: An optional maximum time a build is run before it times out.
For example, "2h15m5s" is 2 hours, 15 minutes, and 5 seconds. If you do
not specify a unit, seconds is assumed. If this value is None, a timeout
is not set.
build_and_push: If True, the created build will have Build and Push steps.
staged_source: An optional GCS object for a staged source repository. The
object must have bucket, name, and generation fields. If this value is
None, the created build will not have a source.
image: The image that will deployed and optionally built beforehand. The
image can include a tag or digest.
dockerfile_path: A path to the source repository's Dockerfile, relative to
the source repository's root directory.
app_name: An app name that is set to a substitution variable.
app_version: An app version that is set to a substitution variable.
config_path: An optional path to the source repository's Kubernetes configs,
relative to the source repository's root directory that is set to a
substitution variable. If this value is None, the substitution variable is
set to '' to indicate its absence.
namespace: A Kubernetes namespace of the cluster to deploy to that
is set to a substitution variable.
expose_port: An optional port that the deployed application listens to that
is set to a substitution variable. If this value is None, the substitution
variable is set to 0 to indicate its absence.
gcs_config_staging_path: An optional path to a GCS subdirectory to copy
application configs that is set to a substitution variable. If this value
is None, the substitution variable is set to '' to indicate its absence.
cluster: The name of the target cluster to deploy to.
location: The zone/region of the target cluster to deploy to.
build_tags: Tags to append to build tags in additional to default tags.
Returns:
messages.Build, the Cloud Build config.
"""
build = messages.Build()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
build.timeout = six.text_type(build_timeout_secs) + 's'
if staged_source:
build.source = messages.Source(storageSource=messages.StorageSource(
bucket=staged_source.bucket,
object=staged_source.name,
generation=staged_source.generation))
if config_path is None:
config_path = ''
if not expose_port:
expose_port = '0'
else:
expose_port = str(expose_port)
build.steps = []
if build_and_push:
build.steps.append(
messages.BuildStep(id=_BUILD_BUILD_STEP_ID,
name='gcr.io/cloud-builders/docker',
args=[
'build', '--network', 'cloudbuild',
'--no-cache', '-t', image, '-f',
'${}'.format(_DOCKERFILE_PATH_SUB_VAR), '.'
]))
build.steps.append(
messages.BuildStep(id=_PUSH_BUILD_STEP_ID,
name='gcr.io/cloud-builders/docker',
args=[
'push',
image,
]))
build.steps.append(
messages.BuildStep(
id=_PREPARE_DEPLOY_BUILD_STEP_ID,
name=_GKE_DEPLOY_PROD,
args=[
'prepare', '--filename=${}'.format(_K8S_YAML_PATH_SUB_VAR),
'--image={}'.format(image),
'--app=${}'.format(_APP_NAME_SUB_VAR),
'--version={}'.format(app_version),
'--namespace=${}'.format(_K8S_NAMESPACE_SUB_VAR),
'--output=output', '--annotation=gcb-build-id=$BUILD_ID',
'--expose=${}'.format(_EXPOSE_PORT_SUB_VAR)
],
))
build.steps.append(
messages.BuildStep(id=_SAVE_CONFIGS_BUILD_STEP_ID,
name='gcr.io/cloud-builders/gsutil',
entrypoint='sh',
args=['-c', _SAVE_CONFIGS_SCRIPT]))
build.steps.append(
messages.BuildStep(
id=_APPLY_DEPLOY_BUILD_STEP_ID,
name=_GKE_DEPLOY_PROD,
args=[
'apply',
'--filename=output/expanded',
'--namespace=${}'.format(_K8S_NAMESPACE_SUB_VAR),
'--cluster=${}'.format(_GKE_CLUSTER_SUB_VAR),
'--location=${}'.format(_GKE_LOCATION_SUB_VAR),
'--timeout=24h' # Set this to max value allowed for a build so that
# this step never times out. We prefer the timeout given to the build
# to take precedence.
],
))
build.substitutions = cloudbuild_util.EncodeSubstitutions(
_BuildSubstitutionsDict(dockerfile_path, app_name, config_path,
namespace, expose_port, cluster, location,
gcs_config_staging_path), messages)
build.tags = _DEFAULT_BUILD_TAGS[:]
if build_tags:
for tag in build_tags:
build.tags.append(tag)
build.options = messages.BuildOptions()
build.options.substitutionOption = messages.BuildOptions.SubstitutionOptionValueValuesEnum.ALLOW_LOOSE
return build
def CreatePRPreviewBuildTrigger(messages, name, description, build_timeout,
github_repo_owner, github_repo_name,
pr_pattern, preview_expiry_days,
comment_control, dockerfile_path, app_name,
config_path, expose_port,
gcs_config_staging_path, cluster, location,
build_tags, build_trigger_tags):
"""Creates the Cloud BuildTrigger config that deploys an application when triggered by a PR create/update.
Args:
messages: Cloud Build messages module. This is the value returned from
cloudbuild_util.GetMessagesModule().
name: Trigger name, which must be unique amongst all triggers in a project.
description: Trigger description.
build_timeout: An optional maximum time a triggered build is run before it
times out. For example, "2h15m5s" is 2 hours, 15 minutes, and 5 seconds.
If you do not specify a unit, seconds is assumed. If this value is None, a
timeout is not set.
github_repo_owner: A GitHub repo owner to be used in the trigger's github
field.
github_repo_name: A GitHub repo name to be used in the trigger's github
field.
pr_pattern: A regex value that is the base branch that the PR is targeting,
which triggers the creation of the PR preview deployment.
preview_expiry_days: How long a deployed preview application can exist
before it is expired, in days, that is set to a substitution variable.
comment_control: Whether or not a user must comment /gcbrun to trigger
the deployment build.
dockerfile_path: An optional path to the source repository's Dockerfile,
relative to the source repository's root directory that is set to a
substitution variable. If this value is not provided, 'Dockerfile' is
used.
app_name: An optional app name that is set to a substitution variable.
If this value is None, the substitution variable is set to '' to indicate
its absence.
config_path: An optional path to the source repository's Kubernetes configs,
relative to the source repository's root directory that is set to a
substitution variable. If this value is None, the substitution variable is
set to '' to indicate its absence.
expose_port: An optional port that the deployed application listens to that
is set to a substitution variable. If this value is None, the substitution
variable is set to 0 to indicate its absence.
gcs_config_staging_path: An optional path to a GCS subdirectory to copy
application configs that is set to a substitution variable. If this value
is None, the substitution variable is set to '' to indicate its absence.
cluster: The name of the target cluster to deploy to that is set to a
substitution variable.
location: The zone/region of the target cluster to deploy to that is set to
a substitution variable.
build_tags: Tags to append to build tags in addition to default tags.
build_trigger_tags: Tags to append to build trigger tags in addition to
default tags.
Returns:
messages.BuildTrigger, the Cloud BuildTrigger config.
"""
substitutions = _BaseBuildSubstitutionsDict(dockerfile_path, app_name,
config_path, expose_port,
cluster, location,
gcs_config_staging_path)
substitutions[_PREVIEW_EXPIRY_SUB_VAR] = six.text_type(preview_expiry_days)
build = messages.Build(
steps=[
_BuildBuildStep(messages, _IMAGE),
_PushBuildStep(messages, _IMAGE),
messages.BuildStep(id=_PREPARE_DEPLOY_BUILD_STEP_ID,
name=_GKE_DEPLOY_PROD,
entrypoint='sh',
args=['-c', _PREPARE_PREVIEW_DEPLOY_SCRIPT]),
_SaveConfigsBuildStep(messages),
messages.BuildStep(id=_APPLY_DEPLOY_BUILD_STEP_ID,
name=_GKE_DEPLOY_PROD,
entrypoint='sh',
args=['-c', _APPLY_PREVIEW_DEPLOY_SCRIPT]),
messages.BuildStep(id=_ANNOTATE_PREVIEW_NAMESPACE_BUILD_STEP_ID,
name='gcr.io/cloud-builders/kubectl',
entrypoint='sh',
args=['-c', _ANNOTATE_PREVIEW_NAMESPACE_SCRIPT])
],
substitutions=cloudbuild_util.EncodeSubstitutions(
substitutions, messages),
options=messages.BuildOptions(
substitutionOption=messages.BuildOptions.
SubstitutionOptionValueValuesEnum.ALLOW_LOOSE),
images=[_IMAGE],
artifacts=messages.Artifacts(
objects=messages.ArtifactObjects(location='gs://' +
_EXPANDED_CONFIGS_PATH_DYNAMIC,
paths=['output/expanded/*'])))
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
build.timeout = six.text_type(build_timeout_secs) + 's'
build.tags = _DEFAULT_PR_PREVIEW_TAGS[:]
if build_tags:
for tag in build_tags:
build.tags.append(tag)
github_config = messages.GitHubEventsConfig(
owner=github_repo_owner,
name=github_repo_name,
pullRequest=messages.PullRequestFilter(branch=pr_pattern))
if comment_control:
github_config.pullRequest.commentControl = messages.PullRequestFilter.CommentControlValueValuesEnum.COMMENTS_ENABLED
build_trigger = messages.BuildTrigger(
name=name,
description=description,
build=build,
github=github_config,
substitutions=cloudbuild_util.EncodeTriggerSubstitutions(
substitutions, messages))
build_trigger.tags = _DEFAULT_PR_PREVIEW_TAGS[:]
if build_trigger_tags:
for tag in build_trigger_tags:
build_trigger.tags.append(tag)
return build_trigger
def _CreateBuildFromArgs(self, args, messages):
"""Creates the Cloud Build config from the arguments.
Args:
args: argsparse object from the DeployGKE command.
messages: Cloud Build messages module.
Returns:
messages.Build, the Cloud Build config.
"""
build = messages.Build(steps=[], tags=_CLOUD_BUILD_DEPLOY_TAGS)
if args.app_name:
build.tags.append(args.app_name)
build_timeout = properties.VALUES.builds.timeout.Get()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
build.timeout = six.text_type(build_timeout_secs) + 's'
if args.source is None:
if args.tag or args.tag_default:
raise c_exceptions.RequiredArgumentException(
'SOURCE',
'required to build container image provided by --tag or --tag-default.'
)
if args.config:
raise c_exceptions.RequiredArgumentException(
'SOURCE',
'required because --config is a relative path in the '
'source directory.')
if args.source and args.image and not args.config:
raise c_exceptions.InvalidArgumentException(
'SOURCE', 'Source should not be provided when no Kubernetes '
'configs and no docker builds are required.')
if args.tag_default:
if args.app_name:
default_name = args.app_name
elif os.path.isdir(args.source):
default_name = os.path.basename(os.path.abspath(args.source))
else:
raise c_exceptions.InvalidArgumentException(
'--tag-default',
'No default container image name available. Please provide an '
'app name with --app-name, or provide a valid --tag.')
if args.app_version:
default_tag = args.app_version
elif git.IsGithubRepository(
args.source) and not git.HasPendingChanges(args.source):
default_tag = git.GetShortGitHeadRevision(args.source)
if not default_tag:
raise c_exceptions.InvalidArgumentException(
'--tag-default',
'No default tag available, no commit sha at HEAD of source '
'repository available for tag. Please provide an app version '
'with --app-version, or provide a valid --tag.')
else:
raise c_exceptions.InvalidArgumentException(
'--tag-default',
'No default container image tag available. Please provide an app '
'version with --app-version, or provide a valid --tag.')
args.tag = 'gcr.io/$PROJECT_ID/{name}:{tag}'.format(
name=default_name, tag=default_tag)
if args.tag:
if (properties.VALUES.builds.check_tag.GetBool()
and 'gcr.io/' not in args.tag):
raise c_exceptions.InvalidArgumentException(
'--tag',
'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.'
)
build.steps.append(
messages.BuildStep(
name='gcr.io/cloud-builders/docker',
args=[
'build', '--network', 'cloudbuild', '--no-cache', '-t',
args.tag, '.'
],
))
build.steps.append(
messages.BuildStep(name='gcr.io/cloud-builders/docker',
args=['push', args.tag]))
if args.image and (properties.VALUES.builds.check_tag.GetBool()
and 'gcr.io/' not in args.image):
raise c_exceptions.InvalidArgumentException(
'--image',
'Image value must be in the gcr.io/* or *.gcr.io/* namespace.')
if args.expose and args.expose < 0:
raise c_exceptions.InvalidArgumentException(
'EXPOSE', 'port number is invalid')
self._StageSourceAndConfigFiles(args, messages, build)
image = args.image if args.image else args.tag
deploy_step = messages.BuildStep(
name=_GKE_DEPLOY_PROD,
args=[
'run',
'--image={}'.format(image),
'--cluster={}'.format(args.cluster),
'--location={}'.format(args.location),
'--namespace={}'.format(args.namespace),
'--output=output',
'--label=gcb-build-id=$BUILD_ID',
],
)
image_name = image.split('/')[-1]
image_with_digest = image_name.split('@')
image_with_tag = image_name.split(':')
if args.app_name:
deploy_step.args.append('--app={}'.format(args.app_name))
else:
if len(image_with_digest) > 1:
deploy_step.args.append('--app={}'.format(
image_with_digest[0]))
else:
deploy_step.args.append('--app={}'.format(image_with_tag[0]))
if args.app_version:
deploy_step.args.append('--version={}'.format(args.app_version))
elif len(image_with_digest) == 1 and len(image_with_tag) > 1:
deploy_step.args.append('--version={}'.format(image_with_tag[1]))
elif args.source:
if git.IsGithubRepository(
args.source) and not git.HasPendingChanges(args.source):
short_sha = git.GetShortGitHeadRevision(args.source)
if short_sha:
deploy_step.args.append('--version={}'.format(short_sha))
if args.config:
deploy_step.args.append('--filename={}'.format(args.config))
if args.expose:
deploy_step.args.append('--expose={}'.format(args.expose))
if build.timeout is not None:
deploy_step.args.append('--timeout={}'.format(build.timeout))
# Append before the gsutil copy step
build.steps.insert(-1, deploy_step)
return build
def _ConvertProtoToIsoDuration(proto_duration_str):
"""Convert a given 'proto duration' string to an ISO8601 duration string."""
return times.FormatDuration(times.ParseDuration(proto_duration_str, True))
def ParseValidityFlag(args): """Parses the validity from args.""" return times.FormatDurationForJson(times.ParseDuration(args.validity))
def CreateBuildConfig(tag, no_cache, messages, substitutions, arg_config,
is_specified_source, no_source, source,
gcs_source_staging_dir, ignore_file, arg_gcs_log_dir,
arg_machine_type, arg_disk_size):
"""Returns a build config."""
# Get the build timeout.
build_timeout = properties.VALUES.builds.timeout.Get()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
timeout_str = six.text_type(build_timeout_secs) + 's'
else:
timeout_str = None
if tag is not None:
if (properties.VALUES.builds.check_tag.GetBool()
and 'gcr.io/' not in tag):
raise c_exceptions.InvalidArgumentException(
'--tag',
'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.')
if properties.VALUES.builds.use_kaniko.GetBool():
if no_cache:
ttl = '0h'
else:
ttl = '{}h'.format(
properties.VALUES.builds.kaniko_cache_ttl.Get())
build_config = messages.Build(
steps=[
messages.BuildStep(
name=properties.VALUES.builds.kaniko_image.Get(),
args=[
'--destination',
tag,
'--cache',
'--cache-ttl',
ttl,
'--cache-dir',
'',
],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(
substitutions, messages))
else:
if no_cache:
raise c_exceptions.InvalidArgumentException(
'no-cache',
'Cannot specify --no-cache if builds/use_kaniko property is '
'False')
build_config = messages.Build(
images=[tag],
steps=[
messages.BuildStep(
name='gcr.io/cloud-builders/docker',
args=[
'build', '--network', 'cloudbuild', '--no-cache',
'-t', tag, '.'
],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(
substitutions, messages))
elif arg_config is not None:
if no_cache:
raise c_exceptions.ConflictingArgumentsException(
'--config', '--no-cache')
if not arg_config:
raise c_exceptions.InvalidArgumentException(
'--config', 'Config file path must not be empty.')
build_config = config.LoadCloudbuildConfigFromPath(
arg_config, messages, params=substitutions)
else:
raise c_exceptions.OneOfArgumentsRequiredException(
['--tag', '--config'],
'Requires either a docker tag or a config file.')
# If timeout was set by flag, overwrite the config file.
if timeout_str:
build_config.timeout = timeout_str
# Set the source for the build config.
default_gcs_source = False
default_bucket_name = None
if gcs_source_staging_dir is None:
default_gcs_source = True
default_bucket_name = staging_bucket_util.GetDefaultStagingBucket()
gcs_source_staging_dir = 'gs://{}/source'.format(default_bucket_name)
gcs_client = storage_api.StorageClient()
# --no-source overrides the default --source.
if not is_specified_source and no_source:
source = None
gcs_source_staging = None
if source:
suffix = '.tgz'
if source.startswith('gs://') or os.path.isfile(source):
_, suffix = os.path.splitext(source)
# Next, stage the source to Cloud Storage.
staged_object = '{stamp}-{uuid}{suffix}'.format(
stamp=times.GetTimeStampFromDateTime(times.Now()),
uuid=uuid.uuid4().hex,
suffix=suffix,
)
gcs_source_staging_dir = resources.REGISTRY.Parse(
gcs_source_staging_dir, collection='storage.objects')
# We create the bucket (if it does not exist) first. If we do an existence
# check and then create the bucket ourselves, it would be possible for an
# attacker to get lucky and beat us to creating the bucket. Block on this
# creation to avoid this race condition.
gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket)
# If no bucket is specified (for the source `default_gcs_source`), check
# that the default bucket is also owned by the project (b/33046325).
if default_gcs_source and not staging_bucket_util.BucketIsInProject(
gcs_client, default_bucket_name):
raise c_exceptions.RequiredArgumentException(
'gcs-source-staging-dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket using '
'--gcs-source-staging-dir.'.format(default_bucket_name))
if gcs_source_staging_dir.object:
staged_object = gcs_source_staging_dir.object + '/' + staged_object
gcs_source_staging = resources.REGISTRY.Create(
collection='storage.objects',
bucket=gcs_source_staging_dir.bucket,
object=staged_object)
if source.startswith('gs://'):
gcs_source = resources.REGISTRY.Parse(source,
collection='storage.objects')
staged_source_obj = gcs_client.Rewrite(gcs_source,
gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
if not os.path.exists(source):
raise c_exceptions.BadFileException(
'could not find source [{src}]'.format(src=source))
if os.path.isdir(source):
source_snapshot = snapshot.Snapshot(source,
ignore_file=ignore_file)
size_str = resource_transform.TransformSize(
source_snapshot.uncompressed_size)
log.status.Print(
'Creating temporary tarball archive of {num_files} file(s)'
' totalling {size} before compression.'.format(
num_files=len(source_snapshot.files), size=size_str))
staged_source_obj = source_snapshot.CopyTarballToGCS(
gcs_client, gcs_source_staging, ignore_file=ignore_file)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
elif os.path.isfile(source):
unused_root, ext = os.path.splitext(source)
if ext not in _ALLOWED_SOURCE_EXT:
raise c_exceptions.BadFileException(
'Local file [{src}] is none of ' +
', '.join(_ALLOWED_SOURCE_EXT))
log.status.Print('Uploading local file [{src}] to '
'[gs://{bucket}/{object}].'.format(
src=source,
bucket=gcs_source_staging.bucket,
object=gcs_source_staging.object,
))
staged_source_obj = gcs_client.CopyFileToGCS(
source, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
# No source
if not no_source:
raise c_exceptions.InvalidArgumentException(
'--no-source', 'To omit source, use the --no-source flag.')
# Set a Google Cloud Storage directory to hold build logs.
if arg_gcs_log_dir:
gcs_log_dir = resources.REGISTRY.Parse(arg_gcs_log_dir,
collection='storage.objects')
build_config.logsBucket = ('gs://' + gcs_log_dir.bucket + '/' +
gcs_log_dir.object)
# Set the machine type used to run the build.
if arg_machine_type is not None:
machine_type = flags.GetMachineType(arg_machine_type)
if not build_config.options:
build_config.options = messages.BuildOptions()
build_config.options.machineType = machine_type
# Set the disk size used to run the build.
if arg_disk_size is not None:
disk_size = compute_utils.BytesToGb(arg_disk_size)
if not build_config.options:
build_config.options = messages.BuildOptions()
build_config.options.diskSizeGb = int(disk_size)
return build_config
def GetConfigurationChanges(args):
"""Returns a list of changes to Configuration, based on the flags set."""
changes = []
changes.extend(_GetScalingChanges(args))
if _HasEnvChanges(args):
changes.append(_GetEnvChanges(args))
if _HasTrafficChanges(args):
changes.append(_GetTrafficChanges(args))
if _HasCloudSQLChanges(args):
region = GetRegion(args)
project = (
getattr(args, 'project', None) or
properties.VALUES.core.project.Get(required=True))
_CheckCloudSQLApiEnablement()
changes.append(config_changes.CloudSQLChanges(project, region, args))
if _HasSecretsChanges(args):
changes.extend(_GetSecretsChanges(args))
if _HasConfigMapsChanges(args):
changes.extend(_GetConfigMapsChanges(args))
if 'no_traffic' in args and args.no_traffic:
changes.append(config_changes.NoTrafficChange())
if 'cpu' in args and args.cpu:
changes.append(config_changes.ResourceChanges(cpu=args.cpu))
if 'memory' in args and args.memory:
changes.append(config_changes.ResourceChanges(memory=args.memory))
if 'concurrency' in args and args.concurrency:
changes.append(config_changes.ConcurrencyChanges(
concurrency=args.concurrency))
if 'timeout' in args and args.timeout:
try:
# A bare number is interpreted as seconds.
timeout_secs = int(args.timeout)
except ValueError:
timeout_duration = times.ParseDuration(args.timeout)
timeout_secs = int(timeout_duration.total_seconds)
if timeout_secs <= 0:
raise ArgumentError(
'The --timeout argument must be a positive time duration.')
changes.append(config_changes.TimeoutChanges(timeout=timeout_secs))
if 'service_account' in args and args.service_account:
changes.append(
config_changes.ServiceAccountChanges(
service_account=args.service_account))
if _HasLabelChanges(args):
additions = (
args.labels
if _FlagIsExplicitlySet(args, 'labels') else args.update_labels)
diff = labels_util.Diff(
additions=additions,
subtractions=args.remove_labels,
clear=args.clear_labels)
if diff.MayHaveUpdates():
changes.append(config_changes.LabelChanges(diff))
if 'revision_suffix' in args and args.revision_suffix:
changes.append(config_changes.RevisionNameChanges(args.revision_suffix))
if 'vpc_connector' in args and args.vpc_connector:
changes.append(config_changes.VpcConnectorChange(args.vpc_connector))
if 'clear_vpc_connector' in args and args.clear_vpc_connector:
changes.append(config_changes.ClearVpcConnectorChange())
if 'connectivity' in args and args.connectivity:
if args.connectivity == 'internal':
changes.append(config_changes.EndpointVisibilityChange(True))
elif args.connectivity == 'external':
changes.append(config_changes.EndpointVisibilityChange(False))
if 'command' in args and args.command is not None:
# Allow passing an empty string here to reset the field
changes.append(config_changes.ContainerCommandChange(args.command))
if 'args' in args and args.args is not None:
# Allow passing an empty string here to reset the field
changes.append(config_changes.ContainerArgsChange(args.args))
if _FlagIsExplicitlySet(args, 'port'):
changes.append(config_changes.ContainerPortChange(port=args.port))
if _FlagIsExplicitlySet(args, 'use_http2'):
changes.append(config_changes.ContainerPortChange(use_http2=args.use_http2))
return changes
def CreateBuild(messages, build_timeout, build_and_push, staged_source, image,
dockerfile_path, app_name, app_version, config_path, namespace,
expose_port, gcs_config_staging_path, cluster, location,
build_tags):
"""Creates the Cloud Build config to run.
Args:
messages: Cloud Build messages module. This is the value returned from
cloudbuild_util.GetMessagesModule().
build_timeout: An optional maximum time a build is run before it times out.
For example, "2h15m5s" is 2 hours, 15 minutes, and 5 seconds. If you do
not specify a unit, seconds is assumed. If this value is None, a timeout
is not set.
build_and_push: If True, the created build will have Build and Push steps.
staged_source: An optional GCS object for a staged source repository. The
object must have bucket, name, and generation fields. If this value is
None, the created build will not have a source.
image: The image that will deployed and optionally built beforehand. The
image can include a tag or digest.
dockerfile_path: An optional path to the source repository's Dockerfile,
relative to the source repository's root directory. If this value is not
provided, 'Dockerfile' is used.
app_name: An optional app name that is set to a substitution variable.
If this value is None, the substitution variable is set to '' to indicate
its absence.
app_version: A app version that is set to the deployed application's
version. If this value is None, the version will be set to '' to indicate
its absence.
config_path: An optional path to the source repository's Kubernetes configs,
relative to the source repository's root directory that is set to a
substitution variable. If this value is None, the substitution variable is
set to '' to indicate its absence.
namespace: An optional Kubernetes namespace of the cluster to deploy to that
is set to a substitution variable. If this value is None, the substitution
variable is set to 'default'.
expose_port: An optional port that the deployed application listens to that
is set to a substitution variable. If this value is None, the substitution
variable is set to 0 to indicate its absence.
gcs_config_staging_path: An optional path to a GCS subdirectory to copy
application configs that is set to a substitution variable. If this value
is None, the substitution variable is set to '' to indicate its absence.
cluster: The name of the target cluster to deploy to.
location: The zone/region of the target cluster to deploy to.
build_tags: Tags to append to build tags in addition to default tags.
Returns:
messages.Build, the Cloud Build config.
"""
build = messages.Build()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
build.timeout = six.text_type(build_timeout_secs) + 's'
if staged_source:
build.source = messages.Source(storageSource=messages.StorageSource(
bucket=staged_source.bucket,
object=staged_source.name,
generation=staged_source.generation))
if config_path is None:
config_path = ''
if not expose_port:
expose_port = '0'
else:
expose_port = six.text_type(expose_port)
if app_version is None:
app_version = ''
build.steps = []
if build_and_push:
build.steps.append(_BuildBuildStep(messages, image))
build.steps.append(_PushBuildStep(messages, image))
build.steps.append(
messages.BuildStep(
id=_PREPARE_DEPLOY_BUILD_STEP_ID,
name=_GKE_DEPLOY_PROD,
args=[
'prepare',
'--filename=${}'.format(_K8S_YAML_PATH_SUB_VAR),
'--image={}'.format(image),
'--app=${}'.format(_APP_NAME_SUB_VAR),
'--version={}'.format(app_version),
'--namespace=${}'.format(_K8S_NAMESPACE_SUB_VAR),
'--output=output',
'--annotation=gcb-build-id=$BUILD_ID,${}'.format(
_K8S_ANNOTATIONS_SUB_VAR
), # You cannot embed a substitution
# variable in another, so gcb-build-id=$BUILD_ID must be hard-coded.
'--expose=${}'.format(_EXPOSE_PORT_SUB_VAR)
],
))
build.steps.append(_SaveConfigsBuildStep(messages))
build.steps.append(
messages.BuildStep(
id=_APPLY_DEPLOY_BUILD_STEP_ID,
name=_GKE_DEPLOY_PROD,
args=[
'apply',
'--filename=output/expanded',
'--namespace=${}'.format(_K8S_NAMESPACE_SUB_VAR),
'--cluster=${}'.format(_GKE_CLUSTER_SUB_VAR),
'--location=${}'.format(_GKE_LOCATION_SUB_VAR),
'--timeout=24h' # Set this to max value allowed for a build so that
# this step never times out. We prefer the timeout given to the build
# to take precedence.
],
))
substitutions = _BaseBuildSubstitutionsDict(dockerfile_path, app_name,
config_path, expose_port,
cluster, location,
gcs_config_staging_path)
if namespace is None:
namespace = 'default'
substitutions[_K8S_NAMESPACE_SUB_VAR] = namespace
build.substitutions = cloudbuild_util.EncodeSubstitutions(
substitutions, messages)
build.tags = _DEFAULT_TAGS[:]
if build_tags:
for tag in build_tags:
build.tags.append(tag)
build.options = messages.BuildOptions()
build.options.substitutionOption = messages.BuildOptions.SubstitutionOptionValueValuesEnum.ALLOW_LOOSE
if build_and_push:
build.images = [image]
build.artifacts = messages.Artifacts(
objects=messages.ArtifactObjects(location='gs://' +
_EXPANDED_CONFIGS_PATH_DYNAMIC,
paths=['output/expanded/*']))
return build