def ParseFutureDatetime(s):
"""Parses a string value into a future Datetime object."""
dt = arg_parsers.Datetime.Parse(s)
if dt < times.Now():
raise arg_parsers.ArgumentTypeError(
'Date/time must be in the future: {0}'.format(s))
return dt
def Run(self, args):
"""Run the list command."""
if args.include_expired:
# The (deprecated, hidden) --include-expired argument is equivalent to
# --include_inactive=unlimited
log.warn(
'The --include-expired flag has been deprecated. Please use '
'--include-inactive=unlimited instead.')
args.include_inactive = None
project_id = properties.VALUES.core.project.Get(required=True)
debugger = debug.Debugger(project_id)
debuggee = debugger.FindDebuggee(args.target)
logpoints = debuggee.ListBreakpoints(
args.id_or_location_regexp,
include_all_users=args.all_users,
include_inactive=(args.include_inactive != 0),
restrict_to_type=debugger.LOGPOINT_TYPE)
# Filter any results more than include_inactive seconds old.
# include_inactive may be None, which means we do not want to filter the
# results.
if args.include_inactive > 0:
cutoff_time = (times.Now(times.UTC) -
datetime.timedelta(seconds=args.include_inactive))
logpoints = [
lp for lp in logpoints if _ShouldInclude(lp, cutoff_time)
]
return logpoints
def ParseExpireTime(expiration_value):
"""Parse flag value into Datetime format for expireTime."""
# expiration_value could be in Datetime format or Duration format.
datetime = (
times.ParseDuration(expiration_value).GetRelativeDateTime(
times.Now(times.UTC)))
parsed_datetime = times.FormatDateTime(
datetime, '%Y-%m-%dT%H:%M:%S.%6f%Ez', tzinfo=times.UTC)
return parsed_datetime
def RecentlyModified(update_time):
"""Checks if the trigger with the given update_time was recently modified.
Args:
update_time: str, the time when the trigger was last modified.
Returns:
True if the trigger was recently modified and might not be ready for use.
"""
update_dt = times.ParseDateTime(update_time)
max_duration = iso_duration.Duration(minutes=MAX_READY_LATENCY_MINUTES)
ready_dt = times.GetDateTimePlusDuration(update_dt, max_duration)
return times.Now() < ready_dt
def _WriteAgentLogs():
"""Writes logs from the GKE Connect agent pod to a temporary file."""
logs, err = kube_client.Logs(namespace, agent_pod_name)
if err:
log.warning('Could not fetch agent pod logs: {}'.format(err))
return
_, tmp_file = tempfile.mkstemp(
suffix='_{}.log'.format(times.Now().strftime('%Y%m%d-%H%M%S')),
prefix='gke_connect_',
)
files.WriteFileContents(tmp_file, logs, private=True)
log.status.Print('GKE Connect pod logs saved to [{}]'.format(tmp_file))
def GenerateCertId():
"""Generate a certificate id with the date and two length 3 alphanum strings.
E.G. YYYYMMDD-ABC-DEF.
Returns:
The generated certificate id string.
"""
alphanum = string.ascii_uppercase + string.digits
alphanum_rand1 = ''.join(random.choice(alphanum) for i in range(3))
alphanum_rand2 = ''.join(random.choice(alphanum) for i in range(3))
date_str = times.FormatDateTime(times.Now(), '%Y%m%d')
return '{}-{}-{}'.format(date_str, alphanum_rand1, alphanum_rand2)
def testListDefaultSort(self):
# Verify that list is sorted by default with active logpoints first,
# inactive logpoints next, and both groups sorted by creation time.
logpoints = [
self.messages.Breakpoint(
id='dummy-id-{0}'.format(i),
action=self.messages.Breakpoint.ActionValueValuesEnum.LOG,
logLevel=self.messages.Breakpoint.LogLevelValueValuesEnum.INFO,
logMessageFormat='message',
location=self.messages.SourceLocation(path='myfile', line=i))
for i in range(0, 10)
]
active_order = [7, 9, 0, 4, 2, 5]
inactive_order = [1, 8, 6, 3]
now = times.Now(times.UTC)
self.StartObjectPatch(times, 'Now', return_value=now)
base_time = now - datetime.timedelta(hours=1, minutes=1)
offset_sec = 0
for i in active_order:
logpoints[i].createTime = (
base_time +
datetime.timedelta(seconds=offset_sec)).isoformat()
offset_sec += 1
offset_sec = 0
for i in inactive_order:
logpoints[i].isFinalState = True
logpoints[i].createTime = (
base_time +
datetime.timedelta(seconds=offset_sec)).isoformat()
logpoints[i].finalTime = now.isoformat()
offset_sec += 1
logpoints = [self.debuggee.AddTargetInfo(lp) for lp in logpoints]
list_mock = self.StartObjectPatch(debug.Debuggee,
'ListBreakpoints',
return_value=logpoints)
self.RunDebug(['logpoints', 'list'])
list_mock.assert_called_once_with(
None,
resource_ids=[],
include_all_users=True,
include_inactive=True,
restrict_to_type=debug.Debugger.LOGPOINT_TYPE)
# Look for all the IDs in the expected order. Remove '\n' characters
# from the output because '.' doesn't match newline, and '(.|\s)' causes
# the Python re.search function to time out.
self.AssertOutputMatches('.*'.join(
logpoints[i].id for i in active_order + inactive_order),
actual_filter=lambda s: s.replace('\n', ' '))
def GetSSHKeyExpirationFromArgs(args):
"""Converts flags to an ssh key expiration in datetime and micros."""
if args.ssh_key_expiration:
# this argument is checked in ParseFutureDatetime to be sure that it
# is not already expired. I.e. the expiration should be in the future.
expiration = args.ssh_key_expiration
elif args.ssh_key_expire_after:
expiration = times.Now() + datetime.timedelta(
seconds=args.ssh_key_expire_after)
else:
return None, None
expiration_micros = times.GetTimeStampFromDateTime(expiration) * 1e6
return expiration, int(expiration_micros)
def _PrepareSSHKeysValue(ssh_keys):
"""Returns a string appropriate for the metadata.
Expired SSH keys are always removed.
Then Values are taken from the tail until either all values are taken or
_MAX_METADATA_VALUE_SIZE_IN_BYTES is reached, whichever comes first. The
selected values are then reversed. Only values at the head of the list will be
subject to removal.
Args:
ssh_keys: A list of keys. Each entry should be one key.
Returns:
A new-line-joined string of SSH keys.
"""
keys = []
bytes_consumed = 0
now = times.LocalizeDateTime(times.Now(), times.UTC)
for key in reversed(ssh_keys):
try:
expiration = _SSHKeyExpiration(key)
expired = expiration is not None and expiration < now
if expired:
continue
except (ValueError, times.DateTimeSyntaxError,
times.DateTimeValueError) as exc:
# Unable to get expiration, so treat it like it is unexpiring.
log.warning(
'Treating {0!r} as unexpiring, since unable to parse: {1}'.
format(key, exc))
num_bytes = len(key + '\n')
if bytes_consumed + num_bytes > constants.MAX_METADATA_VALUE_SIZE_IN_BYTES:
prompt_message = (
'The following SSH key will be removed from your '
'project because your SSH keys metadata value has '
'reached its maximum allowed size of {0} bytes: {1}')
prompt_message = prompt_message.format(
constants.MAX_METADATA_VALUE_SIZE_IN_BYTES, key)
console_io.PromptContinue(message=prompt_message,
cancel_on_no=True)
else:
keys.append(key)
bytes_consumed += num_bytes
keys.reverse()
return '\n'.join(keys)
def ParseExpireTime(s):
"""Return timedelta TTL for a cluster.
Args:
s: expireTime string timestamp in RFC3339 format.
Returns:
datetime.timedelta of time remaining before cluster expiration.
Raises:
TypeError, ValueError if time could not be parsed.
"""
if not s:
return None
expire_dt = times.ParseDateTime(s)
if not expire_dt:
return None
return expire_dt - times.Now(expire_dt.tzinfo)
def Run(self, args):
"""Run the list command."""
project_id = properties.VALUES.core.project.Get(required=True)
debugger = debug.Debugger(project_id)
debuggee = debugger.FindDebuggee(args.target)
snapshots = debuggee.ListBreakpoints(
args.location, resource_ids=args.ids, include_all_users=args.all_users,
include_inactive=(args.include_inactive != 0),
restrict_to_type=debugger.SNAPSHOT_TYPE)
# Filter any results more than include_inactive seconds old.
# include_inactive may be None, which means we do not want to filter the
# results.
if args.include_inactive:
cutoff_time = (times.Now(times.UTC) -
datetime.timedelta(seconds=args.include_inactive))
snapshots = [s for s in snapshots if _ShouldInclude(s, cutoff_time)]
return snapshots
def _WaitForSSHKeysToPropagate(self, ssh_helper, remote, identity_file,
user, instance, options):
"""Waits for SSH keys to propagate in order to SSH to the instance."""
ssh_helper.EnsureSSHKeyExists(
self.client, user, instance,
ssh_helper.GetProject(
self.client,
properties.VALUES.core.project.Get(required=True)),
times.Now() + datetime.timedelta(seconds=300))
ssh_poller = ssh.SSHPoller(remote=remote,
identity_file=identity_file,
options=options,
max_wait_ms=120 * 1000)
try:
ssh_poller.Poll(ssh_helper.env, force_connect=True)
except retry.WaitException:
raise ssh_utils.NetworkError()
def GenerateCertId():
"""Generate a certificate id with the date and two length 3 alphanum strings.
E.G. YYYYMMDD-ABC-DEF.
Returns:
The generated certificate id string.
"""
# Avoid name collisions in certificate id generation. Normal random.choice
# seeds on system time, which might not be sufficiently random of a seed.
# SystemRandom uses other sources from the OS to generate the random ID.
sys_rng = random.SystemRandom()
alphanum = string.ascii_uppercase + string.digits
alphanum_rand1 = ''.join(sys_rng.choice(alphanum) for i in range(3))
alphanum_rand2 = ''.join(sys_rng.choice(alphanum) for i in range(3))
date_str = times.FormatDateTime(times.Now(), '%Y%m%d')
return '{}-{}-{}'.format(date_str, alphanum_rand1, alphanum_rand2)
def testListIncludeExclude(self):
now = times.Now(times.UTC)
self.StartObjectPatch(times, 'Now', return_value=now)
create_time = now - datetime.timedelta(hours=1, minutes=1)
logpoints = [
self.messages.Breakpoint(
id='dummy-id-{0}'.format(i),
createTime=create_time.isoformat(),
action=self.messages.Breakpoint.ActionValueValuesEnum.LOG,
logLevel=self.messages.Breakpoint.LogLevelValueValuesEnum.INFO,
isFinalState=True,
logMessageFormat='message',
location=self.messages.SourceLocation(path='myfile', line=i))
for i in range(0, 10)
]
exclude_indices = [0, 2, 4, 5, 7, 9]
include_indices = [1, 3, 6, 8]
for e in exclude_indices:
# Set to a time before the default cutoff (so it will be excluded).
logpoints[e].finalTime = (
now - datetime.timedelta(seconds=301)).isoformat()
for i in include_indices:
# Set to a time after the default cutoff (so it will be included).
logpoints[i].finalTime = (
now - datetime.timedelta(seconds=299)).isoformat()
included_ids = [logpoints[i].id for i in include_indices]
excluded_ids = [logpoints[e].id for e in exclude_indices]
logpoints = [self.debuggee.AddTargetInfo(lp) for lp in logpoints]
list_mock = self.StartObjectPatch(debug.Debuggee,
'ListBreakpoints',
return_value=logpoints)
self.RunDebug(['logpoints', 'list'])
list_mock.assert_called_once_with(
None,
resource_ids=[],
include_all_users=True,
include_inactive=True,
restrict_to_type=debug.Debugger.LOGPOINT_TYPE)
for i in included_ids:
self.AssertOutputContains(i)
for e in excluded_ids:
self.AssertOutputNotContains(e)
def TriggerActiveTime(event_type, update_time):
"""Computes the time by which the trigger will become active.
Args:
event_type: str, the trigger's event type.
update_time: str, the time when the trigger was last modified.
Returns:
The active time as a string, or None if the trigger is already active.
"""
if not types.IsAuditLogType(event_type):
# The delay only applies to Audit Log triggers.
return None
update_dt = times.ParseDateTime(update_time)
delay = iso_duration.Duration(minutes=MAX_ACTIVE_DELAY_MINUTES)
active_dt = times.GetDateTimePlusDuration(update_dt, delay)
if times.Now() >= active_dt:
return None
return times.FormatDateTime(active_dt, fmt='%H:%M:%S', tzinfo=times.LOCAL)
def _IsTTLSafe(ttl, obj):
"""Determines whether a GCS object is close to end-of-life.
In order to reduce false negative rate (objects that are close to deletion but
aren't marked as such) the returned filter is forward-adjusted with
_TTL_MARGIN.
Args:
ttl: datetime.timedelta, TTL of objects, or None if no TTL.
obj: storage object to check.
Returns:
True if the ojbect is safe or False if it is approaching end of life.
"""
if ttl is None:
return True
now = times.Now(times.UTC)
delta = ttl - _TTL_MARGIN
return (now - obj.timeCreated) <= delta
def CreateCloudEvent(event_id, event_type, event_source, event_data,
event_attributes):
"""Transform args to a valid cloud event.
Args:
event_id: The id of a published event.
event_type: The event type of a published event.
event_source: The event source of a published event.
event_data: The event data of a published event.
event_attributes: The event attributes of a published event. It can be
repeated to add more attributes.
Returns:
valid CloudEvent.
"""
cloud_event = {
"@type": "type.googleapis.com/io.cloudevents.v1.CloudEvent",
"id": event_id,
"source": event_source,
"specVersion": "1.0",
"type": event_type,
"attributes": {
"time": {
"ceTimestamp":
times.FormatDateTime(times.Now())
},
"datacontenttype": {
"ceString": "application/json"
},
},
"textData": event_data
}
# Event attributes could be zero or more
# So it must be serialized into a dictionary
if event_attributes is not None:
for key, value in event_attributes.items():
cloud_event["attributes"][key] = {"ceString": value}
return cloud_event
def _TokenExpiresWithinWindow(expiry_window,
token_expiry_time,
max_window_seconds=3600):
"""Determines if token_expiry_time is within expiry_window_duration.
Calculates the amount of time between utcnow() and token_expiry_time and
returns true, if that amount is less thank the provided duration window. All
calculations are done in number of seconds for consistency.
Args:
expiry_window: string, Duration representing the amount of time between
now and token_expiry_time to compare against.
token_expiry_time: datetime, The time when token expires.
max_window_seconds: int, Maximum size of expiry window, in seconds.
Raises:
ValueError: If expiry_window is invalid or can not be parsed.
Returns:
True if token is expired or will expire with in the provided window,
False otherwise.
"""
try:
min_expiry = times.ParseDuration(expiry_window, default_suffix='s')
if min_expiry.total_seconds > max_window_seconds:
raise ValueError('Invalid expiry window duration [{}]: '
'Must be between 0s and 1h'.format(expiry_window))
except times.Error as e:
message = six.text_type(e).rstrip('.')
raise ValueError('Error Parsing expiry window duration '
'[{}]: {}'.format(expiry_window, message))
token_expiry_time = times.LocalizeDateTime(token_expiry_time,
tzinfo=dateutil.tz.tzutc())
window_end = times.GetDateTimePlusDuration(
times.Now(tzinfo=dateutil.tz.tzutc()), min_expiry)
return token_expiry_time <= window_end
def testListIncludesExpiredUnlimited(self):
now = times.Now(times.UTC)
self.StartObjectPatch(times, 'Now', return_value=now)
create_time = now - datetime.timedelta(days=2)
logpoints = [
self.messages.Breakpoint(
id='dummy-id-{0}'.format(i),
createTime=create_time.isoformat(),
action=self.messages.Breakpoint.ActionValueValuesEnum.LOG,
logLevel=self.messages.Breakpoint.LogLevelValueValuesEnum.INFO,
isFinalState=True,
logMessageFormat='message',
location=self.messages.SourceLocation(path='myfile', line=i))
for i in range(0, 10)
]
# Set logpoint 0 to a final time before well in the past.
logpoints[0].finalTime = (
now - datetime.timedelta(days=1, minutes=1)).isoformat()
logpoints = [self.debuggee.AddTargetInfo(lp) for lp in logpoints]
list_mock = self.StartObjectPatch(debug.Debuggee,
'ListBreakpoints',
return_value=logpoints)
self.RunDebug(['logpoints', 'list', '--include-inactive=unlimited'])
list_mock.assert_called_once_with(
None,
resource_ids=[],
include_all_users=True,
include_inactive=True,
restrict_to_type=debug.Debugger.LOGPOINT_TYPE)
for l in logpoints:
self.AssertOutputContains(l.id)
self.AssertOutputContains(
'USER_EMAIL LOCATION CONDITION LOG_LEVEL LOG_MESSAGE_FORMAT ID STATUS',
normalize_space=True)
def CreateRequest(args,
cleared_fields,
client,
resources,
minimal_action,
max_surge=None):
"""Create request helper for compute instance-groups managed rolling-action.
Args:
args: argparse namespace
cleared_fields: Fields which are left cleared, but should be send in request
client: The compute client
resources: The compute resources
minimal_action: MinimalActionValueValuesEnum value
max_surge: InstanceGroupManagerUpdatePolicy.maxSurge value
Returns:
ComputeInstanceGroupManagersPatchRequest or
ComputeRegionInstanceGroupManagersPatchRequest instance
"""
resource_arg = instance_groups_flags.MULTISCOPE_INSTANCE_GROUP_MANAGER_ARG
default_scope = compute_scope.ScopeEnum.ZONE
scope_lister = flags.GetDefaultScopeLister(client)
igm_ref = resource_arg.ResolveAsResource(args,
resources,
default_scope=default_scope,
scope_lister=scope_lister)
update_policy_type = (client.messages.InstanceGroupManagerUpdatePolicy.
TypeValueValuesEnum.PROACTIVE)
max_unavailable = update_instances_utils.ParseFixedOrPercent(
'--max-unavailable', 'max-unavailable', args.max_unavailable,
client.messages)
igm_info = managed_instance_groups_utils.GetInstanceGroupManagerOrThrow(
igm_ref, client)
versions = (igm_info.versions or [
client.messages.InstanceGroupManagerVersion(
instanceTemplate=igm_info.instanceTemplate)
])
current_time_str = str(times.Now(times.UTC))
for i, version in enumerate(versions):
version.name = '%d/%s' % (i, current_time_str)
update_policy = client.messages.InstanceGroupManagerUpdatePolicy(
maxSurge=max_surge,
maxUnavailable=max_unavailable,
minReadySec=args.min_ready,
minimalAction=minimal_action,
type=update_policy_type)
igm_resource = client.messages.InstanceGroupManager(
instanceTemplate=None, updatePolicy=update_policy, versions=versions)
if igm_ref.Collection() == 'compute.instanceGroupManagers':
service = client.apitools_client.instanceGroupManagers
request = client.messages.ComputeInstanceGroupManagersPatchRequest(
instanceGroupManager=igm_ref.Name(),
instanceGroupManagerResource=igm_resource,
project=igm_ref.project,
zone=igm_ref.zone)
elif igm_ref.Collection() == 'compute.regionInstanceGroupManagers':
service = client.apitools_client.regionInstanceGroupManagers
request = client.messages.ComputeRegionInstanceGroupManagersPatchRequest(
instanceGroupManager=igm_ref.Name(),
instanceGroupManagerResource=igm_resource,
project=igm_ref.project,
region=igm_ref.region)
# Due to 'Patch' semantics, we have to clear either 'fixed' or 'percent'.
# Otherwise, we'll get an error that both 'fixed' and 'percent' are set.
if max_surge is not None:
cleared_fields.append('updatePolicy.maxSurge.fixed' if max_surge.fixed
is None else 'updatePolicy.maxSurge.percent')
if max_unavailable is not None:
cleared_fields.append(
'updatePolicy.maxUnavailable.fixed' if max_unavailable.
fixed is None else 'updatePolicy.maxUnavailable.percent')
return (service, 'Patch', request)
def TransformDuration(r,
start='',
end='',
parts=3,
precision=3,
calendar=True,
unit=1,
undefined=''):
"""Formats the resource as an ISO 8601 duration string.
The [ISO 8601 Duration](https://en.wikipedia.org/wiki/ISO_8601#Durations)
format is: "[-]P[nY][nM][nD][T[nH][nM][n[.m]S]]". The 0 duration is "P0".
Otherwise at least one part will always be displayed. Negative durations are
prefixed by "-". "T" disambiguates months "P2M" to the left of "T" and minutes
"PT5M" to the right.
If the resource is a datetime then the duration of `resource - current_time`
is returned.
Args:
r: A JSON-serializable object.
start: The name of a start time attribute in the resource. The duration of
the `end - start` time attributes in resource is returned. If `end` is
not specified then the current time is used.
end: The name of an end time attribute in the resource. Defaults to
the current time if omitted. Ignored if `start` is not specified.
parts: Format at most this many duration parts starting with largest
non-zero part.
precision: Format the last duration part with precision digits after the
decimal point. Trailing "0" and "." are always stripped.
calendar: Allow time units larger than hours in formated durations if true.
Durations specifying hours or smaller units are exact across daylight
savings time boundaries. On by default. Use calendar=false to disable.
For example, if `calendar=true` then at the daylight savings boundary
2016-03-13T01:00:00 + P1D => 2016-03-14T01:00:00 but 2016-03-13T01:00:00 +
PT24H => 2016-03-14T03:00:00. Similarly, a +P1Y duration will be inexact
but "calendar correct", yielding the same month and day number next year,
even in leap years.
unit: Divide the resource numeric value by _unit_ to yield seconds.
undefined: Returns this value if the resource is not a valid timestamp.
Returns:
The ISO 8601 duration string for r or undefined if r is not a duration.
"""
try:
parts = int(parts)
precision = int(precision)
except ValueError:
return undefined
calendar = GetBooleanArgValue(calendar)
if start:
# Duration of ((end or Now()) - start).
# Get the datetime of both.
try:
start_datetime = times.ParseDateTime(_GetKeyValue(r, start))
end_value = _GetKeyValue(r, end) if end else None
if end_value:
end_datetime = times.ParseDateTime(end_value)
else:
end_datetime = times.Now(tzinfo=start_datetime.tzinfo)
except (AttributeError, ImportError, TypeError, ValueError):
return undefined
# Finally format the duration of the delta.
delta = end_datetime - start_datetime
return times.GetDurationFromTimeDelta(
delta=delta, calendar=calendar).Format(parts=parts,
precision=precision)
# Check if the resource is a float duration.
try:
duration = times.ParseDuration('PT{0}S'.format(float(r) / unit),
calendar=calendar)
return duration.Format(parts=parts, precision=precision)
except (TypeError, ValueError):
pass
# Check if the resource is an ISO 8601 duration.
try:
duration = times.ParseDuration(r)
return duration.Format(parts=parts, precision=precision)
except (AttributeError, TypeError, ValueError):
pass
# Check if the resource is a datetime.
try:
start_datetime = times.ParseDateTime(r)
except (AttributeError, ImportError, TypeError, ValueError):
return undefined
# Format the duration of (now - r).
end_datetime = times.Now(tzinfo=start_datetime.tzinfo)
delta = end_datetime - start_datetime
return times.GetDurationFromTimeDelta(
delta=delta, calendar=calendar).Format(parts=parts,
precision=precision)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
if args.gcs_source_staging_dir is None:
args.gcs_source_staging_dir = 'gs://{project}_cloudbuild/source'.format(
project=properties.VALUES.core.project.Get(), )
if args.gcs_log_dir is None:
args.gcs_log_dir = 'gs://{project}_cloudbuild/logs'.format(
project=properties.VALUES.core.project.Get(), )
client = core_apis.GetClientInstance('cloudbuild', 'v1')
messages = core_apis.GetMessagesModule('cloudbuild', 'v1')
registry = self.context['registry']
gcs_client = storage_api.StorageClient()
# First, create the build request.
build_timeout = properties.VALUES.container.build_timeout.Get()
if build_timeout is not None:
timeout_str = build_timeout + 's'
else:
timeout_str = None
if args.tag:
if 'gcr.io/' not in args.tag:
raise c_exceptions.InvalidArgumentException(
'--tag',
'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.'
)
build_config = messages.Build(
images=[args.tag],
steps=[
messages.BuildStep(
name='gcr.io/cloud-builders/docker',
args=['build', '--no-cache', '-t', args.tag, '.'],
),
],
timeout=timeout_str,
)
elif args.config:
build_config = config.LoadCloudbuildConfig(args.config, messages)
if build_config.timeout is None:
build_config.timeout = timeout_str
suffix = '.tgz'
if args.source.startswith('gs://') or os.path.isfile(args.source):
_, suffix = os.path.splitext(args.source)
# Next, stage the source to Cloud Storage.
staged_object = '{stamp}_{tag_ish}{suffix}'.format(
stamp=times.GetTimeStampFromDateTime(times.Now()),
tag_ish='_'.join(build_config.images or 'null').replace('/', '_'),
suffix=suffix,
)
gcs_source_staging_dir = registry.Parse(args.gcs_source_staging_dir,
collection='storage.objects')
gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket)
if gcs_source_staging_dir.object:
staged_object = gcs_source_staging_dir.object + '/' + staged_object
gcs_source_staging = registry.Create(
collection='storage.objects',
bucket=gcs_source_staging_dir.bucket,
object=staged_object)
if args.source.startswith('gs://'):
gcs_source = registry.Parse(args.source,
collection='storage.objects')
staged_source_obj = gcs_client.Copy(gcs_source, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
if not os.path.exists(args.source):
raise c_exceptions.BadFileException(
'could not find source [{src}]'.format(src=args.source))
if os.path.isdir(args.source):
source_snapshot = snapshot.Snapshot(args.source)
size_str = resource_transform.TransformSize(
source_snapshot.uncompressed_size)
log.status.write(
'Creating temporary tarball archive of {num_files} file(s)'
' totalling {size} before compression.\n'.format(
num_files=len(source_snapshot.files), size=size_str))
staged_source_obj = source_snapshot.CopyTarballToGCS(
gcs_client, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
elif os.path.isfile(args.source):
unused_root, ext = os.path.splitext(args.source)
if ext not in _ALLOWED_SOURCE_EXT:
raise c_exceptions.BadFileException(
'Local file [{src}] is none of ' +
', '.join(_ALLOWED_SOURCE_EXT))
log.status.write('Uploading local file [{src}] to '
'[gs://{bucket}/{object}]\n'.format(
src=args.source,
bucket=gcs_source_staging.bucket,
object=gcs_source_staging.object,
))
staged_source_obj = gcs_client.CopyFileToGCS(
storage_util.BucketReference.FromBucketUrl(
gcs_source_staging.bucket), args.source,
gcs_source_staging.object)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
gcs_log_dir = registry.Parse(args.gcs_log_dir,
collection='storage.objects')
if gcs_log_dir.bucket != gcs_source_staging.bucket:
# Create the logs bucket if it does not yet exist.
gcs_client.CreateBucketIfNotExists(gcs_log_dir.bucket)
build_config.logsBucket = 'gs://' + gcs_log_dir.bucket + '/' + gcs_log_dir.object
log.debug('submitting build: ' + repr(build_config))
# Start the build.
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(
build=build_config,
projectId=properties.VALUES.core.project.Get()))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata,
json).build
build_ref = registry.Create(collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.CreatedResource(build_ref)
if build.logUrl:
log.status.write(
'Logs are permanently available at [{log_url}]\n'.format(
log_url=build.logUrl))
else:
log.status.write('Logs are available in the Cloud Console.\n')
# If the command is run --async, we just print out a reference to the build.
if args. async:
return build
# Otherwise, logs are streamed from GCS.
build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build.status)
return build
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
project = properties.VALUES.core.project.Get(required=True)
safe_project = project.replace(':', '_')
safe_project = safe_project.replace('.', '_')
# The string 'google' is not allowed in bucket names.
safe_project = safe_project.replace('google', 'elgoog')
default_bucket_name = '{}_cloudbuild'.format(safe_project)
default_gcs_source = False
if args.gcs_source_staging_dir is None:
default_gcs_source = True
args.gcs_source_staging_dir = 'gs://{}/source'.format(default_bucket_name)
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
gcs_client = storage_api.StorageClient()
# First, create the build request.
build_timeout = properties.VALUES.builds.timeout.Get()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
timeout_str = six.text_type(build_timeout_secs) + 's'
else:
timeout_str = None
if args.tag is not None:
if (properties.VALUES.builds.check_tag.GetBool() and
'gcr.io/' not in args.tag):
raise c_exceptions.InvalidArgumentException(
'--tag',
'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.')
if properties.VALUES.builds.use_kaniko.GetBool():
if args.no_cache:
ttl = '0h'
else:
ttl = '{}h'.format(properties.VALUES.builds.kaniko_cache_ttl.Get())
build_config = messages.Build(
steps=[
messages.BuildStep(
name=properties.VALUES.builds.kaniko_image.Get(),
args=[
'--destination', args.tag, '--cache', 'true',
'--cache-ttl', ttl
],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(
args.substitutions, messages))
else:
if args.no_cache:
raise c_exceptions.InvalidArgumentException(
'no-cache',
'Cannot specify --no-cache if builds/use_kaniko property is '
'False')
build_config = messages.Build(
images=[args.tag],
steps=[
messages.BuildStep(
name='gcr.io/cloud-builders/docker',
args=[
'build', '--network', 'cloudbuild', '--no-cache', '-t',
args.tag, '.'
],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(
args.substitutions, messages))
elif args.config is not None:
if args.no_cache:
raise c_exceptions.ConflictingArgumentsException(
'--config', '--no-cache')
if not args.config:
raise c_exceptions.InvalidArgumentException(
'--config', 'Config file path must not be empty.')
build_config = config.LoadCloudbuildConfigFromPath(
args.config, messages, params=args.substitutions)
else:
raise c_exceptions.OneOfArgumentsRequiredException(
['--tag', '--config'],
'Requires either a docker tag or a config file.')
# If timeout was set by flag, overwrite the config file.
if timeout_str:
build_config.timeout = timeout_str
# --no-source overrides the default --source.
if not args.IsSpecified('source') and args.no_source:
args.source = None
gcs_source_staging = None
if args.source:
suffix = '.tgz'
if args.source.startswith('gs://') or os.path.isfile(args.source):
_, suffix = os.path.splitext(args.source)
# Next, stage the source to Cloud Storage.
staged_object = '{stamp}-{uuid}{suffix}'.format(
stamp=times.GetTimeStampFromDateTime(times.Now()),
uuid=uuid.uuid4().hex,
suffix=suffix,
)
gcs_source_staging_dir = resources.REGISTRY.Parse(
args.gcs_source_staging_dir, collection='storage.objects')
# We create the bucket (if it does not exist) first. If we do an existence
# check and then create the bucket ourselves, it would be possible for an
# attacker to get lucky and beat us to creating the bucket. Block on this
# creation to avoid this race condition.
gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket)
# If no bucket is specified (for the source `default_gcs_source`), check
# that the default bucket is also owned by the project (b/33046325).
if default_gcs_source:
# This request returns only the buckets owned by the project.
bucket_list_req = gcs_client.messages.StorageBucketsListRequest(
project=project, prefix=default_bucket_name)
bucket_list = gcs_client.client.buckets.List(bucket_list_req)
found_bucket = False
for bucket in bucket_list.items:
if bucket.id == default_bucket_name:
found_bucket = True
break
if not found_bucket:
if default_gcs_source:
raise c_exceptions.RequiredArgumentException(
'gcs_source_staging_dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket using '
'--gcs_source_staging_dir.'.format(default_bucket_name))
if gcs_source_staging_dir.object:
staged_object = gcs_source_staging_dir.object + '/' + staged_object
gcs_source_staging = resources.REGISTRY.Create(
collection='storage.objects',
bucket=gcs_source_staging_dir.bucket,
object=staged_object)
if args.source.startswith('gs://'):
gcs_source = resources.REGISTRY.Parse(
args.source, collection='storage.objects')
staged_source_obj = gcs_client.Rewrite(gcs_source, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
if not os.path.exists(args.source):
raise c_exceptions.BadFileException(
'could not find source [{src}]'.format(src=args.source))
if os.path.isdir(args.source):
source_snapshot = snapshot.Snapshot(args.source,
ignore_file=args.ignore_file)
size_str = resource_transform.TransformSize(
source_snapshot.uncompressed_size)
log.status.Print(
'Creating temporary tarball archive of {num_files} file(s)'
' totalling {size} before compression.'.format(
num_files=len(source_snapshot.files), size=size_str))
staged_source_obj = source_snapshot.CopyTarballToGCS(
gcs_client, gcs_source_staging, ignore_file=args.ignore_file)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
elif os.path.isfile(args.source):
unused_root, ext = os.path.splitext(args.source)
if ext not in _ALLOWED_SOURCE_EXT:
raise c_exceptions.BadFileException(
'Local file [{src}] is none of ' +
', '.join(_ALLOWED_SOURCE_EXT))
log.status.Print('Uploading local file [{src}] to '
'[gs://{bucket}/{object}].'.format(
src=args.source,
bucket=gcs_source_staging.bucket,
object=gcs_source_staging.object,
))
staged_source_obj = gcs_client.CopyFileToGCS(args.source,
gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
# No source
if not args.no_source:
raise c_exceptions.InvalidArgumentException(
'--no-source', 'To omit source, use the --no-source flag.')
if args.gcs_log_dir:
gcs_log_dir = resources.REGISTRY.Parse(
args.gcs_log_dir, collection='storage.objects')
build_config.logsBucket = ('gs://' + gcs_log_dir.bucket + '/' +
gcs_log_dir.object)
# Machine type.
if args.machine_type is not None:
machine_type = Submit._machine_type_flag_map.GetEnumForChoice(
args.machine_type)
if not build_config.options:
build_config.options = messages.BuildOptions()
build_config.options.machineType = machine_type
# Disk size.
if args.disk_size is not None:
disk_size = compute_utils.BytesToGb(args.disk_size)
if not build_config.options:
build_config.options = messages.BuildOptions()
build_config.options.diskSizeGb = int(disk_size)
log.debug('submitting build: ' + repr(build_config))
# Start the build.
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(
build=build_config, projectId=properties.VALUES.core.project.Get()))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build
build_ref = resources.REGISTRY.Create(
collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print(
'Logs are available at [{log_url}].'.format(log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
# If the command is run --async, we just print out a reference to the build.
if args.async:
return build
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def CreateRequest(args,
client,
resources,
minimal_action,
max_surge=None):
"""Create request helper for compute instance-groups managed rolling-action.
Args:
args: argparse namespace
client: The compute client
resources: The compute resources
minimal_action: MinimalActionValueValuesEnum value
max_surge: InstanceGroupManagerUpdatePolicy.maxSurge value
Returns:
ComputeInstanceGroupManagersPatchRequest or
ComputeRegionInstanceGroupManagersPatchRequest instance
Raises:
ValueError: if instance group manager collection path is unknown
"""
resource_arg = instance_groups_flags.MULTISCOPE_INSTANCE_GROUP_MANAGER_ARG
default_scope = compute_scope.ScopeEnum.ZONE
scope_lister = flags.GetDefaultScopeLister(client)
igm_ref = resource_arg.ResolveAsResource(
args, resources, default_scope=default_scope, scope_lister=scope_lister)
if igm_ref.Collection() not in [
'compute.instanceGroupManagers', 'compute.regionInstanceGroupManagers'
]:
raise ValueError('Unknown reference type {0}'.format(igm_ref.Collection()))
update_policy_type = (client.messages.InstanceGroupManagerUpdatePolicy.
TypeValueValuesEnum.PROACTIVE)
max_unavailable = update_instances_utils.ParseFixedOrPercent(
'--max-unavailable', 'max-unavailable', args.max_unavailable,
client.messages)
igm_info = managed_instance_groups_utils.GetInstanceGroupManagerOrThrow(
igm_ref, client)
versions = (igm_info.versions or [
client.messages.InstanceGroupManagerVersion(
instanceTemplate=igm_info.instanceTemplate)
])
current_time_str = str(times.Now(times.UTC))
for i, version in enumerate(versions):
version.name = '%d/%s' % (i, current_time_str)
update_policy = client.messages.InstanceGroupManagerUpdatePolicy(
maxSurge=max_surge,
maxUnavailable=max_unavailable,
minimalAction=minimal_action,
type=update_policy_type)
# min_ready is available in alpha and beta APIs only
if hasattr(args, 'min_ready'):
update_policy.minReadySec = args.min_ready
# replacement_method is available in alpha API only
if hasattr(args, 'replacement_method'):
replacement_method = update_instances_utils.ParseReplacementMethod(
args.replacement_method, client.messages)
update_policy.replacementMethod = replacement_method
igm_resource = client.messages.InstanceGroupManager(
instanceTemplate=None, updatePolicy=update_policy, versions=versions)
if igm_ref.Collection() == 'compute.instanceGroupManagers':
service = client.apitools_client.instanceGroupManagers
request = client.messages.ComputeInstanceGroupManagersPatchRequest(
instanceGroupManager=igm_ref.Name(),
instanceGroupManagerResource=igm_resource,
project=igm_ref.project,
zone=igm_ref.zone)
else:
service = client.apitools_client.regionInstanceGroupManagers
request = client.messages.ComputeRegionInstanceGroupManagersPatchRequest(
instanceGroupManager=igm_ref.Name(),
instanceGroupManagerResource=igm_resource,
project=igm_ref.project,
region=igm_ref.region)
return (service, 'Patch', request)
def CreateRequest(self, args, cleared_fields):
resource_arg = instance_groups_flags.MULTISCOPE_INSTANCE_GROUP_MANAGER_ARG
default_scope = compute_scope.ScopeEnum.ZONE
scope_lister = flags.GetDefaultScopeLister(self.compute_client,
self.project)
igm_ref = resource_arg.ResolveAsResource(args,
self.resources,
default_scope=default_scope,
scope_lister=scope_lister)
update_instances_utils.ValidateUpdateInstancesArgs(args)
update_policy_type = update_instances_utils.ParseUpdatePolicyType(
'--type', args.type, self.messages)
max_surge = update_instances_utils.ParseFixedOrPercent(
'--max-surge', 'max-surge', args.max_surge, self.messages)
max_unavailable = update_instances_utils.ParseFixedOrPercent(
'--max-unavailable', 'max-unavailable', args.max_unavailable,
self.messages)
igm_info = managed_instance_groups_utils.GetInstanceGroupManagerOrThrow(
igm_ref, self.compute_client)
if args.action == 'replace':
versions = []
if args.version_original:
versions.append(
update_instances_utils.ParseVersion(
'--version-original', args.version_original,
self.resources, self.messages))
versions.append(
update_instances_utils.ParseVersion('--version-new',
args.version_new,
self.resources,
self.messages))
managed_instance_groups_utils.ValidateVersions(
igm_info, versions, args.force)
# TODO(b/36056457): Decide what we should do when two versions have the
# same instance template (this can happen with canary restart performed
# using tags).
igm_version_names = {
version.instanceTemplate: version.name
for version in igm_info.versions
}
for version in versions:
version.name = igm_version_names.get(version.instanceTemplate)
version.tag = version.name
minimal_action = (self.messages.InstanceGroupManagerUpdatePolicy.
MinimalActionValueValuesEnum.REPLACE)
elif args.action == 'restart' and igm_info.versions is not None:
versions = (igm_info.versions or [
self.messages.InstanceGroupManagerVersion(
instanceTemplate=igm_info.instanceTemplate)
])
current_time_str = str(times.Now(times.UTC))
for i, version in enumerate(versions):
version.name = '%d/%s' % (i, current_time_str)
version.tag = version.name
minimal_action = (self.messages.InstanceGroupManagerUpdatePolicy.
MinimalActionValueValuesEnum.RESTART)
else:
raise exceptions.InvalidArgumentException('--action',
'unknown action type.')
update_policy = self.messages.InstanceGroupManagerUpdatePolicy(
maxSurge=max_surge,
maxUnavailable=max_unavailable,
minReadySec=args.min_ready,
minimalAction=minimal_action,
type=update_policy_type)
igm_resource = self.messages.InstanceGroupManager(
instanceTemplate=None,
updatePolicy=update_policy,
versions=versions)
if hasattr(igm_ref, 'zone'):
service = self.compute.instanceGroupManagers
request = (self.messages.ComputeInstanceGroupManagersPatchRequest(
instanceGroupManager=igm_ref.Name(),
instanceGroupManagerResource=igm_resource,
project=self.project,
zone=igm_ref.zone))
elif hasattr(igm_ref, 'region'):
service = self.compute.regionInstanceGroupManagers
request = (
self.messages.ComputeRegionInstanceGroupManagersPatchRequest(
instanceGroupManager=igm_ref.Name(),
instanceGroupManagerResource=igm_resource,
project=self.project,
region=igm_ref.region))
# Due to 'Patch' semantics, we have to clear either 'fixed' or 'percent'.
# Otherwise, we'll get an error that both 'fixed' and 'percent' are set.
if max_surge is not None:
cleared_fields.append(
'updatePolicy.maxSurge.fixed' if max_surge.fixed is None else
'updatePolicy.maxSurge.percent')
if max_unavailable is not None:
cleared_fields.append(
'updatePolicy.maxUnavailable.fixed' if max_unavailable.
fixed is None else 'updatePolicy.maxUnavailable.percent')
return (service, 'Patch', request)
def CreateRequests(self, args):
resource_arg = instance_groups_flags.ZONAL_INSTANCE_GROUP_MANAGER_ARG
default_scope = flags.ScopeEnum.ZONE
scope_lister = flags.GetDefaultScopeLister(self.compute_client,
self.project)
igm_ref = resource_arg.ResolveAsResource(args,
self.resources,
default_scope=default_scope,
scope_lister=scope_lister)
update_instances_utils.ValidateUpdateInstancesArgs(args)
update_policy_type = update_instances_utils.ParseUpdatePolicyType(
'--type', args.type, self.messages)
max_surge = update_instances_utils.ParseFixedOrPercent(
'--max-surge', 'max-surge', args.max_surge, self.messages)
max_unavailable = update_instances_utils.ParseFixedOrPercent(
'--max-unavailable', 'max-unavailable', args.max_unavailable,
self.messages)
igm_info = managed_instance_groups_utils.GetInstanceGroupManagerOrThrow(
igm_ref, self.project, self.compute, self.http, self.batch_url)
if args.action == 'replace':
versions = []
if args.version_original:
versions.append(
update_instances_utils.ParseVersion(
'--version-original', args.version_original,
self.resources, self.messages))
versions.append(
update_instances_utils.ParseVersion('--version-new',
args.version_new,
self.resources,
self.messages))
managed_instance_groups_utils.ValidateVersions(
igm_info, versions, args.force)
igm_tags = dict((version.instanceTemplate, version.tag)
for version in igm_info.versions)
for version in versions:
version.tag = igm_tags.get(version.instanceTemplate)
minimal_action = (self.messages.InstanceGroupManagerUpdatePolicy.
MinimalActionValueValuesEnum.REPLACE)
elif args.action == 'restart' and igm_info.versions is not None:
versions = (igm_info.versions or [
self.messages.InstanceGroupManagerVersion(
instanceTemplate=igm_info.instanceTemplate)
])
for version in versions:
version.tag = str(times.Now(times.UTC))
minimal_action = (self.messages.InstanceGroupManagerUpdatePolicy.
MinimalActionValueValuesEnum.RESTART)
else:
raise exceptions.InvalidArgumentException('--action',
'unknown action type.')
update_policy = self.messages.InstanceGroupManagerUpdatePolicy(
maxSurge=max_surge,
maxUnavailable=max_unavailable,
minReadySec=args.min_ready,
minimalAction=minimal_action,
type=update_policy_type,
)
service = self.compute.instanceGroupManagers
request = (self.messages.ComputeInstanceGroupManagersPatchRequest(
instanceGroupManager=igm_ref.Name(),
instanceGroupManagerResource=(self.messages.InstanceGroupManager(
instanceTemplate=None,
updatePolicy=update_policy,
versions=versions)),
project=self.project,
zone=igm_ref.zone))
return [(service, self.method, request)]
def _SetSource(build_config, messages, is_specified_source, no_source, source,
gcs_source_staging_dir, ignore_file):
"""Set the source for the build config."""
default_gcs_source = False
default_bucket_name = None
if gcs_source_staging_dir is None:
default_gcs_source = True
default_bucket_name = staging_bucket_util.GetDefaultStagingBucket()
gcs_source_staging_dir = 'gs://{}/source'.format(default_bucket_name)
gcs_client = storage_api.StorageClient()
# --no-source overrides the default --source.
if not is_specified_source and no_source:
source = None
gcs_source_staging = None
if source:
suffix = '.tgz'
if source.startswith('gs://') or os.path.isfile(source):
_, suffix = os.path.splitext(source)
# Next, stage the source to Cloud Storage.
staged_object = '{stamp}-{uuid}{suffix}'.format(
stamp=times.GetTimeStampFromDateTime(times.Now()),
uuid=uuid.uuid4().hex,
suffix=suffix,
)
gcs_source_staging_dir = resources.REGISTRY.Parse(
gcs_source_staging_dir, collection='storage.objects')
# We create the bucket (if it does not exist) first. If we do an existence
# check and then create the bucket ourselves, it would be possible for an
# attacker to get lucky and beat us to creating the bucket. Block on this
# creation to avoid this race condition.
gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket)
# If no bucket is specified (for the source `default_gcs_source`), check
# that the default bucket is also owned by the project (b/33046325).
if default_gcs_source and not staging_bucket_util.BucketIsInProject(
gcs_client, default_bucket_name):
raise c_exceptions.RequiredArgumentException(
'gcs-source-staging-dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket using '
'--gcs-source-staging-dir.'.format(default_bucket_name))
if gcs_source_staging_dir.object:
staged_object = gcs_source_staging_dir.object + '/' + staged_object
gcs_source_staging = resources.REGISTRY.Create(
collection='storage.objects',
bucket=gcs_source_staging_dir.bucket,
object=staged_object)
if source.startswith('gs://'):
gcs_source = resources.REGISTRY.Parse(source,
collection='storage.objects')
staged_source_obj = gcs_client.Rewrite(gcs_source,
gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
if not os.path.exists(source):
raise c_exceptions.BadFileException(
'could not find source [{src}]'.format(src=source))
if os.path.isdir(source):
source_snapshot = snapshot.Snapshot(source,
ignore_file=ignore_file)
size_str = resource_transform.TransformSize(
source_snapshot.uncompressed_size)
log.status.Print(
'Creating temporary tarball archive of {num_files} file(s)'
' totalling {size} before compression.'.format(
num_files=len(source_snapshot.files), size=size_str))
staged_source_obj = source_snapshot.CopyTarballToGCS(
gcs_client, gcs_source_staging, ignore_file=ignore_file)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
elif os.path.isfile(source):
unused_root, ext = os.path.splitext(source)
if ext not in _ALLOWED_SOURCE_EXT:
raise c_exceptions.BadFileException(
'Local file [{src}] is none of ' +
', '.join(_ALLOWED_SOURCE_EXT))
log.status.Print('Uploading local file [{src}] to '
'[gs://{bucket}/{object}].'.format(
src=source,
bucket=gcs_source_staging.bucket,
object=gcs_source_staging.object,
))
staged_source_obj = gcs_client.CopyFileToGCS(
source, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
# No source
if not no_source:
raise c_exceptions.InvalidArgumentException(
'--no-source', 'To omit source, use the --no-source flag.')
return build_config
def _StageSource(self, source, gcs_staging_dir_bucket,
gcs_staging_dir_object):
"""Stages source onto the provided bucket and returns its reference.
Args:
source: Path to source repo as a directory on a local disk or a
gzipped archive file (.tar.gz) in Google Cloud Storage.
gcs_staging_dir_bucket: Bucket name of staging directory.
gcs_staging_dir_object: Bucket object of staging directory.
Returns:
Reference to the staged source, which has bucket, name, and generation
fields.
"""
suffix = '.tgz'
if source.startswith('gs://') or os.path.isfile(source):
_, suffix = os.path.splitext(source)
source_object = 'source/{stamp}-{uuid}{suffix}'.format(
stamp=times.GetTimeStampFromDateTime(times.Now()),
uuid=uuid.uuid4().hex,
suffix=suffix,
)
if gcs_staging_dir_object:
source_object = gcs_staging_dir_object + '/' + source_object
gcs_source_staging = resources.REGISTRY.Create(
collection='storage.objects',
bucket=gcs_staging_dir_bucket,
object=source_object)
gcs_client = storage_api.StorageClient()
if source.startswith('gs://'):
gcs_source = resources.REGISTRY.Parse(source,
collection='storage.objects')
staged_source = gcs_client.Rewrite(gcs_source, gcs_source_staging)
else:
if not os.path.exists(source):
raise c_exceptions.BadFileException(
'could not find source [{src}]'.format(src=source))
elif os.path.isdir(source):
source_snapshot = snapshot.Snapshot(source)
size_str = resource_transform.TransformSize(
source_snapshot.uncompressed_size)
log.status.Print(
'Creating temporary tarball archive of {num_files} file(s)'
' totalling {size} before compression.'.format(
num_files=len(source_snapshot.files), size=size_str))
staged_source = source_snapshot.CopyTarballToGCS(
gcs_client, gcs_source_staging)
elif os.path.isfile(source):
unused_root, ext = os.path.splitext(source)
if ext not in _ALLOWED_SOURCE_EXT:
raise c_exceptions.BadFileException(
'Local file [{src}] is none of '.format(src=source) +
', '.join(_ALLOWED_SOURCE_EXT))
log.status.Print('Uploading local file [{src}] to '
'[gs://{bucket}/{object}].'.format(
src=source,
bucket=gcs_source_staging.bucket,
object=gcs_source_staging.object,
))
staged_source = gcs_client.CopyFileToGCS(
source, gcs_source_staging)
return staged_source
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
project = properties.VALUES.core.project.Get()
safe_project = project.replace(':', '_')
safe_project = safe_project.replace('.', '_')
# The string 'google' is not allowed in bucket names.
safe_project = safe_project.replace('google', 'elgoog')
default_bucket_name = '{}_cloudbuild'.format(safe_project)
default_gcs_source = False
if args.gcs_source_staging_dir is None:
default_gcs_source = True
args.gcs_source_staging_dir = 'gs://{}/source'.format(default_bucket_name)
default_gcs_log_dir = False
if args.gcs_log_dir is None:
default_gcs_log_dir = True
args.gcs_log_dir = 'gs://{}/logs'.format(default_bucket_name)
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
gcs_client = storage_api.StorageClient()
# First, create the build request.
build_timeout = properties.VALUES.container.build_timeout.Get()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
timeout_str = str(build_timeout_secs) + 's'
else:
timeout_str = None
if args.tag:
if 'gcr.io/' not in args.tag:
raise c_exceptions.InvalidArgumentException(
'--tag',
'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.')
build_config = messages.Build(
images=[args.tag],
steps=[
messages.BuildStep(
name='gcr.io/cloud-builders/docker',
args=['build', '--no-cache', '-t', args.tag, '.'],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(args.substitutions,
messages)
)
elif args.config:
build_config = config.LoadCloudbuildConfigFromPath(
args.config, messages, params=args.substitutions)
# If timeout was set by flag, overwrite the config file.
if timeout_str:
build_config.timeout = timeout_str
suffix = '.tgz'
if args.source.startswith('gs://') or os.path.isfile(args.source):
_, suffix = os.path.splitext(args.source)
# Next, stage the source to Cloud Storage.
staged_object = '{stamp}{suffix}'.format(
stamp=times.GetTimeStampFromDateTime(times.Now()),
suffix=suffix,
)
gcs_source_staging_dir = resources.REGISTRY.Parse(
args.gcs_source_staging_dir, collection='storage.objects')
# We first try to create the bucket, before doing all the checks, in order
# to avoid a race condition. If we do the check first, an attacker could
# be lucky enough to create the bucket after the check and before this
# bucket creation.
gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket)
# If no bucket is specified (for the source `default_gcs_source` or for the
# logs `default_gcs_log_dir`), check that the default bucket is also owned
# by the project (b/33046325).
if default_gcs_source or default_gcs_log_dir:
# This request returns only the buckets owned by the project.
bucket_list_req = gcs_client.messages.StorageBucketsListRequest(
project=project,
prefix=default_bucket_name)
bucket_list = gcs_client.client.buckets.List(bucket_list_req)
found_bucket = False
for bucket in bucket_list.items:
if bucket.id == default_bucket_name:
found_bucket = True
break
if not found_bucket:
if default_gcs_source:
raise c_exceptions.RequiredArgumentException(
'gcs_source_staging_dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket using '
'--gcs_source_staging_dir.'.format(default_bucket_name))
elif default_gcs_log_dir:
raise c_exceptions.RequiredArgumentException(
'gcs-log-dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket to hold build logs '
'using --gcs-log-dir.'.format(default_bucket_name))
if gcs_source_staging_dir.object:
staged_object = gcs_source_staging_dir.object + '/' + staged_object
gcs_source_staging = resources.REGISTRY.Create(
collection='storage.objects',
bucket=gcs_source_staging_dir.bucket,
object=staged_object)
if args.source.startswith('gs://'):
gcs_source = resources.REGISTRY.Parse(
args.source, collection='storage.objects')
staged_source_obj = gcs_client.Rewrite(gcs_source, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
if not os.path.exists(args.source):
raise c_exceptions.BadFileException(
'could not find source [{src}]'.format(src=args.source))
if os.path.isdir(args.source):
source_snapshot = snapshot.Snapshot(args.source)
size_str = resource_transform.TransformSize(
source_snapshot.uncompressed_size)
log.status.Print(
'Creating temporary tarball archive of {num_files} file(s)'
' totalling {size} before compression.'.format(
num_files=len(source_snapshot.files),
size=size_str))
staged_source_obj = source_snapshot.CopyTarballToGCS(
gcs_client, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
elif os.path.isfile(args.source):
unused_root, ext = os.path.splitext(args.source)
if ext not in _ALLOWED_SOURCE_EXT:
raise c_exceptions.BadFileException(
'Local file [{src}] is none of '+', '.join(_ALLOWED_SOURCE_EXT))
log.status.Print(
'Uploading local file [{src}] to '
'[gs://{bucket}/{object}].'.format(
src=args.source,
bucket=gcs_source_staging.bucket,
object=gcs_source_staging.object,
))
staged_source_obj = gcs_client.CopyFileToGCS(
storage_util.BucketReference.FromBucketUrl(
gcs_source_staging.bucket),
args.source, gcs_source_staging.object)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
gcs_log_dir = resources.REGISTRY.Parse(
args.gcs_log_dir, collection='storage.objects')
if gcs_log_dir.bucket != gcs_source_staging.bucket:
# Create the logs bucket if it does not yet exist.
gcs_client.CreateBucketIfNotExists(gcs_log_dir.bucket)
build_config.logsBucket = 'gs://'+gcs_log_dir.bucket+'/'+gcs_log_dir.object
log.debug('submitting build: '+repr(build_config))
# Start the build.
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(
build=build_config,
projectId=properties.VALUES.core.project.Get()))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build
build_ref = resources.REGISTRY.Create(
collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print('Logs are available at [{log_url}].'.format(
log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
# If the command is run --async, we just print out a reference to the build.
if args.async:
return build
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def GetCurrentTime():
"""Returns the current UTC datetime."""
return times.Now(tzinfo=times.UTC)