def CreateRequests(self, args):
"""Returns a list of requests necessary for adding disks."""
size_gb = utils.BytesToGb(args.size)
from_image = args.image or args.image_family
if not size_gb and not args.source_snapshot and not from_image:
if args.type and 'pd-ssd' in args.type:
size_gb = constants.DEFAULT_SSD_DISK_SIZE_GB
else:
size_gb = constants.DEFAULT_STANDARD_DISK_SIZE_GB
utils.WarnIfDiskSizeIsTooSmall(size_gb, args.type)
requests = []
disk_refs = disks_flags.DISKS_ARG.ResolveAsResource(
args,
self.resources,
scope_lister=flags.GetDefaultScopeLister(self.compute_client,
self.project))
# Check if the zone is deprecated or has maintenance coming.
self.WarnForZonalCreation(disk_refs)
if from_image:
source_image_uri, _ = self.ExpandImageFlag(
args, return_image_resource=False)
else:
source_image_uri = None
snapshot_ref = disks_flags.SOURCE_SNAPSHOT_ARG.ResolveAsResource(
args, self.resources)
if snapshot_ref:
snapshot_uri = snapshot_ref.SelfLink()
else:
snapshot_uri = None
# This feature is only exposed in alpha/beta
allow_rsa_encrypted = self.ReleaseTrack() in [
base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA
]
csek_keys = csek_utils.CsekKeyStore.FromArgs(args, allow_rsa_encrypted)
image_key_message_or_none, snapshot_key_message_or_none = (
csek_utils.MaybeLookupKeyMessagesByUri(
csek_keys, self.resources, [source_image_uri, snapshot_uri],
self.compute))
for disk_ref in disk_refs:
if args.type:
type_ref = self.resources.Parse(args.type,
collection='compute.diskTypes',
params={'zone': disk_ref.zone})
type_uri = type_ref.SelfLink()
else:
type_uri = None
if csek_keys:
disk_key_or_none = csek_keys.LookupKey(
disk_ref, args.require_csek_key_create)
disk_key_message_or_none = csek_utils.MaybeToMessage(
disk_key_or_none, self.compute)
kwargs = {
'diskEncryptionKey': disk_key_message_or_none,
'sourceImageEncryptionKey': image_key_message_or_none,
'sourceSnapshotEncryptionKey': snapshot_key_message_or_none
}
else:
kwargs = {}
request = self.messages.ComputeDisksInsertRequest(
disk=self.messages.Disk(name=disk_ref.Name(),
description=args.description,
sizeGb=size_gb,
sourceSnapshot=snapshot_uri,
type=type_uri,
**kwargs),
project=self.project,
sourceImage=source_image_uri,
zone=disk_ref.zone)
requests.append(request)
return requests
def _GetBootDiskSize(self, args): boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) return boot_disk_size_gb
def Run(self, args):
"""Issues an InstanceTemplates.Insert request.
Args:
args: the argparse arguments that this command was invoked with.
Returns:
an InstanceTemplate message object
"""
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
instances_flags.ValidateDockerArgs(args)
instances_flags.ValidateDiskCommonFlags(args)
instances_flags.ValidateLocalSsdFlags(args)
instances_flags.ValidateServiceAccountAndScopeArgs(args)
if instance_utils.UseExistingBootDisk(args.disk or []):
raise exceptions.InvalidArgumentException(
'--disk', 'Boot disk specified for containerized VM.')
boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size)
utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type)
instance_template_ref = (
CreateFromContainer.InstanceTemplateArg.ResolveAsResource(
args, holder.resources))
user_metadata = metadata_utils.ConstructMetadataMessage(
client.messages,
metadata=args.metadata,
metadata_from_file=args.metadata_from_file)
containers_utils.ValidateUserMetadata(user_metadata)
network_interface = instance_template_utils.CreateNetworkInterfaceMessage(
resources=holder.resources,
scope_lister=flags.GetDefaultScopeLister(client),
messages=client.messages,
network=args.network,
region=args.region,
subnet=args.subnet,
address=(instance_template_utils.EPHEMERAL_ADDRESS
if not args.no_address and not args.address else
args.address),
network_tier=args.network_tier)
scheduling = instance_utils.CreateSchedulingMessage(
messages=client.messages,
maintenance_policy=args.maintenance_policy,
preemptible=args.preemptible,
restart_on_failure=args.restart_on_failure)
if args.no_service_account:
service_account = None
else:
service_account = args.service_account
service_accounts = instance_utils.CreateServiceAccountMessages(
messages=client.messages,
scopes=[] if args.no_scopes else args.scopes,
service_account=service_account)
image_uri = containers_utils.ExpandCosImageFlag(client)
machine_type = instance_utils.InterpretMachineType(
machine_type=args.machine_type,
custom_cpu=args.custom_cpu,
custom_memory=args.custom_memory,
ext=getattr(args, 'custom_extensions', None))
metadata = containers_utils.CreateMetadataMessage(
client.messages, args.run_as_privileged, args.container_manifest,
args.docker_image, args.port_mappings, args.run_command,
user_metadata, instance_template_ref.Name())
request = client.messages.ComputeInstanceTemplatesInsertRequest(
instanceTemplate=client.messages.InstanceTemplate(
properties=client.messages.InstanceProperties(
machineType=machine_type,
disks=self._CreateDiskMessages(
holder, args, boot_disk_size_gb, image_uri,
instance_template_ref.project),
canIpForward=args.can_ip_forward,
metadata=metadata,
minCpuPlatform=args.min_cpu_platform,
networkInterfaces=[network_interface],
serviceAccounts=service_accounts,
scheduling=scheduling,
tags=containers_utils.CreateTagsMessage(
client.messages, args.tags),
),
description=args.description,
name=instance_template_ref.Name(),
),
project=instance_template_ref.project)
return client.MakeRequests([(client.apitools_client.instanceTemplates,
'Insert', request)])
def GetClusterConfig(args,
dataproc,
project_id,
compute_resources,
beta=False):
"""Get dataproc cluster configuration.
Args:
args: Arguments parsed from argparse.ArgParser.
dataproc: Dataproc object that contains client, messages, and resources
project_id: Dataproc project ID
compute_resources: compute resource for cluster
beta: use BETA only features
Returns:
cluster_config: Dataproc cluster configuration
"""
master_accelerator_type = None
worker_accelerator_type = None
master_accelerator_count = None
worker_accelerator_count = None
if beta:
if args.master_accelerator:
master_accelerator_type = args.master_accelerator['type']
master_accelerator_count = args.master_accelerator.get('count', 1)
if args.worker_accelerator:
worker_accelerator_type = args.worker_accelerator['type']
worker_accelerator_count = args.worker_accelerator.get('count', 1)
# Resolve non-zonal GCE resources
# We will let the server resolve short names of zonal resources because
# if auto zone is requested, we will not know the zone before sending the
# request
image_ref = args.image and compute_resources.Parse(
args.image,
params={'project': project_id},
collection='compute.images')
network_ref = args.network and compute_resources.Parse(
args.network,
params={'project': project_id},
collection='compute.networks')
subnetwork_ref = args.subnet and compute_resources.Parse(
args.subnet,
params={
'project': project_id,
'region': properties.VALUES.compute.region.GetOrFail,
},
collection='compute.subnetworks')
timeout_str = str(args.initialization_action_timeout) + 's'
init_actions = [
dataproc.messages.NodeInitializationAction(
executableFile=exe, executionTimeout=timeout_str)
for exe in (args.initialization_actions or [])
]
# Increase the client timeout for each initialization action.
args.timeout += args.initialization_action_timeout * len(init_actions)
expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes)
software_config = dataproc.messages.SoftwareConfig(
imageVersion=args.image_version)
master_boot_disk_size_gb = args.master_boot_disk_size_gb
if args.master_boot_disk_size:
master_boot_disk_size_gb = (api_utils.BytesToGb(
args.master_boot_disk_size))
worker_boot_disk_size_gb = args.worker_boot_disk_size_gb
if args.worker_boot_disk_size:
worker_boot_disk_size_gb = (api_utils.BytesToGb(
args.worker_boot_disk_size))
preemptible_worker_boot_disk_size_gb = (api_utils.BytesToGb(
args.preemptible_worker_boot_disk_size))
if args.single_node or args.num_workers == 0:
# Explicitly specifying --num-workers=0 gives you a single node cluster,
# but if --num-workers is omitted, args.num_workers is None (not 0), and
# this property will not be set
args.properties[constants.ALLOW_ZERO_WORKERS_PROPERTY] = 'true'
if args.properties:
software_config.properties = encoding.DictToMessage(
args.properties, dataproc.messages.SoftwareConfig.PropertiesValue)
gce_cluster_config = dataproc.messages.GceClusterConfig(
networkUri=network_ref and network_ref.SelfLink(),
subnetworkUri=subnetwork_ref and subnetwork_ref.SelfLink(),
internalIpOnly=args.no_address,
serviceAccount=args.service_account,
serviceAccountScopes=expanded_scopes,
zoneUri=properties.VALUES.compute.zone.GetOrFail())
if args.tags:
gce_cluster_config.tags = args.tags
if args.metadata:
flat_metadata = dict(
(k, v) for d in args.metadata for k, v in d.items())
gce_cluster_config.metadata = encoding.DictToMessage(
flat_metadata, dataproc.messages.GceClusterConfig.MetadataValue)
master_accelerators = []
if master_accelerator_type:
master_accelerators.append(
dataproc.messages.AcceleratorConfig(
acceleratorTypeUri=master_accelerator_type,
acceleratorCount=master_accelerator_count))
worker_accelerators = []
if worker_accelerator_type:
worker_accelerators.append(
dataproc.messages.AcceleratorConfig(
acceleratorTypeUri=worker_accelerator_type,
acceleratorCount=worker_accelerator_count))
cluster_config = dataproc.messages.ClusterConfig(
configBucket=args.bucket,
gceClusterConfig=gce_cluster_config,
masterConfig=dataproc.messages.InstanceGroupConfig(
numInstances=args.num_masters,
imageUri=image_ref and image_ref.SelfLink(),
machineTypeUri=args.master_machine_type,
accelerators=master_accelerators,
diskConfig=GetDiskConfig(
dataproc,
args.master_boot_disk_type if beta else None,
master_boot_disk_size_gb,
args.num_master_local_ssds,
beta,
)),
workerConfig=dataproc.messages.InstanceGroupConfig(
numInstances=args.num_workers,
imageUri=image_ref and image_ref.SelfLink(),
machineTypeUri=args.worker_machine_type,
accelerators=worker_accelerators,
diskConfig=GetDiskConfig(
dataproc,
args.worker_boot_disk_type if beta else None,
worker_boot_disk_size_gb,
args.num_worker_local_ssds,
beta,
)),
initializationActions=init_actions,
softwareConfig=software_config,
)
if beta:
cluster_config.masterConfig.minCpuPlatform = args.master_min_cpu_platform
cluster_config.workerConfig.minCpuPlatform = args.worker_min_cpu_platform
if beta:
lifecycle_config = dataproc.messages.LifecycleConfig()
changed_config = False
if args.max_age is not None:
lifecycle_config.autoDeleteTtl = str(args.max_age) + 's'
changed_config = True
if args.expiration_time is not None:
lifecycle_config.autoDeleteTime = times.FormatDateTime(
args.expiration_time)
changed_config = True
if args.max_idle is not None:
lifecycle_config.idleDeleteTtl = str(args.max_idle) + 's'
changed_config = True
if changed_config:
cluster_config.lifecycleConfig = lifecycle_config
# Secondary worker group is optional. However, users may specify
# future pVMs configuration at creation time.
if (args.num_preemptible_workers is not None
or preemptible_worker_boot_disk_size_gb is not None
or (beta and (args.preemptible_worker_boot_disk_type is not None
or args.worker_min_cpu_platform is not None))):
cluster_config.secondaryWorkerConfig = (
dataproc.messages.InstanceGroupConfig(
numInstances=args.num_preemptible_workers,
diskConfig=GetDiskConfig(
dataproc,
args.preemptible_worker_boot_disk_type if beta else None,
preemptible_worker_boot_disk_size_gb,
None,
beta,
)))
if beta and args.worker_min_cpu_platform:
cluster_config.secondaryWorkerConfig.minCpuPlatform = (
args.worker_min_cpu_platform)
return cluster_config
def _RunCreate(compute_api,
args,
support_source_instance,
support_kms=False,
support_min_node_cpus=False):
"""Common routine for creating instance template.
This is shared between various release tracks.
Args:
compute_api: The compute api.
args: argparse.Namespace, An object that contains the values for the
arguments specified in the .Args() method.
support_source_instance: indicates whether source instance is supported.
support_kms: Indicate whether KMS is integrated or not.
support_min_node_cpus: Indicate whether the --min-node-cpus flag for
sole tenancy overcommit is supported.
Returns:
A resource object dispatched by display.Displayer().
"""
_ValidateInstancesFlags(args, support_kms=support_kms)
instances_flags.ValidateNetworkTierArgs(args)
client = compute_api.client
boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size)
utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type)
instance_template_ref = (Create.InstanceTemplateArg.ResolveAsResource(
args, compute_api.resources))
metadata = metadata_utils.ConstructMetadataMessage(
client.messages,
metadata=args.metadata,
metadata_from_file=args.metadata_from_file)
if hasattr(args, 'network_interface') and args.network_interface:
network_interfaces = (
instance_template_utils.CreateNetworkInterfaceMessages)(
resources=compute_api.resources,
scope_lister=flags.GetDefaultScopeLister(client),
messages=client.messages,
network_interface_arg=args.network_interface,
region=args.region)
else:
network_tier = getattr(args, 'network_tier', None)
network_interfaces = [
instance_template_utils.CreateNetworkInterfaceMessage(
resources=compute_api.resources,
scope_lister=flags.GetDefaultScopeLister(client),
messages=client.messages,
network=args.network,
region=args.region,
subnet=args.subnet,
address=(instance_template_utils.EPHEMERAL_ADDRESS
if not args.no_address and not args.address else
args.address),
network_tier=network_tier)
]
# Compute the shieldedInstanceConfig message.
shieldedinstance_config_message = BuildShieldedInstanceConfigMessage(
messages=client.messages, args=args)
node_affinities = sole_tenancy_util.GetSchedulingNodeAffinityListFromArgs(
args, client.messages)
min_node_cpus = None
if support_min_node_cpus and args.IsSpecified('min_node_cpus'):
min_node_cpus = args.min_node_cpus
scheduling = instance_utils.CreateSchedulingMessage(
messages=client.messages,
maintenance_policy=args.maintenance_policy,
preemptible=args.preemptible,
restart_on_failure=args.restart_on_failure,
node_affinities=node_affinities,
min_node_cpus=min_node_cpus)
if args.no_service_account:
service_account = None
else:
service_account = args.service_account
service_accounts = instance_utils.CreateServiceAccountMessages(
messages=client.messages,
scopes=[] if args.no_scopes else args.scopes,
service_account=service_account)
create_boot_disk = not instance_utils.UseExistingBootDisk(args.disk or [])
if create_boot_disk:
image_expander = image_utils.ImageExpander(client,
compute_api.resources)
try:
image_uri, _ = image_expander.ExpandImageFlag(
user_project=instance_template_ref.project,
image=args.image,
image_family=args.image_family,
image_project=args.image_project,
return_image_resource=True)
except utils.ImageNotFoundError as e:
if args.IsSpecified('image_project'):
raise e
image_uri, _ = image_expander.ExpandImageFlag(
user_project=instance_template_ref.project,
image=args.image,
image_family=args.image_family,
image_project=args.image_project,
return_image_resource=False)
raise utils.ImageNotFoundError(
'The resource [{}] was not found. Is the image located in another '
'project? Use the --image-project flag to specify the '
'project where the image is located.'.format(image_uri))
else:
image_uri = None
if args.tags:
tags = client.messages.Tags(items=args.tags)
else:
tags = None
persistent_disks = (
instance_template_utils.CreatePersistentAttachedDiskMessages(
client.messages, args.disk or []))
persistent_create_disks = (
instance_template_utils.CreatePersistentCreateDiskMessages(
client,
compute_api.resources,
instance_template_ref.project,
getattr(args, 'create_disk', []),
support_kms=support_kms))
if create_boot_disk:
boot_disk_list = [
instance_template_utils.CreateDefaultBootAttachedDiskMessage(
messages=client.messages,
disk_type=args.boot_disk_type,
disk_device_name=args.boot_disk_device_name,
disk_auto_delete=args.boot_disk_auto_delete,
disk_size_gb=boot_disk_size_gb,
image_uri=image_uri,
kms_args=args,
support_kms=support_kms)
]
else:
boot_disk_list = []
local_nvdimms = instance_utils.CreateLocalNvdimmMessages(
args,
compute_api.resources,
client.messages,
)
local_ssds = instance_utils.CreateLocalSsdMessages(
args,
compute_api.resources,
client.messages,
)
disks = (boot_disk_list + persistent_disks + persistent_create_disks +
local_nvdimms + local_ssds)
machine_type = instance_utils.InterpretMachineType(
machine_type=args.machine_type,
custom_cpu=args.custom_cpu,
custom_memory=args.custom_memory,
ext=getattr(args, 'custom_extensions', None),
vm_type=getattr(args, 'custom_vm_type', None))
guest_accelerators = (
instance_template_utils.CreateAcceleratorConfigMessages(
client.messages, getattr(args, 'accelerator', None)))
instance_template = client.messages.InstanceTemplate(
properties=client.messages.InstanceProperties(
machineType=machine_type,
disks=disks,
canIpForward=args.can_ip_forward,
metadata=metadata,
minCpuPlatform=args.min_cpu_platform,
networkInterfaces=network_interfaces,
serviceAccounts=service_accounts,
scheduling=scheduling,
tags=tags,
guestAccelerators=guest_accelerators,
),
description=args.description,
name=instance_template_ref.Name(),
)
instance_template.properties.shieldedInstanceConfig = shieldedinstance_config_message
instance_template.properties.reservationAffinity = instance_utils.GetReservationAffinity(
args, client)
request = client.messages.ComputeInstanceTemplatesInsertRequest(
instanceTemplate=instance_template,
project=instance_template_ref.project)
request.instanceTemplate.properties.labels = labels_util.ParseCreateArgs(
args, client.messages.InstanceProperties.LabelsValue)
_AddSourceInstanceToTemplate(compute_api, args, instance_template,
support_source_instance)
return client.MakeRequests([(client.apitools_client.instanceTemplates,
'Insert', request)])
def ParseCreateOptionsBase(args):
"""Parses the flags provided with the cluster creation command."""
if args.IsSpecified('addons') and api_adapter.DASHBOARD in args.addons:
log.warning(
'The `KubernetesDashboard` addon is deprecated, and will be removed as '
'an option for new clusters starting in 1.15. It is recommended to use '
'the Cloud Console to manage and monitor your Kubernetes clusters, '
'workloads and applications. See: '
'https://cloud.google.com/kubernetes-engine/docs/concepts/dashboards'
)
flags.MungeBasicAuthFlags(args)
if args.IsSpecified('issue_client_certificate') and not (
args.IsSpecified('enable_basic_auth')
or args.IsSpecified('username')):
log.warning(
'If `--issue-client-certificate` is specified but '
'`--enable-basic-auth` or `--username` is not, our API will '
'treat that as `--no-enable-basic-auth`.')
flags.WarnForUnspecifiedIpAllocationPolicy(args)
enable_autorepair = cmd_util.GetAutoRepair(args)
flags.WarnForNodeModification(args, enable_autorepair)
metadata = metadata_utils.ConstructMetadataDict(args.metadata,
args.metadata_from_file)
return api_adapter.CreateClusterOptions(
accelerators=args.accelerator,
additional_zones=args.additional_zones,
addons=args.addons,
cluster_ipv4_cidr=args.cluster_ipv4_cidr,
cluster_secondary_range_name=args.cluster_secondary_range_name,
cluster_version=args.cluster_version,
node_version=args.node_version,
create_subnetwork=args.create_subnetwork,
disk_type=args.disk_type,
enable_autorepair=enable_autorepair,
enable_autoscaling=args.enable_autoscaling,
enable_autoupgrade=cmd_util.GetAutoUpgrade(args),
enable_binauthz=args.enable_binauthz,
enable_stackdriver_kubernetes=args.enable_stackdriver_kubernetes if args.IsSpecified('enable_stackdriver_kubernetes') else None,
enable_cloud_logging=args.enable_cloud_logging if args.IsSpecified('enable_cloud_logging') else None,
enable_cloud_monitoring=args.enable_cloud_monitoring if args.IsSpecified('enable_cloud_monitoring') else None,
enable_ip_alias=args.enable_ip_alias,
enable_intra_node_visibility=args.enable_intra_node_visibility,
enable_kubernetes_alpha=args.enable_kubernetes_alpha,
enable_cloud_run_alpha=args.enable_cloud_run_alpha if args.IsSpecified('enable_cloud_run_alpha') else None,
enable_legacy_authorization=args.enable_legacy_authorization,
enable_master_authorized_networks=args.enable_master_authorized_networks,
enable_network_policy=args.enable_network_policy,
enable_private_nodes=args.enable_private_nodes,
enable_private_endpoint=args.enable_private_endpoint,
image_type=args.image_type,
image=args.image,
image_project=args.image_project,
image_family=args.image_family,
issue_client_certificate=args.issue_client_certificate,
labels=args.labels,
local_ssd_count=args.local_ssd_count,
maintenance_window=args.maintenance_window,
maintenance_window_start=args.maintenance_window_start,
maintenance_window_end=args.maintenance_window_end,
maintenance_window_recurrence=args.maintenance_window_recurrence,
master_authorized_networks=args.master_authorized_networks,
master_ipv4_cidr=args.master_ipv4_cidr,
max_nodes=args.max_nodes,
max_nodes_per_pool=args.max_nodes_per_pool,
min_cpu_platform=args.min_cpu_platform,
min_nodes=args.min_nodes,
network=args.network,
node_disk_size_gb=utils.BytesToGb(args.disk_size),
node_labels=args.node_labels,
node_locations=args.node_locations,
node_machine_type=args.machine_type,
node_taints=args.node_taints,
num_nodes=args.num_nodes,
password=args.password,
preemptible=args.preemptible,
scopes=args.scopes,
service_account=args.service_account,
services_ipv4_cidr=args.services_ipv4_cidr,
services_secondary_range_name=args.services_secondary_range_name,
subnetwork=args.subnetwork,
tags=args.tags,
user=args.username,
metadata=metadata,
default_max_pods_per_node=args.default_max_pods_per_node,
max_pods_per_node=args.max_pods_per_node,
enable_tpu=args.enable_tpu,
tpu_ipv4_cidr=args.tpu_ipv4_cidr,
resource_usage_bigquery_dataset=args.resource_usage_bigquery_dataset,
enable_network_egress_metering=args.enable_network_egress_metering,
enable_resource_consumption_metering=\
args.enable_resource_consumption_metering,
database_encryption_key=args.database_encryption_key,
enable_vertical_pod_autoscaling=args.enable_vertical_pod_autoscaling,
enable_autoprovisioning=args.enable_autoprovisioning,
autoprovisioning_config_file=args.autoprovisioning_config_file,
autoprovisioning_service_account=args.autoprovisioning_service_account,
autoprovisioning_scopes=args.autoprovisioning_scopes,
autoprovisioning_locations=args.autoprovisioning_locations,
autoprovisioning_max_surge_upgrade=getattr(args, 'autoprovisioning_max_surge_upgrade', None),
autoprovisioning_max_unavailable_upgrade=getattr(args, 'autoprovisioning_max_unavailable_upgrade', None),
enable_autoprovisioning_autorepair=getattr(args, 'enable_autoprovisioning_autorepair', None),
enable_autoprovisioning_autoupgrade=getattr(args, 'enable_autoprovisioning_autoupgrade', None),
min_cpu=args.min_cpu,
max_cpu=args.max_cpu,
min_memory=args.min_memory,
max_memory=args.max_memory,
min_accelerator=args.min_accelerator,
max_accelerator=args.max_accelerator,
shielded_secure_boot=args.shielded_secure_boot,
shielded_integrity_monitoring=args.shielded_integrity_monitoring,
reservation_affinity=getattr(args, 'reservation_affinity', None),
reservation=getattr(args, 'reservation', None),
enable_shielded_nodes=args.enable_shielded_nodes,)
def CreateRequests(self, args):
"""Creates and returns an InstanceTemplates.Insert request.
Args:
args: the argparse arguments that this command was invoked with.
Returns:
request: a ComputeInstanceTemplatesInsertRequest message object
"""
self.ValidateDiskFlags(args)
instances_flags.ValidateLocalSsdFlags(args)
instances_flags.ValidateNicFlags(args)
instances_flags.ValidateServiceAccountAndScopeArgs(args)
boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size)
utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type)
instance_template_ref = (
instance_templates_flags.INSTANCE_TEMPLATE_ARG.ResolveAsResource(
args, self.resources))
metadata = metadata_utils.ConstructMetadataMessage(
self.messages,
metadata=args.metadata,
metadata_from_file=args.metadata_from_file)
if hasattr(args, 'network_interface') and args.network_interface:
network_interfaces = (
instance_template_utils.CreateNetworkInterfaceMessages)(
resources=self.resources,
scope_lister=flags.GetDefaultScopeLister(
self.compute_client),
messages=self.messages,
network_interface_arg=args.network_interface,
region=args.region)
else:
network_interfaces = [
instance_template_utils.CreateNetworkInterfaceMessage(
resources=self.resources,
scope_lister=flags.GetDefaultScopeLister(
self.compute_client),
messages=self.messages,
network=args.network,
region=args.region,
subnet=args.subnet,
address=(instance_template_utils.EPHEMERAL_ADDRESS
if not args.no_address and not args.address else
args.address))
]
scheduling = instance_utils.CreateSchedulingMessage(
messages=self.messages,
maintenance_policy=args.maintenance_policy,
preemptible=args.preemptible,
restart_on_failure=args.restart_on_failure)
if args.no_service_account:
service_account = None
else:
service_account = args.service_account
service_accounts = instance_utils.CreateServiceAccountMessages(
messages=self.messages,
scopes=[] if args.no_scopes else args.scopes,
service_account=service_account)
create_boot_disk = not instance_utils.UseExistingBootDisk(args.disk
or [])
if create_boot_disk:
image_expander = image_utils.ImageExpander(self.compute_client,
self.resources)
image_uri, _ = image_expander.ExpandImageFlag(
user_project=instance_template_ref.project,
image=args.image,
image_family=args.image_family,
image_project=args.image_project,
return_image_resource=True)
else:
image_uri = None
if args.tags:
tags = self.messages.Tags(items=args.tags)
else:
tags = None
persistent_disks = (
instance_template_utils.CreatePersistentAttachedDiskMessages(
self.messages, args.disk or []))
persistent_create_disks = (
instance_template_utils.CreatePersistentCreateDiskMessages(
self, self.messages, getattr(args, 'create_disk', [])))
if create_boot_disk:
boot_disk_list = [
instance_template_utils.CreateDefaultBootAttachedDiskMessage(
messages=self.messages,
disk_type=args.boot_disk_type,
disk_device_name=args.boot_disk_device_name,
disk_auto_delete=args.boot_disk_auto_delete,
disk_size_gb=boot_disk_size_gb,
image_uri=image_uri)
]
else:
boot_disk_list = []
local_ssds = []
for x in args.local_ssd or []:
local_ssd = instance_utils.CreateLocalSsdMessage(
self.resources, self.messages, x.get('device-name'),
x.get('interface'), x.get('size'))
local_ssds.append(local_ssd)
disks = (boot_disk_list + persistent_disks + persistent_create_disks +
local_ssds)
machine_type = instance_utils.InterpretMachineType(
machine_type=args.machine_type,
custom_cpu=args.custom_cpu,
custom_memory=args.custom_memory,
ext=getattr(args, 'custom_extensions', None))
request = self.messages.ComputeInstanceTemplatesInsertRequest(
instanceTemplate=self.messages.InstanceTemplate(
properties=self.messages.InstanceProperties(
machineType=machine_type,
disks=disks,
canIpForward=args.can_ip_forward,
metadata=metadata,
networkInterfaces=network_interfaces,
serviceAccounts=service_accounts,
scheduling=scheduling,
tags=tags,
),
description=args.description,
name=instance_template_ref.Name(),
),
project=instance_template_ref.project)
return [request]
def Run(self, args):
"""Issues an InstanceTemplates.Insert request.
Args:
args: the argparse arguments that this command was invoked with.
Returns:
an InstanceTemplate message object
"""
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
instances_flags.ValidateKonletArgs(args)
instances_flags.ValidateDiskCommonFlags(args)
instances_flags.ValidateLocalSsdFlags(args)
instances_flags.ValidateServiceAccountAndScopeArgs(args)
instances_flags.ValidateNetworkTierArgs(args,
support_network_tier=True)
if instance_utils.UseExistingBootDisk(args.disk or []):
raise exceptions.InvalidArgumentException(
'--disk', 'Boot disk specified for containerized VM.')
boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size)
utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type)
instance_template_ref = (
CreateWithContainer.InstanceTemplateArg.ResolveAsResource(
args, holder.resources))
user_metadata = metadata_utils.ConstructMetadataMessage(
client.messages,
metadata=args.metadata,
metadata_from_file=args.metadata_from_file)
containers_utils.ValidateUserMetadata(user_metadata)
network_interface = instance_template_utils.CreateNetworkInterfaceMessage(
resources=holder.resources,
scope_lister=flags.GetDefaultScopeLister(client),
messages=client.messages,
network=args.network,
region=args.region,
subnet=args.subnet,
address=(instance_template_utils.EPHEMERAL_ADDRESS
if not args.no_address and not args.address else
args.address),
network_tier=getattr(args, 'network_tier', None))
scheduling = instance_utils.CreateSchedulingMessage(
messages=client.messages,
maintenance_policy=args.maintenance_policy,
preemptible=args.preemptible,
restart_on_failure=args.restart_on_failure)
if args.no_service_account:
service_account = None
else:
service_account = args.service_account
service_accounts = instance_utils.CreateServiceAccountMessages(
messages=client.messages,
scopes=[] if args.no_scopes else args.scopes,
service_account=service_account)
if (args.IsSpecified('image') or args.IsSpecified('image_family')
or args.IsSpecified('image_project')):
image_expander = image_utils.ImageExpander(client,
holder.resources)
image_uri, _ = image_expander.ExpandImageFlag(
user_project=instance_template_ref.project,
image=args.image,
image_family=args.image_family,
image_project=args.image_project)
if holder.resources.Parse(image_uri).project != 'cos-cloud':
log.warn(
'This container deployment mechanism requires a '
'Container-Optimized OS image in order to work. Select an '
'image from a cos-cloud project (cost-stable, cos-beta, '
'cos-dev image families).')
else:
image_uri = containers_utils.ExpandKonletCosImageFlag(client)
machine_type = instance_utils.InterpretMachineType(
machine_type=args.machine_type,
custom_cpu=args.custom_cpu,
custom_memory=args.custom_memory,
ext=getattr(args, 'custom_extensions', None))
metadata = containers_utils.CreateKonletMetadataMessage(
client.messages, args, instance_template_ref.Name(), user_metadata)
request = client.messages.ComputeInstanceTemplatesInsertRequest(
instanceTemplate=client.messages.InstanceTemplate(
properties=client.messages.InstanceProperties(
machineType=machine_type,
disks=self._CreateDiskMessages(
holder, args, boot_disk_size_gb, image_uri,
instance_template_ref.project),
canIpForward=args.can_ip_forward,
metadata=metadata,
minCpuPlatform=args.min_cpu_platform,
networkInterfaces=[network_interface],
serviceAccounts=service_accounts,
scheduling=scheduling,
tags=containers_utils.CreateTagsMessage(
client.messages, args.tags),
),
description=args.description,
name=instance_template_ref.Name(),
),
project=instance_template_ref.project)
return client.MakeRequests([(client.apitools_client.instanceTemplates,
'Insert', request)])
def ParseCreateOptionsBase(args):
"""Parses the flags provided with the cluster creation command."""
if not (args.IsSpecified('enable_basic_auth')
or args.IsSpecified('username')):
log.warning('Starting in 1.12, new clusters will have basic '
'authentication disabled by default. Basic authentication '
'can be enabled (or disabled) manually using the '
'`--[no-]enable-basic-auth` flag.')
if not args.IsSpecified('issue_client_certificate'):
log.warning(
'Starting in 1.12, new clusters will not have a client '
'certificate issued. You can manually enable (or disable) the '
'issuance of the client certificate using the '
'`--[no-]issue-client-certificate` flag.')
if args.IsSpecified('addons') and api_adapter.DASHBOARD in args.addons:
log.warning(
'The `KubernetesDashboard` addon is deprecated, and will be removed as '
'an option for new clusters starting in 1.15. It is recommended to use '
'the Cloud Console to manage and monitor your Kubernetes clusters, '
'workloads and applications. See: '
'https://cloud.google.com/kubernetes-engine/docs/concepts/dashboards'
)
flags.MungeBasicAuthFlags(args)
if args.IsSpecified('issue_client_certificate') and not (
args.IsSpecified('enable_basic_auth')
or args.IsSpecified('username')):
log.warning(
'If `--issue-client-certificate` is specified but '
'`--enable-basic-auth` or `--username` is not, our API will '
'treat that as `--no-enable-basic-auth`.')
enable_autorepair = cmd_util.GetAutoRepair(args)
flags.WarnForNodeModification(args, enable_autorepair)
metadata = metadata_utils.ConstructMetadataDict(args.metadata,
args.metadata_from_file)
return api_adapter.CreateClusterOptions(
accelerators=args.accelerator,
additional_zones=args.additional_zones,
addons=args.addons,
cluster_ipv4_cidr=args.cluster_ipv4_cidr,
cluster_secondary_range_name=args.cluster_secondary_range_name,
cluster_version=args.cluster_version,
node_version=args.node_version,
create_subnetwork=args.create_subnetwork,
disk_type=args.disk_type,
enable_autorepair=enable_autorepair,
enable_autoscaling=args.enable_autoscaling,
enable_autoupgrade=cmd_util.GetAutoUpgrade(args),
enable_stackdriver_kubernetes=args.enable_stackdriver_kubernetes,
enable_cloud_logging=args.enable_cloud_logging,
enable_cloud_monitoring=args.enable_cloud_monitoring,
enable_ip_alias=args.enable_ip_alias,
enable_kubernetes_alpha=args.enable_kubernetes_alpha,
enable_legacy_authorization=args.enable_legacy_authorization,
enable_main_authorized_networks=args.enable_main_authorized_networks,
enable_network_policy=args.enable_network_policy,
enable_private_nodes=args.enable_private_nodes,
enable_private_endpoint=args.enable_private_endpoint,
image_type=args.image_type,
image=args.image,
image_project=args.image_project,
image_family=args.image_family,
issue_client_certificate=args.issue_client_certificate,
labels=args.labels,
local_ssd_count=args.local_ssd_count,
maintenance_window=args.maintenance_window,
main_authorized_networks=args.main_authorized_networks,
main_ipv4_cidr=args.main_ipv4_cidr,
max_nodes=args.max_nodes,
max_nodes_per_pool=args.max_nodes_per_pool,
min_cpu_platform=args.min_cpu_platform,
min_nodes=args.min_nodes,
network=args.network,
node_disk_size_gb=utils.BytesToGb(args.disk_size),
node_labels=args.node_labels,
node_locations=args.node_locations,
node_machine_type=args.machine_type,
node_taints=args.node_taints,
num_nodes=args.num_nodes,
password=args.password,
preemptible=args.preemptible,
scopes=args.scopes,
service_account=args.service_account,
services_ipv4_cidr=args.services_ipv4_cidr,
services_secondary_range_name=args.services_secondary_range_name,
subnetwork=args.subnetwork,
tags=args.tags,
user=args.username,
metadata=metadata,
default_max_pods_per_node=args.default_max_pods_per_node,
max_pods_per_node=args.max_pods_per_node,
enable_tpu=args.enable_tpu,
tpu_ipv4_cidr=args.tpu_ipv4_cidr,
resource_usage_bigquery_dataset=args.resource_usage_bigquery_dataset,
enable_network_egress_metering=args.enable_network_egress_metering,
enable_resource_consumption_metering=\
args.enable_resource_consumption_metering)
def CreatePersistentCreateDiskMessages(compute_client,
resources,
csek_keys,
create_disks,
project,
location,
scope,
holder,
enable_kms=False,
enable_snapshots=False,
container_mount_disk=None,
resource_policy=False,
enable_source_snapshot_csek=False,
enable_image_csek=False,
support_replica_zones=False,
use_disk_type_uri=True,
support_multi_writer=False):
"""Returns a list of AttachedDisk messages for newly creating disks.
Args:
compute_client: creates resources,
resources: parser of resources,
csek_keys: customer suplied encryption keys,
create_disks: disk objects - contains following properties * name - the name
of disk, * description - an optional description for the disk, * mode -
'rw' (R/W), 'ro' (R/O) access mode, * disk-size - the size of the disk, *
disk-type - the type of the disk (HDD or SSD), * image - the name of the
image to initialize from, * image-csek-required - the name of the CSK
protected image, * image-family - the image family name, * image-project -
the project name that has the image, * auto-delete - whether disks is
deleted when VM is deleted, * device-name - device name on VM, *
source-snapshot - the snapshot to initialize from, *
source-snapshot-csek-required - CSK protected snapshot, *
disk-resource-policy - resource policies applied to disk. *
enable_source_snapshot_csek - CSK file for snapshot, * enable_image_csek -
CSK file for image
project: Project of instance that will own the new disks.
location: Location of the instance that will own the new disks.
scope: Location type of the instance that will own the new disks.
holder: Convenience class to hold lazy initialized client and resources.
enable_kms: True if KMS keys are supported for the disk.
enable_snapshots: True if snapshot initialization is supported for the disk.
container_mount_disk: list of disks to be mounted to container, if any.
resource_policy: True if resource-policies are enabled
enable_source_snapshot_csek: True if snapshot CSK files are enabled
enable_image_csek: True if image CSK files are enabled
support_replica_zones: True if we allow creation of regional disks
use_disk_type_uri: True to use disk type URI, False if naked type.
support_multi_writer: True if we allow multiple instances to write to disk.
Returns:
list of API messages for attached disks
"""
disks_messages = []
messages = compute_client.messages
compute = compute_client.apitools_client
for disk in create_disks or []:
name = disk.get('name')
# Resolves the mode.
mode_value = disk.get('mode', 'rw')
if mode_value == 'rw':
mode = messages.AttachedDisk.ModeValueValuesEnum.READ_WRITE
else:
mode = messages.AttachedDisk.ModeValueValuesEnum.READ_ONLY
auto_delete_value = disk.get('auto-delete', 'yes')
auto_delete = auto_delete_value == 'yes'
disk_size_gb = utils.BytesToGb(disk.get('size'))
disk_type = disk.get('type')
if disk_type:
if use_disk_type_uri:
disk_type_ref = instance_utils.ParseDiskType(
resources, disk_type, project, location, scope)
disk_type = disk_type_ref.SelfLink()
else:
disk_type = None
img = disk.get('image')
img_family = disk.get('image-family')
img_project = disk.get('image-project')
image_uri = None
if img or img_family:
image_expander = image_utils.ImageExpander(compute_client,
resources)
image_uri, _ = image_expander.ExpandImageFlag(
user_project=project,
image=img,
image_family=img_family,
image_project=img_project,
return_image_resource=False)
image_key = None
disk_key = None
if csek_keys:
image_key = csek_utils.MaybeLookupKeyMessagesByUri(
csek_keys, resources, [image_uri], compute)
if name:
disk_ref = resources.Parse(name,
collection='compute.disks',
params={'zone': location})
disk_key = csek_utils.MaybeLookupKeyMessage(
csek_keys, disk_ref, compute)
if enable_kms:
disk_key = kms_utils.MaybeGetKmsKeyFromDict(
disk, messages, disk_key)
initialize_params = messages.AttachedDiskInitializeParams(
diskName=name,
description=disk.get('description'),
sourceImage=image_uri,
diskSizeGb=disk_size_gb,
diskType=disk_type,
sourceImageEncryptionKey=image_key)
replica_zones = disk.get('replica-zones')
if support_replica_zones and replica_zones:
normalized_zones = []
for zone in replica_zones:
zone_ref = holder.resources.Parse(zone,
collection='compute.zones',
params={'project': project})
normalized_zones.append(zone_ref.SelfLink())
initialize_params.replicaZones = normalized_zones
if enable_snapshots:
snapshot_name = disk.get('source-snapshot')
attached_snapshot_uri = instance_utils.ResolveSnapshotURI(
snapshot=snapshot_name,
user_project=project,
resource_parser=resources)
if attached_snapshot_uri:
initialize_params.sourceImage = None
initialize_params.sourceSnapshot = attached_snapshot_uri
if resource_policy:
policies = disk.get('disk-resource-policy')
if policies:
initialize_params.resourcePolicies = policies
if enable_image_csek:
image_key_file = disk.get('image_csek')
if image_key_file:
initialize_params.imageKeyFile = image_key_file
if enable_source_snapshot_csek:
snapshot_key_file = disk.get('source_snapshot_csek')
if snapshot_key_file:
initialize_params.snapshotKeyFile = snapshot_key_file
boot = disk.get('boot') == 'yes'
multi_writer = disk.get('multi-writer')
if support_multi_writer and multi_writer:
initialize_params.multiWriter = True
device_name = instance_utils.GetDiskDeviceName(disk, name,
container_mount_disk)
create_disk = messages.AttachedDisk(
autoDelete=auto_delete,
boot=boot,
deviceName=device_name,
initializeParams=initialize_params,
mode=mode,
type=messages.AttachedDisk.TypeValueValuesEnum.PERSISTENT,
diskEncryptionKey=disk_key)
disks_messages.append(create_disk)
return disks_messages
def Run(self, args):
"""Creates and runs an InstanceTemplates.Insert request.
Args:
args: argparse.Namespace, An object that contains the values for the
arguments specified in the .Args() method.
Returns:
A resource object dispatched by display.Displayer().
"""
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
self.ValidateDiskFlags(args)
instances_flags.ValidateLocalSsdFlags(args)
instances_flags.ValidateNicFlags(args)
instances_flags.ValidateServiceAccountAndScopeArgs(args)
instances_flags.ValidateAcceleratorArgs(args)
boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size)
utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type)
instance_template_ref = (
Create.InstanceTemplateArg.ResolveAsResource(
args, holder.resources))
metadata = metadata_utils.ConstructMetadataMessage(
client.messages,
metadata=args.metadata,
metadata_from_file=args.metadata_from_file)
if hasattr(args, 'network_interface') and args.network_interface:
network_interfaces = (
instance_template_utils.CreateNetworkInterfaceMessages)(
resources=holder.resources,
scope_lister=flags.GetDefaultScopeLister(client),
messages=client.messages,
network_interface_arg=args.network_interface,
region=args.region,
support_network_tier=self._support_network_tier)
else:
network_tier = getattr(args, 'network_tier', None)
network_interfaces = [
instance_template_utils.CreateNetworkInterfaceMessage(
resources=holder.resources,
scope_lister=flags.GetDefaultScopeLister(client),
messages=client.messages,
network=args.network,
region=args.region,
subnet=args.subnet,
address=(instance_template_utils.EPHEMERAL_ADDRESS
if not args.no_address and not args.address
else args.address),
network_tier=network_tier)
]
scheduling = instance_utils.CreateSchedulingMessage(
messages=client.messages,
maintenance_policy=args.maintenance_policy,
preemptible=args.preemptible,
restart_on_failure=args.restart_on_failure)
if args.no_service_account:
service_account = None
else:
service_account = args.service_account
service_accounts = instance_utils.CreateServiceAccountMessages(
messages=client.messages,
scopes=[] if args.no_scopes else args.scopes,
service_account=service_account)
create_boot_disk = not instance_utils.UseExistingBootDisk(args.disk or [])
if create_boot_disk:
image_expander = image_utils.ImageExpander(client, holder.resources)
image_uri, _ = image_expander.ExpandImageFlag(
user_project=instance_template_ref.project,
image=args.image,
image_family=args.image_family,
image_project=args.image_project,
return_image_resource=True)
else:
image_uri = None
if args.tags:
tags = client.messages.Tags(items=args.tags)
else:
tags = None
persistent_disks = (
instance_template_utils.CreatePersistentAttachedDiskMessages(
client.messages, args.disk or []))
persistent_create_disks = (
instance_template_utils.CreatePersistentCreateDiskMessages(
client, holder.resources, instance_template_ref.project,
getattr(args, 'create_disk', [])))
if create_boot_disk:
boot_disk_list = [
instance_template_utils.CreateDefaultBootAttachedDiskMessage(
messages=client.messages,
disk_type=args.boot_disk_type,
disk_device_name=args.boot_disk_device_name,
disk_auto_delete=args.boot_disk_auto_delete,
disk_size_gb=boot_disk_size_gb,
image_uri=image_uri)]
else:
boot_disk_list = []
local_ssds = []
for x in args.local_ssd or []:
local_ssd = instance_utils.CreateLocalSsdMessage(
holder.resources,
client.messages,
x.get('device-name'),
x.get('interface'),
x.get('size'))
local_ssds.append(local_ssd)
disks = (
boot_disk_list + persistent_disks + persistent_create_disks + local_ssds
)
machine_type = instance_utils.InterpretMachineType(
machine_type=args.machine_type,
custom_cpu=args.custom_cpu,
custom_memory=args.custom_memory,
ext=getattr(args, 'custom_extensions', None))
guest_accelerators = (
instance_template_utils.CreateAcceleratorConfigMessages(
client.messages, getattr(args, 'accelerator', None)))
instance_properties = client.messages.InstanceProperties(
machineType=machine_type,
disks=disks,
canIpForward=args.can_ip_forward,
metadata=metadata,
networkInterfaces=network_interfaces,
serviceAccounts=service_accounts,
scheduling=scheduling,
tags=tags,
)
# TODO(b/36890961): Pass this directly into guestAccelerators once GA.
if guest_accelerators:
instance_properties.guestAccelerators = guest_accelerators
request = client.messages.ComputeInstanceTemplatesInsertRequest(
instanceTemplate=client.messages.InstanceTemplate(
properties=instance_properties,
description=args.description,
name=instance_template_ref.Name(),
),
project=instance_template_ref.project)
if getattr(args, 'min_cpu_platform', None):
request.instanceTemplate.properties.minCpuPlatform = args.min_cpu_platform
return client.MakeRequests([(client.apitools_client.instanceTemplates,
'Insert', request)])
def Run(self, args):
self.ValidateArgs(args)
client = self.context['dataproc_client']
messages = self.context['dataproc_messages']
cluster_ref = util.ParseCluster(args.name, self.context)
config_helper = compute_helpers.ConfigurationHelper.FromContext(
self.context)
compute_uris = config_helper.ResolveGceUris(args.name, args.image,
args.master_machine_type,
args.worker_machine_type,
args.network, args.subnet)
init_actions = []
timeout_str = str(args.initialization_action_timeout) + 's'
if args.initialization_actions:
init_actions = [
messages.NodeInitializationAction(executableFile=exe,
executionTimeout=timeout_str)
for exe in args.initialization_actions
]
expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes)
software_config = messages.SoftwareConfig(
imageVersion=args.image_version)
master_boot_disk_size_gb = args.master_boot_disk_size_gb
if args.master_boot_disk_size:
master_boot_disk_size_gb = (api_utils.BytesToGb(
args.master_boot_disk_size))
worker_boot_disk_size_gb = args.worker_boot_disk_size_gb
if args.worker_boot_disk_size:
worker_boot_disk_size_gb = (api_utils.BytesToGb(
args.worker_boot_disk_size))
preemptible_worker_boot_disk_size_gb = (api_utils.BytesToGb(
args.preemptible_worker_boot_disk_size))
if args.properties:
software_config.properties = encoding.DictToMessage(
args.properties, messages.SoftwareConfig.PropertiesValue)
gce_cluster_config = messages.GceClusterConfig(
networkUri=compute_uris['network'],
subnetworkUri=compute_uris['subnetwork'],
serviceAccountScopes=expanded_scopes,
zoneUri=compute_uris['zone'])
if args.tags:
gce_cluster_config.tags = args.tags
if args.metadata:
flat_metadata = dict(
(k, v) for d in args.metadata for k, v in d.items())
gce_cluster_config.metadata = encoding.DictToMessage(
flat_metadata, messages.GceClusterConfig.MetadataValue)
cluster_config = messages.ClusterConfig(
configBucket=args.bucket,
gceClusterConfig=gce_cluster_config,
masterConfig=messages.InstanceGroupConfig(
imageUri=compute_uris['image'],
machineTypeUri=compute_uris['master_machine_type'],
diskConfig=messages.DiskConfig(
bootDiskSizeGb=master_boot_disk_size_gb,
numLocalSsds=args.num_master_local_ssds,
),
),
workerConfig=messages.InstanceGroupConfig(
numInstances=args.num_workers,
imageUri=compute_uris['image'],
machineTypeUri=compute_uris['worker_machine_type'],
diskConfig=messages.DiskConfig(
bootDiskSizeGb=worker_boot_disk_size_gb,
numLocalSsds=args.num_worker_local_ssds,
),
),
initializationActions=init_actions,
softwareConfig=software_config,
)
# Secondary worker group is optional.
if args.num_preemptible_workers is not None:
cluster_config.secondaryWorkerConfig = (
messages.InstanceGroupConfig(
numInstances=args.num_preemptible_workers,
diskConfig=messages.DiskConfig(
bootDiskSizeGb=preemptible_worker_boot_disk_size_gb, ))
)
cluster = messages.Cluster(config=cluster_config,
clusterName=cluster_ref.clusterName,
projectId=cluster_ref.projectId)
operation = client.projects_regions_clusters.Create(
messages.DataprocProjectsRegionsClustersCreateRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
cluster=cluster))
if args. async:
log.status.write('Creating [{0}] with operation [{1}].'.format(
cluster_ref, operation.name))
return
operation = util.WaitForOperation(
operation, self.context, 'Waiting for cluster creation operation')
cluster = client.projects_regions_clusters.Get(cluster_ref.Request())
if cluster.status.state == (
messages.ClusterStatus.StateValueValuesEnum.RUNNING):
log.CreatedResource(cluster_ref)
else:
log.error('Create cluster failed!')
if operation.details:
log.error('Details:\n' + operation.details)
return cluster
def ParseCreateOptionsBase(args, is_autopilot, get_default, location,
project_id):
"""Parses the flags provided with the cluster creation command."""
if hasattr(args, 'addons') and args.IsSpecified('addons') and \
api_adapter.DASHBOARD in args.addons:
log.warning(
'The `KubernetesDashboard` addon is deprecated, and will be removed as '
'an option for new clusters starting in 1.15. It is recommended to use '
'the Cloud Console to manage and monitor your Kubernetes clusters, '
'workloads and applications. See: '
'https://cloud.google.com/kubernetes-engine/docs/concepts/dashboards'
)
flags.LogBasicAuthDeprecationWarning(args)
flags.MungeBasicAuthFlags(args)
MaybeLogAuthWarning(args)
MaybeLogReleaseChannelDefaultWarning(args)
enable_ip_alias = get_default('enable_ip_alias')
if hasattr(args, 'enable_ip_alias'):
flags.WarnForUnspecifiedIpAllocationPolicy(args)
enable_autorepair = None
if hasattr(args, 'enable_autorepair'):
enable_autorepair = cmd_util.GetAutoRepair(args)
if enable_autorepair:
flags.WarnForNodeModification(args, enable_autorepair)
metadata = metadata_utils.ConstructMetadataDict(
get_default('metadata'), get_default('metadata_from_file'))
cloud_run_config = flags.GetLegacyCloudRunFlag('{}_config', args,
get_default)
flags.ValidateCloudRunConfigCreateArgs(cloud_run_config,
get_default('addons'))
MaybeLogCloudNatHelpText(args, is_autopilot, location, project_id)
flags.ValidateNotificationConfigFlag(args)
return api_adapter.CreateClusterOptions(
accelerators=get_default('accelerator'),
additional_zones=get_default('additional_zones'),
addons=get_default('addons'),
boot_disk_kms_key=get_default('boot_disk_kms_key'),
cluster_ipv4_cidr=get_default('cluster_ipv4_cidr'),
cluster_secondary_range_name=get_default('cluster_secondary_range_name'),
cluster_version=get_default('cluster_version'),
cloud_run_config=cloud_run_config,
node_version=get_default('node_version'),
create_subnetwork=get_default('create_subnetwork'),
disable_default_snat=get_default('disable_default_snat'),
disk_type=get_default('disk_type'),
enable_autorepair=enable_autorepair,
enable_autoscaling=get_default('enable_autoscaling'),
enable_autoupgrade=(cmd_util.GetAutoUpgrade(args) if
hasattr(args, 'enable_autoupgrade')
else None),
enable_binauthz=get_default('enable_binauthz'),
enable_stackdriver_kubernetes=_GetEnableStackdriver(args),
enable_cloud_logging=args.enable_cloud_logging if (hasattr(args, 'enable_cloud_logging') and args.IsSpecified('enable_cloud_logging')) else None,
enable_cloud_monitoring=args.enable_cloud_monitoring if (hasattr(args, 'enable_cloud_monitoring') and args.IsSpecified('enable_cloud_monitoring')) else None,
enable_workload_monitoring_eap=get_default('enable_workload_monitoring_eap'),
enable_ip_alias=enable_ip_alias,
enable_intra_node_visibility=get_default('enable_intra_node_visibility'),
enable_kubernetes_alpha=get_default('enable_kubernetes_alpha'),
enable_cloud_run_alpha=flags.GetLegacyCloudRunFlag('enable_{}_alpha', args, get_default),
enable_legacy_authorization=get_default('enable_legacy_authorization'),
enable_master_authorized_networks=\
get_default('enable_master_authorized_networks'),
enable_master_global_access=get_default('enable_master_global_access'),
enable_network_policy=get_default('enable_network_policy'),
enable_private_nodes=get_default('enable_private_nodes'),
enable_private_endpoint=get_default('enable_private_endpoint'),
enable_gke_oidc=getattr(args, 'enable_gke_oidc', None),
image_type=get_default('image_type'),
image=get_default('image'),
image_project=get_default('image_project'),
image_family=get_default('image_family'),
issue_client_certificate=get_default('issue_client_certificate'),
labels=get_default('labels'),
local_ssd_count=get_default('local_ssd_count'),
maintenance_window=get_default('maintenance_window'),
maintenance_window_start=get_default('maintenance_window_start'),
maintenance_window_end=get_default('maintenance_window_end'),
maintenance_window_recurrence=get_default('maintenance_window_recurrence'),
master_authorized_networks=get_default('master_authorized_networks'),
master_ipv4_cidr=get_default('master_ipv4_cidr'),
max_nodes=get_default('max_nodes'),
max_nodes_per_pool=get_default('max_nodes_per_pool'),
min_cpu_platform=get_default('min_cpu_platform'),
min_nodes=get_default('min_nodes'),
network=get_default('network'),
node_disk_size_gb=utils.BytesToGb(args.disk_size) if hasattr(args, 'disk_size') else None,
node_labels=get_default('node_labels'),
node_locations=get_default('node_locations'),
node_machine_type=get_default('machine_type'),
node_taints=get_default('node_taints'),
num_nodes=get_default('num_nodes'),
password=get_default('password'),
preemptible=get_default('preemptible'),
scopes=get_default('scopes'),
service_account=get_default('service_account'),
services_ipv4_cidr=get_default('services_ipv4_cidr'),
services_secondary_range_name=get_default('services_secondary_range_name'),
subnetwork=get_default('subnetwork'),
system_config_from_file=get_default('system_config_from_file'),
private_ipv6_google_access_type=get_default('private_ipv6_google_access_type'),
tags=get_default('tags'),
user=get_default('username'),
metadata=metadata,
default_max_pods_per_node=get_default('default_max_pods_per_node'),
max_pods_per_node=get_default('max_pods_per_node'),
enable_tpu=get_default('enable_tpu'),
tpu_ipv4_cidr=get_default('tpu_ipv4_cidr'),
resource_usage_bigquery_dataset=get_default('resource_usage_bigquery_dataset'),
enable_network_egress_metering=get_default('enable_network_egress_metering'),
enable_resource_consumption_metering=get_default('enable_resource_consumption_metering'),
database_encryption_key=get_default('database_encryption_key'),
workload_pool=get_default('workload_pool'),
identity_provider=get_default('identity_provider'),
workload_metadata=get_default('workload_metadata'),
workload_metadata_from_node=get_default('workload_metadata_from_node'),
enable_vertical_pod_autoscaling=get_default('enable_vertical_pod_autoscaling'),
enable_autoprovisioning=get_default('enable_autoprovisioning'),
autoprovisioning_config_file=get_default('autoprovisioning_config_file'),
autoprovisioning_service_account=get_default('autoprovisioning_service_account'),
autoprovisioning_scopes=get_default('autoprovisioning_scopes'),
autoprovisioning_locations=get_default('autoprovisioning_locations'),
autoprovisioning_max_surge_upgrade=get_default('autoprovisioning_max_surge_upgrade'),
autoprovisioning_max_unavailable_upgrade=get_default('autoprovisioning_max_unavailable_upgrade'),
enable_autoprovisioning_autorepair=get_default('enable_autoprovisioning_autorepair'),
enable_autoprovisioning_autoupgrade=get_default('enable_autoprovisioning_autoupgrade'),
autoprovisioning_min_cpu_platform=get_default('autoprovisioning_min_cpu_platform'),
min_cpu=get_default('min_cpu'),
max_cpu=get_default('max_cpu'),
min_memory=get_default('min_memory'),
max_memory=get_default('max_memory'),
min_accelerator=get_default('min_accelerator'),
max_accelerator=get_default('max_accelerator'),
shielded_secure_boot=get_default('shielded_secure_boot'),
shielded_integrity_monitoring=get_default('shielded_integrity_monitoring'),
reservation_affinity=get_default('reservation_affinity'),
reservation=get_default('reservation'),
release_channel=get_default('release_channel'),
enable_shielded_nodes=get_default('enable_shielded_nodes'),
max_surge_upgrade=get_default('max_surge_upgrade'),
max_unavailable_upgrade=get_default('max_unavailable_upgrade'),
autopilot=is_autopilot)
def CreateRequests(self, args):
instances_flags.ValidateDockerArgs(args)
instances_flags.ValidateDiskCommonFlags(args)
instances_flags.ValidateLocalSsdFlags(args)
instances_flags.ValidateServiceAccountAndScopeArgs(args)
if instance_utils.UseExistingBootDisk(args.disk or []):
raise exceptions.InvalidArgumentException(
'--disk', 'Boot disk specified for containerized VM.')
scheduling = instance_utils.CreateSchedulingMessage(
messages=self.messages,
maintenance_policy=args.maintenance_policy,
preemptible=args.preemptible,
restart_on_failure=args.restart_on_failure)
if args.no_service_account:
service_account = None
else:
service_account = args.service_account
service_accounts = instance_utils.CreateServiceAccountMessages(
messages=self.messages,
scopes=[] if args.no_scopes else args.scopes,
service_account=service_account)
user_metadata = metadata_utils.ConstructMetadataMessage(
self.messages,
metadata=args.metadata,
metadata_from_file=args.metadata_from_file)
containers_utils.ValidateUserMetadata(user_metadata)
boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size)
utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type)
instance_refs = instances_flags.INSTANCES_ARG.ResolveAsResource(
args,
self.resources,
scope_lister=flags.GetDefaultScopeLister(self.compute_client,
self.project))
# Check if the zone is deprecated or has maintenance coming.
zone_resource_fetcher = zone_utils.ZoneResourceFetcher(
self.compute_client)
zone_resource_fetcher.WarnForZonalCreation(instance_refs)
instances_flags.ValidatePublicDnsFlags(args)
network_interface = instance_utils.CreateNetworkInterfaceMessage(
resources=self.resources,
compute_client=self.compute_client,
network=args.network,
subnet=args.subnet,
private_network_ip=args.private_network_ip,
no_address=args.no_address,
address=args.address,
instance_refs=instance_refs,
network_tier=args.network_tier,
no_public_dns=getattr(args, 'no_public_dns', None),
public_dns=getattr(args, 'public_dns', None),
no_public_ptr=getattr(args, 'no_public_ptr', None),
public_ptr=getattr(args, 'public_ptr', None),
no_public_ptr_domain=getattr(args, 'no_public_ptr_domain', None),
public_ptr_domain=getattr(args, 'public_ptr_domain', None))
machine_type_uris = instance_utils.CreateMachineTypeUris(
resources=self.resources,
compute_client=self.compute_client,
project=self.project,
machine_type=args.machine_type,
custom_cpu=args.custom_cpu,
custom_memory=args.custom_memory,
ext=getattr(args, 'custom_extensions', None),
instance_refs=instance_refs)
image_uri = containers_utils.ExpandCosImageFlag(self.compute_client)
args_labels = getattr(args, 'labels', None)
labels = None
if args_labels:
labels = self.messages.Instance.LabelsValue(additionalProperties=[
self.messages.Instance.LabelsValue.AdditionalProperty(
key=key, value=value)
for key, value in sorted(args.labels.iteritems())
])
requests = []
for instance_ref, machine_type_uri in zip(instance_refs,
machine_type_uris):
metadata = containers_utils.CreateMetadataMessage(
self.messages, args.run_as_privileged, args.container_manifest,
args.docker_image, args.port_mappings, args.run_command,
user_metadata, instance_ref.Name())
request = self.messages.ComputeInstancesInsertRequest(
instance=self.messages.Instance(
canIpForward=args.can_ip_forward,
disks=(self._CreateDiskMessages(args, boot_disk_size_gb,
image_uri, instance_ref)),
description=args.description,
machineType=machine_type_uri,
metadata=metadata,
minCpuPlatform=args.min_cpu_platform,
name=instance_ref.Name(),
networkInterfaces=[network_interface],
serviceAccounts=service_accounts,
scheduling=scheduling,
tags=containers_utils.CreateTagsMessage(
self.messages, args.tags),
),
project=self.project,
zone=instance_ref.zone)
if labels:
request.instance.labels = labels
requests.append(request)
return requests
def CreatePersistentCreateDiskMessages(compute_client,
resources,
csek_keys,
create_disks,
instance_ref,
enable_kms=False,
enable_snapshots=False,
container_mount_disk=None,
resource_policy=False):
"""Returns a list of AttachedDisk messages for newly creating disks.
Args:
compute_client: creates resources,
resources: parser of resources,
csek_keys: customer suplied encryption keys,
create_disks: disk objects - contains following properties
* name - the name of disk,
* description - an optional description for the disk,
* mode - 'rw' (R/W), 'ro' (R/O) access mode,
* disk-size - the size of the disk,
* disk-type - the type of the disk (HDD or SSD),
* image - the name of the image to initialize from,
* image-family - the image family name,
* image-project - the project name that has the image,
* auto-delete - whether disks is deleted when VM is deleted,
* device-name - device name on VM,
* source-snapshot - the snapshot to initialize from,
* disk-resource-policy - resource policies applied to disk.
instance_ref: reference to the instance that will own the new disks.
enable_kms: True if KMS keys are supported for the disk.
enable_snapshots: True if snapshot initialization is supported for the disk.
container_mount_disk: list of disks to be mounted to container, if any.
resource_policy: True if resource-policies are enabled
Returns:
list of API messages for attached disks
"""
disks_messages = []
messages = compute_client.messages
compute = compute_client.apitools_client
for disk in create_disks or []:
name = disk.get('name')
# Resolves the mode.
mode_value = disk.get('mode', 'rw')
if mode_value == 'rw':
mode = messages.AttachedDisk.ModeValueValuesEnum.READ_WRITE
else:
mode = messages.AttachedDisk.ModeValueValuesEnum.READ_ONLY
auto_delete_value = disk.get('auto-delete', 'yes')
auto_delete = auto_delete_value == 'yes'
disk_size_gb = utils.BytesToGb(disk.get('size'))
disk_type = disk.get('type')
if disk_type:
disk_type_ref = resources.Parse(disk_type,
collection='compute.diskTypes',
params={
'project':
instance_ref.project,
'zone': instance_ref.zone
})
disk_type_uri = disk_type_ref.SelfLink()
else:
disk_type_uri = None
img = disk.get('image')
img_family = disk.get('image-family')
img_project = disk.get('image-project')
image_uri = None
if img or img_family:
image_expander = image_utils.ImageExpander(compute_client,
resources)
image_uri, _ = image_expander.ExpandImageFlag(
user_project=instance_ref.project,
image=img,
image_family=img_family,
image_project=img_project,
return_image_resource=False)
image_key = None
disk_key = None
if csek_keys:
image_key = csek_utils.MaybeLookupKeyMessagesByUri(
csek_keys, resources, [image_uri], compute)
if name:
disk_ref = resources.Parse(name,
collection='compute.disks',
params={'zone': instance_ref.zone})
disk_key = csek_utils.MaybeLookupKeyMessage(
csek_keys, disk_ref, compute)
if enable_kms:
disk_key = kms_utils.MaybeGetKmsKeyFromDict(
disk, messages, disk_key)
initialize_params = messages.AttachedDiskInitializeParams(
diskName=name,
description=disk.get('description'),
sourceImage=image_uri,
diskSizeGb=disk_size_gb,
diskType=disk_type_uri,
sourceImageEncryptionKey=image_key)
if enable_snapshots:
snapshot_name = disk.get('source-snapshot')
attached_snapshot_uri = ResolveSnapshotURI(
snapshot=snapshot_name,
user_project=instance_ref.project,
resource_parser=resources)
if attached_snapshot_uri:
initialize_params.sourceImage = None
initialize_params.sourceSnapshot = attached_snapshot_uri
if resource_policy:
policies = disk.get('disk-resource-policy')
if policies:
initialize_params.resourcePolicies = policies
device_name = GetDiskDeviceName(disk, name, container_mount_disk)
create_disk = messages.AttachedDisk(
autoDelete=auto_delete,
boot=False,
deviceName=device_name,
initializeParams=initialize_params,
mode=mode,
type=messages.AttachedDisk.TypeValueValuesEnum.PERSISTENT,
diskEncryptionKey=disk_key)
disks_messages.append(create_disk)
return disks_messages
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
client = cloudbuild_util.GetClientInstanceAlpha()
messages = cloudbuild_util.GetMessagesModuleAlpha()
parent = properties.VALUES.core.project.Get(required=True)
# Get the workerpool proto from either the flags or the specified file.
wp = messages.WorkerPool()
if args.config_from_file is not None:
wp = workerpool_config.LoadWorkerpoolConfigFromPath(
args.config_from_file, messages)
else:
wp.name = args.WORKER_POOL
if args.worker_count is not None:
try:
wp.workerCount = int(args.worker_count)
except ValueError as e:
raise c_exceptions.InvalidArgumentException('--worker-count', e)
if args.regions is not None:
for region_str in args.regions:
region = Create._region_choice_to_enum[region_str]
wp.regions.append(region)
worker_config = messages.WorkerConfig()
if args.worker_machine_type is not None:
worker_config.machineType = args.worker_machine_type
if args.worker_disk_size is not None:
worker_config.diskSizeGb = compute_utils.BytesToGb(
args.worker_disk_size)
if any([
args.worker_network_project is not None,
args.worker_network_name is not None,
args.worker_network_subnet is not None
]):
if not all([
args.worker_network_project is not None,
args.worker_network_name is not None,
args.worker_network_subnet is not None
]):
raise c_exceptions.RequiredArgumentException(
'--worker_network_*',
'The flags --worker_network_project, --worker_network_name, and '
'--worker_network_subnet must all be set if any of them are set.')
# At this point all network flags are set, but possibly empty string.
# The API handles default values.
network = messages.Network()
network.projectId = args.worker_network_project
network.network = args.worker_network_name
network.subnetwork = args.worker_network_subnet
worker_config.network = network
if args.worker_tag is not None:
worker_config.tag = args.worker_tag
wp.workerConfig = worker_config
# Get the parent project ref
parent_resource = resources.REGISTRY.Create(
collection='cloudbuild.projects', projectId=parent)
# Send the Create request
created_wp = client.projects_workerPools.Create(
messages.CloudbuildProjectsWorkerPoolsCreateRequest(
workerPool=wp, parent=parent_resource.RelativeName()))
# Get the workerpool ref
wp_resource = resources.REGISTRY.Parse(
None,
collection='cloudbuild.projects.workerPools',
api_version='v1alpha1',
params={
'projectsId': parent,
'workerPoolsId': wp.name,
})
log.CreatedResource(wp_resource)
return created_wp
def ParseCreateOptionsBase(args):
"""Parses the flags provided with the cluster creation command."""
if not (args.IsSpecified('enable_basic_auth')
or args.IsSpecified('username')):
log.warning('Starting in 1.12, new clusters will have basic '
'authentication disabled by default. Basic authentication '
'can be enabled (or disabled) manually using the '
'`--[no-]enable-basic-auth` flag.')
if not args.IsSpecified('issue_client_certificate'):
log.warning(
'Starting in 1.12, new clusters will not have a client '
'certificate issued. You can manually enable (or disable) the '
'issuance of the client certificate using the '
'`--[no-]issue-client-certificate` flag.')
flags.MungeBasicAuthFlags(args)
if args.IsSpecified('issue_client_certificate') and not (
args.IsSpecified('enable_basic_auth')
or args.IsSpecified('username')):
log.warning(
'If `--issue-client-certificate` is specified but '
'`--enable-basic-auth` or `--username` is not, our API will '
'treat that as `--no-enable-basic-auth`.')
flags.WarnForUnspecifiedIpAllocationPolicy(args)
enable_autorepair = cmd_util.GetAutoRepair(args)
flags.WarnForNodeModification(args, enable_autorepair)
metadata = metadata_utils.ConstructMetadataDict(args.metadata,
args.metadata_from_file)
return api_adapter.CreateClusterOptions(
accelerators=args.accelerator,
additional_zones=args.additional_zones,
addons=args.addons,
cluster_ipv4_cidr=args.cluster_ipv4_cidr,
cluster_secondary_range_name=args.cluster_secondary_range_name,
cluster_version=args.cluster_version,
node_version=args.node_version,
create_subnetwork=args.create_subnetwork,
disk_type=args.disk_type,
enable_autorepair=enable_autorepair,
enable_autoscaling=args.enable_autoscaling,
enable_autoupgrade=cmd_util.GetAutoUpgrade(args),
enable_cloud_logging=args.enable_cloud_logging,
enable_cloud_monitoring=args.enable_cloud_monitoring,
enable_ip_alias=args.enable_ip_alias,
enable_kubernetes_alpha=args.enable_kubernetes_alpha,
enable_legacy_authorization=args.enable_legacy_authorization,
enable_master_authorized_networks=args.
enable_master_authorized_networks,
enable_network_policy=args.enable_network_policy,
enable_private_nodes=args.enable_private_nodes,
enable_private_endpoint=args.enable_private_endpoint,
image_type=args.image_type,
image=args.image,
image_project=args.image_project,
image_family=args.image_family,
issue_client_certificate=args.issue_client_certificate,
labels=args.labels,
local_ssd_count=args.local_ssd_count,
maintenance_window=args.maintenance_window,
master_authorized_networks=args.master_authorized_networks,
master_ipv4_cidr=args.master_ipv4_cidr,
max_nodes=args.max_nodes,
max_nodes_per_pool=args.max_nodes_per_pool,
min_cpu_platform=args.min_cpu_platform,
min_nodes=args.min_nodes,
network=args.network,
node_disk_size_gb=utils.BytesToGb(args.disk_size),
node_labels=args.node_labels,
node_locations=args.node_locations,
node_machine_type=args.machine_type,
node_taints=args.node_taints,
num_nodes=args.num_nodes,
password=args.password,
preemptible=args.preemptible,
scopes=args.scopes,
service_account=args.service_account,
services_ipv4_cidr=args.services_ipv4_cidr,
services_secondary_range_name=args.services_secondary_range_name,
subnetwork=args.subnetwork,
tags=args.tags,
user=args.username,
metadata=metadata,
default_max_pods_per_node=args.default_max_pods_per_node,
max_pods_per_node=args.max_pods_per_node,
enable_tpu=args.enable_tpu,
tpu_ipv4_cidr=args.tpu_ipv4_cidr)
def CreateRequests(self, args):
_ValidateDiskFlags(args)
instance_utils.ValidateLocalSsdFlags(args)
# TODO(user) drop test after CSEK goes GA
if hasattr(args, 'csek_key_file'):
self.csek_keys = csek_utils.CsekKeyStore.FromArgs(args)
else:
self.csek_keys = None
if args.maintenance_policy:
on_host_maintenance = (
self.messages.Scheduling.OnHostMaintenanceValueValuesEnum(
args.maintenance_policy))
else:
on_host_maintenance = None
# Note: We always specify automaticRestart=False for preemptible VMs. This
# makes sense, since no-restart-on-failure is defined as "store-true", and
# thus can't be given an explicit value. Hence it either has its default
# value (in which case we override it for convenience's sake to the only
# setting that makes sense for preemptible VMs), or the user actually
# specified no-restart-on-failure, the only usable setting.
if args.preemptible:
scheduling = self.messages.Scheduling(
automaticRestart=False,
onHostMaintenance=on_host_maintenance,
preemptible=True)
else:
scheduling = self.messages.Scheduling(
automaticRestart=args.restart_on_failure,
onHostMaintenance=on_host_maintenance)
service_accounts = self.CreateServiceAccountMessages(args)
if args.tags:
tags = self.messages.Tags(items=args.tags)
else:
tags = None
metadata = metadata_utils.ConstructMetadataMessage(
self.messages,
metadata=args.metadata,
metadata_from_file=args.metadata_from_file)
# If the user already provided an initial Windows password and
# username through metadata, then there is no need to check
# whether the image or the boot disk is Windows.
boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size)
utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type)
instance_refs = self.CreateZonalReferences(args.names, args.zone)
# Check if the zone is deprecated or has maintenance coming.
self.WarnForZonalCreation(instance_refs)
network_interface = self.CreateNetworkInterfaceMessage(
args, instance_refs)
# The element at index i is the machine type URI for instance
# i. We build this list here because we want to delay work that
# requires API calls as much as possible. This leads to a better
# user experience because the tool can fail fast upon a spelling
# mistake instead of delaying the user by making API calls whose
# purpose has already been rendered moot by the spelling mistake.
machine_type_uris = []
# Setting the machine type
machine_type_name = instance_utils.InterpretMachineType(args)
for instance_ref in instance_refs:
# Check to see if the custom machine type ratio is supported
instance_utils.CheckCustomCpuRamRatio(self, instance_ref.zone,
machine_type_name)
machine_type_uris.append(
self.CreateZonalReference(
machine_type_name,
instance_ref.zone,
resource_type='machineTypes').SelfLink())
create_boot_disk = not _UseExistingBootDisk(args)
if create_boot_disk:
image_uri, _ = self.ExpandImageFlag(args,
return_image_resource=False)
else:
image_uri = None
# A list of lists where the element at index i contains a list of
# disk messages that should be set for the instance at index i.
disks_messages = []
# A mapping of zone to boot disk references for all existing boot
# disks that are being attached.
# TODO(user): Simplify this once resources.Resource becomes
# hashable.
existing_boot_disks = {}
for instance_ref in instance_refs:
persistent_disks, boot_disk_ref = (
self.CreatePersistentAttachedDiskMessages(args, instance_ref))
local_ssds = [
instance_utils.CreateLocalSsdMessage(self,
x.get('device-name'),
x.get('interface'),
instance_ref.zone)
for x in args.local_ssd or []
]
if create_boot_disk:
boot_disk = self.CreateDefaultBootAttachedDiskMessage(
args, boot_disk_size_gb, image_uri, instance_ref)
persistent_disks = [boot_disk] + persistent_disks
else:
existing_boot_disks[boot_disk_ref.zone] = boot_disk_ref
disks_messages.append(persistent_disks + local_ssds)
requests = []
for instance_ref, machine_type_uri, disks in zip(
instance_refs, machine_type_uris, disks_messages):
requests.append(
self.messages.ComputeInstancesInsertRequest(
instance=self.messages.Instance(
canIpForward=args.can_ip_forward,
disks=disks,
description=args.description,
machineType=machine_type_uri,
metadata=metadata,
name=instance_ref.Name(),
networkInterfaces=[network_interface],
serviceAccounts=service_accounts,
scheduling=scheduling,
tags=tags,
),
project=self.project,
zone=instance_ref.zone))
return requests
def ParseCreateOptionsBase(args):
"""Parses the flags provided with the cluster creation command."""
if not (args.IsSpecified('enable_basic_auth')
or args.IsSpecified('username')):
log.warning('Starting in 1.12, new clusters will have basic '
'authentication disabled by default. Basic authentication '
'can be enabled (or disabled) manually using the '
'`--[no-]enable-basic-auth` flag.')
if not args.IsSpecified('issue_client_certificate'):
log.warning(
'Starting in 1.12, new clusters will not have a client '
'certificate issued. You can manually enable (or disable) the '
'issuance of the client certificate using the '
'`--[no-]issue-client-certificate` flag.')
flags.MungeBasicAuthFlags(args)
if (args.IsSpecified('enable_cloud_endpoints')
and properties.VALUES.container.new_scopes_behavior.GetBool()):
raise util.Error(
'Flag --[no-]enable-cloud-endpoints is not allowed if '
'property container/ new_scopes_behavior is set to true.')
if args.IsSpecified('enable_autorepair'):
enable_autorepair = args.enable_autorepair
else:
# Node pools using COS support auto repairs, enable it for them by default.
# Other node pools using (Ubuntu, custom images) don't support node auto
# repairs, attempting to enable autorepair for them will result in API call
# failing so don't do it.
enable_autorepair = ((args.image_type or '').lower() in ['', 'cos'])
flags.WarnForUnspecifiedIpAllocationPolicy(args)
metadata = metadata_utils.ConstructMetadataDict(args.metadata,
args.metadata_from_file)
return api_adapter.CreateClusterOptions(
accelerators=args.accelerator,
additional_zones=args.additional_zones,
addons=args.addons,
cluster_ipv4_cidr=args.cluster_ipv4_cidr,
cluster_secondary_range_name=args.cluster_secondary_range_name,
cluster_version=args.cluster_version,
node_version=args.node_version,
create_subnetwork=args.create_subnetwork,
disk_type=args.disk_type,
enable_autorepair=enable_autorepair,
enable_autoscaling=args.enable_autoscaling,
enable_autoupgrade=args.enable_autoupgrade,
enable_cloud_endpoints=args.enable_cloud_endpoints,
enable_cloud_logging=args.enable_cloud_logging,
enable_cloud_monitoring=args.enable_cloud_monitoring,
enable_ip_alias=args.enable_ip_alias,
enable_kubernetes_alpha=args.enable_kubernetes_alpha,
enable_legacy_authorization=args.enable_legacy_authorization,
enable_master_authorized_networks=args.
enable_master_authorized_networks,
enable_network_policy=args.enable_network_policy,
image_type=args.image_type,
image=args.image,
image_project=args.image_project,
image_family=args.image_family,
issue_client_certificate=args.issue_client_certificate,
labels=args.labels,
local_ssd_count=args.local_ssd_count,
maintenance_window=args.maintenance_window,
master_authorized_networks=args.master_authorized_networks,
max_nodes=args.max_nodes,
max_nodes_per_pool=args.max_nodes_per_pool,
min_cpu_platform=args.min_cpu_platform,
min_nodes=args.min_nodes,
network=args.network,
node_disk_size_gb=utils.BytesToGb(args.disk_size),
node_labels=args.node_labels,
node_locations=args.node_locations,
node_machine_type=args.machine_type,
node_taints=args.node_taints,
num_nodes=args.num_nodes,
password=args.password,
preemptible=args.preemptible,
scopes=args.scopes,
service_account=args.service_account,
services_ipv4_cidr=args.services_ipv4_cidr,
services_secondary_range_name=args.services_secondary_range_name,
subnetwork=args.subnetwork,
tags=args.tags,
user=args.username,
metadata=metadata)
def CreatePersistentCreateDiskMessages(client,
resources,
user_project,
create_disks,
support_kms=False,
container_mount_disk=None):
"""Returns a list of AttachedDisk messages.
Args:
client: Compute client adapter
resources: Compute resources registry
user_project: name of user project
create_disks: disk objects - contains following properties
* name - the name of disk,
* description - an optional description for the disk,
* mode - 'rw' (R/W), 'ro' (R/O) access mode,
* size - the size of the disk,
* type - the type of the disk (HDD or SSD),
* image - the name of the image to initialize from,
* image-family - the image family name,
* image-project - the project name that has the image,
* auto-delete - whether disks is deleted when VM is deleted ('yes'
if True),
* device-name - device name on VM.
support_kms: if KMS is supported
container_mount_disk: list of disks to be mounted to container, if any.
Returns:
list of API messages for attached disks
"""
disks_messages = []
for disk in create_disks or []:
name = disk.get('name')
# Resolves the mode.
mode_value = disk.get('mode', 'rw')
if mode_value == 'rw':
mode = client.messages.AttachedDisk.ModeValueValuesEnum.READ_WRITE
else:
mode = client.messages.AttachedDisk.ModeValueValuesEnum.READ_ONLY
auto_delete = disk.get('auto-delete') == 'yes'
disk_size_gb = utils.BytesToGb(disk.get('size'))
img = disk.get('image')
img_family = disk.get('image-family')
img_project = disk.get('image-project')
image_uri = None
if img or img_family:
image_expander = image_utils.ImageExpander(client, resources)
image_uri, _ = image_expander.ExpandImageFlag(
user_project=user_project,
image=img,
image_family=img_family,
image_project=img_project,
return_image_resource=False)
disk_key = None
if support_kms:
disk_key = kms_utils.MaybeGetKmsKeyFromDict(
disk, client.messages, disk_key)
device_name = instance_utils.GetDiskDeviceName(disk, name,
container_mount_disk)
create_disk = client.messages.AttachedDisk(
autoDelete=auto_delete,
boot=False,
deviceName=device_name,
initializeParams=client.messages.AttachedDiskInitializeParams(
diskName=name,
description=disk.get('description'),
sourceImage=image_uri,
diskSizeGb=disk_size_gb,
diskType=disk.get('type')),
mode=mode,
type=client.messages.AttachedDisk.TypeValueValuesEnum.PERSISTENT,
diskEncryptionKey=disk_key)
disks_messages.append(create_disk)
return disks_messages
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
Raises:
FailedBuildException: If the build is completed and not 'SUCCESS'.
"""
default_gcs_source = False
default_bucket_name = None
if args.gcs_source_staging_dir is None:
default_gcs_source = True
default_bucket_name = staging_bucket_util.GetDefaultStagingBucket()
args.gcs_source_staging_dir = 'gs://{}/source'.format(
default_bucket_name)
client = cloudbuild_util.GetClientInstance()
messages = cloudbuild_util.GetMessagesModule()
gcs_client = storage_api.StorageClient()
# First, create the build request.
build_timeout = properties.VALUES.builds.timeout.Get()
if build_timeout is not None:
try:
# A bare number is interpreted as seconds.
build_timeout_secs = int(build_timeout)
except ValueError:
build_timeout_duration = times.ParseDuration(build_timeout)
build_timeout_secs = int(build_timeout_duration.total_seconds)
timeout_str = six.text_type(build_timeout_secs) + 's'
else:
timeout_str = None
if args.tag is not None:
if (properties.VALUES.builds.check_tag.GetBool()
and 'gcr.io/' not in args.tag):
raise c_exceptions.InvalidArgumentException(
'--tag',
'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.'
)
if properties.VALUES.builds.use_kaniko.GetBool():
if args.no_cache:
ttl = '0h'
else:
ttl = '{}h'.format(
properties.VALUES.builds.kaniko_cache_ttl.Get())
build_config = messages.Build(
steps=[
messages.BuildStep(
name=properties.VALUES.builds.kaniko_image.Get(),
args=[
'--destination',
args.tag,
'--cache',
'--cache-ttl',
ttl,
'--cache-dir',
'',
],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(
args.substitutions, messages))
else:
if args.no_cache:
raise c_exceptions.InvalidArgumentException(
'no-cache',
'Cannot specify --no-cache if builds/use_kaniko property is '
'False')
build_config = messages.Build(
images=[args.tag],
steps=[
messages.BuildStep(
name='gcr.io/cloud-builders/docker',
args=[
'build', '--network', 'cloudbuild',
'--no-cache', '-t', args.tag, '.'
],
),
],
timeout=timeout_str,
substitutions=cloudbuild_util.EncodeSubstitutions(
args.substitutions, messages))
elif args.config is not None:
if args.no_cache:
raise c_exceptions.ConflictingArgumentsException(
'--config', '--no-cache')
if not args.config:
raise c_exceptions.InvalidArgumentException(
'--config', 'Config file path must not be empty.')
build_config = config.LoadCloudbuildConfigFromPath(
args.config, messages, params=args.substitutions)
else:
raise c_exceptions.OneOfArgumentsRequiredException(
['--tag', '--config'],
'Requires either a docker tag or a config file.')
# If timeout was set by flag, overwrite the config file.
if timeout_str:
build_config.timeout = timeout_str
# --no-source overrides the default --source.
if not args.IsSpecified('source') and args.no_source:
args.source = None
gcs_source_staging = None
if args.source:
suffix = '.tgz'
if args.source.startswith('gs://') or os.path.isfile(args.source):
_, suffix = os.path.splitext(args.source)
# Next, stage the source to Cloud Storage.
staged_object = '{stamp}-{uuid}{suffix}'.format(
stamp=times.GetTimeStampFromDateTime(times.Now()),
uuid=uuid.uuid4().hex,
suffix=suffix,
)
gcs_source_staging_dir = resources.REGISTRY.Parse(
args.gcs_source_staging_dir, collection='storage.objects')
# We create the bucket (if it does not exist) first. If we do an existence
# check and then create the bucket ourselves, it would be possible for an
# attacker to get lucky and beat us to creating the bucket. Block on this
# creation to avoid this race condition.
gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket)
# If no bucket is specified (for the source `default_gcs_source`), check
# that the default bucket is also owned by the project (b/33046325).
if default_gcs_source and not staging_bucket_util.BucketIsInProject(
gcs_client, default_bucket_name):
raise c_exceptions.RequiredArgumentException(
'gcs-source-staging-dir',
'A bucket with name {} already exists and is owned by '
'another project. Specify a bucket using '
'--gcs-source-staging-dir.'.format(default_bucket_name))
if gcs_source_staging_dir.object:
staged_object = gcs_source_staging_dir.object + '/' + staged_object
gcs_source_staging = resources.REGISTRY.Create(
collection='storage.objects',
bucket=gcs_source_staging_dir.bucket,
object=staged_object)
if args.source.startswith('gs://'):
gcs_source = resources.REGISTRY.Parse(
args.source, collection='storage.objects')
staged_source_obj = gcs_client.Rewrite(gcs_source,
gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
if not os.path.exists(args.source):
raise c_exceptions.BadFileException(
'could not find source [{src}]'.format(
src=args.source))
if os.path.isdir(args.source):
source_snapshot = snapshot.Snapshot(
args.source, ignore_file=args.ignore_file)
size_str = resource_transform.TransformSize(
source_snapshot.uncompressed_size)
log.status.Print(
'Creating temporary tarball archive of {num_files} file(s)'
' totalling {size} before compression.'.format(
num_files=len(source_snapshot.files),
size=size_str))
staged_source_obj = source_snapshot.CopyTarballToGCS(
gcs_client,
gcs_source_staging,
ignore_file=args.ignore_file)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
elif os.path.isfile(args.source):
unused_root, ext = os.path.splitext(args.source)
if ext not in _ALLOWED_SOURCE_EXT:
raise c_exceptions.BadFileException(
'Local file [{src}] is none of ' +
', '.join(_ALLOWED_SOURCE_EXT))
log.status.Print('Uploading local file [{src}] to '
'[gs://{bucket}/{object}].'.format(
src=args.source,
bucket=gcs_source_staging.bucket,
object=gcs_source_staging.object,
))
staged_source_obj = gcs_client.CopyFileToGCS(
args.source, gcs_source_staging)
build_config.source = messages.Source(
storageSource=messages.StorageSource(
bucket=staged_source_obj.bucket,
object=staged_source_obj.name,
generation=staged_source_obj.generation,
))
else:
# No source
if not args.no_source:
raise c_exceptions.InvalidArgumentException(
'--no-source', 'To omit source, use the --no-source flag.')
if args.gcs_log_dir:
gcs_log_dir = resources.REGISTRY.Parse(
args.gcs_log_dir, collection='storage.objects')
build_config.logsBucket = ('gs://' + gcs_log_dir.bucket + '/' +
gcs_log_dir.object)
# Machine type.
if args.machine_type is not None:
machine_type = flags.GetMachineType(args.machine_type)
if not build_config.options:
build_config.options = messages.BuildOptions()
build_config.options.machineType = machine_type
# Disk size.
if args.disk_size is not None:
disk_size = compute_utils.BytesToGb(args.disk_size)
if not build_config.options:
build_config.options = messages.BuildOptions()
build_config.options.diskSizeGb = int(disk_size)
log.debug('submitting build: ' + repr(build_config))
# Start the build.
op = client.projects_builds.Create(
messages.CloudbuildProjectsBuildsCreateRequest(
build=build_config,
projectId=properties.VALUES.core.project.Get()))
json = encoding.MessageToJson(op.metadata)
build = encoding.JsonToMessage(messages.BuildOperationMetadata,
json).build
build_ref = resources.REGISTRY.Create(
collection='cloudbuild.projects.builds',
projectId=build.projectId,
id=build.id)
log.CreatedResource(build_ref)
if build.logUrl:
log.status.Print('Logs are available at [{log_url}].'.format(
log_url=build.logUrl))
else:
log.status.Print('Logs are available in the Cloud Console.')
# If the command is run --async, we just print out a reference to the build.
if args.async_:
return build
mash_handler = execution.MashHandler(
execution.GetCancelBuildHandler(client, messages, build_ref))
# Otherwise, logs are streamed from GCS.
with execution_utils.CtrlCSection(mash_handler):
build = cb_logs.CloudBuildClient(client,
messages).Stream(build_ref)
if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT:
log.status.Print(
'Your build timed out. Use the [--timeout=DURATION] flag to change '
'the timeout threshold.')
if build.status != messages.Build.StatusValueValuesEnum.SUCCESS:
raise FailedBuildException(build)
return build
def _CreateRequests(self, args, instance_refs, compute_client,
resource_parser):
# gcloud creates default values for some fields in Instance resource
# when no value was specified on command line.
# When --source-instance-template was specified, defaults are taken from
# Instance Template and gcloud flags are used to override them - by default
# fields should not be initialized.
source_instance_template = self.GetSourceInstanceTemplate(
args, resource_parser)
skip_defaults = source_instance_template is not None
# This feature is only exposed in alpha/beta
allow_rsa_encrypted = self.ReleaseTrack() in [
base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA
]
self.csek_keys = csek_utils.CsekKeyStore.FromArgs(
args, allow_rsa_encrypted)
if (skip_defaults and not args.IsSpecified('maintenance_policy')
and not args.IsSpecified('preemptible')
and not args.IsSpecified('restart_on_failure')):
scheduling = None
else:
scheduling = instance_utils.CreateSchedulingMessage(
messages=compute_client.messages,
maintenance_policy=args.maintenance_policy,
preemptible=args.preemptible,
restart_on_failure=args.restart_on_failure)
if args.tags:
tags = compute_client.messages.Tags(items=args.tags)
else:
tags = None
labels = None
args_labels = getattr(args, 'labels', None)
if args_labels:
labels = compute_client.messages.Instance.LabelsValue(
additionalProperties=[
compute_client.messages.Instance.LabelsValue.
AdditionalProperty(key=key, value=value)
for key, value in sorted(args.labels.iteritems())
])
if (skip_defaults and not args.IsSpecified('metadata')
and not args.IsSpecified('metadata_from_file')):
metadata = None
else:
metadata = metadata_utils.ConstructMetadataMessage(
compute_client.messages,
metadata=args.metadata,
metadata_from_file=args.metadata_from_file)
# If the user already provided an initial Windows password and
# username through metadata, then there is no need to check
# whether the image or the boot disk is Windows.
boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size)
utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type)
# Check if the zone is deprecated or has maintenance coming.
zone_resource_fetcher = zone_utils.ZoneResourceFetcher(compute_client)
zone_resource_fetcher.WarnForZonalCreation(instance_refs)
network_interface_arg = getattr(args, 'network_interface', None)
if network_interface_arg:
network_interfaces = instance_utils.CreateNetworkInterfaceMessages(
resources=resource_parser,
compute_client=compute_client,
network_interface_arg=network_interface_arg,
instance_refs=instance_refs,
support_network_tier=self._support_network_tier)
else:
if self._support_public_dns is True:
instances_flags.ValidatePublicDnsFlags(args)
if self._support_public_ptr is True:
instances_flags.ValidatePublicPtrFlags(args)
if (skip_defaults and not args.IsSpecified('network')
and not args.IsSpecified('subnet')
and not args.IsSpecified('private_network_ip')
and not args.IsSpecified('no_address')
and not args.IsSpecified('address')
and not args.IsSpecified('network_tier')
and not args.IsSpecified('no_public_dns')
and not args.IsSpecified('public_dns')
and not args.IsSpecified('no_public_ptr')
and not args.IsSpecified('public_ptr')
and not args.IsSpecified('no_public_ptr_domain')
and not args.IsSpecified('public_ptr_domain')):
network_interfaces = []
else:
network_tier = getattr(args, 'network_tier', None)
network_interfaces = [
instance_utils.CreateNetworkInterfaceMessage(
resources=resource_parser,
compute_client=compute_client,
network=args.network,
subnet=args.subnet,
private_network_ip=args.private_network_ip,
no_address=args.no_address,
address=args.address,
instance_refs=instance_refs,
network_tier=network_tier,
no_public_dns=getattr(args, 'no_public_dns', None),
public_dns=getattr(args, 'public_dns', None),
no_public_ptr=getattr(args, 'no_public_ptr', None),
public_ptr=getattr(args, 'public_ptr', None),
no_public_ptr_domain=getattr(args,
'no_public_ptr_domain',
None),
public_ptr_domain=getattr(args, 'public_ptr_domain',
None))
]
if (skip_defaults and not args.IsSpecified('machine_type')
and not args.IsSpecified('custom_cpu')
and not args.IsSpecified('custom_memory')):
machine_type_uris = [None for _ in instance_refs]
else:
machine_type_uris = instance_utils.CreateMachineTypeUris(
resources=resource_parser,
compute_client=compute_client,
machine_type=args.machine_type,
custom_cpu=args.custom_cpu,
custom_memory=args.custom_memory,
ext=getattr(args, 'custom_extensions', None),
instance_refs=instance_refs)
create_boot_disk = not instance_utils.UseExistingBootDisk(args.disk
or [])
if create_boot_disk:
image_expander = image_utils.ImageExpander(compute_client,
resource_parser)
image_uri, _ = image_expander.ExpandImageFlag(
user_project=instance_refs[0].project,
image=args.image,
image_family=args.image_family,
image_project=args.image_project,
return_image_resource=False)
else:
image_uri = None
# A list of lists where the element at index i contains a list of
# disk messages that should be set for the instance at index i.
disks_messages = []
# A mapping of zone to boot disk references for all existing boot
# disks that are being attached.
# TODO(b/36050875): Simplify since resources.Resource is now hashable.
existing_boot_disks = {}
if (skip_defaults and not args.IsSpecified('disk')
and not args.IsSpecified('create_disk')
and not args.IsSpecified('local_ssd')
and not args.IsSpecified('boot_disk_type')
and not args.IsSpecified('boot_disk_device_name')
and not args.IsSpecified('boot_disk_auto_delete')
and not args.IsSpecified('boot_disk_kms_key')
and not args.IsSpecified('boot_disk_kms_project')
and not args.IsSpecified('boot_disk_kms_location')
and not args.IsSpecified('boot_disk_kms_keyring')
and not args.IsSpecified('require_csek_key_create')):
disks_messages = [[] for _ in instance_refs]
else:
for instance_ref in instance_refs:
persistent_disks, boot_disk_ref = (
instance_utils.CreatePersistentAttachedDiskMessages(
resource_parser, compute_client, self.csek_keys,
args.disk or [], instance_ref))
persistent_create_disks = (
instance_utils.CreatePersistentCreateDiskMessages(
compute_client, resource_parser, self.csek_keys,
getattr(args, 'create_disk', []), instance_ref))
local_ssds = []
for x in args.local_ssd or []:
local_ssds.append(
instance_utils.CreateLocalSsdMessage(
resource_parser, compute_client.messages,
x.get('device-name'), x.get('interface'),
x.get('size'), instance_ref.zone,
instance_ref.project))
if create_boot_disk:
boot_disk = instance_utils.CreateDefaultBootAttachedDiskMessage(
compute_client,
resource_parser,
disk_type=args.boot_disk_type,
disk_device_name=args.boot_disk_device_name,
disk_auto_delete=args.boot_disk_auto_delete,
disk_size_gb=boot_disk_size_gb,
require_csek_key_create=(args.require_csek_key_create
if self.csek_keys else None),
image_uri=image_uri,
instance_ref=instance_ref,
csek_keys=self.csek_keys,
kms_args=args)
persistent_disks = [boot_disk] + persistent_disks
else:
existing_boot_disks[boot_disk_ref.zone] = boot_disk_ref
disks_messages.append(persistent_disks +
persistent_create_disks + local_ssds)
accelerator_args = getattr(args, 'accelerator', None)
project_to_sa = {}
requests = []
for instance_ref, machine_type_uri, disks in zip(
instance_refs, machine_type_uris, disks_messages):
if instance_ref.project not in project_to_sa:
scopes = None
if not args.no_scopes and not args.scopes:
# User didn't provide any input on scopes. If project has no default
# service account then we want to create a VM with no scopes
request = (
compute_client.apitools_client.projects, 'Get',
compute_client.messages.ComputeProjectsGetRequest(
project=instance_ref.project))
errors = []
result = compute_client.MakeRequests([request], errors)
if not errors:
if not result[0].defaultServiceAccount:
scopes = []
log.status.Print(
'There is no default service account for project {}. '
'Instance {} will not have scopes.'.format(
instance_ref.project, instance_ref.Name))
if scopes is None:
scopes = [] if args.no_scopes else args.scopes
if args.no_service_account:
service_account = None
else:
service_account = args.service_account
if (skip_defaults and not args.IsSpecified('scopes')
and not args.IsSpecified('no_scopes')
and not args.IsSpecified('service_account')
and not args.IsSpecified('no_service_account')):
service_accounts = []
else:
service_accounts = instance_utils.CreateServiceAccountMessages(
messages=compute_client.messages,
scopes=scopes,
service_account=service_account)
project_to_sa[instance_ref.project] = service_accounts
if skip_defaults and not args.IsSpecified('can_ip_forward'):
can_ip_forward = None
else:
can_ip_forward = args.can_ip_forward
instance = compute_client.messages.Instance(
canIpForward=can_ip_forward,
disks=disks,
description=args.description,
machineType=machine_type_uri,
metadata=metadata,
minCpuPlatform=args.min_cpu_platform,
name=instance_ref.Name(),
networkInterfaces=network_interfaces,
serviceAccounts=project_to_sa[instance_ref.project],
scheduling=scheduling,
tags=tags)
if getattr(args, 'deletion_protection', None) is not None:
instance.deletionProtection = args.deletion_protection
if labels:
instance.labels = labels
if accelerator_args:
accelerator_type_name = accelerator_args['type']
accelerator_type_ref = resource_parser.Parse(
accelerator_type_name,
collection='compute.acceleratorTypes',
params={
'project': instance_ref.project,
'zone': instance_ref.zone
})
# Accelerator count is default to 1.
accelerator_count = int(accelerator_args.get('count', 1))
accelerators = instance_utils.CreateAcceleratorConfigMessages(
compute_client.messages, accelerator_type_ref,
accelerator_count)
instance.guestAccelerators = accelerators
request = compute_client.messages.ComputeInstancesInsertRequest(
instance=instance,
project=instance_ref.project,
zone=instance_ref.zone)
if source_instance_template:
request.sourceInstanceTemplate = source_instance_template
sole_tenancy_host_arg = getattr(args, 'sole_tenancy_host', None)
if sole_tenancy_host_arg:
sole_tenancy_host_ref = resource_parser.Parse(
sole_tenancy_host_arg,
collection='compute.hosts',
params={
'project': instance_ref.project,
'zone': instance_ref.zone
})
request.instance.host = sole_tenancy_host_ref.SelfLink()
requests.append(
(compute_client.apitools_client.instances, 'Insert', request))
return requests
def GetClusterConfig(args,
dataproc,
project_id,
compute_resources,
beta=False,
include_deprecated=True,
include_ttl_config=False):
"""Get dataproc cluster configuration.
Args:
args: Arguments parsed from argparse.ArgParser.
dataproc: Dataproc object that contains client, messages, and resources
project_id: Dataproc project ID
compute_resources: compute resource for cluster
beta: use BETA only features
include_deprecated: whether to include deprecated args
include_ttl_config: whether to include Scheduled Delete(TTL) args
Returns:
cluster_config: Dataproc cluster configuration
"""
master_accelerator_type = None
worker_accelerator_type = None
master_accelerator_count = None
worker_accelerator_count = None
if beta:
if args.master_accelerator:
master_accelerator_type = args.master_accelerator['type']
master_accelerator_count = args.master_accelerator.get('count', 1)
if args.worker_accelerator:
worker_accelerator_type = args.worker_accelerator['type']
worker_accelerator_count = args.worker_accelerator.get('count', 1)
# Resolve non-zonal GCE resources
# We will let the server resolve short names of zonal resources because
# if auto zone is requested, we will not know the zone before sending the
# request
image_ref = args.image and compute_resources.Parse(
args.image,
params={'project': project_id},
collection='compute.images')
network_ref = args.network and compute_resources.Parse(
args.network,
params={'project': project_id},
collection='compute.networks')
subnetwork_ref = args.subnet and compute_resources.Parse(
args.subnet,
params={
'project': project_id,
'region': properties.VALUES.compute.region.GetOrFail,
},
collection='compute.subnetworks')
timeout_str = str(args.initialization_action_timeout) + 's'
init_actions = [
dataproc.messages.NodeInitializationAction(
executableFile=exe, executionTimeout=timeout_str)
for exe in (args.initialization_actions or [])
]
# Increase the client timeout for each initialization action.
args.timeout += args.initialization_action_timeout * len(init_actions)
expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes)
software_config = dataproc.messages.SoftwareConfig(
imageVersion=args.image_version)
if include_deprecated:
master_boot_disk_size_gb = args.master_boot_disk_size_gb
else:
master_boot_disk_size_gb = None
if args.master_boot_disk_size:
master_boot_disk_size_gb = (api_utils.BytesToGb(
args.master_boot_disk_size))
if include_deprecated:
worker_boot_disk_size_gb = args.worker_boot_disk_size_gb
else:
worker_boot_disk_size_gb = None
if args.worker_boot_disk_size:
worker_boot_disk_size_gb = (api_utils.BytesToGb(
args.worker_boot_disk_size))
preemptible_worker_boot_disk_size_gb = (api_utils.BytesToGb(
args.preemptible_worker_boot_disk_size))
if args.single_node or args.num_workers == 0:
# Explicitly specifying --num-workers=0 gives you a single node cluster,
# but if --num-workers is omitted, args.num_workers is None (not 0), and
# this property will not be set
args.properties[constants.ALLOW_ZERO_WORKERS_PROPERTY] = 'true'
if args.properties:
software_config.properties = encoding.DictToAdditionalPropertyMessage(
args.properties,
dataproc.messages.SoftwareConfig.PropertiesValue,
sort_items=True)
if args.components:
software_config_cls = dataproc.messages.SoftwareConfig
software_config.optionalComponents.extend(
list(
map(
software_config_cls.
OptionalComponentsValueListEntryValuesEnum,
args.components)))
gce_cluster_config = dataproc.messages.GceClusterConfig(
networkUri=network_ref and network_ref.SelfLink(),
subnetworkUri=subnetwork_ref and subnetwork_ref.SelfLink(),
internalIpOnly=args.no_address,
serviceAccount=args.service_account,
serviceAccountScopes=expanded_scopes,
zoneUri=properties.VALUES.compute.zone.GetOrFail())
if beta:
reservation_affinity = GetReservationAffinity(args, dataproc)
gce_cluster_config.reservationAffinity = reservation_affinity
if args.tags:
gce_cluster_config.tags = args.tags
if args.metadata:
flat_metadata = collections.OrderedDict([(k, v) for d in args.metadata
for k, v in d.items()])
gce_cluster_config.metadata = encoding.DictToAdditionalPropertyMessage(
flat_metadata, dataproc.messages.GceClusterConfig.MetadataValue)
master_accelerators = []
if master_accelerator_type:
master_accelerators.append(
dataproc.messages.AcceleratorConfig(
acceleratorTypeUri=master_accelerator_type,
acceleratorCount=master_accelerator_count))
worker_accelerators = []
if worker_accelerator_type:
worker_accelerators.append(
dataproc.messages.AcceleratorConfig(
acceleratorTypeUri=worker_accelerator_type,
acceleratorCount=worker_accelerator_count))
cluster_config = dataproc.messages.ClusterConfig(
configBucket=args.bucket,
gceClusterConfig=gce_cluster_config,
masterConfig=dataproc.messages.InstanceGroupConfig(
numInstances=args.num_masters,
imageUri=image_ref and image_ref.SelfLink(),
machineTypeUri=args.master_machine_type,
accelerators=master_accelerators,
diskConfig=GetDiskConfig(dataproc, args.master_boot_disk_type,
master_boot_disk_size_gb,
args.num_master_local_ssds)),
workerConfig=dataproc.messages.InstanceGroupConfig(
numInstances=args.num_workers,
imageUri=image_ref and image_ref.SelfLink(),
machineTypeUri=args.worker_machine_type,
accelerators=worker_accelerators,
diskConfig=GetDiskConfig(
dataproc,
args.worker_boot_disk_type,
worker_boot_disk_size_gb,
args.num_worker_local_ssds,
)),
initializationActions=init_actions,
softwareConfig=software_config,
)
if args.kerberos_config_file or args.kerberos_root_principal_password_uri:
cluster_config.securityConfig = dataproc.messages.SecurityConfig()
if args.kerberos_config_file:
cluster_config.securityConfig.kerberosConfig = ParseKerberosConfigFile(
dataproc, args.kerberos_config_file)
else:
kerberos_config = dataproc.messages.KerberosConfig()
kerberos_config.enableKerberos = True
if args.kerberos_root_principal_password_uri:
kerberos_config.rootPrincipalPasswordUri = \
args.kerberos_root_principal_password_uri
kerberos_kms_ref = args.CONCEPTS.kerberos_kms_key.Parse()
kerberos_config.kmsKeyUri = kerberos_kms_ref.RelativeName()
cluster_config.securityConfig.kerberosConfig = kerberos_config
if beta:
if args.enable_component_gateway:
cluster_config.endpointConfig = dataproc.messages.EndpointConfig(
enableHttpPortAccess=args.enable_component_gateway)
if args.autoscaling_policy:
cluster_config.autoscalingConfig = dataproc.messages.AutoscalingConfig(
policyUri=args.CONCEPTS.autoscaling_policy.Parse(
).RelativeName())
cluster_config.masterConfig.minCpuPlatform = args.master_min_cpu_platform
cluster_config.workerConfig.minCpuPlatform = args.worker_min_cpu_platform
if include_ttl_config:
lifecycle_config = dataproc.messages.LifecycleConfig()
changed_config = False
if args.max_age is not None:
lifecycle_config.autoDeleteTtl = str(args.max_age) + 's'
changed_config = True
if args.expiration_time is not None:
lifecycle_config.autoDeleteTime = times.FormatDateTime(
args.expiration_time)
changed_config = True
if args.max_idle is not None:
lifecycle_config.idleDeleteTtl = str(args.max_idle) + 's'
changed_config = True
if changed_config:
cluster_config.lifecycleConfig = lifecycle_config
if hasattr(args.CONCEPTS, 'kms_key'):
kms_ref = args.CONCEPTS.kms_key.Parse()
if kms_ref:
encryption_config = dataproc.messages.EncryptionConfig()
encryption_config.gcePdKmsKeyName = kms_ref.RelativeName()
cluster_config.encryptionConfig = encryption_config
else:
# Did user use any gce-pd-kms-key flags?
for keyword in [
'gce-pd-kms-key', 'gce-pd-kms-key-project',
'gce-pd-kms-key-location', 'gce-pd-kms-key-keyring'
]:
if getattr(args, keyword.replace('-', '_'), None):
raise exceptions.ArgumentError(
'--gce-pd-kms-key was not fully specified.')
# Secondary worker group is optional. However, users may specify
# future pVMs configuration at creation time.
if (args.num_preemptible_workers is not None
or preemptible_worker_boot_disk_size_gb is not None
or args.preemptible_worker_boot_disk_type is not None
or args.num_preemptible_worker_local_ssds is not None
or (beta and args.worker_min_cpu_platform is not None)):
cluster_config.secondaryWorkerConfig = (
dataproc.messages.InstanceGroupConfig(
numInstances=args.num_preemptible_workers,
diskConfig=GetDiskConfig(
dataproc,
args.preemptible_worker_boot_disk_type,
preemptible_worker_boot_disk_size_gb,
args.num_preemptible_worker_local_ssds,
)))
if beta and args.worker_min_cpu_platform:
cluster_config.secondaryWorkerConfig.minCpuPlatform = (
args.worker_min_cpu_platform)
return cluster_config
def CreateRequests(self, args):
instances_flags.ValidateDockerArgs(args)
instances_flags.ValidateDiskCommonFlags(args)
instances_flags.ValidateLocalSsdFlags(args)
if instance_utils.UseExistingBootDisk(args.disk or []):
raise exceptions.InvalidArgumentException(
'--disk', 'Boot disk specified for containerized VM.')
scheduling = instance_utils.CreateSchedulingMessage(
messages=self.messages,
maintenance_policy=args.maintenance_policy,
preemptible=args.preemptible,
restart_on_failure=args.restart_on_failure)
service_accounts = instance_utils.CreateServiceAccountMessages(
messages=self.messages,
scopes=([] if args.no_scopes else args.scopes))
user_metadata = metadata_utils.ConstructMetadataMessage(
self.messages,
metadata=args.metadata,
metadata_from_file=args.metadata_from_file)
containers_utils.ValidateUserMetadata(user_metadata)
boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size)
utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type)
instance_refs = instances_flags.INSTANCES_ARG.ResolveAsResource(
args,
self.resources,
scope_lister=flags.GetDefaultScopeLister(self.compute_client,
self.project))
# Check if the zone is deprecated or has maintenance coming.
self.WarnForZonalCreation(instance_refs)
network_interface = instance_utils.CreateNetworkInterfaceMessage(
scope_prompter=self,
compute_client=self.compute_client,
network=args.network,
subnet=args.subnet,
private_network_ip=args.private_network_ip,
no_address=args.no_address,
address=args.address,
instance_refs=instance_refs)
machine_type_uris = instance_utils.CreateMachineTypeUris(
scope_prompter=self,
compute_client=self.compute_client,
project=self.project,
machine_type=args.machine_type,
custom_cpu=args.custom_cpu,
custom_memory=args.custom_memory,
instance_refs=instance_refs)
image_uri = containers_utils.ExpandGciImageFlag(self.compute_client)
requests = []
for instance_ref, machine_type_uri in zip(instance_refs,
machine_type_uris):
metadata = containers_utils.CreateMetadataMessage(
self.messages, args.run_as_privileged, args.container_manifest,
args.docker_image, args.port_mappings, args.run_command,
user_metadata, instance_ref.Name())
requests.append(
self.messages.ComputeInstancesInsertRequest(
instance=self.messages.Instance(
canIpForward=args.can_ip_forward,
disks=(self._CreateDiskMessages(
args, boot_disk_size_gb, image_uri, instance_ref)),
description=args.description,
machineType=machine_type_uri,
metadata=metadata,
name=instance_ref.Name(),
networkInterfaces=[network_interface],
serviceAccounts=service_accounts,
scheduling=scheduling,
tags=containers_utils.CreateTagsMessage(
self.messages, args.tags),
),
project=self.project,
zone=instance_ref.zone))
return requests
def Run(self, args):
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
source_instance_template = self.GetSourceInstanceTemplate(
args, holder.resources)
# gcloud creates default values for some fields in Instance resource
# when no value was specified on command line.
# When --source-instance-template was specified, defaults are taken from
# Instance Template and gcloud flags are used to override them - by default
# fields should not be initialized.
skip_defaults = source_instance_template is not None
instances_flags.ValidateDockerArgs(args)
instances_flags.ValidateDiskCommonFlags(args)
instances_flags.ValidateLocalSsdFlags(args)
instances_flags.ValidateServiceAccountAndScopeArgs(args)
if instance_utils.UseExistingBootDisk(args.disk or []):
raise exceptions.InvalidArgumentException(
'--disk',
'Boot disk specified for containerized VM.')
if (skip_defaults and not args.IsSpecified('maintenance_policy') and
not args.IsSpecified('preemptible') and
not args.IsSpecified('restart_on_failure')):
scheduling = None
else:
scheduling = instance_utils.CreateSchedulingMessage(
messages=client.messages,
maintenance_policy=args.maintenance_policy,
preemptible=args.preemptible,
restart_on_failure=args.restart_on_failure)
if args.no_service_account:
service_account = None
else:
service_account = args.service_account
if (skip_defaults and not args.IsSpecified('scopes') and
not args.IsSpecified('no_scopes') and
not args.IsSpecified('service_account') and
not args.IsSpecified('no_service_account')):
service_accounts = []
else:
service_accounts = instance_utils.CreateServiceAccountMessages(
messages=client.messages,
scopes=[] if args.no_scopes else args.scopes,
service_account=service_account)
user_metadata = metadata_utils.ConstructMetadataMessage(
client.messages,
metadata=args.metadata,
metadata_from_file=args.metadata_from_file)
containers_utils.ValidateUserMetadata(user_metadata)
boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size)
utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type)
instance_refs = instances_flags.INSTANCES_ARG.ResolveAsResource(
args,
holder.resources,
scope_lister=flags.GetDefaultScopeLister(client))
# Check if the zone is deprecated or has maintenance coming.
zone_resource_fetcher = zone_utils.ZoneResourceFetcher(client)
zone_resource_fetcher.WarnForZonalCreation(instance_refs)
instances_flags.ValidatePublicDnsFlags(args)
if (skip_defaults and not args.IsSpecified('network') and
not args.IsSpecified('subnet') and
not args.IsSpecified('private_network_ip') and
not args.IsSpecified('no_address') and
not args.IsSpecified('address') and
not args.IsSpecified('network_tier') and
not args.IsSpecified('no_public_dns') and
not args.IsSpecified('public_dns') and
not args.IsSpecified('no_public_ptr') and
not args.IsSpecified('public_ptr') and
not args.IsSpecified('no_public_ptr_domain') and
not args.IsSpecified('public_ptr_domain')):
network_interfaces = []
else:
network_interfaces = [instance_utils.CreateNetworkInterfaceMessage(
resources=holder.resources,
compute_client=client,
network=args.network,
subnet=args.subnet,
private_network_ip=args.private_network_ip,
no_address=args.no_address,
address=args.address,
instance_refs=instance_refs,
network_tier=args.network_tier,
no_public_dns=getattr(args, 'no_public_dns', None),
public_dns=getattr(args, 'public_dns', None),
no_public_ptr=getattr(args, 'no_public_ptr', None),
public_ptr=getattr(args, 'public_ptr', None),
no_public_ptr_domain=getattr(args, 'no_public_ptr_domain', None),
public_ptr_domain=getattr(args, 'public_ptr_domain', None))]
if (skip_defaults and not args.IsSpecified('machine_type') and
not args.IsSpecified('custom_cpu') and
not args.IsSpecified('custom_memory')):
machine_type_uris = [None for _ in instance_refs]
else:
machine_type_uris = instance_utils.CreateMachineTypeUris(
resources=holder.resources,
compute_client=client,
machine_type=args.machine_type,
custom_cpu=args.custom_cpu,
custom_memory=args.custom_memory,
ext=getattr(args, 'custom_extensions', None),
instance_refs=instance_refs)
image_uri = containers_utils.ExpandCosImageFlag(client)
args_labels = getattr(args, 'labels', None)
labels = None
if args_labels:
labels = client.messages.Instance.LabelsValue(
additionalProperties=[
client.messages.Instance.LabelsValue.AdditionalProperty(
key=key, value=value)
for key, value in sorted(args.labels.iteritems())])
if skip_defaults and not args.IsSpecified('can_ip_forward'):
can_ip_forward = None
else:
can_ip_forward = args.can_ip_forward
requests = []
for instance_ref, machine_type_uri in zip(instance_refs, machine_type_uris):
metadata = containers_utils.CreateMetadataMessage(
client.messages, args.run_as_privileged, args.container_manifest,
args.docker_image, args.port_mappings, args.run_command,
user_metadata, instance_ref.Name())
request = client.messages.ComputeInstancesInsertRequest(
instance=client.messages.Instance(
canIpForward=can_ip_forward,
disks=(self._CreateDiskMessages(holder, args, boot_disk_size_gb,
image_uri, instance_ref,
skip_defaults)),
description=args.description,
machineType=machine_type_uri,
metadata=metadata,
minCpuPlatform=args.min_cpu_platform,
name=instance_ref.Name(),
networkInterfaces=network_interfaces,
serviceAccounts=service_accounts,
scheduling=scheduling,
tags=containers_utils.CreateTagsMessage(client.messages,
args.tags)),
project=instance_ref.project,
zone=instance_ref.zone)
if labels:
request.instance.labels = labels
if source_instance_template:
request.sourceInstanceTemplate = source_instance_template
requests.append((client.apitools_client.instances,
'Insert', request))
return client.MakeRequests(requests)
def CreateRequests(self, args):
"""Returns a list of requests necessary for adding disks."""
size_gb = utils.BytesToGb(args.size)
if not size_gb and not args.source_snapshot and not args.image:
if args.type and 'pd-ssd' in args.type:
size_gb = constants.DEFAULT_SSD_DISK_SIZE_GB
else:
size_gb = constants.DEFAULT_STANDARD_DISK_SIZE_GB
utils.WarnIfDiskSizeIsTooSmall(size_gb, args.type)
requests = []
disk_refs = self.CreateZonalReferences(args.names, args.zone)
# Check if the zone is deprecated or has maintenance coming.
self.WarnForZonalCreation(disk_refs)
if args.image:
source_image_uri, _ = self.ExpandImageFlag(
args, return_image_resource=False)
else:
source_image_uri = None
if args.source_snapshot:
snapshot_ref = self.CreateGlobalReference(
args.source_snapshot, resource_type='snapshots')
snapshot_uri = snapshot_ref.SelfLink()
else:
snapshot_uri = None
if hasattr(args, 'csek_key_file'):
csek_keys = csek_utils.CsekKeyStore.FromArgs(args)
else:
csek_keys = None
image_key_message_or_none, snapshot_key_message_or_none = (
csek_utils.MaybeLookupKeyMessagesByUri(
csek_keys, self.resources, [source_image_uri, snapshot_uri],
self.compute))
for disk_ref in disk_refs:
if args.type:
type_ref = self.CreateZonalReference(args.type,
disk_ref.zone,
resource_type='diskTypes')
type_uri = type_ref.SelfLink()
else:
type_uri = None
if csek_keys:
disk_key_or_none = csek_keys.LookupKey(
disk_ref, args.require_csek_key_create)
disk_key_message_or_none = csek_utils.MaybeToMessage(
disk_key_or_none, self.compute)
kwargs = {
'diskEncryptionKey': disk_key_message_or_none,
'sourceImageEncryptionKey': image_key_message_or_none,
'sourceSnapshotEncryptionKey': snapshot_key_message_or_none
}
else:
kwargs = {}
request = self.messages.ComputeDisksInsertRequest(
disk=self.messages.Disk(name=disk_ref.Name(),
description=args.description,
sizeGb=size_gb,
sourceSnapshot=snapshot_uri,
type=type_uri,
**kwargs),
project=self.project,
sourceImage=source_image_uri,
zone=disk_ref.zone)
requests.append(request)
return requests
def CreatePersistentCreateDiskMessages(scope_prompter, compute_client,
resources, csek_keys, create_disks,
instance_ref):
"""Returns a list of AttachedDisk messages for newly creating disks.
Args:
scope_prompter: Scope prompter object,
compute_client: creates resources,
resources: parser of resources,
csek_keys: customer suplied encryption keys,
create_disks: disk objects - contains following properties
* name - the name of disk,
* mode - 'rw' (R/W), 'ro' (R/O) access mode,
* disk-size - the size of the disk,
* disk-type - the type of the disk (HDD or SSD),
* image - the name of the image to initialize from,
* image-family - the image family name,
* image-project - the project name that has the image,
* auto-delete - whether disks is deleted when VM is deleted,
* device-name - device name on VM.
instance_ref: reference to the instance that will own the new disks.
Returns:
list of API messages for attached disks
"""
disks_messages = []
messages = compute_client.messages
compute = compute_client.apitools_client
for disk in create_disks or []:
name = disk.get('name')
# Resolves the mode.
mode_value = disk.get('mode', 'rw')
if mode_value == 'rw':
mode = messages.AttachedDisk.ModeValueValuesEnum.READ_WRITE
else:
mode = messages.AttachedDisk.ModeValueValuesEnum.READ_ONLY
auto_delete_value = disk.get('auto-delete', 'yes')
auto_delete = auto_delete_value == 'yes'
disk_size_gb = utils.BytesToGb(disk.get('size'))
disk_type = disk.get('type')
if disk_type:
disk_type_ref = resources.Parse(disk_type,
collection='compute.diskTypes',
params={'zone': instance_ref.zone})
disk_type_uri = disk_type_ref.SelfLink()
else:
disk_type_ref = None
disk_type_uri = None
image_uri, _ = scope_prompter.ExpandImageFlag(
user_project=scope_prompter.project,
image=disk.get('image'),
image_family=disk.get('image-family'),
image_project=disk.get('image-project'),
return_image_resource=False)
image_key = None
disk_key = None
if csek_keys:
image_key = csek_utils.MaybeLookupKeyMessagesByUri(csek_keys,
resources,
[image_uri],
compute)
if name:
disk_ref = resources.Parse(name,
collection='compute.disks',
params={'zone': instance_ref.zone})
disk_key = csek_utils.MaybeLookupKeyMessage(csek_keys, disk_ref,
compute)
create_disk = messages.AttachedDisk(
autoDelete=auto_delete,
boot=False,
deviceName=disk.get('device-name'),
initializeParams=messages.AttachedDiskInitializeParams(
diskName=name,
sourceImage=image_uri,
diskSizeGb=disk_size_gb,
diskType=disk_type_uri,
sourceImageEncryptionKey=image_key),
mode=mode,
type=messages.AttachedDisk.TypeValueValuesEnum.PERSISTENT,
diskEncryptionKey=disk_key)
disks_messages.append(create_disk)
return disks_messages
def Run(self, args):
self.ValidateArgs(args)
client = self.context['dataproc_client']
messages = self.context['dataproc_messages']
cluster_ref = util.ParseCluster(args.name, self.context)
compute_resources = compute_helpers.GetComputeResources(
self.ReleaseTrack(), args.name)
master_accelerator_type = None
worker_accelerator_type = None
master_accelerator_count = None
worker_accelerator_count = None
if self.ReleaseTrack() == base.ReleaseTrack.BETA:
if args.master_accelerator:
master_accelerator_type = args.master_accelerator['type']
master_accelerator_count = args.master_accelerator.get(
'count', 1)
if args.worker_accelerator:
worker_accelerator_type = args.worker_accelerator['type']
worker_accelerator_count = args.worker_accelerator.get(
'count', 1)
# Resolve GCE resources
zone_ref = compute_resources.Parse(None, collection='compute.zones')
image_ref = args.image and compute_resources.Parse(
args.image, collection='compute.images')
master_machine_type_ref = (args.master_machine_type
and compute_resources.Parse(
args.master_machine_type,
collection='compute.machineTypes'))
worker_machine_type_ref = (args.worker_machine_type
and compute_resources.Parse(
args.worker_machine_type,
collection='compute.machineTypes'))
network_ref = args.network and compute_resources.Parse(
args.network, collection='compute.networks')
subnetwork_ref = args.subnet and compute_resources.Parse(
args.subnet, collection='compute.subnetworks')
master_accelerator_type_ref = (
master_accelerator_type
and compute_resources.Parse(master_accelerator_type,
collection='compute.acceleratorTypes'))
worker_accelerator_type_ref = (
worker_accelerator_type
and compute_resources.Parse(worker_accelerator_type,
collection='compute.acceleratorTypes'))
init_actions = []
timeout_str = str(args.initialization_action_timeout) + 's'
if args.initialization_actions:
init_actions = [
messages.NodeInitializationAction(executableFile=exe,
executionTimeout=timeout_str)
for exe in args.initialization_actions
]
expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes)
software_config = messages.SoftwareConfig(
imageVersion=args.image_version)
master_boot_disk_size_gb = args.master_boot_disk_size_gb
if args.master_boot_disk_size:
master_boot_disk_size_gb = (api_utils.BytesToGb(
args.master_boot_disk_size))
worker_boot_disk_size_gb = args.worker_boot_disk_size_gb
if args.worker_boot_disk_size:
worker_boot_disk_size_gb = (api_utils.BytesToGb(
args.worker_boot_disk_size))
preemptible_worker_boot_disk_size_gb = (api_utils.BytesToGb(
args.preemptible_worker_boot_disk_size))
if args.single_node:
args.properties[constants.ALLOW_ZERO_WORKERS_PROPERTY] = 'true'
if args.properties:
software_config.properties = encoding.DictToMessage(
args.properties, messages.SoftwareConfig.PropertiesValue)
gce_cluster_config = messages.GceClusterConfig(
networkUri=network_ref and network_ref.SelfLink(),
subnetworkUri=subnetwork_ref and subnetwork_ref.SelfLink(),
serviceAccount=args.service_account,
serviceAccountScopes=expanded_scopes,
zoneUri=zone_ref and zone_ref.SelfLink())
if args.tags:
gce_cluster_config.tags = args.tags
if args.metadata:
flat_metadata = dict(
(k, v) for d in args.metadata for k, v in d.items())
gce_cluster_config.metadata = encoding.DictToMessage(
flat_metadata, messages.GceClusterConfig.MetadataValue)
master_accelerators = []
if master_accelerator_type:
master_accelerators.append(
messages.AcceleratorConfig(
acceleratorTypeUri=master_accelerator_type_ref
and master_accelerator_type_ref.SelfLink(),
acceleratorCount=master_accelerator_count))
worker_accelerators = []
if worker_accelerator_type:
worker_accelerators.append(
messages.AcceleratorConfig(
acceleratorTypeUri=worker_accelerator_type_ref
and worker_accelerator_type_ref.SelfLink(),
acceleratorCount=worker_accelerator_count))
cluster_config = messages.ClusterConfig(
configBucket=args.bucket,
gceClusterConfig=gce_cluster_config,
masterConfig=messages.InstanceGroupConfig(
numInstances=args.num_masters,
imageUri=image_ref and image_ref.SelfLink(),
machineTypeUri=master_machine_type_ref
and master_machine_type_ref.SelfLink(),
accelerators=master_accelerators,
diskConfig=messages.DiskConfig(
bootDiskSizeGb=master_boot_disk_size_gb,
numLocalSsds=args.num_master_local_ssds,
),
),
workerConfig=messages.InstanceGroupConfig(
numInstances=args.num_workers,
imageUri=image_ref and image_ref.SelfLink(),
machineTypeUri=worker_machine_type_ref
and worker_machine_type_ref.SelfLink(),
accelerators=worker_accelerators,
diskConfig=messages.DiskConfig(
bootDiskSizeGb=worker_boot_disk_size_gb,
numLocalSsds=args.num_worker_local_ssds,
),
),
initializationActions=init_actions,
softwareConfig=software_config,
)
# Secondary worker group is optional. However, users may specify
# future pVM disk size at creation time.
if (args.num_preemptible_workers is not None
or preemptible_worker_boot_disk_size_gb is not None):
cluster_config.secondaryWorkerConfig = (
messages.InstanceGroupConfig(
numInstances=args.num_preemptible_workers,
diskConfig=messages.DiskConfig(
bootDiskSizeGb=preemptible_worker_boot_disk_size_gb, ))
)
cluster = messages.Cluster(config=cluster_config,
clusterName=cluster_ref.clusterName,
projectId=cluster_ref.projectId)
self.ConfigureCluster(messages, args, cluster)
operation = client.projects_regions_clusters.Create(
messages.DataprocProjectsRegionsClustersCreateRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
cluster=cluster))
if args. async:
log.status.write('Creating [{0}] with operation [{1}].'.format(
cluster_ref, operation.name))
return
operation = util.WaitForOperation(
operation, self.context, 'Waiting for cluster creation operation')
get_request = messages.DataprocProjectsRegionsClustersGetRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
clusterName=cluster_ref.clusterName)
cluster = client.projects_regions_clusters.Get(get_request)
if cluster.status.state == (
messages.ClusterStatus.StateValueValuesEnum.RUNNING):
log.CreatedResource(cluster_ref)
else:
log.error('Create cluster failed!')
if operation.details:
log.error('Details:\n' + operation.details)
return cluster
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
wp_region = args.region
release_track = self.ReleaseTrack()
client = cloudbuild_util.GetClientInstance(release_track)
messages = cloudbuild_util.GetMessagesModule(release_track)
parent = properties.VALUES.core.project.Get(required=True)
# Get the workerpool proto from either the flags or the specified file.
wp = messages.WorkerPool()
if args.config_from_file is not None:
wp = workerpool_config.LoadWorkerpoolConfigFromPath(
args.config_from_file, messages)
else:
wp.name = args.WORKER_POOL
if args.peered_network is not None:
network_config = messages.NetworkConfig()
network_config.peeredNetwork = args.peered_network
wp.networkConfig = network_config
worker_config = messages.WorkerConfig()
if args.worker_machine_type is not None:
worker_config.machineType = args.worker_machine_type
if args.worker_disk_size is not None:
worker_config.diskSizeGb = compute_utils.BytesToGb(
args.worker_disk_size)
if args.no_external_ip:
worker_config.noExternalIp = True
wp.workerConfig = worker_config
# Get the workerpool ref
wp_resource = resources.REGISTRY.Parse(
None,
collection='cloudbuild.projects.locations.workerPools',
api_version=cloudbuild_util.
RELEASE_TRACK_TO_API_VERSION[release_track],
params={
'projectsId': parent,
'locationsId': wp_region,
'workerPoolsId': wp.name,
})
update_mask = cloudbuild_util.MessageToFieldPaths(wp)
req = messages.CloudbuildProjectsLocationsWorkerPoolsPatchRequest(
name=wp_resource.RelativeName(),
workerPool=wp,
updateMask=','.join(update_mask))
# Send the Update request
updated_op = client.projects_locations_workerPools.Patch(req)
op_resource = resources.REGISTRY.ParseRelativeName(
updated_op.name,
collection='cloudbuild.projects.locations.operations')
updated_wp = waiter.WaitFor(
waiter.CloudOperationPoller(client.projects_locations_workerPools,
client.projects_locations_operations),
op_resource, 'Updating worker pool')
log.UpdatedResource(wp_resource)
# Format the workerpool name for display
try:
updated_wp.name = cloudbuild_util.RegionalWorkerPoolShortName(
updated_wp.name)
except ValueError:
pass # Must be an old version.
return updated_wp
def Run(self, args):
compute_holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = compute_holder.client
disk_refs = self.ValidateAndParse(args, compute_holder)
size_gb = utils.BytesToGb(args.size)
from_image = args.image or args.image_family
if not size_gb and not args.source_snapshot and not from_image:
if args.type and 'pd-ssd' in args.type:
size_gb = constants.DEFAULT_SSD_DISK_SIZE_GB
else:
size_gb = constants.DEFAULT_STANDARD_DISK_SIZE_GB
utils.WarnIfDiskSizeIsTooSmall(size_gb, args.type)
requests = []
# Check if the zone is deprecated or has maintenance coming.
zone_resource_fetcher = zone_utils.ZoneResourceFetcher(client)
zone_resource_fetcher.WarnForZonalCreation(
(ref for ref in disk_refs if ref.Collection() == 'compute.disks'))
# Check if the region is deprecated or has maintenance coming.
region_resource_fetcher = region_utils.RegionResourceFetcher(client)
region_resource_fetcher.WarnForRegionalCreation(
(ref for ref in disk_refs
if ref.Collection() == 'compute.regionDisks'))
project_to_source_image = {}
image_expander = image_utils.ImageExpander(client,
compute_holder.resources)
for disk_ref in disk_refs:
if from_image:
if disk_ref.project not in project_to_source_image:
source_image_uri, _ = image_expander.ExpandImageFlag(
user_project=disk_ref.project,
image=args.image,
image_family=args.image_family,
image_project=args.image_project,
return_image_resource=False)
project_to_source_image[
disk_ref.project] = argparse.Namespace()
project_to_source_image[
disk_ref.project].uri = source_image_uri
else:
project_to_source_image[
disk_ref.project] = argparse.Namespace()
project_to_source_image[disk_ref.project].uri = None
snapshot_ref = disks_flags.SOURCE_SNAPSHOT_ARG.ResolveAsResource(
args, compute_holder.resources)
if snapshot_ref:
snapshot_uri = snapshot_ref.SelfLink()
else:
snapshot_uri = None
# This feature is only exposed in alpha/beta
allow_rsa_encrypted = self.ReleaseTrack() in [
base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA
]
csek_keys = csek_utils.CsekKeyStore.FromArgs(args, allow_rsa_encrypted)
for project in project_to_source_image:
source_image_uri = project_to_source_image[project].uri
project_to_source_image[project].keys = (
csek_utils.MaybeLookupKeyMessagesByUri(
csek_keys, compute_holder.resources,
[source_image_uri, snapshot_uri], client.apitools_client))
for disk_ref in disk_refs:
if args.type:
if disk_ref.Collection() == 'compute.disks':
type_ref = compute_holder.resources.Parse(
args.type,
collection='compute.diskTypes',
params={
'project': disk_ref.project,
'zone': disk_ref.zone
})
elif disk_ref.Collection() == 'compute.regionDisks':
type_ref = compute_holder.resources.Parse(
args.type,
collection='compute.regionDiskTypes',
params={
'project': disk_ref.project,
'region': disk_ref.region
})
type_uri = type_ref.SelfLink()
else:
type_uri = None
if csek_keys:
disk_key_or_none = csek_keys.LookupKey(
disk_ref, args.require_csek_key_create)
disk_key_message_or_none = csek_utils.MaybeToMessage(
disk_key_or_none, client.apitools_client)
kwargs = {
'diskEncryptionKey':
disk_key_message_or_none,
'sourceImageEncryptionKey':
project_to_source_image[disk_ref.project].keys[0],
'sourceSnapshotEncryptionKey':
project_to_source_image[disk_ref.project].keys[1]
}
else:
kwargs = {}
if disk_ref.Collection() == 'compute.disks':
request = client.messages.ComputeDisksInsertRequest(
disk=client.messages.Disk(name=disk_ref.Name(),
description=args.description,
sizeGb=size_gb,
sourceSnapshot=snapshot_uri,
type=type_uri,
**kwargs),
project=disk_ref.project,
sourceImage=project_to_source_image[disk_ref.project].uri,
zone=disk_ref.zone)
request = (client.apitools_client.disks, 'Insert', request)
elif disk_ref.Collection() == 'compute.regionDisks':
def SelfLink(zone, disk_ref):
return compute_holder.resources.Parse(
zone,
collection='compute.zones',
params={
'project': disk_ref.project
}).SelfLink()
zones = [
SelfLink(zone, disk_ref) for zone in args.replica_zones
]
request = client.messages.ComputeRegionDisksInsertRequest(
disk=client.messages.Disk(name=disk_ref.Name(),
description=args.description,
sizeGb=size_gb,
sourceSnapshot=snapshot_uri,
type=type_uri,
replicaZones=zones,
**kwargs),
project=disk_ref.project,
sourceImage=project_to_source_image[disk_ref.project].uri,
region=disk_ref.region)
request = (client.apitools_client.regionDisks, 'Insert',
request)
requests.append(request)
return client.MakeRequests(requests)
