def CreateRequests(self, args): """Returns a list of requests necessary for adding disks.""" size_gb = utils.BytesToGb(args.size) from_image = args.image or args.image_family if not size_gb and not args.source_snapshot and not from_image: if args.type and 'pd-ssd' in args.type: size_gb = constants.DEFAULT_SSD_DISK_SIZE_GB else: size_gb = constants.DEFAULT_STANDARD_DISK_SIZE_GB utils.WarnIfDiskSizeIsTooSmall(size_gb, args.type) requests = [] disk_refs = disks_flags.DISKS_ARG.ResolveAsResource( args, self.resources, scope_lister=flags.GetDefaultScopeLister(self.compute_client, self.project)) # Check if the zone is deprecated or has maintenance coming. self.WarnForZonalCreation(disk_refs) if from_image: source_image_uri, _ = self.ExpandImageFlag( args, return_image_resource=False) else: source_image_uri = None snapshot_ref = disks_flags.SOURCE_SNAPSHOT_ARG.ResolveAsResource( args, self.resources) if snapshot_ref: snapshot_uri = snapshot_ref.SelfLink() else: snapshot_uri = None # This feature is only exposed in alpha/beta allow_rsa_encrypted = self.ReleaseTrack() in [ base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA ] csek_keys = csek_utils.CsekKeyStore.FromArgs(args, allow_rsa_encrypted) image_key_message_or_none, snapshot_key_message_or_none = ( csek_utils.MaybeLookupKeyMessagesByUri( csek_keys, self.resources, [source_image_uri, snapshot_uri], self.compute)) for disk_ref in disk_refs: if args.type: type_ref = self.resources.Parse(args.type, collection='compute.diskTypes', params={'zone': disk_ref.zone}) type_uri = type_ref.SelfLink() else: type_uri = None if csek_keys: disk_key_or_none = csek_keys.LookupKey( disk_ref, args.require_csek_key_create) disk_key_message_or_none = csek_utils.MaybeToMessage( disk_key_or_none, self.compute) kwargs = { 'diskEncryptionKey': disk_key_message_or_none, 'sourceImageEncryptionKey': image_key_message_or_none, 'sourceSnapshotEncryptionKey': snapshot_key_message_or_none } else: kwargs = {} request = self.messages.ComputeDisksInsertRequest( disk=self.messages.Disk(name=disk_ref.Name(), description=args.description, sizeGb=size_gb, sourceSnapshot=snapshot_uri, type=type_uri, **kwargs), project=self.project, sourceImage=source_image_uri, zone=disk_ref.zone) requests.append(request) return requests
def _GetBootDiskSize(self, args): boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) return boot_disk_size_gb
def Run(self, args): """Issues an InstanceTemplates.Insert request. Args: args: the argparse arguments that this command was invoked with. Returns: an InstanceTemplate message object """ holder = base_classes.ComputeApiHolder(self.ReleaseTrack()) client = holder.client instances_flags.ValidateDockerArgs(args) instances_flags.ValidateDiskCommonFlags(args) instances_flags.ValidateLocalSsdFlags(args) instances_flags.ValidateServiceAccountAndScopeArgs(args) if instance_utils.UseExistingBootDisk(args.disk or []): raise exceptions.InvalidArgumentException( '--disk', 'Boot disk specified for containerized VM.') boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) instance_template_ref = ( CreateFromContainer.InstanceTemplateArg.ResolveAsResource( args, holder.resources)) user_metadata = metadata_utils.ConstructMetadataMessage( client.messages, metadata=args.metadata, metadata_from_file=args.metadata_from_file) containers_utils.ValidateUserMetadata(user_metadata) network_interface = instance_template_utils.CreateNetworkInterfaceMessage( resources=holder.resources, scope_lister=flags.GetDefaultScopeLister(client), messages=client.messages, network=args.network, region=args.region, subnet=args.subnet, address=(instance_template_utils.EPHEMERAL_ADDRESS if not args.no_address and not args.address else args.address), network_tier=args.network_tier) scheduling = instance_utils.CreateSchedulingMessage( messages=client.messages, maintenance_policy=args.maintenance_policy, preemptible=args.preemptible, restart_on_failure=args.restart_on_failure) if args.no_service_account: service_account = None else: service_account = args.service_account service_accounts = instance_utils.CreateServiceAccountMessages( messages=client.messages, scopes=[] if args.no_scopes else args.scopes, service_account=service_account) image_uri = containers_utils.ExpandCosImageFlag(client) machine_type = instance_utils.InterpretMachineType( machine_type=args.machine_type, custom_cpu=args.custom_cpu, custom_memory=args.custom_memory, ext=getattr(args, 'custom_extensions', None)) metadata = containers_utils.CreateMetadataMessage( client.messages, args.run_as_privileged, args.container_manifest, args.docker_image, args.port_mappings, args.run_command, user_metadata, instance_template_ref.Name()) request = client.messages.ComputeInstanceTemplatesInsertRequest( instanceTemplate=client.messages.InstanceTemplate( properties=client.messages.InstanceProperties( machineType=machine_type, disks=self._CreateDiskMessages( holder, args, boot_disk_size_gb, image_uri, instance_template_ref.project), canIpForward=args.can_ip_forward, metadata=metadata, minCpuPlatform=args.min_cpu_platform, networkInterfaces=[network_interface], serviceAccounts=service_accounts, scheduling=scheduling, tags=containers_utils.CreateTagsMessage( client.messages, args.tags), ), description=args.description, name=instance_template_ref.Name(), ), project=instance_template_ref.project) return client.MakeRequests([(client.apitools_client.instanceTemplates, 'Insert', request)])
def GetClusterConfig(args, dataproc, project_id, compute_resources, beta=False): """Get dataproc cluster configuration. Args: args: Arguments parsed from argparse.ArgParser. dataproc: Dataproc object that contains client, messages, and resources project_id: Dataproc project ID compute_resources: compute resource for cluster beta: use BETA only features Returns: cluster_config: Dataproc cluster configuration """ master_accelerator_type = None worker_accelerator_type = None master_accelerator_count = None worker_accelerator_count = None if beta: if args.master_accelerator: master_accelerator_type = args.master_accelerator['type'] master_accelerator_count = args.master_accelerator.get('count', 1) if args.worker_accelerator: worker_accelerator_type = args.worker_accelerator['type'] worker_accelerator_count = args.worker_accelerator.get('count', 1) # Resolve non-zonal GCE resources # We will let the server resolve short names of zonal resources because # if auto zone is requested, we will not know the zone before sending the # request image_ref = args.image and compute_resources.Parse( args.image, params={'project': project_id}, collection='compute.images') network_ref = args.network and compute_resources.Parse( args.network, params={'project': project_id}, collection='compute.networks') subnetwork_ref = args.subnet and compute_resources.Parse( args.subnet, params={ 'project': project_id, 'region': properties.VALUES.compute.region.GetOrFail, }, collection='compute.subnetworks') timeout_str = str(args.initialization_action_timeout) + 's' init_actions = [ dataproc.messages.NodeInitializationAction( executableFile=exe, executionTimeout=timeout_str) for exe in (args.initialization_actions or []) ] # Increase the client timeout for each initialization action. args.timeout += args.initialization_action_timeout * len(init_actions) expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes) software_config = dataproc.messages.SoftwareConfig( imageVersion=args.image_version) master_boot_disk_size_gb = args.master_boot_disk_size_gb if args.master_boot_disk_size: master_boot_disk_size_gb = (api_utils.BytesToGb( args.master_boot_disk_size)) worker_boot_disk_size_gb = args.worker_boot_disk_size_gb if args.worker_boot_disk_size: worker_boot_disk_size_gb = (api_utils.BytesToGb( args.worker_boot_disk_size)) preemptible_worker_boot_disk_size_gb = (api_utils.BytesToGb( args.preemptible_worker_boot_disk_size)) if args.single_node or args.num_workers == 0: # Explicitly specifying --num-workers=0 gives you a single node cluster, # but if --num-workers is omitted, args.num_workers is None (not 0), and # this property will not be set args.properties[constants.ALLOW_ZERO_WORKERS_PROPERTY] = 'true' if args.properties: software_config.properties = encoding.DictToMessage( args.properties, dataproc.messages.SoftwareConfig.PropertiesValue) gce_cluster_config = dataproc.messages.GceClusterConfig( networkUri=network_ref and network_ref.SelfLink(), subnetworkUri=subnetwork_ref and subnetwork_ref.SelfLink(), internalIpOnly=args.no_address, serviceAccount=args.service_account, serviceAccountScopes=expanded_scopes, zoneUri=properties.VALUES.compute.zone.GetOrFail()) if args.tags: gce_cluster_config.tags = args.tags if args.metadata: flat_metadata = dict( (k, v) for d in args.metadata for k, v in d.items()) gce_cluster_config.metadata = encoding.DictToMessage( flat_metadata, dataproc.messages.GceClusterConfig.MetadataValue) master_accelerators = [] if master_accelerator_type: master_accelerators.append( dataproc.messages.AcceleratorConfig( acceleratorTypeUri=master_accelerator_type, acceleratorCount=master_accelerator_count)) worker_accelerators = [] if worker_accelerator_type: worker_accelerators.append( dataproc.messages.AcceleratorConfig( acceleratorTypeUri=worker_accelerator_type, acceleratorCount=worker_accelerator_count)) cluster_config = dataproc.messages.ClusterConfig( configBucket=args.bucket, gceClusterConfig=gce_cluster_config, masterConfig=dataproc.messages.InstanceGroupConfig( numInstances=args.num_masters, imageUri=image_ref and image_ref.SelfLink(), machineTypeUri=args.master_machine_type, accelerators=master_accelerators, diskConfig=GetDiskConfig( dataproc, args.master_boot_disk_type if beta else None, master_boot_disk_size_gb, args.num_master_local_ssds, beta, )), workerConfig=dataproc.messages.InstanceGroupConfig( numInstances=args.num_workers, imageUri=image_ref and image_ref.SelfLink(), machineTypeUri=args.worker_machine_type, accelerators=worker_accelerators, diskConfig=GetDiskConfig( dataproc, args.worker_boot_disk_type if beta else None, worker_boot_disk_size_gb, args.num_worker_local_ssds, beta, )), initializationActions=init_actions, softwareConfig=software_config, ) if beta: cluster_config.masterConfig.minCpuPlatform = args.master_min_cpu_platform cluster_config.workerConfig.minCpuPlatform = args.worker_min_cpu_platform if beta: lifecycle_config = dataproc.messages.LifecycleConfig() changed_config = False if args.max_age is not None: lifecycle_config.autoDeleteTtl = str(args.max_age) + 's' changed_config = True if args.expiration_time is not None: lifecycle_config.autoDeleteTime = times.FormatDateTime( args.expiration_time) changed_config = True if args.max_idle is not None: lifecycle_config.idleDeleteTtl = str(args.max_idle) + 's' changed_config = True if changed_config: cluster_config.lifecycleConfig = lifecycle_config # Secondary worker group is optional. However, users may specify # future pVMs configuration at creation time. if (args.num_preemptible_workers is not None or preemptible_worker_boot_disk_size_gb is not None or (beta and (args.preemptible_worker_boot_disk_type is not None or args.worker_min_cpu_platform is not None))): cluster_config.secondaryWorkerConfig = ( dataproc.messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers, diskConfig=GetDiskConfig( dataproc, args.preemptible_worker_boot_disk_type if beta else None, preemptible_worker_boot_disk_size_gb, None, beta, ))) if beta and args.worker_min_cpu_platform: cluster_config.secondaryWorkerConfig.minCpuPlatform = ( args.worker_min_cpu_platform) return cluster_config
def _RunCreate(compute_api, args, support_source_instance, support_kms=False, support_min_node_cpus=False): """Common routine for creating instance template. This is shared between various release tracks. Args: compute_api: The compute api. args: argparse.Namespace, An object that contains the values for the arguments specified in the .Args() method. support_source_instance: indicates whether source instance is supported. support_kms: Indicate whether KMS is integrated or not. support_min_node_cpus: Indicate whether the --min-node-cpus flag for sole tenancy overcommit is supported. Returns: A resource object dispatched by display.Displayer(). """ _ValidateInstancesFlags(args, support_kms=support_kms) instances_flags.ValidateNetworkTierArgs(args) client = compute_api.client boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) instance_template_ref = (Create.InstanceTemplateArg.ResolveAsResource( args, compute_api.resources)) metadata = metadata_utils.ConstructMetadataMessage( client.messages, metadata=args.metadata, metadata_from_file=args.metadata_from_file) if hasattr(args, 'network_interface') and args.network_interface: network_interfaces = ( instance_template_utils.CreateNetworkInterfaceMessages)( resources=compute_api.resources, scope_lister=flags.GetDefaultScopeLister(client), messages=client.messages, network_interface_arg=args.network_interface, region=args.region) else: network_tier = getattr(args, 'network_tier', None) network_interfaces = [ instance_template_utils.CreateNetworkInterfaceMessage( resources=compute_api.resources, scope_lister=flags.GetDefaultScopeLister(client), messages=client.messages, network=args.network, region=args.region, subnet=args.subnet, address=(instance_template_utils.EPHEMERAL_ADDRESS if not args.no_address and not args.address else args.address), network_tier=network_tier) ] # Compute the shieldedInstanceConfig message. shieldedinstance_config_message = BuildShieldedInstanceConfigMessage( messages=client.messages, args=args) node_affinities = sole_tenancy_util.GetSchedulingNodeAffinityListFromArgs( args, client.messages) min_node_cpus = None if support_min_node_cpus and args.IsSpecified('min_node_cpus'): min_node_cpus = args.min_node_cpus scheduling = instance_utils.CreateSchedulingMessage( messages=client.messages, maintenance_policy=args.maintenance_policy, preemptible=args.preemptible, restart_on_failure=args.restart_on_failure, node_affinities=node_affinities, min_node_cpus=min_node_cpus) if args.no_service_account: service_account = None else: service_account = args.service_account service_accounts = instance_utils.CreateServiceAccountMessages( messages=client.messages, scopes=[] if args.no_scopes else args.scopes, service_account=service_account) create_boot_disk = not instance_utils.UseExistingBootDisk(args.disk or []) if create_boot_disk: image_expander = image_utils.ImageExpander(client, compute_api.resources) try: image_uri, _ = image_expander.ExpandImageFlag( user_project=instance_template_ref.project, image=args.image, image_family=args.image_family, image_project=args.image_project, return_image_resource=True) except utils.ImageNotFoundError as e: if args.IsSpecified('image_project'): raise e image_uri, _ = image_expander.ExpandImageFlag( user_project=instance_template_ref.project, image=args.image, image_family=args.image_family, image_project=args.image_project, return_image_resource=False) raise utils.ImageNotFoundError( 'The resource [{}] was not found. Is the image located in another ' 'project? Use the --image-project flag to specify the ' 'project where the image is located.'.format(image_uri)) else: image_uri = None if args.tags: tags = client.messages.Tags(items=args.tags) else: tags = None persistent_disks = ( instance_template_utils.CreatePersistentAttachedDiskMessages( client.messages, args.disk or [])) persistent_create_disks = ( instance_template_utils.CreatePersistentCreateDiskMessages( client, compute_api.resources, instance_template_ref.project, getattr(args, 'create_disk', []), support_kms=support_kms)) if create_boot_disk: boot_disk_list = [ instance_template_utils.CreateDefaultBootAttachedDiskMessage( messages=client.messages, disk_type=args.boot_disk_type, disk_device_name=args.boot_disk_device_name, disk_auto_delete=args.boot_disk_auto_delete, disk_size_gb=boot_disk_size_gb, image_uri=image_uri, kms_args=args, support_kms=support_kms) ] else: boot_disk_list = [] local_nvdimms = instance_utils.CreateLocalNvdimmMessages( args, compute_api.resources, client.messages, ) local_ssds = instance_utils.CreateLocalSsdMessages( args, compute_api.resources, client.messages, ) disks = (boot_disk_list + persistent_disks + persistent_create_disks + local_nvdimms + local_ssds) machine_type = instance_utils.InterpretMachineType( machine_type=args.machine_type, custom_cpu=args.custom_cpu, custom_memory=args.custom_memory, ext=getattr(args, 'custom_extensions', None), vm_type=getattr(args, 'custom_vm_type', None)) guest_accelerators = ( instance_template_utils.CreateAcceleratorConfigMessages( client.messages, getattr(args, 'accelerator', None))) instance_template = client.messages.InstanceTemplate( properties=client.messages.InstanceProperties( machineType=machine_type, disks=disks, canIpForward=args.can_ip_forward, metadata=metadata, minCpuPlatform=args.min_cpu_platform, networkInterfaces=network_interfaces, serviceAccounts=service_accounts, scheduling=scheduling, tags=tags, guestAccelerators=guest_accelerators, ), description=args.description, name=instance_template_ref.Name(), ) instance_template.properties.shieldedInstanceConfig = shieldedinstance_config_message instance_template.properties.reservationAffinity = instance_utils.GetReservationAffinity( args, client) request = client.messages.ComputeInstanceTemplatesInsertRequest( instanceTemplate=instance_template, project=instance_template_ref.project) request.instanceTemplate.properties.labels = labels_util.ParseCreateArgs( args, client.messages.InstanceProperties.LabelsValue) _AddSourceInstanceToTemplate(compute_api, args, instance_template, support_source_instance) return client.MakeRequests([(client.apitools_client.instanceTemplates, 'Insert', request)])
def ParseCreateOptionsBase(args): """Parses the flags provided with the cluster creation command.""" if args.IsSpecified('addons') and api_adapter.DASHBOARD in args.addons: log.warning( 'The `KubernetesDashboard` addon is deprecated, and will be removed as ' 'an option for new clusters starting in 1.15. It is recommended to use ' 'the Cloud Console to manage and monitor your Kubernetes clusters, ' 'workloads and applications. See: ' 'https://cloud.google.com/kubernetes-engine/docs/concepts/dashboards' ) flags.MungeBasicAuthFlags(args) if args.IsSpecified('issue_client_certificate') and not ( args.IsSpecified('enable_basic_auth') or args.IsSpecified('username')): log.warning( 'If `--issue-client-certificate` is specified but ' '`--enable-basic-auth` or `--username` is not, our API will ' 'treat that as `--no-enable-basic-auth`.') flags.WarnForUnspecifiedIpAllocationPolicy(args) enable_autorepair = cmd_util.GetAutoRepair(args) flags.WarnForNodeModification(args, enable_autorepair) metadata = metadata_utils.ConstructMetadataDict(args.metadata, args.metadata_from_file) return api_adapter.CreateClusterOptions( accelerators=args.accelerator, additional_zones=args.additional_zones, addons=args.addons, cluster_ipv4_cidr=args.cluster_ipv4_cidr, cluster_secondary_range_name=args.cluster_secondary_range_name, cluster_version=args.cluster_version, node_version=args.node_version, create_subnetwork=args.create_subnetwork, disk_type=args.disk_type, enable_autorepair=enable_autorepair, enable_autoscaling=args.enable_autoscaling, enable_autoupgrade=cmd_util.GetAutoUpgrade(args), enable_binauthz=args.enable_binauthz, enable_stackdriver_kubernetes=args.enable_stackdriver_kubernetes if args.IsSpecified('enable_stackdriver_kubernetes') else None, enable_cloud_logging=args.enable_cloud_logging if args.IsSpecified('enable_cloud_logging') else None, enable_cloud_monitoring=args.enable_cloud_monitoring if args.IsSpecified('enable_cloud_monitoring') else None, enable_ip_alias=args.enable_ip_alias, enable_intra_node_visibility=args.enable_intra_node_visibility, enable_kubernetes_alpha=args.enable_kubernetes_alpha, enable_cloud_run_alpha=args.enable_cloud_run_alpha if args.IsSpecified('enable_cloud_run_alpha') else None, enable_legacy_authorization=args.enable_legacy_authorization, enable_master_authorized_networks=args.enable_master_authorized_networks, enable_network_policy=args.enable_network_policy, enable_private_nodes=args.enable_private_nodes, enable_private_endpoint=args.enable_private_endpoint, image_type=args.image_type, image=args.image, image_project=args.image_project, image_family=args.image_family, issue_client_certificate=args.issue_client_certificate, labels=args.labels, local_ssd_count=args.local_ssd_count, maintenance_window=args.maintenance_window, maintenance_window_start=args.maintenance_window_start, maintenance_window_end=args.maintenance_window_end, maintenance_window_recurrence=args.maintenance_window_recurrence, master_authorized_networks=args.master_authorized_networks, master_ipv4_cidr=args.master_ipv4_cidr, max_nodes=args.max_nodes, max_nodes_per_pool=args.max_nodes_per_pool, min_cpu_platform=args.min_cpu_platform, min_nodes=args.min_nodes, network=args.network, node_disk_size_gb=utils.BytesToGb(args.disk_size), node_labels=args.node_labels, node_locations=args.node_locations, node_machine_type=args.machine_type, node_taints=args.node_taints, num_nodes=args.num_nodes, password=args.password, preemptible=args.preemptible, scopes=args.scopes, service_account=args.service_account, services_ipv4_cidr=args.services_ipv4_cidr, services_secondary_range_name=args.services_secondary_range_name, subnetwork=args.subnetwork, tags=args.tags, user=args.username, metadata=metadata, default_max_pods_per_node=args.default_max_pods_per_node, max_pods_per_node=args.max_pods_per_node, enable_tpu=args.enable_tpu, tpu_ipv4_cidr=args.tpu_ipv4_cidr, resource_usage_bigquery_dataset=args.resource_usage_bigquery_dataset, enable_network_egress_metering=args.enable_network_egress_metering, enable_resource_consumption_metering=\ args.enable_resource_consumption_metering, database_encryption_key=args.database_encryption_key, enable_vertical_pod_autoscaling=args.enable_vertical_pod_autoscaling, enable_autoprovisioning=args.enable_autoprovisioning, autoprovisioning_config_file=args.autoprovisioning_config_file, autoprovisioning_service_account=args.autoprovisioning_service_account, autoprovisioning_scopes=args.autoprovisioning_scopes, autoprovisioning_locations=args.autoprovisioning_locations, autoprovisioning_max_surge_upgrade=getattr(args, 'autoprovisioning_max_surge_upgrade', None), autoprovisioning_max_unavailable_upgrade=getattr(args, 'autoprovisioning_max_unavailable_upgrade', None), enable_autoprovisioning_autorepair=getattr(args, 'enable_autoprovisioning_autorepair', None), enable_autoprovisioning_autoupgrade=getattr(args, 'enable_autoprovisioning_autoupgrade', None), min_cpu=args.min_cpu, max_cpu=args.max_cpu, min_memory=args.min_memory, max_memory=args.max_memory, min_accelerator=args.min_accelerator, max_accelerator=args.max_accelerator, shielded_secure_boot=args.shielded_secure_boot, shielded_integrity_monitoring=args.shielded_integrity_monitoring, reservation_affinity=getattr(args, 'reservation_affinity', None), reservation=getattr(args, 'reservation', None), enable_shielded_nodes=args.enable_shielded_nodes,)
def CreateRequests(self, args): """Creates and returns an InstanceTemplates.Insert request. Args: args: the argparse arguments that this command was invoked with. Returns: request: a ComputeInstanceTemplatesInsertRequest message object """ self.ValidateDiskFlags(args) instances_flags.ValidateLocalSsdFlags(args) instances_flags.ValidateNicFlags(args) instances_flags.ValidateServiceAccountAndScopeArgs(args) boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) instance_template_ref = ( instance_templates_flags.INSTANCE_TEMPLATE_ARG.ResolveAsResource( args, self.resources)) metadata = metadata_utils.ConstructMetadataMessage( self.messages, metadata=args.metadata, metadata_from_file=args.metadata_from_file) if hasattr(args, 'network_interface') and args.network_interface: network_interfaces = ( instance_template_utils.CreateNetworkInterfaceMessages)( resources=self.resources, scope_lister=flags.GetDefaultScopeLister( self.compute_client), messages=self.messages, network_interface_arg=args.network_interface, region=args.region) else: network_interfaces = [ instance_template_utils.CreateNetworkInterfaceMessage( resources=self.resources, scope_lister=flags.GetDefaultScopeLister( self.compute_client), messages=self.messages, network=args.network, region=args.region, subnet=args.subnet, address=(instance_template_utils.EPHEMERAL_ADDRESS if not args.no_address and not args.address else args.address)) ] scheduling = instance_utils.CreateSchedulingMessage( messages=self.messages, maintenance_policy=args.maintenance_policy, preemptible=args.preemptible, restart_on_failure=args.restart_on_failure) if args.no_service_account: service_account = None else: service_account = args.service_account service_accounts = instance_utils.CreateServiceAccountMessages( messages=self.messages, scopes=[] if args.no_scopes else args.scopes, service_account=service_account) create_boot_disk = not instance_utils.UseExistingBootDisk(args.disk or []) if create_boot_disk: image_expander = image_utils.ImageExpander(self.compute_client, self.resources) image_uri, _ = image_expander.ExpandImageFlag( user_project=instance_template_ref.project, image=args.image, image_family=args.image_family, image_project=args.image_project, return_image_resource=True) else: image_uri = None if args.tags: tags = self.messages.Tags(items=args.tags) else: tags = None persistent_disks = ( instance_template_utils.CreatePersistentAttachedDiskMessages( self.messages, args.disk or [])) persistent_create_disks = ( instance_template_utils.CreatePersistentCreateDiskMessages( self, self.messages, getattr(args, 'create_disk', []))) if create_boot_disk: boot_disk_list = [ instance_template_utils.CreateDefaultBootAttachedDiskMessage( messages=self.messages, disk_type=args.boot_disk_type, disk_device_name=args.boot_disk_device_name, disk_auto_delete=args.boot_disk_auto_delete, disk_size_gb=boot_disk_size_gb, image_uri=image_uri) ] else: boot_disk_list = [] local_ssds = [] for x in args.local_ssd or []: local_ssd = instance_utils.CreateLocalSsdMessage( self.resources, self.messages, x.get('device-name'), x.get('interface'), x.get('size')) local_ssds.append(local_ssd) disks = (boot_disk_list + persistent_disks + persistent_create_disks + local_ssds) machine_type = instance_utils.InterpretMachineType( machine_type=args.machine_type, custom_cpu=args.custom_cpu, custom_memory=args.custom_memory, ext=getattr(args, 'custom_extensions', None)) request = self.messages.ComputeInstanceTemplatesInsertRequest( instanceTemplate=self.messages.InstanceTemplate( properties=self.messages.InstanceProperties( machineType=machine_type, disks=disks, canIpForward=args.can_ip_forward, metadata=metadata, networkInterfaces=network_interfaces, serviceAccounts=service_accounts, scheduling=scheduling, tags=tags, ), description=args.description, name=instance_template_ref.Name(), ), project=instance_template_ref.project) return [request]
def Run(self, args): """Issues an InstanceTemplates.Insert request. Args: args: the argparse arguments that this command was invoked with. Returns: an InstanceTemplate message object """ holder = base_classes.ComputeApiHolder(self.ReleaseTrack()) client = holder.client instances_flags.ValidateKonletArgs(args) instances_flags.ValidateDiskCommonFlags(args) instances_flags.ValidateLocalSsdFlags(args) instances_flags.ValidateServiceAccountAndScopeArgs(args) instances_flags.ValidateNetworkTierArgs(args, support_network_tier=True) if instance_utils.UseExistingBootDisk(args.disk or []): raise exceptions.InvalidArgumentException( '--disk', 'Boot disk specified for containerized VM.') boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) instance_template_ref = ( CreateWithContainer.InstanceTemplateArg.ResolveAsResource( args, holder.resources)) user_metadata = metadata_utils.ConstructMetadataMessage( client.messages, metadata=args.metadata, metadata_from_file=args.metadata_from_file) containers_utils.ValidateUserMetadata(user_metadata) network_interface = instance_template_utils.CreateNetworkInterfaceMessage( resources=holder.resources, scope_lister=flags.GetDefaultScopeLister(client), messages=client.messages, network=args.network, region=args.region, subnet=args.subnet, address=(instance_template_utils.EPHEMERAL_ADDRESS if not args.no_address and not args.address else args.address), network_tier=getattr(args, 'network_tier', None)) scheduling = instance_utils.CreateSchedulingMessage( messages=client.messages, maintenance_policy=args.maintenance_policy, preemptible=args.preemptible, restart_on_failure=args.restart_on_failure) if args.no_service_account: service_account = None else: service_account = args.service_account service_accounts = instance_utils.CreateServiceAccountMessages( messages=client.messages, scopes=[] if args.no_scopes else args.scopes, service_account=service_account) if (args.IsSpecified('image') or args.IsSpecified('image_family') or args.IsSpecified('image_project')): image_expander = image_utils.ImageExpander(client, holder.resources) image_uri, _ = image_expander.ExpandImageFlag( user_project=instance_template_ref.project, image=args.image, image_family=args.image_family, image_project=args.image_project) if holder.resources.Parse(image_uri).project != 'cos-cloud': log.warn( 'This container deployment mechanism requires a ' 'Container-Optimized OS image in order to work. Select an ' 'image from a cos-cloud project (cost-stable, cos-beta, ' 'cos-dev image families).') else: image_uri = containers_utils.ExpandKonletCosImageFlag(client) machine_type = instance_utils.InterpretMachineType( machine_type=args.machine_type, custom_cpu=args.custom_cpu, custom_memory=args.custom_memory, ext=getattr(args, 'custom_extensions', None)) metadata = containers_utils.CreateKonletMetadataMessage( client.messages, args, instance_template_ref.Name(), user_metadata) request = client.messages.ComputeInstanceTemplatesInsertRequest( instanceTemplate=client.messages.InstanceTemplate( properties=client.messages.InstanceProperties( machineType=machine_type, disks=self._CreateDiskMessages( holder, args, boot_disk_size_gb, image_uri, instance_template_ref.project), canIpForward=args.can_ip_forward, metadata=metadata, minCpuPlatform=args.min_cpu_platform, networkInterfaces=[network_interface], serviceAccounts=service_accounts, scheduling=scheduling, tags=containers_utils.CreateTagsMessage( client.messages, args.tags), ), description=args.description, name=instance_template_ref.Name(), ), project=instance_template_ref.project) return client.MakeRequests([(client.apitools_client.instanceTemplates, 'Insert', request)])
def ParseCreateOptionsBase(args): """Parses the flags provided with the cluster creation command.""" if not (args.IsSpecified('enable_basic_auth') or args.IsSpecified('username')): log.warning('Starting in 1.12, new clusters will have basic ' 'authentication disabled by default. Basic authentication ' 'can be enabled (or disabled) manually using the ' '`--[no-]enable-basic-auth` flag.') if not args.IsSpecified('issue_client_certificate'): log.warning( 'Starting in 1.12, new clusters will not have a client ' 'certificate issued. You can manually enable (or disable) the ' 'issuance of the client certificate using the ' '`--[no-]issue-client-certificate` flag.') if args.IsSpecified('addons') and api_adapter.DASHBOARD in args.addons: log.warning( 'The `KubernetesDashboard` addon is deprecated, and will be removed as ' 'an option for new clusters starting in 1.15. It is recommended to use ' 'the Cloud Console to manage and monitor your Kubernetes clusters, ' 'workloads and applications. See: ' 'https://cloud.google.com/kubernetes-engine/docs/concepts/dashboards' ) flags.MungeBasicAuthFlags(args) if args.IsSpecified('issue_client_certificate') and not ( args.IsSpecified('enable_basic_auth') or args.IsSpecified('username')): log.warning( 'If `--issue-client-certificate` is specified but ' '`--enable-basic-auth` or `--username` is not, our API will ' 'treat that as `--no-enable-basic-auth`.') enable_autorepair = cmd_util.GetAutoRepair(args) flags.WarnForNodeModification(args, enable_autorepair) metadata = metadata_utils.ConstructMetadataDict(args.metadata, args.metadata_from_file) return api_adapter.CreateClusterOptions( accelerators=args.accelerator, additional_zones=args.additional_zones, addons=args.addons, cluster_ipv4_cidr=args.cluster_ipv4_cidr, cluster_secondary_range_name=args.cluster_secondary_range_name, cluster_version=args.cluster_version, node_version=args.node_version, create_subnetwork=args.create_subnetwork, disk_type=args.disk_type, enable_autorepair=enable_autorepair, enable_autoscaling=args.enable_autoscaling, enable_autoupgrade=cmd_util.GetAutoUpgrade(args), enable_stackdriver_kubernetes=args.enable_stackdriver_kubernetes, enable_cloud_logging=args.enable_cloud_logging, enable_cloud_monitoring=args.enable_cloud_monitoring, enable_ip_alias=args.enable_ip_alias, enable_kubernetes_alpha=args.enable_kubernetes_alpha, enable_legacy_authorization=args.enable_legacy_authorization, enable_main_authorized_networks=args.enable_main_authorized_networks, enable_network_policy=args.enable_network_policy, enable_private_nodes=args.enable_private_nodes, enable_private_endpoint=args.enable_private_endpoint, image_type=args.image_type, image=args.image, image_project=args.image_project, image_family=args.image_family, issue_client_certificate=args.issue_client_certificate, labels=args.labels, local_ssd_count=args.local_ssd_count, maintenance_window=args.maintenance_window, main_authorized_networks=args.main_authorized_networks, main_ipv4_cidr=args.main_ipv4_cidr, max_nodes=args.max_nodes, max_nodes_per_pool=args.max_nodes_per_pool, min_cpu_platform=args.min_cpu_platform, min_nodes=args.min_nodes, network=args.network, node_disk_size_gb=utils.BytesToGb(args.disk_size), node_labels=args.node_labels, node_locations=args.node_locations, node_machine_type=args.machine_type, node_taints=args.node_taints, num_nodes=args.num_nodes, password=args.password, preemptible=args.preemptible, scopes=args.scopes, service_account=args.service_account, services_ipv4_cidr=args.services_ipv4_cidr, services_secondary_range_name=args.services_secondary_range_name, subnetwork=args.subnetwork, tags=args.tags, user=args.username, metadata=metadata, default_max_pods_per_node=args.default_max_pods_per_node, max_pods_per_node=args.max_pods_per_node, enable_tpu=args.enable_tpu, tpu_ipv4_cidr=args.tpu_ipv4_cidr, resource_usage_bigquery_dataset=args.resource_usage_bigquery_dataset, enable_network_egress_metering=args.enable_network_egress_metering, enable_resource_consumption_metering=\ args.enable_resource_consumption_metering)
def CreatePersistentCreateDiskMessages(compute_client, resources, csek_keys, create_disks, project, location, scope, holder, enable_kms=False, enable_snapshots=False, container_mount_disk=None, resource_policy=False, enable_source_snapshot_csek=False, enable_image_csek=False, support_replica_zones=False, use_disk_type_uri=True, support_multi_writer=False): """Returns a list of AttachedDisk messages for newly creating disks. Args: compute_client: creates resources, resources: parser of resources, csek_keys: customer suplied encryption keys, create_disks: disk objects - contains following properties * name - the name of disk, * description - an optional description for the disk, * mode - 'rw' (R/W), 'ro' (R/O) access mode, * disk-size - the size of the disk, * disk-type - the type of the disk (HDD or SSD), * image - the name of the image to initialize from, * image-csek-required - the name of the CSK protected image, * image-family - the image family name, * image-project - the project name that has the image, * auto-delete - whether disks is deleted when VM is deleted, * device-name - device name on VM, * source-snapshot - the snapshot to initialize from, * source-snapshot-csek-required - CSK protected snapshot, * disk-resource-policy - resource policies applied to disk. * enable_source_snapshot_csek - CSK file for snapshot, * enable_image_csek - CSK file for image project: Project of instance that will own the new disks. location: Location of the instance that will own the new disks. scope: Location type of the instance that will own the new disks. holder: Convenience class to hold lazy initialized client and resources. enable_kms: True if KMS keys are supported for the disk. enable_snapshots: True if snapshot initialization is supported for the disk. container_mount_disk: list of disks to be mounted to container, if any. resource_policy: True if resource-policies are enabled enable_source_snapshot_csek: True if snapshot CSK files are enabled enable_image_csek: True if image CSK files are enabled support_replica_zones: True if we allow creation of regional disks use_disk_type_uri: True to use disk type URI, False if naked type. support_multi_writer: True if we allow multiple instances to write to disk. Returns: list of API messages for attached disks """ disks_messages = [] messages = compute_client.messages compute = compute_client.apitools_client for disk in create_disks or []: name = disk.get('name') # Resolves the mode. mode_value = disk.get('mode', 'rw') if mode_value == 'rw': mode = messages.AttachedDisk.ModeValueValuesEnum.READ_WRITE else: mode = messages.AttachedDisk.ModeValueValuesEnum.READ_ONLY auto_delete_value = disk.get('auto-delete', 'yes') auto_delete = auto_delete_value == 'yes' disk_size_gb = utils.BytesToGb(disk.get('size')) disk_type = disk.get('type') if disk_type: if use_disk_type_uri: disk_type_ref = instance_utils.ParseDiskType( resources, disk_type, project, location, scope) disk_type = disk_type_ref.SelfLink() else: disk_type = None img = disk.get('image') img_family = disk.get('image-family') img_project = disk.get('image-project') image_uri = None if img or img_family: image_expander = image_utils.ImageExpander(compute_client, resources) image_uri, _ = image_expander.ExpandImageFlag( user_project=project, image=img, image_family=img_family, image_project=img_project, return_image_resource=False) image_key = None disk_key = None if csek_keys: image_key = csek_utils.MaybeLookupKeyMessagesByUri( csek_keys, resources, [image_uri], compute) if name: disk_ref = resources.Parse(name, collection='compute.disks', params={'zone': location}) disk_key = csek_utils.MaybeLookupKeyMessage( csek_keys, disk_ref, compute) if enable_kms: disk_key = kms_utils.MaybeGetKmsKeyFromDict( disk, messages, disk_key) initialize_params = messages.AttachedDiskInitializeParams( diskName=name, description=disk.get('description'), sourceImage=image_uri, diskSizeGb=disk_size_gb, diskType=disk_type, sourceImageEncryptionKey=image_key) replica_zones = disk.get('replica-zones') if support_replica_zones and replica_zones: normalized_zones = [] for zone in replica_zones: zone_ref = holder.resources.Parse(zone, collection='compute.zones', params={'project': project}) normalized_zones.append(zone_ref.SelfLink()) initialize_params.replicaZones = normalized_zones if enable_snapshots: snapshot_name = disk.get('source-snapshot') attached_snapshot_uri = instance_utils.ResolveSnapshotURI( snapshot=snapshot_name, user_project=project, resource_parser=resources) if attached_snapshot_uri: initialize_params.sourceImage = None initialize_params.sourceSnapshot = attached_snapshot_uri if resource_policy: policies = disk.get('disk-resource-policy') if policies: initialize_params.resourcePolicies = policies if enable_image_csek: image_key_file = disk.get('image_csek') if image_key_file: initialize_params.imageKeyFile = image_key_file if enable_source_snapshot_csek: snapshot_key_file = disk.get('source_snapshot_csek') if snapshot_key_file: initialize_params.snapshotKeyFile = snapshot_key_file boot = disk.get('boot') == 'yes' multi_writer = disk.get('multi-writer') if support_multi_writer and multi_writer: initialize_params.multiWriter = True device_name = instance_utils.GetDiskDeviceName(disk, name, container_mount_disk) create_disk = messages.AttachedDisk( autoDelete=auto_delete, boot=boot, deviceName=device_name, initializeParams=initialize_params, mode=mode, type=messages.AttachedDisk.TypeValueValuesEnum.PERSISTENT, diskEncryptionKey=disk_key) disks_messages.append(create_disk) return disks_messages
def Run(self, args): """Creates and runs an InstanceTemplates.Insert request. Args: args: argparse.Namespace, An object that contains the values for the arguments specified in the .Args() method. Returns: A resource object dispatched by display.Displayer(). """ holder = base_classes.ComputeApiHolder(self.ReleaseTrack()) client = holder.client self.ValidateDiskFlags(args) instances_flags.ValidateLocalSsdFlags(args) instances_flags.ValidateNicFlags(args) instances_flags.ValidateServiceAccountAndScopeArgs(args) instances_flags.ValidateAcceleratorArgs(args) boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) instance_template_ref = ( Create.InstanceTemplateArg.ResolveAsResource( args, holder.resources)) metadata = metadata_utils.ConstructMetadataMessage( client.messages, metadata=args.metadata, metadata_from_file=args.metadata_from_file) if hasattr(args, 'network_interface') and args.network_interface: network_interfaces = ( instance_template_utils.CreateNetworkInterfaceMessages)( resources=holder.resources, scope_lister=flags.GetDefaultScopeLister(client), messages=client.messages, network_interface_arg=args.network_interface, region=args.region, support_network_tier=self._support_network_tier) else: network_tier = getattr(args, 'network_tier', None) network_interfaces = [ instance_template_utils.CreateNetworkInterfaceMessage( resources=holder.resources, scope_lister=flags.GetDefaultScopeLister(client), messages=client.messages, network=args.network, region=args.region, subnet=args.subnet, address=(instance_template_utils.EPHEMERAL_ADDRESS if not args.no_address and not args.address else args.address), network_tier=network_tier) ] scheduling = instance_utils.CreateSchedulingMessage( messages=client.messages, maintenance_policy=args.maintenance_policy, preemptible=args.preemptible, restart_on_failure=args.restart_on_failure) if args.no_service_account: service_account = None else: service_account = args.service_account service_accounts = instance_utils.CreateServiceAccountMessages( messages=client.messages, scopes=[] if args.no_scopes else args.scopes, service_account=service_account) create_boot_disk = not instance_utils.UseExistingBootDisk(args.disk or []) if create_boot_disk: image_expander = image_utils.ImageExpander(client, holder.resources) image_uri, _ = image_expander.ExpandImageFlag( user_project=instance_template_ref.project, image=args.image, image_family=args.image_family, image_project=args.image_project, return_image_resource=True) else: image_uri = None if args.tags: tags = client.messages.Tags(items=args.tags) else: tags = None persistent_disks = ( instance_template_utils.CreatePersistentAttachedDiskMessages( client.messages, args.disk or [])) persistent_create_disks = ( instance_template_utils.CreatePersistentCreateDiskMessages( client, holder.resources, instance_template_ref.project, getattr(args, 'create_disk', []))) if create_boot_disk: boot_disk_list = [ instance_template_utils.CreateDefaultBootAttachedDiskMessage( messages=client.messages, disk_type=args.boot_disk_type, disk_device_name=args.boot_disk_device_name, disk_auto_delete=args.boot_disk_auto_delete, disk_size_gb=boot_disk_size_gb, image_uri=image_uri)] else: boot_disk_list = [] local_ssds = [] for x in args.local_ssd or []: local_ssd = instance_utils.CreateLocalSsdMessage( holder.resources, client.messages, x.get('device-name'), x.get('interface'), x.get('size')) local_ssds.append(local_ssd) disks = ( boot_disk_list + persistent_disks + persistent_create_disks + local_ssds ) machine_type = instance_utils.InterpretMachineType( machine_type=args.machine_type, custom_cpu=args.custom_cpu, custom_memory=args.custom_memory, ext=getattr(args, 'custom_extensions', None)) guest_accelerators = ( instance_template_utils.CreateAcceleratorConfigMessages( client.messages, getattr(args, 'accelerator', None))) instance_properties = client.messages.InstanceProperties( machineType=machine_type, disks=disks, canIpForward=args.can_ip_forward, metadata=metadata, networkInterfaces=network_interfaces, serviceAccounts=service_accounts, scheduling=scheduling, tags=tags, ) # TODO(b/36890961): Pass this directly into guestAccelerators once GA. if guest_accelerators: instance_properties.guestAccelerators = guest_accelerators request = client.messages.ComputeInstanceTemplatesInsertRequest( instanceTemplate=client.messages.InstanceTemplate( properties=instance_properties, description=args.description, name=instance_template_ref.Name(), ), project=instance_template_ref.project) if getattr(args, 'min_cpu_platform', None): request.instanceTemplate.properties.minCpuPlatform = args.min_cpu_platform return client.MakeRequests([(client.apitools_client.instanceTemplates, 'Insert', request)])
def Run(self, args): self.ValidateArgs(args) client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) config_helper = compute_helpers.ConfigurationHelper.FromContext( self.context) compute_uris = config_helper.ResolveGceUris(args.name, args.image, args.master_machine_type, args.worker_machine_type, args.network, args.subnet) init_actions = [] timeout_str = str(args.initialization_action_timeout) + 's' if args.initialization_actions: init_actions = [ messages.NodeInitializationAction(executableFile=exe, executionTimeout=timeout_str) for exe in args.initialization_actions ] expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes) software_config = messages.SoftwareConfig( imageVersion=args.image_version) master_boot_disk_size_gb = args.master_boot_disk_size_gb if args.master_boot_disk_size: master_boot_disk_size_gb = (api_utils.BytesToGb( args.master_boot_disk_size)) worker_boot_disk_size_gb = args.worker_boot_disk_size_gb if args.worker_boot_disk_size: worker_boot_disk_size_gb = (api_utils.BytesToGb( args.worker_boot_disk_size)) preemptible_worker_boot_disk_size_gb = (api_utils.BytesToGb( args.preemptible_worker_boot_disk_size)) if args.properties: software_config.properties = encoding.DictToMessage( args.properties, messages.SoftwareConfig.PropertiesValue) gce_cluster_config = messages.GceClusterConfig( networkUri=compute_uris['network'], subnetworkUri=compute_uris['subnetwork'], serviceAccountScopes=expanded_scopes, zoneUri=compute_uris['zone']) if args.tags: gce_cluster_config.tags = args.tags if args.metadata: flat_metadata = dict( (k, v) for d in args.metadata for k, v in d.items()) gce_cluster_config.metadata = encoding.DictToMessage( flat_metadata, messages.GceClusterConfig.MetadataValue) cluster_config = messages.ClusterConfig( configBucket=args.bucket, gceClusterConfig=gce_cluster_config, masterConfig=messages.InstanceGroupConfig( imageUri=compute_uris['image'], machineTypeUri=compute_uris['master_machine_type'], diskConfig=messages.DiskConfig( bootDiskSizeGb=master_boot_disk_size_gb, numLocalSsds=args.num_master_local_ssds, ), ), workerConfig=messages.InstanceGroupConfig( numInstances=args.num_workers, imageUri=compute_uris['image'], machineTypeUri=compute_uris['worker_machine_type'], diskConfig=messages.DiskConfig( bootDiskSizeGb=worker_boot_disk_size_gb, numLocalSsds=args.num_worker_local_ssds, ), ), initializationActions=init_actions, softwareConfig=software_config, ) # Secondary worker group is optional. if args.num_preemptible_workers is not None: cluster_config.secondaryWorkerConfig = ( messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers, diskConfig=messages.DiskConfig( bootDiskSizeGb=preemptible_worker_boot_disk_size_gb, )) ) cluster = messages.Cluster(config=cluster_config, clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) operation = client.projects_regions_clusters.Create( messages.DataprocProjectsRegionsClustersCreateRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, cluster=cluster)) if args. async: log.status.write('Creating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation = util.WaitForOperation( operation, self.context, 'Waiting for cluster creation operation') cluster = client.projects_regions_clusters.Get(cluster_ref.Request()) if cluster.status.state == ( messages.ClusterStatus.StateValueValuesEnum.RUNNING): log.CreatedResource(cluster_ref) else: log.error('Create cluster failed!') if operation.details: log.error('Details:\n' + operation.details) return cluster
def ParseCreateOptionsBase(args, is_autopilot, get_default, location, project_id): """Parses the flags provided with the cluster creation command.""" if hasattr(args, 'addons') and args.IsSpecified('addons') and \ api_adapter.DASHBOARD in args.addons: log.warning( 'The `KubernetesDashboard` addon is deprecated, and will be removed as ' 'an option for new clusters starting in 1.15. It is recommended to use ' 'the Cloud Console to manage and monitor your Kubernetes clusters, ' 'workloads and applications. See: ' 'https://cloud.google.com/kubernetes-engine/docs/concepts/dashboards' ) flags.LogBasicAuthDeprecationWarning(args) flags.MungeBasicAuthFlags(args) MaybeLogAuthWarning(args) MaybeLogReleaseChannelDefaultWarning(args) enable_ip_alias = get_default('enable_ip_alias') if hasattr(args, 'enable_ip_alias'): flags.WarnForUnspecifiedIpAllocationPolicy(args) enable_autorepair = None if hasattr(args, 'enable_autorepair'): enable_autorepair = cmd_util.GetAutoRepair(args) if enable_autorepair: flags.WarnForNodeModification(args, enable_autorepair) metadata = metadata_utils.ConstructMetadataDict( get_default('metadata'), get_default('metadata_from_file')) cloud_run_config = flags.GetLegacyCloudRunFlag('{}_config', args, get_default) flags.ValidateCloudRunConfigCreateArgs(cloud_run_config, get_default('addons')) MaybeLogCloudNatHelpText(args, is_autopilot, location, project_id) flags.ValidateNotificationConfigFlag(args) return api_adapter.CreateClusterOptions( accelerators=get_default('accelerator'), additional_zones=get_default('additional_zones'), addons=get_default('addons'), boot_disk_kms_key=get_default('boot_disk_kms_key'), cluster_ipv4_cidr=get_default('cluster_ipv4_cidr'), cluster_secondary_range_name=get_default('cluster_secondary_range_name'), cluster_version=get_default('cluster_version'), cloud_run_config=cloud_run_config, node_version=get_default('node_version'), create_subnetwork=get_default('create_subnetwork'), disable_default_snat=get_default('disable_default_snat'), disk_type=get_default('disk_type'), enable_autorepair=enable_autorepair, enable_autoscaling=get_default('enable_autoscaling'), enable_autoupgrade=(cmd_util.GetAutoUpgrade(args) if hasattr(args, 'enable_autoupgrade') else None), enable_binauthz=get_default('enable_binauthz'), enable_stackdriver_kubernetes=_GetEnableStackdriver(args), enable_cloud_logging=args.enable_cloud_logging if (hasattr(args, 'enable_cloud_logging') and args.IsSpecified('enable_cloud_logging')) else None, enable_cloud_monitoring=args.enable_cloud_monitoring if (hasattr(args, 'enable_cloud_monitoring') and args.IsSpecified('enable_cloud_monitoring')) else None, enable_workload_monitoring_eap=get_default('enable_workload_monitoring_eap'), enable_ip_alias=enable_ip_alias, enable_intra_node_visibility=get_default('enable_intra_node_visibility'), enable_kubernetes_alpha=get_default('enable_kubernetes_alpha'), enable_cloud_run_alpha=flags.GetLegacyCloudRunFlag('enable_{}_alpha', args, get_default), enable_legacy_authorization=get_default('enable_legacy_authorization'), enable_master_authorized_networks=\ get_default('enable_master_authorized_networks'), enable_master_global_access=get_default('enable_master_global_access'), enable_network_policy=get_default('enable_network_policy'), enable_private_nodes=get_default('enable_private_nodes'), enable_private_endpoint=get_default('enable_private_endpoint'), enable_gke_oidc=getattr(args, 'enable_gke_oidc', None), image_type=get_default('image_type'), image=get_default('image'), image_project=get_default('image_project'), image_family=get_default('image_family'), issue_client_certificate=get_default('issue_client_certificate'), labels=get_default('labels'), local_ssd_count=get_default('local_ssd_count'), maintenance_window=get_default('maintenance_window'), maintenance_window_start=get_default('maintenance_window_start'), maintenance_window_end=get_default('maintenance_window_end'), maintenance_window_recurrence=get_default('maintenance_window_recurrence'), master_authorized_networks=get_default('master_authorized_networks'), master_ipv4_cidr=get_default('master_ipv4_cidr'), max_nodes=get_default('max_nodes'), max_nodes_per_pool=get_default('max_nodes_per_pool'), min_cpu_platform=get_default('min_cpu_platform'), min_nodes=get_default('min_nodes'), network=get_default('network'), node_disk_size_gb=utils.BytesToGb(args.disk_size) if hasattr(args, 'disk_size') else None, node_labels=get_default('node_labels'), node_locations=get_default('node_locations'), node_machine_type=get_default('machine_type'), node_taints=get_default('node_taints'), num_nodes=get_default('num_nodes'), password=get_default('password'), preemptible=get_default('preemptible'), scopes=get_default('scopes'), service_account=get_default('service_account'), services_ipv4_cidr=get_default('services_ipv4_cidr'), services_secondary_range_name=get_default('services_secondary_range_name'), subnetwork=get_default('subnetwork'), system_config_from_file=get_default('system_config_from_file'), private_ipv6_google_access_type=get_default('private_ipv6_google_access_type'), tags=get_default('tags'), user=get_default('username'), metadata=metadata, default_max_pods_per_node=get_default('default_max_pods_per_node'), max_pods_per_node=get_default('max_pods_per_node'), enable_tpu=get_default('enable_tpu'), tpu_ipv4_cidr=get_default('tpu_ipv4_cidr'), resource_usage_bigquery_dataset=get_default('resource_usage_bigquery_dataset'), enable_network_egress_metering=get_default('enable_network_egress_metering'), enable_resource_consumption_metering=get_default('enable_resource_consumption_metering'), database_encryption_key=get_default('database_encryption_key'), workload_pool=get_default('workload_pool'), identity_provider=get_default('identity_provider'), workload_metadata=get_default('workload_metadata'), workload_metadata_from_node=get_default('workload_metadata_from_node'), enable_vertical_pod_autoscaling=get_default('enable_vertical_pod_autoscaling'), enable_autoprovisioning=get_default('enable_autoprovisioning'), autoprovisioning_config_file=get_default('autoprovisioning_config_file'), autoprovisioning_service_account=get_default('autoprovisioning_service_account'), autoprovisioning_scopes=get_default('autoprovisioning_scopes'), autoprovisioning_locations=get_default('autoprovisioning_locations'), autoprovisioning_max_surge_upgrade=get_default('autoprovisioning_max_surge_upgrade'), autoprovisioning_max_unavailable_upgrade=get_default('autoprovisioning_max_unavailable_upgrade'), enable_autoprovisioning_autorepair=get_default('enable_autoprovisioning_autorepair'), enable_autoprovisioning_autoupgrade=get_default('enable_autoprovisioning_autoupgrade'), autoprovisioning_min_cpu_platform=get_default('autoprovisioning_min_cpu_platform'), min_cpu=get_default('min_cpu'), max_cpu=get_default('max_cpu'), min_memory=get_default('min_memory'), max_memory=get_default('max_memory'), min_accelerator=get_default('min_accelerator'), max_accelerator=get_default('max_accelerator'), shielded_secure_boot=get_default('shielded_secure_boot'), shielded_integrity_monitoring=get_default('shielded_integrity_monitoring'), reservation_affinity=get_default('reservation_affinity'), reservation=get_default('reservation'), release_channel=get_default('release_channel'), enable_shielded_nodes=get_default('enable_shielded_nodes'), max_surge_upgrade=get_default('max_surge_upgrade'), max_unavailable_upgrade=get_default('max_unavailable_upgrade'), autopilot=is_autopilot)
def CreateRequests(self, args): instances_flags.ValidateDockerArgs(args) instances_flags.ValidateDiskCommonFlags(args) instances_flags.ValidateLocalSsdFlags(args) instances_flags.ValidateServiceAccountAndScopeArgs(args) if instance_utils.UseExistingBootDisk(args.disk or []): raise exceptions.InvalidArgumentException( '--disk', 'Boot disk specified for containerized VM.') scheduling = instance_utils.CreateSchedulingMessage( messages=self.messages, maintenance_policy=args.maintenance_policy, preemptible=args.preemptible, restart_on_failure=args.restart_on_failure) if args.no_service_account: service_account = None else: service_account = args.service_account service_accounts = instance_utils.CreateServiceAccountMessages( messages=self.messages, scopes=[] if args.no_scopes else args.scopes, service_account=service_account) user_metadata = metadata_utils.ConstructMetadataMessage( self.messages, metadata=args.metadata, metadata_from_file=args.metadata_from_file) containers_utils.ValidateUserMetadata(user_metadata) boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) instance_refs = instances_flags.INSTANCES_ARG.ResolveAsResource( args, self.resources, scope_lister=flags.GetDefaultScopeLister(self.compute_client, self.project)) # Check if the zone is deprecated or has maintenance coming. zone_resource_fetcher = zone_utils.ZoneResourceFetcher( self.compute_client) zone_resource_fetcher.WarnForZonalCreation(instance_refs) instances_flags.ValidatePublicDnsFlags(args) network_interface = instance_utils.CreateNetworkInterfaceMessage( resources=self.resources, compute_client=self.compute_client, network=args.network, subnet=args.subnet, private_network_ip=args.private_network_ip, no_address=args.no_address, address=args.address, instance_refs=instance_refs, network_tier=args.network_tier, no_public_dns=getattr(args, 'no_public_dns', None), public_dns=getattr(args, 'public_dns', None), no_public_ptr=getattr(args, 'no_public_ptr', None), public_ptr=getattr(args, 'public_ptr', None), no_public_ptr_domain=getattr(args, 'no_public_ptr_domain', None), public_ptr_domain=getattr(args, 'public_ptr_domain', None)) machine_type_uris = instance_utils.CreateMachineTypeUris( resources=self.resources, compute_client=self.compute_client, project=self.project, machine_type=args.machine_type, custom_cpu=args.custom_cpu, custom_memory=args.custom_memory, ext=getattr(args, 'custom_extensions', None), instance_refs=instance_refs) image_uri = containers_utils.ExpandCosImageFlag(self.compute_client) args_labels = getattr(args, 'labels', None) labels = None if args_labels: labels = self.messages.Instance.LabelsValue(additionalProperties=[ self.messages.Instance.LabelsValue.AdditionalProperty( key=key, value=value) for key, value in sorted(args.labels.iteritems()) ]) requests = [] for instance_ref, machine_type_uri in zip(instance_refs, machine_type_uris): metadata = containers_utils.CreateMetadataMessage( self.messages, args.run_as_privileged, args.container_manifest, args.docker_image, args.port_mappings, args.run_command, user_metadata, instance_ref.Name()) request = self.messages.ComputeInstancesInsertRequest( instance=self.messages.Instance( canIpForward=args.can_ip_forward, disks=(self._CreateDiskMessages(args, boot_disk_size_gb, image_uri, instance_ref)), description=args.description, machineType=machine_type_uri, metadata=metadata, minCpuPlatform=args.min_cpu_platform, name=instance_ref.Name(), networkInterfaces=[network_interface], serviceAccounts=service_accounts, scheduling=scheduling, tags=containers_utils.CreateTagsMessage( self.messages, args.tags), ), project=self.project, zone=instance_ref.zone) if labels: request.instance.labels = labels requests.append(request) return requests
def CreatePersistentCreateDiskMessages(compute_client, resources, csek_keys, create_disks, instance_ref, enable_kms=False, enable_snapshots=False, container_mount_disk=None, resource_policy=False): """Returns a list of AttachedDisk messages for newly creating disks. Args: compute_client: creates resources, resources: parser of resources, csek_keys: customer suplied encryption keys, create_disks: disk objects - contains following properties * name - the name of disk, * description - an optional description for the disk, * mode - 'rw' (R/W), 'ro' (R/O) access mode, * disk-size - the size of the disk, * disk-type - the type of the disk (HDD or SSD), * image - the name of the image to initialize from, * image-family - the image family name, * image-project - the project name that has the image, * auto-delete - whether disks is deleted when VM is deleted, * device-name - device name on VM, * source-snapshot - the snapshot to initialize from, * disk-resource-policy - resource policies applied to disk. instance_ref: reference to the instance that will own the new disks. enable_kms: True if KMS keys are supported for the disk. enable_snapshots: True if snapshot initialization is supported for the disk. container_mount_disk: list of disks to be mounted to container, if any. resource_policy: True if resource-policies are enabled Returns: list of API messages for attached disks """ disks_messages = [] messages = compute_client.messages compute = compute_client.apitools_client for disk in create_disks or []: name = disk.get('name') # Resolves the mode. mode_value = disk.get('mode', 'rw') if mode_value == 'rw': mode = messages.AttachedDisk.ModeValueValuesEnum.READ_WRITE else: mode = messages.AttachedDisk.ModeValueValuesEnum.READ_ONLY auto_delete_value = disk.get('auto-delete', 'yes') auto_delete = auto_delete_value == 'yes' disk_size_gb = utils.BytesToGb(disk.get('size')) disk_type = disk.get('type') if disk_type: disk_type_ref = resources.Parse(disk_type, collection='compute.diskTypes', params={ 'project': instance_ref.project, 'zone': instance_ref.zone }) disk_type_uri = disk_type_ref.SelfLink() else: disk_type_uri = None img = disk.get('image') img_family = disk.get('image-family') img_project = disk.get('image-project') image_uri = None if img or img_family: image_expander = image_utils.ImageExpander(compute_client, resources) image_uri, _ = image_expander.ExpandImageFlag( user_project=instance_ref.project, image=img, image_family=img_family, image_project=img_project, return_image_resource=False) image_key = None disk_key = None if csek_keys: image_key = csek_utils.MaybeLookupKeyMessagesByUri( csek_keys, resources, [image_uri], compute) if name: disk_ref = resources.Parse(name, collection='compute.disks', params={'zone': instance_ref.zone}) disk_key = csek_utils.MaybeLookupKeyMessage( csek_keys, disk_ref, compute) if enable_kms: disk_key = kms_utils.MaybeGetKmsKeyFromDict( disk, messages, disk_key) initialize_params = messages.AttachedDiskInitializeParams( diskName=name, description=disk.get('description'), sourceImage=image_uri, diskSizeGb=disk_size_gb, diskType=disk_type_uri, sourceImageEncryptionKey=image_key) if enable_snapshots: snapshot_name = disk.get('source-snapshot') attached_snapshot_uri = ResolveSnapshotURI( snapshot=snapshot_name, user_project=instance_ref.project, resource_parser=resources) if attached_snapshot_uri: initialize_params.sourceImage = None initialize_params.sourceSnapshot = attached_snapshot_uri if resource_policy: policies = disk.get('disk-resource-policy') if policies: initialize_params.resourcePolicies = policies device_name = GetDiskDeviceName(disk, name, container_mount_disk) create_disk = messages.AttachedDisk( autoDelete=auto_delete, boot=False, deviceName=device_name, initializeParams=initialize_params, mode=mode, type=messages.AttachedDisk.TypeValueValuesEnum.PERSISTENT, diskEncryptionKey=disk_key) disks_messages.append(create_disk) return disks_messages
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ client = cloudbuild_util.GetClientInstanceAlpha() messages = cloudbuild_util.GetMessagesModuleAlpha() parent = properties.VALUES.core.project.Get(required=True) # Get the workerpool proto from either the flags or the specified file. wp = messages.WorkerPool() if args.config_from_file is not None: wp = workerpool_config.LoadWorkerpoolConfigFromPath( args.config_from_file, messages) else: wp.name = args.WORKER_POOL if args.worker_count is not None: try: wp.workerCount = int(args.worker_count) except ValueError as e: raise c_exceptions.InvalidArgumentException('--worker-count', e) if args.regions is not None: for region_str in args.regions: region = Create._region_choice_to_enum[region_str] wp.regions.append(region) worker_config = messages.WorkerConfig() if args.worker_machine_type is not None: worker_config.machineType = args.worker_machine_type if args.worker_disk_size is not None: worker_config.diskSizeGb = compute_utils.BytesToGb( args.worker_disk_size) if any([ args.worker_network_project is not None, args.worker_network_name is not None, args.worker_network_subnet is not None ]): if not all([ args.worker_network_project is not None, args.worker_network_name is not None, args.worker_network_subnet is not None ]): raise c_exceptions.RequiredArgumentException( '--worker_network_*', 'The flags --worker_network_project, --worker_network_name, and ' '--worker_network_subnet must all be set if any of them are set.') # At this point all network flags are set, but possibly empty string. # The API handles default values. network = messages.Network() network.projectId = args.worker_network_project network.network = args.worker_network_name network.subnetwork = args.worker_network_subnet worker_config.network = network if args.worker_tag is not None: worker_config.tag = args.worker_tag wp.workerConfig = worker_config # Get the parent project ref parent_resource = resources.REGISTRY.Create( collection='cloudbuild.projects', projectId=parent) # Send the Create request created_wp = client.projects_workerPools.Create( messages.CloudbuildProjectsWorkerPoolsCreateRequest( workerPool=wp, parent=parent_resource.RelativeName())) # Get the workerpool ref wp_resource = resources.REGISTRY.Parse( None, collection='cloudbuild.projects.workerPools', api_version='v1alpha1', params={ 'projectsId': parent, 'workerPoolsId': wp.name, }) log.CreatedResource(wp_resource) return created_wp
def ParseCreateOptionsBase(args): """Parses the flags provided with the cluster creation command.""" if not (args.IsSpecified('enable_basic_auth') or args.IsSpecified('username')): log.warning('Starting in 1.12, new clusters will have basic ' 'authentication disabled by default. Basic authentication ' 'can be enabled (or disabled) manually using the ' '`--[no-]enable-basic-auth` flag.') if not args.IsSpecified('issue_client_certificate'): log.warning( 'Starting in 1.12, new clusters will not have a client ' 'certificate issued. You can manually enable (or disable) the ' 'issuance of the client certificate using the ' '`--[no-]issue-client-certificate` flag.') flags.MungeBasicAuthFlags(args) if args.IsSpecified('issue_client_certificate') and not ( args.IsSpecified('enable_basic_auth') or args.IsSpecified('username')): log.warning( 'If `--issue-client-certificate` is specified but ' '`--enable-basic-auth` or `--username` is not, our API will ' 'treat that as `--no-enable-basic-auth`.') flags.WarnForUnspecifiedIpAllocationPolicy(args) enable_autorepair = cmd_util.GetAutoRepair(args) flags.WarnForNodeModification(args, enable_autorepair) metadata = metadata_utils.ConstructMetadataDict(args.metadata, args.metadata_from_file) return api_adapter.CreateClusterOptions( accelerators=args.accelerator, additional_zones=args.additional_zones, addons=args.addons, cluster_ipv4_cidr=args.cluster_ipv4_cidr, cluster_secondary_range_name=args.cluster_secondary_range_name, cluster_version=args.cluster_version, node_version=args.node_version, create_subnetwork=args.create_subnetwork, disk_type=args.disk_type, enable_autorepair=enable_autorepair, enable_autoscaling=args.enable_autoscaling, enable_autoupgrade=cmd_util.GetAutoUpgrade(args), enable_cloud_logging=args.enable_cloud_logging, enable_cloud_monitoring=args.enable_cloud_monitoring, enable_ip_alias=args.enable_ip_alias, enable_kubernetes_alpha=args.enable_kubernetes_alpha, enable_legacy_authorization=args.enable_legacy_authorization, enable_master_authorized_networks=args. enable_master_authorized_networks, enable_network_policy=args.enable_network_policy, enable_private_nodes=args.enable_private_nodes, enable_private_endpoint=args.enable_private_endpoint, image_type=args.image_type, image=args.image, image_project=args.image_project, image_family=args.image_family, issue_client_certificate=args.issue_client_certificate, labels=args.labels, local_ssd_count=args.local_ssd_count, maintenance_window=args.maintenance_window, master_authorized_networks=args.master_authorized_networks, master_ipv4_cidr=args.master_ipv4_cidr, max_nodes=args.max_nodes, max_nodes_per_pool=args.max_nodes_per_pool, min_cpu_platform=args.min_cpu_platform, min_nodes=args.min_nodes, network=args.network, node_disk_size_gb=utils.BytesToGb(args.disk_size), node_labels=args.node_labels, node_locations=args.node_locations, node_machine_type=args.machine_type, node_taints=args.node_taints, num_nodes=args.num_nodes, password=args.password, preemptible=args.preemptible, scopes=args.scopes, service_account=args.service_account, services_ipv4_cidr=args.services_ipv4_cidr, services_secondary_range_name=args.services_secondary_range_name, subnetwork=args.subnetwork, tags=args.tags, user=args.username, metadata=metadata, default_max_pods_per_node=args.default_max_pods_per_node, max_pods_per_node=args.max_pods_per_node, enable_tpu=args.enable_tpu, tpu_ipv4_cidr=args.tpu_ipv4_cidr)
def CreateRequests(self, args): _ValidateDiskFlags(args) instance_utils.ValidateLocalSsdFlags(args) # TODO(user) drop test after CSEK goes GA if hasattr(args, 'csek_key_file'): self.csek_keys = csek_utils.CsekKeyStore.FromArgs(args) else: self.csek_keys = None if args.maintenance_policy: on_host_maintenance = ( self.messages.Scheduling.OnHostMaintenanceValueValuesEnum( args.maintenance_policy)) else: on_host_maintenance = None # Note: We always specify automaticRestart=False for preemptible VMs. This # makes sense, since no-restart-on-failure is defined as "store-true", and # thus can't be given an explicit value. Hence it either has its default # value (in which case we override it for convenience's sake to the only # setting that makes sense for preemptible VMs), or the user actually # specified no-restart-on-failure, the only usable setting. if args.preemptible: scheduling = self.messages.Scheduling( automaticRestart=False, onHostMaintenance=on_host_maintenance, preemptible=True) else: scheduling = self.messages.Scheduling( automaticRestart=args.restart_on_failure, onHostMaintenance=on_host_maintenance) service_accounts = self.CreateServiceAccountMessages(args) if args.tags: tags = self.messages.Tags(items=args.tags) else: tags = None metadata = metadata_utils.ConstructMetadataMessage( self.messages, metadata=args.metadata, metadata_from_file=args.metadata_from_file) # If the user already provided an initial Windows password and # username through metadata, then there is no need to check # whether the image or the boot disk is Windows. boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) instance_refs = self.CreateZonalReferences(args.names, args.zone) # Check if the zone is deprecated or has maintenance coming. self.WarnForZonalCreation(instance_refs) network_interface = self.CreateNetworkInterfaceMessage( args, instance_refs) # The element at index i is the machine type URI for instance # i. We build this list here because we want to delay work that # requires API calls as much as possible. This leads to a better # user experience because the tool can fail fast upon a spelling # mistake instead of delaying the user by making API calls whose # purpose has already been rendered moot by the spelling mistake. machine_type_uris = [] # Setting the machine type machine_type_name = instance_utils.InterpretMachineType(args) for instance_ref in instance_refs: # Check to see if the custom machine type ratio is supported instance_utils.CheckCustomCpuRamRatio(self, instance_ref.zone, machine_type_name) machine_type_uris.append( self.CreateZonalReference( machine_type_name, instance_ref.zone, resource_type='machineTypes').SelfLink()) create_boot_disk = not _UseExistingBootDisk(args) if create_boot_disk: image_uri, _ = self.ExpandImageFlag(args, return_image_resource=False) else: image_uri = None # A list of lists where the element at index i contains a list of # disk messages that should be set for the instance at index i. disks_messages = [] # A mapping of zone to boot disk references for all existing boot # disks that are being attached. # TODO(user): Simplify this once resources.Resource becomes # hashable. existing_boot_disks = {} for instance_ref in instance_refs: persistent_disks, boot_disk_ref = ( self.CreatePersistentAttachedDiskMessages(args, instance_ref)) local_ssds = [ instance_utils.CreateLocalSsdMessage(self, x.get('device-name'), x.get('interface'), instance_ref.zone) for x in args.local_ssd or [] ] if create_boot_disk: boot_disk = self.CreateDefaultBootAttachedDiskMessage( args, boot_disk_size_gb, image_uri, instance_ref) persistent_disks = [boot_disk] + persistent_disks else: existing_boot_disks[boot_disk_ref.zone] = boot_disk_ref disks_messages.append(persistent_disks + local_ssds) requests = [] for instance_ref, machine_type_uri, disks in zip( instance_refs, machine_type_uris, disks_messages): requests.append( self.messages.ComputeInstancesInsertRequest( instance=self.messages.Instance( canIpForward=args.can_ip_forward, disks=disks, description=args.description, machineType=machine_type_uri, metadata=metadata, name=instance_ref.Name(), networkInterfaces=[network_interface], serviceAccounts=service_accounts, scheduling=scheduling, tags=tags, ), project=self.project, zone=instance_ref.zone)) return requests
def ParseCreateOptionsBase(args): """Parses the flags provided with the cluster creation command.""" if not (args.IsSpecified('enable_basic_auth') or args.IsSpecified('username')): log.warning('Starting in 1.12, new clusters will have basic ' 'authentication disabled by default. Basic authentication ' 'can be enabled (or disabled) manually using the ' '`--[no-]enable-basic-auth` flag.') if not args.IsSpecified('issue_client_certificate'): log.warning( 'Starting in 1.12, new clusters will not have a client ' 'certificate issued. You can manually enable (or disable) the ' 'issuance of the client certificate using the ' '`--[no-]issue-client-certificate` flag.') flags.MungeBasicAuthFlags(args) if (args.IsSpecified('enable_cloud_endpoints') and properties.VALUES.container.new_scopes_behavior.GetBool()): raise util.Error( 'Flag --[no-]enable-cloud-endpoints is not allowed if ' 'property container/ new_scopes_behavior is set to true.') if args.IsSpecified('enable_autorepair'): enable_autorepair = args.enable_autorepair else: # Node pools using COS support auto repairs, enable it for them by default. # Other node pools using (Ubuntu, custom images) don't support node auto # repairs, attempting to enable autorepair for them will result in API call # failing so don't do it. enable_autorepair = ((args.image_type or '').lower() in ['', 'cos']) flags.WarnForUnspecifiedIpAllocationPolicy(args) metadata = metadata_utils.ConstructMetadataDict(args.metadata, args.metadata_from_file) return api_adapter.CreateClusterOptions( accelerators=args.accelerator, additional_zones=args.additional_zones, addons=args.addons, cluster_ipv4_cidr=args.cluster_ipv4_cidr, cluster_secondary_range_name=args.cluster_secondary_range_name, cluster_version=args.cluster_version, node_version=args.node_version, create_subnetwork=args.create_subnetwork, disk_type=args.disk_type, enable_autorepair=enable_autorepair, enable_autoscaling=args.enable_autoscaling, enable_autoupgrade=args.enable_autoupgrade, enable_cloud_endpoints=args.enable_cloud_endpoints, enable_cloud_logging=args.enable_cloud_logging, enable_cloud_monitoring=args.enable_cloud_monitoring, enable_ip_alias=args.enable_ip_alias, enable_kubernetes_alpha=args.enable_kubernetes_alpha, enable_legacy_authorization=args.enable_legacy_authorization, enable_master_authorized_networks=args. enable_master_authorized_networks, enable_network_policy=args.enable_network_policy, image_type=args.image_type, image=args.image, image_project=args.image_project, image_family=args.image_family, issue_client_certificate=args.issue_client_certificate, labels=args.labels, local_ssd_count=args.local_ssd_count, maintenance_window=args.maintenance_window, master_authorized_networks=args.master_authorized_networks, max_nodes=args.max_nodes, max_nodes_per_pool=args.max_nodes_per_pool, min_cpu_platform=args.min_cpu_platform, min_nodes=args.min_nodes, network=args.network, node_disk_size_gb=utils.BytesToGb(args.disk_size), node_labels=args.node_labels, node_locations=args.node_locations, node_machine_type=args.machine_type, node_taints=args.node_taints, num_nodes=args.num_nodes, password=args.password, preemptible=args.preemptible, scopes=args.scopes, service_account=args.service_account, services_ipv4_cidr=args.services_ipv4_cidr, services_secondary_range_name=args.services_secondary_range_name, subnetwork=args.subnetwork, tags=args.tags, user=args.username, metadata=metadata)
def CreatePersistentCreateDiskMessages(client, resources, user_project, create_disks, support_kms=False, container_mount_disk=None): """Returns a list of AttachedDisk messages. Args: client: Compute client adapter resources: Compute resources registry user_project: name of user project create_disks: disk objects - contains following properties * name - the name of disk, * description - an optional description for the disk, * mode - 'rw' (R/W), 'ro' (R/O) access mode, * size - the size of the disk, * type - the type of the disk (HDD or SSD), * image - the name of the image to initialize from, * image-family - the image family name, * image-project - the project name that has the image, * auto-delete - whether disks is deleted when VM is deleted ('yes' if True), * device-name - device name on VM. support_kms: if KMS is supported container_mount_disk: list of disks to be mounted to container, if any. Returns: list of API messages for attached disks """ disks_messages = [] for disk in create_disks or []: name = disk.get('name') # Resolves the mode. mode_value = disk.get('mode', 'rw') if mode_value == 'rw': mode = client.messages.AttachedDisk.ModeValueValuesEnum.READ_WRITE else: mode = client.messages.AttachedDisk.ModeValueValuesEnum.READ_ONLY auto_delete = disk.get('auto-delete') == 'yes' disk_size_gb = utils.BytesToGb(disk.get('size')) img = disk.get('image') img_family = disk.get('image-family') img_project = disk.get('image-project') image_uri = None if img or img_family: image_expander = image_utils.ImageExpander(client, resources) image_uri, _ = image_expander.ExpandImageFlag( user_project=user_project, image=img, image_family=img_family, image_project=img_project, return_image_resource=False) disk_key = None if support_kms: disk_key = kms_utils.MaybeGetKmsKeyFromDict( disk, client.messages, disk_key) device_name = instance_utils.GetDiskDeviceName(disk, name, container_mount_disk) create_disk = client.messages.AttachedDisk( autoDelete=auto_delete, boot=False, deviceName=device_name, initializeParams=client.messages.AttachedDiskInitializeParams( diskName=name, description=disk.get('description'), sourceImage=image_uri, diskSizeGb=disk_size_gb, diskType=disk.get('type')), mode=mode, type=client.messages.AttachedDisk.TypeValueValuesEnum.PERSISTENT, diskEncryptionKey=disk_key) disks_messages.append(create_disk) return disks_messages
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. Raises: FailedBuildException: If the build is completed and not 'SUCCESS'. """ default_gcs_source = False default_bucket_name = None if args.gcs_source_staging_dir is None: default_gcs_source = True default_bucket_name = staging_bucket_util.GetDefaultStagingBucket() args.gcs_source_staging_dir = 'gs://{}/source'.format( default_bucket_name) client = cloudbuild_util.GetClientInstance() messages = cloudbuild_util.GetMessagesModule() gcs_client = storage_api.StorageClient() # First, create the build request. build_timeout = properties.VALUES.builds.timeout.Get() if build_timeout is not None: try: # A bare number is interpreted as seconds. build_timeout_secs = int(build_timeout) except ValueError: build_timeout_duration = times.ParseDuration(build_timeout) build_timeout_secs = int(build_timeout_duration.total_seconds) timeout_str = six.text_type(build_timeout_secs) + 's' else: timeout_str = None if args.tag is not None: if (properties.VALUES.builds.check_tag.GetBool() and 'gcr.io/' not in args.tag): raise c_exceptions.InvalidArgumentException( '--tag', 'Tag value must be in the gcr.io/* or *.gcr.io/* namespace.' ) if properties.VALUES.builds.use_kaniko.GetBool(): if args.no_cache: ttl = '0h' else: ttl = '{}h'.format( properties.VALUES.builds.kaniko_cache_ttl.Get()) build_config = messages.Build( steps=[ messages.BuildStep( name=properties.VALUES.builds.kaniko_image.Get(), args=[ '--destination', args.tag, '--cache', '--cache-ttl', ttl, '--cache-dir', '', ], ), ], timeout=timeout_str, substitutions=cloudbuild_util.EncodeSubstitutions( args.substitutions, messages)) else: if args.no_cache: raise c_exceptions.InvalidArgumentException( 'no-cache', 'Cannot specify --no-cache if builds/use_kaniko property is ' 'False') build_config = messages.Build( images=[args.tag], steps=[ messages.BuildStep( name='gcr.io/cloud-builders/docker', args=[ 'build', '--network', 'cloudbuild', '--no-cache', '-t', args.tag, '.' ], ), ], timeout=timeout_str, substitutions=cloudbuild_util.EncodeSubstitutions( args.substitutions, messages)) elif args.config is not None: if args.no_cache: raise c_exceptions.ConflictingArgumentsException( '--config', '--no-cache') if not args.config: raise c_exceptions.InvalidArgumentException( '--config', 'Config file path must not be empty.') build_config = config.LoadCloudbuildConfigFromPath( args.config, messages, params=args.substitutions) else: raise c_exceptions.OneOfArgumentsRequiredException( ['--tag', '--config'], 'Requires either a docker tag or a config file.') # If timeout was set by flag, overwrite the config file. if timeout_str: build_config.timeout = timeout_str # --no-source overrides the default --source. if not args.IsSpecified('source') and args.no_source: args.source = None gcs_source_staging = None if args.source: suffix = '.tgz' if args.source.startswith('gs://') or os.path.isfile(args.source): _, suffix = os.path.splitext(args.source) # Next, stage the source to Cloud Storage. staged_object = '{stamp}-{uuid}{suffix}'.format( stamp=times.GetTimeStampFromDateTime(times.Now()), uuid=uuid.uuid4().hex, suffix=suffix, ) gcs_source_staging_dir = resources.REGISTRY.Parse( args.gcs_source_staging_dir, collection='storage.objects') # We create the bucket (if it does not exist) first. If we do an existence # check and then create the bucket ourselves, it would be possible for an # attacker to get lucky and beat us to creating the bucket. Block on this # creation to avoid this race condition. gcs_client.CreateBucketIfNotExists(gcs_source_staging_dir.bucket) # If no bucket is specified (for the source `default_gcs_source`), check # that the default bucket is also owned by the project (b/33046325). if default_gcs_source and not staging_bucket_util.BucketIsInProject( gcs_client, default_bucket_name): raise c_exceptions.RequiredArgumentException( 'gcs-source-staging-dir', 'A bucket with name {} already exists and is owned by ' 'another project. Specify a bucket using ' '--gcs-source-staging-dir.'.format(default_bucket_name)) if gcs_source_staging_dir.object: staged_object = gcs_source_staging_dir.object + '/' + staged_object gcs_source_staging = resources.REGISTRY.Create( collection='storage.objects', bucket=gcs_source_staging_dir.bucket, object=staged_object) if args.source.startswith('gs://'): gcs_source = resources.REGISTRY.Parse( args.source, collection='storage.objects') staged_source_obj = gcs_client.Rewrite(gcs_source, gcs_source_staging) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) else: if not os.path.exists(args.source): raise c_exceptions.BadFileException( 'could not find source [{src}]'.format( src=args.source)) if os.path.isdir(args.source): source_snapshot = snapshot.Snapshot( args.source, ignore_file=args.ignore_file) size_str = resource_transform.TransformSize( source_snapshot.uncompressed_size) log.status.Print( 'Creating temporary tarball archive of {num_files} file(s)' ' totalling {size} before compression.'.format( num_files=len(source_snapshot.files), size=size_str)) staged_source_obj = source_snapshot.CopyTarballToGCS( gcs_client, gcs_source_staging, ignore_file=args.ignore_file) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) elif os.path.isfile(args.source): unused_root, ext = os.path.splitext(args.source) if ext not in _ALLOWED_SOURCE_EXT: raise c_exceptions.BadFileException( 'Local file [{src}] is none of ' + ', '.join(_ALLOWED_SOURCE_EXT)) log.status.Print('Uploading local file [{src}] to ' '[gs://{bucket}/{object}].'.format( src=args.source, bucket=gcs_source_staging.bucket, object=gcs_source_staging.object, )) staged_source_obj = gcs_client.CopyFileToGCS( args.source, gcs_source_staging) build_config.source = messages.Source( storageSource=messages.StorageSource( bucket=staged_source_obj.bucket, object=staged_source_obj.name, generation=staged_source_obj.generation, )) else: # No source if not args.no_source: raise c_exceptions.InvalidArgumentException( '--no-source', 'To omit source, use the --no-source flag.') if args.gcs_log_dir: gcs_log_dir = resources.REGISTRY.Parse( args.gcs_log_dir, collection='storage.objects') build_config.logsBucket = ('gs://' + gcs_log_dir.bucket + '/' + gcs_log_dir.object) # Machine type. if args.machine_type is not None: machine_type = flags.GetMachineType(args.machine_type) if not build_config.options: build_config.options = messages.BuildOptions() build_config.options.machineType = machine_type # Disk size. if args.disk_size is not None: disk_size = compute_utils.BytesToGb(args.disk_size) if not build_config.options: build_config.options = messages.BuildOptions() build_config.options.diskSizeGb = int(disk_size) log.debug('submitting build: ' + repr(build_config)) # Start the build. op = client.projects_builds.Create( messages.CloudbuildProjectsBuildsCreateRequest( build=build_config, projectId=properties.VALUES.core.project.Get())) json = encoding.MessageToJson(op.metadata) build = encoding.JsonToMessage(messages.BuildOperationMetadata, json).build build_ref = resources.REGISTRY.Create( collection='cloudbuild.projects.builds', projectId=build.projectId, id=build.id) log.CreatedResource(build_ref) if build.logUrl: log.status.Print('Logs are available at [{log_url}].'.format( log_url=build.logUrl)) else: log.status.Print('Logs are available in the Cloud Console.') # If the command is run --async, we just print out a reference to the build. if args.async_: return build mash_handler = execution.MashHandler( execution.GetCancelBuildHandler(client, messages, build_ref)) # Otherwise, logs are streamed from GCS. with execution_utils.CtrlCSection(mash_handler): build = cb_logs.CloudBuildClient(client, messages).Stream(build_ref) if build.status == messages.Build.StatusValueValuesEnum.TIMEOUT: log.status.Print( 'Your build timed out. Use the [--timeout=DURATION] flag to change ' 'the timeout threshold.') if build.status != messages.Build.StatusValueValuesEnum.SUCCESS: raise FailedBuildException(build) return build
def _CreateRequests(self, args, instance_refs, compute_client, resource_parser): # gcloud creates default values for some fields in Instance resource # when no value was specified on command line. # When --source-instance-template was specified, defaults are taken from # Instance Template and gcloud flags are used to override them - by default # fields should not be initialized. source_instance_template = self.GetSourceInstanceTemplate( args, resource_parser) skip_defaults = source_instance_template is not None # This feature is only exposed in alpha/beta allow_rsa_encrypted = self.ReleaseTrack() in [ base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA ] self.csek_keys = csek_utils.CsekKeyStore.FromArgs( args, allow_rsa_encrypted) if (skip_defaults and not args.IsSpecified('maintenance_policy') and not args.IsSpecified('preemptible') and not args.IsSpecified('restart_on_failure')): scheduling = None else: scheduling = instance_utils.CreateSchedulingMessage( messages=compute_client.messages, maintenance_policy=args.maintenance_policy, preemptible=args.preemptible, restart_on_failure=args.restart_on_failure) if args.tags: tags = compute_client.messages.Tags(items=args.tags) else: tags = None labels = None args_labels = getattr(args, 'labels', None) if args_labels: labels = compute_client.messages.Instance.LabelsValue( additionalProperties=[ compute_client.messages.Instance.LabelsValue. AdditionalProperty(key=key, value=value) for key, value in sorted(args.labels.iteritems()) ]) if (skip_defaults and not args.IsSpecified('metadata') and not args.IsSpecified('metadata_from_file')): metadata = None else: metadata = metadata_utils.ConstructMetadataMessage( compute_client.messages, metadata=args.metadata, metadata_from_file=args.metadata_from_file) # If the user already provided an initial Windows password and # username through metadata, then there is no need to check # whether the image or the boot disk is Windows. boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) # Check if the zone is deprecated or has maintenance coming. zone_resource_fetcher = zone_utils.ZoneResourceFetcher(compute_client) zone_resource_fetcher.WarnForZonalCreation(instance_refs) network_interface_arg = getattr(args, 'network_interface', None) if network_interface_arg: network_interfaces = instance_utils.CreateNetworkInterfaceMessages( resources=resource_parser, compute_client=compute_client, network_interface_arg=network_interface_arg, instance_refs=instance_refs, support_network_tier=self._support_network_tier) else: if self._support_public_dns is True: instances_flags.ValidatePublicDnsFlags(args) if self._support_public_ptr is True: instances_flags.ValidatePublicPtrFlags(args) if (skip_defaults and not args.IsSpecified('network') and not args.IsSpecified('subnet') and not args.IsSpecified('private_network_ip') and not args.IsSpecified('no_address') and not args.IsSpecified('address') and not args.IsSpecified('network_tier') and not args.IsSpecified('no_public_dns') and not args.IsSpecified('public_dns') and not args.IsSpecified('no_public_ptr') and not args.IsSpecified('public_ptr') and not args.IsSpecified('no_public_ptr_domain') and not args.IsSpecified('public_ptr_domain')): network_interfaces = [] else: network_tier = getattr(args, 'network_tier', None) network_interfaces = [ instance_utils.CreateNetworkInterfaceMessage( resources=resource_parser, compute_client=compute_client, network=args.network, subnet=args.subnet, private_network_ip=args.private_network_ip, no_address=args.no_address, address=args.address, instance_refs=instance_refs, network_tier=network_tier, no_public_dns=getattr(args, 'no_public_dns', None), public_dns=getattr(args, 'public_dns', None), no_public_ptr=getattr(args, 'no_public_ptr', None), public_ptr=getattr(args, 'public_ptr', None), no_public_ptr_domain=getattr(args, 'no_public_ptr_domain', None), public_ptr_domain=getattr(args, 'public_ptr_domain', None)) ] if (skip_defaults and not args.IsSpecified('machine_type') and not args.IsSpecified('custom_cpu') and not args.IsSpecified('custom_memory')): machine_type_uris = [None for _ in instance_refs] else: machine_type_uris = instance_utils.CreateMachineTypeUris( resources=resource_parser, compute_client=compute_client, machine_type=args.machine_type, custom_cpu=args.custom_cpu, custom_memory=args.custom_memory, ext=getattr(args, 'custom_extensions', None), instance_refs=instance_refs) create_boot_disk = not instance_utils.UseExistingBootDisk(args.disk or []) if create_boot_disk: image_expander = image_utils.ImageExpander(compute_client, resource_parser) image_uri, _ = image_expander.ExpandImageFlag( user_project=instance_refs[0].project, image=args.image, image_family=args.image_family, image_project=args.image_project, return_image_resource=False) else: image_uri = None # A list of lists where the element at index i contains a list of # disk messages that should be set for the instance at index i. disks_messages = [] # A mapping of zone to boot disk references for all existing boot # disks that are being attached. # TODO(b/36050875): Simplify since resources.Resource is now hashable. existing_boot_disks = {} if (skip_defaults and not args.IsSpecified('disk') and not args.IsSpecified('create_disk') and not args.IsSpecified('local_ssd') and not args.IsSpecified('boot_disk_type') and not args.IsSpecified('boot_disk_device_name') and not args.IsSpecified('boot_disk_auto_delete') and not args.IsSpecified('boot_disk_kms_key') and not args.IsSpecified('boot_disk_kms_project') and not args.IsSpecified('boot_disk_kms_location') and not args.IsSpecified('boot_disk_kms_keyring') and not args.IsSpecified('require_csek_key_create')): disks_messages = [[] for _ in instance_refs] else: for instance_ref in instance_refs: persistent_disks, boot_disk_ref = ( instance_utils.CreatePersistentAttachedDiskMessages( resource_parser, compute_client, self.csek_keys, args.disk or [], instance_ref)) persistent_create_disks = ( instance_utils.CreatePersistentCreateDiskMessages( compute_client, resource_parser, self.csek_keys, getattr(args, 'create_disk', []), instance_ref)) local_ssds = [] for x in args.local_ssd or []: local_ssds.append( instance_utils.CreateLocalSsdMessage( resource_parser, compute_client.messages, x.get('device-name'), x.get('interface'), x.get('size'), instance_ref.zone, instance_ref.project)) if create_boot_disk: boot_disk = instance_utils.CreateDefaultBootAttachedDiskMessage( compute_client, resource_parser, disk_type=args.boot_disk_type, disk_device_name=args.boot_disk_device_name, disk_auto_delete=args.boot_disk_auto_delete, disk_size_gb=boot_disk_size_gb, require_csek_key_create=(args.require_csek_key_create if self.csek_keys else None), image_uri=image_uri, instance_ref=instance_ref, csek_keys=self.csek_keys, kms_args=args) persistent_disks = [boot_disk] + persistent_disks else: existing_boot_disks[boot_disk_ref.zone] = boot_disk_ref disks_messages.append(persistent_disks + persistent_create_disks + local_ssds) accelerator_args = getattr(args, 'accelerator', None) project_to_sa = {} requests = [] for instance_ref, machine_type_uri, disks in zip( instance_refs, machine_type_uris, disks_messages): if instance_ref.project not in project_to_sa: scopes = None if not args.no_scopes and not args.scopes: # User didn't provide any input on scopes. If project has no default # service account then we want to create a VM with no scopes request = ( compute_client.apitools_client.projects, 'Get', compute_client.messages.ComputeProjectsGetRequest( project=instance_ref.project)) errors = [] result = compute_client.MakeRequests([request], errors) if not errors: if not result[0].defaultServiceAccount: scopes = [] log.status.Print( 'There is no default service account for project {}. ' 'Instance {} will not have scopes.'.format( instance_ref.project, instance_ref.Name)) if scopes is None: scopes = [] if args.no_scopes else args.scopes if args.no_service_account: service_account = None else: service_account = args.service_account if (skip_defaults and not args.IsSpecified('scopes') and not args.IsSpecified('no_scopes') and not args.IsSpecified('service_account') and not args.IsSpecified('no_service_account')): service_accounts = [] else: service_accounts = instance_utils.CreateServiceAccountMessages( messages=compute_client.messages, scopes=scopes, service_account=service_account) project_to_sa[instance_ref.project] = service_accounts if skip_defaults and not args.IsSpecified('can_ip_forward'): can_ip_forward = None else: can_ip_forward = args.can_ip_forward instance = compute_client.messages.Instance( canIpForward=can_ip_forward, disks=disks, description=args.description, machineType=machine_type_uri, metadata=metadata, minCpuPlatform=args.min_cpu_platform, name=instance_ref.Name(), networkInterfaces=network_interfaces, serviceAccounts=project_to_sa[instance_ref.project], scheduling=scheduling, tags=tags) if getattr(args, 'deletion_protection', None) is not None: instance.deletionProtection = args.deletion_protection if labels: instance.labels = labels if accelerator_args: accelerator_type_name = accelerator_args['type'] accelerator_type_ref = resource_parser.Parse( accelerator_type_name, collection='compute.acceleratorTypes', params={ 'project': instance_ref.project, 'zone': instance_ref.zone }) # Accelerator count is default to 1. accelerator_count = int(accelerator_args.get('count', 1)) accelerators = instance_utils.CreateAcceleratorConfigMessages( compute_client.messages, accelerator_type_ref, accelerator_count) instance.guestAccelerators = accelerators request = compute_client.messages.ComputeInstancesInsertRequest( instance=instance, project=instance_ref.project, zone=instance_ref.zone) if source_instance_template: request.sourceInstanceTemplate = source_instance_template sole_tenancy_host_arg = getattr(args, 'sole_tenancy_host', None) if sole_tenancy_host_arg: sole_tenancy_host_ref = resource_parser.Parse( sole_tenancy_host_arg, collection='compute.hosts', params={ 'project': instance_ref.project, 'zone': instance_ref.zone }) request.instance.host = sole_tenancy_host_ref.SelfLink() requests.append( (compute_client.apitools_client.instances, 'Insert', request)) return requests
def GetClusterConfig(args, dataproc, project_id, compute_resources, beta=False, include_deprecated=True, include_ttl_config=False): """Get dataproc cluster configuration. Args: args: Arguments parsed from argparse.ArgParser. dataproc: Dataproc object that contains client, messages, and resources project_id: Dataproc project ID compute_resources: compute resource for cluster beta: use BETA only features include_deprecated: whether to include deprecated args include_ttl_config: whether to include Scheduled Delete(TTL) args Returns: cluster_config: Dataproc cluster configuration """ master_accelerator_type = None worker_accelerator_type = None master_accelerator_count = None worker_accelerator_count = None if beta: if args.master_accelerator: master_accelerator_type = args.master_accelerator['type'] master_accelerator_count = args.master_accelerator.get('count', 1) if args.worker_accelerator: worker_accelerator_type = args.worker_accelerator['type'] worker_accelerator_count = args.worker_accelerator.get('count', 1) # Resolve non-zonal GCE resources # We will let the server resolve short names of zonal resources because # if auto zone is requested, we will not know the zone before sending the # request image_ref = args.image and compute_resources.Parse( args.image, params={'project': project_id}, collection='compute.images') network_ref = args.network and compute_resources.Parse( args.network, params={'project': project_id}, collection='compute.networks') subnetwork_ref = args.subnet and compute_resources.Parse( args.subnet, params={ 'project': project_id, 'region': properties.VALUES.compute.region.GetOrFail, }, collection='compute.subnetworks') timeout_str = str(args.initialization_action_timeout) + 's' init_actions = [ dataproc.messages.NodeInitializationAction( executableFile=exe, executionTimeout=timeout_str) for exe in (args.initialization_actions or []) ] # Increase the client timeout for each initialization action. args.timeout += args.initialization_action_timeout * len(init_actions) expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes) software_config = dataproc.messages.SoftwareConfig( imageVersion=args.image_version) if include_deprecated: master_boot_disk_size_gb = args.master_boot_disk_size_gb else: master_boot_disk_size_gb = None if args.master_boot_disk_size: master_boot_disk_size_gb = (api_utils.BytesToGb( args.master_boot_disk_size)) if include_deprecated: worker_boot_disk_size_gb = args.worker_boot_disk_size_gb else: worker_boot_disk_size_gb = None if args.worker_boot_disk_size: worker_boot_disk_size_gb = (api_utils.BytesToGb( args.worker_boot_disk_size)) preemptible_worker_boot_disk_size_gb = (api_utils.BytesToGb( args.preemptible_worker_boot_disk_size)) if args.single_node or args.num_workers == 0: # Explicitly specifying --num-workers=0 gives you a single node cluster, # but if --num-workers is omitted, args.num_workers is None (not 0), and # this property will not be set args.properties[constants.ALLOW_ZERO_WORKERS_PROPERTY] = 'true' if args.properties: software_config.properties = encoding.DictToAdditionalPropertyMessage( args.properties, dataproc.messages.SoftwareConfig.PropertiesValue, sort_items=True) if args.components: software_config_cls = dataproc.messages.SoftwareConfig software_config.optionalComponents.extend( list( map( software_config_cls. OptionalComponentsValueListEntryValuesEnum, args.components))) gce_cluster_config = dataproc.messages.GceClusterConfig( networkUri=network_ref and network_ref.SelfLink(), subnetworkUri=subnetwork_ref and subnetwork_ref.SelfLink(), internalIpOnly=args.no_address, serviceAccount=args.service_account, serviceAccountScopes=expanded_scopes, zoneUri=properties.VALUES.compute.zone.GetOrFail()) if beta: reservation_affinity = GetReservationAffinity(args, dataproc) gce_cluster_config.reservationAffinity = reservation_affinity if args.tags: gce_cluster_config.tags = args.tags if args.metadata: flat_metadata = collections.OrderedDict([(k, v) for d in args.metadata for k, v in d.items()]) gce_cluster_config.metadata = encoding.DictToAdditionalPropertyMessage( flat_metadata, dataproc.messages.GceClusterConfig.MetadataValue) master_accelerators = [] if master_accelerator_type: master_accelerators.append( dataproc.messages.AcceleratorConfig( acceleratorTypeUri=master_accelerator_type, acceleratorCount=master_accelerator_count)) worker_accelerators = [] if worker_accelerator_type: worker_accelerators.append( dataproc.messages.AcceleratorConfig( acceleratorTypeUri=worker_accelerator_type, acceleratorCount=worker_accelerator_count)) cluster_config = dataproc.messages.ClusterConfig( configBucket=args.bucket, gceClusterConfig=gce_cluster_config, masterConfig=dataproc.messages.InstanceGroupConfig( numInstances=args.num_masters, imageUri=image_ref and image_ref.SelfLink(), machineTypeUri=args.master_machine_type, accelerators=master_accelerators, diskConfig=GetDiskConfig(dataproc, args.master_boot_disk_type, master_boot_disk_size_gb, args.num_master_local_ssds)), workerConfig=dataproc.messages.InstanceGroupConfig( numInstances=args.num_workers, imageUri=image_ref and image_ref.SelfLink(), machineTypeUri=args.worker_machine_type, accelerators=worker_accelerators, diskConfig=GetDiskConfig( dataproc, args.worker_boot_disk_type, worker_boot_disk_size_gb, args.num_worker_local_ssds, )), initializationActions=init_actions, softwareConfig=software_config, ) if args.kerberos_config_file or args.kerberos_root_principal_password_uri: cluster_config.securityConfig = dataproc.messages.SecurityConfig() if args.kerberos_config_file: cluster_config.securityConfig.kerberosConfig = ParseKerberosConfigFile( dataproc, args.kerberos_config_file) else: kerberos_config = dataproc.messages.KerberosConfig() kerberos_config.enableKerberos = True if args.kerberos_root_principal_password_uri: kerberos_config.rootPrincipalPasswordUri = \ args.kerberos_root_principal_password_uri kerberos_kms_ref = args.CONCEPTS.kerberos_kms_key.Parse() kerberos_config.kmsKeyUri = kerberos_kms_ref.RelativeName() cluster_config.securityConfig.kerberosConfig = kerberos_config if beta: if args.enable_component_gateway: cluster_config.endpointConfig = dataproc.messages.EndpointConfig( enableHttpPortAccess=args.enable_component_gateway) if args.autoscaling_policy: cluster_config.autoscalingConfig = dataproc.messages.AutoscalingConfig( policyUri=args.CONCEPTS.autoscaling_policy.Parse( ).RelativeName()) cluster_config.masterConfig.minCpuPlatform = args.master_min_cpu_platform cluster_config.workerConfig.minCpuPlatform = args.worker_min_cpu_platform if include_ttl_config: lifecycle_config = dataproc.messages.LifecycleConfig() changed_config = False if args.max_age is not None: lifecycle_config.autoDeleteTtl = str(args.max_age) + 's' changed_config = True if args.expiration_time is not None: lifecycle_config.autoDeleteTime = times.FormatDateTime( args.expiration_time) changed_config = True if args.max_idle is not None: lifecycle_config.idleDeleteTtl = str(args.max_idle) + 's' changed_config = True if changed_config: cluster_config.lifecycleConfig = lifecycle_config if hasattr(args.CONCEPTS, 'kms_key'): kms_ref = args.CONCEPTS.kms_key.Parse() if kms_ref: encryption_config = dataproc.messages.EncryptionConfig() encryption_config.gcePdKmsKeyName = kms_ref.RelativeName() cluster_config.encryptionConfig = encryption_config else: # Did user use any gce-pd-kms-key flags? for keyword in [ 'gce-pd-kms-key', 'gce-pd-kms-key-project', 'gce-pd-kms-key-location', 'gce-pd-kms-key-keyring' ]: if getattr(args, keyword.replace('-', '_'), None): raise exceptions.ArgumentError( '--gce-pd-kms-key was not fully specified.') # Secondary worker group is optional. However, users may specify # future pVMs configuration at creation time. if (args.num_preemptible_workers is not None or preemptible_worker_boot_disk_size_gb is not None or args.preemptible_worker_boot_disk_type is not None or args.num_preemptible_worker_local_ssds is not None or (beta and args.worker_min_cpu_platform is not None)): cluster_config.secondaryWorkerConfig = ( dataproc.messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers, diskConfig=GetDiskConfig( dataproc, args.preemptible_worker_boot_disk_type, preemptible_worker_boot_disk_size_gb, args.num_preemptible_worker_local_ssds, ))) if beta and args.worker_min_cpu_platform: cluster_config.secondaryWorkerConfig.minCpuPlatform = ( args.worker_min_cpu_platform) return cluster_config
def CreateRequests(self, args): instances_flags.ValidateDockerArgs(args) instances_flags.ValidateDiskCommonFlags(args) instances_flags.ValidateLocalSsdFlags(args) if instance_utils.UseExistingBootDisk(args.disk or []): raise exceptions.InvalidArgumentException( '--disk', 'Boot disk specified for containerized VM.') scheduling = instance_utils.CreateSchedulingMessage( messages=self.messages, maintenance_policy=args.maintenance_policy, preemptible=args.preemptible, restart_on_failure=args.restart_on_failure) service_accounts = instance_utils.CreateServiceAccountMessages( messages=self.messages, scopes=([] if args.no_scopes else args.scopes)) user_metadata = metadata_utils.ConstructMetadataMessage( self.messages, metadata=args.metadata, metadata_from_file=args.metadata_from_file) containers_utils.ValidateUserMetadata(user_metadata) boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) instance_refs = instances_flags.INSTANCES_ARG.ResolveAsResource( args, self.resources, scope_lister=flags.GetDefaultScopeLister(self.compute_client, self.project)) # Check if the zone is deprecated or has maintenance coming. self.WarnForZonalCreation(instance_refs) network_interface = instance_utils.CreateNetworkInterfaceMessage( scope_prompter=self, compute_client=self.compute_client, network=args.network, subnet=args.subnet, private_network_ip=args.private_network_ip, no_address=args.no_address, address=args.address, instance_refs=instance_refs) machine_type_uris = instance_utils.CreateMachineTypeUris( scope_prompter=self, compute_client=self.compute_client, project=self.project, machine_type=args.machine_type, custom_cpu=args.custom_cpu, custom_memory=args.custom_memory, instance_refs=instance_refs) image_uri = containers_utils.ExpandGciImageFlag(self.compute_client) requests = [] for instance_ref, machine_type_uri in zip(instance_refs, machine_type_uris): metadata = containers_utils.CreateMetadataMessage( self.messages, args.run_as_privileged, args.container_manifest, args.docker_image, args.port_mappings, args.run_command, user_metadata, instance_ref.Name()) requests.append( self.messages.ComputeInstancesInsertRequest( instance=self.messages.Instance( canIpForward=args.can_ip_forward, disks=(self._CreateDiskMessages( args, boot_disk_size_gb, image_uri, instance_ref)), description=args.description, machineType=machine_type_uri, metadata=metadata, name=instance_ref.Name(), networkInterfaces=[network_interface], serviceAccounts=service_accounts, scheduling=scheduling, tags=containers_utils.CreateTagsMessage( self.messages, args.tags), ), project=self.project, zone=instance_ref.zone)) return requests
def Run(self, args): holder = base_classes.ComputeApiHolder(self.ReleaseTrack()) client = holder.client source_instance_template = self.GetSourceInstanceTemplate( args, holder.resources) # gcloud creates default values for some fields in Instance resource # when no value was specified on command line. # When --source-instance-template was specified, defaults are taken from # Instance Template and gcloud flags are used to override them - by default # fields should not be initialized. skip_defaults = source_instance_template is not None instances_flags.ValidateDockerArgs(args) instances_flags.ValidateDiskCommonFlags(args) instances_flags.ValidateLocalSsdFlags(args) instances_flags.ValidateServiceAccountAndScopeArgs(args) if instance_utils.UseExistingBootDisk(args.disk or []): raise exceptions.InvalidArgumentException( '--disk', 'Boot disk specified for containerized VM.') if (skip_defaults and not args.IsSpecified('maintenance_policy') and not args.IsSpecified('preemptible') and not args.IsSpecified('restart_on_failure')): scheduling = None else: scheduling = instance_utils.CreateSchedulingMessage( messages=client.messages, maintenance_policy=args.maintenance_policy, preemptible=args.preemptible, restart_on_failure=args.restart_on_failure) if args.no_service_account: service_account = None else: service_account = args.service_account if (skip_defaults and not args.IsSpecified('scopes') and not args.IsSpecified('no_scopes') and not args.IsSpecified('service_account') and not args.IsSpecified('no_service_account')): service_accounts = [] else: service_accounts = instance_utils.CreateServiceAccountMessages( messages=client.messages, scopes=[] if args.no_scopes else args.scopes, service_account=service_account) user_metadata = metadata_utils.ConstructMetadataMessage( client.messages, metadata=args.metadata, metadata_from_file=args.metadata_from_file) containers_utils.ValidateUserMetadata(user_metadata) boot_disk_size_gb = utils.BytesToGb(args.boot_disk_size) utils.WarnIfDiskSizeIsTooSmall(boot_disk_size_gb, args.boot_disk_type) instance_refs = instances_flags.INSTANCES_ARG.ResolveAsResource( args, holder.resources, scope_lister=flags.GetDefaultScopeLister(client)) # Check if the zone is deprecated or has maintenance coming. zone_resource_fetcher = zone_utils.ZoneResourceFetcher(client) zone_resource_fetcher.WarnForZonalCreation(instance_refs) instances_flags.ValidatePublicDnsFlags(args) if (skip_defaults and not args.IsSpecified('network') and not args.IsSpecified('subnet') and not args.IsSpecified('private_network_ip') and not args.IsSpecified('no_address') and not args.IsSpecified('address') and not args.IsSpecified('network_tier') and not args.IsSpecified('no_public_dns') and not args.IsSpecified('public_dns') and not args.IsSpecified('no_public_ptr') and not args.IsSpecified('public_ptr') and not args.IsSpecified('no_public_ptr_domain') and not args.IsSpecified('public_ptr_domain')): network_interfaces = [] else: network_interfaces = [instance_utils.CreateNetworkInterfaceMessage( resources=holder.resources, compute_client=client, network=args.network, subnet=args.subnet, private_network_ip=args.private_network_ip, no_address=args.no_address, address=args.address, instance_refs=instance_refs, network_tier=args.network_tier, no_public_dns=getattr(args, 'no_public_dns', None), public_dns=getattr(args, 'public_dns', None), no_public_ptr=getattr(args, 'no_public_ptr', None), public_ptr=getattr(args, 'public_ptr', None), no_public_ptr_domain=getattr(args, 'no_public_ptr_domain', None), public_ptr_domain=getattr(args, 'public_ptr_domain', None))] if (skip_defaults and not args.IsSpecified('machine_type') and not args.IsSpecified('custom_cpu') and not args.IsSpecified('custom_memory')): machine_type_uris = [None for _ in instance_refs] else: machine_type_uris = instance_utils.CreateMachineTypeUris( resources=holder.resources, compute_client=client, machine_type=args.machine_type, custom_cpu=args.custom_cpu, custom_memory=args.custom_memory, ext=getattr(args, 'custom_extensions', None), instance_refs=instance_refs) image_uri = containers_utils.ExpandCosImageFlag(client) args_labels = getattr(args, 'labels', None) labels = None if args_labels: labels = client.messages.Instance.LabelsValue( additionalProperties=[ client.messages.Instance.LabelsValue.AdditionalProperty( key=key, value=value) for key, value in sorted(args.labels.iteritems())]) if skip_defaults and not args.IsSpecified('can_ip_forward'): can_ip_forward = None else: can_ip_forward = args.can_ip_forward requests = [] for instance_ref, machine_type_uri in zip(instance_refs, machine_type_uris): metadata = containers_utils.CreateMetadataMessage( client.messages, args.run_as_privileged, args.container_manifest, args.docker_image, args.port_mappings, args.run_command, user_metadata, instance_ref.Name()) request = client.messages.ComputeInstancesInsertRequest( instance=client.messages.Instance( canIpForward=can_ip_forward, disks=(self._CreateDiskMessages(holder, args, boot_disk_size_gb, image_uri, instance_ref, skip_defaults)), description=args.description, machineType=machine_type_uri, metadata=metadata, minCpuPlatform=args.min_cpu_platform, name=instance_ref.Name(), networkInterfaces=network_interfaces, serviceAccounts=service_accounts, scheduling=scheduling, tags=containers_utils.CreateTagsMessage(client.messages, args.tags)), project=instance_ref.project, zone=instance_ref.zone) if labels: request.instance.labels = labels if source_instance_template: request.sourceInstanceTemplate = source_instance_template requests.append((client.apitools_client.instances, 'Insert', request)) return client.MakeRequests(requests)
def CreateRequests(self, args): """Returns a list of requests necessary for adding disks.""" size_gb = utils.BytesToGb(args.size) if not size_gb and not args.source_snapshot and not args.image: if args.type and 'pd-ssd' in args.type: size_gb = constants.DEFAULT_SSD_DISK_SIZE_GB else: size_gb = constants.DEFAULT_STANDARD_DISK_SIZE_GB utils.WarnIfDiskSizeIsTooSmall(size_gb, args.type) requests = [] disk_refs = self.CreateZonalReferences(args.names, args.zone) # Check if the zone is deprecated or has maintenance coming. self.WarnForZonalCreation(disk_refs) if args.image: source_image_uri, _ = self.ExpandImageFlag( args, return_image_resource=False) else: source_image_uri = None if args.source_snapshot: snapshot_ref = self.CreateGlobalReference( args.source_snapshot, resource_type='snapshots') snapshot_uri = snapshot_ref.SelfLink() else: snapshot_uri = None if hasattr(args, 'csek_key_file'): csek_keys = csek_utils.CsekKeyStore.FromArgs(args) else: csek_keys = None image_key_message_or_none, snapshot_key_message_or_none = ( csek_utils.MaybeLookupKeyMessagesByUri( csek_keys, self.resources, [source_image_uri, snapshot_uri], self.compute)) for disk_ref in disk_refs: if args.type: type_ref = self.CreateZonalReference(args.type, disk_ref.zone, resource_type='diskTypes') type_uri = type_ref.SelfLink() else: type_uri = None if csek_keys: disk_key_or_none = csek_keys.LookupKey( disk_ref, args.require_csek_key_create) disk_key_message_or_none = csek_utils.MaybeToMessage( disk_key_or_none, self.compute) kwargs = { 'diskEncryptionKey': disk_key_message_or_none, 'sourceImageEncryptionKey': image_key_message_or_none, 'sourceSnapshotEncryptionKey': snapshot_key_message_or_none } else: kwargs = {} request = self.messages.ComputeDisksInsertRequest( disk=self.messages.Disk(name=disk_ref.Name(), description=args.description, sizeGb=size_gb, sourceSnapshot=snapshot_uri, type=type_uri, **kwargs), project=self.project, sourceImage=source_image_uri, zone=disk_ref.zone) requests.append(request) return requests
def CreatePersistentCreateDiskMessages(scope_prompter, compute_client, resources, csek_keys, create_disks, instance_ref): """Returns a list of AttachedDisk messages for newly creating disks. Args: scope_prompter: Scope prompter object, compute_client: creates resources, resources: parser of resources, csek_keys: customer suplied encryption keys, create_disks: disk objects - contains following properties * name - the name of disk, * mode - 'rw' (R/W), 'ro' (R/O) access mode, * disk-size - the size of the disk, * disk-type - the type of the disk (HDD or SSD), * image - the name of the image to initialize from, * image-family - the image family name, * image-project - the project name that has the image, * auto-delete - whether disks is deleted when VM is deleted, * device-name - device name on VM. instance_ref: reference to the instance that will own the new disks. Returns: list of API messages for attached disks """ disks_messages = [] messages = compute_client.messages compute = compute_client.apitools_client for disk in create_disks or []: name = disk.get('name') # Resolves the mode. mode_value = disk.get('mode', 'rw') if mode_value == 'rw': mode = messages.AttachedDisk.ModeValueValuesEnum.READ_WRITE else: mode = messages.AttachedDisk.ModeValueValuesEnum.READ_ONLY auto_delete_value = disk.get('auto-delete', 'yes') auto_delete = auto_delete_value == 'yes' disk_size_gb = utils.BytesToGb(disk.get('size')) disk_type = disk.get('type') if disk_type: disk_type_ref = resources.Parse(disk_type, collection='compute.diskTypes', params={'zone': instance_ref.zone}) disk_type_uri = disk_type_ref.SelfLink() else: disk_type_ref = None disk_type_uri = None image_uri, _ = scope_prompter.ExpandImageFlag( user_project=scope_prompter.project, image=disk.get('image'), image_family=disk.get('image-family'), image_project=disk.get('image-project'), return_image_resource=False) image_key = None disk_key = None if csek_keys: image_key = csek_utils.MaybeLookupKeyMessagesByUri(csek_keys, resources, [image_uri], compute) if name: disk_ref = resources.Parse(name, collection='compute.disks', params={'zone': instance_ref.zone}) disk_key = csek_utils.MaybeLookupKeyMessage(csek_keys, disk_ref, compute) create_disk = messages.AttachedDisk( autoDelete=auto_delete, boot=False, deviceName=disk.get('device-name'), initializeParams=messages.AttachedDiskInitializeParams( diskName=name, sourceImage=image_uri, diskSizeGb=disk_size_gb, diskType=disk_type_uri, sourceImageEncryptionKey=image_key), mode=mode, type=messages.AttachedDisk.TypeValueValuesEnum.PERSISTENT, diskEncryptionKey=disk_key) disks_messages.append(create_disk) return disks_messages
def Run(self, args): self.ValidateArgs(args) client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) compute_resources = compute_helpers.GetComputeResources( self.ReleaseTrack(), args.name) master_accelerator_type = None worker_accelerator_type = None master_accelerator_count = None worker_accelerator_count = None if self.ReleaseTrack() == base.ReleaseTrack.BETA: if args.master_accelerator: master_accelerator_type = args.master_accelerator['type'] master_accelerator_count = args.master_accelerator.get( 'count', 1) if args.worker_accelerator: worker_accelerator_type = args.worker_accelerator['type'] worker_accelerator_count = args.worker_accelerator.get( 'count', 1) # Resolve GCE resources zone_ref = compute_resources.Parse(None, collection='compute.zones') image_ref = args.image and compute_resources.Parse( args.image, collection='compute.images') master_machine_type_ref = (args.master_machine_type and compute_resources.Parse( args.master_machine_type, collection='compute.machineTypes')) worker_machine_type_ref = (args.worker_machine_type and compute_resources.Parse( args.worker_machine_type, collection='compute.machineTypes')) network_ref = args.network and compute_resources.Parse( args.network, collection='compute.networks') subnetwork_ref = args.subnet and compute_resources.Parse( args.subnet, collection='compute.subnetworks') master_accelerator_type_ref = ( master_accelerator_type and compute_resources.Parse(master_accelerator_type, collection='compute.acceleratorTypes')) worker_accelerator_type_ref = ( worker_accelerator_type and compute_resources.Parse(worker_accelerator_type, collection='compute.acceleratorTypes')) init_actions = [] timeout_str = str(args.initialization_action_timeout) + 's' if args.initialization_actions: init_actions = [ messages.NodeInitializationAction(executableFile=exe, executionTimeout=timeout_str) for exe in args.initialization_actions ] expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes) software_config = messages.SoftwareConfig( imageVersion=args.image_version) master_boot_disk_size_gb = args.master_boot_disk_size_gb if args.master_boot_disk_size: master_boot_disk_size_gb = (api_utils.BytesToGb( args.master_boot_disk_size)) worker_boot_disk_size_gb = args.worker_boot_disk_size_gb if args.worker_boot_disk_size: worker_boot_disk_size_gb = (api_utils.BytesToGb( args.worker_boot_disk_size)) preemptible_worker_boot_disk_size_gb = (api_utils.BytesToGb( args.preemptible_worker_boot_disk_size)) if args.single_node: args.properties[constants.ALLOW_ZERO_WORKERS_PROPERTY] = 'true' if args.properties: software_config.properties = encoding.DictToMessage( args.properties, messages.SoftwareConfig.PropertiesValue) gce_cluster_config = messages.GceClusterConfig( networkUri=network_ref and network_ref.SelfLink(), subnetworkUri=subnetwork_ref and subnetwork_ref.SelfLink(), serviceAccount=args.service_account, serviceAccountScopes=expanded_scopes, zoneUri=zone_ref and zone_ref.SelfLink()) if args.tags: gce_cluster_config.tags = args.tags if args.metadata: flat_metadata = dict( (k, v) for d in args.metadata for k, v in d.items()) gce_cluster_config.metadata = encoding.DictToMessage( flat_metadata, messages.GceClusterConfig.MetadataValue) master_accelerators = [] if master_accelerator_type: master_accelerators.append( messages.AcceleratorConfig( acceleratorTypeUri=master_accelerator_type_ref and master_accelerator_type_ref.SelfLink(), acceleratorCount=master_accelerator_count)) worker_accelerators = [] if worker_accelerator_type: worker_accelerators.append( messages.AcceleratorConfig( acceleratorTypeUri=worker_accelerator_type_ref and worker_accelerator_type_ref.SelfLink(), acceleratorCount=worker_accelerator_count)) cluster_config = messages.ClusterConfig( configBucket=args.bucket, gceClusterConfig=gce_cluster_config, masterConfig=messages.InstanceGroupConfig( numInstances=args.num_masters, imageUri=image_ref and image_ref.SelfLink(), machineTypeUri=master_machine_type_ref and master_machine_type_ref.SelfLink(), accelerators=master_accelerators, diskConfig=messages.DiskConfig( bootDiskSizeGb=master_boot_disk_size_gb, numLocalSsds=args.num_master_local_ssds, ), ), workerConfig=messages.InstanceGroupConfig( numInstances=args.num_workers, imageUri=image_ref and image_ref.SelfLink(), machineTypeUri=worker_machine_type_ref and worker_machine_type_ref.SelfLink(), accelerators=worker_accelerators, diskConfig=messages.DiskConfig( bootDiskSizeGb=worker_boot_disk_size_gb, numLocalSsds=args.num_worker_local_ssds, ), ), initializationActions=init_actions, softwareConfig=software_config, ) # Secondary worker group is optional. However, users may specify # future pVM disk size at creation time. if (args.num_preemptible_workers is not None or preemptible_worker_boot_disk_size_gb is not None): cluster_config.secondaryWorkerConfig = ( messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers, diskConfig=messages.DiskConfig( bootDiskSizeGb=preemptible_worker_boot_disk_size_gb, )) ) cluster = messages.Cluster(config=cluster_config, clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) self.ConfigureCluster(messages, args, cluster) operation = client.projects_regions_clusters.Create( messages.DataprocProjectsRegionsClustersCreateRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, cluster=cluster)) if args. async: log.status.write('Creating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation = util.WaitForOperation( operation, self.context, 'Waiting for cluster creation operation') get_request = messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = client.projects_regions_clusters.Get(get_request) if cluster.status.state == ( messages.ClusterStatus.StateValueValuesEnum.RUNNING): log.CreatedResource(cluster_ref) else: log.error('Create cluster failed!') if operation.details: log.error('Details:\n' + operation.details) return cluster
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ wp_region = args.region release_track = self.ReleaseTrack() client = cloudbuild_util.GetClientInstance(release_track) messages = cloudbuild_util.GetMessagesModule(release_track) parent = properties.VALUES.core.project.Get(required=True) # Get the workerpool proto from either the flags or the specified file. wp = messages.WorkerPool() if args.config_from_file is not None: wp = workerpool_config.LoadWorkerpoolConfigFromPath( args.config_from_file, messages) else: wp.name = args.WORKER_POOL if args.peered_network is not None: network_config = messages.NetworkConfig() network_config.peeredNetwork = args.peered_network wp.networkConfig = network_config worker_config = messages.WorkerConfig() if args.worker_machine_type is not None: worker_config.machineType = args.worker_machine_type if args.worker_disk_size is not None: worker_config.diskSizeGb = compute_utils.BytesToGb( args.worker_disk_size) if args.no_external_ip: worker_config.noExternalIp = True wp.workerConfig = worker_config # Get the workerpool ref wp_resource = resources.REGISTRY.Parse( None, collection='cloudbuild.projects.locations.workerPools', api_version=cloudbuild_util. RELEASE_TRACK_TO_API_VERSION[release_track], params={ 'projectsId': parent, 'locationsId': wp_region, 'workerPoolsId': wp.name, }) update_mask = cloudbuild_util.MessageToFieldPaths(wp) req = messages.CloudbuildProjectsLocationsWorkerPoolsPatchRequest( name=wp_resource.RelativeName(), workerPool=wp, updateMask=','.join(update_mask)) # Send the Update request updated_op = client.projects_locations_workerPools.Patch(req) op_resource = resources.REGISTRY.ParseRelativeName( updated_op.name, collection='cloudbuild.projects.locations.operations') updated_wp = waiter.WaitFor( waiter.CloudOperationPoller(client.projects_locations_workerPools, client.projects_locations_operations), op_resource, 'Updating worker pool') log.UpdatedResource(wp_resource) # Format the workerpool name for display try: updated_wp.name = cloudbuild_util.RegionalWorkerPoolShortName( updated_wp.name) except ValueError: pass # Must be an old version. return updated_wp
def Run(self, args): compute_holder = base_classes.ComputeApiHolder(self.ReleaseTrack()) client = compute_holder.client disk_refs = self.ValidateAndParse(args, compute_holder) size_gb = utils.BytesToGb(args.size) from_image = args.image or args.image_family if not size_gb and not args.source_snapshot and not from_image: if args.type and 'pd-ssd' in args.type: size_gb = constants.DEFAULT_SSD_DISK_SIZE_GB else: size_gb = constants.DEFAULT_STANDARD_DISK_SIZE_GB utils.WarnIfDiskSizeIsTooSmall(size_gb, args.type) requests = [] # Check if the zone is deprecated or has maintenance coming. zone_resource_fetcher = zone_utils.ZoneResourceFetcher(client) zone_resource_fetcher.WarnForZonalCreation( (ref for ref in disk_refs if ref.Collection() == 'compute.disks')) # Check if the region is deprecated or has maintenance coming. region_resource_fetcher = region_utils.RegionResourceFetcher(client) region_resource_fetcher.WarnForRegionalCreation( (ref for ref in disk_refs if ref.Collection() == 'compute.regionDisks')) project_to_source_image = {} image_expander = image_utils.ImageExpander(client, compute_holder.resources) for disk_ref in disk_refs: if from_image: if disk_ref.project not in project_to_source_image: source_image_uri, _ = image_expander.ExpandImageFlag( user_project=disk_ref.project, image=args.image, image_family=args.image_family, image_project=args.image_project, return_image_resource=False) project_to_source_image[ disk_ref.project] = argparse.Namespace() project_to_source_image[ disk_ref.project].uri = source_image_uri else: project_to_source_image[ disk_ref.project] = argparse.Namespace() project_to_source_image[disk_ref.project].uri = None snapshot_ref = disks_flags.SOURCE_SNAPSHOT_ARG.ResolveAsResource( args, compute_holder.resources) if snapshot_ref: snapshot_uri = snapshot_ref.SelfLink() else: snapshot_uri = None # This feature is only exposed in alpha/beta allow_rsa_encrypted = self.ReleaseTrack() in [ base.ReleaseTrack.ALPHA, base.ReleaseTrack.BETA ] csek_keys = csek_utils.CsekKeyStore.FromArgs(args, allow_rsa_encrypted) for project in project_to_source_image: source_image_uri = project_to_source_image[project].uri project_to_source_image[project].keys = ( csek_utils.MaybeLookupKeyMessagesByUri( csek_keys, compute_holder.resources, [source_image_uri, snapshot_uri], client.apitools_client)) for disk_ref in disk_refs: if args.type: if disk_ref.Collection() == 'compute.disks': type_ref = compute_holder.resources.Parse( args.type, collection='compute.diskTypes', params={ 'project': disk_ref.project, 'zone': disk_ref.zone }) elif disk_ref.Collection() == 'compute.regionDisks': type_ref = compute_holder.resources.Parse( args.type, collection='compute.regionDiskTypes', params={ 'project': disk_ref.project, 'region': disk_ref.region }) type_uri = type_ref.SelfLink() else: type_uri = None if csek_keys: disk_key_or_none = csek_keys.LookupKey( disk_ref, args.require_csek_key_create) disk_key_message_or_none = csek_utils.MaybeToMessage( disk_key_or_none, client.apitools_client) kwargs = { 'diskEncryptionKey': disk_key_message_or_none, 'sourceImageEncryptionKey': project_to_source_image[disk_ref.project].keys[0], 'sourceSnapshotEncryptionKey': project_to_source_image[disk_ref.project].keys[1] } else: kwargs = {} if disk_ref.Collection() == 'compute.disks': request = client.messages.ComputeDisksInsertRequest( disk=client.messages.Disk(name=disk_ref.Name(), description=args.description, sizeGb=size_gb, sourceSnapshot=snapshot_uri, type=type_uri, **kwargs), project=disk_ref.project, sourceImage=project_to_source_image[disk_ref.project].uri, zone=disk_ref.zone) request = (client.apitools_client.disks, 'Insert', request) elif disk_ref.Collection() == 'compute.regionDisks': def SelfLink(zone, disk_ref): return compute_holder.resources.Parse( zone, collection='compute.zones', params={ 'project': disk_ref.project }).SelfLink() zones = [ SelfLink(zone, disk_ref) for zone in args.replica_zones ] request = client.messages.ComputeRegionDisksInsertRequest( disk=client.messages.Disk(name=disk_ref.Name(), description=args.description, sizeGb=size_gb, sourceSnapshot=snapshot_uri, type=type_uri, replicaZones=zones, **kwargs), project=disk_ref.project, sourceImage=project_to_source_image[disk_ref.project].uri, region=disk_ref.region) request = (client.apitools_client.regionDisks, 'Insert', request) requests.append(request) return client.MakeRequests(requests)