def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = util.ParseCluster(args.name, dataproc) request = dataproc.messages.DataprocProjectsRegionsClustersDeleteRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId) console_io.PromptContinue( message="The cluster '{0}' and all attached disks will be " 'deleted.'.format(args.name), cancel_on_no=True, cancel_string='Deletion aborted by user.') operation = dataproc.client.projects_regions_clusters.Delete(request) if args. async: log.status.write('Deleting [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster deletion operation', timeout_s=args.timeout) log.DeletedResource(cluster_ref) return operation
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = args.CONCEPTS.cluster.Parse() stop_cluster_request = dataproc.messages.StopClusterRequest( requestId=util.GetUniqueId()) request = dataproc.messages.DataprocProjectsRegionsClustersStopRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, stopClusterRequest=stop_cluster_request) console_io.PromptContinue( message="Cluster '{0}' is stopping.".format( cluster_ref.clusterName), cancel_on_no=True, cancel_string='Stopping cluster aborted by user.') operation = dataproc.client.projects_regions_clusters.Stop(request) if args.async_: log.status.write('Stopping [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster to stop.', timeout_s=args.timeout) return operation
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) request = messages.DataprocProjectsRegionsClustersDeleteRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId) console_io.PromptContinue( message="The cluster '{0}' and all attached disks will be " 'deleted.'.format(args.name), cancel_on_no=True, cancel_string='Deletion aborted by user.') operation = client.projects_regions_clusters.Delete(request) if args. async: log.status.write('Deleting [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation operation = util.WaitForOperation( operation, self.context, 'Waiting for cluster deletion operation') log.DeletedResource(cluster_ref) return operation
def Run(self, args): dataproc = dp.Dataproc() session_id = args.CONCEPTS.session.Parse() console_io.PromptContinue( message="The session '{0}' will be terminated.".format( session_id.Name()), cancel_on_no=True, cancel_string='Termination canceled by user.') request = dataproc.messages.DataprocProjectsLocationsSessionsTerminateRequest( name=session_id.RelativeName()) operation = dataproc.client.projects_locations_sessions.Terminate( request) if args.async_: log.status.write("Terminating session '{0}'".format( session_id.Name())) return operation operation = util.WaitForOperation( dataproc, operation, message="Waiting for session '{0}' to terminate.".format( session_id.Name()), timeout_s=args.timeout) log.DeletedResource(session_id.RelativeName()) return operation
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = args.CONCEPTS.cluster.Parse() start_cluster_request = dataproc.messages.StartClusterRequest( requestId=util.GetUniqueId()) request = dataproc.messages.DataprocProjectsRegionsClustersStartRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, startClusterRequest=start_cluster_request) operation = dataproc.client.projects_regions_clusters.Start(request) if args.async_: log.status.write('Starting [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation operation = util.WaitForOperation( dataproc, operation, message="Waiting for cluster '{0}' to start.".format( cluster_ref.clusterName), timeout_s=args.timeout) return operation
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = args.CONCEPTS.cluster.Parse() request = None if args.tarball_access is not None: tarball_access = arg_utils.ChoiceToEnum( args.tarball_access, dataproc.messages.DiagnoseClusterRequest. TarballAccessValueValuesEnum) diagnose_request = dataproc.messages.DiagnoseClusterRequest( tarballAccess=tarball_access) request = dataproc.messages.DataprocProjectsRegionsClustersDiagnoseRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, diagnoseClusterRequest=diagnose_request) else: request = dataproc.messages.DataprocProjectsRegionsClustersDiagnoseRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId) operation = dataproc.client.projects_regions_clusters.Diagnose(request) # TODO(b/36052522): Stream output during polling. operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster diagnose operation', timeout_s=args.timeout) if not operation.response: raise exceptions.OperationError('Operation is missing response') properties = encoding.MessageToDict(operation.response) output_uri = properties['outputUri'] if not output_uri: raise exceptions.OperationError('Response is missing outputUri') log.err.Print('Output from diagnostic:') log.err.Print('-----------------------------------------------') driver_log_stream = storage_helpers.StorageObjectSeriesStream( output_uri) # A single read might not read whole stream. Try a few times. read_retrier = retry.Retryer(max_retrials=4, jitter_ms=None) try: read_retrier.RetryOnResult( lambda: driver_log_stream.ReadIntoWritable(log.err), sleep_ms=100, should_retry_if=lambda *_: driver_log_stream.open) except retry.MaxRetrialsException: log.warning('Diagnostic finished successfully, ' 'but output did not finish streaming.') log.err.Print('-----------------------------------------------') return output_uri
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) cluster_config = messages.ClusterConfiguration() changed_fields = [] has_changes = False if args.new_num_workers is not None: log.warn( '--new-num-workers parameter is deprecated and will be removed ' 'in a future release. Please use --num-workers instead') args.num_workers = args.new_num_workers if args.num_workers is not None: worker_config = messages.InstanceGroupConfiguration( numInstances=args.num_workers) cluster_config.workerConfiguration = worker_config changed_fields.append( 'configuration.worker_configuration.num_instances') has_changes = True if args.num_preemptible_workers is not None: worker_config = messages.InstanceGroupConfiguration( numInstances=args.num_preemptible_workers) cluster_config.secondaryWorkerConfiguration = worker_config changed_fields.append( 'configuration.secondary_worker_configuration.num_instances') has_changes = True if not has_changes: raise exceptions.ToolException( 'Must specify at least one cluster parameter to update.') cluster = messages.Cluster(configuration=cluster_config, clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) request = messages.DataprocProjectsClustersPatchRequest( clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId, cluster=cluster, updateMask=','.join(changed_fields)) operation = client.projects_clusters.Patch(request) util.WaitForOperation(operation, self.context, message='Waiting for cluster update operation', timeout_s=3600 * 3) cluster = client.projects_clusters.Get(cluster_ref.Request()) log.UpdatedResource(cluster_ref) return cluster
def CreateCluster(dataproc, cluster, is_async, timeout): """Create a cluster. Args: dataproc: Dataproc object that contains client, messages, and resources cluster: Cluster to create is_async: Whether to wait for the operation to complete timeout: Timeout used when waiting for the operation to complete Returns: Created cluster, or None if async """ # Get project id and region. cluster_ref = util.ParseCluster(cluster.clusterName, dataproc) request_id = util.GetUniqueId() request = dataproc.messages.DataprocProjectsRegionsClustersCreateRequest( cluster=cluster, projectId=cluster_ref.projectId, region=cluster_ref.region, requestId=request_id) operation = dataproc.client.projects_regions_clusters.Create(request) if is_async: log.status.write('Creating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster creation operation', timeout_s=timeout) get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(get_request) if cluster.status.state == ( dataproc.messages.ClusterStatus.StateValueValuesEnum.RUNNING): zone_uri = cluster.config.gceClusterConfig.zoneUri zone_short_name = zone_uri.split('/')[-1] # Log the URL of the cluster log.CreatedResource( cluster_ref, # Also indicate which zone the cluster was placed in. This is helpful # if the server picked a zone (auto zone) details='Cluster placed in zone [{0}]'.format(zone_short_name)) else: log.error('Create cluster failed!') if operation.details: log.error('Details:\n' + operation.details) return cluster
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) request = messages.DataprocProjectsRegionsClustersDiagnoseRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId) try: operation = client.projects_regions_clusters.Diagnose(request) # TODO(user): Stream output during polling. operation = util.WaitForOperation( operation, self.context, message='Waiting for cluster diagnose operation') response = operation.response except apitools_exceptions.HttpError as error: raise exceptions.HttpException(util.FormatHttpError(error)) if not response: raise exceptions.ToolException('Operation is missing response') properties = encoding.MessageToDict(response) output_uri = properties['outputUri'] if not output_uri: raise exceptions.ToolException('Response is missing outputUri') log.err.Print('Output from diagnostic:') log.err.Print('-----------------------------------------------') driver_log_stream = storage_helpers.StorageObjectSeriesStream( output_uri) # A single read might not read whole stream. Try a few times. read_retrier = retry.Retryer(max_retrials=4, jitter_ms=None) try: read_retrier.RetryOnResult( lambda: driver_log_stream.ReadIntoWritable(log.err), sleep_ms=100, should_retry_if=lambda *_: driver_log_stream.open) except retry.MaxRetrialsException: log.warn( 'Diagnostic finished succesfully, ' 'but output did not finish streaming.') log.err.Print('-----------------------------------------------') return output_uri
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) request = messages.DataprocProjectsClustersDeleteRequest( clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) if not console_io.PromptContinue( message="The cluster '{0}' and all attached disks will be " 'deleted.'.format(args.name)): raise exceptions.ToolException('Deletion aborted by user.') operation = client.projects_clusters.Delete(request) operation = util.WaitForOperation( operation, self.context, 'Waiting for cluster deletion operation') log.DeletedResource(cluster_ref) return operation
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) request = messages.DataprocProjectsClustersDiagnoseRequest( clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) try: operation = client.projects_clusters.Diagnose(request) operation = util.WaitForOperation( operation, self.context, message='Waiting for cluster diagnose operation') response = operation.response except apitools_base.HttpError as error: raise exceptions.HttpException(util.FormatHttpError(error)) if not response: raise exceptions.ToolException('Operation is missing response') properties = apitools_base.MessageToDict(response) output_uri = properties['outputUri'] if not output_uri: raise exceptions.ToolException('Response is missing outputUri') log.err.Print('Output from diagnostic:') log.err.Print('-----------------------------------------------') driver_log_stream = storage_helpers.StorageObjectSeriesStream( output_uri) driver_log_stream.ReadIntoWritable(log.err) log.err.Print('-----------------------------------------------') return output_uri
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = util.ParseCluster(args.name, dataproc) cluster_config = dataproc.messages.ClusterConfig() changed_fields = [] has_changes = False if args.num_workers is not None: worker_config = dataproc.messages.InstanceGroupConfig( numInstances=args.num_workers) cluster_config.workerConfig = worker_config changed_fields.append('config.worker_config.num_instances') has_changes = True if args.num_preemptible_workers is not None: worker_config = dataproc.messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers) cluster_config.secondaryWorkerConfig = worker_config changed_fields.append( 'config.secondary_worker_config.num_instances') has_changes = True if self.ReleaseTrack() == base.ReleaseTrack.BETA: lifecycle_config = dataproc.messages.LifecycleConfig() changed_config = False if args.max_age is not None: lifecycle_config.autoDeleteTtl = str(args.max_age) + 's' changed_config = True if args.expiration_time is not None: lifecycle_config.autoDeleteTime = times.FormatDateTime( args.expiration_time) changed_config = True if args.max_idle is not None: lifecycle_config.idleDeleteTtl = str(args.max_idle) + 's' changed_config = True if changed_config: cluster_config.lifecycleConfig = lifecycle_config changed_fields.append('config.lifecycle_config') has_changes = True # Update labels if the user requested it labels = None if args.update_labels or args.remove_labels: has_changes = True changed_fields.append('labels') # We need to fetch cluster first so we know what the labels look like. The # labels_util.UpdateLabels will fill out the proto for us with all the # updates and removals, but first we need to provide the current state # of the labels get_cluster_request = ( dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName)) current_cluster = dataproc.client.projects_regions_clusters.Get( get_cluster_request) labels = labels_util.UpdateLabels( current_cluster.labels, dataproc.messages.Cluster.LabelsValue, args.update_labels, args.remove_labels) if not has_changes: raise exceptions.ArgumentError( 'Must specify at least one cluster parameter to update.') cluster = dataproc.messages.Cluster( config=cluster_config, clusterName=cluster_ref.clusterName, labels=labels, projectId=cluster_ref.projectId) request = dataproc.messages.DataprocProjectsRegionsClustersPatchRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, cluster=cluster, updateMask=','.join(changed_fields)) if (self.ReleaseTrack() == base.ReleaseTrack.BETA and args.graceful_decommission_timeout): request.gracefulDecommissionTimeout = ( str(args.graceful_decommission_timeout) + 's') operation = dataproc.client.projects_regions_clusters.Patch(request) if args. async: log.status.write('Updating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return util.WaitForOperation(dataproc, operation, message='Waiting for cluster update operation', timeout_s=args.timeout) request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(request) log.UpdatedResource(cluster_ref) return cluster
def Run(self, args): self.ValidateArgs(args) client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) compute_resources = compute_helpers.GetComputeResources( self.ReleaseTrack(), args.name) main_accelerator_type = None worker_accelerator_type = None main_accelerator_count = None worker_accelerator_count = None if self.ReleaseTrack() == base.ReleaseTrack.BETA: if args.main_accelerator: main_accelerator_type = args.main_accelerator['type'] main_accelerator_count = args.main_accelerator.get('count', 1) if args.worker_accelerator: worker_accelerator_type = args.worker_accelerator['type'] worker_accelerator_count = args.worker_accelerator.get( 'count', 1) # Resolve GCE resources zone_ref = compute_resources.Parse(None, collection='compute.zones') image_ref = args.image and compute_resources.Parse( args.image, collection='compute.images') main_machine_type_ref = (args.main_machine_type and compute_resources.Parse( args.main_machine_type, collection='compute.machineTypes')) worker_machine_type_ref = (args.worker_machine_type and compute_resources.Parse( args.worker_machine_type, collection='compute.machineTypes')) network_ref = args.network and compute_resources.Parse( args.network, collection='compute.networks') subnetwork_ref = args.subnet and compute_resources.Parse( args.subnet, collection='compute.subnetworks') main_accelerator_type_ref = ( main_accelerator_type and compute_resources.Parse( main_accelerator_type, collection='compute.acceleratorTypes')) worker_accelerator_type_ref = ( worker_accelerator_type and compute_resources.Parse(worker_accelerator_type, collection='compute.acceleratorTypes')) init_actions = [] timeout_str = str(args.initialization_action_timeout) + 's' if args.initialization_actions: init_actions = [ messages.NodeInitializationAction(executableFile=exe, executionTimeout=timeout_str) for exe in args.initialization_actions ] expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes) software_config = messages.SoftwareConfig( imageVersion=args.image_version) main_boot_disk_size_gb = args.main_boot_disk_size_gb if args.main_boot_disk_size: main_boot_disk_size_gb = (api_utils.BytesToGb( args.main_boot_disk_size)) worker_boot_disk_size_gb = args.worker_boot_disk_size_gb if args.worker_boot_disk_size: worker_boot_disk_size_gb = (api_utils.BytesToGb( args.worker_boot_disk_size)) preemptible_worker_boot_disk_size_gb = (api_utils.BytesToGb( args.preemptible_worker_boot_disk_size)) if args.single_node: args.properties[constants.ALLOW_ZERO_WORKERS_PROPERTY] = 'true' if args.properties: software_config.properties = encoding.DictToMessage( args.properties, messages.SoftwareConfig.PropertiesValue) gce_cluster_config = messages.GceClusterConfig( networkUri=network_ref and network_ref.SelfLink(), subnetworkUri=subnetwork_ref and subnetwork_ref.SelfLink(), serviceAccount=args.service_account, serviceAccountScopes=expanded_scopes, zoneUri=zone_ref and zone_ref.SelfLink()) if args.tags: gce_cluster_config.tags = args.tags if args.metadata: flat_metadata = dict( (k, v) for d in args.metadata for k, v in d.items()) gce_cluster_config.metadata = encoding.DictToMessage( flat_metadata, messages.GceClusterConfig.MetadataValue) main_accelerators = [] if main_accelerator_type: main_accelerators.append( messages.AcceleratorConfig( acceleratorTypeUri=main_accelerator_type_ref and main_accelerator_type_ref.SelfLink(), acceleratorCount=main_accelerator_count)) worker_accelerators = [] if worker_accelerator_type: worker_accelerators.append( messages.AcceleratorConfig( acceleratorTypeUri=worker_accelerator_type_ref and worker_accelerator_type_ref.SelfLink(), acceleratorCount=worker_accelerator_count)) cluster_config = messages.ClusterConfig( configBucket=args.bucket, gceClusterConfig=gce_cluster_config, mainConfig=messages.InstanceGroupConfig( numInstances=args.num_mains, imageUri=image_ref and image_ref.SelfLink(), machineTypeUri=main_machine_type_ref and main_machine_type_ref.SelfLink(), accelerators=main_accelerators, diskConfig=messages.DiskConfig( bootDiskSizeGb=main_boot_disk_size_gb, numLocalSsds=args.num_main_local_ssds, ), ), workerConfig=messages.InstanceGroupConfig( numInstances=args.num_workers, imageUri=image_ref and image_ref.SelfLink(), machineTypeUri=worker_machine_type_ref and worker_machine_type_ref.SelfLink(), accelerators=worker_accelerators, diskConfig=messages.DiskConfig( bootDiskSizeGb=worker_boot_disk_size_gb, numLocalSsds=args.num_worker_local_ssds, ), ), initializationActions=init_actions, softwareConfig=software_config, ) # Secondary worker group is optional. However, users may specify # future pVM disk size at creation time. if (args.num_preemptible_workers is not None or preemptible_worker_boot_disk_size_gb is not None): cluster_config.secondaryWorkerConfig = ( messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers, diskConfig=messages.DiskConfig( bootDiskSizeGb=preemptible_worker_boot_disk_size_gb, )) ) cluster = messages.Cluster(config=cluster_config, clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) self.ConfigureCluster(messages, args, cluster) operation = client.projects_regions_clusters.Create( messages.DataprocProjectsRegionsClustersCreateRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, cluster=cluster)) if args. async: log.status.write('Creating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation = util.WaitForOperation( operation, self.context, 'Waiting for cluster creation operation') get_request = messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = client.projects_regions_clusters.Get(get_request) if cluster.status.state == ( messages.ClusterStatus.StateValueValuesEnum.RUNNING): log.CreatedResource(cluster_ref) else: log.error('Create cluster failed!') if operation.details: log.error('Details:\n' + operation.details) return cluster
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) cluster_config = messages.ClusterConfig() changed_fields = [] has_changes = False if args.num_workers is not None: worker_config = messages.InstanceGroupConfig( numInstances=args.num_workers) cluster_config.workerConfig = worker_config changed_fields.append('config.worker_config.num_instances') has_changes = True if args.num_preemptible_workers is not None: worker_config = messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers) cluster_config.secondaryWorkerConfig = worker_config changed_fields.append( 'config.secondary_worker_config.num_instances') has_changes = True # Update labels if the user requested it labels = None if args.update_labels or args.remove_labels: has_changes = True changed_fields.append('labels') # We need to fetch cluster first so we know what the labels look like. The # labels_util.UpdateLabels will fill out the proto for us with all the # updates and removals, but first we need to provide the current state # of the labels get_cluster_request = (client.MESSAGES_MODULE. DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName)) current_cluster = client.projects_regions_clusters.Get( get_cluster_request) labels = labels_util.UpdateLabels(current_cluster.labels, messages.Cluster.LabelsValue, args.update_labels, args.remove_labels) if not has_changes: raise exceptions.ArgumentError( 'Must specify at least one cluster parameter to update.') cluster = messages.Cluster(config=cluster_config, clusterName=cluster_ref.clusterName, labels=labels, projectId=cluster_ref.projectId) request = messages.DataprocProjectsRegionsClustersPatchRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, cluster=cluster, updateMask=','.join(changed_fields)) operation = client.projects_regions_clusters.Patch(request) if args. async: log.status.write('Updating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return util.WaitForOperation(operation, self.context, message='Waiting for cluster update operation', timeout_s=3600 * 3) request = client.MESSAGES_MODULE.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = client.projects_regions_clusters.Get(request) log.UpdatedResource(cluster_ref) return cluster
def Run(self, args): self.ValidateArgs(args) client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) config_helper = compute_helpers.ConfigurationHelper.FromContext( self.context) compute_uris = config_helper.ResolveGceUris(args.name, args.image, args.master_machine_type, args.worker_machine_type, args.network, args.subnet) init_actions = [] timeout_str = str(args.initialization_action_timeout) + 's' if args.initialization_actions: init_actions = [ messages.NodeInitializationAction(executableFile=exe, executionTimeout=timeout_str) for exe in args.initialization_actions ] expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes) software_config = messages.SoftwareConfig( imageVersion=args.image_version) master_boot_disk_size_gb = args.master_boot_disk_size_gb if args.master_boot_disk_size: master_boot_disk_size_gb = (api_utils.BytesToGb( args.master_boot_disk_size)) worker_boot_disk_size_gb = args.worker_boot_disk_size_gb if args.worker_boot_disk_size: worker_boot_disk_size_gb = (api_utils.BytesToGb( args.worker_boot_disk_size)) preemptible_worker_boot_disk_size_gb = (api_utils.BytesToGb( args.preemptible_worker_boot_disk_size)) if args.properties: software_config.properties = encoding.DictToMessage( args.properties, messages.SoftwareConfig.PropertiesValue) gce_cluster_config = messages.GceClusterConfig( networkUri=compute_uris['network'], subnetworkUri=compute_uris['subnetwork'], serviceAccountScopes=expanded_scopes, zoneUri=compute_uris['zone']) if args.tags: gce_cluster_config.tags = args.tags if args.metadata: flat_metadata = dict( (k, v) for d in args.metadata for k, v in d.items()) gce_cluster_config.metadata = encoding.DictToMessage( flat_metadata, messages.GceClusterConfig.MetadataValue) cluster_config = messages.ClusterConfig( configBucket=args.bucket, gceClusterConfig=gce_cluster_config, masterConfig=messages.InstanceGroupConfig( numInstances=args.num_masters, imageUri=compute_uris['image'], machineTypeUri=compute_uris['master_machine_type'], diskConfig=messages.DiskConfig( bootDiskSizeGb=master_boot_disk_size_gb, numLocalSsds=args.num_master_local_ssds, ), ), workerConfig=messages.InstanceGroupConfig( numInstances=args.num_workers, imageUri=compute_uris['image'], machineTypeUri=compute_uris['worker_machine_type'], diskConfig=messages.DiskConfig( bootDiskSizeGb=worker_boot_disk_size_gb, numLocalSsds=args.num_worker_local_ssds, ), ), initializationActions=init_actions, softwareConfig=software_config, ) # Secondary worker group is optional. However, users may specify # future pVM disk size at creation time. if (args.num_preemptible_workers is not None or preemptible_worker_boot_disk_size_gb is not None): cluster_config.secondaryWorkerConfig = ( messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers, diskConfig=messages.DiskConfig( bootDiskSizeGb=preemptible_worker_boot_disk_size_gb, )) ) cluster = messages.Cluster(config=cluster_config, clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) operation = client.projects_regions_clusters.Create( messages.DataprocProjectsRegionsClustersCreateRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, cluster=cluster)) if args. async: log.status.write('Creating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation = util.WaitForOperation( operation, self.context, 'Waiting for cluster creation operation') cluster = client.projects_regions_clusters.Get(cluster_ref.Request()) if cluster.status.state == ( messages.ClusterStatus.StateValueValuesEnum.RUNNING): log.CreatedResource(cluster_ref) else: log.error('Create cluster failed!') if operation.details: log.error('Details:\n' + operation.details) return cluster
def Run(self, args): self.ValidateArgs(args) dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = util.ParseCluster(args.name, dataproc) compute_resources = compute_helpers.GetComputeResources( self.ReleaseTrack(), args.name) beta = self.ReleaseTrack() == base.ReleaseTrack.BETA cluster_config = clusters.GetClusterConfig(args, dataproc, cluster_ref.projectId, compute_resources, beta) cluster = dataproc.messages.Cluster( config=cluster_config, clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) self.ConfigureCluster(dataproc.messages, args, cluster) operation = dataproc.client.projects_regions_clusters.Create( dataproc.messages.DataprocProjectsRegionsClustersCreateRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, cluster=cluster)) if args. async: log.status.write('Creating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster creation operation', timeout_s=args.timeout) get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(get_request) if cluster.status.state == ( dataproc.messages.ClusterStatus.StateValueValuesEnum.RUNNING): zone_uri = cluster.config.gceClusterConfig.zoneUri zone_short_name = zone_uri.split('/')[-1] # Log the URL of the cluster log.CreatedResource( cluster_ref, # Also indicate which zone the cluster was placed in. This is helpful # if the server picked a zone (auto zone) details='Cluster placed in zone [{0}]'.format(zone_short_name)) else: log.error('Create cluster failed!') if operation.details: log.error('Details:\n' + operation.details) return cluster
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = util.ParseCluster(args.name, dataproc) cluster_config = dataproc.messages.ClusterConfig() changed_fields = [] has_changes = False if args.num_workers is not None: worker_config = dataproc.messages.InstanceGroupConfig( numInstances=args.num_workers) cluster_config.workerConfig = worker_config changed_fields.append('config.worker_config.num_instances') has_changes = True if args.num_preemptible_workers is not None: worker_config = dataproc.messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers) cluster_config.secondaryWorkerConfig = worker_config changed_fields.append( 'config.secondary_worker_config.num_instances') has_changes = True if self.ReleaseTrack() == base.ReleaseTrack.BETA: if args.autoscaling_policy: cluster_config.autoscalingConfig = dataproc.messages.AutoscalingConfig( policyUri=args.CONCEPTS.autoscaling_policy.Parse( ).RelativeName()) changed_fields.append('config.autoscaling_config.policy_uri') has_changes = True elif args.autoscaling_policy == '' or args.disable_autoscaling: # pylint: disable=g-explicit-bool-comparison # Disabling autoscaling. Don't need to explicitly set # cluster_config.autoscaling_config to None. changed_fields.append('config.autoscaling_config.policy_uri') has_changes = True lifecycle_config = dataproc.messages.LifecycleConfig() changed_config = False if args.max_age is not None: lifecycle_config.autoDeleteTtl = str(args.max_age) + 's' changed_fields.append( 'config.lifecycle_config.auto_delete_ttl') changed_config = True if args.expiration_time is not None: lifecycle_config.autoDeleteTime = times.FormatDateTime( args.expiration_time) changed_fields.append( 'config.lifecycle_config.auto_delete_time') changed_config = True if args.max_idle is not None: lifecycle_config.idleDeleteTtl = str(args.max_idle) + 's' changed_fields.append( 'config.lifecycle_config.idle_delete_ttl') changed_config = True if args.no_max_age: lifecycle_config.autoDeleteTtl = None changed_fields.append( 'config.lifecycle_config.auto_delete_ttl') changed_config = True if args.no_max_idle: lifecycle_config.idleDeleteTtl = None changed_fields.append( 'config.lifecycle_config.idle_delete_ttl') changed_config = True if changed_config: cluster_config.lifecycleConfig = lifecycle_config has_changes = True # Put in a thunk so we only make this call if needed def _GetCurrentLabels(): # We need to fetch cluster first so we know what the labels look like. The # labels_util will fill out the proto for us with all the updates and # removals, but first we need to provide the current state of the labels get_cluster_request = ( dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName)) current_cluster = dataproc.client.projects_regions_clusters.Get( get_cluster_request) return current_cluster.labels labels_update = labels_util.ProcessUpdateArgsLazy( args, dataproc.messages.Cluster.LabelsValue, orig_labels_thunk=_GetCurrentLabels) if labels_update.needs_update: has_changes = True changed_fields.append('labels') labels = labels_update.GetOrNone() if not has_changes: raise exceptions.ArgumentError( 'Must specify at least one cluster parameter to update.') cluster = dataproc.messages.Cluster( config=cluster_config, clusterName=cluster_ref.clusterName, labels=labels, projectId=cluster_ref.projectId) request = dataproc.messages.DataprocProjectsRegionsClustersPatchRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, cluster=cluster, updateMask=','.join(changed_fields), requestId=util.GetUniqueId()) if args.graceful_decommission_timeout is not None: request.gracefulDecommissionTimeout = ( str(args.graceful_decommission_timeout) + 's') operation = dataproc.client.projects_regions_clusters.Patch(request) if args. async: log.status.write('Updating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return util.WaitForOperation(dataproc, operation, message='Waiting for cluster update operation', timeout_s=args.timeout) request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(request) log.UpdatedResource(cluster_ref) return cluster
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = args.CONCEPTS.cluster.Parse() cluster_config = dataproc.messages.ClusterConfig() changed_fields = [] has_changes = False if args.num_workers is not None: worker_config = dataproc.messages.InstanceGroupConfig( numInstances=args.num_workers) cluster_config.workerConfig = worker_config changed_fields.append('config.worker_config.num_instances') has_changes = True num_secondary_workers = _FirstNonNone(args.num_preemptible_workers, args.num_secondary_workers) if num_secondary_workers is not None: worker_config = dataproc.messages.InstanceGroupConfig( numInstances=num_secondary_workers) cluster_config.secondaryWorkerConfig = worker_config changed_fields.append( 'config.secondary_worker_config.num_instances') has_changes = True if args.autoscaling_policy: cluster_config.autoscalingConfig = dataproc.messages.AutoscalingConfig( policyUri=args.CONCEPTS.autoscaling_policy.Parse( ).RelativeName()) changed_fields.append('config.autoscaling_config.policy_uri') has_changes = True elif args.autoscaling_policy == '' or args.disable_autoscaling: # pylint: disable=g-explicit-bool-comparison # Disabling autoscaling. Don't need to explicitly set # cluster_config.autoscaling_config to None. changed_fields.append('config.autoscaling_config.policy_uri') has_changes = True lifecycle_config = dataproc.messages.LifecycleConfig() changed_config = False if args.max_age is not None: lifecycle_config.autoDeleteTtl = six.text_type(args.max_age) + 's' changed_fields.append('config.lifecycle_config.auto_delete_ttl') changed_config = True if args.expiration_time is not None: lifecycle_config.autoDeleteTime = times.FormatDateTime( args.expiration_time) changed_fields.append('config.lifecycle_config.auto_delete_time') changed_config = True if args.max_idle is not None: lifecycle_config.idleDeleteTtl = six.text_type(args.max_idle) + 's' changed_fields.append('config.lifecycle_config.idle_delete_ttl') changed_config = True if args.no_max_age: lifecycle_config.autoDeleteTtl = None changed_fields.append('config.lifecycle_config.auto_delete_ttl') changed_config = True if args.no_max_idle: lifecycle_config.idleDeleteTtl = None changed_fields.append('config.lifecycle_config.idle_delete_ttl') changed_config = True if changed_config: cluster_config.lifecycleConfig = lifecycle_config has_changes = True def _GetCurrentCluster(): # This is used for labels and auxiliary_node_pool_configs get_cluster_request = ( dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName)) current_cluster = dataproc.client.projects_regions_clusters.Get( get_cluster_request) return current_cluster # Put in a thunk so we only make this call if needed def _GetCurrentLabels(): # We need to fetch cluster first so we know what the labels look like. The # labels_util will fill out the proto for us with all the updates and # removals, but first we need to provide the current state of the labels current_cluster = _GetCurrentCluster() return current_cluster.labels labels_update = labels_util.ProcessUpdateArgsLazy( args, dataproc.messages.Cluster.LabelsValue, orig_labels_thunk=_GetCurrentLabels) if labels_update.needs_update: has_changes = True changed_fields.append('labels') labels = labels_update.GetOrNone() if args.driver_pool_size is not None: # Getting the node_pool_ids from the current node_pools and other attrs # that are not shared with the user # Driver pools can only be updated currently with NO other updates # We are relying on our frontend validation to prevent this until # the change is made to allow driver pools to be updated with other fields auxiliary_node_pools = _GetCurrentCluster( ).config.auxiliaryNodePoolConfigs # get the index of the current cluster's driver pool in the auxiliary # node pools list, index_driver_pools is also a list that should have a # length of 1 index_driver_pools = [ i for i, n in enumerate(auxiliary_node_pools) if dataproc.messages.NodePoolConfig. RolesValueListEntryValuesEnum.DRIVER in n.roles ] if len(index_driver_pools) > 1: raise exceptions.ArgumentError( 'At most one driver pool can be specified per cluster.') elif len(index_driver_pools) == 1: index = index_driver_pools[0] auxiliary_node_pools[ index].nodePoolConfig.numInstances = args.driver_pool_size else: # This case is only relevant for scaling from 0 -> N nodes # this will not be supported initially, but will be relying on our # front end validation to prevent or allow worker_config = dataproc.messages.InstanceGroupConfig( numInstances=args.driver_pool_size) node_config = dataproc.messages.NodePoolConfig( nodePoolConfig=worker_config, roles=[ dataproc.messages.NodePoolConfig. RolesValueListEntryValuesEnum.DRIVER ]) auxiliary_node_pools.append(node_config) cluster_config.auxiliaryNodePoolConfigs = auxiliary_node_pools changed_fields.append('config.auxiliary_node_pool_configs') has_changes = True if not has_changes: raise exceptions.ArgumentError( 'Must specify at least one cluster parameter to update.') cluster = dataproc.messages.Cluster( config=cluster_config, clusterName=cluster_ref.clusterName, labels=labels, projectId=cluster_ref.projectId) request = dataproc.messages.DataprocProjectsRegionsClustersPatchRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, cluster=cluster, updateMask=','.join(changed_fields), requestId=util.GetUniqueId()) if args.graceful_decommission_timeout is not None: request.gracefulDecommissionTimeout = ( six.text_type(args.graceful_decommission_timeout) + 's') operation = dataproc.client.projects_regions_clusters.Patch(request) if args.async_: log.status.write('Updating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return util.WaitForOperation(dataproc, operation, message='Waiting for cluster update operation', timeout_s=args.timeout) request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(request) log.UpdatedResource(cluster_ref) return cluster
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) cluster_config = messages.ClusterConfig() changed_fields = [] has_changes = False if args.new_num_workers is not None: log.warn( '--new-num-workers parameter is deprecated and will be removed ' 'in a future release. Please use --num-workers instead') args.num_workers = args.new_num_workers if args.num_workers is not None: worker_config = messages.InstanceGroupConfig( numInstances=args.num_workers) cluster_config.workerConfig = worker_config changed_fields.append('config.worker_config.num_instances') has_changes = True if args.num_preemptible_workers is not None: worker_config = messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers) cluster_config.secondaryWorkerConfig = worker_config changed_fields.append( 'config.secondary_worker_config.num_instances') has_changes = True if not has_changes: raise exceptions.ArgumentError( 'Must specify at least one cluster parameter to update.') cluster = messages.Cluster(config=cluster_config, clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) request = messages.DataprocProjectsRegionsClustersPatchRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, cluster=cluster, updateMask=','.join(changed_fields)) operation = client.projects_regions_clusters.Patch(request) if args. async: log.status.write('Updating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return util.WaitForOperation(operation, self.context, message='Waiting for cluster update operation', timeout_s=3600 * 3) request = client.MESSAGES_MODULE.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = client.projects_regions_clusters.Get(request) log.UpdatedResource(cluster_ref) return cluster
def Run(self, args): self.ValidateArgs(args) dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = util.ParseCluster(args.name, dataproc) compute_resources = compute_helpers.GetComputeResources( self.ReleaseTrack(), args.name) master_accelerator_type = None worker_accelerator_type = None master_accelerator_count = None worker_accelerator_count = None if self.ReleaseTrack() == base.ReleaseTrack.BETA: if args.master_accelerator: master_accelerator_type = args.master_accelerator['type'] master_accelerator_count = args.master_accelerator.get('count', 1) if args.worker_accelerator: worker_accelerator_type = args.worker_accelerator['type'] worker_accelerator_count = args.worker_accelerator.get('count', 1) # Resolve non-zonal GCE resources # We will let the server resolve short names of zonal resources because # if auto zone is requested, we will not know the zone before sending the # request image_ref = args.image and compute_resources.Parse( args.image, params={'project': cluster_ref.projectId}, collection='compute.images') network_ref = args.network and compute_resources.Parse( args.network, params={'project': cluster_ref.projectId}, collection='compute.networks') subnetwork_ref = args.subnet and compute_resources.Parse( args.subnet, params={ 'project': cluster_ref.projectId, 'region': properties.VALUES.compute.region.GetOrFail, }, collection='compute.subnetworks') timeout_str = str(args.initialization_action_timeout) + 's' init_actions = [ dataproc.messages.NodeInitializationAction( executableFile=exe, executionTimeout=timeout_str) for exe in (args.initialization_actions or [])] # Increase the client timeout for each initialization action. args.timeout += args.initialization_action_timeout * len(init_actions) expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes) software_config = dataproc.messages.SoftwareConfig( imageVersion=args.image_version) master_boot_disk_size_gb = args.master_boot_disk_size_gb if args.master_boot_disk_size: master_boot_disk_size_gb = ( api_utils.BytesToGb(args.master_boot_disk_size)) worker_boot_disk_size_gb = args.worker_boot_disk_size_gb if args.worker_boot_disk_size: worker_boot_disk_size_gb = ( api_utils.BytesToGb(args.worker_boot_disk_size)) preemptible_worker_boot_disk_size_gb = ( api_utils.BytesToGb(args.preemptible_worker_boot_disk_size)) if args.single_node: args.properties[constants.ALLOW_ZERO_WORKERS_PROPERTY] = 'true' if args.properties: software_config.properties = encoding.DictToMessage( args.properties, dataproc.messages.SoftwareConfig.PropertiesValue) gce_cluster_config = dataproc.messages.GceClusterConfig( networkUri=network_ref and network_ref.SelfLink(), subnetworkUri=subnetwork_ref and subnetwork_ref.SelfLink(), internalIpOnly=args.no_address, serviceAccount=args.service_account, serviceAccountScopes=expanded_scopes, zoneUri=properties.VALUES.compute.zone.GetOrFail()) if args.tags: gce_cluster_config.tags = args.tags if args.metadata: flat_metadata = dict((k, v) for d in args.metadata for k, v in d.items()) gce_cluster_config.metadata = encoding.DictToMessage( flat_metadata, dataproc.messages.GceClusterConfig.MetadataValue) master_accelerators = [] if master_accelerator_type: master_accelerators.append( dataproc.messages.AcceleratorConfig( acceleratorTypeUri=master_accelerator_type, acceleratorCount=master_accelerator_count)) worker_accelerators = [] if worker_accelerator_type: worker_accelerators.append( dataproc.messages.AcceleratorConfig( acceleratorTypeUri=worker_accelerator_type, acceleratorCount=worker_accelerator_count)) cluster_config = dataproc.messages.ClusterConfig( configBucket=args.bucket, gceClusterConfig=gce_cluster_config, masterConfig=dataproc.messages.InstanceGroupConfig( numInstances=args.num_masters, imageUri=image_ref and image_ref.SelfLink(), machineTypeUri=args.master_machine_type, accelerators=master_accelerators, diskConfig=dataproc.messages.DiskConfig( bootDiskSizeGb=master_boot_disk_size_gb, numLocalSsds=args.num_master_local_ssds,),), workerConfig=dataproc.messages.InstanceGroupConfig( numInstances=args.num_workers, imageUri=image_ref and image_ref.SelfLink(), machineTypeUri=args.worker_machine_type, accelerators=worker_accelerators, diskConfig=dataproc.messages.DiskConfig( bootDiskSizeGb=worker_boot_disk_size_gb, numLocalSsds=args.num_worker_local_ssds,),), initializationActions=init_actions, softwareConfig=software_config,) # Secondary worker group is optional. However, users may specify # future pVM disk size at creation time. if (args.num_preemptible_workers is not None or preemptible_worker_boot_disk_size_gb is not None): cluster_config.secondaryWorkerConfig = ( dataproc.messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers, diskConfig=dataproc.messages.DiskConfig( bootDiskSizeGb=preemptible_worker_boot_disk_size_gb, ))) cluster = dataproc.messages.Cluster( config=cluster_config, clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) self.ConfigureCluster(dataproc.messages, args, cluster) operation = dataproc.client.projects_regions_clusters.Create( dataproc.messages.DataprocProjectsRegionsClustersCreateRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, cluster=cluster)) if args.async: log.status.write( 'Creating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster creation operation', timeout_s=args.timeout) get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(get_request) if cluster.status.state == ( dataproc.messages.ClusterStatus.StateValueValuesEnum.RUNNING): zone_uri = cluster.config.gceClusterConfig.zoneUri zone_short_name = zone_uri.split('/')[-1] # Log the URL of the cluster log.CreatedResource( cluster_ref, # Also indicate which zone the cluster was placed in. This is helpful # if the server picked a zone (auto zone) details='Cluster placed in zone [{0}]'.format(zone_short_name)) else: log.error('Create cluster failed!') if operation.details: log.error('Details:\n' + operation.details) return cluster
def CreateCluster(dataproc, cluster_ref, cluster, is_async, timeout, enable_create_on_gke=False): """Create a cluster. Args: dataproc: Dataproc object that contains client, messages, and resources cluster_ref: Full resource ref of cluster with name, region, and project id cluster: Cluster to create is_async: Whether to wait for the operation to complete timeout: Timeout used when waiting for the operation to complete enable_create_on_gke: Whether to enable creation of GKE-based clusters Returns: Created cluster, or None if async """ # Get project id and region. request_id = util.GetUniqueId() request = dataproc.messages.DataprocProjectsRegionsClustersCreateRequest( cluster=cluster, projectId=cluster_ref.projectId, region=cluster_ref.region, requestId=request_id) operation = dataproc.client.projects_regions_clusters.Create(request) if is_async: log.status.write('Creating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster creation operation', timeout_s=timeout) get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(get_request) if cluster.status.state == ( dataproc.messages.ClusterStatus.StateValueValuesEnum.RUNNING): if enable_create_on_gke and cluster.config.gkeClusterConfig is not None: log.CreatedResource( cluster_ref, details='Cluster created on GKE cluster {0}'.format( cluster.config.gkeClusterConfig. namespacedGkeDeploymentTarget.targetGkeCluster)) else: zone_uri = cluster.config.gceClusterConfig.zoneUri zone_short_name = zone_uri.split('/')[-1] # Log the URL of the cluster log.CreatedResource( cluster_ref, # Also indicate which zone the cluster was placed in. This is helpful # if the server picked a zone (auto zone) details='Cluster placed in zone [{0}]'.format(zone_short_name)) else: # The operation didn't have an error, but the cluster is not RUNNING. log.error('Create cluster failed!') if cluster.status.detail: log.error('Details:\n' + cluster.status.detail) return cluster
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] cluster_ref = util.ParseCluster(args.name, self.context) config_helper = compute_helpers.ConfigurationHelper.FromContext( self.context) compute_uris = config_helper.ResolveGceUris(args.name, args.image, args.master_machine_type, args.worker_machine_type, args.network) init_actions = [] timeout_str = str(args.initialization_action_timeout) + 's' if args.initialization_actions: init_actions = [ messages.NodeInitializationAction(executableFile=exe, executionTimeout=timeout_str) for exe in args.initialization_actions ] expanded_scopes = compute_helpers.ExpandScopeAliases(args.scopes) cluster_config = messages.ClusterConfiguration( configurationBucket=args.bucket, gceClusterConfiguration=messages.GceClusterConfiguration( networkUri=compute_uris['network'], serviceAccountScopes=expanded_scopes, zoneUri=compute_uris['zone'], ), masterConfiguration=messages.InstanceGroupConfiguration( imageUri=compute_uris['image'], machineTypeUri=compute_uris['master_machine_type'], diskConfiguration=messages.DiskConfiguration( bootDiskSizeGb=args.master_boot_disk_size_gb, numLocalSsds=args.num_master_local_ssds, ), ), workerConfiguration=messages.InstanceGroupConfiguration( numInstances=args.num_workers, imageUri=compute_uris['image'], machineTypeUri=compute_uris['worker_machine_type'], diskConfiguration=messages.DiskConfiguration( bootDiskSizeGb=args.worker_boot_disk_size_gb, numLocalSsds=args.num_worker_local_ssds, ), ), initializationActions=init_actions, softwareConfiguration=messages.SoftwareConfiguration( imageVersion=args.image_version), ) # Secondary worker group is optional. if args.num_preemptible_workers is not None: cluster_config.secondaryWorkerConfiguration = ( messages.InstanceGroupConfiguration( numInstances=args.num_preemptible_workers)) cluster = messages.Cluster(configuration=cluster_config, clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) operation = client.projects_clusters.Create(cluster) operation = util.WaitForOperation( operation, self.context, 'Waiting for cluster creation operation') cluster = client.projects_clusters.Get(cluster_ref.Request()) if cluster.status.state == ( messages.ClusterStatus.StateValueValuesEnum.RUNNING): log.CreatedResource(cluster_ref) else: log.error('Create cluster failed!') if operation.details: log.error('Details:\n' + operation.details) return cluster