def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = util.ParseCluster(args.name, dataproc) request = dataproc.messages.DataprocProjectsRegionsClustersDeleteRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, requestId=util.GetUniqueId()) console_io.PromptContinue( message="The cluster '{0}' and all attached disks will be " 'deleted.'.format(args.name), cancel_on_no=True, cancel_string='Deletion aborted by user.') operation = dataproc.client.projects_regions_clusters.Delete(request) if args. async: log.status.write('Deleting [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster deletion operation', timeout_s=args.timeout) log.DeletedResource(cluster_ref) return operation
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = args.CONCEPTS.cluster.Parse() stop_cluster_request = dataproc.messages.StopClusterRequest( requestId=util.GetUniqueId()) request = dataproc.messages.DataprocProjectsRegionsClustersStopRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, stopClusterRequest=stop_cluster_request) console_io.PromptContinue( message="Cluster '{0}' is stopping.".format( cluster_ref.clusterName), cancel_on_no=True, cancel_string='Stopping cluster aborted by user.') operation = dataproc.client.projects_regions_clusters.Stop(request) if args.async_: log.status.write('Stopping [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster to stop.', timeout_s=args.timeout) return operation
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = args.CONCEPTS.cluster.Parse() start_cluster_request = dataproc.messages.StartClusterRequest( requestId=util.GetUniqueId()) request = dataproc.messages.DataprocProjectsRegionsClustersStartRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, startClusterRequest=start_cluster_request) operation = dataproc.client.projects_regions_clusters.Start(request) if args.async_: log.status.write('Starting [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation operation = util.WaitForOperation( dataproc, operation, message="Waiting for cluster '{0}' to start.".format( cluster_ref.clusterName), timeout_s=args.timeout) return operation
def Run(self, args): """This is what gets called when the user runs this command.""" dataproc = dp.Dataproc(self.ReleaseTrack()) request_id = util.GetUniqueId() job_id = args.id if args.id else request_id # Don't use ResourceArgument, because --id is hidden by default job_ref = util.ParseJob(job_id, dataproc) self.PopulateFilesByType(args) cluster_ref = util.ParseCluster(args.cluster, dataproc) request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(request) self._staging_dir = self.GetStagingDir(cluster, job_ref.jobId, bucket=args.bucket) self.ValidateAndStageFiles() job = dataproc.messages.Job( reference=dataproc.messages.JobReference( projectId=job_ref.projectId, jobId=job_ref.jobId), placement=dataproc.messages.JobPlacement(clusterName=args.cluster)) self.ConfigureJob(dataproc.messages, job, args) if args.max_failures_per_hour: scheduling = dataproc.messages.JobScheduling( maxFailuresPerHour=args.max_failures_per_hour) job.scheduling = scheduling request = dataproc.messages.DataprocProjectsRegionsJobsSubmitRequest( projectId=job_ref.projectId, region=job_ref.region, submitJobRequest=dataproc.messages.SubmitJobRequest( job=job, requestId=request_id)) job = dataproc.client.projects_regions_jobs.Submit(request) log.status.Print('Job [{0}] submitted.'.format(job_id)) if not args.async_: job = util.WaitForJobTermination( dataproc, job, job_ref, message='Waiting for job completion', goal_state=dataproc.messages.JobStatus.StateValueValuesEnum. DONE, error_state=dataproc.messages.JobStatus.StateValueValuesEnum. ERROR, stream_driver_log=True) log.status.Print('Job [{0}] finished successfully.'.format(job_id)) return job
def GetRequest(self, args): """Creates a SessionsCreateRequest message. Creates a SessionsCreateRequest message. The factory only handles the arguments added in AddArguments function. User needs to provide session specific message instance. Args: args: Parsed arguments. Returns: SessionsCreateRequest: A configured SessionsCreateRequest. """ kwargs = {} kwargs['parent'] = args.CONCEPTS.session.Parse().Parent().RelativeName( ) kwargs['requestId'] = args.request_id if not kwargs['requestId']: kwargs['requestId'] = util.GetUniqueId() kwargs['sessionId'] = args.session kwargs['session'] = self.session_message_factory.GetMessage(args) return self.dataproc.messages.DataprocProjectsLocationsSessionsCreateRequest( **kwargs)
def GetRequest(self, args, batch_job): """Creates a BatchesCreateRequest message. Creates a BatchesCreateRequest message. The factory only handles the arguments added in AddArguments function. User needs to provide job specific message instance. Args: args: Parsed arguments. batch_job: A batch job typed message instance. Returns: BatchesCreateRequest: A configured BatchesCreateRequest. """ kwargs = {} kwargs['parent'] = args.CONCEPTS.region.Parse().RelativeName() # Recommendation: Always set a request ID for a create batch request. kwargs['requestId'] = args.request_id if not kwargs['requestId']: kwargs['requestId'] = util.GetUniqueId() # This behavior conflicts with protobuf definition. # Remove this if auto assign batch ID on control plane is enabled. kwargs['batchId'] = args.batch if not kwargs['batchId']: kwargs['batchId'] = kwargs['requestId'] kwargs['batch'] = self.batch_message_factory.GetMessage( args, batch_job) return self.dataproc.messages.DataprocProjectsLocationsBatchesCreateRequest( **kwargs)
def CreateCluster(dataproc, cluster, is_async, timeout): """Create a cluster. Args: dataproc: Dataproc object that contains client, messages, and resources cluster: Cluster to create is_async: Whether to wait for the operation to complete timeout: Timeout used when waiting for the operation to complete Returns: Created cluster, or None if async """ # Get project id and region. cluster_ref = util.ParseCluster(cluster.clusterName, dataproc) request_id = util.GetUniqueId() request = dataproc.messages.DataprocProjectsRegionsClustersCreateRequest( cluster=cluster, projectId=cluster_ref.projectId, region=cluster_ref.region, requestId=request_id) operation = dataproc.client.projects_regions_clusters.Create(request) if is_async: log.status.write('Creating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster creation operation', timeout_s=timeout) get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(get_request) if cluster.status.state == ( dataproc.messages.ClusterStatus.StateValueValuesEnum.RUNNING): zone_uri = cluster.config.gceClusterConfig.zoneUri zone_short_name = zone_uri.split('/')[-1] # Log the URL of the cluster log.CreatedResource( cluster_ref, # Also indicate which zone the cluster was placed in. This is helpful # if the server picked a zone (auto zone) details='Cluster placed in zone [{0}]'.format(zone_short_name)) else: log.error('Create cluster failed!') if operation.details: log.error('Details:\n' + operation.details) return cluster
def CreateCluster(dataproc, cluster_ref, cluster, is_async, timeout, enable_create_on_gke=False): """Create a cluster. Args: dataproc: Dataproc object that contains client, messages, and resources cluster_ref: Full resource ref of cluster with name, region, and project id cluster: Cluster to create is_async: Whether to wait for the operation to complete timeout: Timeout used when waiting for the operation to complete enable_create_on_gke: Whether to enable creation of GKE-based clusters Returns: Created cluster, or None if async """ # Get project id and region. request_id = util.GetUniqueId() request = dataproc.messages.DataprocProjectsRegionsClustersCreateRequest( cluster=cluster, projectId=cluster_ref.projectId, region=cluster_ref.region, requestId=request_id) operation = dataproc.client.projects_regions_clusters.Create(request) if is_async: log.status.write('Creating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster creation operation', timeout_s=timeout) get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(get_request) if cluster.status.state == ( dataproc.messages.ClusterStatus.StateValueValuesEnum.RUNNING): if enable_create_on_gke and cluster.config.gkeClusterConfig is not None: log.CreatedResource( cluster_ref, details='Cluster created on GKE cluster {0}'.format( cluster.config.gkeClusterConfig. namespacedGkeDeploymentTarget.targetGkeCluster)) else: zone_uri = cluster.config.gceClusterConfig.zoneUri zone_short_name = zone_uri.split('/')[-1] # Log the URL of the cluster log.CreatedResource( cluster_ref, # Also indicate which zone the cluster was placed in. This is helpful # if the server picked a zone (auto zone) details='Cluster placed in zone [{0}]'.format(zone_short_name)) else: # The operation didn't have an error, but the cluster is not RUNNING. log.error('Create cluster failed!') if cluster.status.detail: log.error('Details:\n' + cluster.status.detail) return cluster
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = util.ParseCluster(args.name, dataproc) cluster_config = dataproc.messages.ClusterConfig() changed_fields = [] has_changes = False if args.num_workers is not None: worker_config = dataproc.messages.InstanceGroupConfig( numInstances=args.num_workers) cluster_config.workerConfig = worker_config changed_fields.append('config.worker_config.num_instances') has_changes = True if args.num_preemptible_workers is not None: worker_config = dataproc.messages.InstanceGroupConfig( numInstances=args.num_preemptible_workers) cluster_config.secondaryWorkerConfig = worker_config changed_fields.append( 'config.secondary_worker_config.num_instances') has_changes = True if self.ReleaseTrack() == base.ReleaseTrack.BETA: if args.autoscaling_policy: cluster_config.autoscalingConfig = dataproc.messages.AutoscalingConfig( policyUri=args.CONCEPTS.autoscaling_policy.Parse( ).RelativeName()) changed_fields.append('config.autoscaling_config.policy_uri') has_changes = True elif args.autoscaling_policy == '' or args.disable_autoscaling: # pylint: disable=g-explicit-bool-comparison # Disabling autoscaling. Don't need to explicitly set # cluster_config.autoscaling_config to None. changed_fields.append('config.autoscaling_config.policy_uri') has_changes = True lifecycle_config = dataproc.messages.LifecycleConfig() changed_config = False if args.max_age is not None: lifecycle_config.autoDeleteTtl = str(args.max_age) + 's' changed_fields.append( 'config.lifecycle_config.auto_delete_ttl') changed_config = True if args.expiration_time is not None: lifecycle_config.autoDeleteTime = times.FormatDateTime( args.expiration_time) changed_fields.append( 'config.lifecycle_config.auto_delete_time') changed_config = True if args.max_idle is not None: lifecycle_config.idleDeleteTtl = str(args.max_idle) + 's' changed_fields.append( 'config.lifecycle_config.idle_delete_ttl') changed_config = True if args.no_max_age: lifecycle_config.autoDeleteTtl = None changed_fields.append( 'config.lifecycle_config.auto_delete_ttl') changed_config = True if args.no_max_idle: lifecycle_config.idleDeleteTtl = None changed_fields.append( 'config.lifecycle_config.idle_delete_ttl') changed_config = True if changed_config: cluster_config.lifecycleConfig = lifecycle_config has_changes = True # Put in a thunk so we only make this call if needed def _GetCurrentLabels(): # We need to fetch cluster first so we know what the labels look like. The # labels_util will fill out the proto for us with all the updates and # removals, but first we need to provide the current state of the labels get_cluster_request = ( dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName)) current_cluster = dataproc.client.projects_regions_clusters.Get( get_cluster_request) return current_cluster.labels labels_update = labels_util.ProcessUpdateArgsLazy( args, dataproc.messages.Cluster.LabelsValue, orig_labels_thunk=_GetCurrentLabels) if labels_update.needs_update: has_changes = True changed_fields.append('labels') labels = labels_update.GetOrNone() if not has_changes: raise exceptions.ArgumentError( 'Must specify at least one cluster parameter to update.') cluster = dataproc.messages.Cluster( config=cluster_config, clusterName=cluster_ref.clusterName, labels=labels, projectId=cluster_ref.projectId) request = dataproc.messages.DataprocProjectsRegionsClustersPatchRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, cluster=cluster, updateMask=','.join(changed_fields), requestId=util.GetUniqueId()) if args.graceful_decommission_timeout is not None: request.gracefulDecommissionTimeout = ( str(args.graceful_decommission_timeout) + 's') operation = dataproc.client.projects_regions_clusters.Patch(request) if args. async: log.status.write('Updating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return util.WaitForOperation(dataproc, operation, message='Waiting for cluster update operation', timeout_s=args.timeout) request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(request) log.UpdatedResource(cluster_ref) return cluster
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = args.CONCEPTS.cluster.Parse() cluster_config = dataproc.messages.ClusterConfig() changed_fields = [] has_changes = False if args.num_workers is not None: worker_config = dataproc.messages.InstanceGroupConfig( numInstances=args.num_workers) cluster_config.workerConfig = worker_config changed_fields.append('config.worker_config.num_instances') has_changes = True num_secondary_workers = _FirstNonNone(args.num_preemptible_workers, args.num_secondary_workers) if num_secondary_workers is not None: worker_config = dataproc.messages.InstanceGroupConfig( numInstances=num_secondary_workers) cluster_config.secondaryWorkerConfig = worker_config changed_fields.append( 'config.secondary_worker_config.num_instances') has_changes = True if args.autoscaling_policy: cluster_config.autoscalingConfig = dataproc.messages.AutoscalingConfig( policyUri=args.CONCEPTS.autoscaling_policy.Parse( ).RelativeName()) changed_fields.append('config.autoscaling_config.policy_uri') has_changes = True elif args.autoscaling_policy == '' or args.disable_autoscaling: # pylint: disable=g-explicit-bool-comparison # Disabling autoscaling. Don't need to explicitly set # cluster_config.autoscaling_config to None. changed_fields.append('config.autoscaling_config.policy_uri') has_changes = True lifecycle_config = dataproc.messages.LifecycleConfig() changed_config = False if args.max_age is not None: lifecycle_config.autoDeleteTtl = six.text_type(args.max_age) + 's' changed_fields.append('config.lifecycle_config.auto_delete_ttl') changed_config = True if args.expiration_time is not None: lifecycle_config.autoDeleteTime = times.FormatDateTime( args.expiration_time) changed_fields.append('config.lifecycle_config.auto_delete_time') changed_config = True if args.max_idle is not None: lifecycle_config.idleDeleteTtl = six.text_type(args.max_idle) + 's' changed_fields.append('config.lifecycle_config.idle_delete_ttl') changed_config = True if args.no_max_age: lifecycle_config.autoDeleteTtl = None changed_fields.append('config.lifecycle_config.auto_delete_ttl') changed_config = True if args.no_max_idle: lifecycle_config.idleDeleteTtl = None changed_fields.append('config.lifecycle_config.idle_delete_ttl') changed_config = True if changed_config: cluster_config.lifecycleConfig = lifecycle_config has_changes = True def _GetCurrentCluster(): # This is used for labels and auxiliary_node_pool_configs get_cluster_request = ( dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName)) current_cluster = dataproc.client.projects_regions_clusters.Get( get_cluster_request) return current_cluster # Put in a thunk so we only make this call if needed def _GetCurrentLabels(): # We need to fetch cluster first so we know what the labels look like. The # labels_util will fill out the proto for us with all the updates and # removals, but first we need to provide the current state of the labels current_cluster = _GetCurrentCluster() return current_cluster.labels labels_update = labels_util.ProcessUpdateArgsLazy( args, dataproc.messages.Cluster.LabelsValue, orig_labels_thunk=_GetCurrentLabels) if labels_update.needs_update: has_changes = True changed_fields.append('labels') labels = labels_update.GetOrNone() if args.driver_pool_size is not None: # Getting the node_pool_ids from the current node_pools and other attrs # that are not shared with the user # Driver pools can only be updated currently with NO other updates # We are relying on our frontend validation to prevent this until # the change is made to allow driver pools to be updated with other fields auxiliary_node_pools = _GetCurrentCluster( ).config.auxiliaryNodePoolConfigs # get the index of the current cluster's driver pool in the auxiliary # node pools list, index_driver_pools is also a list that should have a # length of 1 index_driver_pools = [ i for i, n in enumerate(auxiliary_node_pools) if dataproc.messages.NodePoolConfig. RolesValueListEntryValuesEnum.DRIVER in n.roles ] if len(index_driver_pools) > 1: raise exceptions.ArgumentError( 'At most one driver pool can be specified per cluster.') elif len(index_driver_pools) == 1: index = index_driver_pools[0] auxiliary_node_pools[ index].nodePoolConfig.numInstances = args.driver_pool_size else: # This case is only relevant for scaling from 0 -> N nodes # this will not be supported initially, but will be relying on our # front end validation to prevent or allow worker_config = dataproc.messages.InstanceGroupConfig( numInstances=args.driver_pool_size) node_config = dataproc.messages.NodePoolConfig( nodePoolConfig=worker_config, roles=[ dataproc.messages.NodePoolConfig. RolesValueListEntryValuesEnum.DRIVER ]) auxiliary_node_pools.append(node_config) cluster_config.auxiliaryNodePoolConfigs = auxiliary_node_pools changed_fields.append('config.auxiliary_node_pool_configs') has_changes = True if not has_changes: raise exceptions.ArgumentError( 'Must specify at least one cluster parameter to update.') cluster = dataproc.messages.Cluster( config=cluster_config, clusterName=cluster_ref.clusterName, labels=labels, projectId=cluster_ref.projectId) request = dataproc.messages.DataprocProjectsRegionsClustersPatchRequest( clusterName=cluster_ref.clusterName, region=cluster_ref.region, projectId=cluster_ref.projectId, cluster=cluster, updateMask=','.join(changed_fields), requestId=util.GetUniqueId()) if args.graceful_decommission_timeout is not None: request.gracefulDecommissionTimeout = ( six.text_type(args.graceful_decommission_timeout) + 's') operation = dataproc.client.projects_regions_clusters.Patch(request) if args.async_: log.status.write('Updating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return util.WaitForOperation(dataproc, operation, message='Waiting for cluster update operation', timeout_s=args.timeout) request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(request) log.UpdatedResource(cluster_ref) return cluster
def Run(self, args): self.ValidateArgs(args) dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = util.ParseCluster(args.name, dataproc) compute_resources = compute_helpers.GetComputeResources( self.ReleaseTrack(), args.name) beta = self.ReleaseTrack() == base.ReleaseTrack.BETA cluster_config = clusters.GetClusterConfig(args, dataproc, cluster_ref.projectId, compute_resources, beta) cluster = dataproc.messages.Cluster( config=cluster_config, clusterName=cluster_ref.clusterName, projectId=cluster_ref.projectId) self.ConfigureCluster(dataproc.messages, args, cluster) operation = dataproc.client.projects_regions_clusters.Create( dataproc.messages.DataprocProjectsRegionsClustersCreateRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, cluster=cluster, requestId=util.GetUniqueId())) if args. async: log.status.write('Creating [{0}] with operation [{1}].'.format( cluster_ref, operation.name)) return operation = util.WaitForOperation( dataproc, operation, message='Waiting for cluster creation operation', timeout_s=args.timeout) get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(get_request) if cluster.status.state == ( dataproc.messages.ClusterStatus.StateValueValuesEnum.RUNNING): zone_uri = cluster.config.gceClusterConfig.zoneUri zone_short_name = zone_uri.split('/')[-1] # Log the URL of the cluster log.CreatedResource( cluster_ref, # Also indicate which zone the cluster was placed in. This is helpful # if the server picked a zone (auto zone) details='Cluster placed in zone [{0}]'.format(zone_short_name)) else: log.error('Create cluster failed!') if operation.details: log.error('Details:\n' + operation.details) return cluster