def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
cluster_ref = util.ParseCluster(args.name, dataproc)
request = dataproc.messages.DataprocProjectsRegionsClustersDeleteRequest(
clusterName=cluster_ref.clusterName,
region=cluster_ref.region,
projectId=cluster_ref.projectId,
requestId=util.GetUniqueId())
console_io.PromptContinue(
message="The cluster '{0}' and all attached disks will be "
'deleted.'.format(args.name),
cancel_on_no=True,
cancel_string='Deletion aborted by user.')
operation = dataproc.client.projects_regions_clusters.Delete(request)
if args. async:
log.status.write('Deleting [{0}] with operation [{1}].'.format(
cluster_ref, operation.name))
return operation
operation = util.WaitForOperation(
dataproc,
operation,
message='Waiting for cluster deletion operation',
timeout_s=args.timeout)
log.DeletedResource(cluster_ref)
return operation
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
cluster_ref = args.CONCEPTS.cluster.Parse()
stop_cluster_request = dataproc.messages.StopClusterRequest(
requestId=util.GetUniqueId())
request = dataproc.messages.DataprocProjectsRegionsClustersStopRequest(
clusterName=cluster_ref.clusterName,
region=cluster_ref.region,
projectId=cluster_ref.projectId,
stopClusterRequest=stop_cluster_request)
console_io.PromptContinue(
message="Cluster '{0}' is stopping.".format(
cluster_ref.clusterName),
cancel_on_no=True,
cancel_string='Stopping cluster aborted by user.')
operation = dataproc.client.projects_regions_clusters.Stop(request)
if args.async_:
log.status.write('Stopping [{0}] with operation [{1}].'.format(
cluster_ref, operation.name))
return operation
operation = util.WaitForOperation(
dataproc,
operation,
message='Waiting for cluster to stop.',
timeout_s=args.timeout)
return operation
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
cluster_ref = args.CONCEPTS.cluster.Parse()
start_cluster_request = dataproc.messages.StartClusterRequest(
requestId=util.GetUniqueId())
request = dataproc.messages.DataprocProjectsRegionsClustersStartRequest(
clusterName=cluster_ref.clusterName,
region=cluster_ref.region,
projectId=cluster_ref.projectId,
startClusterRequest=start_cluster_request)
operation = dataproc.client.projects_regions_clusters.Start(request)
if args.async_:
log.status.write('Starting [{0}] with operation [{1}].'.format(
cluster_ref, operation.name))
return operation
operation = util.WaitForOperation(
dataproc,
operation,
message="Waiting for cluster '{0}' to start.".format(
cluster_ref.clusterName),
timeout_s=args.timeout)
return operation
def Run(self, args):
"""This is what gets called when the user runs this command."""
dataproc = dp.Dataproc(self.ReleaseTrack())
request_id = util.GetUniqueId()
job_id = args.id if args.id else request_id
# Don't use ResourceArgument, because --id is hidden by default
job_ref = util.ParseJob(job_id, dataproc)
self.PopulateFilesByType(args)
cluster_ref = util.ParseCluster(args.cluster, dataproc)
request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
clusterName=cluster_ref.clusterName)
cluster = dataproc.client.projects_regions_clusters.Get(request)
self._staging_dir = self.GetStagingDir(cluster,
job_ref.jobId,
bucket=args.bucket)
self.ValidateAndStageFiles()
job = dataproc.messages.Job(
reference=dataproc.messages.JobReference(
projectId=job_ref.projectId, jobId=job_ref.jobId),
placement=dataproc.messages.JobPlacement(clusterName=args.cluster))
self.ConfigureJob(dataproc.messages, job, args)
if args.max_failures_per_hour:
scheduling = dataproc.messages.JobScheduling(
maxFailuresPerHour=args.max_failures_per_hour)
job.scheduling = scheduling
request = dataproc.messages.DataprocProjectsRegionsJobsSubmitRequest(
projectId=job_ref.projectId,
region=job_ref.region,
submitJobRequest=dataproc.messages.SubmitJobRequest(
job=job, requestId=request_id))
job = dataproc.client.projects_regions_jobs.Submit(request)
log.status.Print('Job [{0}] submitted.'.format(job_id))
if not args.async_:
job = util.WaitForJobTermination(
dataproc,
job,
job_ref,
message='Waiting for job completion',
goal_state=dataproc.messages.JobStatus.StateValueValuesEnum.
DONE,
error_state=dataproc.messages.JobStatus.StateValueValuesEnum.
ERROR,
stream_driver_log=True)
log.status.Print('Job [{0}] finished successfully.'.format(job_id))
return job
def GetRequest(self, args):
"""Creates a SessionsCreateRequest message.
Creates a SessionsCreateRequest message. The factory only handles the
arguments added in AddArguments function. User needs to provide session
specific message instance.
Args:
args: Parsed arguments.
Returns:
SessionsCreateRequest: A configured SessionsCreateRequest.
"""
kwargs = {}
kwargs['parent'] = args.CONCEPTS.session.Parse().Parent().RelativeName(
)
kwargs['requestId'] = args.request_id
if not kwargs['requestId']:
kwargs['requestId'] = util.GetUniqueId()
kwargs['sessionId'] = args.session
kwargs['session'] = self.session_message_factory.GetMessage(args)
return self.dataproc.messages.DataprocProjectsLocationsSessionsCreateRequest(
**kwargs)
def GetRequest(self, args, batch_job):
"""Creates a BatchesCreateRequest message.
Creates a BatchesCreateRequest message. The factory only handles the
arguments added in AddArguments function. User needs to provide job
specific message instance.
Args:
args: Parsed arguments.
batch_job: A batch job typed message instance.
Returns:
BatchesCreateRequest: A configured BatchesCreateRequest.
"""
kwargs = {}
kwargs['parent'] = args.CONCEPTS.region.Parse().RelativeName()
# Recommendation: Always set a request ID for a create batch request.
kwargs['requestId'] = args.request_id
if not kwargs['requestId']:
kwargs['requestId'] = util.GetUniqueId()
# This behavior conflicts with protobuf definition.
# Remove this if auto assign batch ID on control plane is enabled.
kwargs['batchId'] = args.batch
if not kwargs['batchId']:
kwargs['batchId'] = kwargs['requestId']
kwargs['batch'] = self.batch_message_factory.GetMessage(
args, batch_job)
return self.dataproc.messages.DataprocProjectsLocationsBatchesCreateRequest(
**kwargs)
def CreateCluster(dataproc, cluster, is_async, timeout):
"""Create a cluster.
Args:
dataproc: Dataproc object that contains client, messages, and resources
cluster: Cluster to create
is_async: Whether to wait for the operation to complete
timeout: Timeout used when waiting for the operation to complete
Returns:
Created cluster, or None if async
"""
# Get project id and region.
cluster_ref = util.ParseCluster(cluster.clusterName, dataproc)
request_id = util.GetUniqueId()
request = dataproc.messages.DataprocProjectsRegionsClustersCreateRequest(
cluster=cluster,
projectId=cluster_ref.projectId,
region=cluster_ref.region,
requestId=request_id)
operation = dataproc.client.projects_regions_clusters.Create(request)
if is_async:
log.status.write('Creating [{0}] with operation [{1}].'.format(
cluster_ref, operation.name))
return
operation = util.WaitForOperation(
dataproc,
operation,
message='Waiting for cluster creation operation',
timeout_s=timeout)
get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
clusterName=cluster_ref.clusterName)
cluster = dataproc.client.projects_regions_clusters.Get(get_request)
if cluster.status.state == (
dataproc.messages.ClusterStatus.StateValueValuesEnum.RUNNING):
zone_uri = cluster.config.gceClusterConfig.zoneUri
zone_short_name = zone_uri.split('/')[-1]
# Log the URL of the cluster
log.CreatedResource(
cluster_ref,
# Also indicate which zone the cluster was placed in. This is helpful
# if the server picked a zone (auto zone)
details='Cluster placed in zone [{0}]'.format(zone_short_name))
else:
log.error('Create cluster failed!')
if operation.details:
log.error('Details:\n' + operation.details)
return cluster
def CreateCluster(dataproc,
cluster_ref,
cluster,
is_async,
timeout,
enable_create_on_gke=False):
"""Create a cluster.
Args:
dataproc: Dataproc object that contains client, messages, and resources
cluster_ref: Full resource ref of cluster with name, region, and project id
cluster: Cluster to create
is_async: Whether to wait for the operation to complete
timeout: Timeout used when waiting for the operation to complete
enable_create_on_gke: Whether to enable creation of GKE-based clusters
Returns:
Created cluster, or None if async
"""
# Get project id and region.
request_id = util.GetUniqueId()
request = dataproc.messages.DataprocProjectsRegionsClustersCreateRequest(
cluster=cluster,
projectId=cluster_ref.projectId,
region=cluster_ref.region,
requestId=request_id)
operation = dataproc.client.projects_regions_clusters.Create(request)
if is_async:
log.status.write('Creating [{0}] with operation [{1}].'.format(
cluster_ref, operation.name))
return
operation = util.WaitForOperation(
dataproc,
operation,
message='Waiting for cluster creation operation',
timeout_s=timeout)
get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
clusterName=cluster_ref.clusterName)
cluster = dataproc.client.projects_regions_clusters.Get(get_request)
if cluster.status.state == (
dataproc.messages.ClusterStatus.StateValueValuesEnum.RUNNING):
if enable_create_on_gke and cluster.config.gkeClusterConfig is not None:
log.CreatedResource(
cluster_ref,
details='Cluster created on GKE cluster {0}'.format(
cluster.config.gkeClusterConfig.
namespacedGkeDeploymentTarget.targetGkeCluster))
else:
zone_uri = cluster.config.gceClusterConfig.zoneUri
zone_short_name = zone_uri.split('/')[-1]
# Log the URL of the cluster
log.CreatedResource(
cluster_ref,
# Also indicate which zone the cluster was placed in. This is helpful
# if the server picked a zone (auto zone)
details='Cluster placed in zone [{0}]'.format(zone_short_name))
else:
# The operation didn't have an error, but the cluster is not RUNNING.
log.error('Create cluster failed!')
if cluster.status.detail:
log.error('Details:\n' + cluster.status.detail)
return cluster
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
cluster_ref = util.ParseCluster(args.name, dataproc)
cluster_config = dataproc.messages.ClusterConfig()
changed_fields = []
has_changes = False
if args.num_workers is not None:
worker_config = dataproc.messages.InstanceGroupConfig(
numInstances=args.num_workers)
cluster_config.workerConfig = worker_config
changed_fields.append('config.worker_config.num_instances')
has_changes = True
if args.num_preemptible_workers is not None:
worker_config = dataproc.messages.InstanceGroupConfig(
numInstances=args.num_preemptible_workers)
cluster_config.secondaryWorkerConfig = worker_config
changed_fields.append(
'config.secondary_worker_config.num_instances')
has_changes = True
if self.ReleaseTrack() == base.ReleaseTrack.BETA:
if args.autoscaling_policy:
cluster_config.autoscalingConfig = dataproc.messages.AutoscalingConfig(
policyUri=args.CONCEPTS.autoscaling_policy.Parse(
).RelativeName())
changed_fields.append('config.autoscaling_config.policy_uri')
has_changes = True
elif args.autoscaling_policy == '' or args.disable_autoscaling: # pylint: disable=g-explicit-bool-comparison
# Disabling autoscaling. Don't need to explicitly set
# cluster_config.autoscaling_config to None.
changed_fields.append('config.autoscaling_config.policy_uri')
has_changes = True
lifecycle_config = dataproc.messages.LifecycleConfig()
changed_config = False
if args.max_age is not None:
lifecycle_config.autoDeleteTtl = str(args.max_age) + 's'
changed_fields.append(
'config.lifecycle_config.auto_delete_ttl')
changed_config = True
if args.expiration_time is not None:
lifecycle_config.autoDeleteTime = times.FormatDateTime(
args.expiration_time)
changed_fields.append(
'config.lifecycle_config.auto_delete_time')
changed_config = True
if args.max_idle is not None:
lifecycle_config.idleDeleteTtl = str(args.max_idle) + 's'
changed_fields.append(
'config.lifecycle_config.idle_delete_ttl')
changed_config = True
if args.no_max_age:
lifecycle_config.autoDeleteTtl = None
changed_fields.append(
'config.lifecycle_config.auto_delete_ttl')
changed_config = True
if args.no_max_idle:
lifecycle_config.idleDeleteTtl = None
changed_fields.append(
'config.lifecycle_config.idle_delete_ttl')
changed_config = True
if changed_config:
cluster_config.lifecycleConfig = lifecycle_config
has_changes = True
# Put in a thunk so we only make this call if needed
def _GetCurrentLabels():
# We need to fetch cluster first so we know what the labels look like. The
# labels_util will fill out the proto for us with all the updates and
# removals, but first we need to provide the current state of the labels
get_cluster_request = (
dataproc.messages.DataprocProjectsRegionsClustersGetRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
clusterName=cluster_ref.clusterName))
current_cluster = dataproc.client.projects_regions_clusters.Get(
get_cluster_request)
return current_cluster.labels
labels_update = labels_util.ProcessUpdateArgsLazy(
args,
dataproc.messages.Cluster.LabelsValue,
orig_labels_thunk=_GetCurrentLabels)
if labels_update.needs_update:
has_changes = True
changed_fields.append('labels')
labels = labels_update.GetOrNone()
if not has_changes:
raise exceptions.ArgumentError(
'Must specify at least one cluster parameter to update.')
cluster = dataproc.messages.Cluster(
config=cluster_config,
clusterName=cluster_ref.clusterName,
labels=labels,
projectId=cluster_ref.projectId)
request = dataproc.messages.DataprocProjectsRegionsClustersPatchRequest(
clusterName=cluster_ref.clusterName,
region=cluster_ref.region,
projectId=cluster_ref.projectId,
cluster=cluster,
updateMask=','.join(changed_fields),
requestId=util.GetUniqueId())
if args.graceful_decommission_timeout is not None:
request.gracefulDecommissionTimeout = (
str(args.graceful_decommission_timeout) + 's')
operation = dataproc.client.projects_regions_clusters.Patch(request)
if args. async:
log.status.write('Updating [{0}] with operation [{1}].'.format(
cluster_ref, operation.name))
return
util.WaitForOperation(dataproc,
operation,
message='Waiting for cluster update operation',
timeout_s=args.timeout)
request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
clusterName=cluster_ref.clusterName)
cluster = dataproc.client.projects_regions_clusters.Get(request)
log.UpdatedResource(cluster_ref)
return cluster
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
cluster_ref = args.CONCEPTS.cluster.Parse()
cluster_config = dataproc.messages.ClusterConfig()
changed_fields = []
has_changes = False
if args.num_workers is not None:
worker_config = dataproc.messages.InstanceGroupConfig(
numInstances=args.num_workers)
cluster_config.workerConfig = worker_config
changed_fields.append('config.worker_config.num_instances')
has_changes = True
num_secondary_workers = _FirstNonNone(args.num_preemptible_workers,
args.num_secondary_workers)
if num_secondary_workers is not None:
worker_config = dataproc.messages.InstanceGroupConfig(
numInstances=num_secondary_workers)
cluster_config.secondaryWorkerConfig = worker_config
changed_fields.append(
'config.secondary_worker_config.num_instances')
has_changes = True
if args.autoscaling_policy:
cluster_config.autoscalingConfig = dataproc.messages.AutoscalingConfig(
policyUri=args.CONCEPTS.autoscaling_policy.Parse(
).RelativeName())
changed_fields.append('config.autoscaling_config.policy_uri')
has_changes = True
elif args.autoscaling_policy == '' or args.disable_autoscaling: # pylint: disable=g-explicit-bool-comparison
# Disabling autoscaling. Don't need to explicitly set
# cluster_config.autoscaling_config to None.
changed_fields.append('config.autoscaling_config.policy_uri')
has_changes = True
lifecycle_config = dataproc.messages.LifecycleConfig()
changed_config = False
if args.max_age is not None:
lifecycle_config.autoDeleteTtl = six.text_type(args.max_age) + 's'
changed_fields.append('config.lifecycle_config.auto_delete_ttl')
changed_config = True
if args.expiration_time is not None:
lifecycle_config.autoDeleteTime = times.FormatDateTime(
args.expiration_time)
changed_fields.append('config.lifecycle_config.auto_delete_time')
changed_config = True
if args.max_idle is not None:
lifecycle_config.idleDeleteTtl = six.text_type(args.max_idle) + 's'
changed_fields.append('config.lifecycle_config.idle_delete_ttl')
changed_config = True
if args.no_max_age:
lifecycle_config.autoDeleteTtl = None
changed_fields.append('config.lifecycle_config.auto_delete_ttl')
changed_config = True
if args.no_max_idle:
lifecycle_config.idleDeleteTtl = None
changed_fields.append('config.lifecycle_config.idle_delete_ttl')
changed_config = True
if changed_config:
cluster_config.lifecycleConfig = lifecycle_config
has_changes = True
def _GetCurrentCluster():
# This is used for labels and auxiliary_node_pool_configs
get_cluster_request = (
dataproc.messages.DataprocProjectsRegionsClustersGetRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
clusterName=cluster_ref.clusterName))
current_cluster = dataproc.client.projects_regions_clusters.Get(
get_cluster_request)
return current_cluster
# Put in a thunk so we only make this call if needed
def _GetCurrentLabels():
# We need to fetch cluster first so we know what the labels look like. The
# labels_util will fill out the proto for us with all the updates and
# removals, but first we need to provide the current state of the labels
current_cluster = _GetCurrentCluster()
return current_cluster.labels
labels_update = labels_util.ProcessUpdateArgsLazy(
args,
dataproc.messages.Cluster.LabelsValue,
orig_labels_thunk=_GetCurrentLabels)
if labels_update.needs_update:
has_changes = True
changed_fields.append('labels')
labels = labels_update.GetOrNone()
if args.driver_pool_size is not None:
# Getting the node_pool_ids from the current node_pools and other attrs
# that are not shared with the user
# Driver pools can only be updated currently with NO other updates
# We are relying on our frontend validation to prevent this until
# the change is made to allow driver pools to be updated with other fields
auxiliary_node_pools = _GetCurrentCluster(
).config.auxiliaryNodePoolConfigs
# get the index of the current cluster's driver pool in the auxiliary
# node pools list, index_driver_pools is also a list that should have a
# length of 1
index_driver_pools = [
i for i, n in enumerate(auxiliary_node_pools)
if dataproc.messages.NodePoolConfig.
RolesValueListEntryValuesEnum.DRIVER in n.roles
]
if len(index_driver_pools) > 1:
raise exceptions.ArgumentError(
'At most one driver pool can be specified per cluster.')
elif len(index_driver_pools) == 1:
index = index_driver_pools[0]
auxiliary_node_pools[
index].nodePoolConfig.numInstances = args.driver_pool_size
else:
# This case is only relevant for scaling from 0 -> N nodes
# this will not be supported initially, but will be relying on our
# front end validation to prevent or allow
worker_config = dataproc.messages.InstanceGroupConfig(
numInstances=args.driver_pool_size)
node_config = dataproc.messages.NodePoolConfig(
nodePoolConfig=worker_config,
roles=[
dataproc.messages.NodePoolConfig.
RolesValueListEntryValuesEnum.DRIVER
])
auxiliary_node_pools.append(node_config)
cluster_config.auxiliaryNodePoolConfigs = auxiliary_node_pools
changed_fields.append('config.auxiliary_node_pool_configs')
has_changes = True
if not has_changes:
raise exceptions.ArgumentError(
'Must specify at least one cluster parameter to update.')
cluster = dataproc.messages.Cluster(
config=cluster_config,
clusterName=cluster_ref.clusterName,
labels=labels,
projectId=cluster_ref.projectId)
request = dataproc.messages.DataprocProjectsRegionsClustersPatchRequest(
clusterName=cluster_ref.clusterName,
region=cluster_ref.region,
projectId=cluster_ref.projectId,
cluster=cluster,
updateMask=','.join(changed_fields),
requestId=util.GetUniqueId())
if args.graceful_decommission_timeout is not None:
request.gracefulDecommissionTimeout = (
six.text_type(args.graceful_decommission_timeout) + 's')
operation = dataproc.client.projects_regions_clusters.Patch(request)
if args.async_:
log.status.write('Updating [{0}] with operation [{1}].'.format(
cluster_ref, operation.name))
return
util.WaitForOperation(dataproc,
operation,
message='Waiting for cluster update operation',
timeout_s=args.timeout)
request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
clusterName=cluster_ref.clusterName)
cluster = dataproc.client.projects_regions_clusters.Get(request)
log.UpdatedResource(cluster_ref)
return cluster
def Run(self, args):
self.ValidateArgs(args)
dataproc = dp.Dataproc(self.ReleaseTrack())
cluster_ref = util.ParseCluster(args.name, dataproc)
compute_resources = compute_helpers.GetComputeResources(
self.ReleaseTrack(), args.name)
beta = self.ReleaseTrack() == base.ReleaseTrack.BETA
cluster_config = clusters.GetClusterConfig(args, dataproc,
cluster_ref.projectId,
compute_resources, beta)
cluster = dataproc.messages.Cluster(
config=cluster_config,
clusterName=cluster_ref.clusterName,
projectId=cluster_ref.projectId)
self.ConfigureCluster(dataproc.messages, args, cluster)
operation = dataproc.client.projects_regions_clusters.Create(
dataproc.messages.DataprocProjectsRegionsClustersCreateRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
cluster=cluster,
requestId=util.GetUniqueId()))
if args. async:
log.status.write('Creating [{0}] with operation [{1}].'.format(
cluster_ref, operation.name))
return
operation = util.WaitForOperation(
dataproc,
operation,
message='Waiting for cluster creation operation',
timeout_s=args.timeout)
get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
clusterName=cluster_ref.clusterName)
cluster = dataproc.client.projects_regions_clusters.Get(get_request)
if cluster.status.state == (
dataproc.messages.ClusterStatus.StateValueValuesEnum.RUNNING):
zone_uri = cluster.config.gceClusterConfig.zoneUri
zone_short_name = zone_uri.split('/')[-1]
# Log the URL of the cluster
log.CreatedResource(
cluster_ref,
# Also indicate which zone the cluster was placed in. This is helpful
# if the server picked a zone (auto zone)
details='Cluster placed in zone [{0}]'.format(zone_short_name))
else:
log.error('Create cluster failed!')
if operation.details:
log.error('Details:\n' + operation.details)
return cluster
