def Run(self, args): """This is what gets called when the user runs this command.""" dataproc = dp.Dataproc(self.ReleaseTrack()) request_id = util.GetUniqueId() job_id = args.id if args.id else request_id # Don't use ResourceArgument, because --id is hidden by default job_ref = util.ParseJob(job_id, dataproc) self.PopulateFilesByType(args) cluster_ref = util.ParseCluster(args.cluster, dataproc) request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(request) self._staging_dir = self.GetStagingDir(cluster, job_ref.jobId, bucket=args.bucket) self.ValidateAndStageFiles() job = dataproc.messages.Job( reference=dataproc.messages.JobReference( projectId=job_ref.projectId, jobId=job_ref.jobId), placement=dataproc.messages.JobPlacement(clusterName=args.cluster)) self.ConfigureJob(dataproc.messages, job, args) if args.max_failures_per_hour: scheduling = dataproc.messages.JobScheduling( maxFailuresPerHour=args.max_failures_per_hour) job.scheduling = scheduling request = dataproc.messages.DataprocProjectsRegionsJobsSubmitRequest( projectId=job_ref.projectId, region=job_ref.region, submitJobRequest=dataproc.messages.SubmitJobRequest( job=job, requestId=request_id)) job = dataproc.client.projects_regions_jobs.Submit(request) log.status.Print('Job [{0}] submitted.'.format(job_id)) if not args.async_: job = util.WaitForJobTermination( dataproc, job, job_ref, message='Waiting for job completion', goal_state=dataproc.messages.JobStatus.StateValueValuesEnum. DONE, error_state=dataproc.messages.JobStatus.StateValueValuesEnum. ERROR, stream_driver_log=True) log.status.Print('Job [{0}] finished successfully.'.format(job_id)) return job
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) job_ref = util.ParseJob(args.id, dataproc) request = dataproc.messages.DataprocProjectsRegionsJobsDeleteRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId) console_io.PromptContinue( message="The job '{0}' will be deleted.".format(args.id), cancel_on_no=True, cancel_string='Deletion aborted by user.') dataproc.client.projects_regions_jobs.Delete(request) def _GetJob(job_ref): return dataproc.client.projects_regions_jobs.Get( dataproc.messages.DataprocProjectsRegionsJobsGetRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId)) util.WaitForResourceDeletion(_GetJob, job_ref, message='Waiting for job deletion') log.DeletedResource(job_ref)
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] job_ref = util.ParseJob(args.id, self.context) request = messages.DataprocProjectsRegionsJobsDeleteRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId) if not console_io.PromptContinue( message="The job '{0}' will be deleted.".format(args.id)): raise exceptions.ToolException('Deletion aborted by user.') client.projects_regions_jobs.Delete(request) def _GetJob(job_ref): return client.projects_regions_jobs.Get( messages.DataprocProjectsRegionsJobsGetRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId)) util.WaitForResourceDeletion(_GetJob, job_ref, message='Waiting for job deletion') log.DeletedResource(job_ref)
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) job_ref = util.ParseJob(args.job, dataproc) changed_fields = [] orig_job = dataproc.client.projects_regions_jobs.Get( dataproc.messages.DataprocProjectsRegionsJobsGetRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId)) # Update labels if the user requested it labels_update_result = labels_util.Diff.FromUpdateArgs(args).Apply( dataproc.messages.Job.LabelsValue, orig_job.labels) if labels_update_result.needs_update: changed_fields.append('labels') updated_job = orig_job updated_job.labels = labels_update_result.GetOrNone() request = dataproc.messages.DataprocProjectsRegionsJobsPatchRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId, job=updated_job, updateMask=','.join(changed_fields)) returned_job = dataproc.client.projects_regions_jobs.Patch(request) log.UpdatedResource(returned_job) return returned_job
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] job_ref = util.ParseJob(args.id, self.context) request = messages.DataprocProjectsRegionsJobsCancelRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId, cancelJobRequest=messages.CancelJobRequest()) # TODO(user) Check if Job is still running and fail or handle 401. if not console_io.PromptContinue( message="The job '{0}' will be killed.".format(args.id)): raise exceptions.ToolException('Cancellation aborted by user.') job = client.projects_regions_jobs.Cancel(request) log.status.Print('Job cancellation initiated for [{0}].'.format( job_ref.jobId)) job = util.WaitForJobTermination( job, self.context, message='Waiting for job cancellation', goal_state=messages.JobStatus.StateValueValuesEnum.CANCELLED) log.status.Print('Killed [{0}].'.format(job_ref)) return job
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) job_ref = util.ParseJob(args.job, dataproc) request = dataproc.messages.DataprocProjectsRegionsJobsCancelRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId, cancelJobRequest=dataproc.messages.CancelJobRequest()) # TODO(b/36049788) Check if Job is still running and fail or handle 401. console_io.PromptContinue( message="The job '{0}' will be killed.".format(args.job), cancel_on_no=True, cancel_string='Cancellation aborted by user.') job = dataproc.client.projects_regions_jobs.Cancel(request) log.status.Print('Job cancellation initiated for [{0}].'.format( job_ref.jobId)) job = util.WaitForJobTermination( dataproc, job, message='Waiting for job cancellation', goal_state=dataproc.messages.JobStatus.StateValueValuesEnum. CANCELLED) log.status.Print('Killed [{0}].'.format(job_ref)) return job
def Run(self, args): client = self.context['dataproc_client'] job_ref = util.ParseJob(args.id, self.context) request = job_ref.Request() job = client.projects_regions_jobs.Get(request) return job
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages job = util.ParseJob(args.job, dataproc) request = msgs.DataprocProjectsRegionsJobsGetIamPolicyRequest( resource=job.RelativeName()) return dataproc.client.projects_regions_jobs.GetIamPolicy(request)
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) job_ref = util.ParseJob(args.job, dataproc) return dataproc.client.projects_regions_jobs.Get( dataproc.messages.DataprocProjectsRegionsJobsGetRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId))
def Run(self, args): client = self.context['dataproc_client'] job_ref = util.ParseJob(args.id, self.context) return client.projects_regions_jobs.Get( client.MESSAGES_MODULE.DataprocProjectsRegionsJobsGetRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId))
def Run(self, args): """This is what gets called when the user runs this command.""" client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] job_id = util.GetJobId(args.id) job_ref = util.ParseJob(job_id, self.context) self.PopulateFilesByType(args) cluster_ref = util.ParseCluster(args.cluster, self.context) request = messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) try: cluster = client.projects_regions_clusters.Get(request) except apitools_exceptions.HttpError as error: raise exceptions.HttpException(error) self._staging_dir = self.GetStagingDir(cluster, job_ref.jobId) self.ValidateAndStageFiles() job = messages.Job( reference=messages.JobReference(projectId=job_ref.projectId, jobId=job_ref.jobId), placement=messages.JobPlacement(clusterName=args.cluster)) self.ConfigureJob(job, args) request = messages.DataprocProjectsRegionsJobsSubmitRequest( projectId=job_ref.projectId, region=job_ref.region, submitJobRequest=messages.SubmitJobRequest(job=job)) try: job = client.projects_regions_jobs.Submit(request) except apitools_exceptions.HttpError as error: raise exceptions.HttpException(error) log.status.Print('Job [{0}] submitted.'.format(job_id)) if not args. async: job = util.WaitForJobTermination( job, self.context, message='Waiting for job completion', goal_state=messages.JobStatus.StateValueValuesEnum.DONE, stream_driver_log=True) log.status.Print('Job [{0}] finished successfully.'.format(job_id)) return job
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages policy = iam_util.ParsePolicyFile(args.policy_file, msgs.Policy) set_iam_policy_request = msgs.SetIamPolicyRequest(policy=policy) job = util.ParseJob(args.job, dataproc) request = msgs.DataprocProjectsRegionsJobsSetIamPolicyRequest( resource=job.RelativeName(), setIamPolicyRequest=set_iam_policy_request) return dataproc.client.projects_regions_jobs.SetIamPolicy(request)
def Run(self, args): """This is what gets called when the user runs this command.""" dataproc = dp.Dataproc(self.ReleaseTrack()) job_id = util.GetJobId(args.id) job_ref = util.ParseJob(job_id, dataproc) self.PopulateFilesByType(args) cluster_ref = util.ParseCluster(args.cluster, dataproc) request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=cluster_ref.projectId, region=cluster_ref.region, clusterName=cluster_ref.clusterName) cluster = dataproc.client.projects_regions_clusters.Get(request) self._staging_dir = self.GetStagingDir( cluster, job_ref.jobId, bucket=args.bucket) self.ValidateAndStageFiles() job = dataproc.messages.Job( reference=dataproc.messages.JobReference( projectId=job_ref.projectId, jobId=job_ref.jobId), placement=dataproc.messages.JobPlacement( clusterName=args.cluster)) self.ConfigureJob(dataproc.messages, job, args) request = dataproc.messages.DataprocProjectsRegionsJobsSubmitRequest( projectId=job_ref.projectId, region=job_ref.region, submitJobRequest=dataproc.messages.SubmitJobRequest( job=job)) job = dataproc.client.projects_regions_jobs.Submit(request) log.status.Print('Job [{0}] submitted.'.format(job_id)) if not args.async: job = util.WaitForJobTermination( dataproc, job, message='Waiting for job completion', goal_state=dataproc.messages.JobStatus.StateValueValuesEnum.DONE, stream_driver_log=True) log.status.Print('Job [{0}] finished successfully.'.format(job_id)) return job
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] job_ref = util.ParseJob(args.id, self.context) changed_fields = [] has_changes = False # Update labels if the user requested it labels = None if args.update_labels or args.remove_labels: has_changes = True changed_fields.append('labels') # We need to fetch the job first so we know what the labels look like. The # labels_util.UpdateLabels will fill out the proto for us with all the # updates and removals, but first we need to provide the current state # of the labels orig_job = client.projects_regions_jobs.Get( client.MESSAGES_MODULE.DataprocProjectsRegionsJobsGetRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId)) labels = labels_util.UpdateLabels(orig_job.labels, messages.Job.LabelsValue, args.update_labels, args.remove_labels) if not has_changes: raise exceptions.ArgumentError( 'Must specify at least one job parameter to update.') updated_job = orig_job updated_job.labels = labels request = messages.DataprocProjectsRegionsJobsPatchRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId, job=updated_job, updateMask=','.join(changed_fields)) returned_job = client.projects_regions_jobs.Patch(request) log.UpdatedResource(returned_job) return returned_job
def Run(self, args): client = self.context['dataproc_client'] messages = self.context['dataproc_messages'] job_ref = util.ParseJob(args.id, self.context) request = job_ref.Request() job = client.projects_regions_jobs.Get(request) # TODO(user) Check if Job is still running and fail or handle 401. job = util.WaitForJobTermination( job, self.context, message='Waiting for job completion', goal_state=messages.JobStatus.StateValueValuesEnum.DONE, stream_driver_log=True) log.status.Print('Job [{0}] finished successfully.'.format(args.id)) return job
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) job_ref = util.ParseJob(args.id, dataproc) changed_fields = [] # Update labels if the user requested it labels = None if args.update_labels or args.remove_labels: changed_fields.append('labels') # We need to fetch the job first so we know what the labels look like. The # labels_util.UpdateLabels will fill out the proto for us with all the # updates and removals, but first we need to provide the current state # of the labels orig_job = dataproc.client.projects_regions_jobs.Get( dataproc.messages.DataprocProjectsRegionsJobsGetRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId)) labels = labels_util.UpdateLabels( orig_job.labels, dataproc.messages.Job.LabelsValue, args.update_labels, args.remove_labels) updated_job = orig_job updated_job.labels = labels request = dataproc.messages.DataprocProjectsRegionsJobsPatchRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId, job=updated_job, updateMask=','.join(changed_fields)) returned_job = dataproc.client.projects_regions_jobs.Patch(request) log.UpdatedResource(returned_job) return returned_job
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) job_ref = util.ParseJob(args.job, dataproc) job = dataproc.client.projects_regions_jobs.Get( dataproc.messages.DataprocProjectsRegionsJobsGetRequest( projectId=job_ref.projectId, region=job_ref.region, jobId=job_ref.jobId)) # TODO(b/36050945) Check if Job is still running and fail or handle 401. job = util.WaitForJobTermination( dataproc, job, message='Waiting for job completion', goal_state=dataproc.messages.JobStatus.StateValueValuesEnum.DONE, stream_driver_log=True) log.status.Print('Job [{0}] finished successfully.'.format(args.job)) return job