def Run(self, args):
"""This is what gets called when the user runs this command."""
dataproc = dp.Dataproc(self.ReleaseTrack())
job_id = util.GetJobId(args.id)
job_ref = util.ParseJob(job_id, dataproc)
self.PopulateFilesByType(args) # pytype: disable=attribute-error
cluster_ref = util.ParseCluster(args.cluster, dataproc)
request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
clusterName=cluster_ref.clusterName)
cluster = dataproc.client.projects_regions_clusters.Get(request)
self._staging_dir = self.GetStagingDir( # pytype: disable=attribute-error
cluster,
job_ref.jobId,
bucket=args.bucket)
self.ValidateAndStageFiles() # pytype: disable=attribute-error
job = dataproc.messages.Job(
reference=dataproc.messages.JobReference(
projectId=job_ref.projectId, jobId=job_ref.jobId),
placement=dataproc.messages.JobPlacement(clusterName=args.cluster))
self.ConfigureJob(dataproc.messages, job, args)
if args.max_failures_per_hour:
scheduling = dataproc.messages.JobScheduling(
maxFailuresPerHour=args.max_failures_per_hour)
job.scheduling = scheduling
request = dataproc.messages.DataprocProjectsRegionsJobsSubmitRequest(
projectId=job_ref.projectId,
region=job_ref.region,
submitJobRequest=dataproc.messages.SubmitJobRequest(job=job))
job = dataproc.client.projects_regions_jobs.Submit(request)
log.status.Print('Job [{0}] submitted.'.format(job_id))
if not args. async:
job = util.WaitForJobTermination(
dataproc,
job,
message='Waiting for job completion',
goal_state=dataproc.messages.JobStatus.StateValueValuesEnum.
DONE,
stream_driver_log=True)
log.status.Print('Job [{0}] finished successfully.'.format(job_id))
return job
def Run(self, args):
"""This is what gets called when the user runs this command."""
client = self.context['dataproc_client']
messages = self.context['dataproc_messages']
job_id = util.GetJobId(args.id)
job_ref = util.ParseJob(job_id, self.context)
self.PopulateFilesByType(args)
cluster_ref = util.ParseCluster(args.cluster, self.context)
request = messages.DataprocProjectsRegionsClustersGetRequest(
projectId=cluster_ref.projectId,
region=cluster_ref.region,
clusterName=cluster_ref.clusterName)
try:
cluster = client.projects_regions_clusters.Get(request)
except apitools_exceptions.HttpError as error:
raise exceptions.HttpException(error)
self._staging_dir = self.GetStagingDir(cluster, job_ref.jobId)
self.ValidateAndStageFiles()
job = messages.Job(
reference=messages.JobReference(projectId=job_ref.projectId,
jobId=job_ref.jobId),
placement=messages.JobPlacement(clusterName=args.cluster))
self.ConfigureJob(job, args)
request = messages.DataprocProjectsRegionsJobsSubmitRequest(
projectId=job_ref.projectId,
region=job_ref.region,
submitJobRequest=messages.SubmitJobRequest(job=job))
try:
job = client.projects_regions_jobs.Submit(request)
except apitools_exceptions.HttpError as error:
raise exceptions.HttpException(error)
log.status.Print('Job [{0}] submitted.'.format(job_id))
if not args. async:
job = util.WaitForJobTermination(
job,
self.context,
message='Waiting for job completion',
goal_state=messages.JobStatus.StateValueValuesEnum.DONE,
stream_driver_log=True)
log.status.Print('Job [{0}] finished successfully.'.format(job_id))
return job
