def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
template = util.ParseWorkflowTemplates(args.template, dataproc)
workflow_template = dataproc.GetRegionsWorkflowTemplate(
template, args.version)
cluster_name = template.workflowTemplatesId
compute_resources = compute_helpers.GetComputeResources(
self.ReleaseTrack(), cluster_name)
use_accelerators = self.ReleaseTrack() == base.ReleaseTrack.BETA
use_auto_delete_ttl = self.ReleaseTrack() == base.ReleaseTrack.BETA
use_min_cpu_platform = self.ReleaseTrack() == base.ReleaseTrack.BETA
cluster_config = clusters.GetClusterConfig(
args, dataproc, template.projectsId, compute_resources,
use_accelerators, use_auto_delete_ttl, use_min_cpu_platform)
labels = labels_util.ParseCreateArgs(
args, dataproc.messages.ManagedCluster.LabelsValue)
managed_cluster = dataproc.messages.ManagedCluster(
clusterName=cluster_name, config=cluster_config, labels=labels)
workflow_template.placement = dataproc.messages.WorkflowTemplatePlacement(
managedCluster=managed_cluster)
response = dataproc.client.projects_regions_workflowTemplates.Update(
workflow_template)
return response
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
template = util.ParseWorkflowTemplates(args.template, dataproc)
workflow_template = dataproc.GetRegionsWorkflowTemplate(
template, args.version)
jobs = workflow_template.jobs
job_removed = False
new_jobs = []
for ordered_job in jobs:
if ordered_job.stepId != args.step_id:
new_jobs.append(ordered_job)
else:
console_io.PromptContinue(
message=('The job [{0}] will be removed from workflow template '
'[{1}].').format(args.step_id, workflow_template.id),
cancel_on_no=True)
job_removed = True
if not job_removed:
log.error('Step id [{0}] is not found in workflow template [{1}].'.format(
args.step_id, workflow_template.id))
return # do not update workflow template if job is not removed.
workflow_template.jobs = new_jobs
response = dataproc.client.projects_regions_workflowTemplates.Update(
workflow_template)
return response
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
template = util.ParseWorkflowTemplates(args.template, dataproc)
instantiate_request = dataproc.messages.InstantiateWorkflowTemplateRequest(
)
instantiate_request.instanceId = uuid.uuid4().hex # request UUID
if args.parameters:
instantiate_request.parameters = encoding.DictToMessage(
args.parameters,
msgs.InstantiateWorkflowTemplateRequest.ParametersValue)
request = msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateRequest(
instantiateWorkflowTemplateRequest=instantiate_request,
name=template.RelativeName())
operation = dataproc.client.projects_regions_workflowTemplates.Instantiate(
request)
if args. async:
log.status.Print(
'Instantiating [{0}] with operation [{1}].'.format(
template.Name(), operation.name))
return
operation = util.WaitForWorkflowTemplateOperation(
dataproc, operation, timeout_s=args.timeout)
return operation
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
messages = dataproc.messages
template = util.ParseWorkflowTemplates(args.template, dataproc)
workflow_template = dataproc.GetRegionsWorkflowTemplate(
template, args.version)
# load workflow template from YAML or JSON file
local_workflow_yaml = workflow_templates.ParseYamlOrJsonToWorkflowTemplate(
args.config_file, messages.WorkflowTemplate)
if local_workflow_yaml.id != workflow_template.id:
raise exceptions.WorkflowTemplateError(
'Workflow template ID [{0}] in YAML or JSON file does not match the '
'workflow template ID [{1}] in Dataproc.'.format(
local_workflow_yaml.id, workflow_template.id))
if local_workflow_yaml.name != workflow_template.name:
raise exceptions.WorkflowTemplateError(
'Workflow template name [{0}] in YAML or JSON file does not match '
'the workflow template name [{1}] in Dataproc.'.format(
local_workflow_yaml.name, workflow_template.name))
# update version fields in local yaml workflow template
local_workflow_yaml.version = workflow_template.version
response = dataproc.client.projects_regions_workflowTemplates.Update(
local_workflow_yaml)
return response
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
template = util.ParseWorkflowTemplates(args.template, dataproc)
workflow_template = dataproc.GetRegionsWorkflowTemplate(
template, args.version)
return workflow_template
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
template = util.ParseWorkflowTemplates(args.template, dataproc)
request = msgs.DataprocProjectsRegionsWorkflowTemplatesGetIamPolicyRequest(
resource=template.RelativeName())
return dataproc.client.projects_regions_workflowTemplates.GetIamPolicy(
request)
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
policy = iam_util.ParsePolicyFile(args.policy_file, msgs.Policy)
set_iam_policy_request = msgs.SetIamPolicyRequest(policy=policy)
template = util.ParseWorkflowTemplates(args.template, dataproc)
request = msgs.DataprocProjectsRegionsWorkflowTemplatesSetIamPolicyRequest(
resource=template.RelativeName(),
setIamPolicyRequest=set_iam_policy_request)
return dataproc.client.projects_regions_workflowTemplates.SetIamPolicy(
request)
def AddJobToWorkflowTemplate(args, dataproc, ordered_job):
"""Add an ordered job to the workflow template."""
template = util.ParseWorkflowTemplates(args.workflow_template, dataproc)
workflow_template = dataproc.GetRegionsWorkflowTemplate(
template, args.version)
jobs = workflow_template.jobs if workflow_template.jobs is not None else []
jobs.append(ordered_job)
workflow_template.jobs = jobs
response = dataproc.client.projects_regions_workflowTemplates.Update(
workflow_template)
return response
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
messages = dataproc.messages
template = util.ParseWorkflowTemplates(args.template, dataproc)
request = messages.DataprocProjectsRegionsWorkflowTemplatesDeleteRequest(
name=template.RelativeName())
console_io.PromptContinue(
message="The workflow template '[{0}]' will be deleted.".format(
template.Name()),
cancel_on_no=True)
dataproc.client.projects_regions_workflowTemplates.Delete(request)
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
messages = dataproc.messages
template_ref = util.ParseWorkflowTemplates(args.template, dataproc)
regions_ref = util.ParseRegion(dataproc)
workflow_template = messages.WorkflowTemplate(
id=args.template, name=template_ref.RelativeName(),
labels=labels_util.ParseCreateArgs(
args, messages.WorkflowTemplate.LabelsValue))
request = messages.DataprocProjectsRegionsWorkflowTemplatesCreateRequest(
parent=regions_ref.RelativeName(), workflowTemplate=workflow_template)
template = dataproc.client.projects_regions_workflowTemplates.Create(
request)
return template
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
template = util.ParseWorkflowTemplates(args.template, dataproc)
workflow_template = dataproc.GetRegionsWorkflowTemplate(
template, args.version)
labels = labels_util.Diff(additions=args.cluster_labels).Apply(
dataproc.messages.ClusterSelector.ClusterLabelsValue)
cluster_selector = dataproc.messages.ClusterSelector(
clusterLabels=labels, zone=properties.VALUES.compute.zone.GetOrFail())
workflow_template.placement = dataproc.messages.WorkflowTemplatePlacement(
clusterSelector=cluster_selector)
response = dataproc.client.projects_regions_workflowTemplates.Update(
workflow_template)
return response
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
template = util.ParseWorkflowTemplates(args.template, dataproc)
instantiate_request = dataproc.messages.InstantiateWorkflowTemplateRequest()
request = msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateRequest(
instantiateWorkflowTemplateRequest=instantiate_request,
name=template.RelativeName())
operation = dataproc.client.projects_regions_workflowTemplates.Instantiate(
request)
if args.async:
log.status.Print('Running [{0}].'.format(template.Name()))
return
operation = util.WaitForWorkflowTemplateOperation(
dataproc, operation, timeout_s=args.timeout)
return operation
def Run(self, args):
# TODO (b/68774667): deprecate Run command in favor of Instantiate command.
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
template = util.ParseWorkflowTemplates(args.template, dataproc)
instantiate_request = dataproc.messages.InstantiateWorkflowTemplateRequest()
instantiate_request.instanceId = uuid.uuid4().hex # request UUID
request = msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateRequest(
instantiateWorkflowTemplateRequest=instantiate_request,
name=template.RelativeName())
operation = dataproc.client.projects_regions_workflowTemplates.Instantiate(
request)
if args.async:
log.status.Print('Running [{0}].'.format(template.Name()))
return
operation = util.WaitForWorkflowTemplateOperation(
dataproc, operation, timeout_s=args.timeout)
return operation
