def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
# Generate uuid for request.
instance_id = uuid.uuid4().hex
regions_ref = dp_util.ParseRegion(dataproc)
# Read template from YAML file and validate it using a schema.
data = console_io.ReadFromFileOrStdin(args.file or '-', binary=False)
template = export_util.Import(message_type=msgs.WorkflowTemplate,
stream=data,
schema_path=export_util.GetSchemaPath(
'dataproc',
api_version='v1beta2',
message_name='WorkflowTemplate'))
# Send instantiate inline request.
request = \
msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateInlineRequest(
instanceId=instance_id,
parent=regions_ref.RelativeName(),
workflowTemplate=template)
operation = \
dataproc.client.projects_regions_workflowTemplates.InstantiateInline(
request)
if args.async_:
log.status.Print('Instantiating with operation [{0}].'.format(
operation.name))
return
operation = dp_util.WaitForWorkflowTemplateOperation(
dataproc, operation)
return operation
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
# Generate uuid for request.
instance_id = uuid.uuid4().hex
regions_ref = util.ParseRegion(dataproc)
# Read template from YAML file.
template = util.ReadYaml(args.file, msgs.WorkflowTemplate)
# Send instantiate inline request.
request = \
msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateInlineRequest(
instanceId=instance_id,
parent=regions_ref.RelativeName(),
workflowTemplate=template)
operation = \
dataproc.client.projects_regions_workflowTemplates.InstantiateInline(
request)
if args. async:
log.status.Print(
'Instantiating [{0}] with operation [{1}].'.format(
template.id, operation.name))
return
operation = util.WaitForWorkflowTemplateOperation(dataproc, operation)
return operation
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
# Generate uuid for request.
instance_id = uuid.uuid4().hex
regions_ref = util.ParseRegion(dataproc)
# Read template from YAML file and validate it using a schema.
with files.FileReader(args.file) as stream:
template = util.ReadYaml(message_type=msgs.WorkflowTemplate,
stream=stream,
schema_path=V1_BETA2_SCHEMA_PATH)
# Send instantiate inline request.
request = \
msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateInlineRequest(
instanceId=instance_id,
parent=regions_ref.RelativeName(),
workflowTemplate=template)
operation = \
dataproc.client.projects_regions_workflowTemplates.InstantiateInline(
request)
if args. async:
log.status.Print('Instantiating with operation [{0}].'.format(
operation.name))
return
operation = util.WaitForWorkflowTemplateOperation(dataproc, operation)
return operation
def testWaitForWorkflowTemplateOperationTimeout(self):
operation = self.MakeOperation()
exception_message = 'Operation [{0}] timed out.'.format(operation.name)
with self.AssertRaisesExceptionMatches(
exceptions.OperationTimeoutError, exception_message):
util.WaitForWorkflowTemplateOperation(self.dataproc_mock,
operation, 0, 1)
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
template = util.ParseWorkflowTemplates(args.template, dataproc)
instantiate_request = dataproc.messages.InstantiateWorkflowTemplateRequest(
)
instantiate_request.instanceId = uuid.uuid4().hex # request UUID
if args.parameters:
instantiate_request.parameters = encoding.DictToMessage(
args.parameters,
msgs.InstantiateWorkflowTemplateRequest.ParametersValue)
request = msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateRequest(
instantiateWorkflowTemplateRequest=instantiate_request,
name=template.RelativeName())
operation = dataproc.client.projects_regions_workflowTemplates.Instantiate(
request)
if args. async:
log.status.Print(
'Instantiating [{0}] with operation [{1}].'.format(
template.Name(), operation.name))
return
operation = util.WaitForWorkflowTemplateOperation(
dataproc, operation, timeout_s=args.timeout)
return operation
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
# Generate uuid for request.
instance_id = uuid.uuid4().hex
regions_ref = dp_util.ParseRegion(dataproc)
data = console_io.ReadFromFileOrStdin(args.file or '-', binary=False)
template = export_util.Import(message_type=msgs.WorkflowTemplate,
stream=data)
# Send instantiate inline request.
request = \
msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateInlineRequest(
parent=regions_ref.RelativeName(),
workflowTemplate=template)
if dataproc.api_version == 'v1':
# Deprecated field in v1beta2
request.requestId = instance_id
else:
# new field not in v1
request.instanceId = instance_id
operation = \
dataproc.client.projects_regions_workflowTemplates.InstantiateInline(
request)
if args.async_:
log.status.Print('Instantiating with operation [{0}].'.format(
operation.name))
return
operation = dp_util.WaitForWorkflowTemplateOperation(
dataproc, operation)
return operation
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
# Generate uuid for request.
instance_id = uuid.uuid4().hex
regions_ref = dp_util.ParseRegion(dataproc)
if args.file.startswith('gs://'):
data = storage_helpers.ReadObject(args.file)
else:
data = console_io.ReadFromFileOrStdin(args.file, binary=False)
template = export_util.Import(
message_type=msgs.WorkflowTemplate, stream=data)
# Send instantiate inline request.
request = \
msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateInlineRequest(
parent=regions_ref.RelativeName(),
workflowTemplate=template)
request.requestId = instance_id
operation = \
dataproc.client.projects_regions_workflowTemplates.InstantiateInline(
request)
if args.async_:
log.status.Print('Instantiating with operation [{0}].'.format(
operation.name))
return
operation = dp_util.WaitForWorkflowTemplateOperation(dataproc, operation)
return operation
def testWaitForWorkflowTemplateOperation(self):
expected = self.MakeCompletedOperation()
self.ExpectGetOperation()
self.ExpectGetOperation()
self.ExpectGetOperation(operation=expected)
result = util.WaitForWorkflowTemplateOperation(self.dataproc_mock,
self.MakeOperation(),
10, 0)
self.assertEqual(result, expected)
def testWaitForWorkflowTemplateOperationError(self):
operation = self.MakeOperation()
rpc_error = self.MakeRpcError()
self.ExpectGetOperation()
self.ExpectGetOperation()
self.ExpectGetOperation(operation=self.MakeCompletedOperation(
error=rpc_error))
exception_message = 'Operation [{0}] failed: {1}.'.format(
operation.name, util.FormatRpcError(rpc_error))
with self.AssertRaisesExceptionMatches(exceptions.OperationError,
exception_message):
util.WaitForWorkflowTemplateOperation(self.dataproc_mock,
operation, 10, 0)
def testWaitForWorkflowTemplateOperationDeleteClusterError(self):
operation = self.MakeCompletedOperation(
metadata={
'deleteCluster':
collections.OrderedDict([(
'error', 'delete error.'), ('operationId', 'test id')])
})
self.ExpectGetOperation()
self.ExpectGetOperation()
self.ExpectGetOperation(operation=operation)
exception_message = 'Operation [{0}] failed: {1}.'.format(
'test id', 'delete error')
with self.AssertRaisesExceptionMatches(exceptions.OperationError,
exception_message):
util.WaitForWorkflowTemplateOperation(self.dataproc_mock,
operation, 10, 0)
def Run(self, args):
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
template = util.ParseWorkflowTemplates(args.template, dataproc)
instantiate_request = dataproc.messages.InstantiateWorkflowTemplateRequest()
request = msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateRequest(
instantiateWorkflowTemplateRequest=instantiate_request,
name=template.RelativeName())
operation = dataproc.client.projects_regions_workflowTemplates.Instantiate(
request)
if args.async:
log.status.Print('Running [{0}].'.format(template.Name()))
return
operation = util.WaitForWorkflowTemplateOperation(
dataproc, operation, timeout_s=args.timeout)
return operation
def Run(self, args):
# TODO (b/68774667): deprecate Run command in favor of Instantiate command.
dataproc = dp.Dataproc(self.ReleaseTrack())
msgs = dataproc.messages
template_ref = args.CONCEPTS.template.Parse()
instantiate_request = dataproc.messages.InstantiateWorkflowTemplateRequest()
instantiate_request.requestId = uuid.uuid4().hex # request UUID
request = msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateRequest(
instantiateWorkflowTemplateRequest=instantiate_request,
name=template_ref.RelativeName())
operation = dataproc.client.projects_regions_workflowTemplates.Instantiate(
request)
if args.async_:
log.status.Print('Running [{0}].'.format(template_ref.Name()))
return
operation = util.WaitForWorkflowTemplateOperation(
dataproc, operation, timeout_s=args.timeout)
return operation
