def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages # Generate uuid for request. instance_id = uuid.uuid4().hex regions_ref = dp_util.ParseRegion(dataproc) # Read template from YAML file and validate it using a schema. data = console_io.ReadFromFileOrStdin(args.file or '-', binary=False) template = export_util.Import(message_type=msgs.WorkflowTemplate, stream=data, schema_path=export_util.GetSchemaPath( 'dataproc', api_version='v1beta2', message_name='WorkflowTemplate')) # Send instantiate inline request. request = \ msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateInlineRequest( instanceId=instance_id, parent=regions_ref.RelativeName(), workflowTemplate=template) operation = \ dataproc.client.projects_regions_workflowTemplates.InstantiateInline( request) if args.async_: log.status.Print('Instantiating with operation [{0}].'.format( operation.name)) return operation = dp_util.WaitForWorkflowTemplateOperation( dataproc, operation) return operation
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages # Generate uuid for request. instance_id = uuid.uuid4().hex regions_ref = util.ParseRegion(dataproc) # Read template from YAML file. template = util.ReadYaml(args.file, msgs.WorkflowTemplate) # Send instantiate inline request. request = \ msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateInlineRequest( instanceId=instance_id, parent=regions_ref.RelativeName(), workflowTemplate=template) operation = \ dataproc.client.projects_regions_workflowTemplates.InstantiateInline( request) if args. async: log.status.Print( 'Instantiating [{0}] with operation [{1}].'.format( template.id, operation.name)) return operation = util.WaitForWorkflowTemplateOperation(dataproc, operation) return operation
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages # Generate uuid for request. instance_id = uuid.uuid4().hex regions_ref = util.ParseRegion(dataproc) # Read template from YAML file and validate it using a schema. with files.FileReader(args.file) as stream: template = util.ReadYaml(message_type=msgs.WorkflowTemplate, stream=stream, schema_path=V1_BETA2_SCHEMA_PATH) # Send instantiate inline request. request = \ msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateInlineRequest( instanceId=instance_id, parent=regions_ref.RelativeName(), workflowTemplate=template) operation = \ dataproc.client.projects_regions_workflowTemplates.InstantiateInline( request) if args. async: log.status.Print('Instantiating with operation [{0}].'.format( operation.name)) return operation = util.WaitForWorkflowTemplateOperation(dataproc, operation) return operation
def testWaitForWorkflowTemplateOperationTimeout(self): operation = self.MakeOperation() exception_message = 'Operation [{0}] timed out.'.format(operation.name) with self.AssertRaisesExceptionMatches( exceptions.OperationTimeoutError, exception_message): util.WaitForWorkflowTemplateOperation(self.dataproc_mock, operation, 0, 1)
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages template = util.ParseWorkflowTemplates(args.template, dataproc) instantiate_request = dataproc.messages.InstantiateWorkflowTemplateRequest( ) instantiate_request.instanceId = uuid.uuid4().hex # request UUID if args.parameters: instantiate_request.parameters = encoding.DictToMessage( args.parameters, msgs.InstantiateWorkflowTemplateRequest.ParametersValue) request = msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateRequest( instantiateWorkflowTemplateRequest=instantiate_request, name=template.RelativeName()) operation = dataproc.client.projects_regions_workflowTemplates.Instantiate( request) if args. async: log.status.Print( 'Instantiating [{0}] with operation [{1}].'.format( template.Name(), operation.name)) return operation = util.WaitForWorkflowTemplateOperation( dataproc, operation, timeout_s=args.timeout) return operation
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages # Generate uuid for request. instance_id = uuid.uuid4().hex regions_ref = dp_util.ParseRegion(dataproc) data = console_io.ReadFromFileOrStdin(args.file or '-', binary=False) template = export_util.Import(message_type=msgs.WorkflowTemplate, stream=data) # Send instantiate inline request. request = \ msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateInlineRequest( parent=regions_ref.RelativeName(), workflowTemplate=template) if dataproc.api_version == 'v1': # Deprecated field in v1beta2 request.requestId = instance_id else: # new field not in v1 request.instanceId = instance_id operation = \ dataproc.client.projects_regions_workflowTemplates.InstantiateInline( request) if args.async_: log.status.Print('Instantiating with operation [{0}].'.format( operation.name)) return operation = dp_util.WaitForWorkflowTemplateOperation( dataproc, operation) return operation
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages # Generate uuid for request. instance_id = uuid.uuid4().hex regions_ref = dp_util.ParseRegion(dataproc) if args.file.startswith('gs://'): data = storage_helpers.ReadObject(args.file) else: data = console_io.ReadFromFileOrStdin(args.file, binary=False) template = export_util.Import( message_type=msgs.WorkflowTemplate, stream=data) # Send instantiate inline request. request = \ msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateInlineRequest( parent=regions_ref.RelativeName(), workflowTemplate=template) request.requestId = instance_id operation = \ dataproc.client.projects_regions_workflowTemplates.InstantiateInline( request) if args.async_: log.status.Print('Instantiating with operation [{0}].'.format( operation.name)) return operation = dp_util.WaitForWorkflowTemplateOperation(dataproc, operation) return operation
def testWaitForWorkflowTemplateOperation(self): expected = self.MakeCompletedOperation() self.ExpectGetOperation() self.ExpectGetOperation() self.ExpectGetOperation(operation=expected) result = util.WaitForWorkflowTemplateOperation(self.dataproc_mock, self.MakeOperation(), 10, 0) self.assertEqual(result, expected)
def testWaitForWorkflowTemplateOperationError(self): operation = self.MakeOperation() rpc_error = self.MakeRpcError() self.ExpectGetOperation() self.ExpectGetOperation() self.ExpectGetOperation(operation=self.MakeCompletedOperation( error=rpc_error)) exception_message = 'Operation [{0}] failed: {1}.'.format( operation.name, util.FormatRpcError(rpc_error)) with self.AssertRaisesExceptionMatches(exceptions.OperationError, exception_message): util.WaitForWorkflowTemplateOperation(self.dataproc_mock, operation, 10, 0)
def testWaitForWorkflowTemplateOperationDeleteClusterError(self): operation = self.MakeCompletedOperation( metadata={ 'deleteCluster': collections.OrderedDict([( 'error', 'delete error.'), ('operationId', 'test id')]) }) self.ExpectGetOperation() self.ExpectGetOperation() self.ExpectGetOperation(operation=operation) exception_message = 'Operation [{0}] failed: {1}.'.format( 'test id', 'delete error') with self.AssertRaisesExceptionMatches(exceptions.OperationError, exception_message): util.WaitForWorkflowTemplateOperation(self.dataproc_mock, operation, 10, 0)
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages template = util.ParseWorkflowTemplates(args.template, dataproc) instantiate_request = dataproc.messages.InstantiateWorkflowTemplateRequest() request = msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateRequest( instantiateWorkflowTemplateRequest=instantiate_request, name=template.RelativeName()) operation = dataproc.client.projects_regions_workflowTemplates.Instantiate( request) if args.async: log.status.Print('Running [{0}].'.format(template.Name())) return operation = util.WaitForWorkflowTemplateOperation( dataproc, operation, timeout_s=args.timeout) return operation
def Run(self, args): # TODO (b/68774667): deprecate Run command in favor of Instantiate command. dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages template_ref = args.CONCEPTS.template.Parse() instantiate_request = dataproc.messages.InstantiateWorkflowTemplateRequest() instantiate_request.requestId = uuid.uuid4().hex # request UUID request = msgs.DataprocProjectsRegionsWorkflowTemplatesInstantiateRequest( instantiateWorkflowTemplateRequest=instantiate_request, name=template_ref.RelativeName()) operation = dataproc.client.projects_regions_workflowTemplates.Instantiate( request) if args.async_: log.status.Print('Running [{0}].'.format(template_ref.Name())) return operation = util.WaitForWorkflowTemplateOperation( dataproc, operation, timeout_s=args.timeout) return operation