def WaitForOperation(self, operation, message, timeout_s, poll_period_s=5): """Poll dataproc Operation until its status is done or timeout reached. Args: operation: Operation, message of the operation to be polled. message: str, message to display to user while polling. timeout_s: number, seconds to poll with retries before timing out. poll_period_s: number, delay in seconds between requests. Returns: Operation: the return value of the last successful operations.get request. Raises: OperationError: if the operation times out or finishes with an error. """ request = self.messages.DataprocProjectsRegionsOperationsGetRequest( name=operation.name) log.status.Print('Waiting on operation [{0}].'.format(operation.name)) start_time = time.time() warnings_so_far = 0 is_tty = console_io.IsInteractive(error=True) tracker_separator = '\n' if is_tty else '' def _LogWarnings(warnings): new_warnings = warnings[warnings_so_far:] if new_warnings: # Drop a line to print nicely with the progress tracker. log.err.write(tracker_separator) for warning in new_warnings: log.warn(warning) with progress_tracker.ProgressTracker(message, autotick=True): while timeout_s > (time.time() - start_time): try: operation = self.client.projects_regions_operations.Get( request) metadata = self.ParseOperationJsonMetadata( operation.metadata) _LogWarnings(metadata.warnings) warnings_so_far = len(metadata.warnings) if operation.done: break except apitools_exceptions.HttpError: # Keep trying until we timeout in case error is transient. pass time.sleep(poll_period_s) metadata = self.ParseOperationJsonMetadata(operation.metadata) _LogWarnings(metadata.warnings) if not operation.done: raise exceptions.OperationTimeoutError( 'Operation [{0}] timed out.'.format(operation.name)) elif operation.error: raise exceptions.OperationError( 'Operation [{0}] failed: {1}.'.format( operation.name, util.FormatRpcError(operation.error))) log.info('Operation [%s] finished after %.3f seconds', operation.name, (time.time() - start_time)) return operation
def WaitForWorkflowTemplateOperation(dataproc, operation, timeout_s=None, poll_period_s=5): """Poll dataproc Operation until its status is done or timeout reached. Args: dataproc: wrapper for Dataproc messages, resources, and client operation: Operation, message of the operation to be polled. timeout_s: number, seconds to poll with retries before timing out. poll_period_s: number, delay in seconds between requests. Returns: Operation: the return value of the last successful operations.get request. Raises: OperationError: if the operation times out or finishes with an error. """ request = dataproc.messages.DataprocProjectsRegionsOperationsGetRequest( name=operation.name) log.status.Print('Waiting on operation [{0}].'.format(operation.name)) start_time = time.time() operations = {'createCluster': None, 'deleteCluster': None} status = {} errors = {} # If no timeout is specified, poll forever. while timeout_s is None or timeout_s > (time.time() - start_time): try: operation = dataproc.client.projects_regions_operations.Get(request) metadata = ParseOperationJsonMetadata(operation.metadata, dataproc.messages.WorkflowMetadata) PrintWorkflowMetadata(metadata, status, operations, errors) if operation.done: break except apitools_exceptions.HttpError as http_exception: # Do not retry on 4xx errors. if IsClientHttpException(http_exception): raise time.sleep(poll_period_s) metadata = ParseOperationJsonMetadata(operation.metadata, dataproc.messages.WorkflowMetadata) if not operation.done: raise exceptions.OperationTimeoutError( 'Operation [{0}] timed out.'.format(operation.name)) elif operation.error: raise exceptions.OperationError('Operation [{0}] failed: {1}.'.format( operation.name, FormatRpcError(operation.error))) for op in ['createCluster', 'deleteCluster']: if op in operations and operations[op] is not None and operations[op].error: raise exceptions.OperationError('Operation [{0}] failed: {1}.'.format( operations[op].operationId, operations[op].error)) log.info('Operation [%s] finished after %.3f seconds', operation.name, (time.time() - start_time)) return operation
def WaitForOperation(operation, context, message, timeout_s=2100, poll_period_s=5): """Poll dataproc Operation until its status is done or timeout reached. Args: operation: Operation, message of the operation to be polled. context: dict, dataproc Command context. message: str, message to display to user while polling. timeout_s: number, seconds to poll with retries before timing out. poll_period_s: number, delay in seconds between requests. Returns: Operation: the return value of the last successful operations.get request. Raises: OperationError: if the operation times out or finishes with an error. """ client = context['dataproc_client'] messages = context['dataproc_messages'] request = messages.DataprocProjectsRegionsOperationsGetRequest( name=operation.name) log.status.Print('Waiting on operation [{0}].'.format(operation.name)) start_time = time.time() with progress_tracker.ProgressTracker(message, autotick=True): while timeout_s > (time.time() - start_time): try: operation = client.projects_regions_operations.Get(request) if operation.done: break except apitools_exceptions.HttpError: # Keep trying until we timeout in case error is transient. pass time.sleep(poll_period_s) # TODO(user): Parse operation metadata. log.debug('Operation:\n' + encoding.MessageToJson(operation)) if not operation.done: raise exceptions.OperationTimeoutError( 'Operation [{0}] timed out.'.format(operation.name)) elif operation.error: raise exceptions.OperationError('Operation [{0}] failed: {1}.'.format( operation.name, FormatRpcError(operation.error))) log.info('Operation [%s] finished after %.3f seconds', operation.name, (time.time() - start_time)) return operation
def WaitForResourceDeletion(request_method, resource_ref, message, timeout_s=60, poll_period_s=5): """Poll Dataproc resource until it no longer exists.""" with progress_tracker.ProgressTracker(message, autotick=True): start_time = time.time() while timeout_s > (time.time() - start_time): try: request_method(resource_ref) except apitools_exceptions.HttpError as error: if error.status_code == 404: # Object deleted return log.debug('Get request for [{0}] failed:\n{1}', resource_ref, error) # Keep trying until we timeout in case error is transient. time.sleep(poll_period_s) raise exceptions.OperationTimeoutError( 'Deleting resource [{0}] timed out.'.format(resource_ref))
def _GetResult(self, session): """Handles errors. Error handling for sessions. This happen after the session reaches one of the complete states. Overrides. Args: session: The session resource. Returns: None. The result is directly output to log.err. Raises: OperationTimeoutError: When waiter timed out. OperationError: When remote session creation is failed. """ if not session: # Session resource is None but polling is considered done. # This only happens when the waiter timed out. raise exceptions.OperationTimeoutError( 'Timed out while waiting for session creation.') if (session.state == self.dataproc.messages.Session.StateValueValuesEnum.FAILED): err_message = 'Session creation is FAILED.' if session.stateMessage: err_message = '{} Detail: {}'.format(err_message, session.stateMessage) if err_message[-1] != '.': err_message += '.' raise exceptions.OperationError(err_message) # Nothing to return, since the result is directly output to users. return self._GetOutputUri(session)
def WaitForResourceDeletion(request_method, resource_ref, message, timeout_s=60, poll_period_s=5): """Poll Dataproc resource until it no longer exists.""" with progress_tracker.ProgressTracker(message, autotick=True): start_time = time.time() while timeout_s > (time.time() - start_time): try: request_method(resource_ref) except apitools_exceptions.HttpNotFoundError: # Object deleted return except apitools_exceptions.HttpError as error: log.debug('Get request for [{0}] failed:\n{1}', resource_ref, error) # Do not retry on 4xx errors if IsClientHttpException(error): raise time.sleep(poll_period_s) raise exceptions.OperationTimeoutError( 'Deleting resource [{0}] timed out.'.format(resource_ref))