def WaitUntilSSHable(user, host, env, key_file, host_key_alias=None, plain=False, strict_host_key_checking=None, timeout=_DEFAULT_TIMEOUT): """Blocks until SSHing to the given host succeeds.""" ssh_args_for_polling = [env.ssh] ssh_args_for_polling.extend(GetDefaultFlags(key_file)) ssh_args_for_polling.extend( GetHostKeyArgs(host_key_alias, plain, strict_host_key_checking)) ssh_args_for_polling.append(UserHost(user, host)) ssh_args_for_polling.append('true') ssh_args_for_polling = LocalizeCommand(ssh_args_for_polling, env) start_sec = time_util.CurrentTimeSec() while True: log.debug('polling instance for SSHability') retval = subprocess.call(ssh_args_for_polling) if retval == 0: break if time_util.CurrentTimeSec() - start_sec > timeout: # TODO(b/33467618): Create another exception raise exceptions.ToolException( 'Could not SSH to the instance. It is possible that ' 'your SSH key has not propagated to the instance yet. ' 'Try running this command again. If you still cannot connect, ' 'verify that the firewall and instance are set to accept ' 'ssh traffic.') time_util.Sleep(5)
def WaitForOperation(client, operation_ref, message): """Waits until the given operation finishes. Wait loop terminates when the operation's status becomes 'DONE'. Args: client: interface to the Cloud Updater API operation_ref: operation to poll message: message to be displayed by progress tracker Returns: True iff the operation finishes with success """ with progress_tracker.ProgressTracker(message, autotick=False) as pt: while True: operation = client.zoneOperations.Get( client.MESSAGES_MODULE. ReplicapoolupdaterZoneOperationsGetRequest( project=operation_ref.project, zone=operation_ref.zone, operation=operation_ref.operation)) if operation.error: return False if operation.status == 'DONE': return True pt.Tick() time_util.Sleep(2)
def _PollUntilDoneUsingOperationGet(self, timeout_sec=_POLLING_TIMEOUT_SEC): """Polls the operation with operation Get method.""" get_request = self.OperationGetRequest() start = time_util.CurrentTimeSec() poll_time_interval = 0 max_poll_interval = 5 # 5 seconds while True: if time_util.CurrentTimeSec() - start > timeout_sec: self.errors.append( (None, 'operation {} timed out'.format(self.operation.name))) _RecordProblems(self.operation, self.warnings, self.errors) return try: self.operation = self._CallService(self.operation_service.Get, get_request) except apitools_exceptions.HttpError: return if self.IsDone(): _RecordProblems(self.operation, self.warnings, self.errors) return poll_time_interval = min(poll_time_interval + 1, max_poll_interval) time_util.Sleep(poll_time_interval)
def WaitUntilSSHable(self, args, user, host, host_key_alias, timeout=_DEFAULT_TIMEOUT): """Blocks until SSHing to the given host succeeds.""" ssh_args_for_polling = [self.ssh_executable] ssh_args_for_polling.extend(self.GetDefaultFlags()) ssh_args_for_polling.extend(self.GetHostKeyArgs(args, host_key_alias)) ssh_args_for_polling.append(UserHost(user, host)) ssh_args_for_polling.append('true') ssh_args_for_polling = self.LocalizeCommand(ssh_args_for_polling) start_sec = time_util.CurrentTimeSec() while True: logging.debug('polling instance for SSHability') retval = subprocess.call(ssh_args_for_polling) if retval == 0: break if time_util.CurrentTimeSec() - start_sec > timeout: raise exceptions.ToolException( 'Could not SSH to the instance. It is possible that ' 'your SSH key has not propagated to the instance yet. ' 'Try running this command again. If you still cannot connect, ' 'verify that the firewall and instance are set to accept ' 'ssh traffic.') time_util.Sleep(5)
def _GetEncryptedPasswordFromSerialPort(self, client, instance_ref, search_modulus): """Returns the decrypted password from the data in the serial port.""" encrypted_password_data = {} start_time = time_util.CurrentTimeSec() count = 1 agent_ready = False while not encrypted_password_data: log.debug('Get Serial Port Output, Try {0}'.format(count)) if (time_util.CurrentTimeSec() > (start_time + WINDOWS_PASSWORD_TIMEOUT_SEC)): raise utils.TimeoutError( TIMEOUT_ERROR.format(time_util.CurrentDatetimeUtc())) serial_port_output = self._GetSerialPortOutput(client, instance_ref, port=4).split('\n') for line in reversed(serial_port_output): try: encrypted_password_dict = json.loads(line) # Sometimes the serial port output only contains a partial entry. except ValueError: continue modulus = encrypted_password_dict.get('modulus') if modulus or encrypted_password_dict.get('ready'): agent_ready = True # Ignore any output that doesn't contain an encrypted password. if not encrypted_password_dict.get('encryptedPassword'): continue if (core_encoding.Decode(search_modulus) == core_encoding.Decode(modulus)): encrypted_password_data = encrypted_password_dict break if not agent_ready: if self.old_metadata_keys: message = OLD_WINDOWS_BUILD_ERROR.format( instance_ref.instance, instance_ref.zone) raise utils.WrongInstanceTypeError(message) else: message = NOT_READY_ERROR raise utils.InstanceNotReadyError(message) time_util.Sleep(POLLING_SEC) count += 1 encrypted_password = encrypted_password_data['encryptedPassword'] return encrypted_password
def Run(self, args): start = time_util.CurrentTimeSec() group_ref = self.CreateGroupReference(args) while True: responses, errors = self._GetResources(group_ref) if errors: utils.RaiseToolException(errors) if wait_info.IsGroupStable(responses[0]): break log.out.Print(wait_info.CreateWaitText(responses[0])) time_util.Sleep(WaitUntilStable._TIME_BETWEEN_POLLS_SEC) if args.timeout and time_util.CurrentTimeSec( ) - start > args.timeout: raise utils.TimeoutError( 'Timeout while waiting for group to become ' 'stable.') log.out.Print('Group is stable')
def Run(self, args): """Issues requests necessary to wait until stable on a MIG.""" holder = base_classes.ComputeApiHolder(self.ReleaseTrack()) client = holder.client start = time_util.CurrentTimeSec() group_ref = self.CreateGroupReference(client, holder.resources, args) while True: responses, errors = self._GetResources(client, group_ref) if errors: utils.RaiseToolException(errors) if wait_info.IsGroupStable(responses[0]): break log.out.Print(wait_info.CreateWaitText(responses[0])) time_util.Sleep(WaitUntilStable._TIME_BETWEEN_POLLS_SEC) if args.timeout and time_util.CurrentTimeSec() - start > args.timeout: raise utils.TimeoutError('Timeout while waiting for group to become ' 'stable.') log.out.Print('Group is stable')
def WaitForOperations(operations_data, http, batch_url, warnings, errors, progress_tracker=None, timeout=None, log_result=True): """Blocks until the given operations are done or until a timeout is reached. Args: operations_data: A list of OperationData objects holding Operations to poll. http: An HTTP object. batch_url: The URL to which batch requests should be sent. warnings: An output parameter for capturing warnings. errors: An output parameter for capturing errors. progress_tracker: progress tracker to tick while waiting for operations to finish. timeout: The maximum amount of time, in seconds, to wait for the operations to reach the DONE state. log_result: Whether the Operation Waiter should print the result in past tense of each request. Yields: The resources pointed to by the operations' targetLink fields if the operation type is not delete. Only resources whose corresponding operations reach done are yielded. """ if not operations_data: return timeout = timeout or _POLLING_TIMEOUT_SEC # Operation -> OperationData mapping will be used to reify operation_service # and resource_service from operation_service.Get(operation) response. # It is necessary because poll operation is returning only response, but we # also need to get operation details to know the service to poll for all # unprocessed_operations. operation_details = {} unprocessed_operations = [] for operation in operations_data: operation_details[operation.operation.selfLink] = operation unprocessed_operations.append(operation.operation) start = time_util.CurrentTimeSec() sleep_sec = 0 # There is only one type of operation in compute API. # We pick the type of the first operation in the list. operation_type = operations_data[0].operation_service.GetResponseType( 'Get') while unprocessed_operations: if progress_tracker: progress_tracker.Tick() resource_requests = [] operation_requests = [] log.debug('Operations to inspect: %s', unprocessed_operations) for operation in unprocessed_operations: # Reify operation data = operation_details[operation.selfLink] # Need to update the operation since old operation may not have all the # required information. data.SetOperation(operation) operation_service = data.operation_service resource_service = data.resource_service if operation.status == operation_type.StatusValueValuesEnum.DONE: # The operation has reached the DONE state, so we record any # problems it contains (if any) and proceed to get the target # resource if there were no problems and the operation is not # a deletion. _RecordProblems(operation, warnings, errors) # We shouldn't attempt to get the target resource if there was # anything wrong with the operation. Note that # httpErrorStatusCode is set only when the operation is not # successful. if (operation.httpErrorStatusCode and operation.httpErrorStatusCode != 200): # httplib.OK continue # Just in case the server did not set httpErrorStatusCode but # the operation did fail, we check the "error" field. if operation.error: continue # We shouldn't get the target resource if the operation type # is delete because there will be no resource left. if not _IsDeleteOp( operation.operationType) and not data.no_followup: request = data.ResourceGetRequest() # Some operations do not have target and should not send get request. if request: resource_requests.append( (resource_service, 'Get', request)) # Only log when there is target link in the operation. if operation.targetLink and log_result: log.status.write('{0} [{1}].\n'.format( _HumanFriendlyNameForOpPastTense( operation.operationType).capitalize(), operation.targetLink)) else: # The operation has not reached the DONE state, so we add a request # to poll the operation. # TODO(b/129413862): Global org operation service supports wait API. if data.IsGlobalOrganizationOperation(): request = data.OperationGetRequest() operation_requests.append( (operation_service, 'Get', request)) else: request = data.OperationWaitRequest() operation_requests.append( (operation_service, 'Wait', request)) requests = resource_requests + operation_requests if not requests: break responses, request_errors = batch_helper.MakeRequests( requests=requests, http=http, batch_url=batch_url) errors.extend(request_errors) all_done = True unprocessed_operations = [] for response in responses: if isinstance(response, operation_type): unprocessed_operations.append(response) if response.status != operation_type.StatusValueValuesEnum.DONE: all_done = False else: yield response # If there are no more operations, we are done. if not unprocessed_operations: break # If all of the operations are done, we should ignore the timeout and ignore # the sleep. if all_done: continue # Did we time out? If so, record the operations that timed out so # they can be reported to the user. if time_util.CurrentTimeSec() - start > timeout: log.debug('Timeout of %ss reached.', timeout) _RecordUnfinishedOperations(unprocessed_operations, errors) break # Sleeps before trying to poll the operations again. sleep_sec = min(sleep_sec + 1, _MAX_TIME_BETWEEN_POLLS_SEC) log.debug('Sleeping for %ss.', sleep_sec) time_util.Sleep(sleep_sec)
def WaitForOperations( operations_data, http, batch_url, warnings, errors, progress_tracker=None, timeout=None): """Blocks until the given operations are done or until a timeout is reached. Args: operations_data: A list of OperationData objects holding Operations to poll. http: An HTTP object. batch_url: The URL to which batch requests should be sent. warnings: An output parameter for capturing warnings. errors: An output parameter for capturing errors. progress_tracker: progress tracker to tick while waiting for operations to finish. timeout: The maximum amount of time, in seconds, to wait for the operations to reach the DONE state. Yields: The resources pointed to by the operations' targetLink fields if the operation type is not delete. Only resources whose corresponding operations reach done are yielded. """ timeout = timeout or _POLLING_TIMEOUT_SEC # Operation -> OperationData mapping will be used to reify operation_service # and resource_service from operation_service.Get(operation) response. # It is necessary because poll operation is returning only response, but we # also need to get operation details to know the service to poll for all # unfinished_operations. operation_details = {} unfinished_operations = [] for operation in operations_data: operation_details[operation.operation.selfLink] = operation unfinished_operations.append(operation.operation) responses = [] start = time_util.CurrentTimeSec() sleep_sec = 0 while unfinished_operations: if progress_tracker: progress_tracker.Tick() resource_requests = [] operation_requests = [] log.debug('Operations to inspect: %s', unfinished_operations) for operation in unfinished_operations: # Reify operation data = operation_details[operation.selfLink] project = data.project operation_service = data.operation_service resource_service = data.resource_service operation_type = operation_service.GetResponseType('Get') if operation.status == operation_type.StatusValueValuesEnum.DONE: # The operation has reached the DONE state, so we record any # problems it contains (if any) and proceed to get the target # resource if there were no problems and the operation is not # a deletion. _RecordProblems(operation, warnings, errors) # We shouldn't attempt to get the target resource if there was # anything wrong with the operation. Note that # httpErrorStatusCode is set only when the operation is not # successful. if (operation.httpErrorStatusCode and operation.httpErrorStatusCode != 200): # httplib.OK continue # Just in case the server did not set httpErrorStatusCode but # the operation did fail, we check the "error" field. if operation.error: continue target_link = operation.targetLink # We shouldn't get the target resource if the operation type # is delete because there will be no resource left. if not _IsDeleteOp(operation.operationType): request = resource_service.GetRequestType('Get')(project=project) if operation.zone: request.zone = path_simplifier.Name(operation.zone) elif operation.region: request.region = path_simplifier.Name(operation.region) name_field = resource_service.GetMethodConfig( 'Get').ordered_params[-1] setattr(request, name_field, path_simplifier.Name(operation.targetLink)) resource_requests.append((resource_service, 'Get', request)) log.status.write('{0} [{1}].\n'.format( _HumanFrieldlyNameForOpPastTense( operation.operationType).capitalize(), target_link)) else: # The operation has not reached the DONE state, so we add a # get request to poll the operation. request = operation_service.GetRequestType('Get')( operation=operation.name, project=project) if operation.zone: request.zone = path_simplifier.Name(operation.zone) elif operation.region: request.region = path_simplifier.Name(operation.region) operation_requests.append((operation_service, 'Get', request)) requests = resource_requests + operation_requests if not requests: break responses, request_errors = batch_helper.MakeRequests( requests=requests, http=http, batch_url=batch_url) errors.extend(request_errors) unfinished_operations = [] for response in responses: if isinstance(response, operation_type): unfinished_operations.append(response) else: yield response # If there are no more operations, we are done. if not unfinished_operations: break # Did we time out? If so, record the operations that timed out so # they can be reported to the user. if time_util.CurrentTimeSec() - start > timeout: log.debug('Timeout of %ss reached.', timeout) _RecordUnfinishedOperations(unfinished_operations, errors) break # Sleeps before trying to poll the operations again. sleep_sec += 1 # Don't re-use sleep_sec, since we want to keep the same time increment sleep_time = min(sleep_sec, _MAX_TIME_BETWEEN_POLLS_SEC) log.debug('Sleeping for %ss.', sleep_time) time_util.Sleep(sleep_time)
def WaitForOperations(operations, project, operation_service, resource_service, http, batch_url, warnings, errors, custom_get_requests=None, timeout=None): """Blocks until the given operations are done or until a timeout is reached. Args: operations: A list of Operation objects to poll. project: The project to which the resources belog. operation_service: The service that can be used to get operation objects. resource_service: The service of the collection being mutated by the operations. If the operation type is not delete, this service is used to fetch the mutated objects after the operations are done. http: An HTTP object. batch_url: The URL to which batch requests should be sent. warnings: An output parameter for capturing warnings. errors: An output parameter for capturing errors. custom_get_requests: A mapping of resource names to requests. If this is provided, when an operation is DONE, instead of performing a get on the targetLink, this function will consult custom_get_requests and perform the request dictated by custom_get_requests. timeout: The maximum amount of time, in seconds, to wait for the operations to reach the DONE state. Yields: The resources pointed to by the operations' targetLink fields if the operation type is not delete. Only resources whose corresponding operations reach done are yielded. """ timeout = timeout or _POLLING_TIMEOUT_SEC operation_type = operation_service.GetResponseType('Get') responses = [] start = time_util.CurrentTimeSec() sleep_sec = 0 while operations: resource_requests = [] operation_requests = [] log.debug('Operations to inspect: %s', operations) for operation in operations: if operation.status == operation_type.StatusValueValuesEnum.DONE: # The operation has reached the DONE state, so we record any # problems it contains (if any) and proceed to get the target # resource if there were no problems and the operation is not # a deletion. _RecordProblems(operation, warnings, errors) # We shouldn't attempt to get the target resource if there was # anything wrong with the operation. Note that # httpErrorStatusCode is set only when the operation is not # successful. if (operation.httpErrorStatusCode and operation.httpErrorStatusCode != httplib.OK): continue # Just in case the server did not set httpErrorStatusCode but # the operation did fail, we check the "error" field. if operation.error: continue target_link = operation.targetLink if custom_get_requests: target_link, service, request_protobuf = ( custom_get_requests[operation.targetLink]) resource_requests.append( (service, 'Get', request_protobuf)) # We shouldn't get the target resource if the operation type # is delete because there will be no resource left. elif not _IsDeleteOp(operation.operationType): request = resource_service.GetRequestType('Get')( project=project) if operation.zone: request.zone = path_simplifier.Name(operation.zone) elif operation.region: request.region = path_simplifier.Name(operation.region) name_field = resource_service.GetMethodConfig( 'Get').ordered_params[-1] setattr(request, name_field, path_simplifier.Name(operation.targetLink)) resource_requests.append( (resource_service, 'Get', request)) log.status.write('{0} [{1}].\n'.format( _HumanFrieldlyNameForOpPastTense( operation.operationType).capitalize(), target_link)) else: # The operation has not reached the DONE state, so we add a # get request to poll the operation. request = operation_service.GetRequestType('Get')( operation=operation.name, project=project) if operation.zone: request.zone = path_simplifier.Name(operation.zone) elif operation.region: request.region = path_simplifier.Name(operation.region) operation_requests.append((operation_service, 'Get', request)) requests = resource_requests + operation_requests if not requests: break responses, request_errors = batch_helper.MakeRequests( requests=requests, http=http, batch_url=batch_url) errors.extend(request_errors) operations = [] for response in responses: if isinstance(response, operation_type): operations.append(response) else: yield response # If there are no more operations, we are done. if not operations: break # Did we time out? If so, record the operations that timed out so # they can be reported to the user. if time_util.CurrentTimeSec() - start > timeout: log.debug('Timeout of %ss reached.', timeout) _RecordUnfinishedOperations(operations, errors) break # Sleeps before trying to poll the operations again. sleep_sec += 1 # Don't re-use sleep_sec, since we want to keep the same time increment sleep_time = min(sleep_sec, _MAX_TIME_BETWEEN_POLLS_SEC) log.debug('Sleeping for %ss.', sleep_time) time_util.Sleep(sleep_time)