def WaitUntilSSHable(user,
host,
env,
key_file,
host_key_alias=None,
plain=False,
strict_host_key_checking=None,
timeout=_DEFAULT_TIMEOUT):
"""Blocks until SSHing to the given host succeeds."""
ssh_args_for_polling = [env.ssh]
ssh_args_for_polling.extend(GetDefaultFlags(key_file))
ssh_args_for_polling.extend(
GetHostKeyArgs(host_key_alias, plain, strict_host_key_checking))
ssh_args_for_polling.append(UserHost(user, host))
ssh_args_for_polling.append('true')
ssh_args_for_polling = LocalizeCommand(ssh_args_for_polling, env)
start_sec = time_util.CurrentTimeSec()
while True:
log.debug('polling instance for SSHability')
retval = subprocess.call(ssh_args_for_polling)
if retval == 0:
break
if time_util.CurrentTimeSec() - start_sec > timeout:
# TODO(b/33467618): Create another exception
raise exceptions.ToolException(
'Could not SSH to the instance. It is possible that '
'your SSH key has not propagated to the instance yet. '
'Try running this command again. If you still cannot connect, '
'verify that the firewall and instance are set to accept '
'ssh traffic.')
time_util.Sleep(5)
def WaitUntilSSHable(self, args, user, host, host_key_alias,
timeout=_DEFAULT_TIMEOUT):
"""Blocks until SSHing to the given host succeeds."""
ssh_args_for_polling = [self.ssh_executable]
ssh_args_for_polling.extend(self.GetDefaultFlags())
ssh_args_for_polling.extend(self.GetHostKeyArgs(args, host_key_alias))
ssh_args_for_polling.append(UserHost(user, host))
ssh_args_for_polling.append('true')
ssh_args_for_polling = self.LocalizeCommand(ssh_args_for_polling)
start_sec = time_util.CurrentTimeSec()
while True:
logging.debug('polling instance for SSHability')
retval = subprocess.call(ssh_args_for_polling)
if retval == 0:
break
if time_util.CurrentTimeSec() - start_sec > timeout:
raise exceptions.ToolException(
'Could not SSH to the instance. It is possible that '
'your SSH key has not propagated to the instance yet. '
'Try running this command again. If you still cannot connect, '
'verify that the firewall and instance are set to accept '
'ssh traffic.')
time_util.Sleep(5)
def _PollUntilDoneUsingOperationGet(self,
timeout_sec=_POLLING_TIMEOUT_SEC):
"""Polls the operation with operation Get method."""
get_request = self.OperationGetRequest()
start = time_util.CurrentTimeSec()
poll_time_interval = 0
max_poll_interval = 5 # 5 seconds
while True:
if time_util.CurrentTimeSec() - start > timeout_sec:
self.errors.append(
(None,
'operation {} timed out'.format(self.operation.name)))
_RecordProblems(self.operation, self.warnings, self.errors)
return
try:
self.operation = self._CallService(self.operation_service.Get,
get_request)
except apitools_exceptions.HttpError:
return
if self.IsDone():
_RecordProblems(self.operation, self.warnings, self.errors)
return
poll_time_interval = min(poll_time_interval + 1, max_poll_interval)
time_util.Sleep(poll_time_interval)
def _GetEncryptedPasswordFromSerialPort(self, client, instance_ref,
search_modulus):
"""Returns the decrypted password from the data in the serial port."""
encrypted_password_data = {}
start_time = time_util.CurrentTimeSec()
count = 1
agent_ready = False
while not encrypted_password_data:
log.debug('Get Serial Port Output, Try {0}'.format(count))
if (time_util.CurrentTimeSec() >
(start_time + WINDOWS_PASSWORD_TIMEOUT_SEC)):
raise utils.TimeoutError(
TIMEOUT_ERROR.format(time_util.CurrentDatetimeUtc()))
serial_port_output = self._GetSerialPortOutput(client,
instance_ref,
port=4).split('\n')
for line in reversed(serial_port_output):
try:
encrypted_password_dict = json.loads(line)
# Sometimes the serial port output only contains a partial entry.
except ValueError:
continue
modulus = encrypted_password_dict.get('modulus')
if modulus or encrypted_password_dict.get('ready'):
agent_ready = True
# Ignore any output that doesn't contain an encrypted password.
if not encrypted_password_dict.get('encryptedPassword'):
continue
if (core_encoding.Decode(search_modulus) ==
core_encoding.Decode(modulus)):
encrypted_password_data = encrypted_password_dict
break
if not agent_ready:
if self.old_metadata_keys:
message = OLD_WINDOWS_BUILD_ERROR.format(
instance_ref.instance, instance_ref.zone)
raise utils.WrongInstanceTypeError(message)
else:
message = NOT_READY_ERROR
raise utils.InstanceNotReadyError(message)
time_util.Sleep(POLLING_SEC)
count += 1
encrypted_password = encrypted_password_data['encryptedPassword']
return encrypted_password
def _PollUntilDoneUsingOperationWait(self, timeout_sec=_POLLING_TIMEOUT_SEC):
"""Polls the operation with operation method."""
wait_request = self.OperationWaitRequest()
start = time_util.CurrentTimeSec()
while not self.IsDone():
if time_util.CurrentTimeSec() - start > timeout_sec:
self.errors.append(
(None, 'operation {} timed out'.format(self.operation.name)))
_RecordProblems(self.operation, self.warnings, self.errors)
return
try:
self.operation = self._CallService(self.operation_service.Wait,
wait_request)
except apitools_exceptions.HttpError:
return
_RecordProblems(self.operation, self.warnings, self.errors)
def Run(self, args):
start = time_util.CurrentTimeSec()
group_ref = self.CreateGroupReference(args)
while True:
responses, errors = self._GetResources(group_ref)
if errors:
utils.RaiseToolException(errors)
if wait_info.IsGroupStable(responses[0]):
break
log.out.Print(wait_info.CreateWaitText(responses[0]))
time_util.Sleep(WaitUntilStable._TIME_BETWEEN_POLLS_SEC)
if args.timeout and time_util.CurrentTimeSec(
) - start > args.timeout:
raise utils.TimeoutError(
'Timeout while waiting for group to become '
'stable.')
log.out.Print('Group is stable')
def Run(self, args):
"""Issues requests necessary to wait until stable on a MIG."""
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
start = time_util.CurrentTimeSec()
group_ref = self.CreateGroupReference(client, holder.resources, args)
while True:
responses, errors = self._GetResources(client, group_ref)
if errors:
utils.RaiseToolException(errors)
if wait_info.IsGroupStable(responses[0]):
break
log.out.Print(wait_info.CreateWaitText(responses[0]))
time_util.Sleep(WaitUntilStable._TIME_BETWEEN_POLLS_SEC)
if args.timeout and time_util.CurrentTimeSec() - start > args.timeout:
raise utils.TimeoutError('Timeout while waiting for group to become '
'stable.')
log.out.Print('Group is stable')
def WaitForOperations(operations_data,
http,
batch_url,
warnings,
errors,
progress_tracker=None,
timeout=None,
log_result=True):
"""Blocks until the given operations are done or until a timeout is reached.
Args:
operations_data: A list of OperationData objects holding Operations to poll.
http: An HTTP object.
batch_url: The URL to which batch requests should be sent.
warnings: An output parameter for capturing warnings.
errors: An output parameter for capturing errors.
progress_tracker: progress tracker to tick while waiting for operations to
finish.
timeout: The maximum amount of time, in seconds, to wait for the
operations to reach the DONE state.
log_result: Whether the Operation Waiter should print the result in past
tense of each request.
Yields:
The resources pointed to by the operations' targetLink fields if
the operation type is not delete. Only resources whose
corresponding operations reach done are yielded.
"""
if not operations_data:
return
timeout = timeout or _POLLING_TIMEOUT_SEC
# Operation -> OperationData mapping will be used to reify operation_service
# and resource_service from operation_service.Get(operation) response.
# It is necessary because poll operation is returning only response, but we
# also need to get operation details to know the service to poll for all
# unprocessed_operations.
operation_details = {}
unprocessed_operations = []
for operation in operations_data:
operation_details[operation.operation.selfLink] = operation
unprocessed_operations.append(operation.operation)
start = time_util.CurrentTimeSec()
sleep_sec = 0
# There is only one type of operation in compute API.
# We pick the type of the first operation in the list.
operation_type = operations_data[0].operation_service.GetResponseType(
'Get')
while unprocessed_operations:
if progress_tracker:
progress_tracker.Tick()
resource_requests = []
operation_requests = []
log.debug('Operations to inspect: %s', unprocessed_operations)
for operation in unprocessed_operations:
# Reify operation
data = operation_details[operation.selfLink]
# Need to update the operation since old operation may not have all the
# required information.
data.SetOperation(operation)
operation_service = data.operation_service
resource_service = data.resource_service
if operation.status == operation_type.StatusValueValuesEnum.DONE:
# The operation has reached the DONE state, so we record any
# problems it contains (if any) and proceed to get the target
# resource if there were no problems and the operation is not
# a deletion.
_RecordProblems(operation, warnings, errors)
# We shouldn't attempt to get the target resource if there was
# anything wrong with the operation. Note that
# httpErrorStatusCode is set only when the operation is not
# successful.
if (operation.httpErrorStatusCode and
operation.httpErrorStatusCode != 200): # httplib.OK
continue
# Just in case the server did not set httpErrorStatusCode but
# the operation did fail, we check the "error" field.
if operation.error:
continue
# We shouldn't get the target resource if the operation type
# is delete because there will be no resource left.
if not _IsDeleteOp(
operation.operationType) and not data.no_followup:
request = data.ResourceGetRequest()
# Some operations do not have target and should not send get request.
if request:
resource_requests.append(
(resource_service, 'Get', request))
# Only log when there is target link in the operation.
if operation.targetLink and log_result:
log.status.write('{0} [{1}].\n'.format(
_HumanFriendlyNameForOpPastTense(
operation.operationType).capitalize(),
operation.targetLink))
else:
# The operation has not reached the DONE state, so we add a request
# to poll the operation.
# TODO(b/129413862): Global org operation service supports wait API.
if data.IsGlobalOrganizationOperation():
request = data.OperationGetRequest()
operation_requests.append(
(operation_service, 'Get', request))
else:
request = data.OperationWaitRequest()
operation_requests.append(
(operation_service, 'Wait', request))
requests = resource_requests + operation_requests
if not requests:
break
responses, request_errors = batch_helper.MakeRequests(
requests=requests, http=http, batch_url=batch_url)
errors.extend(request_errors)
all_done = True
unprocessed_operations = []
for response in responses:
if isinstance(response, operation_type):
unprocessed_operations.append(response)
if response.status != operation_type.StatusValueValuesEnum.DONE:
all_done = False
else:
yield response
# If there are no more operations, we are done.
if not unprocessed_operations:
break
# If all of the operations are done, we should ignore the timeout and ignore
# the sleep.
if all_done:
continue
# Did we time out? If so, record the operations that timed out so
# they can be reported to the user.
if time_util.CurrentTimeSec() - start > timeout:
log.debug('Timeout of %ss reached.', timeout)
_RecordUnfinishedOperations(unprocessed_operations, errors)
break
# Sleeps before trying to poll the operations again.
sleep_sec = min(sleep_sec + 1, _MAX_TIME_BETWEEN_POLLS_SEC)
log.debug('Sleeping for %ss.', sleep_sec)
time_util.Sleep(sleep_sec)
def Run(self, args):
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
start = time_util.CurrentTimeSec()
# Set up Encryption utilities.
openssl_executable = files.FindExecutableOnPath('openssl')
if windows_encryption_utils:
crypt = windows_encryption_utils.WinCrypt()
elif openssl_executable:
crypt = openssl_encryption_utils.OpensslCrypt(openssl_executable)
else:
raise utils.MissingDependencyError(
'Your platform does not support OpenSSL.')
# Get Authenticated email address and default username.
email = properties.VALUES.core.account.GetOrFail()
if args.user:
user = args.user
else:
user = gaia.MapGaiaEmailToDefaultAccountName(email)
if args.instance_name == user:
raise utils.InvalidUserError(
MACHINE_USERNAME_SAME_ERROR.format(user, args.instance_name))
# Warn user (This warning doesn't show for non-interactive sessions).
message = RESET_PASSWORD_WARNING.format(user)
prompt_string = (
'Would you like to set or reset the password for [{0}]'.format(
user))
console_io.PromptContinue(message=message,
prompt_string=prompt_string,
cancel_on_no=True)
log.status.Print(
'Resetting and retrieving password for [{0}] on [{1}]'.format(
user, args.instance_name))
# Get Encryption Keys.
key = crypt.GetKeyPair()
modulus, exponent = crypt.GetModulusExponentFromPublicKey(
crypt.GetPublicKey(key))
# Create Windows key entry.
self.windows_key_entry = self._ConstructWindowsKeyEntry(
user, modulus, exponent, email)
# Call ReadWriteCommad.Run() which will fetch the instance and update
# the metadata (using the data in self.windows_key_entry).
instance_ref = self.CreateReference(client, holder.resources, args)
get_request = self.GetGetRequest(client, instance_ref)
objects = client.MakeRequests([get_request])
new_object = self.Modify(client, objects[0])
# If existing object is equal to the proposed object or if
# Modify() returns None, then there is no work to be done, so we
# print the resource and return.
if objects[0] == new_object:
log.status.Print(
'No change requested; skipping update for [{0}].'.format(
objects[0].name))
return objects
updated_instance = client.MakeRequests(
[self.GetSetRequest(client, instance_ref, new_object)])[0]
# Retrieve and Decrypt the password from the serial console.
enc_password = self._GetEncryptedPasswordFromSerialPort(
client, instance_ref, modulus)
password = crypt.DecryptMessage(key, enc_password)
# Get External IP address.
try:
access_configs = updated_instance.networkInterfaces[
0].accessConfigs
external_ip_address = access_configs[0].natIP
except (KeyError, IndexError) as _:
log.warning(NO_IP_WARNING.format(updated_instance.name))
external_ip_address = None
# Check for old Windows credentials.
if self.old_metadata_keys:
log.warning(
OLD_KEYS_WARNING.format(instance_ref.instance,
instance_ref.instance,
instance_ref.zone,
','.join(self.old_metadata_keys)))
log.info('Total Elapsed Time: {0}'.format(time_util.CurrentTimeSec() -
start))
# The connection info resource.
connection_info = {
'username': user,
'password': password,
'ip_address': external_ip_address
}
return connection_info
def Run(self, args):
start = time_util.CurrentTimeSec()
# Set up Encryption utilities.
openssl_executable = files.FindExecutableOnPath('openssl')
if windows_encryption_utils:
crypt = windows_encryption_utils.WinCrypt()
elif openssl_executable:
crypt = openssl_encryption_utils.OpensslCrypt(openssl_executable)
else:
raise utils.MissingDependencyError(
'Your platform does not support OpenSSL.')
# Get Authenticated email address and default username.
email = gaia.GetAuthenticatedGaiaEmail(self.http)
if args.user:
user = args.user
else:
user = gaia.MapGaiaEmailToDefaultAccountName(email)
if args.name == user:
raise utils.InvalidUserError(
MACHINE_USERNAME_SAME_ERROR.format(user, args.name))
# Warn user (This warning doesn't show for non-interactive sessions).
message = RESET_PASSWORD_WARNING.format(user)
prompt_string = (
'Would you like to set or reset the password for [{0}]'.format(
user))
console_io.PromptContinue(message=message,
prompt_string=prompt_string,
cancel_on_no=True)
log.status.Print(
'Resetting and retrieving password for [{0}] on [{1}]'.format(
user, args.name))
# Get Encryption Keys.
key = crypt.GetKeyPair()
modulus, exponent = crypt.GetModulusExponentFromPublicKey(
crypt.GetPublicKey(key))
# Create Windows key entry.
self.windows_key_entry = self._ConstructWindowsKeyEntry(
user, modulus, exponent, email)
# Call ReadWriteCommad.Run() which will fetch the instance and update
# the metadata (using the data in self.windows_key_entry).
objects = super(ResetWindowsPassword, self).Run(args)
updated_instance = list(objects)[0]
# Retrieve and Decrypt the password from the serial console.
enc_password = self._GetEncryptedPasswordFromSerialPort(modulus)
password = crypt.DecryptMessage(key, enc_password)
# Get External IP address.
try:
access_configs = updated_instance['networkInterfaces'][0][
'accessConfigs']
external_ip_address = access_configs[0]['natIP']
except KeyError:
log.warn(NO_IP_WARNING.format(updated_instance['name']))
external_ip_address = None
# Check for old Windows credentials.
if self.old_metadata_keys:
log.warn(
OLD_KEYS_WARNING.format(self.ref.Name(), self.ref.Name(),
self.ref.zone,
','.join(self.old_metadata_keys)))
log.info('Total Elapsed Time: {0}'.format(time_util.CurrentTimeSec() -
start))
# The connection info resource.
connection_info = {
'username': user,
'password': password,
'ip_address': external_ip_address
}
return connection_info
def WaitForOperations(
operations_data, http, batch_url, warnings, errors,
progress_tracker=None, timeout=None):
"""Blocks until the given operations are done or until a timeout is reached.
Args:
operations_data: A list of OperationData objects holding Operations to poll.
http: An HTTP object.
batch_url: The URL to which batch requests should be sent.
warnings: An output parameter for capturing warnings.
errors: An output parameter for capturing errors.
progress_tracker: progress tracker to tick while waiting for operations to
finish.
timeout: The maximum amount of time, in seconds, to wait for the
operations to reach the DONE state.
Yields:
The resources pointed to by the operations' targetLink fields if
the operation type is not delete. Only resources whose
corresponding operations reach done are yielded.
"""
timeout = timeout or _POLLING_TIMEOUT_SEC
# Operation -> OperationData mapping will be used to reify operation_service
# and resource_service from operation_service.Get(operation) response.
# It is necessary because poll operation is returning only response, but we
# also need to get operation details to know the service to poll for all
# unfinished_operations.
operation_details = {}
unfinished_operations = []
for operation in operations_data:
operation_details[operation.operation.selfLink] = operation
unfinished_operations.append(operation.operation)
responses = []
start = time_util.CurrentTimeSec()
sleep_sec = 0
while unfinished_operations:
if progress_tracker:
progress_tracker.Tick()
resource_requests = []
operation_requests = []
log.debug('Operations to inspect: %s', unfinished_operations)
for operation in unfinished_operations:
# Reify operation
data = operation_details[operation.selfLink]
project = data.project
operation_service = data.operation_service
resource_service = data.resource_service
operation_type = operation_service.GetResponseType('Get')
if operation.status == operation_type.StatusValueValuesEnum.DONE:
# The operation has reached the DONE state, so we record any
# problems it contains (if any) and proceed to get the target
# resource if there were no problems and the operation is not
# a deletion.
_RecordProblems(operation, warnings, errors)
# We shouldn't attempt to get the target resource if there was
# anything wrong with the operation. Note that
# httpErrorStatusCode is set only when the operation is not
# successful.
if (operation.httpErrorStatusCode and
operation.httpErrorStatusCode != 200): # httplib.OK
continue
# Just in case the server did not set httpErrorStatusCode but
# the operation did fail, we check the "error" field.
if operation.error:
continue
target_link = operation.targetLink
# We shouldn't get the target resource if the operation type
# is delete because there will be no resource left.
if not _IsDeleteOp(operation.operationType):
request = resource_service.GetRequestType('Get')(project=project)
if operation.zone:
request.zone = path_simplifier.Name(operation.zone)
elif operation.region:
request.region = path_simplifier.Name(operation.region)
name_field = resource_service.GetMethodConfig(
'Get').ordered_params[-1]
setattr(request, name_field,
path_simplifier.Name(operation.targetLink))
resource_requests.append((resource_service, 'Get', request))
log.status.write('{0} [{1}].\n'.format(
_HumanFrieldlyNameForOpPastTense(
operation.operationType).capitalize(),
target_link))
else:
# The operation has not reached the DONE state, so we add a
# get request to poll the operation.
request = operation_service.GetRequestType('Get')(
operation=operation.name,
project=project)
if operation.zone:
request.zone = path_simplifier.Name(operation.zone)
elif operation.region:
request.region = path_simplifier.Name(operation.region)
operation_requests.append((operation_service, 'Get', request))
requests = resource_requests + operation_requests
if not requests:
break
responses, request_errors = batch_helper.MakeRequests(
requests=requests,
http=http,
batch_url=batch_url)
errors.extend(request_errors)
unfinished_operations = []
for response in responses:
if isinstance(response, operation_type):
unfinished_operations.append(response)
else:
yield response
# If there are no more operations, we are done.
if not unfinished_operations:
break
# Did we time out? If so, record the operations that timed out so
# they can be reported to the user.
if time_util.CurrentTimeSec() - start > timeout:
log.debug('Timeout of %ss reached.', timeout)
_RecordUnfinishedOperations(unfinished_operations, errors)
break
# Sleeps before trying to poll the operations again.
sleep_sec += 1
# Don't re-use sleep_sec, since we want to keep the same time increment
sleep_time = min(sleep_sec, _MAX_TIME_BETWEEN_POLLS_SEC)
log.debug('Sleeping for %ss.', sleep_time)
time_util.Sleep(sleep_time)
def WaitForOperations(operations,
project,
operation_service,
resource_service,
http,
batch_url,
warnings,
errors,
custom_get_requests=None,
timeout=None):
"""Blocks until the given operations are done or until a timeout is reached.
Args:
operations: A list of Operation objects to poll.
project: The project to which the resources belog.
operation_service: The service that can be used to get operation
objects.
resource_service: The service of the collection being mutated by
the operations. If the operation type is not delete, this service
is used to fetch the mutated objects after the operations are done.
http: An HTTP object.
batch_url: The URL to which batch requests should be sent.
warnings: An output parameter for capturing warnings.
errors: An output parameter for capturing errors.
custom_get_requests: A mapping of resource names to requests. If
this is provided, when an operation is DONE, instead of performing
a get on the targetLink, this function will consult custom_get_requests
and perform the request dictated by custom_get_requests.
timeout: The maximum amount of time, in seconds, to wait for the
operations to reach the DONE state.
Yields:
The resources pointed to by the operations' targetLink fields if
the operation type is not delete. Only resources whose
corresponding operations reach done are yielded.
"""
timeout = timeout or _POLLING_TIMEOUT_SEC
operation_type = operation_service.GetResponseType('Get')
responses = []
start = time_util.CurrentTimeSec()
sleep_sec = 0
while operations:
resource_requests = []
operation_requests = []
log.debug('Operations to inspect: %s', operations)
for operation in operations:
if operation.status == operation_type.StatusValueValuesEnum.DONE:
# The operation has reached the DONE state, so we record any
# problems it contains (if any) and proceed to get the target
# resource if there were no problems and the operation is not
# a deletion.
_RecordProblems(operation, warnings, errors)
# We shouldn't attempt to get the target resource if there was
# anything wrong with the operation. Note that
# httpErrorStatusCode is set only when the operation is not
# successful.
if (operation.httpErrorStatusCode
and operation.httpErrorStatusCode != httplib.OK):
continue
# Just in case the server did not set httpErrorStatusCode but
# the operation did fail, we check the "error" field.
if operation.error:
continue
target_link = operation.targetLink
if custom_get_requests:
target_link, service, request_protobuf = (
custom_get_requests[operation.targetLink])
resource_requests.append(
(service, 'Get', request_protobuf))
# We shouldn't get the target resource if the operation type
# is delete because there will be no resource left.
elif not _IsDeleteOp(operation.operationType):
request = resource_service.GetRequestType('Get')(
project=project)
if operation.zone:
request.zone = path_simplifier.Name(operation.zone)
elif operation.region:
request.region = path_simplifier.Name(operation.region)
name_field = resource_service.GetMethodConfig(
'Get').ordered_params[-1]
setattr(request, name_field,
path_simplifier.Name(operation.targetLink))
resource_requests.append(
(resource_service, 'Get', request))
log.status.write('{0} [{1}].\n'.format(
_HumanFrieldlyNameForOpPastTense(
operation.operationType).capitalize(), target_link))
else:
# The operation has not reached the DONE state, so we add a
# get request to poll the operation.
request = operation_service.GetRequestType('Get')(
operation=operation.name, project=project)
if operation.zone:
request.zone = path_simplifier.Name(operation.zone)
elif operation.region:
request.region = path_simplifier.Name(operation.region)
operation_requests.append((operation_service, 'Get', request))
requests = resource_requests + operation_requests
if not requests:
break
responses, request_errors = batch_helper.MakeRequests(
requests=requests, http=http, batch_url=batch_url)
errors.extend(request_errors)
operations = []
for response in responses:
if isinstance(response, operation_type):
operations.append(response)
else:
yield response
# If there are no more operations, we are done.
if not operations:
break
# Did we time out? If so, record the operations that timed out so
# they can be reported to the user.
if time_util.CurrentTimeSec() - start > timeout:
log.debug('Timeout of %ss reached.', timeout)
_RecordUnfinishedOperations(operations, errors)
break
# Sleeps before trying to poll the operations again.
sleep_sec += 1
# Don't re-use sleep_sec, since we want to keep the same time increment
sleep_time = min(sleep_sec, _MAX_TIME_BETWEEN_POLLS_SEC)
log.debug('Sleeping for %ss.', sleep_time)
time_util.Sleep(sleep_time)
