def wait_for_encryption(enc_svc,
progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT):
err_count = 0
max_errs = 10
start_time = time.time()
last_log_time = start_time
progress_deadline = Deadline(progress_timeout)
last_progress = 0
last_state = ''
while err_count < max_errs:
try:
status = enc_svc.get_status()
err_count = 0
except Exception as e:
log.warn("Failed getting encryption status: %s", e)
log.warn("Retrying. . .")
err_count += 1
sleep(10)
continue
state = status['state']
percent_complete = status['percent_complete']
log.debug('state=%s, percent_complete=%d', state, percent_complete)
# Make sure that encryption progress hasn't stalled.
if progress_deadline.is_expired():
raise EncryptionError(
'Waited for encryption progress for longer than %s seconds' %
progress_timeout
)
if percent_complete > last_progress or state != last_state:
last_progress = percent_complete
last_state = state
progress_deadline = Deadline(progress_timeout)
# Log progress once a minute.
now = time.time()
if now - last_log_time >= 60:
if state == ENCRYPT_INITIALIZING:
log.info('Encryption process is initializing')
else:
state_display = 'Encryption'
if state == ENCRYPT_DOWNLOADING:
state_display = 'Download from cloud storage'
log.info(
'%s is %d%% complete', state_display, percent_complete)
last_log_time = now
if state == ENCRYPT_SUCCESSFUL:
log.info('Encrypted root drive created.')
return
elif state == ENCRYPT_FAILED:
log.error('Encryption status: %s', json.dumps(status))
_handle_failure_code(status.get('failure_code'))
sleep(10)
# We've failed to get encryption status for _max_errs_ consecutive tries.
# Assume that the server has crashed.
raise EncryptionError('Encryption service unavailable')
def wait_for_instance(
aws_svc, instance_id, timeout=600, state='running'):
""" Wait for up to timeout seconds for an instance to be in the
given state. Sleep for 2 seconds between checks.
:return: The Instance object
:raises InstanceError if a timeout occurs or the instance unexpectedly
goes into an error or terminated state
"""
log.debug(
'Waiting for %s, timeout=%d, state=%s',
instance_id, timeout, state)
deadline = Deadline(timeout)
while not deadline.is_expired():
instance = aws_svc.get_instance(instance_id)
log.debug('Instance %s state=%s', instance.id, instance.state)
if instance.state == state:
return instance
if instance.state == 'error':
raise InstanceError(
'Instance %s is in an error state. Cannot proceed.' %
instance_id
)
if state != 'terminated' and instance.state == 'terminated':
raise InstanceError(
'Instance %s was unexpectedly terminated.' % instance_id
)
sleep(2)
raise InstanceError(
'Timed out waiting for %s to be in the %s state' %
(instance_id, state)
)
def _wait_for_instance(
aws_svc, instance_id, timeout=300, state='running'):
""" Wait for up to timeout seconds for an instance to be in the
'running' state. Sleep for 2 seconds between checks.
:return: The Instance object, or None if a timeout occurred
"""
log.debug(
'Waiting for %s, timeout=%d, state=%s',
instance_id, timeout, state)
# Wait for AWS eventual consistency to catch up.
instance = _safe_get_instance(aws_svc, instance_id)
deadline = Deadline(timeout)
while not deadline.is_expired():
log.debug('Instance %s state=%s', instance.id, instance.state)
if instance.state == state:
return instance
if instance.state == 'error':
raise Exception(
'Instance %s is in an error state. Cannot proceed.'
)
_sleep(2)
instance = aws_svc.get_instance(instance_id)
raise Exception(
'Timed out waiting for %s to be in the %s state' %
(instance_id, state)
)
def wait_for_encryption(enc_svc,
progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT):
err_count = 0
max_errs = 10
start_time = time.time()
last_log_time = start_time
progress_deadline = Deadline(progress_timeout)
last_progress = 0
while err_count < max_errs:
try:
status = enc_svc.get_status()
err_count = 0
except Exception as e:
log.warn("Failed getting encryption status: %s", e)
err_count += 1
sleep(10)
continue
state = status['state']
percent_complete = status['percent_complete']
log.debug('state=%s, percent_complete=%d', state, percent_complete)
# Make sure that encryption progress hasn't stalled.
if progress_deadline.is_expired():
raise EncryptionError(
'Waited for encryption progress for longer than %s seconds' %
progress_timeout
)
if percent_complete > last_progress:
last_progress = percent_complete
progress_deadline = Deadline(progress_timeout)
# Log progress once a minute.
now = time.time()
if now - last_log_time >= 60:
log.info('Encryption is %d%% complete', percent_complete)
last_log_time = now
if state == encryptor_service.ENCRYPT_SUCCESSFUL:
log.info('Encrypted root drive created.')
return
elif state == encryptor_service.ENCRYPT_FAILED:
failure_code = status.get('failure_code')
log.debug('failure_code=%s', failure_code)
if failure_code == \
encryptor_service.FAILURE_CODE_UNSUPPORTED_GUEST:
raise UnsupportedGuestError(
'The specified AMI uses an unsupported operating system')
raise EncryptionError('Encryption failed')
sleep(10)
# We've failed to get encryption status for _max_errs_ consecutive tries.
# Assume that the server has crashed.
raise EncryptionError('Encryption service unavailable')
def _wait_for_security_group(aws_svc, sg_id):
log.debug('Waiting for security group %s', sg_id)
deadline = Deadline(EVENTUAL_CONSISTENCY_TIMEOUT)
while not deadline.is_expired():
try:
return aws_svc.get_security_group(sg_id)
except EC2ResponseError as e:
if e.error_code == 'InvalidGroup.NotFound':
_sleep(2)
else:
raise
raise Exception('Timed out waiting for security group ' + sg_id)
def wait_for_volume(
aws_svc, volume, timeout=300, state='available'):
log.debug(
'Waiting for %s, timeout=%d, state=%s',
volume.id, timeout, state)
deadline = Deadline(timeout)
while not deadline.is_expired():
volume = aws_svc.get_volume(volume.id)
if volume.status == state:
return volume
sleep(2)
raise InstanceError(
'Timed out waiting for %s to be in the %s state' %
(volume.id, state)
)
def _write_console_output(aws_svc, instance_id, timeout=GET_CONSOLE_TIMEOUT):
deadline = Deadline(timeout)
while not deadline.is_expired():
try:
console_output = aws_svc.get_console_output(instance_id)
if console_output.output:
prefix = instance_id + '-'
with tempfile.NamedTemporaryFile(
prefix=prefix, suffix='.log', delete=False) as t:
t.write(console_output.output)
return t
except:
pass
log.info('Waiting on console output from %s' % instance_id)
_sleep(5)
log.warn('Timed out waiting for console output from %s' % instance_id)
return None
def _safe_get_instance(aws_svc, instance_id):
""" Get the instance and handle AWS eventual consistency lag.
"""
deadline = Deadline(EVENTUAL_CONSISTENCY_TIMEOUT)
instance = None
while instance is None:
try:
instance = aws_svc.get_instance(instance_id)
except EC2ResponseError as e:
if e.error_code == 'InvalidInstanceID.NotFound':
log.debug('Instance was not found. Sleeping.')
_sleep(2)
else:
raise
if deadline.is_expired():
raise Exception('Invalid instance id: ' + instance_id)
return instance
def wait_for_volume(aws_svc, volume_id, timeout=600.0, state='available'):
""" Wait for the volume to be in the specified state.
:return the Volume object
:raise VolumeError if the timeout is exceeded
"""
log.debug(
'Waiting for %s, timeout=%.02f, state=%s',
volume_id, timeout, state)
deadline = Deadline(timeout)
sleep_time = 0.5
while not deadline.is_expired():
volume = aws_svc.get_volume(volume_id)
if volume.status == state:
return volume
util.sleep(sleep_time)
sleep_time *= 2
raise VolumeError(
'Timed out waiting for %s to be in the %s state' %
(volume_id, state)
)
def wait_for_encryption(enc_svc, progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT):
err_count = 0
max_errs = 10
start_time = time.time()
last_log_time = start_time
progress_deadline = Deadline(progress_timeout)
last_progress = 0
last_state = ''
while err_count < max_errs:
try:
status = enc_svc.get_status()
err_count = 0
except Exception as e:
log.warn("Failed getting encryption status: %s", e)
log.warn("Retrying. . .")
err_count += 1
sleep(10)
continue
state = status['state']
percent_complete = status['percent_complete']
log.debug('state=%s, percent_complete=%d', state, percent_complete)
# Make sure that encryption progress hasn't stalled.
if progress_deadline.is_expired():
raise EncryptionError(
'Waited for encryption progress for longer than %s seconds' %
progress_timeout)
if percent_complete > last_progress or state != last_state:
last_progress = percent_complete
last_state = state
progress_deadline = Deadline(progress_timeout)
# Log progress once a minute.
now = time.time()
if now - last_log_time >= 60:
if state == ENCRYPT_INITIALIZING:
log.info('Encryption process is initializing')
else:
state_display = 'Encryption'
if state == ENCRYPT_DOWNLOADING:
state_display = 'Download from S3'
log.info('%s is %d%% complete', state_display,
percent_complete)
last_log_time = now
if state == ENCRYPT_SUCCESSFUL:
log.info('Encrypted root drive created.')
return
elif state == ENCRYPT_FAILED:
log.debug('Encryption failed with status %s', status)
failure_code = status.get('failure_code')
if failure_code == \
FAILURE_CODE_UNSUPPORTED_GUEST:
raise UnsupportedGuestError(
'The specified AMI uses an unsupported operating system')
if failure_code == FAILURE_CODE_AWS_PERMISSIONS:
raise AWSPermissionsError(
'The specified IAM profile has insufficient permissions')
if failure_code == \
FAILURE_CODE_INVALID_NTP_SERVERS:
raise InvalidNtpServerError('Invalid NTP servers provided.')
msg = 'Encryption failed'
if failure_code:
msg += ' with code %s' % failure_code
raise EncryptionError(msg)
sleep(10)
# We've failed to get encryption status for _max_errs_ consecutive tries.
# Assume that the server has crashed.
raise EncryptionError('Encryption service unavailable')
def wait_for_encryption(enc_svc,
progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT):
err_count = 0
max_errs = 10
start_time = time.time()
last_log_time = start_time
progress_deadline = Deadline(progress_timeout)
last_progress = 0
last_state = ''
while err_count < max_errs:
try:
status = enc_svc.get_status()
err_count = 0
except Exception as e:
log.warn("Failed getting encryption status: %s", e)
log.warn("Retrying. . .")
err_count += 1
sleep(10)
continue
state = status['state']
percent_complete = status['percent_complete']
log.debug('state=%s, percent_complete=%d', state, percent_complete)
# Make sure that encryption progress hasn't stalled.
if progress_deadline.is_expired():
raise EncryptionError(
'Waited for encryption progress for longer than %s seconds' %
progress_timeout
)
if percent_complete > last_progress or state != last_state:
last_progress = percent_complete
last_state = state
progress_deadline = Deadline(progress_timeout)
# Log progress once a minute.
now = time.time()
if now - last_log_time >= 60:
if state == encryptor_service.ENCRYPT_INITIALIZING:
log.info('Encryption process is initializing')
else:
state_display = 'Encryption'
if state == encryptor_service.ENCRYPT_DOWNLOADING:
state_display = 'Download from S3'
log.info(
'%s is %d%% complete', state_display, percent_complete)
last_log_time = now
if state == encryptor_service.ENCRYPT_SUCCESSFUL:
log.info('Encrypted root drive created.')
return
elif state == encryptor_service.ENCRYPT_FAILED:
failure_code = status.get('failure_code')
log.debug('failure_code=%s', failure_code)
if failure_code == \
encryptor_service.FAILURE_CODE_UNSUPPORTED_GUEST:
raise UnsupportedGuestError(
'The specified AMI uses an unsupported operating system')
if failure_code == encryptor_service.FAILURE_CODE_AWS_PERMISSIONS:
raise AWSPermissionsError(
'The specified IAM profile has insufficient permissions')
raise EncryptionError('Encryption failed')
sleep(10)
# We've failed to get encryption status for _max_errs_ consecutive tries.
# Assume that the server has crashed.
raise EncryptionError('Encryption service unavailable')
