def wait_for_volume_attached(aws_svc, instance_id, device):
""" Wait until the device appears in the block device mapping of the
given instance.
:return: the Instance object
"""
# Wait for attachment to complete.
log.debug(
'Waiting for %s in block device mapping of %s.',
device,
instance_id
)
found = False
instance = None
for _ in xrange(20):
instance = aws_svc.get_instance(instance_id)
bdm = instance.block_device_mapping
log.debug('Found devices: %s', bdm.keys())
if device in bdm:
found = True
break
else:
sleep(5)
if not found:
raise BracketError(
'Timed out waiting for %s to attach to %s' %
(device, instance_id)
)
return instance
def wait_for_encryption(enc_svc,
progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT):
err_count = 0
max_errs = 10
start_time = time.time()
last_log_time = start_time
progress_deadline = Deadline(progress_timeout)
last_progress = 0
last_state = ''
while err_count < max_errs:
try:
status = enc_svc.get_status()
err_count = 0
except Exception as e:
log.warn("Failed getting encryption status: %s", e)
log.warn("Retrying. . .")
err_count += 1
sleep(10)
continue
state = status['state']
percent_complete = status['percent_complete']
log.debug('state=%s, percent_complete=%d', state, percent_complete)
# Make sure that encryption progress hasn't stalled.
if progress_deadline.is_expired():
raise EncryptionError(
'Waited for encryption progress for longer than %s seconds' %
progress_timeout
)
if percent_complete > last_progress or state != last_state:
last_progress = percent_complete
last_state = state
progress_deadline = Deadline(progress_timeout)
# Log progress once a minute.
now = time.time()
if now - last_log_time >= 60:
if state == ENCRYPT_INITIALIZING:
log.info('Encryption process is initializing')
else:
state_display = 'Encryption'
if state == ENCRYPT_DOWNLOADING:
state_display = 'Download from cloud storage'
log.info(
'%s is %d%% complete', state_display, percent_complete)
last_log_time = now
if state == ENCRYPT_SUCCESSFUL:
log.info('Encrypted root drive created.')
return
elif state == ENCRYPT_FAILED:
log.error('Encryption status: %s', json.dumps(status))
_handle_failure_code(status.get('failure_code'))
sleep(10)
# We've failed to get encryption status for _max_errs_ consecutive tries.
# Assume that the server has crashed.
raise EncryptionError('Encryption service unavailable')
def wait_for_instance(
aws_svc, instance_id, timeout=600, state='running'):
""" Wait for up to timeout seconds for an instance to be in the
given state. Sleep for 2 seconds between checks.
:return: The Instance object
:raises InstanceError if a timeout occurs or the instance unexpectedly
goes into an error or terminated state
"""
log.debug(
'Waiting for %s, timeout=%d, state=%s',
instance_id, timeout, state)
deadline = Deadline(timeout)
while not deadline.is_expired():
instance = aws_svc.get_instance(instance_id)
log.debug('Instance %s state=%s', instance.id, instance.state)
if instance.state == state:
return instance
if instance.state == 'error':
raise InstanceError(
'Instance %s is in an error state. Cannot proceed.' %
instance_id
)
if state != 'terminated' and instance.state == 'terminated':
raise InstanceError(
'Instance %s was unexpectedly terminated.' % instance_id
)
sleep(2)
raise InstanceError(
'Timed out waiting for %s to be in the %s state' %
(instance_id, state)
)
def wait_for_encryptor_up(enc_svc, deadline):
start = time.time()
while not deadline.is_expired():
if enc_svc.is_encryptor_up():
log.debug('Encryption service is up after %.1f seconds',
time.time() - start)
return
sleep(5)
raise BracketError('Unable to contact encryptor instance at %s.' %
', '.join(enc_svc.hostnames))
def wait_for_encryptor_up(enc_svc, deadline):
start = time.time()
while not deadline.is_expired():
if enc_svc.is_encryptor_up():
log.debug(
'Encryption service is up after %.1f seconds',
time.time() - start
)
return
sleep(5)
raise BracketError(
'Unable to contact encryptor instance at %s.' %
', '.join(enc_svc.hostnames)
)
def wait_for_image(aws_svc, image_id):
log.debug('Waiting for %s to become available.', image_id)
for i in range(180):
sleep(5)
try:
image = aws_svc.get_image(image_id)
except EC2ResponseError, e:
if e.error_code == 'InvalidAMIID.NotFound':
log.debug('AWS threw a NotFound, ignoring')
continue
else:
log.warn('Unknown AWS error: %s', e)
# These two attributes are optional in the response and only
# show up sometimes. So we have to getattr them.
reason = repr(getattr(image, 'stateReason', None))
code = repr(getattr(image, 'code', None))
log.debug("%s: %s reason: %s code: %s",
image.id, image.state, reason, code)
if image.state == 'available':
break
if image.state == 'failed':
raise BracketError('Image state became failed')
def wait_for_volume(aws_svc, volume_id, timeout=600.0, state='available'):
""" Wait for the volume to be in the specified state.
:return the Volume object
:raise VolumeError if the timeout is exceeded
"""
log.debug(
'Waiting for %s, timeout=%.02f, state=%s',
volume_id, timeout, state)
deadline = Deadline(timeout)
sleep_time = 0.5
while not deadline.is_expired():
volume = aws_svc.get_volume(volume_id)
if volume.status == state:
return volume
util.sleep(sleep_time)
sleep_time *= 2
raise VolumeError(
'Timed out waiting for %s to be in the %s state' %
(volume_id, state)
)
def wait_for_snapshots(aws_svc, *snapshot_ids):
log.debug('Waiting for status "completed" for %s', str(snapshot_ids))
last_progress_log = time.time()
# Give AWS some time to propagate the snapshot creation.
# If we create and get immediately, AWS may return 400.
sleep(20)
while True:
snapshots = aws_svc.get_snapshots(*snapshot_ids)
log.debug('%s', {s.id: s.status for s in snapshots})
done = True
error_ids = []
for snapshot in snapshots:
if snapshot.status == 'error':
error_ids.append(snapshot.id)
if snapshot.status != 'completed':
done = False
if error_ids:
# Get rid of unicode markers in error the message.
error_ids = [str(id) for id in error_ids]
raise SnapshotError(
'Snapshots in error state: %s. Cannot continue.' %
str(error_ids)
)
if done:
return
# Log progress if necessary.
now = time.time()
if now - last_progress_log > 60:
log.info(_get_snapshot_progress_text(snapshots))
last_progress_log = now
sleep(5)
def wait_for_encryption(enc_svc, progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT):
err_count = 0
max_errs = 10
start_time = time.time()
last_log_time = start_time
progress_deadline = Deadline(progress_timeout)
last_progress = 0
last_state = ''
while err_count < max_errs:
try:
status = enc_svc.get_status()
err_count = 0
except Exception as e:
log.warn("Failed getting encryption status: %s", e)
log.warn("Retrying. . .")
err_count += 1
sleep(10)
continue
state = status['state']
percent_complete = status['percent_complete']
log.debug('state=%s, percent_complete=%d', state, percent_complete)
# Make sure that encryption progress hasn't stalled.
if progress_deadline.is_expired():
raise EncryptionError(
'Waited for encryption progress for longer than %s seconds' %
progress_timeout)
if percent_complete > last_progress or state != last_state:
last_progress = percent_complete
last_state = state
progress_deadline = Deadline(progress_timeout)
# Log progress once a minute.
now = time.time()
if now - last_log_time >= 60:
if state == ENCRYPT_INITIALIZING:
log.info('Encryption process is initializing')
else:
state_display = 'Encryption'
if state == ENCRYPT_DOWNLOADING:
state_display = 'Download from S3'
log.info('%s is %d%% complete', state_display,
percent_complete)
last_log_time = now
if state == ENCRYPT_SUCCESSFUL:
log.info('Encrypted root drive created.')
return
elif state == ENCRYPT_FAILED:
log.debug('Encryption failed with status %s', status)
failure_code = status.get('failure_code')
if failure_code == \
FAILURE_CODE_UNSUPPORTED_GUEST:
raise UnsupportedGuestError(
'The specified AMI uses an unsupported operating system')
if failure_code == FAILURE_CODE_AWS_PERMISSIONS:
raise AWSPermissionsError(
'The specified IAM profile has insufficient permissions')
if failure_code == \
FAILURE_CODE_INVALID_NTP_SERVERS:
raise InvalidNtpServerError('Invalid NTP servers provided.')
msg = 'Encryption failed'
if failure_code:
msg += ' with code %s' % failure_code
raise EncryptionError(msg)
sleep(10)
# We've failed to get encryption status for _max_errs_ consecutive tries.
# Assume that the server has crashed.
raise EncryptionError('Encryption service unavailable')
def wait_for_encryption(enc_svc,
progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT):
err_count = 0
max_errs = 10
start_time = time.time()
last_log_time = start_time
progress_deadline = Deadline(progress_timeout)
last_progress = 0
last_state = ''
while err_count < max_errs:
try:
status = enc_svc.get_status()
err_count = 0
except Exception as e:
log.warn("Failed getting encryption status: %s", e)
log.warn("Retrying. . .")
err_count += 1
sleep(10)
continue
state = status['state']
percent_complete = status['percent_complete']
log.debug('state=%s, percent_complete=%d', state, percent_complete)
# Make sure that encryption progress hasn't stalled.
if progress_deadline.is_expired():
raise EncryptionError(
'Waited for encryption progress for longer than %s seconds' %
progress_timeout
)
if percent_complete > last_progress or state != last_state:
last_progress = percent_complete
last_state = state
progress_deadline = Deadline(progress_timeout)
# Log progress once a minute.
now = time.time()
if now - last_log_time >= 60:
if state == ENCRYPT_INITIALIZING:
log.info('Encryption process is initializing')
else:
state_display = 'Encryption'
if state == ENCRYPT_DOWNLOADING:
state_display = 'Download from S3'
log.info(
'%s is %d%% complete', state_display, percent_complete)
last_log_time = now
if state == ENCRYPT_SUCCESSFUL:
log.info('Encrypted root drive created.')
return
elif state == ENCRYPT_FAILED:
log.debug('Encryption failed with status %s', status)
failure_code = status.get('failure_code')
if failure_code == \
FAILURE_CODE_UNSUPPORTED_GUEST:
raise UnsupportedGuestError(
'The specified AMI uses an unsupported operating system')
if failure_code == FAILURE_CODE_AWS_PERMISSIONS:
raise AWSPermissionsError(
'The specified IAM profile has insufficient permissions')
if failure_code == \
FAILURE_CODE_INVALID_NTP_SERVERS:
raise InvalidNtpServerError(
'Invalid NTP servers provided.')
msg = 'Encryption failed'
if failure_code:
msg += ' with code %s' % failure_code
raise EncryptionError(msg)
sleep(10)
# We've failed to get encryption status for _max_errs_ consecutive tries.
# Assume that the server has crashed.
raise EncryptionError('Encryption service unavailable')
