def wait_for_volume_attached(aws_svc, instance_id, device): """ Wait until the device appears in the block device mapping of the given instance. :return: the Instance object """ # Wait for attachment to complete. log.debug( 'Waiting for %s in block device mapping of %s.', device, instance_id ) found = False instance = None for _ in xrange(20): instance = aws_svc.get_instance(instance_id) bdm = instance.block_device_mapping log.debug('Found devices: %s', bdm.keys()) if device in bdm: found = True break else: sleep(5) if not found: raise BracketError( 'Timed out waiting for %s to attach to %s' % (device, instance_id) ) return instance
def wait_for_encryption(enc_svc, progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT): err_count = 0 max_errs = 10 start_time = time.time() last_log_time = start_time progress_deadline = Deadline(progress_timeout) last_progress = 0 last_state = '' while err_count < max_errs: try: status = enc_svc.get_status() err_count = 0 except Exception as e: log.warn("Failed getting encryption status: %s", e) log.warn("Retrying. . .") err_count += 1 sleep(10) continue state = status['state'] percent_complete = status['percent_complete'] log.debug('state=%s, percent_complete=%d', state, percent_complete) # Make sure that encryption progress hasn't stalled. if progress_deadline.is_expired(): raise EncryptionError( 'Waited for encryption progress for longer than %s seconds' % progress_timeout ) if percent_complete > last_progress or state != last_state: last_progress = percent_complete last_state = state progress_deadline = Deadline(progress_timeout) # Log progress once a minute. now = time.time() if now - last_log_time >= 60: if state == ENCRYPT_INITIALIZING: log.info('Encryption process is initializing') else: state_display = 'Encryption' if state == ENCRYPT_DOWNLOADING: state_display = 'Download from cloud storage' log.info( '%s is %d%% complete', state_display, percent_complete) last_log_time = now if state == ENCRYPT_SUCCESSFUL: log.info('Encrypted root drive created.') return elif state == ENCRYPT_FAILED: log.error('Encryption status: %s', json.dumps(status)) _handle_failure_code(status.get('failure_code')) sleep(10) # We've failed to get encryption status for _max_errs_ consecutive tries. # Assume that the server has crashed. raise EncryptionError('Encryption service unavailable')
def wait_for_instance( aws_svc, instance_id, timeout=600, state='running'): """ Wait for up to timeout seconds for an instance to be in the given state. Sleep for 2 seconds between checks. :return: The Instance object :raises InstanceError if a timeout occurs or the instance unexpectedly goes into an error or terminated state """ log.debug( 'Waiting for %s, timeout=%d, state=%s', instance_id, timeout, state) deadline = Deadline(timeout) while not deadline.is_expired(): instance = aws_svc.get_instance(instance_id) log.debug('Instance %s state=%s', instance.id, instance.state) if instance.state == state: return instance if instance.state == 'error': raise InstanceError( 'Instance %s is in an error state. Cannot proceed.' % instance_id ) if state != 'terminated' and instance.state == 'terminated': raise InstanceError( 'Instance %s was unexpectedly terminated.' % instance_id ) sleep(2) raise InstanceError( 'Timed out waiting for %s to be in the %s state' % (instance_id, state) )
def wait_for_encryptor_up(enc_svc, deadline): start = time.time() while not deadline.is_expired(): if enc_svc.is_encryptor_up(): log.debug('Encryption service is up after %.1f seconds', time.time() - start) return sleep(5) raise BracketError('Unable to contact encryptor instance at %s.' % ', '.join(enc_svc.hostnames))
def wait_for_encryptor_up(enc_svc, deadline): start = time.time() while not deadline.is_expired(): if enc_svc.is_encryptor_up(): log.debug( 'Encryption service is up after %.1f seconds', time.time() - start ) return sleep(5) raise BracketError( 'Unable to contact encryptor instance at %s.' % ', '.join(enc_svc.hostnames) )
def wait_for_image(aws_svc, image_id): log.debug('Waiting for %s to become available.', image_id) for i in range(180): sleep(5) try: image = aws_svc.get_image(image_id) except EC2ResponseError, e: if e.error_code == 'InvalidAMIID.NotFound': log.debug('AWS threw a NotFound, ignoring') continue else: log.warn('Unknown AWS error: %s', e) # These two attributes are optional in the response and only # show up sometimes. So we have to getattr them. reason = repr(getattr(image, 'stateReason', None)) code = repr(getattr(image, 'code', None)) log.debug("%s: %s reason: %s code: %s", image.id, image.state, reason, code) if image.state == 'available': break if image.state == 'failed': raise BracketError('Image state became failed')
def wait_for_volume(aws_svc, volume_id, timeout=600.0, state='available'): """ Wait for the volume to be in the specified state. :return the Volume object :raise VolumeError if the timeout is exceeded """ log.debug( 'Waiting for %s, timeout=%.02f, state=%s', volume_id, timeout, state) deadline = Deadline(timeout) sleep_time = 0.5 while not deadline.is_expired(): volume = aws_svc.get_volume(volume_id) if volume.status == state: return volume util.sleep(sleep_time) sleep_time *= 2 raise VolumeError( 'Timed out waiting for %s to be in the %s state' % (volume_id, state) )
def wait_for_snapshots(aws_svc, *snapshot_ids): log.debug('Waiting for status "completed" for %s', str(snapshot_ids)) last_progress_log = time.time() # Give AWS some time to propagate the snapshot creation. # If we create and get immediately, AWS may return 400. sleep(20) while True: snapshots = aws_svc.get_snapshots(*snapshot_ids) log.debug('%s', {s.id: s.status for s in snapshots}) done = True error_ids = [] for snapshot in snapshots: if snapshot.status == 'error': error_ids.append(snapshot.id) if snapshot.status != 'completed': done = False if error_ids: # Get rid of unicode markers in error the message. error_ids = [str(id) for id in error_ids] raise SnapshotError( 'Snapshots in error state: %s. Cannot continue.' % str(error_ids) ) if done: return # Log progress if necessary. now = time.time() if now - last_progress_log > 60: log.info(_get_snapshot_progress_text(snapshots)) last_progress_log = now sleep(5)
def wait_for_encryption(enc_svc, progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT): err_count = 0 max_errs = 10 start_time = time.time() last_log_time = start_time progress_deadline = Deadline(progress_timeout) last_progress = 0 last_state = '' while err_count < max_errs: try: status = enc_svc.get_status() err_count = 0 except Exception as e: log.warn("Failed getting encryption status: %s", e) log.warn("Retrying. . .") err_count += 1 sleep(10) continue state = status['state'] percent_complete = status['percent_complete'] log.debug('state=%s, percent_complete=%d', state, percent_complete) # Make sure that encryption progress hasn't stalled. if progress_deadline.is_expired(): raise EncryptionError( 'Waited for encryption progress for longer than %s seconds' % progress_timeout) if percent_complete > last_progress or state != last_state: last_progress = percent_complete last_state = state progress_deadline = Deadline(progress_timeout) # Log progress once a minute. now = time.time() if now - last_log_time >= 60: if state == ENCRYPT_INITIALIZING: log.info('Encryption process is initializing') else: state_display = 'Encryption' if state == ENCRYPT_DOWNLOADING: state_display = 'Download from S3' log.info('%s is %d%% complete', state_display, percent_complete) last_log_time = now if state == ENCRYPT_SUCCESSFUL: log.info('Encrypted root drive created.') return elif state == ENCRYPT_FAILED: log.debug('Encryption failed with status %s', status) failure_code = status.get('failure_code') if failure_code == \ FAILURE_CODE_UNSUPPORTED_GUEST: raise UnsupportedGuestError( 'The specified AMI uses an unsupported operating system') if failure_code == FAILURE_CODE_AWS_PERMISSIONS: raise AWSPermissionsError( 'The specified IAM profile has insufficient permissions') if failure_code == \ FAILURE_CODE_INVALID_NTP_SERVERS: raise InvalidNtpServerError('Invalid NTP servers provided.') msg = 'Encryption failed' if failure_code: msg += ' with code %s' % failure_code raise EncryptionError(msg) sleep(10) # We've failed to get encryption status for _max_errs_ consecutive tries. # Assume that the server has crashed. raise EncryptionError('Encryption service unavailable')
def wait_for_encryption(enc_svc, progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT): err_count = 0 max_errs = 10 start_time = time.time() last_log_time = start_time progress_deadline = Deadline(progress_timeout) last_progress = 0 last_state = '' while err_count < max_errs: try: status = enc_svc.get_status() err_count = 0 except Exception as e: log.warn("Failed getting encryption status: %s", e) log.warn("Retrying. . .") err_count += 1 sleep(10) continue state = status['state'] percent_complete = status['percent_complete'] log.debug('state=%s, percent_complete=%d', state, percent_complete) # Make sure that encryption progress hasn't stalled. if progress_deadline.is_expired(): raise EncryptionError( 'Waited for encryption progress for longer than %s seconds' % progress_timeout ) if percent_complete > last_progress or state != last_state: last_progress = percent_complete last_state = state progress_deadline = Deadline(progress_timeout) # Log progress once a minute. now = time.time() if now - last_log_time >= 60: if state == ENCRYPT_INITIALIZING: log.info('Encryption process is initializing') else: state_display = 'Encryption' if state == ENCRYPT_DOWNLOADING: state_display = 'Download from S3' log.info( '%s is %d%% complete', state_display, percent_complete) last_log_time = now if state == ENCRYPT_SUCCESSFUL: log.info('Encrypted root drive created.') return elif state == ENCRYPT_FAILED: log.debug('Encryption failed with status %s', status) failure_code = status.get('failure_code') if failure_code == \ FAILURE_CODE_UNSUPPORTED_GUEST: raise UnsupportedGuestError( 'The specified AMI uses an unsupported operating system') if failure_code == FAILURE_CODE_AWS_PERMISSIONS: raise AWSPermissionsError( 'The specified IAM profile has insufficient permissions') if failure_code == \ FAILURE_CODE_INVALID_NTP_SERVERS: raise InvalidNtpServerError( 'Invalid NTP servers provided.') msg = 'Encryption failed' if failure_code: msg += ' with code %s' % failure_code raise EncryptionError(msg) sleep(10) # We've failed to get encryption status for _max_errs_ consecutive tries. # Assume that the server has crashed. raise EncryptionError('Encryption service unavailable')