def wait_for_encryption(enc_svc, progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT): err_count = 0 max_errs = 10 start_time = time.time() last_log_time = start_time progress_deadline = Deadline(progress_timeout) last_progress = 0 last_state = '' while err_count < max_errs: try: status = enc_svc.get_status() err_count = 0 except Exception as e: log.warn("Failed getting encryption status: %s", e) log.warn("Retrying. . .") err_count += 1 sleep(10) continue state = status['state'] percent_complete = status['percent_complete'] log.debug('state=%s, percent_complete=%d', state, percent_complete) # Make sure that encryption progress hasn't stalled. if progress_deadline.is_expired(): raise EncryptionError( 'Waited for encryption progress for longer than %s seconds' % progress_timeout ) if percent_complete > last_progress or state != last_state: last_progress = percent_complete last_state = state progress_deadline = Deadline(progress_timeout) # Log progress once a minute. now = time.time() if now - last_log_time >= 60: if state == ENCRYPT_INITIALIZING: log.info('Encryption process is initializing') else: state_display = 'Encryption' if state == ENCRYPT_DOWNLOADING: state_display = 'Download from cloud storage' log.info( '%s is %d%% complete', state_display, percent_complete) last_log_time = now if state == ENCRYPT_SUCCESSFUL: log.info('Encrypted root drive created.') return elif state == ENCRYPT_FAILED: log.error('Encryption status: %s', json.dumps(status)) _handle_failure_code(status.get('failure_code')) sleep(10) # We've failed to get encryption status for _max_errs_ consecutive tries. # Assume that the server has crashed. raise EncryptionError('Encryption service unavailable')
def wait_for_instance( aws_svc, instance_id, timeout=600, state='running'): """ Wait for up to timeout seconds for an instance to be in the given state. Sleep for 2 seconds between checks. :return: The Instance object :raises InstanceError if a timeout occurs or the instance unexpectedly goes into an error or terminated state """ log.debug( 'Waiting for %s, timeout=%d, state=%s', instance_id, timeout, state) deadline = Deadline(timeout) while not deadline.is_expired(): instance = aws_svc.get_instance(instance_id) log.debug('Instance %s state=%s', instance.id, instance.state) if instance.state == state: return instance if instance.state == 'error': raise InstanceError( 'Instance %s is in an error state. Cannot proceed.' % instance_id ) if state != 'terminated' and instance.state == 'terminated': raise InstanceError( 'Instance %s was unexpectedly terminated.' % instance_id ) sleep(2) raise InstanceError( 'Timed out waiting for %s to be in the %s state' % (instance_id, state) )
def _wait_for_instance( aws_svc, instance_id, timeout=300, state='running'): """ Wait for up to timeout seconds for an instance to be in the 'running' state. Sleep for 2 seconds between checks. :return: The Instance object, or None if a timeout occurred """ log.debug( 'Waiting for %s, timeout=%d, state=%s', instance_id, timeout, state) # Wait for AWS eventual consistency to catch up. instance = _safe_get_instance(aws_svc, instance_id) deadline = Deadline(timeout) while not deadline.is_expired(): log.debug('Instance %s state=%s', instance.id, instance.state) if instance.state == state: return instance if instance.state == 'error': raise Exception( 'Instance %s is in an error state. Cannot proceed.' ) _sleep(2) instance = aws_svc.get_instance(instance_id) raise Exception( 'Timed out waiting for %s to be in the %s state' % (instance_id, state) )
def wait_for_encryption(enc_svc, progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT): err_count = 0 max_errs = 10 start_time = time.time() last_log_time = start_time progress_deadline = Deadline(progress_timeout) last_progress = 0 while err_count < max_errs: try: status = enc_svc.get_status() err_count = 0 except Exception as e: log.warn("Failed getting encryption status: %s", e) err_count += 1 sleep(10) continue state = status['state'] percent_complete = status['percent_complete'] log.debug('state=%s, percent_complete=%d', state, percent_complete) # Make sure that encryption progress hasn't stalled. if progress_deadline.is_expired(): raise EncryptionError( 'Waited for encryption progress for longer than %s seconds' % progress_timeout ) if percent_complete > last_progress: last_progress = percent_complete progress_deadline = Deadline(progress_timeout) # Log progress once a minute. now = time.time() if now - last_log_time >= 60: log.info('Encryption is %d%% complete', percent_complete) last_log_time = now if state == encryptor_service.ENCRYPT_SUCCESSFUL: log.info('Encrypted root drive created.') return elif state == encryptor_service.ENCRYPT_FAILED: failure_code = status.get('failure_code') log.debug('failure_code=%s', failure_code) if failure_code == \ encryptor_service.FAILURE_CODE_UNSUPPORTED_GUEST: raise UnsupportedGuestError( 'The specified AMI uses an unsupported operating system') raise EncryptionError('Encryption failed') sleep(10) # We've failed to get encryption status for _max_errs_ consecutive tries. # Assume that the server has crashed. raise EncryptionError('Encryption service unavailable')
def _wait_for_security_group(aws_svc, sg_id): log.debug('Waiting for security group %s', sg_id) deadline = Deadline(EVENTUAL_CONSISTENCY_TIMEOUT) while not deadline.is_expired(): try: return aws_svc.get_security_group(sg_id) except EC2ResponseError as e: if e.error_code == 'InvalidGroup.NotFound': _sleep(2) else: raise raise Exception('Timed out waiting for security group ' + sg_id)
def wait_for_volume( aws_svc, volume, timeout=300, state='available'): log.debug( 'Waiting for %s, timeout=%d, state=%s', volume.id, timeout, state) deadline = Deadline(timeout) while not deadline.is_expired(): volume = aws_svc.get_volume(volume.id) if volume.status == state: return volume sleep(2) raise InstanceError( 'Timed out waiting for %s to be in the %s state' % (volume.id, state) )
def _write_console_output(aws_svc, instance_id, timeout=GET_CONSOLE_TIMEOUT): deadline = Deadline(timeout) while not deadline.is_expired(): try: console_output = aws_svc.get_console_output(instance_id) if console_output.output: prefix = instance_id + '-' with tempfile.NamedTemporaryFile( prefix=prefix, suffix='.log', delete=False) as t: t.write(console_output.output) return t except: pass log.info('Waiting on console output from %s' % instance_id) _sleep(5) log.warn('Timed out waiting for console output from %s' % instance_id) return None
def _safe_get_instance(aws_svc, instance_id): """ Get the instance and handle AWS eventual consistency lag. """ deadline = Deadline(EVENTUAL_CONSISTENCY_TIMEOUT) instance = None while instance is None: try: instance = aws_svc.get_instance(instance_id) except EC2ResponseError as e: if e.error_code == 'InvalidInstanceID.NotFound': log.debug('Instance was not found. Sleeping.') _sleep(2) else: raise if deadline.is_expired(): raise Exception('Invalid instance id: ' + instance_id) return instance
def wait_for_volume(aws_svc, volume_id, timeout=600.0, state='available'): """ Wait for the volume to be in the specified state. :return the Volume object :raise VolumeError if the timeout is exceeded """ log.debug( 'Waiting for %s, timeout=%.02f, state=%s', volume_id, timeout, state) deadline = Deadline(timeout) sleep_time = 0.5 while not deadline.is_expired(): volume = aws_svc.get_volume(volume_id) if volume.status == state: return volume util.sleep(sleep_time) sleep_time *= 2 raise VolumeError( 'Timed out waiting for %s to be in the %s state' % (volume_id, state) )
def wait_for_encryption(enc_svc, progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT): err_count = 0 max_errs = 10 start_time = time.time() last_log_time = start_time progress_deadline = Deadline(progress_timeout) last_progress = 0 last_state = '' while err_count < max_errs: try: status = enc_svc.get_status() err_count = 0 except Exception as e: log.warn("Failed getting encryption status: %s", e) log.warn("Retrying. . .") err_count += 1 sleep(10) continue state = status['state'] percent_complete = status['percent_complete'] log.debug('state=%s, percent_complete=%d', state, percent_complete) # Make sure that encryption progress hasn't stalled. if progress_deadline.is_expired(): raise EncryptionError( 'Waited for encryption progress for longer than %s seconds' % progress_timeout) if percent_complete > last_progress or state != last_state: last_progress = percent_complete last_state = state progress_deadline = Deadline(progress_timeout) # Log progress once a minute. now = time.time() if now - last_log_time >= 60: if state == ENCRYPT_INITIALIZING: log.info('Encryption process is initializing') else: state_display = 'Encryption' if state == ENCRYPT_DOWNLOADING: state_display = 'Download from S3' log.info('%s is %d%% complete', state_display, percent_complete) last_log_time = now if state == ENCRYPT_SUCCESSFUL: log.info('Encrypted root drive created.') return elif state == ENCRYPT_FAILED: log.debug('Encryption failed with status %s', status) failure_code = status.get('failure_code') if failure_code == \ FAILURE_CODE_UNSUPPORTED_GUEST: raise UnsupportedGuestError( 'The specified AMI uses an unsupported operating system') if failure_code == FAILURE_CODE_AWS_PERMISSIONS: raise AWSPermissionsError( 'The specified IAM profile has insufficient permissions') if failure_code == \ FAILURE_CODE_INVALID_NTP_SERVERS: raise InvalidNtpServerError('Invalid NTP servers provided.') msg = 'Encryption failed' if failure_code: msg += ' with code %s' % failure_code raise EncryptionError(msg) sleep(10) # We've failed to get encryption status for _max_errs_ consecutive tries. # Assume that the server has crashed. raise EncryptionError('Encryption service unavailable')
def wait_for_encryption(enc_svc, progress_timeout=ENCRYPTION_PROGRESS_TIMEOUT): err_count = 0 max_errs = 10 start_time = time.time() last_log_time = start_time progress_deadline = Deadline(progress_timeout) last_progress = 0 last_state = '' while err_count < max_errs: try: status = enc_svc.get_status() err_count = 0 except Exception as e: log.warn("Failed getting encryption status: %s", e) log.warn("Retrying. . .") err_count += 1 sleep(10) continue state = status['state'] percent_complete = status['percent_complete'] log.debug('state=%s, percent_complete=%d', state, percent_complete) # Make sure that encryption progress hasn't stalled. if progress_deadline.is_expired(): raise EncryptionError( 'Waited for encryption progress for longer than %s seconds' % progress_timeout ) if percent_complete > last_progress or state != last_state: last_progress = percent_complete last_state = state progress_deadline = Deadline(progress_timeout) # Log progress once a minute. now = time.time() if now - last_log_time >= 60: if state == encryptor_service.ENCRYPT_INITIALIZING: log.info('Encryption process is initializing') else: state_display = 'Encryption' if state == encryptor_service.ENCRYPT_DOWNLOADING: state_display = 'Download from S3' log.info( '%s is %d%% complete', state_display, percent_complete) last_log_time = now if state == encryptor_service.ENCRYPT_SUCCESSFUL: log.info('Encrypted root drive created.') return elif state == encryptor_service.ENCRYPT_FAILED: failure_code = status.get('failure_code') log.debug('failure_code=%s', failure_code) if failure_code == \ encryptor_service.FAILURE_CODE_UNSUPPORTED_GUEST: raise UnsupportedGuestError( 'The specified AMI uses an unsupported operating system') if failure_code == encryptor_service.FAILURE_CODE_AWS_PERMISSIONS: raise AWSPermissionsError( 'The specified IAM profile has insufficient permissions') raise EncryptionError('Encryption failed') sleep(10) # We've failed to get encryption status for _max_errs_ consecutive tries. # Assume that the server has crashed. raise EncryptionError('Encryption service unavailable')