def __wait_ready( self, instance, from_states, first_boot ):
"""
Wait until the given instance transistions from stopped or pending state to being fully
running and accessible via SSH.
:param instance: the instance to wait for
:type instance: boto.ec2.instance.Instance
:param from_states: the set of states the instance may be in when this methods is
invoked, any other state will raise an exception.
:type from_states: set of str
:param first_boot: True if the instance is currently booting for the first time,
None if the instance isn't booting, False if the instance is booting but not for the
first time.
"""
log.info( "... waiting for instance %s ... ", instance.id )
wait_transition( instance, from_states, 'running' )
self._on_instance_running( instance, first_boot )
log.info( "... running, waiting for assignment of public IP ... " )
self.__wait_public_ip_assigned( instance )
log.info( "... assigned, waiting for SSH port ... " )
self.__wait_ssh_port_open( )
log.info( "... open ... " )
if first_boot is not None:
log.info( "... testing SSH ... " )
self.__wait_ssh_working( )
log.info( "... SSH working ..., " )
log.info( "... instance ready." )
self._on_instance_ready( first_boot )
def _terminateInstances(cls, instances, ctx):
instanceIDs = [x.id for x in instances]
cls._terminateIDs(instanceIDs, ctx)
logger.info('... Waiting for instance(s) to shut down...')
for instance in instances:
wait_transition(instance, {'pending', 'running', 'shutting-down'}, 'terminated')
logger.info('Instance(s) terminated.')
def _terminateInstances(cls, instances, ctx):
instanceIDs = [x.id for x in instances]
cls._terminateIDs(instanceIDs, ctx)
logger.info('... Waiting for instance(s) to shut down...')
for instance in instances:
wait_transition(instance, {'pending', 'running', 'shutting-down'}, 'terminated')
logger.info('Instance(s) terminated.')
def stop( self ):
"""
Stop the EC2 instance represented by this box. Stopped instances can be started later using
:py:func:`Box.start`.
"""
instance = self.__assert_state( 'running' )
log.info( 'Stopping instance ...' )
self.ctx.ec2.stop_instances( [ instance.id ] )
wait_transition( instance,
from_states={ 'running', 'stopping' },
to_state='stopped' )
log.info( '... instance stopped.' )
def _terminateInstances(cls, instances, ctx):
instanceIDs = [x.id for x in instances]
# get all volume ids now. After termination the block device mapping attr
# will not be populated
ebsIDs = [x.block_device_mapping["/dev/xvda"].volume_id for x in instances]
logger.info('Terminating instance(s): %s', instanceIDs)
cls._terminateIDs(instanceIDs, ctx)
logger.info('... Waiting for instance(s) to shut down...')
for instance in instances:
wait_transition(instance, {'running', 'shutting-down'}, 'terminated')
cls._deleteRootEBS(ebsIDs, ctx)
logger.info('Instance(s) terminated.')
def terminate( self, wait=True ):
"""
Terminate the EC2 instance represented by this box.
"""
if self.instance_id is not None:
instance = self.get_instance( )
if instance.state != 'terminated':
log.info( 'Terminating instance ...' )
self.ctx.ec2.terminate_instances( [ self.instance_id ] )
if wait:
wait_transition( instance,
from_states={ 'running', 'shutting-down', 'stopped' },
to_state='terminated' )
log.info( '... instance terminated.' )
def _getLeader(cls, clusterName, wait=False, zone=None):
ctx = cls._buildContext(clusterName=clusterName, zone=zone)
instances = cls.__getNodesInCluster(ctx, clusterName, both=True)
instances.sort(key=lambda x: x.launch_time)
leader = instances[0] # assume leader was launched first
if wait:
logger.info("Waiting for leader to enter 'running' state...")
wait_transition(leader, {'pending'}, 'running')
logger.info('... leader is running')
cls._waitForIP(leader)
leaderIP = leader.ip_address
cls._waitForSSHPort(leaderIP)
# wait here so docker commands can be used reliably afterwards
cls._waitForDockerDaemon(leaderIP)
cls._waitForAppliance(leaderIP)
return leader
def _getLeader(cls, clusterName, wait=False, zone=None):
ctx = cls._buildContext(clusterName=clusterName, zone=zone)
instances = cls.__getNodesInCluster(ctx, clusterName, both=True)
instances.sort(key=lambda x: x.launch_time)
leader = instances[0] # assume leader was launched first
if wait:
logger.info("Waiting for leader to enter 'running' state...")
wait_transition(leader, {'pending'}, 'running')
logger.info('... leader is running')
cls._waitForIP(leader)
leaderIP = leader.ip_address
cls._waitForSSHPort(leaderIP)
# wait here so docker commands can be used reliably afterwards
cls._waitForDockerDaemon(leaderIP)
cls._waitForAppliance(leaderIP)
return leader
def image( self ):
"""
Create an image (AMI) of the EC2 instance represented by this box and return its ID.
The EC2 instance needs to use an EBS-backed root volume. The box must be stopped or
an exception will be raised.
"""
self.__assert_state( 'stopped' )
log.info( "Creating image ..." )
timestamp = time.strftime( '%Y-%m-%d_%H-%M-%S' )
image_name = self.ctx.to_aws_name( self._image_name_prefix( ) + "_" + timestamp )
image_id = self.ctx.ec2.create_image(
instance_id=self.instance_id,
name=image_name,
block_device_mapping=self._image_block_device_mapping( ) )
while True:
try:
image = self.ctx.ec2.get_image( image_id )
self.generation += 1
try:
self._tag_object_persistently( image, self._get_instance_options( ) )
finally:
self.generation -= 1
wait_transition( image, { 'pending' }, 'available' )
log.info( "... created %s (%s).", image.id, image.name )
break
except self.ctx.ec2.ResponseError as e:
# FIXME: I don't think get_image can throw this, it should be outside the try
if e.error_code != 'InvalidAMIID.NotFound':
raise
# There seems to be another race condition in EC2 that causes a freshly created image to
# not be included in queries other than by AMI ID.
log.info( 'Checking if image %s is discoverable ...' % image_id )
while True:
if image_id in (_.id for _ in self.list_images( )):
log.info( '... image now discoverable.' )
break
log.info( '... image %s not yet discoverable, trying again in %is ...' % (
image_id, a_short_time ) )
time.sleep( a_short_time )
return image_id
def wait_running(instance):
wait_transition(instance,
from_states={'pending'},
to_state='running')
def wait_running(instance):
wait_transition(instance, from_states={'pending'}, to_state='running')