def _stop_ec2_instances(awsclient, ec2_instances, wait=True):
"""Helper to stop ec2 instances.
By default it waits for instances to stop.
:param awsclient:
:param ec2_instances:
:param wait: waits for instances to stop
:return:
"""
if len(ec2_instances) == 0:
return
client_ec2 = awsclient.get_client('ec2')
# get running instances
running_instances = all_pages(
client_ec2.describe_instance_status,
{
'InstanceIds': ec2_instances,
'Filters': [{
'Name': 'instance-state-name',
'Values': ['pending', 'running']
}]
},
lambda r: [i['InstanceId'] for i in r.get('InstanceStatuses', [])],
)
if running_instances:
log.info('Stopping EC2 instances: %s', running_instances)
client_ec2.stop_instances(InstanceIds=running_instances)
if wait:
# wait for instances to stop
waiter_inst_stopped = client_ec2.get_waiter('instance_stopped')
waiter_inst_stopped.wait(InstanceIds=running_instances)
def test_ec2_instance_stop_start(awsclient, simple_cloudformation_stack_with_ec2):
def _get_instance_status(ec2_instance):
# helper to check the status
client_ec2 = awsclient.get_client('ec2')
instances_status = all_pages(
client_ec2.describe_instance_status,
{
'InstanceIds': [ec2_instance],
'IncludeAllInstances': True
},
lambda r: [i['InstanceState']['Name'] for i in r.get('InstanceStatuses', [])],
)[0]
return instances_status
stack_name = _get_stack_name(config_ec2_stack)
client_cfn = awsclient.get_client('cloudformation')
resources = all_pages(
client_cfn.list_stack_resources,
{ 'StackName': stack_name },
lambda r: r['StackResourceSummaries']
)
instances = [
r['PhysicalResourceId'] for r in resources
if r['ResourceType'] == 'AWS::EC2::Instance'
]
assert _get_instance_status(instances[0]) == 'running'
_stop_ec2_instances(awsclient, instances, wait=True)
assert _get_instance_status(instances[0]) == 'stopped'
_start_ec2_instances(awsclient, instances, wait=True)
assert _get_instance_status(instances[0]) == 'running'
def _get_instance_status(ec2_instance):
# helper to check the status
client_ec2 = awsclient.get_client('ec2')
instances_status = all_pages(
client_ec2.describe_instance_status,
{
'InstanceIds': [ec2_instance],
'IncludeAllInstances': True
},
lambda r: [i['InstanceState']['Name'] for i in r.get('InstanceStatuses', [])],
)[0]
return instances_status
def test_all_pages_no_condition():
state = {'counter': 0}
def dummy_method(**kwargs):
# I represent the service method
nextToken = kwargs.pop('nextToken', None)
if nextToken:
assert nextToken == state['counter']
if state['counter'] < 5:
state['counter'] += 1
kwargs['nextToken'] = state['counter']
return kwargs
actual = all_pages(dummy_method, {'foo': 'bar'},
lambda r: r['foo'] + str(r.get('nextToken', '')))
assert actual == ['bar1', 'bar2', 'bar3', 'bar4', 'bar5', 'bar']
def _start_ec2_instances(awsclient, ec2_instances, wait=True):
"""Helper to start ec2 instances
:param awsclient:
:param ec2_instances:
:param wait: waits for instances to start
:return:
"""
if len(ec2_instances) == 0:
return
client_ec2 = awsclient.get_client('ec2')
# get stopped instances
stopped_instances = all_pages(
client_ec2.describe_instance_status,
{
'InstanceIds': ec2_instances,
'Filters': [{
'Name': 'instance-state-name',
'Values': ['stopping', 'stopped']
}],
'IncludeAllInstances': True
},
lambda r: [i['InstanceId'] for i in r.get('InstanceStatuses', [])],
)
if stopped_instances:
# start all stopped instances
log.info('Starting EC2 instances: %s', stopped_instances)
client_ec2.start_instances(InstanceIds=stopped_instances)
if wait:
# wait for instances to come up
waiter_inst_running = client_ec2.get_waiter('instance_running')
waiter_inst_running.wait(InstanceIds=stopped_instances)
# wait for status checks
waiter_status_ok = client_ec2.get_waiter('instance_status_ok')
waiter_status_ok.wait(InstanceIds=stopped_instances)
def start_stack(awsclient, conf, use_suspend=False):
"""Start an existing stack on AWS cloud.
:param awsclient:
:param conf:
:param use_suspend: use suspend and resume on the autoscaling group
:return: exit_code
"""
stack_name = _get_stack_name(conf)
exit_code = 0
# check for DisableStop
disable_stop = conf.get('deployment', {}).get('DisableStop', False)
if disable_stop:
log.warn('\'DisableStop\' is set - nothing to do!')
else:
if not stack_exists(awsclient, stack_name):
log.warn('Stack \'%s\' not deployed - nothing to do!', stack_name)
else:
client_cfn = awsclient.get_client('cloudformation')
client_autoscaling = awsclient.get_client('autoscaling')
client_rds = awsclient.get_client('rds')
resources = all_pages(
client_cfn.list_stack_resources,
{ 'StackName': stack_name },
lambda r: r['StackResourceSummaries']
)
autoscaling_groups = [
r for r in resources
if r['ResourceType'] == 'AWS::AutoScaling::AutoScalingGroup'
]
# lookup all types of scaling processes
# [Launch, Terminate, HealthCheck, ReplaceUnhealthy, AZRebalance
# AlarmNotification, ScheduledActions, AddToLoadBalancer]
response = client_autoscaling.describe_scaling_process_types()
scaling_process_types = [t['ProcessName'] for t in response.get('Processes', [])]
# starting db instances
db_instances = [
r['PhysicalResourceId'] for r in resources
if r['ResourceType'] == 'AWS::RDS::DBInstance'
]
stopped_db_instances = _filter_db_instances_by_status(
awsclient, db_instances, ['stopped']
)
for db in stopped_db_instances:
log.info('Starting RDS instance \'%s\'', db)
client_rds.start_db_instance(DBInstanceIdentifier=db)
# wait for db instances to become available
for db in stopped_db_instances:
waiter_db_available = client_rds.get_waiter('db_instance_available')
waiter_db_available.wait(DBInstanceIdentifier=db)
# starting ec2 instances
instances = [
r['PhysicalResourceId'] for r in resources
if r['ResourceType'] == 'AWS::EC2::Instance'
]
_start_ec2_instances(awsclient, instances)
if autoscaling_groups and not use_suspend:
# get template and parameters from cloudformation
response = client_cfn.get_template(
StackName=stack_name,
TemplateStage='Processed'
)
template = response.get('TemplateBody', {})
response = client_cfn.describe_stacks(
StackName=stack_name
)
parameters = response['Stacks'][0].get('Parameters', {})
for asg in autoscaling_groups:
if use_suspend:
# alternative implementation to speed up start
# only problem is that instances must survive stop & start
# find instances in autoscaling group
instances = all_pages(
client_autoscaling.describe_auto_scaling_instances,
{},
lambda r: [i['InstanceId'] for i in r.get('AutoScalingInstances', [])
if i['AutoScalingGroupName'] == asg['PhysicalResourceId']],
)
_start_ec2_instances(awsclient, instances)
# resume all autoscaling processes
log.info('Resuming all autoscaling processes for \'%s\'',
asg['LogicalResourceId'])
response = client_autoscaling.resume_processes(
AutoScalingGroupName=asg['PhysicalResourceId'],
ScalingProcesses=scaling_process_types
)
else:
# resize autoscaling group back to its original values
log.info('Resize autoscaling group \'%s\' back to original values',
asg['LogicalResourceId'])
min, max = _get_autoscaling_min_max(
template, parameters, asg['LogicalResourceId'])
response = client_autoscaling.update_auto_scaling_group(
AutoScalingGroupName=asg['PhysicalResourceId'],
MinSize=min,
MaxSize=max
)
return exit_code
def stop_stack(awsclient, conf, use_suspend=False):
"""Stop an existing stack on AWS cloud.
:param awsclient:
:param conf:
:param use_suspend: use suspend and resume on the autoscaling group
:return: exit_code
"""
stack_name = _get_stack_name(conf)
exit_code = 0
# check for DisableStop
disable_stop = conf.get('deployment', {}).get('DisableStop', False)
if disable_stop:
log.warn('\'DisableStop\' is set - nothing to do!')
else:
if not stack_exists(awsclient, stack_name):
log.warn('Stack \'%s\' not deployed - nothing to do!', stack_name)
else:
client_cfn = awsclient.get_client('cloudformation')
client_autoscaling = awsclient.get_client('autoscaling')
client_rds = awsclient.get_client('rds')
client_ec2 = awsclient.get_client('ec2')
resources = all_pages(
client_cfn.list_stack_resources,
{ 'StackName': stack_name },
lambda r: r['StackResourceSummaries']
)
autoscaling_groups = [
r for r in resources
if r['ResourceType'] == 'AWS::AutoScaling::AutoScalingGroup'
]
# lookup all types of scaling processes
# [Launch, Terminate, HealthCheck, ReplaceUnhealthy, AZRebalance
# AlarmNotification, ScheduledActions, AddToLoadBalancer]
response = client_autoscaling.describe_scaling_process_types()
scaling_process_types = [t['ProcessName'] for t in response.get('Processes', [])]
for asg in autoscaling_groups:
# find instances in autoscaling group
ec2_instances = all_pages(
client_autoscaling.describe_auto_scaling_instances,
{},
lambda r: [i['InstanceId'] for i in r.get('AutoScalingInstances', [])
if i['AutoScalingGroupName'] == asg['PhysicalResourceId']],
)
if use_suspend:
# alternative implementation to speed up start
# only problem is that instances must survive stop & start
# suspend all autoscaling processes
log.info('Suspending all autoscaling processes for \'%s\'',
asg['LogicalResourceId'])
response = client_autoscaling.suspend_processes(
AutoScalingGroupName=asg['PhysicalResourceId'],
ScalingProcesses=scaling_process_types
)
_stop_ec2_instances(awsclient, ec2_instances)
else:
running_instances = all_pages(
client_ec2.describe_instance_status,
{
'InstanceIds': ec2_instances,
'Filters': [{
'Name': 'instance-state-name',
'Values': ['pending', 'running']
}]
},
lambda r: [i['InstanceId'] for i in r.get('InstanceStatuses', [])],
)
# resize autoscaling group (min, max = 0)
log.info('Resize autoscaling group \'%s\' to minSize=0, maxSize=0',
asg['LogicalResourceId'])
response = client_autoscaling.update_auto_scaling_group(
AutoScalingGroupName=asg['PhysicalResourceId'],
MinSize=0,
MaxSize=0
)
if running_instances:
# wait for instances to terminate
waiter_inst_terminated = client_ec2.get_waiter('instance_terminated')
waiter_inst_terminated.wait(InstanceIds=running_instances)
# stopping ec2 instances
instances = [
r['PhysicalResourceId'] for r in resources
if r['ResourceType'] == 'AWS::EC2::Instance'
]
_stop_ec2_instances(awsclient, instances)
# stopping db instances
db_instances = [
r['PhysicalResourceId'] for r in resources
if r['ResourceType'] == 'AWS::RDS::DBInstance'
]
running_db_instances = _filter_db_instances_by_status(
awsclient, db_instances, ['available']
)
for db in running_db_instances:
log.info('Stopping RDS instance \'%s\'', db)
client_rds.stop_db_instance(DBInstanceIdentifier=db)
return exit_code