def remove_instance_alarm(instance):
"""Remove the instance alarm from the ec2 instance
"""
assert isinstance(instance, EC2Instance)
if instance.termination_alarm:
connection = aws_tools.create_cloudwatch_connection(instance.ec2_pool.vpc.access_key)
try:
connection.delete_alarms([instance.termination_alarm])
instance.termination_alarm = None
instance.save()
except Exception, e:
log.exception(e)
return e
def remove_instance_alarm(instance):
"""Remove the instance alarm from the ec2 instance
"""
assert isinstance(instance, EC2Instance)
if instance.termination_alarm:
connection = aws_tools.create_cloudwatch_connection(instance.ec2_pool.vpc.access_key)
try:
connection.delete_alarms([instance.termination_alarm])
instance.termination_alarm = None
instance.save()
except Exception as e:
log.exception(e)
return e
return None
def add_instance_alarm(instance):
"""Add a termination alarm to an EC2 instance. Alarm parameters are taken from ec2_config. Assumes that there is no alarm already present.
"""
assert isinstance(instance, EC2Instance)
#Only go forward if a termination alarm hasn't already been set
if not instance.termination_alarm:
log.debug('Adding termination alarm for instance %s' %instance.instance_id)
connection = aws_tools.create_cloudwatch_connection(instance.ec2_pool.vpc.access_key)
#Get the appropriate metric for creating the alarm
metrics = connection.list_metrics(dimensions={'InstanceId': instance.instance_id}, metric_name='CPUUtilization')
if len(metrics) == 0:
log.debug('Metric not found yet, try again later')
return
#else continue
assert len(metrics) == 1
log.debug('Metric found')
metric = metrics[0]
#Create alarm for this metric
alarm_name = 'cpu_termination_alarm_%s' % instance.instance_id
log.debug('Adding termination alarm for instance %s'%instance.instance_id)
alarm = metric.create_alarm(name=alarm_name,
comparison='<=',
threshold=ec2_config.DOWNSCALE_CPU_THRESHOLD,
period=ec2_config.DONWSCALE_CPU_PERIOD,
evaluation_periods=ec2_config.DOWNSCALE_CPU_EVALUATION_PERIODS,
statistic='Average',
alarm_actions=[instance.ec2_pool.alarm_notify_topic_arn],
)
instance.termination_alarm = alarm_name
assert isinstance(alarm, cloudwatch.MetricAlarm)
instance.save()
else:
pass#Instance alarm already applied. Do nothing
def add_instance_alarm(instance):
"""Add a termination alarm to an EC2 instance. Alarm parameters are taken from ec2_config. Assumes that there is no alarm already present.
"""
assert isinstance(instance, EC2Instance)
#Only go forward if a termination alarm hasn't already been set
if not instance.termination_alarm:
log.debug('Adding termination alarm for instance %s' %instance.instance_id)
connection = aws_tools.create_cloudwatch_connection(instance.ec2_pool.vpc.access_key)
#Get the appropriate metric for creating the alarm
metrics = connection.list_metrics(dimensions={'InstanceId': instance.instance_id}, metric_name='CPUUtilization')
if len(metrics) == 0:
log.debug('Metric not found yet, try again later')
return
#else continue
assert len(metrics) == 1
log.debug('Metric found')
metric = metrics[0]
#Create alarm for this metric
alarm_name = 'cpu_termination_alarm_%s' % instance.instance_id
log.debug('Adding termination alarm for instance %s'%instance.instance_id)
alarm = metric.create_alarm(name=alarm_name,
comparison='<=',
threshold=ec2_config.DOWNSCALE_CPU_THRESHOLD,
period=ec2_config.DONWSCALE_CPU_PERIOD,
evaluation_periods=ec2_config.DOWNSCALE_CPU_EVALUATION_PERIODS,
statistic='Average',
alarm_actions=[instance.ec2_pool.alarm_notify_topic_arn],
)
instance.termination_alarm = alarm_name
assert isinstance(alarm, cloudwatch.MetricAlarm)
instance.save()
else:
pass#Instance alarm already applied. Do nothing
def terminate_instances(instances):
"""Terminate the selected instances. Will also involve terminating any associated alarms and spot requests
instances: iterable EC2Instances, list or queryset
"""
vpc_connection, ec2_connection = aws_tools.create_connections(instances[0].ec2_pool.vpc.access_key)
#Terminate any spot requests first
spot_requests_to_terminate = SpotRequest.objects.filter(ec2_instance__in=instances)
spot_request_ids = [request.request_id for request in spot_requests_to_terminate]
try:
if spot_request_ids != []:
log.debug('Cancelling %d spot requests'%len(spot_request_ids))
ec2_connection.cancel_spot_instance_requests(request_ids=spot_request_ids)
for spot_request in spot_requests_to_terminate:
spot_request.delete()
except Exception as e:
log.exception(e)
log.debug('Deleting termination alarms')
for instance in instances:
try:
cloudwatch_connection = aws_tools.create_cloudwatch_connection(instance.ec2_pool.vpc.access_key)
if instance.termination_alarm:
cloudwatch_connection.delete_alarms([instance.termination_alarm])
except Exception as e:
log.exception(e)
instance_ids = [instance.instance_id for instance in instances]
log.debug('Terminating instances')
#TODO: terminate the necessary alarms and spot requests before terminating the instances themselves.
ec2_connection.terminate_instances(instance_ids)
#Terminate any spot requests first
spot_requests_to_terminate = SpotRequest.objects.filter(ec2_instance__in=instances)
spot_request_ids = [request.request_id for request in spot_requests_to_terminate]
try:
if spot_request_ids != []:
log.debug('Cancelling %d spot requests'%len(spot_request_ids))
ec2_connection.cancel_spot_instance_requests(request_ids=spot_request_ids)
for spot_request in spot_requests_to_terminate:
spot_request.delete()
except Exception, e:
log.exception(e)
log.debug('Deleting termination alarms')
for instance in instances:
try:
cloudwatch_connection = aws_tools.create_cloudwatch_connection(instance.ec2_pool.vpc.access_key)
if instance.termination_alarm:
cloudwatch_connection.delete_alarms([instance.termination_alarm])
except Exception, e:
log.exception(e)
instance_ids = [instance.instance_id for instance in instances]
log.debug('Terminating instances')
#TODO: terminate the necessary alarms and spot requests before terminating the instances themselves.
ec2_connection.terminate_instances(instance_ids)
