def create_cloudwatch_alarms(self, scale_up_policy_arn, scale_down_policy_arn):
self.logger.log("Creating CloudWatch alarms ...")
conn = boto.ec2.cloudwatch.connect_to_region(
self.region,
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key
)
alarm_dimensions = {'AutoScalingGroupName' : 'cloudscale-as'}
scale_up_alarm = MetricAlarm(
name='scale_up_on_cpu', namespace='AWS/EC2',
metric='CPUUtilization', statistic='Average',
comparison='>', threshold='70',
period='60', evaluation_periods=1,
alarm_actions=[scale_up_policy_arn],
dimensions=alarm_dimensions)
scale_down_alarm = MetricAlarm(
name='scale_down_on_cpu', namespace='AWS/EC2',
metric='CPUUtilization', statistic='Average',
comparison='<', threshold='20',
period='60', evaluation_periods=1,
alarm_actions=[scale_down_policy_arn],
dimensions=alarm_dimensions)
conn.create_alarm(scale_up_alarm)
conn.create_alarm(scale_down_alarm)
def scaling_policy(my_group):
'define the scaing policy for the group'
scale_up_policy = ScalingPolicy(name='scale_up',
adjustment_type='ChangeInCapacity',
as_name=my_group,
scaling_adjustment=1,
cooldown=180)
scale_down_policy = ScalingPolicy(name='scale_down',
adjustment_type='ChangeInCapacity',
as_name=my_group,
scaling_adjustment=-1,
cooldown=180)
##########Submitting policy to AWS
as_conn.create_scaling_policy(scale_up_policy)
as_conn.create_scaling_policy(scale_down_policy)
## need to refresh scaling policy by requesting them back
scale_up_policy = as_conn.get_all_policies(as_group=my_group,
policy_names=['scale_up'])[0]
scale_down_policy = as_conn.get_all_policies(as_group=my_group,
policy_names=['scale_down'
])[0]
###CloudWatch alarms that will define when to run the Auto Scaling Policies
alarm_dimensions = {"AutoScalingGroupName": my_group}
#def set_metricAlarm():
'set alarm for scaing the instances in as_group'
scale_up_alarm = MetricAlarm(name='scale_up_on_cpu_"my_group"',
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
comparison='>',
threshold='70',
period='300',
evaluation_periods=2,
alarm_actions=[scale_up_policy.policy_arn],
dimensions=alarm_dimensions)
cloudwatch.create_alarm(scale_up_alarm)
scale_down_alarm = MetricAlarm(
name='scale_down_on_cpu_"my_group"',
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
comparison='<',
threshold='40',
period='300',
evaluation_periods=2,
alarm_actions=[scale_down_policy.policy_arn],
dimensions=alarm_dimensions)
cloudwatch.create_alarm(scale_down_alarm)
def create_alarm(name='test-alarm',
metric='CPUUtilization',
namespace='AWS/EC2',
statistic='Average',
comparison='>=',
threshold=90,
period=500,
evaluation_periods=1,
unit='Percent',
description='Test Alarm',
dimensions=None):
if dimensions is None:
dimensions = {}
cw_conn = boto.connect_cloudwatch('us-east')
metric_alarm = MetricAlarm(name=name,
metric=metric,
namespace=namespace,
statistic=statistic,
comparison=comparison,
threshold=threshold,
period=period,
evaluation_periods=evaluation_periods,
unit=unit,
description=description,
dimensions=dimensions)
alarm_created = cw_conn.put_metric_alarm(metric_alarm)
return cw_conn, alarm_created
def create_autoscaling_policy(autoscale, cluster_name, opts):
scale_up_policy = ScalingPolicy(
name='scale_up',
adjustment_type='ChangeInCapacity',
as_name=cluster_name + "-ag",
scaling_adjustment=opts.scale_up_nodes_amount,
cooldown=opts.scale_up_cooldown)
scale_down_policy = ScalingPolicy(
name='scale_down',
adjustment_type='ChangeInCapacity',
as_name=cluster_name + "-ag",
scaling_adjustment=-opts.scale_down_nodes_amount,
cooldown=opts.scale_down_cooldown)
autoscale.create_scaling_policy(scale_up_policy)
autoscale.create_scaling_policy(scale_down_policy)
scale_up_policy = autoscale.get_all_policies(as_group=cluster_name + "-ag",
policy_names=['scale_up'])[0]
scale_down_policy = autoscale.get_all_policies(
as_group=cluster_name + "-ag", policy_names=['scale_down'])[0]
alarm_dimensions = {"AutoScalingGroupName": cluster_name + "-ag"}
cloudwatch = boto.ec2.cloudwatch.connect_to_region(opts.region)
scale_up_alarm = MetricAlarm(name='scale_up_on_cpu',
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
comparison='>',
threshold='50',
period='60',
evaluation_periods=1,
alarm_actions=[scale_up_policy.policy_arn],
dimensions=alarm_dimensions)
cloudwatch.create_alarm(scale_up_alarm)
scale_down_alarm = MetricAlarm(
name='scale_down_on_cpu',
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
comparison='<',
threshold='40',
period='60',
evaluation_periods=1,
alarm_actions=[scale_down_policy.policy_arn],
dimensions=alarm_dimensions)
cloudwatch.create_alarm(scale_down_alarm)
def cloudwatch_alarms_create(self):
""" Cloudwatch Alarm Create
Only used by the create alarm dialog on scaling group policy page.
Newer alarm dialogs use cloudwatch_alarms PUT method below.
"""
location = self.request.route_path('cloudwatch_alarms')
redirect_location = self.request.params.get('redirect_location')
if redirect_location:
location = self.sanitize_url(redirect_location)
if self.create_form.validate():
with boto_error_handler(self.request, location):
metric = self.request.params.get('metric')
name = self.request.params.get('name')
namespace = self.request.params.get('namespace')
statistic = self.request.params.get('statistic')
comparison = self.request.params.get('comparison')
threshold = self.request.params.get('threshold')
# Convert to seconds
period = int(self.request.params.get('period', 5)) * 60
evaluation_periods = self.request.params.get(
'evaluation_periods')
unit = self.request.params.get('unit')
description = self.request.params.get('description', '')
dimension_param = self.request.params.get('dimension')
dimension = self.get_dimension_name(dimension_param)
dimension_value = self.get_dimension_value(dimension_param)
dimensions = {dimension: dimension_value}
self.log_request(_(u"Creating alarm {0}").format(name))
alarm = MetricAlarm(name=name,
metric=metric,
namespace=namespace,
statistic=statistic,
comparison=comparison,
threshold=threshold,
period=period,
evaluation_periods=evaluation_periods,
unit=unit,
description=description,
dimensions=dimensions)
self.cloudwatch_conn.put_metric_alarm(alarm)
prefix = _(u'Successfully created alarm')
msg = u'{0} {1}'.format(prefix, alarm.name)
if self.request.is_xhr:
resp = JSONResponse()
resp.body = json.dumps(dict(new_alarm=name))
return resp
else:
self.request.session.flash(msg, queue=Notification.SUCCESS)
return HTTPFound(location=location)
else:
error_msg_list = self.create_form.get_errors_list()
if self.request.is_xhr:
return JSONResponse(status=400,
message=', '.join(error_msg_list))
self.request.error_messages = error_msg_list
return self.render_dict
def GetUpMetric(self, queue_name, arn):
return MetricAlarm(name='%sscale_up_sqs' % self.prefix,
namespace='AWS/SQS',
metric='ApproximateNumberOfMessagesVisible',
statistic='Sum',
comparison='>=',
threshold='1',
period=self.period,
evaluation_periods=self.evaluation_periods,
alarm_actions=[arn],
dimensions={"QueueName": queue_name})
def GetDownMetric(self, queue_name, arn):
return MetricAlarm(name='%sscale_down_sqs' % self.prefix,
namespace='AWS/SQS',
metric='NumberOfEmptyReceives',
statistic='Sum',
comparison='>',
threshold='1',
period=self.period,
evaluation_periods=self.evaluation_periods,
alarm_actions=[arn],
dimensions={"QueueName": queue_name})
def create_additional_node(ec2_connection, cw_connection):
# create new instance
new_node = ec2_connection.run_instances(
INSTANCE_TEMPLATE_ID,
security_group_ids=SECURITY_GROUP_IDS,
instance_type="m1.medium",
user_data=("#!/bin/bash\n"
"sudo /usr/local/bin/docker-compose -f"
"/home/ec2-user/croc_cloud_tz/docker-compose.yml up"))
new_node_instance_id = new_node.instances[0].id
new_instance = None
while True:
start_time = time.clock()
while time.clock() - start_time < 10:
# wait until new_node_instance_id would be accessible
try:
new_instance = ec2_connection.get_all_instances(
[new_node_instance_id])[0].instances[0]
break
except EC2ResponseError:
continue
# If dont get new instance end trying
if not new_instance:
return None
# waiting for instance
if new_instance.state != 'running':
time.sleep(1)
continue
# get elastic ip of created instance
elastic_ip = new_instance.ip_address
if not elastic_ip:
# create elastic ip if new instance dont have ip
elastic_ip = ec2_connection.allocate_address().public_ip
ec2_connection.associate_address(new_node_instance_id,
elastic_ip,
allow_reassociation=True)
break
# Create alarm for CPU. Alarm if CPU > 70%. Check it every 2 minutes
alarm = MetricAlarm(
cw_connection,
'{0} CPU Alarm'.format(new_node_instance_id),
'CPUUtilization',
'AWS/EC2',
'Maximum',
'>',
70.0,
60,
1,
dimensions={'InstanceId': [new_node_instance_id]},
)
cw_connection.put_metric_alarm(alarm)
return elastic_ip
def _cw_define(name, alarm_actions=[], *args, **kwargs):
_obj = cw_exists(name)
if _obj:
mico.output.info("use existent cloudwatch alarm: %s" % name)
return _obj
if "namespace" not in kwargs:
kwargs["namespace"] = "AWS/EC2"
_x = MetricAlarm(name=name, alarm_actions=alarm_actions, *args, **kwargs)
# XXX: boto does not handle very well the alarm_actions list when the
# same connection is used for two different cloudwatch alarms, so the
# actions appears to be duplicated in both alarms. We need to force the
# internal list to be empty.
_x.alarm_actions = []
for action in alarm_actions:
mico.output.debug("add new alarm for metric %s: %s" % (name, action.name))
_x.add_alarm_action(action.policy_arn)
return _x
def _cw_define(name, alarm_actions=[], *args, **kwargs):
_obj = cw_exists(name)
if _obj:
mico.output.info("use existent cloudwatch alarm: %s" % name)
return _obj
if "namespace" not in kwargs:
kwargs["namespace"] = "AWS/EC2"
_x = MetricAlarm(name=name, alarm_actions=alarm_actions, *args, **kwargs)
# XXX: boto does not handle very well the alarm_actions list when the
# same connection is used for two different cloudwatch alarms, so the
# actions appears to be duplicated in both alarms. We need to force the
# internal list to be empty.
_x.alarm_actions = []
for action in alarm_actions:
mico.output.debug("add new alarm for metric %s: %s" %
(name, action.name))
_x.add_alarm_action(action.policy_arn)
return _x
def create_scale_alarm(conn_as,
conn_cw,
as_group_name,
alarm_name,
scaling_policy_name='scale_up',
metric_name='CPUUtilization',
comparison='>',
threshold=70,
period=60,
eval_periods=2,
statistics='Average'):
""" This method will be the responsible for establishing an alarm, alarm that will trigger one of the predefined
policies for the Auto Scaling Group when a certain condition is met, all the parameters will be customizable """
try:
# Now we create the alarm dimensions, in this case, the alarm will apply the metric over the whole group
# instead of over each of the instances
alarm_dimensions = {"AutoScalingGroupName": as_group_name}
# We recover the scaling policy the user decided to use, and warn the user if a warning was thrown
scaling_policy = conn_as.get_all_policies(
as_group=as_group_name, policy_names=[scaling_policy_name])[0]
if not scaling_policy:
print "Scaling policy", scaling_policy_name, "not found"
return False
# Now we create the metric based alarm and pass the parameters the user choose in the menu
scale_up_alarm = MetricAlarm(
name=alarm_name,
namespace='AWS/EC2',
metric=metric_name,
statistic=statistics,
comparison=comparison,
threshold=threshold,
period=period,
evaluation_periods=eval_periods,
alarm_actions=[scaling_policy.policy_arn],
dimensions=alarm_dimensions)
# Once the alarm is created, we push the alarm to AWS for it to work.
conn_cw.create_alarm(scale_up_alarm)
print "Alarm created"
return True
# If an exception has been thrown, we advice the user to try again.
except exception.BotoServerError:
print "One or more of the fields typed are incorrect, \n" \
"Have you created the policies?? \n" \
"please, try again and follow instructions."
return False
def scale_up_alarm(self, as_conn, group_name, cw_conn):
""" Creates an alarm which launches new instances when triggered """
scale_up_policy = ScalingPolicy(name='scale_up', adjustment_type='ChangeInCapacity',
as_name=group_name, scaling_adjustment=1, cooldown=180)
as_conn.create_scaling_policy(scale_up_policy)
scale_up_policy = as_conn.get_all_policies(as_group=group_name, policy_names=['scale_up'])[0]
alarm_dimensions = {"AutoScalingGroupName": group_name}
scale_up_alarm = MetricAlarm(name='scale_up_on_cpu', namespace='AWS/EC2', metric='CPUUtilization',
statistic='Average', comparison='>', threshold=70.0, period=60,
evaluation_periods=2, alarm_actions=[scale_up_policy.policy_arn],
dimensions=alarm_dimensions)
alarm = cw_conn.create_alarm(scale_up_alarm)
if alarm:
print "Alarm has been created,", alarm
else:
print "Error creating alarm"
def create_alarm():
cwConnection = connect_to_cloudwatch()
# scale_up_policy = find policy
scale_up_alarm = MetricAlarm(
name='scale_up_on_cpu',
metric='Number Of Messages Ready', #'CPUUtilization',
namespace='SSRMQ:List Processing', #'AWS/EC2',
statistic='Sum', #'Average',
comparison='>',
threshold='0',
period='60',
evaluation_periods=0,
alarm_actions=[scale_up_policy.policy_arn] #,
#dimensions=alarm_dimensions
)
cwConnection.create_alarm(scale_up_alarm)
def get_cpuutilization_alarms_to_create(rds_instances,
threshold,
aws_cw_connect,
sns_topic_arn):
"""
Creates a CPUUtilization alarm for all RDS instances
Args:
rds_instances (list) List of all RDS instances
threshold (int) The upper limit after which alarm activates
aws_cw_connect (CloudWatchConnection)
sns_topic_arn (str)
Returns:
(set) All CPUUtilization alarms that will be created
"""
assert isinstance(rds_instances, list)
assert isinstance(aws_cw_connect,
boto.ec2.cloudwatch.CloudWatchConnection)
assert isinstance(threshold, int)
assert isinstance(sns_topic_arn, str)
alarms_to_create = set()
existing_alarms = get_existing_cpuutilization_alarm_names(aws_cw_connect)
for instance in rds_instances:
# initiate a CPUUtilization MetricAlarm object for each RDS instance
cpu_utilization_alarm = MetricAlarm(
name=u'RDS-{}-Low-Free-Storage-Space'.format(
instance[u'DBInstanceIdentifier']
),
namespace=u'AWS/RDS',
metric=u'FreeStorageSpace', statistic='Average',
comparison=u'<',
threshold=(0.20*instance[u'AllocatedStorage'])*1000000000,
period=60, evaluation_periods=10,
alarm_actions=[sns_topic_arn],
dimensions={u'DBInstanceIdentifier':
instance[u'DBInstanceIdentifier']
}
)
if cpu_utilization_alarm.name not in existing_alarms:
alarms_to_create.add(cpu_utilization_alarm)
return alarms_to_create
def to_metric_alarm(self, policy_arn):
"""
Returns a MetricAlarm for a given policy.
"""
return MetricAlarm(
alarm_actions=[policy_arn],
ok_actions=[policy_arn],
comparison='>' if self.threshold_type == "max" else "<",
dimensions=self.dimensions,
evaluation_periods=self.duration,
metric=self.metric_name,
name=self.name,
namespace=self.namespace,
period=self.period,
statistic=self.statistic,
threshold=self.threshold,
)
def setAlarm():
sns = boto.sns.connect_to_region('eu-west-1',
aws_access_key_id=key_id,
aws_secret_access_key=secret_key)
cw = boto.ec2.cloudwatch.connect_to_region(
"eu-west-1",
aws_access_key_id=key_id,
aws_secret_access_key=secret_key)
print "\nEnabling Cloudwatch monitoring on running instances..\n"
topics = sns.get_all_topics()
topic = topics[u'ListTopicsResponse']['ListTopicsResult']['Topics'][0][
'TopicArn']
emailAddr = raw_input(
"Please enter an email address to receive alarm notifications: ")
print "\nSNS email address set to %s\n" % emailAddr
any_running = False
for reservation in conn.get_all_reservations():
for instance in reservation.instances:
if instance.state == u'running':
any_running = True
conn.monitor_instance(instance.id)
alarm_name = 'CPU Utilization < 40 for instance: %s' % instance.id
alarm_dimensions = {"InstanceId": instance.id}
alarm = MetricAlarm(name=alarm_name,
comparison='<',
threshold=40,
period=300,
evaluation_periods=2,
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
alarm_actions=[topic],
dimensions=alarm_dimensions)
cw.create_alarm(alarm)
print "Alarm set for instance: %s" % instance.id
sns.subscribe(topic, "email", emailAddr)
if not any_running:
print "\nNo running instances found. No alarm set."
def create_metric_alarm(connection, module):
name = module.params.get('name')
metric = module.params.get('metric')
namespace = module.params.get('namespace')
statistic = module.params.get('statistic')
comparison = module.params.get('comparison')
threshold = module.params.get('threshold')
period = module.params.get('period')
evaluation_periods = module.params.get('evaluation_periods')
unit = module.params.get('unit')
description = module.params.get('description')
dimensions = module.params.get('dimensions')
alarm_actions = module.params.get('alarm_actions')
insufficient_data_actions = module.params.get('insufficient_data_actions')
ok_actions = module.params.get('ok_actions')
alarms = connection.describe_alarms(alarm_names=[name])
if not alarms:
alm = MetricAlarm(
name=name,
metric=metric,
namespace=namespace,
statistic=statistic,
comparison=comparison,
threshold=threshold,
period=period,
evaluation_periods=evaluation_periods,
unit=unit,
description=description,
dimensions=dimensions,
alarm_actions=alarm_actions,
insufficient_data_actions=insufficient_data_actions,
ok_actions=ok_actions
)
try:
connection.create_alarm(alm)
changed = True
alarms = connection.describe_alarms(alarm_names=[name])
except BotoServerError, e:
module.fail_json(msg=str(e))
def make_sleepy(self, parameters, instance_id, period='3600'):
print "Making instance", instance_id, "sleepy..."
credentials = parameters[self.PARAM_CREDENTIALS]
ec2 = boto.connect_cloudwatch(str(credentials['EC2_ACCESS_KEY']), str(credentials['EC2_SECRET_KEY']))
region = "us-east-1"
terminate_arn = 'arn:aws:automate:{0}:ec2:terminate'.format(region)
alarm_name = 'ec2_shutdown_sleepy_{0}'.format(instance_id)
# define our alarm to terminate the instance if it gets sleepy
# i.e. if CPU utilisation is less than 10% for 1 x 4 hr intervals
sleepy_alarm = MetricAlarm(
name=alarm_name, namespace='AWS/EC2',
metric='CPUUtilization', statistic='Average',
comparison='<', threshold='10',
period=period, evaluation_periods=4,
alarm_actions=[terminate_arn],
dimensions={'InstanceId': instance_id})
# create the alarm.. Zzzz!
ec2.put_metric_alarm(sleepy_alarm)
def make_instance_sleepy(self, instance_id):
print "Setting alarm to stop instance after 4 hours of idle cpu."
ec2 = boto.ec2.cloudwatch.connect_to_region(
self.region,
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key)
stop_arn = 'arn:aws:automate:{0}:ec2:stop'.format(self.region)
alarm_name = 'ec2_shutdown_sleepy_{0}'.format(instance_id)
# define our alarm to stop the instance if it gets sleepy
# i.e. if CPU utilization is less than 10% for 1 x 4 hr intervals
sleepy_alarm = MetricAlarm(name=alarm_name,
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
comparison='<',
threshold='10',
period='3600',
evaluation_periods=4,
alarm_actions=[stop_arn],
dimensions={'InstanceId': instance_id})
ec2.create_alarm(sleepy_alarm)
conn.create_scaling_policy(scale_up_policy)
conn.create_scaling_policy(scale_down_policy)
scale_up_policy = conn.get_all_policies(
as_group=AUTO_SCALE_GROUP, policy_names=['scale_up'])[0]
scale_down_policy = conn.get_all_policies(
as_group=AUTO_SCALE_GROUP, policy_names=['scale_down'])[0]
cloudwatch = boto.ec2.cloudwatch.connect_to_region("us-east-1", aws_access_key_id=ACCESS_KEY_ID, aws_secret_access_key=SECRET_ACCESS_KEY)
alarm_dimensions = {"AutoScalingGroupName": AUTO_SCALE_GROUP}
scale_up_alarm = MetricAlarm(
name='scale_up_on_cpu', namespace='AWS/EC2',
metric='CPUUtilization', statistic='Average',
comparison='>', threshold='65',
period='180', evaluation_periods=1,
alarm_actions=[scale_up_policy.policy_arn],
dimensions=alarm_dimensions)
cloudwatch.create_alarm(scale_up_alarm)
scale_down_alarm = MetricAlarm(
name='scale_down_on_cpu', namespace='AWS/EC2',
metric='CPUUtilization', statistic='Average',
comparison='<', threshold='25',
period='240', evaluation_periods=1,
alarm_actions=[scale_down_policy.policy_arn],
dimensions=alarm_dimensions)
cloudwatch.create_alarm(scale_down_alarm)
time.sleep(60)
#!/usr/bin/env python
#objective: create a cloudwatch alarm using Boto
from boto.ec2.cloudwatch import connect_to_region, MetricAlarm
aws_connection = connect_to_region('us-east-1')
print aws_connection.__class__
new_alarm = MetricAlarm(
name='new_alarm',
metric='CPUUtilization',
namespace='AWS/EC2',
statistic='Average',
comparison='>=',
threshold=90.0, # threshold is float
period=60, # period is int, measured in seconds
evaluation_periods=5,
unit='Percent',
description='',
dimensions={'InstanceId': 'i-xxxxxxxx'},
alarm_actions=None,
insufficient_data_actions=None,
ok_actions=None)
print new_alarm.__class__
aws_connection.create_alarm(new_alarm)
def cloudwatch_alarms_update(self):
message = json.loads(self.request.body)
alarm = message.get('alarm', {})
update = alarm.get('update', False)
token = message.get('csrf_token')
flash = message.get('flash')
if not self.is_csrf_valid(token):
return JSONResponse(status=400, message="missing CSRF token")
name = alarm.get('name')
metric = alarm.get('metric')
namespace = alarm.get('namespace')
statistic = alarm.get('statistic')
comparison = alarm.get('comparison')
threshold = alarm.get('threshold')
period = alarm.get('period')
evaluation_periods = alarm.get('evaluation_periods')
unit = alarm.get('unit')
description = alarm.get('description')
dimensions = alarm.get('dimensions')
if isinstance(
dimensions,
str): # Copy Alarm dialog sends dimensions as a JSON string
dimensions = json.loads(dimensions)
insufficient_data_actions = alarm.get('insufficient_data_actions')
alarm_actions = alarm.get('alarm_actions')
ok_actions = alarm.get('ok_actions')
metric_alarm = MetricAlarm(
name=name,
metric=metric,
namespace=namespace,
statistic=statistic,
comparison=comparison,
threshold=threshold,
period=period,
evaluation_periods=evaluation_periods,
unit=unit,
description=description,
dimensions=dimensions,
alarm_actions=alarm_actions,
ok_actions=ok_actions,
insufficient_data_actions=insufficient_data_actions)
with boto_error_handler(self.request):
self.log_request(
_(u'Updating alarm {0}').format(alarm.get('name')))
action = self.cloudwatch_conn.put_metric_alarm(metric_alarm)
if action:
if update:
prefix = _(u'Successfully updated alarm')
else:
prefix = _(u'Successfully created alarm')
else:
if update:
prefix = _(u'There was a problem updating alarm')
else:
prefix = _(u'There was a problem creating alarm')
msg = u'{0} {1}'.format(prefix, alarm.get('name'))
if flash is not None:
self.request.session.flash(msg, queue=Notification.SUCCESS)
return dict(success=action, message=msg)
def create_AutoScaling():
print "Creating AutoScaling..."
# establish connection
as_conn = AutoScaleConnection(AWSAccessKeyId, AWSSecretKey)
# create launch configuration
global lc
lc = LaunchConfiguration(name='lc',
image_id=DATA_CEN_AMI,
key_name=ACCESS_KEY,
instance_monitoring=True,
security_groups=[SECURITY_GRP],
instance_type=MACHINE_TYPE)
as_conn.create_launch_configuration(lc)
# create tag for autoscaling group
as_tag = Tag(key="Project",
value="2.2",
propagate_at_launch=True,
resource_id='my_group')
# create aotoscaling group
global ag
ag = AutoScalingGroup(group_name='my_group',
load_balancers=['myELB'],
availability_zones=['us-east-1a'],
launch_config=lc,
min_size=MIN_SIZE,
max_size=MAX_SIZE,
connection=as_conn,
tags=[as_tag])
# associate the autoscaling group with launch configuration
as_conn.create_auto_scaling_group(ag)
# build the scale policy
scale_up_policy = ScalingPolicy(name='scale_up',
adjustment_type='ChangeInCapacity',
as_name='my_group',
scaling_adjustment=1,
cooldown=60)
scale_down_policy = ScalingPolicy(name='scale_down',
adjustment_type='ChangeInCapacity',
as_name='my_group',
scaling_adjustment=-1,
cooldown=60)
# register the scale policy
as_conn.create_scaling_policy(scale_up_policy)
as_conn.create_scaling_policy(scale_down_policy)
# refresh the scale policy for extra information
scale_up_policy = as_conn.get_all_policies(as_group='my_group',
policy_names=['scale_up'])[0]
scale_down_policy = as_conn.get_all_policies(as_group='my_group',
policy_names=['scale_down'
])[0]
# create cloudwatch alarm
cloudwatch = CloudWatchConnection(aws_access_key_id=AWSAccessKeyId,
aws_secret_access_key=AWSSecretKey,
is_secure=True)
# region='us-east-1a')
# assocate cloudwatch with alarm
alarm_dimensions = {"AutoScalingGroupName": 'my_group'}
# create scale up alarm
scale_up_alarm = MetricAlarm(name='scale_up_on_cpu',
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
comparison='>',
threshold='50',
period='60',
evaluation_periods=2,
alarm_actions=[scale_up_policy.policy_arn],
dimensions=alarm_dimensions)
cloudwatch.create_alarm(scale_up_alarm)
# create scale down alarm
scale_down_alarm = MetricAlarm(
name='scale_down_on_cpu',
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
comparison='<',
threshold='20',
period='60',
evaluation_periods=1,
alarm_actions=[scale_down_policy.policy_arn],
dimensions=alarm_dimensions)
cloudwatch.create_alarm(scale_down_alarm)
print "AutoScaling created successfully"
def update_or_create(self):
alarm = MetricAlarm(name=self.name, **self.configuration)
self.resource = self.cloudwatch.create_alarm(alarm)
#redeclare policies to populate the ARN fields
policyResults = conn_as.get_all_policies(as_group=autoscaleGroup.name, policy_names=[scaleUpPolicy.name])
scaleUpPolicy = policyResults[0]
policyResults = conn_as.get_all_policies(as_group=autoscaleGroup.name, policy_names=[scaleDownPolicy.name])
scaleDownPolicy = policyResults[0]
#connect to Cloud Watch
cw_conn = boto.ec2.cloudwatch.connect_to_region(REGION)
#create the following alarms: ScaleUp @ Avg CPU >60% over 2 periods OR ELB latency >= 0.5sec. ScaleDown @ Avg CPU <30% over 2 periods
dimensions = {"AutoScalingGroupName" : autoscaleGroup.name}
dimensions_elb = {"LoadBalancerName" : ELB_NAME}
scaleUpAlarmCPU = MetricAlarm(name='Alfresco-HighCPU', namespace='AWS/EC2',metric='CPUUtilization', statistic='Average', comparison='>', threshold='60', evaluation_periods=2, period=60, unit='Percent' , alarm_actions=[scaleUpPolicy.policy_arn], dimensions=dimensions)
scaleDownAlarmCPU = MetricAlarm(name='Alfresco-LowCPU', namespace='AWS/EC2',metric='CPUUtilization', statistic='Average', comparison='<', threshold='30', evaluation_periods=2, period=60, unit='Percent', alarm_actions=[scaleDownPolicy.policy_arn], dimensions=dimensions)
scaleUpAlarmLatency = MetricAlarm(name='Alfresco-HighLatency', namespace='AWS/ELB', metric='Latency', statistic='Average', comparison='>', threshold='1', evaluation_periods=2, period=60, unit='Seconds', alarm_actions=[scaleUpPolicy.policy_arn],dimensions=dimensions_elb)
cw_conn.create_alarm(scaleUpAlarmCPU)
cw_conn.create_alarm(scaleDownAlarmCPU)
cw_conn.create_alarm(scaleUpAlarmLatency)
## TODO: uncomment terminate instance command
#Terminate this setup instance now that auto-scaling is configured
conn.terminate_instances(INSTANCE)
#done
def setAlarm():
#An SNS connection object is created by passing the region, key and secret key to the connection_to_region() method
sns = boto.sns.connect_to_region('eu-west-1',
aws_access_key_id=key_id,
aws_secret_access_key=secret_key)
#An cloudwatch connection object is created by passing the region, key and secret key to the connection_to_region() method
cw = boto.ec2.cloudwatch.connect_to_region(
"eu-west-1",
aws_access_key_id=key_id,
aws_secret_access_key=secret_key)
print "\nEnabling Cloudwatch monitoring on running instances..\n"
#Invoke the get_all_topics method to return a dictionary containing all topic arns on the account
topics = sns.get_all_topics()
#Get the first [0] topic arm from the dictionary
topic = topics[u'ListTopicsResponse']['ListTopicsResult']['Topics'][0][
'TopicArn']
emailAddr = raw_input(
"Please enter an email address to receive alarm notifications: ")
#emailAddr = "*****@*****.**"
print "\nSNS email address set to %s\n" % emailAddr
#Iterate through the list of reservation objects returned by get_all_reservations()
#This method returns a list of all EC2 instance objects
for reservation in conn.get_all_reservations():
#iterate through all instances in reservation
for instance in reservation.instances:
#Set boolean to alert use if no running instances found
any_running = False
#for any instances that have their state set to running
if instance.state == u'running':
#Call the monitor_instance method on the EC2 connection object with the specified instance id to enable monitoring
conn.monitor_instance(instance.id)
alarm_name = 'CPU Utilization < 40 for instance: %s' % instance.id
#Set the dimensions for the alarm using the current instance id
alarm_dimensions = {"InstanceId": instance.id}
#Set up the alarm by passing in relevant arguments to the Metric alarm method
#Notable arguments would be threshold being the % to check against and dimensions being the instances to activate the alarm on
alarm = MetricAlarm(name=alarm_name,
comparison='<',
threshold=40,
period=300,
evaluation_periods=2,
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
alarm_actions=[topic],
dimensions=alarm_dimensions)
#Create the alarm
cw.create_alarm(alarm)
print "Alarm set for instance: %s" % instance.id
#Subscribe to the alarm using the specified email address and the email protocol
sns.subscribe(topic, "email", emailAddr)
any_running = True
if not any_running:
print "\nNo running instances found. No alarm set."
def create_metric_alarm(connection, module):
name = module.params.get('name')
metric = module.params.get('metric')
namespace = module.params.get('namespace')
statistic = module.params.get('statistic')
comparison = module.params.get('comparison')
threshold = module.params.get('threshold')
period = module.params.get('period')
evaluation_periods = module.params.get('evaluation_periods')
unit = module.params.get('unit')
description = module.params.get('description')
dimensions = module.params.get('dimensions')
alarm_actions = module.params.get('alarm_actions')
insufficient_data_actions = module.params.get('insufficient_data_actions')
ok_actions = module.params.get('ok_actions')
alarms = connection.describe_alarms(alarm_names=[name])
if not alarms:
alm = MetricAlarm(name=name,
metric=metric,
namespace=namespace,
statistic=statistic,
comparison=comparison,
threshold=threshold,
period=period,
evaluation_periods=evaluation_periods,
unit=unit,
description=description,
dimensions=dimensions,
alarm_actions=alarm_actions,
insufficient_data_actions=insufficient_data_actions,
ok_actions=ok_actions)
try:
connection.create_alarm(alm)
changed = True
alarms = connection.describe_alarms(alarm_names=[name])
except BotoServerError as e:
module.fail_json(msg=str(e))
else:
alarm = alarms[0]
changed = False
for attr in ('comparison', 'metric', 'namespace', 'statistic',
'threshold', 'period', 'evaluation_periods', 'unit',
'description'):
if getattr(alarm, attr) != module.params.get(attr):
changed = True
setattr(alarm, attr, module.params.get(attr))
#this is to deal with a current bug where you cannot assign '<=>' to the comparator when modifying an existing alarm
comparison = alarm.comparison
comparisons = {
'<=': 'LessThanOrEqualToThreshold',
'<': 'LessThanThreshold',
'>=': 'GreaterThanOrEqualToThreshold',
'>': 'GreaterThanThreshold'
}
alarm.comparison = comparisons[comparison]
dim1 = module.params.get('dimensions')
dim2 = alarm.dimensions
for keys in dim1:
if not isinstance(dim1[keys], list):
dim1[keys] = [dim1[keys]]
if keys not in dim2 or dim1[keys] != dim2[keys]:
changed = True
setattr(alarm, 'dimensions', dim1)
for attr in ('alarm_actions', 'insufficient_data_actions',
'ok_actions'):
action = module.params.get(attr) or []
if getattr(alarm, attr) != action:
changed = True
setattr(alarm, attr, module.params.get(attr))
try:
if changed:
connection.create_alarm(alarm)
except BotoServerError as e:
module.fail_json(msg=str(e))
result = alarms[0]
module.exit_json(
changed=changed,
name=result.name,
actions_enabled=result.actions_enabled,
alarm_actions=result.alarm_actions,
alarm_arn=result.alarm_arn,
comparison=result.comparison,
description=result.description,
dimensions=result.dimensions,
evaluation_periods=result.evaluation_periods,
insufficient_data_actions=result.insufficient_data_actions,
last_updated=result.last_updated,
metric=result.metric,
namespace=result.namespace,
ok_actions=result.ok_actions,
period=result.period,
state_reason=result.state_reason,
state_value=result.state_value,
statistic=result.statistic,
threshold=result.threshold,
unit=result.unit)
def get_cpuutilization_alarms_to_create(rds_instances,
threshold,
aws_cw_connect,
sns_topic_arn):
"""
Creates a CPUUtilization alarm for all RDS instances
Args:
rds_instances (list) List of all RDS instances
threshold (int) The upper limit after which alarm activates
aws_cw_connect (CloudWatchConnection)
sns_topic_arn (str)
Returns:
(set) All CPUUtilization alarms that will be created
"""
assert isinstance(rds_instances, list)
assert isinstance(aws_cw_connect,
boto.ec2.cloudwatch.CloudWatchConnection)
assert isinstance(threshold, int)
assert isinstance(sns_topic_arn, str)
alarms_to_create = set()
existing_alarms = get_existing_cpuutilization_alarm_names(aws_cw_connect)
db_classes = {
'db.t1.micro': 0.615,
'db.m1.small': 1.7,
'db.m1.medium': 3.75,
'db.m1.large': 7.5,
'db.m1.xlarge': 15,
'db.m4.large': 8,
'db.m4.xlarge': 16,
'db.m4.2xlarge': 32,
'db.m4.4xlarge': 64,
'db.m4.10xlarge': 160,
'db.r3.large': 15,
'db.r3.xlarge': 30.5,
'db.r3.2xlarge': 61,
'db.r3.4xlarge': 122,
'db.r3.8xlarge': 244,
'db.t2.micro': 1,
'db.t2.small': 2,
'db.t2.medium': 4,
'db.t2.large': 8,
'db.m3.medium': 3.75,
'db.m3.large': 7.5,
'db.m3.xlarge': 15,
'db.m3.2xlarge': 30,
'db.m2.xlarge': 17.1,
'db.m2.2xlarge': 34.2,
'db.m2.4xlarge': 68.4,
'db.cr1.8xlarge': 244,
}
for instance in rds_instances:
# initiate a CPUUtilization MetricAlarm object for each RDS instance
cpu_utilization_alarm = MetricAlarm(
name=u'RDS-{}-Very-Low-Memory'.format(
instance[u'DBInstanceIdentifier']
),
namespace=u'AWS/RDS',
metric=u'FreeableMemory', statistic='Average',
comparison=u'<',
threshold=0.20*(db_classes[instance[u'DBInstanceClass']]*1000000000),
period=60, evaluation_periods=10,
alarm_actions=[sns_topic_arn],
dimensions={u'DBInstanceIdentifier':
instance[u'DBInstanceIdentifier']
}
)
if cpu_utilization_alarm.name not in existing_alarms:
alarms_to_create.add(cpu_utilization_alarm)
return alarms_to_create
def run(self):
all_lcs = self.as_conn.get_all_launch_configurations()
lc_by_group = defaultdict(list)
lc_max_num_by_group = defaultdict(int)
for lc in all_lcs:
name, num = lc.name.split('-')
num = int(num)
lc_by_group[name].append(lc)
if num > lc_max_num_by_group[name]:
lc_max_num_by_group[name] = num
all_ags = self.as_conn.get_all_groups()
ag_by_name = {}
for ag in all_ags:
ag_by_name[ag.name] = ag
for group_name, config in self.as_config["groups"].iteritems():
print "Configuring %s" % group_name
use_lc = None
lc_to_delete = []
for lc in lc_by_group[group_name]:
if use_lc is None and \
lc.image_id == config['ami'] and \
lc.key_name == config['ssh_key'] and \
lc.instance_type == config['instance_type'] and \
lc.security_groups == [config['security_group']] and \
lc.user_data == self.user_data:
print " Found LaunchConfig %s that matches profile" % \
lc.name
use_lc = lc
else:
lc_to_delete.append(lc)
print " Found %d LaunchConfigurations to delete" % len(
lc_to_delete)
if not use_lc:
print " Making LaunchConfiguration for %s" % group_name
lc_num = lc_max_num_by_group[group_name] + 1
use_lc = LaunchConfiguration(
name="%s-%d" % (group_name, lc_num),
image_id=config['ami'],
key_name=config['ssh_key'],
instance_type=config['instance_type'],
security_groups=[config['security_group']],
user_data=self.user_data)
self.as_conn.create_launch_configuration(use_lc)
if group_name in ag_by_name:
print " Found existing AutoScalingGroup, updating"
ag = ag_by_name[group_name]
ag_exists = True
else:
print " Making new AutoScalingGroup"
ag = AutoScalingGroup()
ag_exists = False
# config ASG as we want it
ag.name = group_name
ag.launch_config_name = use_lc.name
ag.availability_zones = config['zones']
ag.desired_capacity = config['capacity']
ag.min_size = config['min_size']
ag.max_size = config['max_size']
# create or update as appropriate
if ag_exists:
ag.update()
else:
self.as_conn.create_auto_scaling_group(ag)
# make it send e-mail whenever it does something
if 'notification_topic' in self.as_config:
# NOTE [adam Sept/18/12]: this is a hack designed to work
# around that boto support for this isn't in a release yet.
# when the next release is out, we should uncomment the
# code below.
params = {
'AutoScalingGroupName': ag.name,
'TopicARN': self.as_config['notification_topic']
}
self.as_conn.build_list_params(params, self.AS_NOTIFICATIONS,
'NotificationTypes')
self.as_conn.get_status('PutNotificationConfiguration', params)
#as_conn.put_notification_configuration(
# ag.name,
# self.as_config['notification_topic'],
# self.AS_NOTIFICATIONS)
tags = []
for tag_name, tag_value in config.get('tags', {}).iteritems():
print " Adding tag %s = %s" % (tag_name, tag_value)
tags.append(
Tag(key=tag_name,
value=tag_value,
propagate_at_launch=True,
resource_id=ag.name))
self.as_conn.create_or_update_tags(tags)
for lc in lc_to_delete:
print " Deleting old LaunchConfiguration %s" % lc.name
lc.delete()
for alarm_name, alarm_cfg in config.get('alarms', {}).iteritems():
alarm_policy_arn = self.make_policy(group_name,
alarm_cfg['policy'])
alarm_name = '%s|%s|%s' % (group_name, alarm_cfg['policy'],
alarm_cfg['metric'])
alarm = MetricAlarm(
name=alarm_name,
namespace=alarm_cfg['namespace'],
metric=alarm_cfg['metric'],
statistic='Average',
dimensions={'AutoScalingGroupName': group_name},
comparison=alarm_cfg['comparison'],
threshold=alarm_cfg['threshold'],
period=alarm_cfg['period'],
evaluation_periods=alarm_cfg.get('evaluation_periods', 1),
alarm_actions=[alarm_policy_arn])
self.cw_conn.put_metric_alarm(alarm)
def setup(CONF):
global out
lookup_tbl = {
'name': CONF['NAME'],
}
conn = AutoScaleConnection()
out['conn'] = conn
# Launch Configurations
LC = CONF['LC']
LC['name'] = LC['name'] % lookup_tbl
lc = LaunchConfiguration(**LC)
conn.create_launch_configuration(lc)
out['lc'] = lc
# Auto Scaling Group
ASG = CONF['ASG']
ASG['group_name'] = ASG['group_name'] % lookup_tbl
ASG['launch_config'] = lc
groups = conn.get_all_groups(names=[ASG['group_name']])
if (len(groups) > 0):
# update
asg = groups[0]
for k in ASG :
# asg not iterable, try-except to make sure asg[k] exists
try: asg.__getattribute__(k)
except: continue
asg.__setattr__(k, ASG[k])
asg.launch_config_name = LC['name']
asg.update()
out['asg'] = asg
else:
#create
asg = AutoScalingGroup(**ASG)
conn.create_auto_scaling_group(asg)
# ASG Tags
ASG_TAGS = CONF['ASG_TAGS']
for i in ASG_TAGS:
if 'propagate_at_launch' not in i:
i['propagate_at_launch'] = True
i['key'] = i['key'] % lookup_tbl
i['value'] = i['value'] % lookup_tbl
tags = [
Tag(**dict(x.items() + [('resource_id', ASG['group_name'])])) for x in ASG_TAGS
]
conn.create_or_update_tags(tags)
# Triggers (Scaling Policy / Cloudwatch Alarm)
conn_cw = connect_to_region(CONF['REGION'])
TRIGGERS = CONF['TRIGGERS']
for T in TRIGGERS:
T['policy']['name'] = T['policy']['name'] % lookup_tbl
T['policy']['as_name'] = ASG['group_name']
T['alarm']['dimensions'] = {'AutoScalingGroupName': ASG['group_name']}
T['alarm']['alarm_actions'] = None
if 'name' in T['alarm']:
T['alarm']['name'] = T['alarm']['name'] % lookup_tbl
else:
T['alarm']['name'] = T['policy']['name']
# Policies are safely overwritten, so not checked for existence
conn.create_scaling_policy(ScalingPolicy(**T['policy']))
policy = conn.get_all_policies(as_group=ASG['group_name'], policy_names=[T['policy']['name']])[0]
T['alarm']['alarm_actions'] = [policy.policy_arn]
hits = conn_cw.describe_alarms(alarm_names=[T['alarm']['name']])
conn_cw.create_alarm(MetricAlarm(**T['alarm']))
instances = [i for r in all_instances for i in r.instances]
for instance in instances:
if instance.__dict__['id'] == thisInstanceId:
thisAutoScalename = instance.__dict__['tags']['aws:autoscaling:groupName']
# Define the ScaleDownPolicy
ScalingDownPolicy = ScalingPolicy(name='ctScaleDown',
adjustment_type='ChangeInCapacity',
as_name=thisAutoScalename,
scaling_adjustment=-1,
cooldown=180)
asconn.create_scaling_policy(ScalingDownPolicy)
ScaleDownPolicy = asconn.get_all_policies(as_group=thisAutoScalename, policy_names=['ctScaleDown'])[0]
alarm_actions = []
alarm_actions.append(ScaleDownPolicy.policy_arn)
ApacheStatusAlarm = MetricAlarm(name=alarmname,
namespace=namespace,
metric=metricname,
statistic=unitname,
comparison='>',
threshold='2',
period='60',
evaluation_periods=2,
alarm_actions=alarm_actions,
dimensions=dimensions)
clconn.create_alarm(ApacheStatusAlarm)
conn.create_scaling_policy(scale_up_policy)
conn.create_scaling_policy(scale_down_policy)
scale_up_policy = conn.get_all_policies(as_group='my_autoscale_group',
policy_names=['scale_up'])[0]
scale_down_policy = conn.get_all_policies(as_group='my_autoscale_group',
policy_names=['scale_down'])[0]
cloudwatch = boto.ec2.cloudwatch.connect_to_region('us-east-1')
alarm_dimensions = {"AutoScalingGroupName": 'my_autoscale_group'}
scale_up_alarm = MetricAlarm(name='scale_up_on_cpu',
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
comparison='>',
threshold='70',
period='60',
evaluation_periods=2,
alarm_actions=[scale_up_policy.policy_arn],
dimensions=alarm_dimensions)
cloudwatch.create_alarm(scale_up_alarm)
scale_down_alarm = MetricAlarm(name='scale_down_on_cpu',
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
comparison='<',
threshold='40',
period='60',
evaluation_periods=2,