def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(_('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
LOG.exception(_('in fixed duration looping call'))
done.send_exception(*sys.exc_info())
return
else:
done.send(True)
def _inner():
if initial_delay:
greenthread.sleep(initial_delay)
try:
while self._running:
start = timeutils.utcnow()
self.f(*self.args, **self.kw)
end = timeutils.utcnow()
if not self._running:
break
delay = interval - timeutils.delta_seconds(start, end)
if delay <= 0:
LOG.warn(
_('task run outlasted interval by %s sec') %
-delay)
greenthread.sleep(delay if delay > 0 else 0)
except LoopingCallDone as e:
self.stop()
done.send(e.retvalue)
except Exception:
LOG.exception(_('in fixed duration looping call'))
done.send_exception(*sys.exc_info())
return
else:
done.send(True)
def test_evaluate_suspend(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
now = timeutils.utcnow()
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().MultipleTimes().AndReturn(now)
self.m.ReplayAll()
# Now data breaches Threshold, but we're suspended
last = now - datetime.timedelta(seconds=300)
data = WatchData(35, now - datetime.timedelta(seconds=150))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
self.wr.state_set(self.wr.SUSPENDED)
actions = self.wr.evaluate()
self.assertEqual(self.wr.SUSPENDED, self.wr.state)
self.assertEqual([], actions)
def test_evaluate_suspend(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
now = timeutils.utcnow()
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().MultipleTimes().AndReturn(now)
self.m.ReplayAll()
# Now data breaches Threshold, but we're suspended
last = now - datetime.timedelta(seconds=300)
data = WatchData(35, now - datetime.timedelta(seconds=150))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
self.wr.state_set(self.wr.SUSPENDED)
actions = self.wr.evaluate()
self.assertEqual(self.wr.SUSPENDED, self.wr.state)
self.assertEqual([], actions)
def _wait_for_resource_status(self, stack_identifier, resource_name,
status, failure_pattern='^.*_FAILED$',
success_on_not_found=False):
"""Waits for a Resource to reach a given status."""
fail_regexp = re.compile(failure_pattern)
build_timeout = self.conf.build_timeout
build_interval = self.conf.build_interval
start = timeutils.utcnow()
while timeutils.delta_seconds(start,
timeutils.utcnow()) < build_timeout:
try:
res = self.client.resources.get(
stack_identifier, resource_name)
except heat_exceptions.HTTPNotFound:
if success_on_not_found:
return
# ignore this, as the resource may not have
# been created yet
else:
if res.resource_status == status:
return
if fail_regexp.search(res.resource_status):
raise exceptions.StackResourceBuildErrorException(
resource_name=res.resource_name,
stack_identifier=stack_identifier,
resource_status=res.resource_status,
resource_status_reason=res.resource_status_reason)
time.sleep(build_interval)
message = ('Resource %s failed to reach %s status within '
'the required time (%s s).' %
(res.resource_name, status, build_timeout))
raise exceptions.TimeoutException(message)
def test_evaluate_suspend(self):
rule = {
"EvaluationPeriods": "1",
"MetricName": "test_metric",
"Period": "300",
"Statistic": "Maximum",
"ComparisonOperator": "GreaterThanOrEqualToThreshold",
"Threshold": "30",
}
now = timeutils.utcnow()
self.m.StubOutWithMock(timeutils, "utcnow")
timeutils.utcnow().MultipleTimes().AndReturn(now)
self.m.ReplayAll()
# Now data breaches Threshold, but we're suspended
last = now - datetime.timedelta(seconds=300)
data = WatchData(35, now - datetime.timedelta(seconds=150))
self.wr = watchrule.WatchRule(
context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last,
)
self.wr.state_set(self.wr.SUSPENDED)
actions = self.wr.evaluate()
self.assertEqual(self.wr.state, self.wr.SUSPENDED)
self.assertEqual(actions, [])
def _wait_for_resource_status(self, stack_identifier, resource_name,
status, failure_pattern='^.*_FAILED$',
success_on_not_found=False):
"""Waits for a Resource to reach a given status."""
fail_regexp = re.compile(failure_pattern)
build_timeout = self.conf.build_timeout
build_interval = self.conf.build_interval
start = timeutils.utcnow()
while timeutils.delta_seconds(start,
timeutils.utcnow()) < build_timeout:
try:
res = self.client.resources.get(
stack_identifier, resource_name)
except heat_exceptions.HTTPNotFound:
if success_on_not_found:
return
# ignore this, as the resource may not have
# been created yet
else:
if res.resource_status == status:
return
if fail_regexp.search(res.resource_status):
raise exceptions.StackResourceBuildErrorException(
resource_name=res.resource_name,
stack_identifier=stack_identifier,
resource_status=res.resource_status,
resource_status_reason=res.resource_status_reason)
time.sleep(build_interval)
message = ('Resource %s failed to reach %s status within '
'the required time (%s s).' %
(res.resource_name, status, build_timeout))
raise exceptions.TimeoutException(message)
def test_evaluate(self):
rule = {
'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'
}
now = timeutils.utcnow()
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().MultipleTimes().AndReturn(now)
self.m.ReplayAll()
# It's not time to evaluate, so should stay NODATA
last = now - datetime.timedelta(seconds=299)
data = WatchData(25, now - datetime.timedelta(seconds=150))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = self.wr.evaluate()
self.assertEqual(self.wr.state, 'NODATA')
self.assertEqual(actions, [])
# now - last == Period, so should set NORMAL
last = now - datetime.timedelta(seconds=300)
data = WatchData(25, now - datetime.timedelta(seconds=150))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = self.wr.evaluate()
self.assertEqual(self.wr.state, 'NORMAL')
self.assertEqual(self.wr.last_evaluated, now)
self.assertEqual(actions, [])
# Now data breaches Threshold, so should set ALARM
last = now - datetime.timedelta(seconds=300)
data = WatchData(35, now - datetime.timedelta(seconds=150))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = self.wr.evaluate()
self.assertEqual(self.wr.state, 'ALARM')
self.assertEqual(self.wr.last_evaluated, now)
self.assertEqual(actions, [])
def test_scaling_policy_cooldown_none(self):
t = template_format.parse(as_template)
stack = utils.parse_stack(t, params=self.params)
# Create initial group
self._stub_lb_reload(1)
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
rsrc = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = rsrc
self.assertEqual(['WebServerGroup-0'], rsrc.get_instance_names())
# Create the scaling policy no Cooldown property, should behave the
# same as when Cooldown==0
properties = t['Resources']['WebServerScaleUpPolicy']['Properties']
del(properties['Cooldown'])
self._stub_lb_reload(2)
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.StubOutWithMock(asc.ScalingPolicy, 'keystone')
asc.ScalingPolicy.keystone().MultipleTimes().AndReturn(
self.fc)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
up_policy.signal()
self.assertEqual(['WebServerGroup-0', 'WebServerGroup-1'],
rsrc.get_instance_names())
# Now trigger another scale-up without changing time, should work
previous_meta = {timeutils.strtime(now): 'ChangeInCapacity : 1'}
self.m.VerifyAll()
self.m.UnsetStubs()
self.m.StubOutWithMock(Metadata, '__get__')
Metadata.__get__(mox.IgnoreArg(), up_policy, mox.IgnoreArg()
).AndReturn(previous_meta)
Metadata.__get__(mox.IgnoreArg(), rsrc, mox.IgnoreArg()
).AndReturn(previous_meta)
self._stub_lb_reload(3, unset=False)
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy.signal()
self.assertEqual(['WebServerGroup-0', 'WebServerGroup-1',
'WebServerGroup-2'],
rsrc.get_instance_names())
rsrc.delete()
self.m.VerifyAll()
def test_evaluate(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
now = timeutils.utcnow()
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().MultipleTimes().AndReturn(now)
self.m.ReplayAll()
# It's not time to evaluate, so should stay NODATA
last = now - datetime.timedelta(seconds=299)
data = WatchData(25, now - datetime.timedelta(seconds=150))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = self.wr.evaluate()
self.assertEqual('NODATA', self.wr.state)
self.assertEqual([], actions)
# now - last == Period, so should set NORMAL
last = now - datetime.timedelta(seconds=300)
data = WatchData(25, now - datetime.timedelta(seconds=150))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = self.wr.evaluate()
self.assertEqual('NORMAL', self.wr.state)
self.assertEqual(now, self.wr.last_evaluated)
self.assertEqual([], actions)
# Now data breaches Threshold, so should set ALARM
last = now - datetime.timedelta(seconds=300)
data = WatchData(35, now - datetime.timedelta(seconds=150))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = self.wr.evaluate()
self.assertEqual('ALARM', self.wr.state)
self.assertEqual(now, self.wr.last_evaluated)
self.assertEqual([], actions)
def test_scaling_policy_cooldown_toosoon(self):
t = template_format.parse(as_template)
stack = parse_stack(t, params=self.params)
# Create initial group
self._stub_lb_reload(1)
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
rsrc = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = rsrc
self.assertEqual('WebServerGroup-0', rsrc.resource_id)
# Scale up one
self._stub_lb_reload(2)
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.StubOutWithMock(asc.ScalingPolicy, 'keystone')
asc.ScalingPolicy.keystone().MultipleTimes().AndReturn(
self.fc)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
up_policy.signal()
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
rsrc.resource_id)
# Now move time on 10 seconds - Cooldown in template is 60
# so this should not update the policy metadata, and the
# scaling group instances should be unchanged
# Note we have to stub Metadata.__get__ since up_policy isn't
# stored in the DB (because the stack hasn't really been created)
previous_meta = {timeutils.strtime(now): 'ChangeInCapacity : 1'}
self.m.VerifyAll()
self.m.UnsetStubs()
now = now + datetime.timedelta(seconds=10)
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().MultipleTimes().AndReturn(now)
self.m.StubOutWithMock(Metadata, '__get__')
Metadata.__get__(mox.IgnoreArg(), up_policy, mox.IgnoreArg()
).AndReturn(previous_meta)
self.m.ReplayAll()
up_policy.signal()
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
rsrc.resource_id)
rsrc.delete()
self.m.VerifyAll()
def _action_set_stubs(self, now, action_expected=True):
# Setup stubs for the action tests
self.m.StubOutWithMock(timeutils, "utcnow")
timeutils.utcnow().MultipleTimes().AndReturn(now)
if action_expected:
dummy_action = DummyAction()
self.m.StubOutWithMock(parser.Stack, "__getitem__")
parser.Stack.__getitem__(mox.IgnoreArg()).MultipleTimes().AndReturn(dummy_action)
self.m.ReplayAll()
def _action_set_stubs(self, now):
# Setup stubs for the action tests
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().MultipleTimes().AndReturn(now)
dummy_action = DummyAction()
self.m.StubOutWithMock(parser.Stack, '__getitem__')
parser.Stack.__getitem__(mox.IgnoreArg()
).MultipleTimes().AndReturn(dummy_action)
self.m.ReplayAll()
def test_scaling_policy_cooldown_none(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0', resource.resource_id)
# Create the scaling policy no Cooldown property, should behave the
# same as when Cooldown==0
properties = t['Resources']['WebServerScaleUpPolicy']['Properties']
del (properties['Cooldown'])
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
up_policy.alarm()
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# Now trigger another scale-up without changing time, should work
previous_meta = {timeutils.strtime(now): 'ChangeInCapacity : 1'}
self.m.VerifyAll()
self.m.UnsetStubs()
self.m.StubOutWithMock(Metadata, '__get__')
Metadata.__get__(mox.IgnoreArg(), up_policy,
mox.IgnoreArg()).AndReturn(previous_meta)
Metadata.__get__(mox.IgnoreArg(), resource,
mox.IgnoreArg()).AndReturn(previous_meta)
self._stub_lb_reload(
['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2'],
unset=False)
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy.alarm()
self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2',
resource.resource_id)
resource.delete()
self.m.VerifyAll()
def _action_set_stubs(self, now, action_expected=True):
# Setup stubs for the action tests
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().MultipleTimes().AndReturn(now)
if action_expected:
dummy_action = DummyAction()
self.m.StubOutWithMock(parser.Stack, 'resource_by_refid')
parser.Stack.resource_by_refid(mox.IgnoreArg()).\
MultipleTimes().AndReturn(dummy_action)
self.m.ReplayAll()
def _action_set_stubs(self, now, action_expected=True):
# Setup stubs for the action tests
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().MultipleTimes().AndReturn(now)
if action_expected:
dummy_action = DummyAction()
self.m.StubOutWithMock(parser.Stack, 'resource_by_refid')
parser.Stack.resource_by_refid(mox.IgnoreArg()).\
MultipleTimes().AndReturn(dummy_action)
self.m.ReplayAll()
def __init__(self, context, watch_name, rule, stack_name, state=NORMAL,
wid=None, watch_data=[], last_evaluated=timeutils.utcnow()):
self.context = context
self.now = timeutils.utcnow()
self.name = watch_name
self.state = state
self.rule = rule
self.stack_name = stack_name
self.timeperiod = datetime.timedelta(seconds=int(rule['Period']))
self.id = wid
self.watch_data = watch_data
self.last_evaluated = last_evaluated
def test_scaling_policy_cooldown_none(self):
t = template_format.parse(as_template)
stack = utils.parse_stack(t, params=self.params)
# Create initial group
self._stub_lb_reload(1)
now = timeutils.utcnow()
self._stub_meta_expected(now, "ExactCapacity : 1")
self._stub_create(1)
self.m.ReplayAll()
rsrc = self.create_scaling_group(t, stack, "WebServerGroup")
stack.resources["WebServerGroup"] = rsrc
self.assertEqual(["WebServerGroup-0"], rsrc.get_instance_names())
# Create the scaling policy no Cooldown property, should behave the
# same as when Cooldown==0
properties = t["Resources"]["WebServerScaleUpPolicy"]["Properties"]
del (properties["Cooldown"])
self._stub_lb_reload(2)
now = timeutils.utcnow()
self._stub_meta_expected(now, "ChangeInCapacity : 1", 2)
self._stub_create(1)
self.m.StubOutWithMock(asc.ScalingPolicy, "keystone")
asc.ScalingPolicy.keystone().MultipleTimes().AndReturn(self.fc)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack, "WebServerScaleUpPolicy")
up_policy.signal()
self.assertEqual(["WebServerGroup-0", "WebServerGroup-1"], rsrc.get_instance_names())
# Now trigger another scale-up without changing time, should work
previous_meta = {timeutils.strtime(now): "ChangeInCapacity : 1"}
self.m.VerifyAll()
self.m.UnsetStubs()
self.m.StubOutWithMock(Metadata, "__get__")
Metadata.__get__(mox.IgnoreArg(), up_policy, mox.IgnoreArg()).AndReturn(previous_meta)
Metadata.__get__(mox.IgnoreArg(), rsrc, mox.IgnoreArg()).AndReturn(previous_meta)
# stub for the metadata accesses while creating the addtional instance
Metadata.__get__(mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg())
self._stub_lb_reload(3, unset=False)
self._stub_meta_expected(now, "ChangeInCapacity : 1", 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy.signal()
self.assertEqual(["WebServerGroup-0", "WebServerGroup-1", "WebServerGroup-2"], rsrc.get_instance_names())
rsrc.delete()
self.m.VerifyAll()
def test_scaling_group_cooldown_toosoon(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group, 2 instances, Cooldown 60s
properties = t['Resources']['WebServerGroup']['Properties']
properties['DesiredCapacity'] = '2'
properties['Cooldown'] = '60'
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# reduce by 50%
self._stub_lb_reload(['WebServerGroup-0'])
self._stub_meta_expected(now, 'PercentChangeInCapacity : -50')
self.m.ReplayAll()
resource.adjust(-50, 'PercentChangeInCapacity')
self.assertEqual('WebServerGroup-0', resource.resource_id)
# Now move time on 10 seconds - Cooldown in template is 60
# so this should not update the policy metadata, and the
# scaling group instances should be unchanged
# Note we have to stub Metadata.__get__ since up_policy isn't
# stored in the DB (because the stack hasn't really been created)
previous_meta = {
timeutils.strtime(now): 'PercentChangeInCapacity : -50'
}
self.m.VerifyAll()
self.m.UnsetStubs()
now = now + datetime.timedelta(seconds=10)
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().AndReturn(now)
self.m.StubOutWithMock(Metadata, '__get__')
Metadata.__get__(mox.IgnoreArg(), resource,
mox.IgnoreArg()).AndReturn(previous_meta)
self.m.ReplayAll()
# raise by 200%, too soon for Cooldown so there should be no change
resource.adjust(200, 'PercentChangeInCapacity')
self.assertEqual('WebServerGroup-0', resource.resource_id)
resource.delete()
def test_scaling_policy_cooldown_none(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0', resource.resource_id)
# Create the scaling policy no Cooldown property, should behave the
# same as when Cooldown==0
properties = t['Resources']['WebServerScaleUpPolicy']['Properties']
del(properties['Cooldown'])
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
up_policy.alarm()
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# Now trigger another scale-up without changing time, should work
previous_meta = {timeutils.strtime(now): 'ChangeInCapacity : 1'}
self.m.VerifyAll()
self.m.UnsetStubs()
self.m.StubOutWithMock(Metadata, '__get__')
Metadata.__get__(mox.IgnoreArg(), up_policy, mox.IgnoreArg()
).AndReturn(previous_meta)
Metadata.__get__(mox.IgnoreArg(), resource, mox.IgnoreArg()
).AndReturn(previous_meta)
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
'WebServerGroup-2'], unset=False)
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy.alarm()
self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2',
resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_scaling_group_cooldown_toosoon(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group, 2 instances, Cooldown 60s
properties = t['Resources']['WebServerGroup']['Properties']
properties['DesiredCapacity'] = '2'
properties['Cooldown'] = '60'
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# reduce by 50%
self._stub_lb_reload(['WebServerGroup-0'])
self._stub_meta_expected(now, 'PercentChangeInCapacity : -50')
self.m.ReplayAll()
resource.adjust(-50, 'PercentChangeInCapacity')
self.assertEqual('WebServerGroup-0',
resource.resource_id)
# Now move time on 10 seconds - Cooldown in template is 60
# so this should not update the policy metadata, and the
# scaling group instances should be unchanged
# Note we have to stub Metadata.__get__ since up_policy isn't
# stored in the DB (because the stack hasn't really been created)
previous_meta = {timeutils.strtime(now):
'PercentChangeInCapacity : -50'}
self.m.VerifyAll()
self.m.UnsetStubs()
now = now + datetime.timedelta(seconds=10)
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().AndReturn(now)
self.m.StubOutWithMock(Metadata, '__get__')
Metadata.__get__(mox.IgnoreArg(), resource, mox.IgnoreArg()
).AndReturn(previous_meta)
self.m.ReplayAll()
# raise by 200%, too soon for Cooldown so there should be no change
resource.adjust(200, 'PercentChangeInCapacity')
self.assertEqual('WebServerGroup-0', resource.resource_id)
resource.delete()
def _stub_meta_expected(self, now, data, nmeta=1):
# Stop time at now
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().MultipleTimes().AndReturn(now)
# Then set a stub to ensure the metadata update is as
# expected based on the timestamp and data
self.m.StubOutWithMock(Metadata, '__set__')
expected = {timeutils.strtime(now): data}
# Note for ScalingPolicy, we expect to get a metadata
# update for the policy and autoscaling group, so pass nmeta=2
for x in range(nmeta):
Metadata.__set__(mox.IgnoreArg(), expected).AndReturn(None)
def _stub_meta_expected(self, now, data, nmeta=1):
# Stop time at now
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().MultipleTimes().AndReturn(now)
# Then set a stub to ensure the metadata update is as
# expected based on the timestamp and data
self.m.StubOutWithMock(Metadata, '__set__')
expected = {timeutils.strtime(now): data}
# Note for ScalingPolicy, we expect to get a metadata
# update for the policy and autoscaling group, so pass nmeta=2
for x in range(nmeta):
Metadata.__set__(mox.IgnoreArg(), expected).AndReturn(None)
def __init__(self, context, watch_name, rule, stack_id=None,
state=NODATA, wid=None, watch_data=[],
last_evaluated=timeutils.utcnow()):
self.context = context
self.now = timeutils.utcnow()
self.name = watch_name
self.state = state
self.rule = rule
self.stack_id = stack_id
self.timeperiod = datetime.timedelta(seconds=int(rule['Period']))
self.id = wid
self.watch_data = watch_data
self.last_evaluated = last_evaluated
def test_scaling_policy_cooldown_toosoon(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0', resource.resource_id)
# Scale up one
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
up_policy.alarm()
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# Now move time on 10 seconds - Cooldown in template is 60
# so this should not update the policy metadata, and the
# scaling group instances should be unchanged
# Note we have to stub Metadata.__get__ since up_policy isn't
# stored in the DB (because the stack hasn't really been created)
previous_meta = {timeutils.strtime(now): 'ChangeInCapacity : 1'}
self.m.VerifyAll()
self.m.UnsetStubs()
now = now + datetime.timedelta(seconds=10)
self.m.StubOutWithMock(timeutils, 'utcnow')
timeutils.utcnow().AndReturn(now)
self.m.StubOutWithMock(Metadata, '__get__')
Metadata.__get__(mox.IgnoreArg(), up_policy,
mox.IgnoreArg()).AndReturn(previous_meta)
self.m.ReplayAll()
up_policy.alarm()
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_scaling_group_cooldown_toosoon(self):
t = template_format.parse(as_template)
stack = utils.parse_stack(t, params=self.params)
# Create initial group, 2 instances, Cooldown 60s
properties = t["Resources"]["WebServerGroup"]["Properties"]
properties["DesiredCapacity"] = "2"
properties["Cooldown"] = "60"
self._stub_lb_reload(2)
now = timeutils.utcnow()
self._stub_meta_expected(now, "ExactCapacity : 2")
self._stub_create(2)
self.m.ReplayAll()
rsrc = self.create_scaling_group(t, stack, "WebServerGroup")
stack.resources["WebServerGroup"] = rsrc
self.assertEqual(["WebServerGroup-0", "WebServerGroup-1"], rsrc.get_instance_names())
# reduce by 50%
self._stub_lb_reload(1)
self._stub_validate()
self._stub_meta_expected(now, "PercentChangeInCapacity : -50")
self.m.ReplayAll()
rsrc.adjust(-50, "PercentChangeInCapacity")
self.assertEqual(["WebServerGroup-0"], rsrc.get_instance_names())
# Now move time on 10 seconds - Cooldown in template is 60
# so this should not update the policy metadata, and the
# scaling group instances should be unchanged
# Note we have to stub Metadata.__get__ since up_policy isn't
# stored in the DB (because the stack hasn't really been created)
previous_meta = {timeutils.strtime(now): "PercentChangeInCapacity : -50"}
self.m.VerifyAll()
self.m.UnsetStubs()
now = now + datetime.timedelta(seconds=10)
self.m.StubOutWithMock(timeutils, "utcnow")
timeutils.utcnow().MultipleTimes().AndReturn(now)
self.m.StubOutWithMock(Metadata, "__get__")
Metadata.__get__(mox.IgnoreArg(), rsrc, mox.IgnoreArg()).AndReturn(previous_meta)
self.m.ReplayAll()
# raise by 200%, too soon for Cooldown so there should be no change
rsrc.adjust(200, "PercentChangeInCapacity")
self.assertEqual(["WebServerGroup-0"], rsrc.get_instance_names())
rsrc.delete()
def test_state_set(self):
rule = {
'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'AlarmActions': ['DummyAction'],
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'
}
last = timeutils.utcnow()
watcher = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch_set_state",
rule=rule,
watch_data=[],
stack_id=self.stack_id,
last_evaluated=last)
watcher.state_set(watcher.SUSPENDED)
self.assertEqual(watcher.state, watcher.SUSPENDED)
check = watchrule.WatchRule.load(context=self.ctx,
watch_name="testwatch_set_state")
self.assertEqual(check.state, watchrule.WatchRule.SUSPENDED)
def test_scaling_group_update_ok_desired_remove(self):
t = self.load_template()
properties = t['Resources']['WebServerGroup']['Properties']
properties['DesiredCapacity'] = '2'
stack = self.parse_stack(t)
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# Remove DesiredCapacity from the updated template, which should
# have no effect, it's an optional parameter
update_snippet = copy.deepcopy(resource.parsed_template())
del (update_snippet['Properties']['DesiredCapacity'])
self.assertEqual(asc.AutoScalingGroup.UPDATE_COMPLETE,
resource.handle_update(update_snippet))
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_scaling_group_percent(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group, 2 instances
properties = t['Resources']['WebServerGroup']['Properties']
properties['DesiredCapacity'] = '2'
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_create(2)
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# reduce by 50%
self._stub_lb_reload(['WebServerGroup-0'])
self._stub_meta_expected(now, 'PercentChangeInCapacity : -50')
self.m.ReplayAll()
resource.adjust(-50, 'PercentChangeInCapacity')
self.assertEqual('WebServerGroup-0', resource.resource_id)
# raise by 200%
self._stub_lb_reload(
['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2'])
self._stub_meta_expected(now, 'PercentChangeInCapacity : 200')
self._stub_create(2)
self.m.ReplayAll()
resource.adjust(200, 'PercentChangeInCapacity')
self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2',
resource.resource_id)
resource.delete()
def test_set_watch_state(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'AlarmActions': ['DummyAction'],
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
now = timeutils.utcnow()
self._action_set_stubs(now)
# Set data so rule evaluates to ALARM state
last = now - datetime.timedelta(seconds=200)
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[],
stack_id=self.stack_id,
last_evaluated=last)
actions = self.wr.set_watch_state(watchrule.WatchRule.NODATA)
self.assertEqual([], actions)
actions = self.wr.set_watch_state(watchrule.WatchRule.NORMAL)
self.assertEqual([], actions)
actions = self.wr.set_watch_state(watchrule.WatchRule.ALARM)
self.assertEqual(['DummyAction'], actions)
self.m.VerifyAll()
def test_destroy(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'AlarmActions': ['DummyAction'],
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
last = timeutils.utcnow()
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch_destroy",
rule=rule,
watch_data=[],
stack_id=self.stack_id,
last_evaluated=last)
self.wr.store()
check = watchrule.WatchRule.load(context=self.ctx,
watch_name="testwatch_destroy")
self.assertIsInstance(check, watchrule.WatchRule)
self.wr.destroy()
self.assertRaises(exception.WatchRuleNotFound,
watchrule.WatchRule.load, context=self.ctx,
watch_name="testwatch_destroy")
def test_rule_actions_alarm_two_actions(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'AlarmActions': ['DummyAction', 'AnotherDummyAction'],
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
now = timeutils.utcnow()
self._action_set_stubs(now)
# Set data so rule evaluates to ALARM state
last = now - datetime.timedelta(seconds=300)
data = WatchData(35, now - datetime.timedelta(seconds=150))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = self.wr.evaluate()
self.assertEqual('ALARM', self.wr.state)
self.assertEqual(['DummyAction', 'DummyAction'], actions)
self.m.VerifyAll()
def test_ave(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'Period': '300',
'Statistic': 'Average',
'ComparisonOperator': 'GreaterThanThreshold',
'Threshold': '100'}
now = timeutils.utcnow()
last = now - datetime.timedelta(seconds=320)
data = [WatchData(117, now - datetime.timedelta(seconds=100))]
data.append(WatchData(23, now - datetime.timedelta(seconds=150)))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
self.wr.now = now
new_state = self.wr.get_alarm_state()
self.assertEqual('NORMAL', new_state)
data.append(WatchData(195, now - datetime.timedelta(seconds=250)))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
self.wr.now = now
new_state = self.wr.get_alarm_state()
self.assertEqual('ALARM', new_state)
def test_scaling_policy_down(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group, 2 instances
properties = t['Resources']['WebServerGroup']['Properties']
properties['DesiredCapacity'] = '2'
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# Scale down one
self._stub_lb_reload(['WebServerGroup-0'])
self._stub_meta_expected(now, 'ChangeInCapacity : -1', 2)
self.m.ReplayAll()
down_policy = self.create_scaling_policy(t, stack,
'WebServerScaleDownPolicy')
down_policy.alarm()
self.assertEqual('WebServerGroup-0', resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_scaling_policy_down(self):
t = template_format.parse(as_template)
stack = parse_stack(t)
# Create initial group, 2 instances
properties = t["Resources"]["WebServerGroup"]["Properties"]
properties["DesiredCapacity"] = "2"
self._stub_lb_reload(["WebServerGroup-0", "WebServerGroup-1"])
now = timeutils.utcnow()
self._stub_meta_expected(now, "ExactCapacity : 2")
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, "WebServerGroup")
stack.resources["WebServerGroup"] = resource
self.assertEqual("WebServerGroup-0,WebServerGroup-1", resource.resource_id)
# Scale down one
self._stub_lb_reload(["WebServerGroup-0"])
self._stub_meta_expected(now, "ChangeInCapacity : -1", 2)
self.m.ReplayAll()
down_policy = self.create_scaling_policy(t, stack, "WebServerScaleDownPolicy")
down_policy.alarm()
self.assertEqual("WebServerGroup-0", resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_rule_actions_alarm_normal(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'AlarmActions': ['DummyAction'],
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
now = timeutils.utcnow()
self._action_set_stubs(now, action_expected=False)
# Set data so rule evaluates to NORMAL state
last = now - datetime.timedelta(seconds=300)
data = WatchData(25, now - datetime.timedelta(seconds=150))
watcher = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = watcher.evaluate()
self.assertEqual(watcher.state, 'NORMAL')
self.assertEqual(actions, [])
self.m.VerifyAll()
def test_scaling_group_update_ok_desired(self):
t = template_format.parse(as_template)
properties = t['Resources']['WebServerGroup']['Properties']
properties['MinSize'] = '1'
properties['MaxSize'] = '3'
stack = parse_stack(t)
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
rsrc = self.create_scaling_group(t, stack, 'WebServerGroup')
self.assertEqual('WebServerGroup-0', rsrc.resource_id)
# Increase min size to 2 via DesiredCapacity, should adjust
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(1)
self.m.ReplayAll()
update_snippet = copy.deepcopy(rsrc.parsed_template())
update_snippet['Properties']['DesiredCapacity'] = '2'
self.assertEqual(None, rsrc.update(update_snippet))
self.assertEqual('WebServerGroup-0,WebServerGroup-1', rsrc.resource_id)
self.assertEqual('2', rsrc.properties['DesiredCapacity'])
rsrc.delete()
self.m.VerifyAll()
def test_scaling_group_update_ok_desired_remove(self):
t = self.load_template()
properties = t['Resources']['WebServerGroup']['Properties']
properties['DesiredCapacity'] = '2'
stack = self.parse_stack(t)
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# Remove DesiredCapacity from the updated template, which should
# have no effect, it's an optional parameter
update_snippet = copy.deepcopy(resource.parsed_template())
del(update_snippet['Properties']['DesiredCapacity'])
self.assertEqual(asc.AutoScalingGroup.UPDATE_COMPLETE,
resource.handle_update(update_snippet))
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_rule_actions_alarm_alarm(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'AlarmActions': ['DummyAction'],
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
now = timeutils.utcnow()
self._action_set_stubs(now)
# Set data so rule evaluates to ALARM state
last = now - datetime.timedelta(seconds=300)
data = WatchData(35, now - datetime.timedelta(seconds=150))
watcher = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = watcher.evaluate()
self.assertEqual(watcher.state, 'ALARM')
self.assertEqual(actions, ['DummyAction'])
# re-set last_evaluated so the rule will be evaluated again,
# but since we're already in ALARM state, we should not generate
# any additional actions
last = now - datetime.timedelta(seconds=300)
watcher.last_evaluated = last
actions = watcher.evaluate()
self.assertEqual(watcher.state, 'ALARM')
self.assertEqual(actions, [])
self.m.VerifyAll()
def test_scaling_group_nochange(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group, 2 instances
properties = t['Resources']['WebServerGroup']['Properties']
properties['DesiredCapacity'] = '2'
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# raise above the max
resource.adjust(2)
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# lower below the min
resource.adjust(-2)
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# no change
resource.adjust(0)
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_rule_actions_alarm_normal(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'AlarmActions': ['DummyAction'],
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
now = timeutils.utcnow()
self._action_set_stubs(now, action_expected=False)
# Set data so rule evaluates to NORMAL state
last = now - datetime.timedelta(seconds=300)
data = WatchData(25, now - datetime.timedelta(seconds=150))
watcher = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = watcher.evaluate()
self.assertEqual(watcher.state, 'NORMAL')
self.assertEqual(actions, [])
self.m.VerifyAll()
def test_scaling_group_update_ok_desired(self):
t = self.load_template()
properties = t['Resources']['WebServerGroup']['Properties']
properties['MinSize'] = '1'
properties['MaxSize'] = '3'
stack = self.parse_stack(t)
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
self.assertEqual('WebServerGroup-0', resource.resource_id)
# Increase min size to 2 via DesiredCapacity, should adjust
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(1)
self.m.ReplayAll()
update_snippet = copy.deepcopy(resource.parsed_template())
update_snippet['Properties']['DesiredCapacity'] = '2'
self.assertEqual(asc.AutoScalingGroup.UPDATE_COMPLETE,
resource.handle_update(update_snippet))
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_sum(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'Period': '300',
'Statistic': 'Sum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '100'}
now = timeutils.utcnow()
last = now - datetime.timedelta(seconds=320)
data = [WatchData(17, now - datetime.timedelta(seconds=100))]
data.append(WatchData(23, now - datetime.timedelta(seconds=150)))
# all < 40 -> NORMAL
watcher = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
watcher.now = now
new_state = watcher.get_alarm_state()
self.assertEqual(new_state, 'NORMAL')
# sum > 100 -> ALARM
data.append(WatchData(85, now - datetime.timedelta(seconds=150)))
watcher = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
watcher.now = now
new_state = watcher.get_alarm_state()
self.assertEqual(new_state, 'ALARM')
def test_scaling_policy_up(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0', resource.resource_id)
# Scale up one
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
up_policy.alarm()
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_minimum(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'Period': '300',
'Statistic': 'Minimum',
'ComparisonOperator': 'LessThanOrEqualToThreshold',
'Threshold': '50'}
now = timeutils.utcnow()
last = now - datetime.timedelta(seconds=320)
data = [WatchData(77, now - datetime.timedelta(seconds=100))]
data.append(WatchData(53, now - datetime.timedelta(seconds=150)))
# all > 50 -> NORMAL
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
new_state = self.wr.get_alarm_state()
self.assertEqual('NORMAL', new_state)
data.append(WatchData(25, now - datetime.timedelta(seconds=250)))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
new_state = self.wr.get_alarm_state()
self.assertEqual('ALARM', new_state)
def test_scaling_policy_down(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group, 2 instances
properties = t['Resources']['WebServerGroup']['Properties']
properties['DesiredCapacity'] = '2'
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# Scale down one
self._stub_lb_reload(['WebServerGroup-0'])
self._stub_meta_expected(now, 'ChangeInCapacity : -1', 2)
self.m.ReplayAll()
down_policy = self.create_scaling_policy(t, stack,
'WebServerScaleDownPolicy')
down_policy.alarm()
self.assertEqual('WebServerGroup-0', resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_sum(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'Period': '300',
'Statistic': 'Sum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '100'}
now = timeutils.utcnow()
last = now - datetime.timedelta(seconds=320)
data = [WatchData(17, now - datetime.timedelta(seconds=100))]
data.append(WatchData(23, now - datetime.timedelta(seconds=150)))
# all < 40 -> NORMAL
watcher = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
watcher.now = now
new_state = watcher.get_alarm_state()
self.assertEqual(new_state, 'NORMAL')
# sum > 100 -> ALARM
data.append(WatchData(85, now - datetime.timedelta(seconds=150)))
watcher = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
watcher.now = now
new_state = watcher.get_alarm_state()
self.assertEqual(new_state, 'ALARM')
def test_scaling_group_nochange(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group, 2 instances
properties = t['Resources']['WebServerGroup']['Properties']
properties['DesiredCapacity'] = '2'
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# raise above the max
resource.adjust(2)
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# lower below the min
resource.adjust(-2)
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
# no change
resource.adjust(0)
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_rule_actions_alarm_alarm(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'AlarmActions': ['DummyAction'],
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
now = timeutils.utcnow()
self._action_set_stubs(now)
# Set data so rule evaluates to ALARM state
last = now - datetime.timedelta(seconds=300)
data = WatchData(35, now - datetime.timedelta(seconds=150))
watcher = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = watcher.evaluate()
self.assertEqual(watcher.state, 'ALARM')
self.assertEqual(actions, ['DummyAction'])
# re-set last_evaluated so the rule will be evaluated again,
# but since we're already in ALARM state, we should not generate
# any additional actions
last = now - datetime.timedelta(seconds=300)
watcher.last_evaluated = last
actions = watcher.evaluate()
self.assertEqual(watcher.state, 'ALARM')
self.assertEqual(actions, [])
self.m.VerifyAll()
def test_scaling_policy_up(self):
t = self.load_template()
stack = self.parse_stack(t)
# Create initial group
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
self.assertEqual('WebServerGroup-0', resource.resource_id)
# Scale up one
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
up_policy.alarm()
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_set_watch_state(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'AlarmActions': ['DummyAction'],
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
now = timeutils.utcnow()
self._action_set_stubs(now)
# Set data so rule evaluates to ALARM state
last = now - datetime.timedelta(seconds=200)
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[],
stack_id=self.stack_id,
last_evaluated=last)
actions = self.wr.set_watch_state(watchrule.WatchRule.NODATA)
self.assertEqual([], actions)
actions = self.wr.set_watch_state(watchrule.WatchRule.NORMAL)
self.assertEqual([], actions)
actions = self.wr.set_watch_state(watchrule.WatchRule.ALARM)
self.assertEqual(['DummyAction'], actions)
self.m.VerifyAll()
def test_minimum(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'Period': '300',
'Statistic': 'Minimum',
'ComparisonOperator': 'LessThanOrEqualToThreshold',
'Threshold': '50'}
now = timeutils.utcnow()
last = now - datetime.timedelta(seconds=320)
data = [WatchData(77, now - datetime.timedelta(seconds=100))]
data.append(WatchData(53, now - datetime.timedelta(seconds=150)))
# all > 50 -> NORMAL
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
new_state = self.wr.get_alarm_state()
self.assertEqual('NORMAL', new_state)
data.append(WatchData(25, now - datetime.timedelta(seconds=250)))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
new_state = self.wr.get_alarm_state()
self.assertEqual('ALARM', new_state)
def test_scaling_group_update_ok_desired(self):
t = self.load_template()
properties = t['Resources']['WebServerGroup']['Properties']
properties['MinSize'] = '1'
properties['MaxSize'] = '3'
stack = self.parse_stack(t)
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
self.assertEqual('WebServerGroup-0', resource.resource_id)
# Increase min size to 2 via DesiredCapacity, should adjust
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(1)
self.m.ReplayAll()
update_snippet = copy.deepcopy(resource.parsed_template())
update_snippet['Properties']['DesiredCapacity'] = '2'
self.assertEqual(asc.AutoScalingGroup.UPDATE_COMPLETE,
resource.handle_update(update_snippet))
self.assertEqual('WebServerGroup-0,WebServerGroup-1',
resource.resource_id)
resource.delete()
self.m.VerifyAll()
def test_rule_actions_alarm_two_actions(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'AlarmActions': ['DummyAction', 'AnotherDummyAction'],
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
now = timeutils.utcnow()
self._action_set_stubs(now)
# Set data so rule evaluates to ALARM state
last = now - datetime.timedelta(seconds=300)
data = WatchData(35, now - datetime.timedelta(seconds=150))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=[data],
stack_id=self.stack_id,
last_evaluated=last)
actions = self.wr.evaluate()
self.assertEqual('ALARM', self.wr.state)
self.assertEqual(['DummyAction', 'DummyAction'], actions)
self.m.VerifyAll()
def test_destroy(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'AlarmActions': ['DummyAction'],
'Period': '300',
'Statistic': 'Maximum',
'ComparisonOperator': 'GreaterThanOrEqualToThreshold',
'Threshold': '30'}
last = timeutils.utcnow()
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch_destroy",
rule=rule,
watch_data=[],
stack_id=self.stack_id,
last_evaluated=last)
self.wr.store()
check = watchrule.WatchRule.load(context=self.ctx,
watch_name="testwatch_destroy")
self.assertIsInstance(check, watchrule.WatchRule)
self.wr.destroy()
self.assertRaises(exception.WatchRuleNotFound,
watchrule.WatchRule.load, context=self.ctx,
watch_name="testwatch_destroy")
def test_ave(self):
rule = {'EvaluationPeriods': '1',
'MetricName': 'test_metric',
'Period': '300',
'Statistic': 'Average',
'ComparisonOperator': 'GreaterThanThreshold',
'Threshold': '100'}
now = timeutils.utcnow()
last = now - datetime.timedelta(seconds=320)
data = [WatchData(117, now - datetime.timedelta(seconds=100))]
data.append(WatchData(23, now - datetime.timedelta(seconds=150)))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
self.wr.now = now
new_state = self.wr.get_alarm_state()
self.assertEqual('NORMAL', new_state)
data.append(WatchData(195, now - datetime.timedelta(seconds=250)))
self.wr = watchrule.WatchRule(context=self.ctx,
watch_name="testwatch",
rule=rule,
watch_data=data,
stack_id=self.stack_id,
last_evaluated=last)
self.wr.now = now
new_state = self.wr.get_alarm_state()
self.assertEqual('ALARM', new_state)
def evaluate(self):
if self.state == self.SUSPENDED:
return []
# has enough time progressed to run the rule
self.now = timeutils.utcnow()
if self.now < (self.last_evaluated + self.timeperiod):
return []
return self.run_rule()
def soft_delete(self, synchronize_session='evaluate'):
return self.update(
{
'deleted': literal_column('id'),
'updated_at': literal_column('updated_at'),
'deleted_at': timeutils.utcnow()
},
synchronize_session=synchronize_session)
def create_watch_rule(ctx, stack, **kwargs):
values = {
'name': 'test_rule',
'rule': loads('{"foo": "123"}'),
'state': 'normal',
'last_evaluated': timeutils.utcnow(),
'stack_id': stack.id,
}
values.update(kwargs)
return db_api.watch_rule_create(ctx, values)
def delete(self, session=None):
"""Delete this object."""
self.deleted = True
self.deleted_at = timeutils.utcnow()
if not session:
session = Session.object_session(self)
if not session:
session = get_session()
session.delete(self)
session.flush()
