def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn(_('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: LOG.exception(_('in fixed duration looping call')) done.send_exception(*sys.exc_info()) return else: done.send(True)
def _inner(): if initial_delay: greenthread.sleep(initial_delay) try: while self._running: start = timeutils.utcnow() self.f(*self.args, **self.kw) end = timeutils.utcnow() if not self._running: break delay = interval - timeutils.delta_seconds(start, end) if delay <= 0: LOG.warn( _('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: LOG.exception(_('in fixed duration looping call')) done.send_exception(*sys.exc_info()) return else: done.send(True)
def test_evaluate_suspend(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'Period': '300', 'Statistic': 'Maximum', 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', 'Threshold': '30'} now = timeutils.utcnow() self.m.StubOutWithMock(timeutils, 'utcnow') timeutils.utcnow().MultipleTimes().AndReturn(now) self.m.ReplayAll() # Now data breaches Threshold, but we're suspended last = now - datetime.timedelta(seconds=300) data = WatchData(35, now - datetime.timedelta(seconds=150)) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[data], stack_id=self.stack_id, last_evaluated=last) self.wr.state_set(self.wr.SUSPENDED) actions = self.wr.evaluate() self.assertEqual(self.wr.SUSPENDED, self.wr.state) self.assertEqual([], actions)
def _wait_for_resource_status(self, stack_identifier, resource_name, status, failure_pattern='^.*_FAILED$', success_on_not_found=False): """Waits for a Resource to reach a given status.""" fail_regexp = re.compile(failure_pattern) build_timeout = self.conf.build_timeout build_interval = self.conf.build_interval start = timeutils.utcnow() while timeutils.delta_seconds(start, timeutils.utcnow()) < build_timeout: try: res = self.client.resources.get( stack_identifier, resource_name) except heat_exceptions.HTTPNotFound: if success_on_not_found: return # ignore this, as the resource may not have # been created yet else: if res.resource_status == status: return if fail_regexp.search(res.resource_status): raise exceptions.StackResourceBuildErrorException( resource_name=res.resource_name, stack_identifier=stack_identifier, resource_status=res.resource_status, resource_status_reason=res.resource_status_reason) time.sleep(build_interval) message = ('Resource %s failed to reach %s status within ' 'the required time (%s s).' % (res.resource_name, status, build_timeout)) raise exceptions.TimeoutException(message)
def test_evaluate_suspend(self): rule = { "EvaluationPeriods": "1", "MetricName": "test_metric", "Period": "300", "Statistic": "Maximum", "ComparisonOperator": "GreaterThanOrEqualToThreshold", "Threshold": "30", } now = timeutils.utcnow() self.m.StubOutWithMock(timeutils, "utcnow") timeutils.utcnow().MultipleTimes().AndReturn(now) self.m.ReplayAll() # Now data breaches Threshold, but we're suspended last = now - datetime.timedelta(seconds=300) data = WatchData(35, now - datetime.timedelta(seconds=150)) self.wr = watchrule.WatchRule( context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[data], stack_id=self.stack_id, last_evaluated=last, ) self.wr.state_set(self.wr.SUSPENDED) actions = self.wr.evaluate() self.assertEqual(self.wr.state, self.wr.SUSPENDED) self.assertEqual(actions, [])
def test_evaluate(self): rule = { 'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'Period': '300', 'Statistic': 'Maximum', 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', 'Threshold': '30' } now = timeutils.utcnow() self.m.StubOutWithMock(timeutils, 'utcnow') timeutils.utcnow().MultipleTimes().AndReturn(now) self.m.ReplayAll() # It's not time to evaluate, so should stay NODATA last = now - datetime.timedelta(seconds=299) data = WatchData(25, now - datetime.timedelta(seconds=150)) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[data], stack_id=self.stack_id, last_evaluated=last) actions = self.wr.evaluate() self.assertEqual(self.wr.state, 'NODATA') self.assertEqual(actions, []) # now - last == Period, so should set NORMAL last = now - datetime.timedelta(seconds=300) data = WatchData(25, now - datetime.timedelta(seconds=150)) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[data], stack_id=self.stack_id, last_evaluated=last) actions = self.wr.evaluate() self.assertEqual(self.wr.state, 'NORMAL') self.assertEqual(self.wr.last_evaluated, now) self.assertEqual(actions, []) # Now data breaches Threshold, so should set ALARM last = now - datetime.timedelta(seconds=300) data = WatchData(35, now - datetime.timedelta(seconds=150)) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[data], stack_id=self.stack_id, last_evaluated=last) actions = self.wr.evaluate() self.assertEqual(self.wr.state, 'ALARM') self.assertEqual(self.wr.last_evaluated, now) self.assertEqual(actions, [])
def test_scaling_policy_cooldown_none(self): t = template_format.parse(as_template) stack = utils.parse_stack(t, params=self.params) # Create initial group self._stub_lb_reload(1) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') self._stub_create(1) self.m.ReplayAll() rsrc = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = rsrc self.assertEqual(['WebServerGroup-0'], rsrc.get_instance_names()) # Create the scaling policy no Cooldown property, should behave the # same as when Cooldown==0 properties = t['Resources']['WebServerScaleUpPolicy']['Properties'] del(properties['Cooldown']) self._stub_lb_reload(2) now = timeutils.utcnow() self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) self._stub_create(1) self.m.StubOutWithMock(asc.ScalingPolicy, 'keystone') asc.ScalingPolicy.keystone().MultipleTimes().AndReturn( self.fc) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy') up_policy.signal() self.assertEqual(['WebServerGroup-0', 'WebServerGroup-1'], rsrc.get_instance_names()) # Now trigger another scale-up without changing time, should work previous_meta = {timeutils.strtime(now): 'ChangeInCapacity : 1'} self.m.VerifyAll() self.m.UnsetStubs() self.m.StubOutWithMock(Metadata, '__get__') Metadata.__get__(mox.IgnoreArg(), up_policy, mox.IgnoreArg() ).AndReturn(previous_meta) Metadata.__get__(mox.IgnoreArg(), rsrc, mox.IgnoreArg() ).AndReturn(previous_meta) self._stub_lb_reload(3, unset=False) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) self._stub_create(1) self.m.ReplayAll() up_policy.signal() self.assertEqual(['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2'], rsrc.get_instance_names()) rsrc.delete() self.m.VerifyAll()
def test_evaluate(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'Period': '300', 'Statistic': 'Maximum', 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', 'Threshold': '30'} now = timeutils.utcnow() self.m.StubOutWithMock(timeutils, 'utcnow') timeutils.utcnow().MultipleTimes().AndReturn(now) self.m.ReplayAll() # It's not time to evaluate, so should stay NODATA last = now - datetime.timedelta(seconds=299) data = WatchData(25, now - datetime.timedelta(seconds=150)) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[data], stack_id=self.stack_id, last_evaluated=last) actions = self.wr.evaluate() self.assertEqual('NODATA', self.wr.state) self.assertEqual([], actions) # now - last == Period, so should set NORMAL last = now - datetime.timedelta(seconds=300) data = WatchData(25, now - datetime.timedelta(seconds=150)) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[data], stack_id=self.stack_id, last_evaluated=last) actions = self.wr.evaluate() self.assertEqual('NORMAL', self.wr.state) self.assertEqual(now, self.wr.last_evaluated) self.assertEqual([], actions) # Now data breaches Threshold, so should set ALARM last = now - datetime.timedelta(seconds=300) data = WatchData(35, now - datetime.timedelta(seconds=150)) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[data], stack_id=self.stack_id, last_evaluated=last) actions = self.wr.evaluate() self.assertEqual('ALARM', self.wr.state) self.assertEqual(now, self.wr.last_evaluated) self.assertEqual([], actions)
def test_scaling_policy_cooldown_toosoon(self): t = template_format.parse(as_template) stack = parse_stack(t, params=self.params) # Create initial group self._stub_lb_reload(1) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') self._stub_create(1) self.m.ReplayAll() rsrc = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = rsrc self.assertEqual('WebServerGroup-0', rsrc.resource_id) # Scale up one self._stub_lb_reload(2) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) self._stub_create(1) self.m.StubOutWithMock(asc.ScalingPolicy, 'keystone') asc.ScalingPolicy.keystone().MultipleTimes().AndReturn( self.fc) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy') up_policy.signal() self.assertEqual('WebServerGroup-0,WebServerGroup-1', rsrc.resource_id) # Now move time on 10 seconds - Cooldown in template is 60 # so this should not update the policy metadata, and the # scaling group instances should be unchanged # Note we have to stub Metadata.__get__ since up_policy isn't # stored in the DB (because the stack hasn't really been created) previous_meta = {timeutils.strtime(now): 'ChangeInCapacity : 1'} self.m.VerifyAll() self.m.UnsetStubs() now = now + datetime.timedelta(seconds=10) self.m.StubOutWithMock(timeutils, 'utcnow') timeutils.utcnow().MultipleTimes().AndReturn(now) self.m.StubOutWithMock(Metadata, '__get__') Metadata.__get__(mox.IgnoreArg(), up_policy, mox.IgnoreArg() ).AndReturn(previous_meta) self.m.ReplayAll() up_policy.signal() self.assertEqual('WebServerGroup-0,WebServerGroup-1', rsrc.resource_id) rsrc.delete() self.m.VerifyAll()
def _action_set_stubs(self, now, action_expected=True): # Setup stubs for the action tests self.m.StubOutWithMock(timeutils, "utcnow") timeutils.utcnow().MultipleTimes().AndReturn(now) if action_expected: dummy_action = DummyAction() self.m.StubOutWithMock(parser.Stack, "__getitem__") parser.Stack.__getitem__(mox.IgnoreArg()).MultipleTimes().AndReturn(dummy_action) self.m.ReplayAll()
def _action_set_stubs(self, now): # Setup stubs for the action tests self.m.StubOutWithMock(timeutils, 'utcnow') timeutils.utcnow().MultipleTimes().AndReturn(now) dummy_action = DummyAction() self.m.StubOutWithMock(parser.Stack, '__getitem__') parser.Stack.__getitem__(mox.IgnoreArg() ).MultipleTimes().AndReturn(dummy_action) self.m.ReplayAll()
def test_scaling_policy_cooldown_none(self): t = self.load_template() stack = self.parse_stack(t) # Create initial group self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') self._stub_create(1) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource self.assertEqual('WebServerGroup-0', resource.resource_id) # Create the scaling policy no Cooldown property, should behave the # same as when Cooldown==0 properties = t['Resources']['WebServerScaleUpPolicy']['Properties'] del (properties['Cooldown']) self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) self._stub_create(1) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy') up_policy.alarm() self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # Now trigger another scale-up without changing time, should work previous_meta = {timeutils.strtime(now): 'ChangeInCapacity : 1'} self.m.VerifyAll() self.m.UnsetStubs() self.m.StubOutWithMock(Metadata, '__get__') Metadata.__get__(mox.IgnoreArg(), up_policy, mox.IgnoreArg()).AndReturn(previous_meta) Metadata.__get__(mox.IgnoreArg(), resource, mox.IgnoreArg()).AndReturn(previous_meta) self._stub_lb_reload( ['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2'], unset=False) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) self._stub_create(1) self.m.ReplayAll() up_policy.alarm() self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2', resource.resource_id) resource.delete() self.m.VerifyAll()
def _action_set_stubs(self, now, action_expected=True): # Setup stubs for the action tests self.m.StubOutWithMock(timeutils, 'utcnow') timeutils.utcnow().MultipleTimes().AndReturn(now) if action_expected: dummy_action = DummyAction() self.m.StubOutWithMock(parser.Stack, 'resource_by_refid') parser.Stack.resource_by_refid(mox.IgnoreArg()).\ MultipleTimes().AndReturn(dummy_action) self.m.ReplayAll()
def __init__(self, context, watch_name, rule, stack_name, state=NORMAL, wid=None, watch_data=[], last_evaluated=timeutils.utcnow()): self.context = context self.now = timeutils.utcnow() self.name = watch_name self.state = state self.rule = rule self.stack_name = stack_name self.timeperiod = datetime.timedelta(seconds=int(rule['Period'])) self.id = wid self.watch_data = watch_data self.last_evaluated = last_evaluated
def test_scaling_policy_cooldown_none(self): t = template_format.parse(as_template) stack = utils.parse_stack(t, params=self.params) # Create initial group self._stub_lb_reload(1) now = timeutils.utcnow() self._stub_meta_expected(now, "ExactCapacity : 1") self._stub_create(1) self.m.ReplayAll() rsrc = self.create_scaling_group(t, stack, "WebServerGroup") stack.resources["WebServerGroup"] = rsrc self.assertEqual(["WebServerGroup-0"], rsrc.get_instance_names()) # Create the scaling policy no Cooldown property, should behave the # same as when Cooldown==0 properties = t["Resources"]["WebServerScaleUpPolicy"]["Properties"] del (properties["Cooldown"]) self._stub_lb_reload(2) now = timeutils.utcnow() self._stub_meta_expected(now, "ChangeInCapacity : 1", 2) self._stub_create(1) self.m.StubOutWithMock(asc.ScalingPolicy, "keystone") asc.ScalingPolicy.keystone().MultipleTimes().AndReturn(self.fc) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, "WebServerScaleUpPolicy") up_policy.signal() self.assertEqual(["WebServerGroup-0", "WebServerGroup-1"], rsrc.get_instance_names()) # Now trigger another scale-up without changing time, should work previous_meta = {timeutils.strtime(now): "ChangeInCapacity : 1"} self.m.VerifyAll() self.m.UnsetStubs() self.m.StubOutWithMock(Metadata, "__get__") Metadata.__get__(mox.IgnoreArg(), up_policy, mox.IgnoreArg()).AndReturn(previous_meta) Metadata.__get__(mox.IgnoreArg(), rsrc, mox.IgnoreArg()).AndReturn(previous_meta) # stub for the metadata accesses while creating the addtional instance Metadata.__get__(mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()) self._stub_lb_reload(3, unset=False) self._stub_meta_expected(now, "ChangeInCapacity : 1", 2) self._stub_create(1) self.m.ReplayAll() up_policy.signal() self.assertEqual(["WebServerGroup-0", "WebServerGroup-1", "WebServerGroup-2"], rsrc.get_instance_names()) rsrc.delete() self.m.VerifyAll()
def test_scaling_group_cooldown_toosoon(self): t = self.load_template() stack = self.parse_stack(t) # Create initial group, 2 instances, Cooldown 60s properties = t['Resources']['WebServerGroup']['Properties'] properties['DesiredCapacity'] = '2' properties['Cooldown'] = '60' self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') self._stub_create(2) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # reduce by 50% self._stub_lb_reload(['WebServerGroup-0']) self._stub_meta_expected(now, 'PercentChangeInCapacity : -50') self.m.ReplayAll() resource.adjust(-50, 'PercentChangeInCapacity') self.assertEqual('WebServerGroup-0', resource.resource_id) # Now move time on 10 seconds - Cooldown in template is 60 # so this should not update the policy metadata, and the # scaling group instances should be unchanged # Note we have to stub Metadata.__get__ since up_policy isn't # stored in the DB (because the stack hasn't really been created) previous_meta = { timeutils.strtime(now): 'PercentChangeInCapacity : -50' } self.m.VerifyAll() self.m.UnsetStubs() now = now + datetime.timedelta(seconds=10) self.m.StubOutWithMock(timeutils, 'utcnow') timeutils.utcnow().AndReturn(now) self.m.StubOutWithMock(Metadata, '__get__') Metadata.__get__(mox.IgnoreArg(), resource, mox.IgnoreArg()).AndReturn(previous_meta) self.m.ReplayAll() # raise by 200%, too soon for Cooldown so there should be no change resource.adjust(200, 'PercentChangeInCapacity') self.assertEqual('WebServerGroup-0', resource.resource_id) resource.delete()
def test_scaling_policy_cooldown_none(self): t = self.load_template() stack = self.parse_stack(t) # Create initial group self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') self._stub_create(1) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource self.assertEqual('WebServerGroup-0', resource.resource_id) # Create the scaling policy no Cooldown property, should behave the # same as when Cooldown==0 properties = t['Resources']['WebServerScaleUpPolicy']['Properties'] del(properties['Cooldown']) self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) self._stub_create(1) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy') up_policy.alarm() self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # Now trigger another scale-up without changing time, should work previous_meta = {timeutils.strtime(now): 'ChangeInCapacity : 1'} self.m.VerifyAll() self.m.UnsetStubs() self.m.StubOutWithMock(Metadata, '__get__') Metadata.__get__(mox.IgnoreArg(), up_policy, mox.IgnoreArg() ).AndReturn(previous_meta) Metadata.__get__(mox.IgnoreArg(), resource, mox.IgnoreArg() ).AndReturn(previous_meta) self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2'], unset=False) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) self._stub_create(1) self.m.ReplayAll() up_policy.alarm() self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2', resource.resource_id) resource.delete() self.m.VerifyAll()
def test_scaling_group_cooldown_toosoon(self): t = self.load_template() stack = self.parse_stack(t) # Create initial group, 2 instances, Cooldown 60s properties = t['Resources']['WebServerGroup']['Properties'] properties['DesiredCapacity'] = '2' properties['Cooldown'] = '60' self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') self._stub_create(2) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # reduce by 50% self._stub_lb_reload(['WebServerGroup-0']) self._stub_meta_expected(now, 'PercentChangeInCapacity : -50') self.m.ReplayAll() resource.adjust(-50, 'PercentChangeInCapacity') self.assertEqual('WebServerGroup-0', resource.resource_id) # Now move time on 10 seconds - Cooldown in template is 60 # so this should not update the policy metadata, and the # scaling group instances should be unchanged # Note we have to stub Metadata.__get__ since up_policy isn't # stored in the DB (because the stack hasn't really been created) previous_meta = {timeutils.strtime(now): 'PercentChangeInCapacity : -50'} self.m.VerifyAll() self.m.UnsetStubs() now = now + datetime.timedelta(seconds=10) self.m.StubOutWithMock(timeutils, 'utcnow') timeutils.utcnow().AndReturn(now) self.m.StubOutWithMock(Metadata, '__get__') Metadata.__get__(mox.IgnoreArg(), resource, mox.IgnoreArg() ).AndReturn(previous_meta) self.m.ReplayAll() # raise by 200%, too soon for Cooldown so there should be no change resource.adjust(200, 'PercentChangeInCapacity') self.assertEqual('WebServerGroup-0', resource.resource_id) resource.delete()
def _stub_meta_expected(self, now, data, nmeta=1): # Stop time at now self.m.StubOutWithMock(timeutils, 'utcnow') timeutils.utcnow().MultipleTimes().AndReturn(now) # Then set a stub to ensure the metadata update is as # expected based on the timestamp and data self.m.StubOutWithMock(Metadata, '__set__') expected = {timeutils.strtime(now): data} # Note for ScalingPolicy, we expect to get a metadata # update for the policy and autoscaling group, so pass nmeta=2 for x in range(nmeta): Metadata.__set__(mox.IgnoreArg(), expected).AndReturn(None)
def __init__(self, context, watch_name, rule, stack_id=None, state=NODATA, wid=None, watch_data=[], last_evaluated=timeutils.utcnow()): self.context = context self.now = timeutils.utcnow() self.name = watch_name self.state = state self.rule = rule self.stack_id = stack_id self.timeperiod = datetime.timedelta(seconds=int(rule['Period'])) self.id = wid self.watch_data = watch_data self.last_evaluated = last_evaluated
def test_scaling_policy_cooldown_toosoon(self): t = self.load_template() stack = self.parse_stack(t) # Create initial group self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') self._stub_create(1) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource self.assertEqual('WebServerGroup-0', resource.resource_id) # Scale up one self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) self._stub_create(1) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy') up_policy.alarm() self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # Now move time on 10 seconds - Cooldown in template is 60 # so this should not update the policy metadata, and the # scaling group instances should be unchanged # Note we have to stub Metadata.__get__ since up_policy isn't # stored in the DB (because the stack hasn't really been created) previous_meta = {timeutils.strtime(now): 'ChangeInCapacity : 1'} self.m.VerifyAll() self.m.UnsetStubs() now = now + datetime.timedelta(seconds=10) self.m.StubOutWithMock(timeutils, 'utcnow') timeutils.utcnow().AndReturn(now) self.m.StubOutWithMock(Metadata, '__get__') Metadata.__get__(mox.IgnoreArg(), up_policy, mox.IgnoreArg()).AndReturn(previous_meta) self.m.ReplayAll() up_policy.alarm() self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) resource.delete() self.m.VerifyAll()
def test_scaling_group_cooldown_toosoon(self): t = template_format.parse(as_template) stack = utils.parse_stack(t, params=self.params) # Create initial group, 2 instances, Cooldown 60s properties = t["Resources"]["WebServerGroup"]["Properties"] properties["DesiredCapacity"] = "2" properties["Cooldown"] = "60" self._stub_lb_reload(2) now = timeutils.utcnow() self._stub_meta_expected(now, "ExactCapacity : 2") self._stub_create(2) self.m.ReplayAll() rsrc = self.create_scaling_group(t, stack, "WebServerGroup") stack.resources["WebServerGroup"] = rsrc self.assertEqual(["WebServerGroup-0", "WebServerGroup-1"], rsrc.get_instance_names()) # reduce by 50% self._stub_lb_reload(1) self._stub_validate() self._stub_meta_expected(now, "PercentChangeInCapacity : -50") self.m.ReplayAll() rsrc.adjust(-50, "PercentChangeInCapacity") self.assertEqual(["WebServerGroup-0"], rsrc.get_instance_names()) # Now move time on 10 seconds - Cooldown in template is 60 # so this should not update the policy metadata, and the # scaling group instances should be unchanged # Note we have to stub Metadata.__get__ since up_policy isn't # stored in the DB (because the stack hasn't really been created) previous_meta = {timeutils.strtime(now): "PercentChangeInCapacity : -50"} self.m.VerifyAll() self.m.UnsetStubs() now = now + datetime.timedelta(seconds=10) self.m.StubOutWithMock(timeutils, "utcnow") timeutils.utcnow().MultipleTimes().AndReturn(now) self.m.StubOutWithMock(Metadata, "__get__") Metadata.__get__(mox.IgnoreArg(), rsrc, mox.IgnoreArg()).AndReturn(previous_meta) self.m.ReplayAll() # raise by 200%, too soon for Cooldown so there should be no change rsrc.adjust(200, "PercentChangeInCapacity") self.assertEqual(["WebServerGroup-0"], rsrc.get_instance_names()) rsrc.delete()
def test_state_set(self): rule = { 'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'AlarmActions': ['DummyAction'], 'Period': '300', 'Statistic': 'Maximum', 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', 'Threshold': '30' } last = timeutils.utcnow() watcher = watchrule.WatchRule(context=self.ctx, watch_name="testwatch_set_state", rule=rule, watch_data=[], stack_id=self.stack_id, last_evaluated=last) watcher.state_set(watcher.SUSPENDED) self.assertEqual(watcher.state, watcher.SUSPENDED) check = watchrule.WatchRule.load(context=self.ctx, watch_name="testwatch_set_state") self.assertEqual(check.state, watchrule.WatchRule.SUSPENDED)
def test_scaling_group_update_ok_desired_remove(self): t = self.load_template() properties = t['Resources']['WebServerGroup']['Properties'] properties['DesiredCapacity'] = '2' stack = self.parse_stack(t) self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') self._stub_create(2) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # Remove DesiredCapacity from the updated template, which should # have no effect, it's an optional parameter update_snippet = copy.deepcopy(resource.parsed_template()) del (update_snippet['Properties']['DesiredCapacity']) self.assertEqual(asc.AutoScalingGroup.UPDATE_COMPLETE, resource.handle_update(update_snippet)) self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) resource.delete() self.m.VerifyAll()
def test_scaling_group_percent(self): t = self.load_template() stack = self.parse_stack(t) # Create initial group, 2 instances properties = t['Resources']['WebServerGroup']['Properties'] properties['DesiredCapacity'] = '2' self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) self._stub_create(2) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # reduce by 50% self._stub_lb_reload(['WebServerGroup-0']) self._stub_meta_expected(now, 'PercentChangeInCapacity : -50') self.m.ReplayAll() resource.adjust(-50, 'PercentChangeInCapacity') self.assertEqual('WebServerGroup-0', resource.resource_id) # raise by 200% self._stub_lb_reload( ['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2']) self._stub_meta_expected(now, 'PercentChangeInCapacity : 200') self._stub_create(2) self.m.ReplayAll() resource.adjust(200, 'PercentChangeInCapacity') self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2', resource.resource_id) resource.delete()
def test_set_watch_state(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'AlarmActions': ['DummyAction'], 'Period': '300', 'Statistic': 'Maximum', 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', 'Threshold': '30'} now = timeutils.utcnow() self._action_set_stubs(now) # Set data so rule evaluates to ALARM state last = now - datetime.timedelta(seconds=200) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[], stack_id=self.stack_id, last_evaluated=last) actions = self.wr.set_watch_state(watchrule.WatchRule.NODATA) self.assertEqual([], actions) actions = self.wr.set_watch_state(watchrule.WatchRule.NORMAL) self.assertEqual([], actions) actions = self.wr.set_watch_state(watchrule.WatchRule.ALARM) self.assertEqual(['DummyAction'], actions) self.m.VerifyAll()
def test_destroy(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'AlarmActions': ['DummyAction'], 'Period': '300', 'Statistic': 'Maximum', 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', 'Threshold': '30'} last = timeutils.utcnow() self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch_destroy", rule=rule, watch_data=[], stack_id=self.stack_id, last_evaluated=last) self.wr.store() check = watchrule.WatchRule.load(context=self.ctx, watch_name="testwatch_destroy") self.assertIsInstance(check, watchrule.WatchRule) self.wr.destroy() self.assertRaises(exception.WatchRuleNotFound, watchrule.WatchRule.load, context=self.ctx, watch_name="testwatch_destroy")
def test_rule_actions_alarm_two_actions(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'AlarmActions': ['DummyAction', 'AnotherDummyAction'], 'Period': '300', 'Statistic': 'Maximum', 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', 'Threshold': '30'} now = timeutils.utcnow() self._action_set_stubs(now) # Set data so rule evaluates to ALARM state last = now - datetime.timedelta(seconds=300) data = WatchData(35, now - datetime.timedelta(seconds=150)) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[data], stack_id=self.stack_id, last_evaluated=last) actions = self.wr.evaluate() self.assertEqual('ALARM', self.wr.state) self.assertEqual(['DummyAction', 'DummyAction'], actions) self.m.VerifyAll()
def test_ave(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'Period': '300', 'Statistic': 'Average', 'ComparisonOperator': 'GreaterThanThreshold', 'Threshold': '100'} now = timeutils.utcnow() last = now - datetime.timedelta(seconds=320) data = [WatchData(117, now - datetime.timedelta(seconds=100))] data.append(WatchData(23, now - datetime.timedelta(seconds=150))) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=data, stack_id=self.stack_id, last_evaluated=last) self.wr.now = now new_state = self.wr.get_alarm_state() self.assertEqual('NORMAL', new_state) data.append(WatchData(195, now - datetime.timedelta(seconds=250))) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=data, stack_id=self.stack_id, last_evaluated=last) self.wr.now = now new_state = self.wr.get_alarm_state() self.assertEqual('ALARM', new_state)
def test_scaling_policy_down(self): t = self.load_template() stack = self.parse_stack(t) # Create initial group, 2 instances properties = t['Resources']['WebServerGroup']['Properties'] properties['DesiredCapacity'] = '2' self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') self._stub_create(2) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # Scale down one self._stub_lb_reload(['WebServerGroup-0']) self._stub_meta_expected(now, 'ChangeInCapacity : -1', 2) self.m.ReplayAll() down_policy = self.create_scaling_policy(t, stack, 'WebServerScaleDownPolicy') down_policy.alarm() self.assertEqual('WebServerGroup-0', resource.resource_id) resource.delete() self.m.VerifyAll()
def test_scaling_policy_down(self): t = template_format.parse(as_template) stack = parse_stack(t) # Create initial group, 2 instances properties = t["Resources"]["WebServerGroup"]["Properties"] properties["DesiredCapacity"] = "2" self._stub_lb_reload(["WebServerGroup-0", "WebServerGroup-1"]) now = timeutils.utcnow() self._stub_meta_expected(now, "ExactCapacity : 2") self._stub_create(2) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, "WebServerGroup") stack.resources["WebServerGroup"] = resource self.assertEqual("WebServerGroup-0,WebServerGroup-1", resource.resource_id) # Scale down one self._stub_lb_reload(["WebServerGroup-0"]) self._stub_meta_expected(now, "ChangeInCapacity : -1", 2) self.m.ReplayAll() down_policy = self.create_scaling_policy(t, stack, "WebServerScaleDownPolicy") down_policy.alarm() self.assertEqual("WebServerGroup-0", resource.resource_id) resource.delete() self.m.VerifyAll()
def test_rule_actions_alarm_normal(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'AlarmActions': ['DummyAction'], 'Period': '300', 'Statistic': 'Maximum', 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', 'Threshold': '30'} now = timeutils.utcnow() self._action_set_stubs(now, action_expected=False) # Set data so rule evaluates to NORMAL state last = now - datetime.timedelta(seconds=300) data = WatchData(25, now - datetime.timedelta(seconds=150)) watcher = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[data], stack_id=self.stack_id, last_evaluated=last) actions = watcher.evaluate() self.assertEqual(watcher.state, 'NORMAL') self.assertEqual(actions, []) self.m.VerifyAll()
def test_scaling_group_update_ok_desired(self): t = template_format.parse(as_template) properties = t['Resources']['WebServerGroup']['Properties'] properties['MinSize'] = '1' properties['MaxSize'] = '3' stack = parse_stack(t) self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') self._stub_create(1) self.m.ReplayAll() rsrc = self.create_scaling_group(t, stack, 'WebServerGroup') self.assertEqual('WebServerGroup-0', rsrc.resource_id) # Increase min size to 2 via DesiredCapacity, should adjust self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) self._stub_meta_expected(now, 'ExactCapacity : 2') self._stub_create(1) self.m.ReplayAll() update_snippet = copy.deepcopy(rsrc.parsed_template()) update_snippet['Properties']['DesiredCapacity'] = '2' self.assertEqual(None, rsrc.update(update_snippet)) self.assertEqual('WebServerGroup-0,WebServerGroup-1', rsrc.resource_id) self.assertEqual('2', rsrc.properties['DesiredCapacity']) rsrc.delete() self.m.VerifyAll()
def test_scaling_group_update_ok_desired_remove(self): t = self.load_template() properties = t['Resources']['WebServerGroup']['Properties'] properties['DesiredCapacity'] = '2' stack = self.parse_stack(t) self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') self._stub_create(2) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # Remove DesiredCapacity from the updated template, which should # have no effect, it's an optional parameter update_snippet = copy.deepcopy(resource.parsed_template()) del(update_snippet['Properties']['DesiredCapacity']) self.assertEqual(asc.AutoScalingGroup.UPDATE_COMPLETE, resource.handle_update(update_snippet)) self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) resource.delete() self.m.VerifyAll()
def test_rule_actions_alarm_alarm(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'AlarmActions': ['DummyAction'], 'Period': '300', 'Statistic': 'Maximum', 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', 'Threshold': '30'} now = timeutils.utcnow() self._action_set_stubs(now) # Set data so rule evaluates to ALARM state last = now - datetime.timedelta(seconds=300) data = WatchData(35, now - datetime.timedelta(seconds=150)) watcher = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=[data], stack_id=self.stack_id, last_evaluated=last) actions = watcher.evaluate() self.assertEqual(watcher.state, 'ALARM') self.assertEqual(actions, ['DummyAction']) # re-set last_evaluated so the rule will be evaluated again, # but since we're already in ALARM state, we should not generate # any additional actions last = now - datetime.timedelta(seconds=300) watcher.last_evaluated = last actions = watcher.evaluate() self.assertEqual(watcher.state, 'ALARM') self.assertEqual(actions, []) self.m.VerifyAll()
def test_scaling_group_nochange(self): t = self.load_template() stack = self.parse_stack(t) # Create initial group, 2 instances properties = t['Resources']['WebServerGroup']['Properties'] properties['DesiredCapacity'] = '2' self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') self._stub_create(2) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # raise above the max resource.adjust(2) self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # lower below the min resource.adjust(-2) self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) # no change resource.adjust(0) self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) resource.delete() self.m.VerifyAll()
def test_scaling_group_update_ok_desired(self): t = self.load_template() properties = t['Resources']['WebServerGroup']['Properties'] properties['MinSize'] = '1' properties['MaxSize'] = '3' stack = self.parse_stack(t) self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') self._stub_create(1) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') self.assertEqual('WebServerGroup-0', resource.resource_id) # Increase min size to 2 via DesiredCapacity, should adjust self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) self._stub_meta_expected(now, 'ExactCapacity : 2') self._stub_create(1) self.m.ReplayAll() update_snippet = copy.deepcopy(resource.parsed_template()) update_snippet['Properties']['DesiredCapacity'] = '2' self.assertEqual(asc.AutoScalingGroup.UPDATE_COMPLETE, resource.handle_update(update_snippet)) self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) resource.delete() self.m.VerifyAll()
def test_sum(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'Period': '300', 'Statistic': 'Sum', 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', 'Threshold': '100'} now = timeutils.utcnow() last = now - datetime.timedelta(seconds=320) data = [WatchData(17, now - datetime.timedelta(seconds=100))] data.append(WatchData(23, now - datetime.timedelta(seconds=150))) # all < 40 -> NORMAL watcher = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=data, stack_id=self.stack_id, last_evaluated=last) watcher.now = now new_state = watcher.get_alarm_state() self.assertEqual(new_state, 'NORMAL') # sum > 100 -> ALARM data.append(WatchData(85, now - datetime.timedelta(seconds=150))) watcher = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=data, stack_id=self.stack_id, last_evaluated=last) watcher.now = now new_state = watcher.get_alarm_state() self.assertEqual(new_state, 'ALARM')
def test_scaling_policy_up(self): t = self.load_template() stack = self.parse_stack(t) # Create initial group self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') self._stub_create(1) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource self.assertEqual('WebServerGroup-0', resource.resource_id) # Scale up one self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) self._stub_create(1) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy') up_policy.alarm() self.assertEqual('WebServerGroup-0,WebServerGroup-1', resource.resource_id) resource.delete() self.m.VerifyAll()
def test_minimum(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric', 'Period': '300', 'Statistic': 'Minimum', 'ComparisonOperator': 'LessThanOrEqualToThreshold', 'Threshold': '50'} now = timeutils.utcnow() last = now - datetime.timedelta(seconds=320) data = [WatchData(77, now - datetime.timedelta(seconds=100))] data.append(WatchData(53, now - datetime.timedelta(seconds=150))) # all > 50 -> NORMAL self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=data, stack_id=self.stack_id, last_evaluated=last) new_state = self.wr.get_alarm_state() self.assertEqual('NORMAL', new_state) data.append(WatchData(25, now - datetime.timedelta(seconds=250))) self.wr = watchrule.WatchRule(context=self.ctx, watch_name="testwatch", rule=rule, watch_data=data, stack_id=self.stack_id, last_evaluated=last) new_state = self.wr.get_alarm_state() self.assertEqual('ALARM', new_state)
def evaluate(self): if self.state == self.SUSPENDED: return [] # has enough time progressed to run the rule self.now = timeutils.utcnow() if self.now < (self.last_evaluated + self.timeperiod): return [] return self.run_rule()
def soft_delete(self, synchronize_session='evaluate'): return self.update( { 'deleted': literal_column('id'), 'updated_at': literal_column('updated_at'), 'deleted_at': timeutils.utcnow() }, synchronize_session=synchronize_session)
def create_watch_rule(ctx, stack, **kwargs): values = { 'name': 'test_rule', 'rule': loads('{"foo": "123"}'), 'state': 'normal', 'last_evaluated': timeutils.utcnow(), 'stack_id': stack.id, } values.update(kwargs) return db_api.watch_rule_create(ctx, values)
def delete(self, session=None): """Delete this object.""" self.deleted = True self.deleted_at = timeutils.utcnow() if not session: session = Session.object_session(self) if not session: session = get_session() session.delete(self) session.flush()