def test_scale_paas_app_handles_deployments(self, mock_get_statsd_client,
mock_paas_client, _, caplog):
caplog.set_level(logging.INFO)
app_name = 'app-name-1'
app_guid = '11111-11111-11111111-1111'
cf_info = {'name': app_name, 'instances': 4, 'guid': app_guid}
app = self._get_mock_app(app_name, cf_info)
app.get_desired_instance_count = Mock(return_value=6)
mock_paas_client.return_value.update.side_effect = InvalidStatusCode(
status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
body={
"description":
"Cannot scale this process while a deployment is in flight.",
"error_code": "CF-ScaleDisabledDuringDeployment",
"code": 390016
})
autoscaler = Autoscaler()
autoscaler.scale(app)
mock_paas_client.return_value.update.assert_called_once_with(
app_guid, 6)
assert caplog.record_tuples == [
('root', logging.INFO, 'Scaling app-name-1 from 4 to 6'),
('root', logging.INFO, 'Cannot scale during deployment app-name-1')
]
def setup(self):
self.config = {
"poll_interval_seconds":
60,
"metric_stores": [{
"name": "monitoring",
"type": "prometheus",
"prometheus": {
"url": "http://localhost:9090"
}
}],
"autoscale_rules": [{
"service_name": "web",
"scale_min": 1,
"scale_max": 3,
"metric_store": "monitoring",
"metric_query": "scalar(avg(http_requests_total))",
"scale_up_threshold": 300,
"scale_down_threshold": 250,
"scale_step": 1
}]
}
self.docker_client = Mock()
self.metric_store_factory = Mock()
self.monitoring_metric_store = Mock()
self.scheduler = Mock()
self.mock_datetime = Mock()
self.metric_store_factory.get_metric_store.return_value = self.monitoring_metric_store
self.autoscaler = Autoscaler(self.config, self.docker_client,
self.metric_store_factory, self.scheduler,
self.mock_datetime)
def test_scale_paas_app_much_fewer_instances(self, mock_get_statsd_client,
mock_paas_client, *args):
""" We don't scale down more than 1 instance at a time """
app_guid = '11111-11111-11111111-1111'
app_name = 'app-name-1'
cf_info = {'name': app_name, 'instances': 4, 'guid': app_guid}
app = self._get_mock_app(app_name, cf_info)
app.get_desired_instance_count = Mock(return_value=1)
autoscaler = Autoscaler()
autoscaler.cooldown_seconds_after_scale_up = SCALEUP_COOLDOWN_SECONDS
autoscaler.cooldown_seconds_after_scale_down = SCALEDOWN_COOLDOWN_SECONDS
# we scaled down 600 seconds ago, scaled up 325 seconds ago
autoscaler._set_last_scale(
'last_scale_down', app_name,
(self._now() - SCALEDOWN_COOLDOWN_SECONDS * 10))
autoscaler._set_last_scale('last_scale_up', app_name,
(self._now() -
(SCALEUP_COOLDOWN_SECONDS + 25)))
autoscaler.scale(app)
assert float(autoscaler.redis_client.hget('last_scale_down',
app_name)) == self._now()
mock_get_statsd_client.return_value.gauge.assert_called_once_with(
"{}.instance-count".format(app_name), 3)
mock_paas_client.return_value.update.assert_called_once_with(
app_guid, 3)
def test_scale_paas_app_handles_unexpected_errors(self,
mock_get_statsd_client,
mock_paas_client, _,
caplog):
caplog.set_level(logging.INFO)
app_name = 'app-name-1'
app_guid = '11111-11111-11111111-1111'
cf_info = {'name': app_name, 'instances': 4, 'guid': app_guid}
app = self._get_mock_app(app_name, cf_info)
app.get_desired_instance_count = Mock(return_value=6)
mock_paas_client.return_value.update.side_effect = InvalidStatusCode(
status_code=HTTPStatus.BAD_REQUEST,
body={'description': 'something bad'})
autoscaler = Autoscaler()
autoscaler.scale(app)
mock_paas_client.return_value.update.assert_called_once_with(
app_guid, 6)
assert caplog.record_tuples == [
('root', logging.INFO, 'Scaling app-name-1 from 4 to 6'),
('root', logging.ERROR,
'Failed to scale app-name-1: BAD_REQUEST : {"description": "something bad"}'
)
]
def test_scale_paas_app_same_instance_count(self, mock_get_statsd_client, mock_paas_client, *args):
app_name = 'app-name-1'
app_guid = '11111-11111-11111111-1111'
cf_info = {'name': app_name, 'instances': 4, 'guid': app_guid}
app = self._get_mock_app(app_name, cf_info)
app.get_desired_instance_count = Mock(return_value=4)
autoscaler = Autoscaler()
autoscaler.scale(app)
mock_get_statsd_client.return_value.gauge.assert_called_once_with("{}.instance-count".format(app_name), 4)
mock_paas_client.return_value.assert_not_called()
def test_scale_paas_app_more_instances(self, mock_get_statsd_client, mock_paas_client, *args):
app_guid = '11111-11111-11111111-1111'
app_name = 'app-name-1'
cf_info = {'name': app_name, 'instances': 4, 'guid': app_guid}
app = self._get_mock_app(app_name, cf_info)
app.get_desired_instance_count = Mock(return_value=6)
autoscaler = Autoscaler()
autoscaler.scale(app)
assert float(autoscaler.redis_client.hget('last_scale_up', app_name)) == self._now()
mock_get_statsd_client.return_value.gauge.assert_called_once_with("{}.instance-count".format(app_name), 6)
mock_paas_client.return_value.update.assert_called_once_with(app_guid, 6)
def test_scale_paas_app_fewer_instances_recent_scale_down(self, mock_get_statsd_client, mock_paas_client, *args):
app_guid = '11111-11111-11111111-1111'
app_name = 'app-name-1'
cf_info = {'name': app_name, 'instances': 4, 'guid': app_guid}
app = self._get_mock_app(app_name, cf_info)
app.get_desired_instance_count = Mock(return_value=3)
autoscaler = Autoscaler()
autoscaler.cooldown_seconds_after_scale_up = SCALEUP_COOLDOWN_SECONDS
autoscaler.cooldown_seconds_after_scale_down = SCALEDOWN_COOLDOWN_SECONDS
# we scaled up 600 seconds ago, scaled down 30 seconds ago
autoscaler._set_last_scale('last_scale_down', app_name, (self._now() - 30))
autoscaler._set_last_scale('last_scale_up', app_name, (self._now() - 600))
autoscaler.scale(app)
mock_get_statsd_client.return_value.gauge.assert_called_once_with("{}.instance-count".format(app_name), 4)
mock_paas_client.return_value.assert_not_called()
def setup(self):
self.config = {
"poll_interval_seconds":
10,
"autoscale_rules": [{
"service_name": "bouncer",
"network_name": "network_1",
"scale_min": 1,
"scale_max": 3,
"scale_up_threshold": 300,
"scale_down_threshold": 250,
"scale_step": 1,
"stablization_window": 300
}]
}
self.scaling_client = Mock()
self.metric_store_factory = Mock()
self.monitoring_metric_store = Mock()
self.scheduler = Mock()
self.mock_datetime = Mock()
self.metric_store_factory.get_metric_store.return_value = self.monitoring_metric_store
self.autoscaler = Autoscaler(self.config, self.scaling_client,
self.metric_store_factory, self.scheduler,
self.mock_datetime)
class TestAutoscaler(object):
def setup(self):
self.config = {
"poll_interval_seconds":
10,
"autoscale_rules": [{
"service_name": "bouncer",
"network_name": "network_1",
"scale_min": 1,
"scale_max": 3,
"scale_up_threshold": 300,
"scale_down_threshold": 250,
"scale_step": 1,
"stablization_window": 300
}]
}
self.scaling_client = Mock()
self.metric_store_factory = Mock()
self.monitoring_metric_store = Mock()
self.scheduler = Mock()
self.mock_datetime = Mock()
self.metric_store_factory.get_metric_store.return_value = self.monitoring_metric_store
self.autoscaler = Autoscaler(self.config, self.scaling_client,
self.metric_store_factory, self.scheduler,
self.mock_datetime)
def test_run_gets_replica_count_of_the_service_from_scaling_client(self):
self.scaling_client.get_bouncer_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 100
self.autoscaler.run()
self.scaling_client.get_bouncer_replica_count.assert_called_once_with(
network_name="network_1", service_name="bouncer")
def test_run_gets_metric_value_from_metric_strore_for_configured_metric_query(
self):
self.scaling_client.get_bouncer_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 100
self.autoscaler.run()
self.monitoring_metric_store.get_metric_value.assert_called_once_with(
300)
def test_run_scales_up_service_replicas_when_metric_value_is_above_scale_up_threshold(
self):
self.scaling_client.get_bouncer_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 301
self.autoscaler.run()
self.scaling_client.scale_service.assert_called_once_with(
network_name="network_1", service_name="bouncer", replica_count=2)
def test_run_scales_up_service_replicas_when_metric_value_is_above_scale_up_threshold_and_current_replica_count_is_close_to_scale_max(
self):
self.scaling_client.get_bouncer_replica_count.return_value = 2
self.monitoring_metric_store.get_metric_value.return_value = 301
self.autoscaler.run()
self.scaling_client.scale_service.assert_called_once_with(
network_name="network_1", service_name="bouncer", replica_count=3)
def test_run_scales_down_service_replicas_when_metric_value_is_below_scale_down_threshold(
self):
self.scaling_client.get_bouncer_replica_count.return_value = 3
self.monitoring_metric_store.get_metric_value.return_value = 249
self.autoscaler.run()
self.scaling_client.scale_service.assert_called_once_with(
network_name="network_1", service_name="bouncer", replica_count=2)
def test_run_scales_down_service_replicas_when_metric_value_is_below_scale_down_threshold_and_current_replica_count_close_to_scale_down_threshold(
self):
self.scaling_client.get_bouncer_replica_count.return_value = 2
self.monitoring_metric_store.get_metric_value.return_value = 249
self.autoscaler.run()
self.scaling_client.scale_service.assert_called_once_with(
network_name="network_1", service_name="bouncer", replica_count=1)
def test_run_does_not_scale_up_service_replicas_when_current_replica_count_equals_scale_max(
self):
self.scaling_client.get_bouncer_replica_count.return_value = 3
self.monitoring_metric_store.get_metric_value.return_value = 301
self.autoscaler.run()
self.scaling_client.scale_service.assert_not_called()
def test_run_does_not_scale_up_service_replicas_when_metric_value_equals_scale_up_threshold(
self):
self.scaling_client.get_bouncer_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 300
self.autoscaler.run()
self.scaling_client.scale_service.assert_not_called()
def test_run_does_not_scale_up_service_replicas_when_metric_value_is_below_scale_up_threshold(
self):
self.scaling_client.get_bouncer_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 299
self.autoscaler.run()
self.scaling_client.scale_service.assert_not_called()
def test_run_does_not_scale_down_service_replicas_when_current_replica_count_equals_scale_min(
self):
self.scaling_client.get_bouncer_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 249
self.autoscaler.run()
self.scaling_client.scale_service.assert_not_called()
def test_run_does_not_scale_down_service_replicas_when_metric_value_equals_scale_down_threshold(
self):
self.scaling_client.get_bouncer_replica_count.return_value = 2
self.monitoring_metric_store.get_metric_value.return_value = 250
self.autoscaler.run()
self.scaling_client.scale_service.assert_not_called()
def test_run_does_not_scale_down_service_replicas_when_metric_value_is_above_scale_down_threshold(
self):
self.scaling_client.get_bouncer_replica_count.return_value = 2
self.monitoring_metric_store.get_metric_value.return_value = 251
self.autoscaler.run()
self.scaling_client.scale_service.assert_not_called()
def test_start_starts_the_scheduler(self):
self.scheduler.timezone = utc
self.autoscaler.start()
self.scheduler.start.assert_called_once()
def test_start_schedules_the_job_to_run_autoscaler_for_each_poll_interval_seconds(
self):
self.scheduler.timezone = utc
self.autoscaler.start()
self.scheduler.add_job.assert_called_once_with(self.autoscaler.run,
'interval',
seconds=60)
def test_start_runs_the_job_to_run_autoscaler_immediately(self):
job = Mock()
current_time = datetime.now()
self.scheduler.timezone = utc
self.mock_datetime.now.return_value = current_time
self.scheduler.add_job.return_value = job
self.autoscaler.start()
self.mock_datetime.now.assert_called_once_with(utc)
job.modify.assert_called_once_with(next_run_time=current_time)
class TestAutoscaler(object):
def setup(self):
self.config = {
"poll_interval_seconds":
60,
"metric_stores": [{
"name": "monitoring",
"type": "prometheus",
"prometheus": {
"url": "http://localhost:9090"
}
}],
"autoscale_rules": [{
"service_name": "web",
"scale_min": 1,
"scale_max": 3,
"metric_store": "monitoring",
"metric_query": "scalar(avg(http_requests_total))",
"scale_up_threshold": 300,
"scale_down_threshold": 250,
"scale_step": 1
}]
}
self.docker_client = Mock()
self.metric_store_factory = Mock()
self.monitoring_metric_store = Mock()
self.scheduler = Mock()
self.mock_datetime = Mock()
self.metric_store_factory.get_metric_store.return_value = self.monitoring_metric_store
self.autoscaler = Autoscaler(self.config, self.docker_client,
self.metric_store_factory, self.scheduler,
self.mock_datetime)
def test_run_gets_replica_count_of_the_service_from_docker_client(self):
self.docker_client.get_service_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 100
self.autoscaler.run()
self.docker_client.get_service_replica_count.assert_called_once_with(
service_name="web")
def test_run_gets_metric_value_from_metric_strore_for_configured_metric_query(
self):
self.docker_client.get_service_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 100
self.autoscaler.run()
self.monitoring_metric_store.get_metric_value.assert_called_once_with(
"scalar(avg(http_requests_total))")
def test_run_scales_up_service_replicas_when_metric_value_is_above_scale_up_threshold(
self):
self.docker_client.get_service_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 301
self.autoscaler.run()
self.docker_client.scale_service.assert_called_once_with(
service_name="web", replica_count=2)
def test_run_scales_up_service_replicas_when_metric_value_is_above_scale_up_threshold_and_current_replica_count_is_close_to_scale_max(
self):
self.docker_client.get_service_replica_count.return_value = 2
self.monitoring_metric_store.get_metric_value.return_value = 301
self.autoscaler.run()
self.docker_client.scale_service.assert_called_once_with(
service_name="web", replica_count=3)
def test_run_scales_down_service_replicas_when_metric_value_is_below_scale_down_threshold(
self):
self.docker_client.get_service_replica_count.return_value = 3
self.monitoring_metric_store.get_metric_value.return_value = 249
self.autoscaler.run()
self.docker_client.scale_service.assert_called_once_with(
service_name="web", replica_count=2)
def test_run_scales_down_service_replicas_when_metric_value_is_below_scale_down_threshold_and_current_replica_count_close_to_scale_down_threshold(
self):
self.docker_client.get_service_replica_count.return_value = 2
self.monitoring_metric_store.get_metric_value.return_value = 249
self.autoscaler.run()
self.docker_client.scale_service.assert_called_once_with(
service_name="web", replica_count=1)
def test_run_does_not_scale_up_service_replicas_when_current_replica_count_equals_scale_max(
self):
self.docker_client.get_service_replica_count.return_value = 3
self.monitoring_metric_store.get_metric_value.return_value = 301
self.autoscaler.run()
self.docker_client.scale_service.assert_not_called()
def test_run_does_not_scale_up_service_replicas_when_metric_value_equals_scale_up_threshold(
self):
self.docker_client.get_service_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 300
self.autoscaler.run()
self.docker_client.scale_service.assert_not_called()
def test_run_does_not_scale_up_service_replicas_when_metric_value_is_below_scale_up_threshold(
self):
self.docker_client.get_service_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 299
self.autoscaler.run()
self.docker_client.scale_service.assert_not_called()
def test_run_does_not_scale_down_service_replicas_when_current_replica_count_equals_scale_min(
self):
self.docker_client.get_service_replica_count.return_value = 1
self.monitoring_metric_store.get_metric_value.return_value = 249
self.autoscaler.run()
self.docker_client.scale_service.assert_not_called()
def test_run_does_not_scale_down_service_replicas_when_metric_value_equals_scale_down_threshold(
self):
self.docker_client.get_service_replica_count.return_value = 2
self.monitoring_metric_store.get_metric_value.return_value = 250
self.autoscaler.run()
self.docker_client.scale_service.assert_not_called()
def test_run_does_not_scale_down_service_replicas_when_metric_value_is_above_scale_down_threshold(
self):
self.docker_client.get_service_replica_count.return_value = 2
self.monitoring_metric_store.get_metric_value.return_value = 251
self.autoscaler.run()
self.docker_client.scale_service.assert_not_called()
def test_start_starts_the_scheduler(self):
self.scheduler.timezone = utc
self.autoscaler.start()
self.scheduler.start.assert_called_once()
def test_start_schedules_the_job_to_run_autoscaler_for_each_poll_interval_seconds(
self):
self.scheduler.timezone = utc
self.autoscaler.start()
self.scheduler.add_job.assert_called_once_with(self.autoscaler.run,
'interval',
seconds=60)
def test_start_runs_the_job_to_run_autoscaler_immediately(self):
job = Mock()
current_time = datetime.now()
self.scheduler.timezone = utc
self.mock_datetime.now.return_value = current_time
self.scheduler.add_job.return_value = job
self.autoscaler.start()
self.mock_datetime.now.assert_called_once_with(utc)
job.modify.assert_called_once_with(next_run_time=current_time)
def test_scale_up(self, mocker):
"""Test consequent scalings on and off schedule"""
app_name = 'test-api-app'
app_config = {
'name':
app_name,
'min_instances':
5,
'max_instances':
10,
'scalers': [{
'type': 'ElbScaler',
'elb_name': 'my-elb',
'threshold': 300
}, {
'type':
'ScheduleScaler',
'schedule':
'''
---
workdays:
- 08:00-19:00
weekends:
- 09:00-17:00
scale_factor: 0.8
'''
}]
}
mocker.patch.object(AwsBaseScaler, '_get_boto3_client')
mocker.patch.object(ElbScaler, '_get_boto3_client')
mocker.patch.object(ElbScaler, 'gauge')
mocker.patch.object(ElbScaler,
'_get_request_counts',
return_value=[1300, 1500, 1600, 1700, 1700])
mock_paas_client = mocker.patch('app.autoscaler.PaasClient')
mocker.patch('app.autoscaler.Redis', fakeredis.FakeRedis)
mock_get_statsd_client = mocker.patch(
'app.autoscaler.get_statsd_client')
mock_paas_client.return_value.get_paas_apps.return_value = {
app_name: {
'name': app_name,
'instances': 5,
'guid': app_name + '-guid'
},
}
with freeze_time("Thursday 31 May 2018 06:00:00") as frozen_time:
# to trigger a scale up we need at least one value greater than min_instances * threshold
app_config['scalers'][1]['schedule'] = yaml.safe_load(
app_config['scalers'][1]['schedule'])
app = App(**app_config)
autoscaler = Autoscaler()
autoscaler.cooldown_seconds_after_scale_up = 300
autoscaler.cooldown_seconds_after_scale_down = 60
autoscaler._schedule = Mock()
autoscaler.autoscaler_apps = [app]
autoscaler.run_task()
mock_get_statsd_client.return_value.gauge.assert_called_once_with(
"{}.instance-count".format(app_name), 6)
mock_paas_client.return_value.update.assert_called_once_with(
app_name + '-guid', 6)
# emulate that we are running in schedule now, which means max_instances * scale_factor
frozen_time.move_to("Thursday 31 May 2018 13:15:00")
mock_get_statsd_client.return_value.reset_mock()
mock_paas_client.return_value.update.reset_mock()
autoscaler.run_task()
mock_get_statsd_client.return_value.gauge.assert_called_once_with(
"{}.instance-count".format(app_name), 8)
mock_paas_client.return_value.update.assert_called_once_with(
app_name + '-guid', 8)
import logging
import sys
from app.autoscaler import Autoscaler
logging.basicConfig(level=logging.WARNING,
handlers=[
logging.FileHandler('/home/vcap/logs/app.log'),
logging.StreamHandler(sys.stdout),
])
autoscaler = Autoscaler()
autoscaler.run()
import yaml
from pytz import utc
from apscheduler.schedulers.blocking import BlockingScheduler
from app.autoscaler import Autoscaler
from app.scaling_client.scaling_client import ScalingClient
from app.metricstores import MetricStoreFactory
print("AutoScaler Test")
with open("example/autoscaler.yml") as data:
config = yaml.load(data, Loader=yaml.Loader)
# print("config: {}".format(yaml.dump(config)))
scalingHandler = ScalingClient()
metric_store_factory = MetricStoreFactory()
scheduler = BlockingScheduler(timezone=utc)
a_scale = Autoscaler(config=config,
scaling_client=scalingHandler,
metric_store_factory=metric_store_factory,
scheduler=scheduler)
a_scale.start()