def process_default(self, event):
self.log.debug(event)
self.watch_new_folder(event)
event = self.filter_event(event)
if event:
self.log.debug("Public config changed on disk, loading new config")
try:
new_config = load_system_paasta_config()
except ValueError:
self.log.error("Couldn't load public config, the JSON is invalid!")
return
service_instances = []
if new_config != self.public_config:
self.log.info("Public config has changed, now checking if it affects any services config shas")
self.public_config = new_config
all_service_instances = get_services_for_cluster(cluster=self.public_config.get_cluster(),
instance_type='marathon',
soa_dir=DEFAULT_SOA_DIR)
service_instances = get_service_instances_with_changed_id(self.marathon_client,
all_service_instances,
self.public_config.get_cluster())
if service_instances:
self.log.info("Found config change affecting {} service instances, "
"now doing a staggered bounce".format(len(service_instances)))
bounce_rate = self.public_config.get_deployd_big_bounce_rate()
service_instances = rate_limit_instances(instances=service_instances,
number_per_minute=bounce_rate,
watcher_name=self.__class__.__name__)
for service_instance in service_instances:
self.filewatcher.inbox_q.put(service_instance)
def add_all_services(self):
instances = get_services_for_cluster(cluster=self.config.get_cluster(),
instance_type='marathon',
soa_dir=DEFAULT_SOA_DIR)
instances_to_add = rate_limit_instances(instances=instances,
number_per_minute=self.config.get_deployd_startup_bounce_rate(),
watcher_name='daemon_start')
for service_instance in instances_to_add:
self.inbox_q.put(service_instance)
def add_all_services(self):
instances = get_services_for_cluster(
cluster=self.config.get_cluster(),
instance_type='marathon',
soa_dir=DEFAULT_SOA_DIR,
)
instances_to_add = rate_limit_instances(
instances=instances,
cluster=self.config.get_cluster(),
number_per_minute=self.config.get_deployd_startup_bounce_rate(),
watcher_name='daemon_start',
priority=99,
)
for service_instance in instances_to_add:
self.instances_that_need_to_be_bounced_in_the_future.put(
service_instance)
def test_rate_limit_instances():
with mock.patch('time.time', autospec=True) as mock_time:
mock_time.return_value = 1
mock_si_1 = ('universe', 'c137')
mock_si_2 = ('universe', 'c138')
ret = rate_limit_instances([mock_si_1, mock_si_2], 2, "Custos")
expected = [
ServiceInstance(service='universe',
instance='c137',
watcher='Custos',
bounce_by=1,
bounce_timers=None),
ServiceInstance(service='universe',
instance='c138',
watcher='Custos',
bounce_by=31,
bounce_timers=None)
]
assert ret == expected
def process_default(self, event):
self.log.debug(event)
self.watch_new_folder(event)
event = self.filter_event(event)
if event:
self.log.debug(
"Public config changed on disk, loading new config.")
try:
new_config = load_system_paasta_config()
except ValueError:
self.log.error(
"Couldn't load public config, the JSON is invalid!")
return
service_instances: List[Tuple[str, str]] = []
if new_config != self.public_config:
self.log.info(
"Public config has changed, now checking if it affects any services config shas."
)
self.public_config = new_config
all_service_instances = get_services_for_cluster(
cluster=self.public_config.get_cluster(),
instance_type='marathon',
soa_dir=DEFAULT_SOA_DIR,
)
service_instances = get_service_instances_needing_update(
self.marathon_clients,
all_service_instances,
self.public_config.get_cluster(),
)
if service_instances:
self.log.info(
f"{len(service_instances)} service instances affected. Doing a staggered bounce."
)
bounce_rate = self.public_config.get_deployd_big_bounce_rate()
for service_instance in rate_limit_instances(
instances=service_instances,
cluster=self.public_config.get_cluster(),
number_per_minute=bounce_rate,
watcher_name=type(self).__name__,
priority=99,
):
self.filewatcher.instances_that_need_to_be_bounced_in_the_future.put(
service_instance)
def test_rate_limit_instances():
with mock.patch(
'paasta_tools.deployd.common.get_priority',
autospec=True,
return_value=0,
), mock.patch(
'time.time',
autospec=True,
) as mock_time:
mock_time.return_value = 1
mock_si_1 = ('universe', 'c137')
mock_si_2 = ('universe', 'c138')
ret = rate_limit_instances([mock_si_1, mock_si_2], "westeros-prod", 2,
"Custos")
expected = [
BaseServiceInstance(
service='universe',
instance='c137',
watcher='Custos',
priority=0,
bounce_by=1,
bounce_timers=None,
failures=0,
),
BaseServiceInstance(
service='universe',
instance='c138',
watcher='Custos',
priority=0,
bounce_by=31,
bounce_timers=None,
failures=0,
),
]
assert ret == expected
ret = rate_limit_instances([mock_si_1, mock_si_2],
"westeros-prod",
2,
"Custos",
priority=99)
expected = [
BaseServiceInstance(
service='universe',
instance='c137',
watcher='Custos',
priority=99,
bounce_by=1,
bounce_timers=None,
failures=0,
),
BaseServiceInstance(
service='universe',
instance='c138',
watcher='Custos',
priority=99,
bounce_by=31,
bounce_timers=None,
failures=0,
),
]
assert ret == expected
