def bounce_service(self, service_name):
self.log.info(f"Checking if any instances for {service_name} need bouncing")
instances = list_all_instances_for_service(
service=service_name,
clusters=[self.filewatcher.cluster],
instance_type='marathon',
cache=False,
)
self.log.debug(instances)
service_instances = [(service_name, instance) for instance in instances]
service_instances = get_service_instances_needing_update(
self.marathon_clients,
service_instances,
self.filewatcher.cluster,
)
for service, instance in service_instances:
self.log.info(f"{service}.{instance} has a new marathon app ID, and so needs bouncing")
service_instances_to_queue = [
# https://github.com/python/mypy/issues/2852
ServiceInstance( # type: ignore
service=service,
instance=instance,
cluster=self.filewatcher.cluster,
bounce_by=int(time.time()),
watcher=type(self).__name__,
bounce_timers=None,
failures=0,
)
for service, instance in service_instances
]
for service_instance in service_instances_to_queue:
self.filewatcher.inbox_q.put(service_instance)
def bounce_service(self, service_name):
self.log.info(
f"Checking if any marathon instances of {service_name} need bouncing."
)
instances = list_all_instances_for_service(
service=service_name,
clusters=[self.filewatcher.cluster],
instance_type="marathon",
cache=False,
)
self.log.debug(instances)
service_instance_configs = get_service_instances_needing_update(
self.marathon_clients,
[(service_name, instance) for instance in instances],
self.filewatcher.cluster,
)
for service, instance, config in service_instance_configs:
self.log.info(
f"{service}.{instance} has a new marathon app ID. Enqueuing it to be bounced."
)
now = time.time()
self.filewatcher.instances_to_bounce.put(
ServiceInstance( # type: ignore
service=service,
instance=instance,
cluster=self.filewatcher.cluster,
bounce_by=now + config.get_bounce_start_deadline(),
wait_until=now,
watcher=type(self).__name__,
bounce_timers=None,
failures=0,
))
def bounce_service(self, service_name):
self.log.info("Checking if any instances for {} need bouncing".format(
service_name))
instances = list_all_instances_for_service(
service=service_name,
clusters=[self.filewatcher.cluster],
instance_type='marathon',
cache=False,
)
self.log.debug(instances)
service_instances = [(service_name, instance)
for instance in instances]
service_instances = get_service_instances_needing_update(
self.marathon_client,
service_instances,
self.filewatcher.cluster,
)
for service, instance in service_instances:
self.log.info(
"{}.{} has a new marathon app ID, and so needs bouncing".
format(service, instance))
service_instances = [
ServiceInstance(
service=service,
instance=instance,
cluster=self.filewatcher.cluster,
bounce_by=int(time.time()),
watcher=type(self).__name__,
bounce_timers=None,
failures=0,
) for service, instance in service_instances
]
for service_instance in service_instances:
self.filewatcher.inbox_q.put(service_instance)
def process_default(self, event: pyinotify.Event) -> None:
self.log.debug(event)
self.watch_new_folder(event)
event = self.filter_event(event)
if event:
self.log.debug(
"Public config changed on disk, loading new config.")
try:
new_config = load_system_paasta_config()
except ValueError:
self.log.error(
"Couldn't load public config, the JSON is invalid!")
return
service_instance_configs: List[Tuple[str, str,
MarathonServiceConfig,
str]] = []
if new_config != self.public_config:
self.log.info(
"Public config has changed, now checking if it affects any services config shas."
)
self.public_config = new_config
all_service_instances = get_services_for_cluster(
cluster=self.public_config.get_cluster(),
instance_type="marathon",
soa_dir=DEFAULT_SOA_DIR,
)
service_instance_configs = get_service_instances_needing_update(
self.marathon_clients,
all_service_instances,
self.public_config.get_cluster(),
)
if service_instance_configs:
self.log.info(
f"{len(service_instance_configs)} service instances affected. Doing a staggered bounce."
)
for service, instance, config, _ in service_instance_configs:
self.filewatcher.instances_to_bounce.put(
ServiceInstance(
service=service,
instance=instance,
watcher=type(self).__name__,
bounce_by=time.time() + self.public_config.
get_deployd_big_bounce_deadline(),
wait_until=time.time(),
enqueue_time=time.time(),
bounce_start_time=time.time(),
))
def process_default(self, event):
self.log.debug(event)
self.watch_new_folder(event)
event = self.filter_event(event)
if event:
self.log.debug("Public config changed on disk, loading new config")
try:
new_config = load_system_paasta_config()
except ValueError:
self.log.error(
"Couldn't load public config, the JSON is invalid!")
return
service_instances = []
if new_config != self.public_config:
self.log.info(
"Public config has changed, now checking if it affects any services config shas"
)
self.public_config = new_config
all_service_instances = get_services_for_cluster(
cluster=self.public_config.get_cluster(),
instance_type='marathon',
soa_dir=DEFAULT_SOA_DIR,
)
service_instances = get_service_instances_needing_update(
self.marathon_client,
all_service_instances,
self.public_config.get_cluster(),
)
if service_instances:
self.log.info(
"Found config change affecting {} service instances, "
"now doing a staggered bounce".format(
len(service_instances)))
bounce_rate = self.public_config.get_deployd_big_bounce_rate()
service_instances = rate_limit_instances(
instances=service_instances,
cluster=self.public_config.get_cluster(),
number_per_minute=bounce_rate,
watcher_name=type(self).__name__,
priority=99,
)
for service_instance in service_instances:
self.filewatcher.inbox_q.put(service_instance)
def test_get_service_instances_needing_update():
with mock.patch(
"paasta_tools.deployd.common.get_all_marathon_apps", autospec=True
) as mock_get_marathon_apps, mock.patch(
"paasta_tools.deployd.common.load_marathon_service_config_no_cache",
autospec=True,
) as mock_load_marathon_service_config:
mock_marathon_apps = [
mock.Mock(id="/universe.c137.c1.g1", instances=2),
mock.Mock(id="/universe.c138.c1.g1", instances=2),
]
mock_get_marathon_apps.return_value = mock_marathon_apps
mock_service_instances = [("universe", "c137"), ("universe", "c138")]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
"id": "universe.c137.c1.g1",
"instances": 2
})),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
"id": "universe.c138.c2.g2",
"instances": 2
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
"westeros-prod")
assert mock_get_marathon_apps.called
calls = [
mock.call(
service="universe",
instance="c137",
cluster="westeros-prod",
soa_dir=DEFAULT_SOA_DIR,
),
mock.call(
service="universe",
instance="c138",
cluster="westeros-prod",
soa_dir=DEFAULT_SOA_DIR,
),
]
mock_load_marathon_service_config.assert_has_calls(calls)
assert ret == [("universe", "c138", mock.ANY)]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
"id": "universe.c137.c1.g1",
"instances": 3
})),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
"id": "universe.c138.c2.g2",
"instances": 2
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
"westeros-prod")
assert ret == [("universe", "c137", mock.ANY),
("universe", "c138", mock.ANY)]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(
side_effect=NoDockerImageError)),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
"id": "universe.c138.c2.g2",
"instances": 2
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
"westeros-prod")
assert ret == [("universe", "c138", mock.ANY)]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(
side_effect=NoSlavesAvailableError)),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
"id": "universe.c138.c2.g2",
"instances": 2
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
"westeros-prod")
assert ret == [("universe", "c138", mock.ANY)]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(
side_effect=InvalidJobNameError)),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
"id": "universe.c138.c2.g2",
"instances": 2
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
"westeros-prod")
assert ret == [("universe", "c138", mock.ANY)]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(
side_effect=NoDeploymentsAvailable)),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
"id": "universe.c138.c2.g2",
"instances": 2
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
"westeros-prod")
assert ret == [("universe", "c138", mock.ANY)]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(
side_effect=Exception)),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
"id": "universe.c138.c2.g2",
"instances": 2
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
"westeros-prod")
assert ret == [("universe", "c138", mock.ANY)]
def test_get_service_instances_needing_update():
with mock.patch(
'paasta_tools.deployd.common.get_all_marathon_apps',
autospec=True,
) as mock_get_marathon_apps, mock.patch(
'paasta_tools.deployd.common.load_marathon_service_config_no_cache',
autospec=True,
) as mock_load_marathon_service_config:
mock_marathon_apps = [
mock.Mock(id='/universe.c137.c1.g1', instances=2),
mock.Mock(id='/universe.c138.c1.g1', instances=2),
]
mock_get_marathon_apps.return_value = mock_marathon_apps
mock_service_instances = [('universe', 'c137'), ('universe', 'c138')]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c137.c1.g1',
'instances': 2,
})),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c138.c2.g2',
'instances': 2,
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
'westeros-prod')
assert mock_get_marathon_apps.called
calls = [
mock.call(
service='universe',
instance='c137',
cluster='westeros-prod',
soa_dir=DEFAULT_SOA_DIR,
),
mock.call(
service='universe',
instance='c138',
cluster='westeros-prod',
soa_dir=DEFAULT_SOA_DIR,
),
]
mock_load_marathon_service_config.assert_has_calls(calls)
assert ret == [('universe', 'c138')]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c137.c1.g1',
'instances': 3,
})),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c138.c2.g2',
'instances': 2,
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
'westeros-prod')
assert ret == [('universe', 'c137'), ('universe', 'c138')]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(
side_effect=NoDockerImageError)),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c138.c2.g2',
'instances': 2,
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
'westeros-prod')
assert ret == [('universe', 'c138')]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(
side_effect=InvalidJobNameError)),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c138.c2.g2',
'instances': 2,
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
'westeros-prod')
assert ret == [('universe', 'c138')]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(
side_effect=NoDeploymentsAvailable)),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c138.c2.g2',
'instances': 2,
})),
]
mock_load_marathon_service_config.side_effect = mock_configs
mock_client = mock.Mock(servers=["foo"])
fake_clients = MarathonClients(current=[mock_client],
previous=[mock_client])
ret = get_service_instances_needing_update(fake_clients,
mock_service_instances,
'westeros-prod')
assert ret == [('universe', 'c138')]
def test_get_service_instances_needing_update():
with mock.patch(
'paasta_tools.deployd.common.get_all_marathon_apps', autospec=True
) as mock_get_marathon_apps, mock.patch(
'paasta_tools.deployd.common.load_marathon_service_config_no_cache',
autospec=True) as mock_load_marathon_service_config:
mock_marathon_apps = [
mock.Mock(id='/universe.c137.c1.g1', instances=2),
mock.Mock(id='/universe.c138.c1.g1', instances=2)
]
mock_get_marathon_apps.return_value = mock_marathon_apps
mock_service_instances = [('universe', 'c137'), ('universe', 'c138')]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c137.c1.g1',
'instances': 2
})),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c138.c2.g2',
'instances': 2
}))
]
mock_load_marathon_service_config.side_effect = mock_configs
ret = get_service_instances_needing_update(mock.Mock(),
mock_service_instances,
'westeros-prod')
assert mock_get_marathon_apps.called
calls = [
mock.call(service='universe',
instance='c137',
cluster='westeros-prod',
soa_dir=DEFAULT_SOA_DIR),
mock.call(service='universe',
instance='c138',
cluster='westeros-prod',
soa_dir=DEFAULT_SOA_DIR)
]
mock_load_marathon_service_config.assert_has_calls(calls)
assert ret == [('universe', 'c138')]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c137.c1.g1',
'instances': 3
})),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c138.c2.g2',
'instances': 2
}))
]
mock_load_marathon_service_config.side_effect = mock_configs
ret = get_service_instances_needing_update(mock.Mock(),
mock_service_instances,
'westeros-prod')
assert ret == [('universe', 'c137'), ('universe', 'c138')]
mock_configs = [
mock.Mock(format_marathon_app_dict=mock.Mock(
side_effect=NoDockerImageError)),
mock.Mock(format_marathon_app_dict=mock.Mock(return_value={
'id': 'universe.c138.c2.g2',
'instances': 2
}))
]
mock_load_marathon_service_config.side_effect = mock_configs
ret = get_service_instances_needing_update(mock.Mock(),
mock_service_instances,
'westeros-prod')
assert ret == [('universe', 'c137'), ('universe', 'c138')]
