def test_get_at_risk_service_instances(self):
with mock.patch('paasta_tools.deployd.watchers.get_all_marathon_apps',
autospec=True) as mock_get_marathon_apps, mock.patch(
'time.time', autospec=True, return_value=1):
mock_marathon_apps = [
mock.Mock(tasks=[
mock.Mock(host='host1',
app_id='/universe.c137.configsha.gitsha'),
mock.Mock(host='host2',
app_id='/universe.c138.configsha.gitsha')
]),
mock.Mock(tasks=[
mock.Mock(host='host1',
app_id='/universe.c139.configsha.gitsha')
]),
mock.Mock(tasks=[
mock.Mock(host='host1',
app_id='/universe.c139.configsha.gitsha')
])
]
mock_get_marathon_apps.return_value = mock_marathon_apps
ret = self.watcher.get_at_risk_service_instances(['host1'])
expected = [
ServiceInstance(service='universe',
instance='c137',
bounce_by=1,
watcher=self.watcher.__class__.__name__,
bounce_timers=None),
ServiceInstance(service='universe',
instance='c139',
bounce_by=1,
watcher=self.watcher.__class__.__name__,
bounce_timers=None)
]
assert ret == expected
def test_prioritise_bouncing_services(self):
with mock.patch(
'paasta_tools.deployd.master.get_service_instances_that_need_bouncing',
autospec=True
) as mock_get_service_instances_that_need_bouncing, mock.patch(
'time.time', autospec=True, return_value=1):
mock_changed_instances = (
x for x in {'universe.c137', 'universe.c138'})
mock_get_service_instances_that_need_bouncing.return_value = mock_changed_instances
self.deployd.prioritise_bouncing_services()
mock_get_service_instances_that_need_bouncing.assert_called_with(
self.deployd.marathon_client, '/nail/etc/services')
calls = [
mock.call(
ServiceInstance(service='universe',
instance='c138',
watcher='DeployDaemon',
bounce_by=1,
bounce_timers=None,
failures=0)),
mock.call(
ServiceInstance(service='universe',
instance='c137',
watcher='DeployDaemon',
bounce_by=1,
bounce_timers=None,
failures=0))
]
self.deployd.inbox_q.put.assert_has_calls(calls, any_order=True)
def test_process_node_event(self):
with mock.patch('paasta_tools.deployd.watchers.EventType',
autospec=True) as mock_event_type, mock.patch(
'time.time', autospec=True, return_value=1):
mock_event_other = mock_event_type.DELETED
mock_event = mock.Mock(
type=mock_event_other,
path='/autoscaling/service/instance/instances')
assert not self.mock_inbox_q.put.called
mock_event_created = mock_event_type.CREATED
mock_event = mock.Mock(
type=mock_event_created,
path='/autoscaling/service/instance/instances')
self.watcher.process_node_event(mock.Mock(), mock.Mock(),
mock_event)
self.mock_inbox_q.put.assert_called_with(
ServiceInstance(service='service',
instance='instance',
bounce_by=1,
bounce_timers=None,
watcher=self.watcher.__class__.__name__))
mock_event_changed = mock_event_type.CHANGED
mock_event = mock.Mock(
type=mock_event_changed,
path='/autoscaling/service/instance/instances')
self.watcher.process_node_event(mock.Mock(), mock.Mock(),
mock_event)
self.mock_inbox_q.put.assert_called_with(
ServiceInstance(service='service',
instance='instance',
bounce_by=1,
bounce_timers=None,
watcher=self.watcher.__class__.__name__))
def test_get_at_risk_service_instances(self):
with mock.patch(
"paasta_tools.deployd.watchers.get_marathon_apps_with_clients",
autospec=True,
) as mock_get_marathon_apps, mock.patch("time.time",
autospec=True,
return_value=1):
mock_marathon_apps = [
mock.Mock(tasks=[
mock.Mock(host="host1",
app_id="/universe.c137.configsha.gitsha"),
mock.Mock(host="host2",
app_id="/universe.c138.configsha.gitsha"),
]),
mock.Mock(tasks=[
mock.Mock(host="host1",
app_id="/universe.c139.configsha.gitsha")
]),
mock.Mock(tasks=[
mock.Mock(host="host1",
app_id="/universe.c139.configsha.gitsha")
]),
]
mock_client = mock.Mock()
mock_get_marathon_apps.return_value = [
(app, mock_client) for app in mock_marathon_apps
]
ret = self.watcher.get_at_risk_service_instances(["host1"])
expected = [
ServiceInstance(
service="universe",
instance="c137",
bounce_by=1,
wait_until=1,
watcher=self.watcher.__class__.__name__,
failures=0,
processed_count=0,
enqueue_time=1,
bounce_start_time=1,
),
ServiceInstance(
service="universe",
instance="c139",
bounce_by=1,
wait_until=1,
watcher=self.watcher.__class__.__name__,
failures=0,
processed_count=0,
enqueue_time=1,
bounce_start_time=1,
),
]
assert ret == expected
def test___new__(self):
with mock.patch(
'paasta_tools.deployd.common.ServiceInstance.get_priority',
autospec=True,
):
expected = BaseServiceInstance(
service='universe',
instance='c137',
watcher='mywatcher',
bounce_by=0,
failures=0,
bounce_timers=None,
priority=1,
)
assert self.service_instance == expected
expected = BaseServiceInstance(
service='universe',
instance='c137',
watcher='mywatcher',
bounce_by=0,
failures=0,
bounce_timers=None,
priority=2,
)
assert ServiceInstance(
service='universe',
instance='c137',
watcher='mywatcher',
cluster='westeros-prod',
bounce_by=0,
priority=2,
) == expected
def get_at_risk_service_instances(self, draining_hosts):
marathon_apps = get_all_marathon_apps(self.marathon_client,
embed_tasks=True)
at_risk_tasks = [
task for app in marathon_apps for task in app.tasks
if task.host in draining_hosts
]
self.log.info("At risk tasks: {}".format(at_risk_tasks))
service_instances = []
for task in at_risk_tasks:
app_id = task.app_id.strip('/')
service, instance, _, __ = deformat_job_id(app_id)
# check we haven't already added this instance,
# no need to add the same instance to the bounce queue
# more than once
if not any([(service, instance) == (si.service, si.instance)
for si in service_instances]):
service_instances.append(
ServiceInstance(
service=service,
instance=instance,
cluster=self.config.get_cluster(),
bounce_by=int(time.time()),
watcher=type(self).__name__,
bounce_timers=None,
failures=0,
))
return service_instances
def bounce_service(self, service_name):
self.log.info("Checking if any instances for {} need bouncing".format(
service_name))
instances = list_all_instances_for_service(
service=service_name,
clusters=[self.filewatcher.cluster],
instance_type='marathon',
cache=False)
self.log.debug(instances)
service_instances = [(service_name, instance)
for instance in instances]
service_instances = get_service_instances_with_changed_id(
self.marathon_client, service_instances, self.filewatcher.cluster)
for service, instance in service_instances:
self.log.info(
"{}.{} has a new marathon app ID, and so needs bouncing".
format(service, instance))
service_instances = [
ServiceInstance(service=service,
instance=instance,
bounce_by=int(time.time()),
watcher=self.__class__.__name__,
bounce_timers=None)
for service, instance in service_instances
]
for service_instance in service_instances:
self.filewatcher.inbox_q.put(service_instance)
def test___new__(self):
expected = BaseServiceInstance(
service="universe",
instance="c137",
watcher="mywatcher",
bounce_by=0,
wait_until=0,
failures=0,
bounce_timers=None,
processed_count=0,
)
assert self.service_instance == expected
expected = BaseServiceInstance(
service="universe",
instance="c137",
watcher="mywatcher",
bounce_by=0,
wait_until=0,
failures=0,
bounce_timers=None,
processed_count=0,
)
# https://github.com/python/mypy/issues/2852
assert (ServiceInstance( # type: ignore
service="universe",
instance="c137",
watcher="mywatcher",
cluster="westeros-prod",
bounce_by=0,
wait_until=0,
) == expected)
def test_process_default(self):
with mock.patch(
"paasta_tools.deployd.watchers.PublicConfigEventHandler.filter_event",
autospec=True,
) as mock_filter_event, mock.patch(
"paasta_tools.deployd.watchers.PublicConfigEventHandler.watch_new_folder",
autospec=True,
), mock.patch(
"paasta_tools.deployd.watchers.get_services_for_cluster",
autospec=True
) as mock_get_services_for_cluster, mock.patch(
"paasta_tools.deployd.watchers.load_system_paasta_config",
autospec=True
) as mock_load_system_config, mock.patch(
"paasta_tools.deployd.watchers.get_service_instances_needing_update",
autospec=True,
) as mock_get_service_instances_needing_update, mock.patch(
"time.time", return_value=1.0, autospec=True):
mock_event = mock.Mock()
mock_filter_event.return_value = mock_event
mock_load_system_config.return_value = self.mock_config
self.handler.process_default(mock_event)
assert mock_load_system_config.called
assert not mock_get_services_for_cluster.called
assert not mock_get_service_instances_needing_update.called
assert not self.mock_filewatcher.instances_to_bounce.put.called
mock_load_system_config.return_value = mock.Mock(
get_cluster=mock.Mock())
mock_get_service_instances_needing_update.return_value = []
self.handler.process_default(mock_event)
assert mock_load_system_config.called
assert mock_get_services_for_cluster.called
assert mock_get_service_instances_needing_update.called
assert not self.mock_filewatcher.instances_to_bounce.put.called
mock_load_system_config.return_value = mock.Mock(
get_deployd_big_bounce_deadline=mock.Mock(return_value=100.0))
fake_si = (
"someservice",
"someinstance",
mock.Mock(),
"someservice.someinstance.stuff.otherstuff",
)
mock_get_service_instances_needing_update.return_value = [fake_si]
self.handler.process_default(mock_event)
assert mock_load_system_config.called
assert mock_get_services_for_cluster.called
assert mock_get_service_instances_needing_update.called
# call objects are (name, posargs, kwargs), so this grabs the first posarg of the most recent call.
assert self.mock_filewatcher.instances_to_bounce.put.mock_calls[
-1][1][0] == ServiceInstance(
service="someservice",
instance="someinstance",
watcher="PublicConfigEventHandler",
bounce_by=101.0,
wait_until=1.0,
enqueue_time=1.0,
bounce_start_time=1.0,
)
def main():
system_paasta_config = load_system_paasta_config()
args = parse_args(system_paasta_config.get_deployd_startup_bounce_deadline())
service, instance = args.service_instance.split(".", 1)
try:
validate_service_instance(
service,
instance,
cluster=system_paasta_config.get_cluster(),
soa_dir=DEFAULT_SOA_DIR,
)
except NoConfigurationForServiceError as e:
paasta_print(PaastaColors.red(str(e)))
sys.exit(1)
service_instance = ServiceInstance(
service=service,
instance=instance,
bounce_by=time.time() + args.bounce_by_delay_secs,
wait_until=time.time(),
watcher="manually_added",
failures=0,
enqueue_time=time.time(),
bounce_start_time=time.time(),
)
zk_client = KazooClient(hosts=system_paasta_config.get_zk_hosts())
zk_client.start()
queue = ZKDelayDeadlineQueue(client=zk_client)
queue.put(service_instance)
def bounce_service(self, service_name):
self.log.info(
f"Checking if any marathon instances of {service_name} need bouncing."
)
instances = list_all_instances_for_service(
service=service_name,
clusters=[self.filewatcher.cluster],
instance_type="marathon",
cache=False,
)
self.log.debug(instances)
service_instance_configs = get_service_instances_needing_update(
self.marathon_clients,
[(service_name, instance) for instance in instances],
self.filewatcher.cluster,
)
for service, instance, config in service_instance_configs:
self.log.info(
f"{service}.{instance} has a new marathon app ID. Enqueuing it to be bounced."
)
now = time.time()
self.filewatcher.instances_to_bounce.put(
ServiceInstance( # type: ignore
service=service,
instance=instance,
cluster=self.filewatcher.cluster,
bounce_by=now + config.get_bounce_start_deadline(),
wait_until=now,
watcher=type(self).__name__,
bounce_timers=None,
failures=0,
))
def get_at_risk_service_instances(
self, draining_hosts: List[str]) -> List[ServiceInstance]:
marathon_apps_with_clients = get_marathon_apps_with_clients(
clients=self.marathon_clients.get_all_clients(), embed_tasks=True)
at_risk_tasks = []
for app, client in marathon_apps_with_clients:
for task in app.tasks:
if task.host in draining_hosts:
at_risk_tasks.append(task)
self.log.info(f"At risk tasks: {at_risk_tasks}")
service_instances: List[ServiceInstance] = []
for task in at_risk_tasks:
app_id = task.app_id.strip("/")
service, instance, _, __ = deformat_job_id(app_id)
# check we haven't already added this instance,
# no need to add the same instance to the bounce queue
# more than once
if not any([(service, instance) == (si.service, si.instance)
for si in service_instances]):
service_instances.append(
ServiceInstance(
service=service,
instance=instance,
bounce_by=time.time(),
wait_until=time.time(),
watcher=type(self).__name__,
failures=0,
enqueue_time=time.time(),
bounce_start_time=time.time(),
))
return service_instances
def bounce_service(self, service_name):
self.log.info(f"Checking if any instances for {service_name} need bouncing")
instances = list_all_instances_for_service(
service=service_name,
clusters=[self.filewatcher.cluster],
instance_type='marathon',
cache=False,
)
self.log.debug(instances)
service_instances = [(service_name, instance) for instance in instances]
service_instances = get_service_instances_needing_update(
self.marathon_clients,
service_instances,
self.filewatcher.cluster,
)
for service, instance in service_instances:
self.log.info(f"{service}.{instance} has a new marathon app ID, and so needs bouncing")
service_instances_to_queue = [
# https://github.com/python/mypy/issues/2852
ServiceInstance( # type: ignore
service=service,
instance=instance,
cluster=self.filewatcher.cluster,
bounce_by=int(time.time()),
watcher=type(self).__name__,
bounce_timers=None,
failures=0,
)
for service, instance in service_instances
]
for service_instance in service_instances_to_queue:
self.filewatcher.inbox_q.put(service_instance)
def process_default(self, event):
self.log.debug(event)
self.watch_new_folder(event)
event = self.filter_event(event)
if event:
self.log.info("Change of {} in {}".format(event.name, event.path))
service_name = event.path.split('/')[-1]
self.log.info("Checking if any instances for {} need bouncing".format(service_name))
instances = list_all_instances_for_service(service=service_name,
clusters=[self.filewatcher.cluster],
instance_type='marathon')
self.log.debug(instances)
service_instances = [(service_name, instance) for instance in instances]
service_instances = get_service_instances_with_changed_id(self.marathon_client,
service_instances,
self.filewatcher.cluster)
for service, instance in service_instances:
self.log.info("{}.{} has a new marathon app ID, and so needs bouncing".format(service, instance))
service_instances = [ServiceInstance(service=service,
instance=instance,
bounce_by=int(time.time()),
watcher=self.__class__.__name__,
bounce_timers=None)
for service, instance in service_instances]
for service_instance in service_instances:
self.filewatcher.inbox_q.put(service_instance)
def get_at_risk_service_instances(self, draining_hosts) -> List[ServiceInstance]:
marathon_apps_with_clients = get_marathon_apps_with_clients(
clients=self.marathon_clients.get_all_clients(),
embed_tasks=True,
)
at_risk_tasks = []
for app, client in marathon_apps_with_clients:
for task in app.tasks:
if task.host in draining_hosts:
at_risk_tasks.append(task)
self.log.info(f"At risk tasks: {at_risk_tasks}")
service_instances: List[ServiceInstance] = []
for task in at_risk_tasks:
app_id = task.app_id.strip('/')
service, instance, _, __ = deformat_job_id(app_id)
# check we haven't already added this instance,
# no need to add the same instance to the bounce queue
# more than once
if not any([(service, instance) == (si.service, si.instance) for si in service_instances]):
# https://github.com/python/mypy/issues/2852
service_instances.append(ServiceInstance( # type: ignore
service=service,
instance=instance,
cluster=self.config.get_cluster(),
bounce_by=int(time.time()),
watcher=type(self).__name__,
bounce_timers=None,
failures=0,
))
return service_instances
def test_bounce_service(self):
with mock.patch(
'paasta_tools.deployd.watchers.list_all_instances_for_service',
autospec=True
) as mock_list_instances, mock.patch(
'paasta_tools.deployd.watchers.get_service_instances_with_changed_id',
autospec=True
) as mock_get_service_instances_with_changed_id, mock.patch(
'time.time', autospec=True, return_value=1):
mock_list_instances.return_value = ['c137', 'c138']
mock_get_service_instances_with_changed_id.return_value = [
('universe', 'c137')
]
self.handler.bounce_service('universe')
mock_list_instances.assert_called_with(
service='universe',
clusters=[self.handler.filewatcher.cluster],
instance_type='marathon',
cache=False)
mock_get_service_instances_with_changed_id.assert_called_with(
self.handler.marathon_client, [('universe', 'c137'),
('universe', 'c138')],
self.handler.filewatcher.cluster)
expected_si = ServiceInstance(service='universe',
instance='c137',
bounce_by=1,
watcher='YelpSoaEventHandler',
bounce_timers=None,
failures=0)
self.mock_filewatcher.inbox_q.put.assert_called_with(expected_si)
assert self.mock_filewatcher.inbox_q.put.call_count == 1
def run(self):
self.log.info(f"{self.name} starting up")
while True:
service_instance = self.instances_to_bounce_now.get()
try:
bounce_again_in_seconds, return_code, bounce_timers = self.process_service_instance(service_instance)
except Exception as e:
self.log.error("Worker failed to process service instance and will retry. "
"Caused by exception: {}".format(e))
return_code = -2
bounce_timers = service_instance.bounce_timers
failures = service_instance.failures
if return_code != 0:
failures = service_instance.failures + 1
bounce_again_in_seconds = exponential_back_off(
failures=failures,
factor=self.config.get_deployd_worker_failure_backoff_factor(),
base=2,
max_time=6000,
)
if bounce_again_in_seconds:
service_instance = ServiceInstance(
service=service_instance.service,
instance=service_instance.instance,
cluster=self.config.get_cluster(),
bounce_by=int(time.time()) + bounce_again_in_seconds,
watcher=self.name,
bounce_timers=bounce_timers,
priority=service_instance.priority,
failures=failures,
)
self.instances_to_bounce_later.put(service_instance)
time.sleep(0.1)
def test___new__(self):
with mock.patch(
'paasta_tools.deployd.common.get_priority',
autospec=True,
):
expected = BaseServiceInstance(
service='universe',
instance='c137',
watcher='mywatcher',
bounce_by=0,
failures=0,
bounce_timers=None,
priority=1,
)
assert self.service_instance == expected
expected = BaseServiceInstance(
service='universe',
instance='c137',
watcher='mywatcher',
bounce_by=0,
failures=0,
bounce_timers=None,
priority=2,
)
# https://github.com/python/mypy/issues/2852
assert ServiceInstance( # type: ignore
service='universe',
instance='c137',
watcher='mywatcher',
cluster='westeros-prod',
bounce_by=0,
priority=2,
) == expected
def test_process_node_event(self):
with mock.patch("paasta_tools.deployd.watchers.EventType",
autospec=True) as mock_event_type, mock.patch(
"time.time", autospec=True, return_value=1):
mock_event_other = mock_event_type.DELETED
mock_event = mock.Mock(
type=mock_event_other,
path="/autoscaling/service/instance/instances")
assert not self.mock_instances_to_bounce.put.called
mock_event_created = mock_event_type.CREATED
mock_event = mock.Mock(
type=mock_event_created,
path="/autoscaling/service/instance/instances")
self.watcher.process_node_event(mock.Mock(), mock.Mock(),
mock_event)
self.mock_instances_to_bounce.put.assert_called_with(
ServiceInstance(
service="service",
instance="instance",
bounce_by=1,
wait_until=1,
watcher=self.watcher.__class__.__name__,
failures=0,
processed_count=0,
enqueue_time=1,
bounce_start_time=1,
))
mock_event_changed = mock_event_type.CHANGED
mock_event = mock.Mock(
type=mock_event_changed,
path="/autoscaling/service/instance/instances")
self.watcher.process_node_event(mock.Mock(), mock.Mock(),
mock_event)
self.mock_instances_to_bounce.put.assert_called_with(
ServiceInstance(
service="service",
instance="instance",
bounce_by=1,
wait_until=1,
watcher=self.watcher.__class__.__name__,
failures=0,
processed_count=0,
enqueue_time=1,
bounce_start_time=1,
))
def run(self):
self.log.info("{} starting up".format(self.name))
while True:
service_instance = self.bounce_q.get()
failures = service_instance.failures
bounce_timers = self.setup_timers(service_instance)
self.log.info("{} processing {}.{}".format(
self.name, service_instance.service,
service_instance.instance))
marathon_apps = marathon_tools.get_all_marathon_apps(
self.marathon_client, embed_failures=True)
bounce_timers.setup_marathon.start()
try:
return_code, bounce_again_in_seconds = deploy_marathon_service(
service=service_instance.service,
instance=service_instance.instance,
client=self.marathon_client,
soa_dir=marathon_tools.DEFAULT_SOA_DIR,
marathon_config=self.marathon_config,
marathon_apps=marathon_apps)
except Exception as e:
self.log.warning(
"deploy_marathon_service caused exception: {}".format(e))
return_code = -2
if return_code != 0:
failures += 1
bounce_again_in_seconds = exponential_back_off(
failures=failures,
factor=self.config.
get_deployd_worker_failure_backoff_factor(),
base=2,
max_time=6000)
bounce_timers.setup_marathon.stop()
self.log.info(
"setup marathon completed with exit code {} for {}.{}".format(
return_code, service_instance.service,
service_instance.instance))
if bounce_again_in_seconds:
bounce_timers.processed_by_worker.start()
self.log.info(
"{}.{} not in steady state so bouncing again in {} "
"seconds".format(service_instance.service,
service_instance.instance,
bounce_again_in_seconds))
service_instance = ServiceInstance(
service=service_instance.service,
instance=service_instance.instance,
bounce_by=int(time.time()) + bounce_again_in_seconds,
watcher=self.name,
bounce_timers=bounce_timers,
failures=failures)
self.inbox_q.put(service_instance)
else:
bounce_timers.bounce_length.stop()
self.log.info("{}.{} in steady state".format(
service_instance.service, service_instance.instance))
time.sleep(0.1)
def setUp(self):
self.service_instance = ServiceInstance( # type: ignore
service="universe",
instance="c137",
watcher="mywatcher",
cluster="westeros-prod",
bounce_by=0,
wait_until=0,
)
def test_rate_limit_instances():
with mock.patch('time.time', autospec=True) as mock_time:
mock_time.return_value = 1
mock_si_1 = ('universe', 'c137')
mock_si_2 = ('universe', 'c138')
ret = rate_limit_instances([mock_si_1, mock_si_2], 2, "Custos")
expected = [
ServiceInstance(service='universe',
instance='c137',
watcher='Custos',
bounce_by=1,
bounce_timers=None),
ServiceInstance(service='universe',
instance='c138',
watcher='Custos',
bounce_by=31,
bounce_timers=None)
]
assert ret == expected
def test_prioritise_bouncing_services(self):
with mock.patch(
"paasta_tools.deployd.master.get_service_instances_that_need_bouncing",
autospec=True,
) as mock_get_service_instances_that_need_bouncing, mock.patch(
"time.time", autospec=True, return_value=1):
mock_changed_instances = (
x for x in {"universe.c137", "universe.c138"})
mock_get_service_instances_that_need_bouncing.return_value = (
mock_changed_instances)
self.deployd.prioritise_bouncing_services()
mock_get_service_instances_that_need_bouncing.assert_called_with(
self.deployd.marathon_clients, "/nail/etc/services")
calls = [
mock.call(
ServiceInstance(
service="universe",
instance="c138",
watcher="DeployDaemon",
bounce_by=1,
wait_until=1,
failures=0,
processed_count=0,
enqueue_time=1,
bounce_start_time=1,
)),
mock.call(
ServiceInstance(
service="universe",
instance="c137",
watcher="DeployDaemon",
bounce_by=1,
wait_until=1,
failures=0,
processed_count=0,
enqueue_time=1,
bounce_start_time=1,
)),
]
self.deployd.instances_to_bounce.put.assert_has_calls(
calls, any_order=True)
def test_run(self):
with mock.patch(
'time.sleep', autospec=True
) as mock_sleep, mock.patch(
'paasta_tools.deployd.workers.marathon_tools.get_all_marathon_apps', autospec=True
) as mock_get_all_marathon_apps, mock.patch(
'paasta_tools.deployd.workers.PaastaDeployWorker.setup_timers', autospec=True
) as mock_setup_timers, mock.patch(
'paasta_tools.deployd.workers.deploy_marathon_service', autospec=True
) as mock_deploy_marathon_service, mock.patch(
'time.time', autospec=True, return_value=1
):
mock_sleep.side_effect = LoopBreak
self.worker.marathon_client = mock.Mock()
self.worker.marathon_config = mock.Mock()
mock_deploy_marathon_service.return_value = (0, None)
mock_si = mock.Mock(service='universe',
instance='c137')
self.mock_bounce_q.get.return_value = mock_si
with raises(LoopBreak):
self.worker.run()
mock_setup_timers.assert_called_with(self.worker, mock_si)
assert mock_setup_timers.return_value.setup_marathon.start.called
mock_deploy_marathon_service.assert_called_with(service='universe',
instance='c137',
client=self.worker.marathon_client,
soa_dir=DEFAULT_SOA_DIR,
marathon_config=self.worker.marathon_config,
marathon_apps=mock_get_all_marathon_apps.return_value)
assert mock_setup_timers.return_value.setup_marathon.stop.called
assert not mock_setup_timers.return_value.processed_by_worker.start.called
assert not self.mock_inbox_q.put.called
assert mock_setup_timers.return_value.bounce_length.stop.called
mock_deploy_marathon_service.return_value = (0, 60)
mock_setup_timers.return_value.bounce_length.stop.reset_mock()
with raises(LoopBreak):
self.worker.run()
mock_setup_timers.assert_called_with(self.worker, mock_si)
assert mock_setup_timers.return_value.setup_marathon.start.called
mock_deploy_marathon_service.assert_called_with(service='universe',
instance='c137',
client=self.worker.marathon_client,
soa_dir=DEFAULT_SOA_DIR,
marathon_config=self.worker.marathon_config,
marathon_apps=mock_get_all_marathon_apps.return_value)
assert mock_setup_timers.return_value.setup_marathon.stop.called
assert mock_setup_timers.return_value.processed_by_worker.start.called
self.mock_inbox_q.put.assert_called_with(ServiceInstance(service='universe',
instance='c137',
bounce_by=61,
watcher='Worker1',
bounce_timers=mock_setup_timers.return_value))
assert not mock_setup_timers.return_value.bounce_length.stop.called
def process_node_event(self, data, stat, event):
self.log.debug("Node change: {}".format(event))
if event and (event.type == EventType.CREATED or event.type == EventType.CHANGED):
service, instance = event.path.split('/')[-3:-1]
self.log.info("Number of instances changed or autoscaling enabled for first time"
" for {}.{}".format(service, instance))
service_instance = ServiceInstance(service=service,
instance=instance,
bounce_by=int(time.time()),
bounce_timers=None,
watcher=self.__class__.__name__)
self.inbox_q.put(service_instance)
def run(self) -> None:
"""Takes things from the to_bounce_now queue, processes them, then
might put them on the bounce_later queue for future processing"""
self.log.info(f"{self.name} starting up")
while True:
with self.instances_to_bounce.get() as service_instance:
self.busy = True
try:
(
bounce_again_in_seconds,
return_code,
) = self.process_service_instance(service_instance)
except Exception:
self.log.error(
f"{self.name} Worker failed to process service instance and will retry. "
f"Caused by exception: {traceback.format_exc()}")
return_code = -2
failures = service_instance.failures
if return_code != 0:
failures = service_instance.failures + 1
bounce_again_in_seconds = exponential_back_off(
failures=failures,
factor=self.config.
get_deployd_worker_failure_backoff_factor(),
base=2,
max_time=6000,
)
if bounce_again_in_seconds:
if failures >= self.max_failures:
self.log.info(
f"{self.name} Worker removing "
f"{service_instance.service}.{service_instance.instance} "
f"from queue because it has failed {failures} times "
f"(max is {self.max_failures})")
else:
bounce_by = int(time.time()) + bounce_again_in_seconds
service_instance = ServiceInstance(
service=service_instance.service,
instance=service_instance.instance,
bounce_by=bounce_by,
wait_until=bounce_by,
watcher=self.name,
failures=failures,
processed_count=service_instance.processed_count +
1,
bounce_start_time=service_instance.
bounce_start_time,
enqueue_time=time.time(),
)
self.instances_to_bounce.put(service_instance)
self.busy = False
time.sleep(0.1)
def _parse_data(self, entry_data: bytes) -> ServiceInstance:
now = time.time()
defaults = {
"watcher": "unknown",
"bounce_by": now,
"wait_until": now,
"enqueue_time": now,
"bounce_start_time": now,
"failures": 0,
}
si_dict = json.loads(entry_data.decode("utf-8"))
merged = {**defaults, **si_dict}
return ServiceInstance(**merged) # type: ignore
def setUp(self):
with mock.patch(
'paasta_tools.deployd.common.ServiceInstance.get_priority',
autospec=True,
) as mock_get_priority:
mock_get_priority.return_value = 1
self.service_instance = ServiceInstance(
service='universe',
instance='c137',
watcher='mywatcher',
cluster='westeros-prod',
bounce_by=0,
)
def setUp(self):
with mock.patch(
'paasta_tools.deployd.common.get_priority',
autospec=True,
) as mock_get_priority:
mock_get_priority.return_value = 1
# https://github.com/python/mypy/issues/2852
self.service_instance = ServiceInstance( # type: ignore
service='universe',
instance='c137',
watcher='mywatcher',
cluster='westeros-prod',
bounce_by=0,
)
def make_si(wait_until, bounce_by):
"""Just using mock.Mock(wait_until=wait_until, bounce_by=bounce_by) mostly works, but our PriorityQueues
occasionally will compare two ServiceInstances directly, and Mocks aren't comparable unless you define an __eq__."""
return ServiceInstance(
service="service",
instance="instance",
bounce_by=bounce_by,
wait_until=wait_until,
watcher="watcher",
failures=0,
processed_count=0,
enqueue_time=1,
bounce_start_time=1,
)
