def test_get_at_risk_service_instances(self): with mock.patch('paasta_tools.deployd.watchers.get_all_marathon_apps', autospec=True) as mock_get_marathon_apps, mock.patch( 'time.time', autospec=True, return_value=1): mock_marathon_apps = [ mock.Mock(tasks=[ mock.Mock(host='host1', app_id='/universe.c137.configsha.gitsha'), mock.Mock(host='host2', app_id='/universe.c138.configsha.gitsha') ]), mock.Mock(tasks=[ mock.Mock(host='host1', app_id='/universe.c139.configsha.gitsha') ]), mock.Mock(tasks=[ mock.Mock(host='host1', app_id='/universe.c139.configsha.gitsha') ]) ] mock_get_marathon_apps.return_value = mock_marathon_apps ret = self.watcher.get_at_risk_service_instances(['host1']) expected = [ ServiceInstance(service='universe', instance='c137', bounce_by=1, watcher=self.watcher.__class__.__name__, bounce_timers=None), ServiceInstance(service='universe', instance='c139', bounce_by=1, watcher=self.watcher.__class__.__name__, bounce_timers=None) ] assert ret == expected
def test_prioritise_bouncing_services(self): with mock.patch( 'paasta_tools.deployd.master.get_service_instances_that_need_bouncing', autospec=True ) as mock_get_service_instances_that_need_bouncing, mock.patch( 'time.time', autospec=True, return_value=1): mock_changed_instances = ( x for x in {'universe.c137', 'universe.c138'}) mock_get_service_instances_that_need_bouncing.return_value = mock_changed_instances self.deployd.prioritise_bouncing_services() mock_get_service_instances_that_need_bouncing.assert_called_with( self.deployd.marathon_client, '/nail/etc/services') calls = [ mock.call( ServiceInstance(service='universe', instance='c138', watcher='DeployDaemon', bounce_by=1, bounce_timers=None, failures=0)), mock.call( ServiceInstance(service='universe', instance='c137', watcher='DeployDaemon', bounce_by=1, bounce_timers=None, failures=0)) ] self.deployd.inbox_q.put.assert_has_calls(calls, any_order=True)
def test_process_node_event(self): with mock.patch('paasta_tools.deployd.watchers.EventType', autospec=True) as mock_event_type, mock.patch( 'time.time', autospec=True, return_value=1): mock_event_other = mock_event_type.DELETED mock_event = mock.Mock( type=mock_event_other, path='/autoscaling/service/instance/instances') assert not self.mock_inbox_q.put.called mock_event_created = mock_event_type.CREATED mock_event = mock.Mock( type=mock_event_created, path='/autoscaling/service/instance/instances') self.watcher.process_node_event(mock.Mock(), mock.Mock(), mock_event) self.mock_inbox_q.put.assert_called_with( ServiceInstance(service='service', instance='instance', bounce_by=1, bounce_timers=None, watcher=self.watcher.__class__.__name__)) mock_event_changed = mock_event_type.CHANGED mock_event = mock.Mock( type=mock_event_changed, path='/autoscaling/service/instance/instances') self.watcher.process_node_event(mock.Mock(), mock.Mock(), mock_event) self.mock_inbox_q.put.assert_called_with( ServiceInstance(service='service', instance='instance', bounce_by=1, bounce_timers=None, watcher=self.watcher.__class__.__name__))
def test_get_at_risk_service_instances(self): with mock.patch( "paasta_tools.deployd.watchers.get_marathon_apps_with_clients", autospec=True, ) as mock_get_marathon_apps, mock.patch("time.time", autospec=True, return_value=1): mock_marathon_apps = [ mock.Mock(tasks=[ mock.Mock(host="host1", app_id="/universe.c137.configsha.gitsha"), mock.Mock(host="host2", app_id="/universe.c138.configsha.gitsha"), ]), mock.Mock(tasks=[ mock.Mock(host="host1", app_id="/universe.c139.configsha.gitsha") ]), mock.Mock(tasks=[ mock.Mock(host="host1", app_id="/universe.c139.configsha.gitsha") ]), ] mock_client = mock.Mock() mock_get_marathon_apps.return_value = [ (app, mock_client) for app in mock_marathon_apps ] ret = self.watcher.get_at_risk_service_instances(["host1"]) expected = [ ServiceInstance( service="universe", instance="c137", bounce_by=1, wait_until=1, watcher=self.watcher.__class__.__name__, failures=0, processed_count=0, enqueue_time=1, bounce_start_time=1, ), ServiceInstance( service="universe", instance="c139", bounce_by=1, wait_until=1, watcher=self.watcher.__class__.__name__, failures=0, processed_count=0, enqueue_time=1, bounce_start_time=1, ), ] assert ret == expected
def test___new__(self): with mock.patch( 'paasta_tools.deployd.common.ServiceInstance.get_priority', autospec=True, ): expected = BaseServiceInstance( service='universe', instance='c137', watcher='mywatcher', bounce_by=0, failures=0, bounce_timers=None, priority=1, ) assert self.service_instance == expected expected = BaseServiceInstance( service='universe', instance='c137', watcher='mywatcher', bounce_by=0, failures=0, bounce_timers=None, priority=2, ) assert ServiceInstance( service='universe', instance='c137', watcher='mywatcher', cluster='westeros-prod', bounce_by=0, priority=2, ) == expected
def get_at_risk_service_instances(self, draining_hosts): marathon_apps = get_all_marathon_apps(self.marathon_client, embed_tasks=True) at_risk_tasks = [ task for app in marathon_apps for task in app.tasks if task.host in draining_hosts ] self.log.info("At risk tasks: {}".format(at_risk_tasks)) service_instances = [] for task in at_risk_tasks: app_id = task.app_id.strip('/') service, instance, _, __ = deformat_job_id(app_id) # check we haven't already added this instance, # no need to add the same instance to the bounce queue # more than once if not any([(service, instance) == (si.service, si.instance) for si in service_instances]): service_instances.append( ServiceInstance( service=service, instance=instance, cluster=self.config.get_cluster(), bounce_by=int(time.time()), watcher=type(self).__name__, bounce_timers=None, failures=0, )) return service_instances
def bounce_service(self, service_name): self.log.info("Checking if any instances for {} need bouncing".format( service_name)) instances = list_all_instances_for_service( service=service_name, clusters=[self.filewatcher.cluster], instance_type='marathon', cache=False) self.log.debug(instances) service_instances = [(service_name, instance) for instance in instances] service_instances = get_service_instances_with_changed_id( self.marathon_client, service_instances, self.filewatcher.cluster) for service, instance in service_instances: self.log.info( "{}.{} has a new marathon app ID, and so needs bouncing". format(service, instance)) service_instances = [ ServiceInstance(service=service, instance=instance, bounce_by=int(time.time()), watcher=self.__class__.__name__, bounce_timers=None) for service, instance in service_instances ] for service_instance in service_instances: self.filewatcher.inbox_q.put(service_instance)
def test___new__(self): expected = BaseServiceInstance( service="universe", instance="c137", watcher="mywatcher", bounce_by=0, wait_until=0, failures=0, bounce_timers=None, processed_count=0, ) assert self.service_instance == expected expected = BaseServiceInstance( service="universe", instance="c137", watcher="mywatcher", bounce_by=0, wait_until=0, failures=0, bounce_timers=None, processed_count=0, ) # https://github.com/python/mypy/issues/2852 assert (ServiceInstance( # type: ignore service="universe", instance="c137", watcher="mywatcher", cluster="westeros-prod", bounce_by=0, wait_until=0, ) == expected)
def test_process_default(self): with mock.patch( "paasta_tools.deployd.watchers.PublicConfigEventHandler.filter_event", autospec=True, ) as mock_filter_event, mock.patch( "paasta_tools.deployd.watchers.PublicConfigEventHandler.watch_new_folder", autospec=True, ), mock.patch( "paasta_tools.deployd.watchers.get_services_for_cluster", autospec=True ) as mock_get_services_for_cluster, mock.patch( "paasta_tools.deployd.watchers.load_system_paasta_config", autospec=True ) as mock_load_system_config, mock.patch( "paasta_tools.deployd.watchers.get_service_instances_needing_update", autospec=True, ) as mock_get_service_instances_needing_update, mock.patch( "time.time", return_value=1.0, autospec=True): mock_event = mock.Mock() mock_filter_event.return_value = mock_event mock_load_system_config.return_value = self.mock_config self.handler.process_default(mock_event) assert mock_load_system_config.called assert not mock_get_services_for_cluster.called assert not mock_get_service_instances_needing_update.called assert not self.mock_filewatcher.instances_to_bounce.put.called mock_load_system_config.return_value = mock.Mock( get_cluster=mock.Mock()) mock_get_service_instances_needing_update.return_value = [] self.handler.process_default(mock_event) assert mock_load_system_config.called assert mock_get_services_for_cluster.called assert mock_get_service_instances_needing_update.called assert not self.mock_filewatcher.instances_to_bounce.put.called mock_load_system_config.return_value = mock.Mock( get_deployd_big_bounce_deadline=mock.Mock(return_value=100.0)) fake_si = ( "someservice", "someinstance", mock.Mock(), "someservice.someinstance.stuff.otherstuff", ) mock_get_service_instances_needing_update.return_value = [fake_si] self.handler.process_default(mock_event) assert mock_load_system_config.called assert mock_get_services_for_cluster.called assert mock_get_service_instances_needing_update.called # call objects are (name, posargs, kwargs), so this grabs the first posarg of the most recent call. assert self.mock_filewatcher.instances_to_bounce.put.mock_calls[ -1][1][0] == ServiceInstance( service="someservice", instance="someinstance", watcher="PublicConfigEventHandler", bounce_by=101.0, wait_until=1.0, enqueue_time=1.0, bounce_start_time=1.0, )
def main(): system_paasta_config = load_system_paasta_config() args = parse_args(system_paasta_config.get_deployd_startup_bounce_deadline()) service, instance = args.service_instance.split(".", 1) try: validate_service_instance( service, instance, cluster=system_paasta_config.get_cluster(), soa_dir=DEFAULT_SOA_DIR, ) except NoConfigurationForServiceError as e: paasta_print(PaastaColors.red(str(e))) sys.exit(1) service_instance = ServiceInstance( service=service, instance=instance, bounce_by=time.time() + args.bounce_by_delay_secs, wait_until=time.time(), watcher="manually_added", failures=0, enqueue_time=time.time(), bounce_start_time=time.time(), ) zk_client = KazooClient(hosts=system_paasta_config.get_zk_hosts()) zk_client.start() queue = ZKDelayDeadlineQueue(client=zk_client) queue.put(service_instance)
def bounce_service(self, service_name): self.log.info( f"Checking if any marathon instances of {service_name} need bouncing." ) instances = list_all_instances_for_service( service=service_name, clusters=[self.filewatcher.cluster], instance_type="marathon", cache=False, ) self.log.debug(instances) service_instance_configs = get_service_instances_needing_update( self.marathon_clients, [(service_name, instance) for instance in instances], self.filewatcher.cluster, ) for service, instance, config in service_instance_configs: self.log.info( f"{service}.{instance} has a new marathon app ID. Enqueuing it to be bounced." ) now = time.time() self.filewatcher.instances_to_bounce.put( ServiceInstance( # type: ignore service=service, instance=instance, cluster=self.filewatcher.cluster, bounce_by=now + config.get_bounce_start_deadline(), wait_until=now, watcher=type(self).__name__, bounce_timers=None, failures=0, ))
def get_at_risk_service_instances( self, draining_hosts: List[str]) -> List[ServiceInstance]: marathon_apps_with_clients = get_marathon_apps_with_clients( clients=self.marathon_clients.get_all_clients(), embed_tasks=True) at_risk_tasks = [] for app, client in marathon_apps_with_clients: for task in app.tasks: if task.host in draining_hosts: at_risk_tasks.append(task) self.log.info(f"At risk tasks: {at_risk_tasks}") service_instances: List[ServiceInstance] = [] for task in at_risk_tasks: app_id = task.app_id.strip("/") service, instance, _, __ = deformat_job_id(app_id) # check we haven't already added this instance, # no need to add the same instance to the bounce queue # more than once if not any([(service, instance) == (si.service, si.instance) for si in service_instances]): service_instances.append( ServiceInstance( service=service, instance=instance, bounce_by=time.time(), wait_until=time.time(), watcher=type(self).__name__, failures=0, enqueue_time=time.time(), bounce_start_time=time.time(), )) return service_instances
def bounce_service(self, service_name): self.log.info(f"Checking if any instances for {service_name} need bouncing") instances = list_all_instances_for_service( service=service_name, clusters=[self.filewatcher.cluster], instance_type='marathon', cache=False, ) self.log.debug(instances) service_instances = [(service_name, instance) for instance in instances] service_instances = get_service_instances_needing_update( self.marathon_clients, service_instances, self.filewatcher.cluster, ) for service, instance in service_instances: self.log.info(f"{service}.{instance} has a new marathon app ID, and so needs bouncing") service_instances_to_queue = [ # https://github.com/python/mypy/issues/2852 ServiceInstance( # type: ignore service=service, instance=instance, cluster=self.filewatcher.cluster, bounce_by=int(time.time()), watcher=type(self).__name__, bounce_timers=None, failures=0, ) for service, instance in service_instances ] for service_instance in service_instances_to_queue: self.filewatcher.inbox_q.put(service_instance)
def process_default(self, event): self.log.debug(event) self.watch_new_folder(event) event = self.filter_event(event) if event: self.log.info("Change of {} in {}".format(event.name, event.path)) service_name = event.path.split('/')[-1] self.log.info("Checking if any instances for {} need bouncing".format(service_name)) instances = list_all_instances_for_service(service=service_name, clusters=[self.filewatcher.cluster], instance_type='marathon') self.log.debug(instances) service_instances = [(service_name, instance) for instance in instances] service_instances = get_service_instances_with_changed_id(self.marathon_client, service_instances, self.filewatcher.cluster) for service, instance in service_instances: self.log.info("{}.{} has a new marathon app ID, and so needs bouncing".format(service, instance)) service_instances = [ServiceInstance(service=service, instance=instance, bounce_by=int(time.time()), watcher=self.__class__.__name__, bounce_timers=None) for service, instance in service_instances] for service_instance in service_instances: self.filewatcher.inbox_q.put(service_instance)
def get_at_risk_service_instances(self, draining_hosts) -> List[ServiceInstance]: marathon_apps_with_clients = get_marathon_apps_with_clients( clients=self.marathon_clients.get_all_clients(), embed_tasks=True, ) at_risk_tasks = [] for app, client in marathon_apps_with_clients: for task in app.tasks: if task.host in draining_hosts: at_risk_tasks.append(task) self.log.info(f"At risk tasks: {at_risk_tasks}") service_instances: List[ServiceInstance] = [] for task in at_risk_tasks: app_id = task.app_id.strip('/') service, instance, _, __ = deformat_job_id(app_id) # check we haven't already added this instance, # no need to add the same instance to the bounce queue # more than once if not any([(service, instance) == (si.service, si.instance) for si in service_instances]): # https://github.com/python/mypy/issues/2852 service_instances.append(ServiceInstance( # type: ignore service=service, instance=instance, cluster=self.config.get_cluster(), bounce_by=int(time.time()), watcher=type(self).__name__, bounce_timers=None, failures=0, )) return service_instances
def test_bounce_service(self): with mock.patch( 'paasta_tools.deployd.watchers.list_all_instances_for_service', autospec=True ) as mock_list_instances, mock.patch( 'paasta_tools.deployd.watchers.get_service_instances_with_changed_id', autospec=True ) as mock_get_service_instances_with_changed_id, mock.patch( 'time.time', autospec=True, return_value=1): mock_list_instances.return_value = ['c137', 'c138'] mock_get_service_instances_with_changed_id.return_value = [ ('universe', 'c137') ] self.handler.bounce_service('universe') mock_list_instances.assert_called_with( service='universe', clusters=[self.handler.filewatcher.cluster], instance_type='marathon', cache=False) mock_get_service_instances_with_changed_id.assert_called_with( self.handler.marathon_client, [('universe', 'c137'), ('universe', 'c138')], self.handler.filewatcher.cluster) expected_si = ServiceInstance(service='universe', instance='c137', bounce_by=1, watcher='YelpSoaEventHandler', bounce_timers=None, failures=0) self.mock_filewatcher.inbox_q.put.assert_called_with(expected_si) assert self.mock_filewatcher.inbox_q.put.call_count == 1
def run(self): self.log.info(f"{self.name} starting up") while True: service_instance = self.instances_to_bounce_now.get() try: bounce_again_in_seconds, return_code, bounce_timers = self.process_service_instance(service_instance) except Exception as e: self.log.error("Worker failed to process service instance and will retry. " "Caused by exception: {}".format(e)) return_code = -2 bounce_timers = service_instance.bounce_timers failures = service_instance.failures if return_code != 0: failures = service_instance.failures + 1 bounce_again_in_seconds = exponential_back_off( failures=failures, factor=self.config.get_deployd_worker_failure_backoff_factor(), base=2, max_time=6000, ) if bounce_again_in_seconds: service_instance = ServiceInstance( service=service_instance.service, instance=service_instance.instance, cluster=self.config.get_cluster(), bounce_by=int(time.time()) + bounce_again_in_seconds, watcher=self.name, bounce_timers=bounce_timers, priority=service_instance.priority, failures=failures, ) self.instances_to_bounce_later.put(service_instance) time.sleep(0.1)
def test___new__(self): with mock.patch( 'paasta_tools.deployd.common.get_priority', autospec=True, ): expected = BaseServiceInstance( service='universe', instance='c137', watcher='mywatcher', bounce_by=0, failures=0, bounce_timers=None, priority=1, ) assert self.service_instance == expected expected = BaseServiceInstance( service='universe', instance='c137', watcher='mywatcher', bounce_by=0, failures=0, bounce_timers=None, priority=2, ) # https://github.com/python/mypy/issues/2852 assert ServiceInstance( # type: ignore service='universe', instance='c137', watcher='mywatcher', cluster='westeros-prod', bounce_by=0, priority=2, ) == expected
def test_process_node_event(self): with mock.patch("paasta_tools.deployd.watchers.EventType", autospec=True) as mock_event_type, mock.patch( "time.time", autospec=True, return_value=1): mock_event_other = mock_event_type.DELETED mock_event = mock.Mock( type=mock_event_other, path="/autoscaling/service/instance/instances") assert not self.mock_instances_to_bounce.put.called mock_event_created = mock_event_type.CREATED mock_event = mock.Mock( type=mock_event_created, path="/autoscaling/service/instance/instances") self.watcher.process_node_event(mock.Mock(), mock.Mock(), mock_event) self.mock_instances_to_bounce.put.assert_called_with( ServiceInstance( service="service", instance="instance", bounce_by=1, wait_until=1, watcher=self.watcher.__class__.__name__, failures=0, processed_count=0, enqueue_time=1, bounce_start_time=1, )) mock_event_changed = mock_event_type.CHANGED mock_event = mock.Mock( type=mock_event_changed, path="/autoscaling/service/instance/instances") self.watcher.process_node_event(mock.Mock(), mock.Mock(), mock_event) self.mock_instances_to_bounce.put.assert_called_with( ServiceInstance( service="service", instance="instance", bounce_by=1, wait_until=1, watcher=self.watcher.__class__.__name__, failures=0, processed_count=0, enqueue_time=1, bounce_start_time=1, ))
def run(self): self.log.info("{} starting up".format(self.name)) while True: service_instance = self.bounce_q.get() failures = service_instance.failures bounce_timers = self.setup_timers(service_instance) self.log.info("{} processing {}.{}".format( self.name, service_instance.service, service_instance.instance)) marathon_apps = marathon_tools.get_all_marathon_apps( self.marathon_client, embed_failures=True) bounce_timers.setup_marathon.start() try: return_code, bounce_again_in_seconds = deploy_marathon_service( service=service_instance.service, instance=service_instance.instance, client=self.marathon_client, soa_dir=marathon_tools.DEFAULT_SOA_DIR, marathon_config=self.marathon_config, marathon_apps=marathon_apps) except Exception as e: self.log.warning( "deploy_marathon_service caused exception: {}".format(e)) return_code = -2 if return_code != 0: failures += 1 bounce_again_in_seconds = exponential_back_off( failures=failures, factor=self.config. get_deployd_worker_failure_backoff_factor(), base=2, max_time=6000) bounce_timers.setup_marathon.stop() self.log.info( "setup marathon completed with exit code {} for {}.{}".format( return_code, service_instance.service, service_instance.instance)) if bounce_again_in_seconds: bounce_timers.processed_by_worker.start() self.log.info( "{}.{} not in steady state so bouncing again in {} " "seconds".format(service_instance.service, service_instance.instance, bounce_again_in_seconds)) service_instance = ServiceInstance( service=service_instance.service, instance=service_instance.instance, bounce_by=int(time.time()) + bounce_again_in_seconds, watcher=self.name, bounce_timers=bounce_timers, failures=failures) self.inbox_q.put(service_instance) else: bounce_timers.bounce_length.stop() self.log.info("{}.{} in steady state".format( service_instance.service, service_instance.instance)) time.sleep(0.1)
def setUp(self): self.service_instance = ServiceInstance( # type: ignore service="universe", instance="c137", watcher="mywatcher", cluster="westeros-prod", bounce_by=0, wait_until=0, )
def test_rate_limit_instances(): with mock.patch('time.time', autospec=True) as mock_time: mock_time.return_value = 1 mock_si_1 = ('universe', 'c137') mock_si_2 = ('universe', 'c138') ret = rate_limit_instances([mock_si_1, mock_si_2], 2, "Custos") expected = [ ServiceInstance(service='universe', instance='c137', watcher='Custos', bounce_by=1, bounce_timers=None), ServiceInstance(service='universe', instance='c138', watcher='Custos', bounce_by=31, bounce_timers=None) ] assert ret == expected
def test_prioritise_bouncing_services(self): with mock.patch( "paasta_tools.deployd.master.get_service_instances_that_need_bouncing", autospec=True, ) as mock_get_service_instances_that_need_bouncing, mock.patch( "time.time", autospec=True, return_value=1): mock_changed_instances = ( x for x in {"universe.c137", "universe.c138"}) mock_get_service_instances_that_need_bouncing.return_value = ( mock_changed_instances) self.deployd.prioritise_bouncing_services() mock_get_service_instances_that_need_bouncing.assert_called_with( self.deployd.marathon_clients, "/nail/etc/services") calls = [ mock.call( ServiceInstance( service="universe", instance="c138", watcher="DeployDaemon", bounce_by=1, wait_until=1, failures=0, processed_count=0, enqueue_time=1, bounce_start_time=1, )), mock.call( ServiceInstance( service="universe", instance="c137", watcher="DeployDaemon", bounce_by=1, wait_until=1, failures=0, processed_count=0, enqueue_time=1, bounce_start_time=1, )), ] self.deployd.instances_to_bounce.put.assert_has_calls( calls, any_order=True)
def test_run(self): with mock.patch( 'time.sleep', autospec=True ) as mock_sleep, mock.patch( 'paasta_tools.deployd.workers.marathon_tools.get_all_marathon_apps', autospec=True ) as mock_get_all_marathon_apps, mock.patch( 'paasta_tools.deployd.workers.PaastaDeployWorker.setup_timers', autospec=True ) as mock_setup_timers, mock.patch( 'paasta_tools.deployd.workers.deploy_marathon_service', autospec=True ) as mock_deploy_marathon_service, mock.patch( 'time.time', autospec=True, return_value=1 ): mock_sleep.side_effect = LoopBreak self.worker.marathon_client = mock.Mock() self.worker.marathon_config = mock.Mock() mock_deploy_marathon_service.return_value = (0, None) mock_si = mock.Mock(service='universe', instance='c137') self.mock_bounce_q.get.return_value = mock_si with raises(LoopBreak): self.worker.run() mock_setup_timers.assert_called_with(self.worker, mock_si) assert mock_setup_timers.return_value.setup_marathon.start.called mock_deploy_marathon_service.assert_called_with(service='universe', instance='c137', client=self.worker.marathon_client, soa_dir=DEFAULT_SOA_DIR, marathon_config=self.worker.marathon_config, marathon_apps=mock_get_all_marathon_apps.return_value) assert mock_setup_timers.return_value.setup_marathon.stop.called assert not mock_setup_timers.return_value.processed_by_worker.start.called assert not self.mock_inbox_q.put.called assert mock_setup_timers.return_value.bounce_length.stop.called mock_deploy_marathon_service.return_value = (0, 60) mock_setup_timers.return_value.bounce_length.stop.reset_mock() with raises(LoopBreak): self.worker.run() mock_setup_timers.assert_called_with(self.worker, mock_si) assert mock_setup_timers.return_value.setup_marathon.start.called mock_deploy_marathon_service.assert_called_with(service='universe', instance='c137', client=self.worker.marathon_client, soa_dir=DEFAULT_SOA_DIR, marathon_config=self.worker.marathon_config, marathon_apps=mock_get_all_marathon_apps.return_value) assert mock_setup_timers.return_value.setup_marathon.stop.called assert mock_setup_timers.return_value.processed_by_worker.start.called self.mock_inbox_q.put.assert_called_with(ServiceInstance(service='universe', instance='c137', bounce_by=61, watcher='Worker1', bounce_timers=mock_setup_timers.return_value)) assert not mock_setup_timers.return_value.bounce_length.stop.called
def process_node_event(self, data, stat, event): self.log.debug("Node change: {}".format(event)) if event and (event.type == EventType.CREATED or event.type == EventType.CHANGED): service, instance = event.path.split('/')[-3:-1] self.log.info("Number of instances changed or autoscaling enabled for first time" " for {}.{}".format(service, instance)) service_instance = ServiceInstance(service=service, instance=instance, bounce_by=int(time.time()), bounce_timers=None, watcher=self.__class__.__name__) self.inbox_q.put(service_instance)
def run(self) -> None: """Takes things from the to_bounce_now queue, processes them, then might put them on the bounce_later queue for future processing""" self.log.info(f"{self.name} starting up") while True: with self.instances_to_bounce.get() as service_instance: self.busy = True try: ( bounce_again_in_seconds, return_code, ) = self.process_service_instance(service_instance) except Exception: self.log.error( f"{self.name} Worker failed to process service instance and will retry. " f"Caused by exception: {traceback.format_exc()}") return_code = -2 failures = service_instance.failures if return_code != 0: failures = service_instance.failures + 1 bounce_again_in_seconds = exponential_back_off( failures=failures, factor=self.config. get_deployd_worker_failure_backoff_factor(), base=2, max_time=6000, ) if bounce_again_in_seconds: if failures >= self.max_failures: self.log.info( f"{self.name} Worker removing " f"{service_instance.service}.{service_instance.instance} " f"from queue because it has failed {failures} times " f"(max is {self.max_failures})") else: bounce_by = int(time.time()) + bounce_again_in_seconds service_instance = ServiceInstance( service=service_instance.service, instance=service_instance.instance, bounce_by=bounce_by, wait_until=bounce_by, watcher=self.name, failures=failures, processed_count=service_instance.processed_count + 1, bounce_start_time=service_instance. bounce_start_time, enqueue_time=time.time(), ) self.instances_to_bounce.put(service_instance) self.busy = False time.sleep(0.1)
def _parse_data(self, entry_data: bytes) -> ServiceInstance: now = time.time() defaults = { "watcher": "unknown", "bounce_by": now, "wait_until": now, "enqueue_time": now, "bounce_start_time": now, "failures": 0, } si_dict = json.loads(entry_data.decode("utf-8")) merged = {**defaults, **si_dict} return ServiceInstance(**merged) # type: ignore
def setUp(self): with mock.patch( 'paasta_tools.deployd.common.ServiceInstance.get_priority', autospec=True, ) as mock_get_priority: mock_get_priority.return_value = 1 self.service_instance = ServiceInstance( service='universe', instance='c137', watcher='mywatcher', cluster='westeros-prod', bounce_by=0, )
def setUp(self): with mock.patch( 'paasta_tools.deployd.common.get_priority', autospec=True, ) as mock_get_priority: mock_get_priority.return_value = 1 # https://github.com/python/mypy/issues/2852 self.service_instance = ServiceInstance( # type: ignore service='universe', instance='c137', watcher='mywatcher', cluster='westeros-prod', bounce_by=0, )
def make_si(wait_until, bounce_by): """Just using mock.Mock(wait_until=wait_until, bounce_by=bounce_by) mostly works, but our PriorityQueues occasionally will compare two ServiceInstances directly, and Mocks aren't comparable unless you define an __eq__.""" return ServiceInstance( service="service", instance="instance", bounce_by=bounce_by, wait_until=wait_until, watcher="watcher", failures=0, processed_count=0, enqueue_time=1, bounce_start_time=1, )