def bounce_service(self, service_name): self.log.info(f"Checking if any instances for {service_name} need bouncing") instances = list_all_instances_for_service( service=service_name, clusters=[self.filewatcher.cluster], instance_type='marathon', cache=False, ) self.log.debug(instances) service_instances = [(service_name, instance) for instance in instances] service_instances = get_service_instances_needing_update( self.marathon_clients, service_instances, self.filewatcher.cluster, ) for service, instance in service_instances: self.log.info(f"{service}.{instance} has a new marathon app ID, and so needs bouncing") service_instances_to_queue = [ # https://github.com/python/mypy/issues/2852 ServiceInstance( # type: ignore service=service, instance=instance, cluster=self.filewatcher.cluster, bounce_by=int(time.time()), watcher=type(self).__name__, bounce_timers=None, failures=0, ) for service, instance in service_instances ] for service_instance in service_instances_to_queue: self.filewatcher.inbox_q.put(service_instance)
def bounce_service(self, service_name): self.log.info( f"Checking if any marathon instances of {service_name} need bouncing." ) instances = list_all_instances_for_service( service=service_name, clusters=[self.filewatcher.cluster], instance_type="marathon", cache=False, ) self.log.debug(instances) service_instance_configs = get_service_instances_needing_update( self.marathon_clients, [(service_name, instance) for instance in instances], self.filewatcher.cluster, ) for service, instance, config in service_instance_configs: self.log.info( f"{service}.{instance} has a new marathon app ID. Enqueuing it to be bounced." ) now = time.time() self.filewatcher.instances_to_bounce.put( ServiceInstance( # type: ignore service=service, instance=instance, cluster=self.filewatcher.cluster, bounce_by=now + config.get_bounce_start_deadline(), wait_until=now, watcher=type(self).__name__, bounce_timers=None, failures=0, ))
def bounce_service(self, service_name): self.log.info("Checking if any instances for {} need bouncing".format( service_name)) instances = list_all_instances_for_service( service=service_name, clusters=[self.filewatcher.cluster], instance_type='marathon', cache=False, ) self.log.debug(instances) service_instances = [(service_name, instance) for instance in instances] service_instances = get_service_instances_needing_update( self.marathon_client, service_instances, self.filewatcher.cluster, ) for service, instance in service_instances: self.log.info( "{}.{} has a new marathon app ID, and so needs bouncing". format(service, instance)) service_instances = [ ServiceInstance( service=service, instance=instance, cluster=self.filewatcher.cluster, bounce_by=int(time.time()), watcher=type(self).__name__, bounce_timers=None, failures=0, ) for service, instance in service_instances ] for service_instance in service_instances: self.filewatcher.inbox_q.put(service_instance)
def process_default(self, event: pyinotify.Event) -> None: self.log.debug(event) self.watch_new_folder(event) event = self.filter_event(event) if event: self.log.debug( "Public config changed on disk, loading new config.") try: new_config = load_system_paasta_config() except ValueError: self.log.error( "Couldn't load public config, the JSON is invalid!") return service_instance_configs: List[Tuple[str, str, MarathonServiceConfig, str]] = [] if new_config != self.public_config: self.log.info( "Public config has changed, now checking if it affects any services config shas." ) self.public_config = new_config all_service_instances = get_services_for_cluster( cluster=self.public_config.get_cluster(), instance_type="marathon", soa_dir=DEFAULT_SOA_DIR, ) service_instance_configs = get_service_instances_needing_update( self.marathon_clients, all_service_instances, self.public_config.get_cluster(), ) if service_instance_configs: self.log.info( f"{len(service_instance_configs)} service instances affected. Doing a staggered bounce." ) for service, instance, config, _ in service_instance_configs: self.filewatcher.instances_to_bounce.put( ServiceInstance( service=service, instance=instance, watcher=type(self).__name__, bounce_by=time.time() + self.public_config. get_deployd_big_bounce_deadline(), wait_until=time.time(), enqueue_time=time.time(), bounce_start_time=time.time(), ))
def process_default(self, event): self.log.debug(event) self.watch_new_folder(event) event = self.filter_event(event) if event: self.log.debug("Public config changed on disk, loading new config") try: new_config = load_system_paasta_config() except ValueError: self.log.error( "Couldn't load public config, the JSON is invalid!") return service_instances = [] if new_config != self.public_config: self.log.info( "Public config has changed, now checking if it affects any services config shas" ) self.public_config = new_config all_service_instances = get_services_for_cluster( cluster=self.public_config.get_cluster(), instance_type='marathon', soa_dir=DEFAULT_SOA_DIR, ) service_instances = get_service_instances_needing_update( self.marathon_client, all_service_instances, self.public_config.get_cluster(), ) if service_instances: self.log.info( "Found config change affecting {} service instances, " "now doing a staggered bounce".format( len(service_instances))) bounce_rate = self.public_config.get_deployd_big_bounce_rate() service_instances = rate_limit_instances( instances=service_instances, cluster=self.public_config.get_cluster(), number_per_minute=bounce_rate, watcher_name=type(self).__name__, priority=99, ) for service_instance in service_instances: self.filewatcher.inbox_q.put(service_instance)
def test_get_service_instances_needing_update(): with mock.patch( "paasta_tools.deployd.common.get_all_marathon_apps", autospec=True ) as mock_get_marathon_apps, mock.patch( "paasta_tools.deployd.common.load_marathon_service_config_no_cache", autospec=True, ) as mock_load_marathon_service_config: mock_marathon_apps = [ mock.Mock(id="/universe.c137.c1.g1", instances=2), mock.Mock(id="/universe.c138.c1.g1", instances=2), ] mock_get_marathon_apps.return_value = mock_marathon_apps mock_service_instances = [("universe", "c137"), ("universe", "c138")] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ "id": "universe.c137.c1.g1", "instances": 2 })), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ "id": "universe.c138.c2.g2", "instances": 2 })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, "westeros-prod") assert mock_get_marathon_apps.called calls = [ mock.call( service="universe", instance="c137", cluster="westeros-prod", soa_dir=DEFAULT_SOA_DIR, ), mock.call( service="universe", instance="c138", cluster="westeros-prod", soa_dir=DEFAULT_SOA_DIR, ), ] mock_load_marathon_service_config.assert_has_calls(calls) assert ret == [("universe", "c138", mock.ANY)] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ "id": "universe.c137.c1.g1", "instances": 3 })), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ "id": "universe.c138.c2.g2", "instances": 2 })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, "westeros-prod") assert ret == [("universe", "c137", mock.ANY), ("universe", "c138", mock.ANY)] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock( side_effect=NoDockerImageError)), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ "id": "universe.c138.c2.g2", "instances": 2 })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, "westeros-prod") assert ret == [("universe", "c138", mock.ANY)] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock( side_effect=NoSlavesAvailableError)), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ "id": "universe.c138.c2.g2", "instances": 2 })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, "westeros-prod") assert ret == [("universe", "c138", mock.ANY)] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock( side_effect=InvalidJobNameError)), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ "id": "universe.c138.c2.g2", "instances": 2 })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, "westeros-prod") assert ret == [("universe", "c138", mock.ANY)] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock( side_effect=NoDeploymentsAvailable)), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ "id": "universe.c138.c2.g2", "instances": 2 })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, "westeros-prod") assert ret == [("universe", "c138", mock.ANY)] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock( side_effect=Exception)), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ "id": "universe.c138.c2.g2", "instances": 2 })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, "westeros-prod") assert ret == [("universe", "c138", mock.ANY)]
def test_get_service_instances_needing_update(): with mock.patch( 'paasta_tools.deployd.common.get_all_marathon_apps', autospec=True, ) as mock_get_marathon_apps, mock.patch( 'paasta_tools.deployd.common.load_marathon_service_config_no_cache', autospec=True, ) as mock_load_marathon_service_config: mock_marathon_apps = [ mock.Mock(id='/universe.c137.c1.g1', instances=2), mock.Mock(id='/universe.c138.c1.g1', instances=2), ] mock_get_marathon_apps.return_value = mock_marathon_apps mock_service_instances = [('universe', 'c137'), ('universe', 'c138')] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c137.c1.g1', 'instances': 2, })), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c138.c2.g2', 'instances': 2, })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, 'westeros-prod') assert mock_get_marathon_apps.called calls = [ mock.call( service='universe', instance='c137', cluster='westeros-prod', soa_dir=DEFAULT_SOA_DIR, ), mock.call( service='universe', instance='c138', cluster='westeros-prod', soa_dir=DEFAULT_SOA_DIR, ), ] mock_load_marathon_service_config.assert_has_calls(calls) assert ret == [('universe', 'c138')] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c137.c1.g1', 'instances': 3, })), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c138.c2.g2', 'instances': 2, })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, 'westeros-prod') assert ret == [('universe', 'c137'), ('universe', 'c138')] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock( side_effect=NoDockerImageError)), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c138.c2.g2', 'instances': 2, })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, 'westeros-prod') assert ret == [('universe', 'c138')] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock( side_effect=InvalidJobNameError)), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c138.c2.g2', 'instances': 2, })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, 'westeros-prod') assert ret == [('universe', 'c138')] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock( side_effect=NoDeploymentsAvailable)), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c138.c2.g2', 'instances': 2, })), ] mock_load_marathon_service_config.side_effect = mock_configs mock_client = mock.Mock(servers=["foo"]) fake_clients = MarathonClients(current=[mock_client], previous=[mock_client]) ret = get_service_instances_needing_update(fake_clients, mock_service_instances, 'westeros-prod') assert ret == [('universe', 'c138')]
def test_get_service_instances_needing_update(): with mock.patch( 'paasta_tools.deployd.common.get_all_marathon_apps', autospec=True ) as mock_get_marathon_apps, mock.patch( 'paasta_tools.deployd.common.load_marathon_service_config_no_cache', autospec=True) as mock_load_marathon_service_config: mock_marathon_apps = [ mock.Mock(id='/universe.c137.c1.g1', instances=2), mock.Mock(id='/universe.c138.c1.g1', instances=2) ] mock_get_marathon_apps.return_value = mock_marathon_apps mock_service_instances = [('universe', 'c137'), ('universe', 'c138')] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c137.c1.g1', 'instances': 2 })), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c138.c2.g2', 'instances': 2 })) ] mock_load_marathon_service_config.side_effect = mock_configs ret = get_service_instances_needing_update(mock.Mock(), mock_service_instances, 'westeros-prod') assert mock_get_marathon_apps.called calls = [ mock.call(service='universe', instance='c137', cluster='westeros-prod', soa_dir=DEFAULT_SOA_DIR), mock.call(service='universe', instance='c138', cluster='westeros-prod', soa_dir=DEFAULT_SOA_DIR) ] mock_load_marathon_service_config.assert_has_calls(calls) assert ret == [('universe', 'c138')] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c137.c1.g1', 'instances': 3 })), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c138.c2.g2', 'instances': 2 })) ] mock_load_marathon_service_config.side_effect = mock_configs ret = get_service_instances_needing_update(mock.Mock(), mock_service_instances, 'westeros-prod') assert ret == [('universe', 'c137'), ('universe', 'c138')] mock_configs = [ mock.Mock(format_marathon_app_dict=mock.Mock( side_effect=NoDockerImageError)), mock.Mock(format_marathon_app_dict=mock.Mock(return_value={ 'id': 'universe.c138.c2.g2', 'instances': 2 })) ] mock_load_marathon_service_config.side_effect = mock_configs ret = get_service_instances_needing_update(mock.Mock(), mock_service_instances, 'westeros-prod') assert ret == [('universe', 'c137'), ('universe', 'c138')]