def test_clean_success(mocker, mock_switches): tree_hub = TreeHub(mocker.Mock()) tree_holder = mocker.MagicMock() cleaner = TreeHolderCleaner(tree_hub) cleaner._old_offset = 0 application_name = 'foo.test' type_name = 'config' mock_switches({ SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK: True, SWITCH_ENABLE_TREE_HOLDER_CLEANER_CLEAN: True, }) tree_hub.tree_map[(application_name, type_name)] = tree_holder tree_hub.tree_map[(application_name, 'switch')] = mocker.MagicMock() tree_hub.tree_map[(application_name, 'service')] = mocker.MagicMock() cleaner.track(application_name, type_name) cleaner.track(application_name, 'service') with freeze_time(datetime.datetime.now() + datetime.timedelta(days=1)): cleaner.track(application_name, 'switch') items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 3 assert (application_name, type_name) in tree_hub.tree_map assert (application_name, 'switch') in tree_hub.tree_map assert (application_name, 'service') in tree_hub.tree_map mocker.patch.object(settings, 'TREE_HOLDER_CLEANER_CONDITION', 'True') cleaner.clean() items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 1 assert (application_name, type_name) not in tree_hub.tree_map assert (application_name, 'switch') in tree_hub.tree_map assert (application_name, 'service') not in tree_hub.tree_map assert tree_holder.close.called
def test_clean_thread(mocker, mock_switches): mocker.patch.object(settings, 'TREE_HOLDER_CLEANER_CONDITION', 'True') mocker.patch.object(settings, 'TREE_HOLDER_CLEANER_OLD_OFFSET', 0) mocker.patch.object(settings, 'TREE_HOLDER_CLEANER_PERIOD', 1) mock_switches({ SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK: True, SWITCH_ENABLE_TREE_HOLDER_CLEANER_CLEAN: True, }) application_name = 'foo.test' type_name = 'config' tree_holder = mocker.MagicMock() tree_hub = TreeHub(mocker.Mock()) tree_hub.tree_map[(application_name, type_name)] = tree_holder tree_hub.tree_map[(application_name, 'switch')] = mocker.MagicMock() tree_hub.tree_map[(application_name, 'service')] = mocker.MagicMock() cleaner = TreeHolderCleaner(tree_hub) cleaner.spawn_cleaning_thread() gevent.sleep(0.1) cleaner.track(application_name, type_name) items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 1 assert (application_name, type_name) in tree_hub.tree_map gevent.sleep(2) items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 0 assert (application_name, type_name) not in tree_hub.tree_map cleaner._stopped.set() cleaner.spawn_cleaning_thread() gevent.sleep(0.1)
def test_start(test_application_name): hub = TreeHub(huskar_client) holder = hub.get_tree_holder(test_application_name, 'config') # It is okay to get it again from holder assert hub.get_tree_holder(test_application_name, 'config') is holder # It will be initialized finally holder.block_until_initialized(5)
def start_concurrently(startup_max_concurrency): hub = TreeHub(huskar_client, startup_max_concurrency) hub.tree_holder_class = RoughTreeHolder execution_order = [] def handle_request(seq): application_name = '%s.%d' % (prefix, seq) holder = hub.get_tree_holder(application_name, 'config') holder.block_until_initialized(5) execution_order.append(seq) hub.release_tree_holder(application_name, 'config') joinall([spawn(handle_request, i) for i in range(5)]) return execution_order, hub.throttle
def test_track_failed_or_skipped(mocker, mock_switches): cleaner = TreeHolderCleaner(TreeHub(mocker.Mock())) application_name = 'foo.test' type_name = 'config' # switch off, skip mock_switches({ SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK: False, }) cleaner.track(application_name, type_name) items = redis_client.zrange(REDIS_KEY, 0, -1, withscores=True) assert not items # exception, ignore error mock_switches({ SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK: True, }) mocker.patch.object(redis_client, 'zadd', side_effect=Exception()) logger = mocker.patch('huskar_api.models.tree.cleaner.logger') cleaner.track(application_name, type_name) items = redis_client.zrange(REDIS_KEY, 0, -1, withscores=True) assert not items assert logger.warning.called call_args = logger.warning.call_args_list[0][0] assert call_args[0] == 'tree holder cleaner track item failed: %s'
def test_start_timeout_with_throttle(test_application_name, zk): path = '/huskar/config/%s/overall/TEST_KEY' % test_application_name zk.create(path, value=b'{}', makepath=True) condition = Event() class ConditionTreeHolder(TreeHolder): def dispatch_signal(self, event): if event.event_type == TreeEvent.INITIALIZED: condition.wait() return TreeHolder.dispatch_signal(self, event) hub = TreeHub(huskar_client, 1) hub.tree_holder_class = ConditionTreeHolder assert not hub.throttle.locked() holder = hub.get_tree_holder(test_application_name, 'config') assert hub.throttle.locked() with raises(TreeTimeoutError): holder.block_until_initialized(1) assert not hub.throttle.locked() hub.release_tree_holder(test_application_name, 'config') assert not hub.throttle.locked() condition.set() assert holder.initialized.wait(1) assert holder._started assert holder._closed assert_semaphore_is_zero(hub.throttle, 1)
def test_track_success(mocker, mock_switches): mock_switches({ SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK: True, }) cleaner = TreeHolderCleaner(TreeHub(mocker.Mock())) application_name = 'foo.test' type_name = 'config' n = time.time() cleaner.track(application_name, type_name) name = '{}:{}'.format(application_name, type_name) items = redis_client.zrange(REDIS_KEY, 0, -1, withscores=True) assert len(items) == 1 assert items[0][0] == name assert n < items[0][1] < time.time()
def test_clean_old_redis_data(mocker, mock_switches): mocker.patch.object(settings, 'TREE_HOLDER_CLEANER_CONDITION', 'memory > 0') mocker.patch.object(settings, 'TREE_HOLDER_CLEANER_OLD_OFFSET', 2) tree_hub = TreeHub(mocker.Mock()) cleaner = TreeHolderCleaner(tree_hub) application_name = 'foo.test' mock_switches({ SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK: True, SWITCH_ENABLE_TREE_HOLDER_CLEANER_CLEAN: True, }) now = datetime.datetime.now() cleaner.track(application_name, 'config') with freeze_time(now + datetime.timedelta(days=3)): cleaner.track(application_name, 'switch') with freeze_time(now + datetime.timedelta(days=-3)): cleaner.track(application_name, 'service') with freeze_time(now + datetime.timedelta(days=-4)): cleaner.track(application_name, 'foo') with freeze_time(now + datetime.timedelta(days=-8)): cleaner.track(application_name, '233') with freeze_time(now + datetime.timedelta(days=-10)): cleaner.track(application_name, '666') items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 6 cleaner.clean() items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 4 assert set(items) == set([ '{}:config'.format(application_name), '{}:switch'.format(application_name), '{}:service'.format(application_name), '{}:foo'.format(application_name), ]) # ignore error mocker.patch.object(redis_client, 'zremrangebyscore', side_effect=Exception()) logger = mocker.patch('huskar_api.models.tree.cleaner.logger') cleaner.clean() assert logger.warning.called call_args = logger.warning.call_args_list[0][0] assert call_args[0] == 'clean tree holder cleaner old data failed: %s'
from huskar_api import settings from huskar_api.extras.raven import capture_exception from huskar_api.extras.concurrent_limiter import release_after_iterator_end from huskar_api.models import huskar_client from huskar_api.models.auth import Authority from huskar_api.models.route import RouteManagement from huskar_api.models.route.hijack import RouteHijack from huskar_api.models.tree import TreeHub, TreeHolderCleaner from huskar_api.service.admin.application_auth import (check_application, check_application_auth) from huskar_api.service.utils import check_cluster_name from huskar_api.switch import (switch, SWITCH_ENABLE_DECLARE_UPSTREAM) from .utils import login_required, get_life_span from .schema import event_subscribe_schema tree_hub = TreeHub(huskar_client, settings.TREE_HOLDER_STARTUP_MAX_CONCURRENCY) tree_holder_cleaner = TreeHolderCleaner(tree_hub) tree_holder_cleaner.spawn_cleaning_thread() class LongPollingView(MethodView): @login_required def post(self): """Subscribes changes of service, switch or config via HTTP long polling connection. This API accepts requests in following schema:: { "service": { # "service" / "switch" / "config" "base.foo": [ # the application name
def hub(): return TreeHub(huskar_client)
def test_clean_failed_or_skipped(mocker, mock_switches): tree_hub = TreeHub(mocker.Mock()) tree_holder = mocker.MagicMock() cleaner = TreeHolderCleaner(tree_hub) cleaner._old_offset = 0 application_name = 'foo.test' type_name = 'config' mock_switches({ SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK: False, SWITCH_ENABLE_TREE_HOLDER_CLEANER_CLEAN: True, }) tree_hub.tree_map[(application_name, type_name)] = tree_holder tree_hub.tree_map[(application_name, 'switch')] = mocker.MagicMock() tree_hub.tree_map[(application_name, 'service')] = mocker.MagicMock() cleaner.track(application_name, type_name) cleaner.track(application_name, 'service') cleaner.track(application_name, 'switch') # track switch off, skip cleaner.clean() items = redis_client.zrange(REDIS_KEY, 0, -1) assert not items # clean switch off, skip mock_switches({ SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK: True, SWITCH_ENABLE_TREE_HOLDER_CLEANER_CLEAN: False, }) cleaner.track(application_name, type_name) cleaner.track(application_name, 'service') with freeze_time(datetime.datetime.now() + datetime.timedelta(days=1)): cleaner.track(application_name, 'switch') cleaner.clean() items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 3 assert len(tree_hub.tree_map) == 3 # condition empty, skip mock_switches({ SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK: True, SWITCH_ENABLE_TREE_HOLDER_CLEANER_CLEAN: True, }) mocker.patch.object(settings, 'TREE_HOLDER_CLEANER_CONDITION', '') cleaner.clean() items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 3 assert len(tree_hub.tree_map) == 3 # condition false, skip mocker.patch.object(settings, 'TREE_HOLDER_CLEANER_CONDITION', 'cpu < 0') cleaner.clean() items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 3 assert len(tree_hub.tree_map) == 3 # condition invalid, skip mocker.patch.object(settings, 'TREE_HOLDER_CLEANER_CONDITION', 'a') logger = mocker.patch('huskar_api.models.tree.cleaner.logger') cleaner.clean() items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 3 assert len(tree_hub.tree_map) == 3 assert logger.error.called call_args = logger.error.call_args_list[0][0] assert call_args[0] == 'invalid tree holder cleaner condition: %r %s' logger.reset_mock() # get redis data failed, skip mocker.patch.object(settings, 'TREE_HOLDER_CLEANER_CONDITION', 'True') logger = mocker.patch('huskar_api.models.tree.cleaner.logger') orig_zrangebyscore = redis_client.zrangebyscore mocker.patch.object(redis_client, 'zrangebyscore', side_effect=Exception()) cleaner.clean() mocker.patch.object(redis_client, 'zrangebyscore', orig_zrangebyscore) items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 3 assert len(tree_hub.tree_map) == 3 assert logger.warning.called call_args = logger.warning.call_args_list[0][0] assert call_args[0] == 'get tree holder cleaner data failed: %s' logger.reset_mock() tree_holder.close.reset_mock() cleaner.clean() tree_holder.close.reset_mock() cleaner.track(application_name, type_name) cleaner.clean() items = redis_client.zrange(REDIS_KEY, 0, -1) assert len(items) == 1 assert len(tree_hub.tree_map) == 1 assert (application_name, type_name) not in tree_hub.tree_map assert (application_name, 'service') not in tree_hub.tree_map assert not tree_holder.close.called