def test_read_load_balancing(self, *args): reporting_dbs = { 'ucr': { 'WRITE': 'ucr', 'READ': [('ucr', 8), ('other', 1), ('default', 1)] }, } with override_settings(REPORTING_DATABASES=reporting_dbs): manager = ConnectionManager() self.assertEqual(manager.db_connection_map, { 'default': 'postgresql+psycopg2://:@localhost:5432/default', 'ucr': 'postgresql+psycopg2://:@localhost:5432/ucr', 'other': 'postgresql+psycopg2://:@localhost:5432/other' }) # test that load balancing works with a 10% margin for randomness total_requests = 10000 randomness_margin = total_requests * 0.1 total_weighting = sum(db[1] for db in reporting_dbs['ucr']['READ']) expected = { alias: weight * total_requests // total_weighting for alias, weight in reporting_dbs['ucr']['READ'] } balanced = Counter(manager.get_load_balanced_read_db_alais('ucr') for i in range(total_requests)) for db, requests in balanced.items(): self.assertAlmostEqual(requests, expected[db], delta=randomness_margin) with override_settings(REPORTING_DATABASES={'default': 'default'}): manager = ConnectionManager() self.assertEqual( ['default', 'default', 'default'], [manager.get_load_balanced_read_db_alais('default') for i in range(3)] )
def test_read_load_balancing_session(self, *args): reporting_dbs = { 'ucr': { 'WRITE': 'ucr', 'READ': [('ucr', 8), ('other', 1), ('default', 1)] }, } with override_settings(REPORTING_DATABASES=reporting_dbs): manager = ConnectionManager() self.assertEqual(manager.engine_id_django_db_map, { 'default': 'default', 'ucr': 'ucr', }) urls = { manager.get_connection_string(alias) for alias, _ in reporting_dbs['ucr']['READ'] } self.assertEqual(len(urls), 3) # withing 50 iterations we should have seen all 3 databases at least once for i in range(50): url = manager.get_session_helper('ucr', readonly=True).url if url in urls: urls.remove(url) if not urls: break if urls: self.fail(f'DBs skipped in load balancing: {urls}')
def test_load_balanced_read_apps(self, mock): load_balanced_apps = { 'users': [ ('users_db1', 5), ] } with override_settings( LOAD_BALANCED_APPS=load_balanced_apps, DATABASES = { 'default': _get_db_config('default'), 'users_db1': _get_db_config('users_db1')}): manager = ConnectionManager() self.assertEqual( manager.get_load_balanced_read_db_alais('users', default="default_option"), 'users_db1' ) with override_settings(LOAD_BALANCED_APPS=load_balanced_apps): # load_balanced_db should be part of settings.DATABASES with self.assertRaises(AssertionError): ConnectionManager().get_load_balanced_read_db_alais('users') # If `LOAD_BALANCED_APPS` is not set for an app, it should point to default kwarg manager = ConnectionManager() self.assertEqual( manager.get_load_balanced_read_db_alais('users', default='default_option'), 'default_option' )
def test_standby_filtering(self, *args): reporting_dbs = { 'ucr_engine': { 'WRITE': 'ucr', 'READ': [('ucr', 8), ('other', 1)] }, } with override_settings(REPORTING_DATABASES=reporting_dbs): # should always return the `ucr` db since `other` has bad replication delay manager = ConnectionManager() self.assertEqual(['ucr', 'ucr', 'ucr'], [ manager.get_load_balanced_read_db_alias('ucr_engine') for i in range(3) ])
def test_standby_filtering(self, *args): reporting_dbs = { 'ucr_engine': { 'WRITE': 'ucr', 'READ': [('ucr', 8), ('other', 1)] }, } with override_settings(REPORTING_DATABASES=reporting_dbs): # should always return the `other` db since `ucr` has bad replication delay manager = ConnectionManager() self.assertEqual( ['other', 'other', 'other'], [manager.get_load_balanced_read_db_alais('ucr_engine') for i in range(3)] )
def test_new_settings(self): manager = ConnectionManager() self.assertEqual(manager.engine_id_django_db_map, { 'default': 'default', 'ucr': 'ucr', 'other': 'other' })
def test_new_settings(self): manager = ConnectionManager() self.assertEqual(manager.db_connection_map, { 'default': 'postgresql+psycopg2://:@localhost:5432/default', 'ucr': 'postgresql+psycopg2://:@localhost:5432/ucr', 'other': 'postgresql+psycopg2://:@localhost:5432/other' })
def test_load_balanced_read_apps(self, mock): load_balanced_apps = { 'users': [ ('users_db1', 5), ] } with override_settings(LOAD_BALANCED_APPS=load_balanced_apps, DATABASES={ 'default': _get_db_config('default'), 'users_db1': _get_db_config('users_db1') }): manager = ConnectionManager() self.assertEqual( manager.get_load_balanced_read_db_alias( 'users', default="default_option"), 'users_db1') with override_settings(LOAD_BALANCED_APPS=load_balanced_apps): # load_balanced_db should be part of settings.DATABASES with self.assertRaises(AssertionError): ConnectionManager().get_load_balanced_read_db_alias('users') # If `LOAD_BALANCED_APPS` is not set for an app, it should point to default kwarg manager = ConnectionManager() self.assertEqual( manager.get_load_balanced_read_db_alias('users', default='default_option'), 'default_option')
def test_read_load_balancing(self): reporting_dbs = { 'ucr': { 'WRITE': 'ucr', 'READ': [('ucr', 8), ('other', 1), ('default', 1)] }, } with override_settings(REPORTING_DATABASES=reporting_dbs): manager = ConnectionManager() self.assertEqual(manager.db_connection_map, { 'default': 'postgresql+psycopg2://:@localhost:5432/default', 'ucr': 'postgresql+psycopg2://:@localhost:5432/ucr', 'other': 'postgresql+psycopg2://:@localhost:5432/other' }) # test that load balancing works with a 10% margin for randomness total_requests = 10000 randomness_margin = total_requests * 0.1 total_weighting = sum(db[1] for db in reporting_dbs['ucr']['READ']) expected = { alias: weight * total_requests // total_weighting for alias, weight in reporting_dbs['ucr']['READ'] } balanced = Counter(manager.get_load_balanced_read_engine_id('ucr') for i in range(total_requests)) for db, requests in balanced.items(): self.assertAlmostEqual(requests, expected[db], delta=randomness_margin) with override_settings(REPORTING_DATABASES={'default': 'default'}): manager = ConnectionManager() self.assertEqual( ['default', 'default', 'default'], [manager.get_load_balanced_read_engine_id('default') for i in range(3)] )
def test_read_load_balancing(self, *args): reporting_dbs = { 'ucr': { 'WRITE': 'ucr', 'READ': [('ucr', 8), ('other', 1), ('default', 1)] }, } with override_settings(REPORTING_DATABASES=reporting_dbs): manager = ConnectionManager() self.assertEqual(manager.engine_id_django_db_map, { 'default': 'default', 'ucr': 'ucr', }) # test that load balancing works with a 10% margin for randomness total_requests = 10000 randomness_margin = total_requests * 0.1 total_weighting = sum(db[1] for db in reporting_dbs['ucr']['READ']) expected = { alias: weight * total_requests // total_weighting for alias, weight in reporting_dbs['ucr']['READ'] } balanced = Counter(manager.get_load_balanced_read_db_alias('ucr') for i in range(total_requests)) for db, requests in balanced.items(): self.assertAlmostEqual(requests, expected[db], delta=randomness_margin) with override_settings(REPORTING_DATABASES={'default': DEFAULT_DB_ALIAS}): manager = ConnectionManager() self.assertEqual( [DEFAULT_DB_ALIAS] * 3, [manager.get_load_balanced_read_db_alias(DEFAULT_DB_ALIAS) for i in range(3)] )
def check_for_ucr_tables_without_existing_domain(): all_domain_names = Domain.get_all_names() connection_name = ConnectionManager().get_django_db_alias(UCR_ENGINE_ID) table_names = connections[connection_name].introspection.table_names() ucr_table_names = [name for name in table_names if name.startswith('config_report')] missing_domains_to_tables = defaultdict(list) for ucr_table_name in ucr_table_names: table_domain = ucr_table_name.split('_')[2] if table_domain not in all_domain_names: missing_domains_to_tables[table_domain].append(ucr_table_name) if missing_domains_to_tables: for missing_domain in missing_domains_to_tables: mail_admins_async.delay( 'Missing domain "%s" has remaining UCR tables' % missing_domain, six.text_type(missing_domains_to_tables[missing_domain]) ) elif _is_monday(): mail_admins_async.delay('All UCR tables belong to valid domains', '')