def test_sharded_replica_database(self):
simple_config = {
'sharded_databases': [{
'name':
'DB01',
'environment_variable':
'SOME_OTHER_USELESS_ENV',
'default_database_url':
self.default_database_url,
'shard_group':
'testing',
'replicas': [{
'name': 'DB01_replica',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
}]
}]
}
result = database_configs(simple_config)
DB01 = {'SHARD_GROUP': 'testing', 'TEST': {}}
DB01.update(self.dj_database_config)
DB01_replica = {
'SHARD_GROUP': 'testing',
'PRIMARY': 'DB01',
'TEST': {
'MIRROR': 'DB01'
}
}
DB01_replica.update(self.dj_database_config)
self.assertEqual(result, {'DB01': DB01, 'DB01_replica': DB01_replica})
def test_sharded_databases_shard_id(self):
simple_config = {
'sharded_databases': [
{
'name': 'DB01',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'shard_group': 'testing'
},
{
'name': 'DB02',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'shard_group': 'testing'
},
{
'name': 'DB03',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'shard_group': 'default'
},
{
'name': 'DB04',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'shard_group': 'default'
},
{
'name': 'DB05',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'shard_group': 'default'
}
]
}
result = database_configs(simple_config)
DB01 = {'SHARD_GROUP': 'testing', 'TEST': {}, 'SHARD_ID': 0}
DB02 = {'SHARD_GROUP': 'testing', 'TEST': {}, 'SHARD_ID': 1}
DB03 = {'SHARD_GROUP': 'default', 'TEST': {}, 'SHARD_ID': 0}
DB04 = {'SHARD_GROUP': 'default', 'TEST': {}, 'SHARD_ID': 1}
DB05 = {'SHARD_GROUP': 'default', 'TEST': {}, 'SHARD_ID': 2}
DB01.update(self.dj_database_config)
DB02.update(self.dj_database_config)
DB03.update(self.dj_database_config)
DB04.update(self.dj_database_config)
DB05.update(self.dj_database_config)
self.assertEqual(result['DB01'], DB01)
self.assertEqual(result['DB02'], DB02)
self.assertEqual(result['DB03'], DB03)
self.assertEqual(result['DB04'], DB04)
self.assertEqual(result['DB05'], DB05)
def test_skips_databases_with_no_envvar_value_or_deafult_url(self):
simple_config = {
'unsharded_databases': [{
'name': 'DB01',
'environment_variable': 'ENV',
'default_database_url': ''
}]
}
result = database_configs(simple_config)
expected_result = {}
self.assertEqual(result, expected_result)
def test_sharded_databases_shard_id(self):
simple_config = {
'sharded_databases': [
{
'name': 'DB01',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'shard_group': 'testing'
},
{
'name': 'DB02',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'shard_group': 'testing'
},
{
'name': 'DB03',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'shard_group': 'default'
},
{
'name': 'DB04',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'shard_group': 'default'
},
{
'name': 'DB05',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'shard_group': 'default'
}
]
}
result = database_configs(simple_config)
DB01 = {'SHARD_GROUP': 'testing', 'TEST': {}, 'SHARD_ID': 0}
DB02 = {'SHARD_GROUP': 'testing', 'TEST': {}, 'SHARD_ID': 1}
DB03 = {'SHARD_GROUP': 'default', 'TEST': {}, 'SHARD_ID': 0}
DB04 = {'SHARD_GROUP': 'default', 'TEST': {}, 'SHARD_ID': 1}
DB05 = {'SHARD_GROUP': 'default', 'TEST': {}, 'SHARD_ID': 2}
DB01.update(self.db_config)
DB02.update(self.db_config)
DB03.update(self.db_config)
DB04.update(self.db_config)
DB05.update(self.db_config)
self.assertEqual(result['DB01'], DB01)
self.assertEqual(result['DB02'], DB02)
self.assertEqual(result['DB03'], DB03)
self.assertEqual(result['DB04'], DB04)
self.assertEqual(result['DB05'], DB05)
def test_skips_databases_with_no_envvar_value_or_deafult_url(self):
simple_config = {
'unsharded_databases': [
{
'name': 'DB01',
'environment_variable': 'ENV',
'default_database_url': ''
}
]
}
result = database_configs(simple_config)
expected_result = {}
self.assertEqual(result, expected_result)
def test_sharded_databases_shard_group(self):
simple_config = {
'sharded_databases': [{
'name': 'DB01',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'shard_group': 'testing'
}]
}
result = database_configs(simple_config)
DB01 = {'SHARD_GROUP': 'testing', 'TEST': {}}
DB01.update(self.dj_database_config)
self.assertEqual(result, {'DB01': DB01})
def test_sharded_databases_default_shard_group(self):
simple_config = {
'sharded_databases': [
{
'name': 'DB01',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
}
]
}
result = database_configs(simple_config)
DB01 = {'SHARD_GROUP': 'default', 'TEST': {}, 'SHARD_ID': 0}
DB01.update(self.dj_database_config)
self.assertEqual(result, {'DB01': DB01})
def test_databases_with_default_url_unset_env(self):
simple_config = {
'unsharded_databases': [
{
'name': 'DB01',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url
}
]
}
result = database_configs(simple_config)
DB01 = {'SHARD_GROUP': None, 'TEST': {}}
DB01.update(self.dj_database_config)
self.assertEqual(result, {'DB01': DB01})
def test_database_name_overrides_url(self):
simple_config = {
'unsharded_databases': [{
'name': 'DB01',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'database_name': 'another_db'
}]
}
result = database_configs(simple_config)
DB01 = {'SHARD_GROUP': None, 'TEST': {}}
DB01.update(self.dj_database_config)
DB01['NAME'] = 'another_db'
self.assertEqual(result, {'DB01': DB01})
def test_databases_with_default_url_unset_env(self):
simple_config = {
'unsharded_databases': [
{
'name': 'DB01',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url
}
]
}
result = database_configs(simple_config)
DB01 = {'SHARD_GROUP': None, 'TEST': {}}
DB01.update(self.db_config)
self.assertEqual(result, {'DB01': DB01})
def test_databases_with_envvar_value_and_no_default_url(self):
import os
os.environ['SOME_USELESS_ENV'] = self.default_database_url
simple_config = {
'unsharded_databases': [{
'name': 'DB01',
'environment_variable': 'SOME_USELESS_ENV',
'default_database_url': ''
}]
}
result = database_configs(simple_config)
del os.environ['SOME_USELESS_ENV']
DB01 = {'SHARD_GROUP': None, 'TEST': {}}
DB01.update(self.dj_database_config)
self.assertEqual(result, {'DB01': DB01})
def test_databases_with_envvar_value_and_no_default_url(self):
import os
os.environ['SOME_USELESS_ENV'] = self.default_database_url
simple_config = {
'unsharded_databases': [
{
'name': 'DB01',
'environment_variable': 'SOME_USELESS_ENV',
'default_database_url': ''
}
]
}
result = database_configs(simple_config)
del os.environ['SOME_USELESS_ENV']
DB01 = {'SHARD_GROUP': None, 'TEST': {}}
DB01.update(self.dj_database_config)
self.assertEqual(result, {'DB01': DB01})
def test_unsharded_replica_database(self):
simple_config = {
'unsharded_databases': [
{
'name': 'DB01',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
'replicas': [{
'name': 'DB01_replica',
'environment_variable': 'SOME_OTHER_USELESS_ENV',
'default_database_url': self.default_database_url,
}]
}
]
}
result = database_configs(simple_config)
DB01 = {'SHARD_GROUP': None, 'TEST': {}}
DB01.update(self.dj_database_config)
DB01_replica = {'SHARD_GROUP': None, 'PRIMARY': 'DB01', 'TEST': {'MIRROR': 'DB01'}}
DB01_replica.update(self.dj_database_config)
self.assertEqual(result, {'DB01': DB01, 'DB01_replica': DB01_replica})
DATABASES = database_configs(databases_dict={
'unsharded_databases': [
{
'name': 'default',
'environment_variable': 'DATABASE_URL',
'default_database_url': 'postgres://*****:*****@localhost/sharding'
}
],
'sharded_databases': [
{
'name': 'app_shard_001',
'environment_variable': 'SHARD_001_DATABASE_URL',
'default_database_url': 'postgres://*****:*****@localhost/sharding_001',
'replicas': [
{
'name': 'app_shard_001_replica_001',
'environment_variable': 'REPLICA_001_DATABASE_URL',
'default_database_url': 'postgres://*****:*****@localhost/sharding_replica_001'
},
{
'name': 'app_shard_001_replica_002',
'environment_variable': 'REPLICA_002_DATABASE_URL',
'default_database_url': 'postgres://*****:*****@localhost/sharding_replica_002'
},
]
},
{
'name': 'app_shard_002',
'environment_variable': 'SHARD_002_DATABASE_URL',
'default_database_url': 'mysql://*****:*****@localhost/sharding_002'
},
]
})
def pytest_configure():
from datetime import datetime
import time
try:
import django
from django.conf import settings
from django_sharding_library.settings_helpers import database_configs
except ImportError:
import traceback
traceback.print_exc()
raise ImportError(
"To fix this error, run: pip install -r requirements-test.txt")
DATABASES = database_configs(
databases_dict={
'unsharded_databases':
[{
'name': 'default',
'environment_variable': 'DATABASE_URL',
'default_database_url': 'sqlite://testing123'
}],
'sharded_databases': [
{
'name':
'app_shard_001',
'environment_variable':
'SHARD_001_DATABASE_URL',
'default_database_url':
'sqlite://testing124',
'replicas': [
{
'name': 'app_shard_001_replica_001',
'environment_variable': 'REPLICA_001_DATABASE_URL',
'default_database_url': 'sqlite://testing125'
},
{
'name': 'app_shard_001_replica_002',
'environment_variable': 'REPLICA_002_DATABASE_URL',
'default_database_url': 'sqlite://testing126'
},
]
},
{
'name': 'app_shard_002',
'environment_variable': 'SHARD_002_DATABASE_URL',
'default_database_url': 'sqlite://testing127'
},
{
'name': 'app_shard_003',
'shard_group': 'postgres',
'environment_variable': 'SHARD_003_DATABASE_URL',
'default_database_url': 'sqlite://testing125'
},
{
'name': 'app_shard_004',
'shard_group': 'postgres',
'environment_variable': 'SHARD_004_DATABASE_URL',
'default_database_url': 'sqlite://testing125'
},
]
})
settings.configure(
DEBUG=True,
USE_TZ=True,
DATABASES=DATABASES,
DATABASE_ROUTERS=['django_sharding_library.router.ShardedRouter'],
AUTH_USER_MODEL='tests.User',
INSTALLED_APPS=[
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sites",
"django_sharding",
"django_nose",
"tests",
],
SITE_ID=1,
MIDDLEWARE_CLASSES=(),
SHARD_EPOCH=int(time.mktime(datetime(2016, 1, 1).timetuple()) * 1000),
)
django.setup()
DATABASES = database_configs(
databases_dict={
'unsharded_databases': [{
'name': 'default',
'environment_variable': 'DATABASE_URL',
'default_database_url': 'sqlite://./default.sqlite3',
'shard_group': 'system'
}],
'sharded_databases': [
{
'name':
'shard_001',
'environment_variable':
'SHARD_001_DATABASE_URL',
'default_database_url':
'sqlite://./sharding.sqlite3',
'shard_group':
'default',
'replicas': [
{
'name':
'shard_001_replica_001',
'environment_variable':
'REPLICA_001_DATABASE_URL',
'default_database_url':
'sqlite://./sharding_1_replica_1.sqlite3',
},
{
'name':
'shard_001_replica_002',
'environment_variable':
'REPLICA_002_DATABASE_URL',
'default_database_url':
'sqlite://./sharding_1_replica_2.sqlite3',
},
]
},
{
'name': 'shard_002',
'environment_variable': 'SHARD_002_DATABASE_URL',
'default_database_url': 'sqlite://./sharding_2.sqlite3',
'shard_group': 'default'
},
]
})
DATABASES = database_configs(databases_dict={
'unsharded_databases': [
{
'name': 'default',
'environment_variable': 'DATABASE_URL',
'default_database_url': 'sqlite://testing123'
}
],
'sharded_databases': [
{
'name': 'app_shard_001',
'environment_variable': 'SHARD_001_DATABASE_URL',
'default_database_url': 'sqlite://testing124',
'replicas': [
{
'name': 'app_shard_001_replica_001',
'environment_variable': 'REPLICA_001_DATABASE_URL',
'default_database_url': 'sqlite://testing125'
},
{
'name': 'app_shard_001_replica_002',
'environment_variable': 'REPLICA_002_DATABASE_URL',
'default_database_url': 'sqlite://testing126'
},
]
},
{
'name': 'app_shard_002',
'environment_variable': 'SHARD_002_DATABASE_URL',
'default_database_url': 'sqlite://testing127'
},
{
'name': 'app_shard_003',
'shard_group': 'postgres',
'environment_variable': 'SHARD_003_DATABASE_URL',
'default_database_url': 'sqlite://testing125'
},
{
'name': 'app_shard_004',
'shard_group': 'postgres',
'environment_variable': 'SHARD_004_DATABASE_URL',
'default_database_url': 'sqlite://testing125'
},
]
})
shard_idx = big_shard_idx * shard_group_settings['small_shard_count'] + small_shard_idx
shard_name = name_sub_regex.sub('_' + str(shard_idx + 1), shard_group)
connection_environment_variable = shard_name.upper() + '_DATABASE_URL'
connection_url = 'sqlite:///{}'.format(os.path.join(os.path.dirname(BASE_DIR), 'db', '{}.sqlite3'.format(shard_name)))
sharded_databases.append({
'shard_group': shard_group,
'name': shard_name,
'environment_variable': connection_environment_variable,
'default_database_url': connection_url,
})
DATABASES = database_configs(databases_dict={
'unsharded_databases': [
{
'name': 'default',
'environment_variable': 'DATABASE_URL',
'default_database_url': 'sqlite:///{}'.format(os.path.join(BASE_DIR, 'db.sqlite3')),
}
],
'sharded_databases': sharded_databases,
})
DJANGO_SHARDING_SETTINGS = {
'items_shard_group': {
'SKIP_ADD_SHARDED_SIGNAL': True,
'BUCKETING_STRATEGY': DivModBucketingStrategy(
shard_group='items_shard_group',
databases=DATABASES,
big_shard_count=SHARDING_SETTINGS['items_shard_group']['big_shard_count'],
small_shard_count=SHARDING_SETTINGS['items_shard_group']['small_shard_count'],
small_shard_size=SHARDING_SETTINGS['items_shard_group']['small_shard_size'],
),
# }
# }
DATABASES = database_configs(
databases_dict={
'unsharded_databases': [
{
'name': 'default',
'environment_variable': 'SUMMER_DATABASE_URL',
'default_database_url': env(
'SUMMER_DATABASE_URL'), # sqlite:///./summer.sqlite3
},
],
'sharded_databases': [
{
'shard_group': 'user_group', # 샤드 그룹은 굳이 정해주지 않아도 됨.
'name': 'user_g1', # name은 실제 DB명을 안 써도 됨.
'environment_variable':
'SUMMER_ONE_DATABASE_URL', # env를 설정했다면, 이 db url을 가진 사용한 변수명 여기에 써주면 됨.
'default_database_url': env('SUMMER_ONE_DATABASE_URL'),
},
{
'shard_group': 'user_group',
'name': 'user_g2',
'environment_variable': 'SUMMER_TWO_DATABASE_URL',
'default_database_url': env('SUMMER_TWO_DATABASE_URL'),
},
],
})
from .sharding_functions import UserGroupBucketingStrategy # 커스텀 전략
# from django_sharding_library.sharding_functions import 원하는 거 # 라이브러리에서 제공하는 전략을 사용하고 싶다면 여기서 import
DATABASES = database_configs(
databases_dict={
'unsharded_databases': [{
'name':
'default',
'environment_variable':
'DATABASE_URL',
'default_database_url':
'postgres://postgres:@localhost/default'
if TRAVISCI else 'sqlite://testing123'
}],
'sharded_databases': [
{
'name':
'app_shard_001',
'environment_variable':
'SHARD_001_DATABASE_URL',
'default_database_url':
'postgres://postgres:@localhost/sharding_001'
if TRAVISCI else 'sqlite://testing124',
'replicas': [
{
'name':
'app_shard_001_replica_001',
'environment_variable':
'REPLICA_001_DATABASE_URL',
'default_database_url':
'postgres://postgres:@localhost/sharding_replica_001'
if TRAVISCI else 'sqlite://testing125'
},
{
'name':
'app_shard_001_replica_002',
'environment_variable':
'REPLICA_002_DATABASE_URL',
'default_database_url':
'postgres://postgres:@localhost/sharding_replica_002'
if TRAVISCI else 'sqlite://testing126'
},
]
},
{
'name':
'app_shard_002',
'environment_variable':
'SHARD_002_DATABASE_URL',
'default_database_url':
'mysql://travis:@localhost/sharding_002'
if TRAVISCI else 'sqlite://testing127'
},
]
})
# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
DATABASES = database_configs(databases_dict={
'unsharded_databases': [
{
'name': 'default',
'environment_variable': 'DATABASE_URL',
'default_database_url': 'sqlite:///%s' % os.path.join(BASE_DIR, 'db_default.sqlite3'),
}
],
'sharded_databases': [
{
'name': 'app_shard_001',
'environment_variable': 'SHARD_001_DATABASE_URL',
'default_database_url': 'sqlite:///%s' % os.path.join(BASE_DIR, 'db_shard1.sqlite3'),
'shard_group': 'shard_blog',
},
{
'name': 'app_shard_002',
'environment_variable': 'SHARD_002_DATABASE_URL',
'default_database_url': 'sqlite:///%s' % os.path.join(BASE_DIR, 'db_shard2.sqlite3'),
'shard_group': 'shard_blog',
},
]
})
# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/
DATABASES = database_configs(databases_dict={
'unsharded_databases': [
{
'name': 'default',
'environment_variable': 'DATABASE_URL',
'default_database_url': 'postgres://postgres:@localhost/default' if TRAVISCI else 'sqlite://testing123'
}
],
'sharded_databases': [
{
'name': 'app_shard_001',
'environment_variable': 'SHARD_001_DATABASE_URL',
'default_database_url': 'postgres://postgres:@localhost/sharding_001' if TRAVISCI else 'sqlite://testing124',
'replicas': [
{
'name': 'app_shard_001_replica_001',
'environment_variable': 'REPLICA_001_DATABASE_URL',
'default_database_url': 'postgres://postgres:@localhost/sharding_replica_001' if TRAVISCI else 'sqlite://testing125'
},
{
'name': 'app_shard_001_replica_002',
'environment_variable': 'REPLICA_002_DATABASE_URL',
'default_database_url': 'postgres://postgres:@localhost/sharding_replica_002' if TRAVISCI else 'sqlite://testing126'
},
]
},
{
'name': 'app_shard_002',
'environment_variable': 'SHARD_002_DATABASE_URL',
'default_database_url': 'mysql://travis:@localhost/sharding_002' if TRAVISCI else 'sqlite://testing127'
},
]
})
DATABASES = database_configs(
databases_dict={
'unsharded_databases': [{
'name': 'default',
'environment_variable': 'DATABASE_URL',
'default_database_url': 'sqlite://testing123'
}],
'sharded_databases': [
{
'name':
'app_shard_001',
'environment_variable':
'SHARD_001_DATABASE_URL',
'default_database_url':
'sqlite://testing124',
'replicas': [
{
'name': 'app_shard_001_replica_001',
'environment_variable': 'REPLICA_001_DATABASE_URL',
'default_database_url': 'sqlite://testing125'
},
{
'name': 'app_shard_001_replica_002',
'environment_variable': 'REPLICA_002_DATABASE_URL',
'default_database_url': 'sqlite://testing126'
},
]
},
{
'name': 'app_shard_002',
'environment_variable': 'SHARD_002_DATABASE_URL',
'default_database_url': 'sqlite://testing127'
},
{
'name': 'app_shard_003',
'shard_group': 'postgres',
'environment_variable': 'SHARD_003_DATABASE_URL',
'default_database_url': 'sqlite://testing125'
},
{
'name': 'app_shard_004',
'shard_group': 'postgres',
'environment_variable': 'SHARD_004_DATABASE_URL',
'default_database_url': 'sqlite://testing125'
},
]
})
