class TestConfig(unittest.TestCase):
@patch('os.path.isfile', Mock(return_value=True))
@patch('json.load', Mock(side_effect=Exception))
@patch.object(builtins, 'open', MagicMock())
def setUp(self):
sys.argv = ['patroni.py']
os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
self.config = Config()
def test_no_config(self):
self.assertRaises(SystemExit, Config)
def test_set_dynamic_configuration(self):
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
self.assertTrue(self.config.set_dynamic_configuration({'synchronous_mode': True, 'standby_cluster': {}}))
def test_reload_local_configuration(self):
os.environ.update({
'PATRONI_NAME': 'postgres0',
'PATRONI_NAMESPACE': '/patroni/',
'PATRONI_SCOPE': 'batman2',
'PATRONI_LOGLEVEL': 'ERROR',
'PATRONI_LOG_LOGGERS': 'patroni.postmaster: WARNING, urllib3: DEBUG',
'PATRONI_RESTAPI_USERNAME': '******',
'PATRONI_RESTAPI_PASSWORD': '******',
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
'PATRONI_RESTAPI_CERTFILE': '/certfile',
'PATRONI_RESTAPI_KEYFILE': '/keyfile',
'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_CONFIG_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_PGPASS': '******',
'PATRONI_ETCD_HOST': '127.0.0.1:2379',
'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',
'PATRONI_ETCD_PROXY': 'http://127.0.0.1:2379',
'PATRONI_ETCD_SRV': 'test',
'PATRONI_ETCD_CACERT': '/cacert',
'PATRONI_ETCD_CERT': '/cert',
'PATRONI_ETCD_KEY': '/key',
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
'PATRONI_CONSUL_REGISTER_SERVICE': 'on',
'PATRONI_KUBERNETES_LABELS': 'a: b: c',
'PATRONI_KUBERNETES_SCOPE_LABEL': 'a',
'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]',
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
'PATRONI_EXHIBITOR_PORT': '8181',
'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list
'PATRONI_SUPERUSER_USERNAME': '******',
'PATRONI_SUPERUSER_PASSWORD': '******',
'PATRONI_REPLICATION_USERNAME': '******',
'PATRONI_REPLICATION_PASSWORD': '******',
'PATRONI_admin_PASSWORD': '******',
'PATRONI_admin_OPTIONS': 'createrole,createdb'
})
sys.argv = ['patroni.py', 'postgres0.yml']
config = Config()
with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
config.reload_local_configuration()
self.assertTrue(config.reload_local_configuration())
self.assertIsNone(config.reload_local_configuration())
@patch('tempfile.mkstemp', Mock(return_value=[3000, 'blabla']))
@patch('os.path.exists', Mock(return_value=True))
@patch('os.remove', Mock(side_effect=IOError))
@patch('os.close', Mock(side_effect=IOError))
@patch('shutil.move', Mock(return_value=None))
@patch('json.dump', Mock())
def test_save_cache(self):
self.config.set_dynamic_configuration({'ttl': 30, 'postgresql': {'foo': 'bar'}})
with patch('os.fdopen', Mock(side_effect=IOError)):
self.config.save_cache()
with patch('os.fdopen', MagicMock()):
self.config.save_cache()
def test_standby_cluster_parameters(self):
dynamic_configuration = {
'standby_cluster': {
'create_replica_methods': ['wal_e', 'basebackup'],
'host': 'localhost',
'port': 5432
}
}
self.config.set_dynamic_configuration(dynamic_configuration)
for name, value in dynamic_configuration['standby_cluster'].items():
self.assertEqual(self.config['standby_cluster'][name], value)
class TestConfig(unittest.TestCase):
@patch('os.path.isfile', Mock(return_value=True))
@patch('json.load', Mock(side_effect=Exception))
@patch.object(builtins, 'open', MagicMock())
def setUp(self):
sys.argv = ['patroni.py']
os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
self.config = Config()
def test_no_config(self):
self.assertRaises(SystemExit, Config)
def test_set_dynamic_configuration(self):
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
self.assertTrue(self.config.set_dynamic_configuration({'synchronous_mode': True}))
def test_reload_local_configuration(self):
os.environ.update({
'PATRONI_NAME': 'postgres0',
'PATRONI_NAMESPACE': '/patroni/',
'PATRONI_SCOPE': 'batman2',
'PATRONI_RESTAPI_USERNAME': '******',
'PATRONI_RESTAPI_PASSWORD': '******',
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
'PATRONI_RESTAPI_CERTFILE': '/certfile',
'PATRONI_RESTAPI_KEYFILE': '/keyfile',
'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_PGPASS': '******',
'PATRONI_ETCD_HOST': '127.0.0.1:2379',
'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',
'PATRONI_ETCD_PROXY': 'http://127.0.0.1:2379',
'PATRONI_ETCD_SRV': 'test',
'PATRONI_ETCD_CACERT': '/cacert',
'PATRONI_ETCD_CERT': '/cert',
'PATRONI_ETCD_KEY': '/key',
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
'PATRONI_EXHIBITOR_PORT': '8181',
'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list
'PATRONI_SUPERUSER_USERNAME': '******',
'PATRONI_SUPERUSER_PASSWORD': '******',
'PATRONI_REPLICATION_USERNAME': '******',
'PATRONI_REPLICATION_PASSWORD': '******',
'PATRONI_admin_PASSWORD': '******',
'PATRONI_admin_OPTIONS': 'createrole,createdb'
})
sys.argv = ['patroni.py', 'postgres0.yml']
config = Config()
with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
self.assertRaises(Exception, config.reload_local_configuration, True)
self.assertTrue(config.reload_local_configuration(True))
self.assertTrue(config.reload_local_configuration())
@patch('tempfile.mkstemp', Mock(return_value=[3000, 'blabla']))
@patch('os.path.exists', Mock(return_value=True))
@patch('os.remove', Mock(side_effect=IOError))
@patch('os.close', Mock(side_effect=IOError))
@patch('os.rename', Mock(return_value=None))
@patch('json.dump', Mock())
def test_save_cache(self):
self.config.set_dynamic_configuration({'ttl': 30, 'postgresql': {'foo': 'bar'}})
with patch('os.fdopen', Mock(side_effect=IOError)):
self.config.save_cache()
with patch('os.fdopen', MagicMock()):
self.config.save_cache()
class Patroni(object):
def __init__(self):
from patroni.api import RestApiServer
from patroni.config import Config
from patroni.dcs import get_dcs
from patroni.ha import Ha
from patroni.postgresql import Postgresql
from patroni.version import __version__
from patroni.watchdog import Watchdog
self.setup_signal_handlers()
self.version = __version__
self.config = Config()
self.dcs = get_dcs(self.config)
self.watchdog = Watchdog(self.config)
self.load_dynamic_configuration()
self.postgresql = Postgresql(self.config['postgresql'])
self.api = RestApiServer(self, self.config['restapi'])
self.ha = Ha(self)
self.tags = self.get_tags()
self.next_run = time.time()
self.scheduled_restart = {}
def load_dynamic_configuration(self):
from patroni.exceptions import DCSError
while True:
try:
cluster = self.dcs.get_cluster()
if cluster and cluster.config and cluster.config.data:
if self.config.set_dynamic_configuration(cluster.config):
self.dcs.reload_config(self.config)
self.watchdog.reload_config(self.config)
elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
if self.config.set_dynamic_configuration(
self.config['bootstrap']['dcs']):
self.dcs.reload_config(self.config)
break
except DCSError:
logger.warning('Can not get cluster from dcs')
def get_tags(self):
return {
tag: value
for tag, value in self.config.get('tags', {}).items()
if tag not in ('clonefrom', 'nofailover', 'noloadbalance',
'nosync') or value
}
@property
def nofailover(self):
return bool(self.tags.get('nofailover', False))
@property
def nosync(self):
return bool(self.tags.get('nosync', False))
def reload_config(self):
try:
self.tags = self.get_tags()
self.dcs.reload_config(self.config)
self.watchdog.reload_config(self.config)
self.api.reload_config(self.config['restapi'])
self.postgresql.reload_config(self.config['postgresql'])
except Exception:
logger.exception('Failed to reload config_file=%s',
self.config.config_file)
@property
def replicatefrom(self):
return self.tags.get('replicatefrom')
def sighup_handler(self, *args):
self._received_sighup = True
def sigterm_handler(self, *args):
if not self._received_sigterm:
self._received_sigterm = True
sys.exit()
@property
def noloadbalance(self):
return bool(self.tags.get('noloadbalance', False))
def schedule_next_run(self):
self.next_run += self.dcs.loop_wait
current_time = time.time()
nap_time = self.next_run - current_time
if nap_time <= 0:
self.next_run = current_time
# Release the GIL so we don't starve anyone waiting on async_executor lock
time.sleep(0.001)
# Warn user that Patroni is not keeping up
logger.warning("Loop time exceeded, rescheduling immediately.")
elif self.ha.watch(nap_time):
self.next_run = time.time()
def run(self):
self.api.start()
self.next_run = time.time()
while not self._received_sigterm:
if self._received_sighup:
self._received_sighup = False
if self.config.reload_local_configuration():
self.reload_config()
logger.info(self.ha.run_cycle())
if self.dcs.cluster and self.dcs.cluster.config and self.dcs.cluster.config.data \
and self.config.set_dynamic_configuration(self.dcs.cluster.config):
self.reload_config()
if self.postgresql.role != 'uninitialized':
self.config.save_cache()
self.schedule_next_run()
def setup_signal_handlers(self):
self._received_sighup = False
self._received_sigterm = False
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGTERM, self.sigterm_handler)
def shutdown(self):
self.api.shutdown()
self.ha.shutdown()
class TestConfig(unittest.TestCase):
@patch('os.path.isfile', Mock(return_value=True))
@patch('json.load', Mock(side_effect=Exception))
@patch.object(builtins, 'open', MagicMock())
def setUp(self):
sys.argv = ['patroni.py']
os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
self.config = Config(None)
def test_set_dynamic_configuration(self):
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
self.assertTrue(self.config.set_dynamic_configuration({'synchronous_mode': True, 'standby_cluster': {}}))
def test_reload_local_configuration(self):
os.environ.update({
'PATRONI_NAME': 'postgres0',
'PATRONI_NAMESPACE': '/patroni/',
'PATRONI_SCOPE': 'batman2',
'PATRONI_LOGLEVEL': 'ERROR',
'PATRONI_LOG_LOGGERS': 'patroni.postmaster: WARNING, urllib3: DEBUG',
'PATRONI_LOG_FILE_NUM': '5',
'PATRONI_RESTAPI_USERNAME': '******',
'PATRONI_RESTAPI_PASSWORD': '******',
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
'PATRONI_RESTAPI_CERTFILE': '/certfile',
'PATRONI_RESTAPI_KEYFILE': '/keyfile',
'PATRONI_RESTAPI_ALLOWLIST_INCLUDE_MEMBERS': 'on',
'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_CONFIG_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_PGPASS': '******',
'PATRONI_ETCD_HOST': '127.0.0.1:2379',
'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',
'PATRONI_ETCD_PROXY': 'http://127.0.0.1:2379',
'PATRONI_ETCD_SRV': 'test',
'PATRONI_ETCD_CACERT': '/cacert',
'PATRONI_ETCD_CERT': '/cert',
'PATRONI_ETCD_KEY': '/key',
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
'PATRONI_CONSUL_REGISTER_SERVICE': 'on',
'PATRONI_KUBERNETES_LABELS': 'a: b: c',
'PATRONI_KUBERNETES_SCOPE_LABEL': 'a',
'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]',
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
'PATRONI_EXHIBITOR_PORT': '8181',
'PATRONI_RAFT_PARTNER_ADDRS': "'host1:1234','host2:1234'",
'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list
'PATRONI_SUPERUSER_USERNAME': '******',
'PATRONI_SUPERUSER_PASSWORD': '******',
'PATRONI_REPLICATION_USERNAME': '******',
'PATRONI_REPLICATION_PASSWORD': '******',
'PATRONI_admin_PASSWORD': '******',
'PATRONI_admin_OPTIONS': 'createrole,createdb'
})
config = Config('postgres0.yml')
with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
config.reload_local_configuration()
self.assertTrue(config.reload_local_configuration())
self.assertIsNone(config.reload_local_configuration())
@patch('tempfile.mkstemp', Mock(return_value=[3000, 'blabla']))
@patch('os.path.exists', Mock(return_value=True))
@patch('os.remove', Mock(side_effect=IOError))
@patch('os.close', Mock(side_effect=IOError))
@patch('shutil.move', Mock(return_value=None))
@patch('json.dump', Mock())
def test_save_cache(self):
self.config.set_dynamic_configuration({'ttl': 30, 'postgresql': {'foo': 'bar'}})
with patch('os.fdopen', Mock(side_effect=IOError)):
self.config.save_cache()
with patch('os.fdopen', MagicMock()):
self.config.save_cache()
def test_standby_cluster_parameters(self):
dynamic_configuration = {
'standby_cluster': {
'create_replica_methods': ['wal_e', 'basebackup'],
'host': 'localhost',
'port': 5432
}
}
self.config.set_dynamic_configuration(dynamic_configuration)
for name, value in dynamic_configuration['standby_cluster'].items():
self.assertEqual(self.config['standby_cluster'][name], value)
@patch('os.path.exists', Mock(return_value=True))
@patch('os.path.isfile', Mock(side_effect=lambda fname: fname != 'postgres0'))
@patch('os.path.isdir', Mock(return_value=True))
@patch('os.listdir', Mock(return_value=['01-specific.yml', '00-base.yml']))
def test_configuration_directory(self):
def open_mock(fname, *args, **kwargs):
if fname.endswith('00-base.yml'):
return io.StringIO(
u'''
test: True
test2:
child-1: somestring
child-2: 5
child-3: False
test3: True
test4:
- abc: 3
- abc: 4
''')
elif fname.endswith('01-specific.yml'):
return io.StringIO(
u'''
test: False
test2:
child-2: 10
child-3: !!null
test4:
- ab: 5
new-attr: True
''')
with patch.object(builtins, 'open', MagicMock(side_effect=open_mock)):
config = Config('postgres0')
self.assertEqual(config._local_configuration,
{'test': False, 'test2': {'child-1': 'somestring', 'child-2': 10},
'test3': True, 'test4': [{'ab': 5}], 'new-attr': True})
@patch('os.path.exists', Mock(return_value=True))
@patch('os.path.isfile', Mock(return_value=False))
@patch('os.path.isdir', Mock(return_value=False))
def test_invalid_path(self):
self.assertRaises(ConfigParseError, Config, 'postgres0')
class Patroni(object):
def __init__(self):
self.version = __version__
self.config = Config()
self.dcs = get_dcs(self.config)
self.load_dynamic_configuration()
self.postgresql = Postgresql(self.config['postgresql'])
self.api = RestApiServer(self, self.config['restapi'])
self.ha = Ha(self)
self.tags = self.get_tags()
self.nap_time = self.config['loop_wait']
self.next_run = time.time()
self._reload_config_scheduled = False
self._received_sighup = False
self._received_sigterm = False
def load_dynamic_configuration(self):
while True:
try:
cluster = self.dcs.get_cluster()
if cluster and cluster.config:
self.config.set_dynamic_configuration(cluster.config)
elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
self.config.set_dynamic_configuration(
self.config['bootstrap']['dcs'])
break
except DCSError:
logger.warning('Can not get cluster from dcs')
def get_tags(self):
return {
tag: value
for tag, value in self.config.get('tags', {}).items()
if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value
}
def reload_config(self):
try:
self.tags = self.get_tags()
self.nap_time = self.config['loop_wait']
self.dcs.set_ttl(self.config.get('ttl') or 30)
self.dcs.set_retry_timeout(
self.config.get('retry_timeout') or self.nap_time)
self.api.reload_config(self.config['restapi'])
self.postgresql.reload_config(self.config['postgresql'])
except Exception:
logger.exception('Failed to reload config_file=%s',
self.config.config_file)
def sighup_handler(self, *args):
self._received_sighup = True
def sigterm_handler(self, *args):
if not self._received_sigterm:
self._received_sigterm = True
sys.exit()
@property
def noloadbalance(self):
return self.tags.get('noloadbalance', False)
@property
def nofailover(self):
return self.tags.get('nofailover', False)
@property
def replicatefrom(self):
return self.tags.get('replicatefrom')
def schedule_next_run(self):
self.next_run += self.nap_time
current_time = time.time()
nap_time = self.next_run - current_time
if nap_time <= 0:
self.next_run = current_time
elif self.dcs.watch(nap_time):
self.next_run = time.time()
def run(self):
self.api.start()
self.next_run = time.time()
while not self._received_sigterm:
if self._received_sighup:
self._received_sighup = False
if self.config.reload_local_configuration():
self.reload_config()
logger.info(self.ha.run_cycle())
cluster = self.dcs.cluster
if cluster and cluster.config and self.config.set_dynamic_configuration(
cluster.config):
self.reload_config()
if not self.postgresql.data_directory_empty():
self.config.save_cache()
reap_children()
self.schedule_next_run()
def setup_signal_handlers(self):
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGTERM, self.sigterm_handler)
signal.signal(signal.SIGCHLD, sigchld_handler)
class TestConfig(unittest.TestCase):
@patch('os.path.isfile', Mock(return_value=True))
@patch('json.load', Mock(side_effect=Exception))
@patch.object(builtins, 'open', MagicMock())
def setUp(self):
sys.argv = ['patroni.py']
os.environ[
Config.
PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
self.config = Config()
def test_no_config(self):
self.assertRaises(SystemExit, Config)
@patch.object(Config, '_build_effective_configuration',
Mock(side_effect=Exception))
def test_set_dynamic_configuration(self):
self.assertIsNone(self.config.set_dynamic_configuration({'foo':
'bar'}))
def test_reload_local_configuration(self):
os.environ.update({
'PATRONI_NAME': 'postgres0',
'PATRONI_NAMESPACE': '/patroni/',
'PATRONI_SCOPE': 'batman2',
'PATRONI_RESTAPI_USERNAME': '******',
'PATRONI_RESTAPI_PASSWORD': '******',
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
'PATRONI_RESTAPI_CERTFILE': '/certfile',
'PATRONI_RESTAPI_KEYFILE': '/keyfile',
'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_PGPASS': '******',
'PATRONI_ETCD_HOST': '127.0.0.1:2379',
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
'PATRONI_EXHIBITOR_PORT': '8181',
'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list
'PATRONI_SUPERUSER_USERNAME': '******',
'PATRONI_SUPERUSER_PASSWORD': '******',
'PATRONI_REPLICATION_USERNAME': '******',
'PATRONI_REPLICATION_PASSWORD': '******',
'PATRONI_admin_PASSWORD': '******',
'PATRONI_admin_OPTIONS': 'createrole,createdb'
})
sys.argv = ['patroni.py', 'postgres0.yml']
config = Config()
with patch.object(Config, '_load_config_file',
Mock(return_value={'restapi': {}})):
with patch.object(Config, '_build_effective_configuration',
Mock(side_effect=Exception)):
self.assertRaises(Exception, config.reload_local_configuration,
True)
self.assertTrue(config.reload_local_configuration(True))
self.assertTrue(config.reload_local_configuration())
@patch('tempfile.mkstemp', Mock(return_value=[3000, 'blabla']))
@patch('os.path.exists', Mock(return_value=True))
@patch('os.remove', Mock(side_effect=IOError))
@patch('os.close', Mock(side_effect=IOError))
@patch('os.rename', Mock(return_value=None))
@patch('json.dump', Mock())
def test_save_cache(self):
self.config.set_dynamic_configuration({
'ttl': 30,
'postgresql': {
'foo': 'bar'
}
})
with patch('os.fdopen', Mock(side_effect=IOError)):
self.config.save_cache()
with patch('os.fdopen', MagicMock()):
self.config.save_cache()
class Patroni(object):
def __init__(self):
from patroni.api import RestApiServer
from patroni.config import Config
from patroni.dcs import get_dcs
from patroni.ha import Ha
from patroni.postgresql import Postgresql
from patroni.version import __version__
self.setup_signal_handlers()
self.version = __version__
self.config = Config()
self.dcs = get_dcs(self.config)
self.load_dynamic_configuration()
self.postgresql = Postgresql(self.config['postgresql'])
self.api = RestApiServer(self, self.config['restapi'])
self.ha = Ha(self)
self.tags = self.get_tags()
self.next_run = time.time()
self.scheduled_restart = {}
def load_dynamic_configuration(self):
from patroni.exceptions import DCSError
while True:
try:
cluster = self.dcs.get_cluster()
if cluster and cluster.config:
if self.config.set_dynamic_configuration(cluster.config):
self.dcs.reload_config(self.config)
elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
if self.config.set_dynamic_configuration(self.config['bootstrap']['dcs']):
self.dcs.reload_config(self.config)
break
except DCSError:
logger.warning('Can not get cluster from dcs')
def get_tags(self):
return {tag: value for tag, value in self.config.get('tags', {}).items()
if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
@property
def nofailover(self):
return bool(self.tags.get('nofailover', False))
@property
def nosync(self):
return bool(self.tags.get('nosync', False))
def reload_config(self):
try:
self.tags = self.get_tags()
self.dcs.reload_config(self.config)
self.api.reload_config(self.config['restapi'])
self.postgresql.reload_config(self.config['postgresql'])
except Exception:
logger.exception('Failed to reload config_file=%s', self.config.config_file)
@property
def replicatefrom(self):
return self.tags.get('replicatefrom')
def sighup_handler(self, *args):
self._received_sighup = True
def sigterm_handler(self, *args):
if not self._received_sigterm:
self._received_sigterm = True
sys.exit()
@property
def noloadbalance(self):
return bool(self.tags.get('noloadbalance', False))
def schedule_next_run(self):
self.next_run += self.dcs.loop_wait
current_time = time.time()
nap_time = self.next_run - current_time
if nap_time <= 0:
self.next_run = current_time
# Release the GIL so we don't starve anyone waiting on async_executor lock
time.sleep(0.001)
# Warn user that Patroni is not keeping up
logger.warning("Loop time exceeded, rescheduling immediately.")
elif self.ha.watch(nap_time):
self.next_run = time.time()
def run(self):
self.api.start()
self.next_run = time.time()
while not self._received_sigterm:
if self._received_sighup:
self._received_sighup = False
if self.config.reload_local_configuration():
self.reload_config()
logger.info(self.ha.run_cycle())
cluster = self.dcs.cluster
if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
self.reload_config()
if not self.postgresql.data_directory_empty():
self.config.save_cache()
self.schedule_next_run()
def setup_signal_handlers(self):
self._received_sighup = False
self._received_sigterm = False
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGTERM, self.sigterm_handler)
class Patroni(object):
def __init__(self):
self.setup_signal_handlers()
self.version = __version__
self.config = Config()
self.dcs = get_dcs(self.config)
self.load_dynamic_configuration()
self.postgresql = Postgresql(self.config['postgresql'])
self.api = RestApiServer(self, self.config['restapi'])
self.ha = Ha(self)
self.tags = self.get_tags()
self.next_run = time.time()
self.scheduled_restart = {}
def load_dynamic_configuration(self):
while True:
try:
cluster = self.dcs.get_cluster()
if cluster and cluster.config:
self.config.set_dynamic_configuration(cluster.config)
elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
self.config.set_dynamic_configuration(self.config['bootstrap']['dcs'])
break
except DCSError:
logger.warning('Can not get cluster from dcs')
def get_tags(self):
return {tag: value for tag, value in self.config.get('tags', {}).items()
if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value}
@property
def nofailover(self):
return self.tags.get('nofailover', False)
def reload_config(self):
try:
self.tags = self.get_tags()
self.dcs.reload_config(self.config)
self.api.reload_config(self.config['restapi'])
self.postgresql.reload_config(self.config['postgresql'])
except Exception:
logger.exception('Failed to reload config_file=%s', self.config.config_file)
@property
def replicatefrom(self):
return self.tags.get('replicatefrom')
def sighup_handler(self, *args):
self._received_sighup = True
def sigterm_handler(self, *args):
if not self._received_sigterm:
self._received_sigterm = True
sys.exit()
@property
def noloadbalance(self):
return self.tags.get('noloadbalance', False)
def schedule_next_run(self):
self.next_run += self.dcs.loop_wait
current_time = time.time()
nap_time = self.next_run - current_time
if nap_time <= 0:
self.next_run = current_time
elif self.dcs.watch(nap_time):
self.next_run = time.time()
def run(self):
self.api.start()
self.next_run = time.time()
while not self._received_sigterm:
if self._received_sighup:
self._received_sighup = False
if self.config.reload_local_configuration():
self.reload_config()
logger.info(self.ha.run_cycle())
cluster = self.dcs.cluster
if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
self.reload_config()
if not self.postgresql.data_directory_empty():
self.config.save_cache()
reap_children()
self.schedule_next_run()
def setup_signal_handlers(self):
self._received_sighup = False
self._received_sigterm = False
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGTERM, self.sigterm_handler)
signal.signal(signal.SIGCHLD, sigchld_handler)
