def test_reload_local_configuration(self):
os.environ.update({
'PATRONI_NAME': 'postgres0',
'PATRONI_NAMESPACE': '/patroni/',
'PATRONI_SCOPE': 'batman2',
'PATRONI_RESTAPI_USERNAME': '******',
'PATRONI_RESTAPI_PASSWORD': '******',
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
'PATRONI_RESTAPI_CERTFILE': '/certfile',
'PATRONI_RESTAPI_KEYFILE': '/keyfile',
'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_PGPASS': '******',
'PATRONI_ETCD_HOST': '127.0.0.1:2379',
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
'PATRONI_EXHIBITOR_PORT': '8181',
'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list
'PATRONI_SUPERUSER_USERNAME': '******',
'PATRONI_SUPERUSER_PASSWORD': '******',
'PATRONI_REPLICATION_USERNAME': '******',
'PATRONI_REPLICATION_PASSWORD': '******',
'PATRONI_admin_PASSWORD': '******',
'PATRONI_admin_OPTIONS': 'createrole,createdb'
})
sys.argv = ['patroni.py', 'postgres0.yml']
config = Config()
with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
self.assertRaises(Exception, config.reload_local_configuration, True)
self.assertTrue(config.reload_local_configuration(True))
self.assertTrue(config.reload_local_configuration())
def test_reload_local_configuration(self):
os.environ.update({
'PATRONI_NAME': 'postgres0',
'PATRONI_NAMESPACE': '/patroni/',
'PATRONI_SCOPE': 'batman2',
'PATRONI_LOGLEVEL': 'ERROR',
'PATRONI_LOG_LOGGERS':
'patroni.postmaster: WARNING, urllib3: DEBUG',
'PATRONI_RESTAPI_USERNAME': '******',
'PATRONI_RESTAPI_PASSWORD': '******',
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
'PATRONI_RESTAPI_CERTFILE': '/certfile',
'PATRONI_RESTAPI_KEYFILE': '/keyfile',
'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_CONFIG_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_PGPASS': '******',
'PATRONI_ETCD_HOST': '127.0.0.1:2379',
'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',
'PATRONI_ETCD_PROXY': 'http://127.0.0.1:2379',
'PATRONI_ETCD_SRV': 'test',
'PATRONI_ETCD_CACERT': '/cacert',
'PATRONI_ETCD_CERT': '/cert',
'PATRONI_ETCD_KEY': '/key',
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
'PATRONI_CONSUL_REGISTER_SERVICE': 'on',
'PATRONI_KUBERNETES_LABELS': 'a: b: c',
'PATRONI_KUBERNETES_SCOPE_LABEL': 'a',
'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]',
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
'PATRONI_EXHIBITOR_PORT': '8181',
'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list
'PATRONI_SUPERUSER_USERNAME': '******',
'PATRONI_SUPERUSER_PASSWORD': '******',
'PATRONI_REPLICATION_USERNAME': '******',
'PATRONI_REPLICATION_PASSWORD': '******',
'PATRONI_admin_PASSWORD': '******',
'PATRONI_admin_OPTIONS': 'createrole,createdb'
})
sys.argv = ['patroni.py', 'postgres0.yml']
config = Config()
with patch.object(Config, '_load_config_file',
Mock(return_value={'restapi': {}})):
with patch.object(Config, '_build_effective_configuration',
Mock(side_effect=Exception)):
self.assertRaises(Exception, config.reload_local_configuration,
True)
self.assertTrue(config.reload_local_configuration(True))
self.assertTrue(config.reload_local_configuration())
self.assertIsNone(config.reload_local_configuration())
def test_reload_local_configuration(self):
os.environ.update({
'PATRONI_NAME': 'postgres0',
'PATRONI_NAMESPACE': '/patroni/',
'PATRONI_SCOPE': 'batman2',
'PATRONI_RESTAPI_USERNAME': '******',
'PATRONI_RESTAPI_PASSWORD': '******',
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
'PATRONI_RESTAPI_CERTFILE': '/certfile',
'PATRONI_RESTAPI_KEYFILE': '/keyfile',
'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_PGPASS': '******',
'PATRONI_ETCD_HOST': '127.0.0.1:2379',
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
'PATRONI_EXHIBITOR_PORT': '8181',
'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list
'PATRONI_SUPERUSER_USERNAME': '******',
'PATRONI_SUPERUSER_PASSWORD': '******',
'PATRONI_REPLICATION_USERNAME': '******',
'PATRONI_REPLICATION_PASSWORD': '******',
'PATRONI_admin_PASSWORD': '******',
'PATRONI_admin_OPTIONS': 'createrole,createdb'
})
sys.argv = ['patroni.py', 'postgres0.yml']
config = Config()
with patch.object(Config, '_load_config_file',
Mock(return_value={'restapi': {}})):
with patch.object(Config, '_build_effective_configuration',
Mock(side_effect=Exception)):
self.assertRaises(Exception, config.reload_local_configuration,
True)
self.assertTrue(config.reload_local_configuration(True))
self.assertTrue(config.reload_local_configuration())
class Patroni(object):
def __init__(self):
from patroni.api import RestApiServer
from patroni.config import Config
from patroni.dcs import get_dcs
from patroni.ha import Ha
from patroni.postgresql import Postgresql
from patroni.version import __version__
from patroni.watchdog import Watchdog
self.setup_signal_handlers()
self.version = __version__
self.config = Config()
self.dcs = get_dcs(self.config)
self.watchdog = Watchdog(self.config)
self.load_dynamic_configuration()
self.postgresql = Postgresql(self.config['postgresql'])
self.api = RestApiServer(self, self.config['restapi'])
self.ha = Ha(self)
self.tags = self.get_tags()
self.next_run = time.time()
self.scheduled_restart = {}
def load_dynamic_configuration(self):
from patroni.exceptions import DCSError
while True:
try:
cluster = self.dcs.get_cluster()
if cluster and cluster.config and cluster.config.data:
if self.config.set_dynamic_configuration(cluster.config):
self.dcs.reload_config(self.config)
self.watchdog.reload_config(self.config)
elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
if self.config.set_dynamic_configuration(
self.config['bootstrap']['dcs']):
self.dcs.reload_config(self.config)
break
except DCSError:
logger.warning('Can not get cluster from dcs')
def get_tags(self):
return {
tag: value
for tag, value in self.config.get('tags', {}).items()
if tag not in ('clonefrom', 'nofailover', 'noloadbalance',
'nosync') or value
}
@property
def nofailover(self):
return bool(self.tags.get('nofailover', False))
@property
def nosync(self):
return bool(self.tags.get('nosync', False))
def reload_config(self):
try:
self.tags = self.get_tags()
self.dcs.reload_config(self.config)
self.watchdog.reload_config(self.config)
self.api.reload_config(self.config['restapi'])
self.postgresql.reload_config(self.config['postgresql'])
except Exception:
logger.exception('Failed to reload config_file=%s',
self.config.config_file)
@property
def replicatefrom(self):
return self.tags.get('replicatefrom')
def sighup_handler(self, *args):
self._received_sighup = True
def sigterm_handler(self, *args):
if not self._received_sigterm:
self._received_sigterm = True
sys.exit()
@property
def noloadbalance(self):
return bool(self.tags.get('noloadbalance', False))
def schedule_next_run(self):
self.next_run += self.dcs.loop_wait
current_time = time.time()
nap_time = self.next_run - current_time
if nap_time <= 0:
self.next_run = current_time
# Release the GIL so we don't starve anyone waiting on async_executor lock
time.sleep(0.001)
# Warn user that Patroni is not keeping up
logger.warning("Loop time exceeded, rescheduling immediately.")
elif self.ha.watch(nap_time):
self.next_run = time.time()
def run(self):
self.api.start()
self.next_run = time.time()
while not self._received_sigterm:
if self._received_sighup:
self._received_sighup = False
if self.config.reload_local_configuration():
self.reload_config()
logger.info(self.ha.run_cycle())
if self.dcs.cluster and self.dcs.cluster.config and self.dcs.cluster.config.data \
and self.config.set_dynamic_configuration(self.dcs.cluster.config):
self.reload_config()
if self.postgresql.role != 'uninitialized':
self.config.save_cache()
self.schedule_next_run()
def setup_signal_handlers(self):
self._received_sighup = False
self._received_sigterm = False
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGTERM, self.sigterm_handler)
def shutdown(self):
self.api.shutdown()
self.ha.shutdown()
class Patroni(object):
def __init__(self):
self.version = __version__
self.config = Config()
self.dcs = get_dcs(self.config)
self.load_dynamic_configuration()
self.postgresql = Postgresql(self.config['postgresql'])
self.api = RestApiServer(self, self.config['restapi'])
self.ha = Ha(self)
self.tags = self.get_tags()
self.nap_time = self.config['loop_wait']
self.next_run = time.time()
self._reload_config_scheduled = False
self._received_sighup = False
self._received_sigterm = False
def load_dynamic_configuration(self):
while True:
try:
cluster = self.dcs.get_cluster()
if cluster and cluster.config:
self.config.set_dynamic_configuration(cluster.config)
elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
self.config.set_dynamic_configuration(
self.config['bootstrap']['dcs'])
break
except DCSError:
logger.warning('Can not get cluster from dcs')
def get_tags(self):
return {
tag: value
for tag, value in self.config.get('tags', {}).items()
if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value
}
def reload_config(self):
try:
self.tags = self.get_tags()
self.nap_time = self.config['loop_wait']
self.dcs.set_ttl(self.config.get('ttl') or 30)
self.dcs.set_retry_timeout(
self.config.get('retry_timeout') or self.nap_time)
self.api.reload_config(self.config['restapi'])
self.postgresql.reload_config(self.config['postgresql'])
except Exception:
logger.exception('Failed to reload config_file=%s',
self.config.config_file)
def sighup_handler(self, *args):
self._received_sighup = True
def sigterm_handler(self, *args):
if not self._received_sigterm:
self._received_sigterm = True
sys.exit()
@property
def noloadbalance(self):
return self.tags.get('noloadbalance', False)
@property
def nofailover(self):
return self.tags.get('nofailover', False)
@property
def replicatefrom(self):
return self.tags.get('replicatefrom')
def schedule_next_run(self):
self.next_run += self.nap_time
current_time = time.time()
nap_time = self.next_run - current_time
if nap_time <= 0:
self.next_run = current_time
elif self.dcs.watch(nap_time):
self.next_run = time.time()
def run(self):
self.api.start()
self.next_run = time.time()
while not self._received_sigterm:
if self._received_sighup:
self._received_sighup = False
if self.config.reload_local_configuration():
self.reload_config()
logger.info(self.ha.run_cycle())
cluster = self.dcs.cluster
if cluster and cluster.config and self.config.set_dynamic_configuration(
cluster.config):
self.reload_config()
if not self.postgresql.data_directory_empty():
self.config.save_cache()
reap_children()
self.schedule_next_run()
def setup_signal_handlers(self):
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGTERM, self.sigterm_handler)
signal.signal(signal.SIGCHLD, sigchld_handler)
class Patroni(object):
def __init__(self):
from patroni.api import RestApiServer
from patroni.config import Config
from patroni.dcs import get_dcs
from patroni.ha import Ha
from patroni.postgresql import Postgresql
from patroni.version import __version__
self.setup_signal_handlers()
self.version = __version__
self.config = Config()
self.dcs = get_dcs(self.config)
self.load_dynamic_configuration()
self.postgresql = Postgresql(self.config['postgresql'])
self.api = RestApiServer(self, self.config['restapi'])
self.ha = Ha(self)
self.tags = self.get_tags()
self.next_run = time.time()
self.scheduled_restart = {}
def load_dynamic_configuration(self):
from patroni.exceptions import DCSError
while True:
try:
cluster = self.dcs.get_cluster()
if cluster and cluster.config:
if self.config.set_dynamic_configuration(cluster.config):
self.dcs.reload_config(self.config)
elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
if self.config.set_dynamic_configuration(self.config['bootstrap']['dcs']):
self.dcs.reload_config(self.config)
break
except DCSError:
logger.warning('Can not get cluster from dcs')
def get_tags(self):
return {tag: value for tag, value in self.config.get('tags', {}).items()
if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
@property
def nofailover(self):
return bool(self.tags.get('nofailover', False))
@property
def nosync(self):
return bool(self.tags.get('nosync', False))
def reload_config(self):
try:
self.tags = self.get_tags()
self.dcs.reload_config(self.config)
self.api.reload_config(self.config['restapi'])
self.postgresql.reload_config(self.config['postgresql'])
except Exception:
logger.exception('Failed to reload config_file=%s', self.config.config_file)
@property
def replicatefrom(self):
return self.tags.get('replicatefrom')
def sighup_handler(self, *args):
self._received_sighup = True
def sigterm_handler(self, *args):
if not self._received_sigterm:
self._received_sigterm = True
sys.exit()
@property
def noloadbalance(self):
return bool(self.tags.get('noloadbalance', False))
def schedule_next_run(self):
self.next_run += self.dcs.loop_wait
current_time = time.time()
nap_time = self.next_run - current_time
if nap_time <= 0:
self.next_run = current_time
# Release the GIL so we don't starve anyone waiting on async_executor lock
time.sleep(0.001)
# Warn user that Patroni is not keeping up
logger.warning("Loop time exceeded, rescheduling immediately.")
elif self.ha.watch(nap_time):
self.next_run = time.time()
def run(self):
self.api.start()
self.next_run = time.time()
while not self._received_sigterm:
if self._received_sighup:
self._received_sighup = False
if self.config.reload_local_configuration():
self.reload_config()
logger.info(self.ha.run_cycle())
cluster = self.dcs.cluster
if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
self.reload_config()
if not self.postgresql.data_directory_empty():
self.config.save_cache()
self.schedule_next_run()
def setup_signal_handlers(self):
self._received_sighup = False
self._received_sigterm = False
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGTERM, self.sigterm_handler)
class Patroni(object):
def __init__(self):
self.setup_signal_handlers()
self.version = __version__
self.config = Config()
self.dcs = get_dcs(self.config)
self.load_dynamic_configuration()
self.postgresql = Postgresql(self.config['postgresql'])
self.api = RestApiServer(self, self.config['restapi'])
self.ha = Ha(self)
self.tags = self.get_tags()
self.next_run = time.time()
self.scheduled_restart = {}
def load_dynamic_configuration(self):
while True:
try:
cluster = self.dcs.get_cluster()
if cluster and cluster.config:
self.config.set_dynamic_configuration(cluster.config)
elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
self.config.set_dynamic_configuration(self.config['bootstrap']['dcs'])
break
except DCSError:
logger.warning('Can not get cluster from dcs')
def get_tags(self):
return {tag: value for tag, value in self.config.get('tags', {}).items()
if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value}
@property
def nofailover(self):
return self.tags.get('nofailover', False)
def reload_config(self):
try:
self.tags = self.get_tags()
self.dcs.reload_config(self.config)
self.api.reload_config(self.config['restapi'])
self.postgresql.reload_config(self.config['postgresql'])
except Exception:
logger.exception('Failed to reload config_file=%s', self.config.config_file)
@property
def replicatefrom(self):
return self.tags.get('replicatefrom')
def sighup_handler(self, *args):
self._received_sighup = True
def sigterm_handler(self, *args):
if not self._received_sigterm:
self._received_sigterm = True
sys.exit()
@property
def noloadbalance(self):
return self.tags.get('noloadbalance', False)
def schedule_next_run(self):
self.next_run += self.dcs.loop_wait
current_time = time.time()
nap_time = self.next_run - current_time
if nap_time <= 0:
self.next_run = current_time
elif self.dcs.watch(nap_time):
self.next_run = time.time()
def run(self):
self.api.start()
self.next_run = time.time()
while not self._received_sigterm:
if self._received_sighup:
self._received_sighup = False
if self.config.reload_local_configuration():
self.reload_config()
logger.info(self.ha.run_cycle())
cluster = self.dcs.cluster
if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
self.reload_config()
if not self.postgresql.data_directory_empty():
self.config.save_cache()
reap_children()
self.schedule_next_run()
def setup_signal_handlers(self):
self._received_sighup = False
self._received_sigterm = False
signal.signal(signal.SIGHUP, self.sighup_handler)
signal.signal(signal.SIGTERM, self.sigterm_handler)
signal.signal(signal.SIGCHLD, sigchld_handler)
