def patroni_main():
import argparse
from patroni.config import Config, ConfigParseError
parser = argparse.ArgumentParser()
parser.add_argument('--version',
action='version',
version='%(prog)s {0}'.format(__version__))
parser.add_argument(
'configfile',
nargs='?',
default='',
help=
'Patroni may also read the configuration from the {0} environment variable'
.format(Config.PATRONI_CONFIG_VARIABLE))
args = parser.parse_args()
try:
conf = Config(args.configfile)
except ConfigParseError as e:
if e.value:
print(e.value)
parser.print_help()
sys.exit(1)
patroni = Patroni(conf)
try:
patroni.run()
except KeyboardInterrupt:
pass
finally:
patroni.shutdown()
def __init__(self):
from patroni.api import RestApiServer
from patroni.config import Config
from patroni.dcs import get_dcs
from patroni.ha import Ha
from patroni.log import PatroniLogger
from patroni.postgresql import Postgresql
from patroni.version import __version__
from patroni.watchdog import Watchdog
self.setup_signal_handlers()
self.version = __version__
self.logger = PatroniLogger()
self.config = Config()
self.logger.reload_config(self.config.get('log', {}))
self.dcs = get_dcs(self.config)
self.watchdog = Watchdog(self.config)
self.load_dynamic_configuration()
self.postgresql = Postgresql(self.config['postgresql'])
self.api = RestApiServer(self, self.config['restapi'])
self.ha = Ha(self)
self.tags = self.get_tags()
self.next_run = time.time()
self.scheduled_restart = {}
def test_patroni_logger(self):
config = {
'log': {
'dir': 'foo',
'file_size': 4096,
'file_num': 5,
'loggers': {
'foo.bar': 'INFO'
}
},
'restapi': {},
'postgresql': {
'data_dir': 'foo'
}
}
sys.argv = ['patroni.py']
os.environ[Config.PATRONI_CONFIG_VARIABLE] = yaml.dump(
config, default_flow_style=False)
logger = PatroniLogger()
patroni_config = Config()
logger.reload_config(patroni_config['log'])
self.assertEqual(logger.handler.maxBytes, config['log']['file_size'])
self.assertEqual(logger.handler.backupCount, config['log']['file_num'])
config['log'].pop('dir')
logger.reload_config(config['log'])
def test_configuration_directory(self):
def open_mock(fname, *args, **kwargs):
if fname.endswith('00-base.yml'):
return io.StringIO(
u'''
test: True
test2:
child-1: somestring
child-2: 5
child-3: False
test3: True
test4:
- abc: 3
- abc: 4
''')
elif fname.endswith('01-specific.yml'):
return io.StringIO(
u'''
test: False
test2:
child-2: 10
child-3: !!null
test4:
- ab: 5
new-attr: True
''')
with patch.object(builtins, 'open', MagicMock(side_effect=open_mock)):
config = Config('postgres0')
self.assertEqual(config._local_configuration,
{'test': False, 'test2': {'child-1': 'somestring', 'child-2': 10},
'test3': True, 'test4': [{'ab': 5}], 'new-attr': True})
def main():
from pg_upgrade import PostgresqlUpgrade
from patroni.config import Config
from patroni.utils import polling_loop
from spilo_commons import get_binary_version
config = Config(sys.argv[1])
upgrade = PostgresqlUpgrade(config)
bin_version = get_binary_version(upgrade.pgcommand(''))
cluster_version = upgrade.get_cluster_version()
if cluster_version == bin_version:
return 0
logger.info('Cluster version: %s, bin version: %s', cluster_version,
bin_version)
assert float(cluster_version) < float(bin_version)
logger.info('Trying to start the cluster with old postgres')
if not upgrade.start_old_cluster(config['bootstrap'], cluster_version):
raise Exception('Failed to start the cluster with old postgres')
for _ in polling_loop(upgrade.config.get('pg_ctl_timeout'), 10):
upgrade.reset_cluster_info_state()
if upgrade.is_leader():
break
logger.info('waiting for end of recovery of the old cluster')
if not upgrade.bootstrap.call_post_bootstrap(config['bootstrap']):
upgrade.stop(block_callbacks=True, checkpoint=False)
raise Exception('Failed to run bootstrap.post_init')
if not upgrade.prepare_new_pgdata(bin_version):
raise Exception('initdb failed')
try:
upgrade.drop_possibly_incompatible_objects()
except Exception:
upgrade.stop(block_callbacks=True, checkpoint=False)
raise
logger.info('Doing a clean shutdown of the cluster before pg_upgrade')
if not upgrade.stop(block_callbacks=True, checkpoint=False):
raise Exception('Failed to stop the cluster with old postgres')
if not upgrade.do_upgrade():
raise Exception('Failed to upgrade cluster from {0} to {1}'.format(
cluster_version, bin_version))
logger.info('Starting the cluster with new postgres after upgrade')
if not upgrade.start():
raise Exception('Failed to start the cluster with new postgres')
try:
upgrade.update_extensions()
except Exception as e:
logger.error('Failed to update extensions: %r', e)
upgrade.analyze()
def __init__(self, p, d):
os.environ[Config.PATRONI_CONFIG_VARIABLE] = """
restapi:
listen: 0.0.0.0:8008
bootstrap:
users:
replicator:
password: rep-pass
options:
- replication
postgresql:
name: foo
data_dir: data/postgresql0
pg_rewind:
username: postgres
password: postgres
zookeeper:
exhibitor:
hosts: [localhost]
port: 8181
"""
self.config = Config()
self.postgresql = p
self.dcs = d
self.api = Mock()
self.tags = {'foo': 'bar'}
self.nofailover = None
self.replicatefrom = None
self.api.connection_string = 'http://127.0.0.1:8008'
self.clonefrom = None
self.nosync = False
self.scheduled_restart = {
'schedule': future_restart_time,
'postmaster_start_time': str(postmaster_start_time)
}
def main():
from patroni.config import Config
from patroni.utils import polling_loop
from pg_upgrade import PostgresqlUpgrade
config = Config()
upgrade = PostgresqlUpgrade(config['postgresql'])
bin_version = upgrade.get_binary_version()
cluster_version = upgrade.get_cluster_version()
if cluster_version == bin_version:
return 0
logger.info('Cluster version: %s, bin version: %s', cluster_version,
bin_version)
assert float(cluster_version) < float(bin_version)
upgrade.set_bin_dir(cluster_version)
upgrade.config['pg_ctl_timeout'] = 3600 * 24 * 7
upgrade.config['callbacks'] = {}
bootstrap_config = config['bootstrap']
bootstrap_config[bootstrap_config['method']]['command'] = 'true'
logger.info('Trying to start the cluster with old postgres')
if not upgrade.bootstrap(bootstrap_config):
raise Exception('Failed to start the cluster with old postgres')
for _ in polling_loop(upgrade.config['pg_ctl_timeout'], 10):
upgrade.reset_cluster_info_state()
if upgrade.is_leader():
break
logger.info('waiting for end of recovery of the old cluster')
if not upgrade.run_bootstrap_post_init(bootstrap_config):
raise Exception('Failed to run bootstrap.post_init')
locale = upgrade.query('SHOW lc_collate').fetchone()[0]
encoding = upgrade.query('SHOW server_encoding').fetchone()[0]
initdb_config = [{'locale': locale}, {'encoding': encoding}]
if upgrade.query('SHOW data_checksums').fetchone()[0]:
initdb_config.append('data-checksums')
logger.info('Doing a clean shutdown of the cluster before pg_upgrade')
if not upgrade.stop(block_callbacks=True, checkpoint=False):
raise Exception('Failed to stop the cluster with old postgres')
logger.info('initdb config: %s', initdb_config)
logger.info('Executing pg_upgrade')
if not upgrade.do_upgrade(bin_version, {'initdb': initdb_config}):
raise Exception('Failed to upgrade cluster from {0} to {1}'.format(
cluster_version, bin_version))
logger.info('Starting the cluster with new postgres after upgrade')
if not upgrade.start():
raise Exception('Failed to start the cluster with new postgres')
upgrade.analyze()
def patroni_main():
import argparse
from multiprocessing import freeze_support
from patroni.config import Config, ConfigParseError
from patroni.validator import schema
freeze_support()
parser = argparse.ArgumentParser()
parser.add_argument('--version',
action='version',
version='%(prog)s {0}'.format(__version__))
parser.add_argument('--validate-config',
action='store_true',
help='Run config validator and exit')
parser.add_argument(
'configfile',
nargs='?',
default='',
help=
'Patroni may also read the configuration from the {0} environment variable'
.format(Config.PATRONI_CONFIG_VARIABLE))
args = parser.parse_args()
try:
if args.validate_config:
conf = Config(args.configfile, validator=schema)
sys.exit()
else:
conf = Config(args.configfile)
except ConfigParseError as e:
if e.value:
print(e.value)
parser.print_help()
sys.exit(1)
patroni = Patroni(conf)
try:
patroni.run()
except KeyboardInterrupt:
pass
finally:
patroni.shutdown()
def test_patroni_logger(self):
config = {
'log': {
'traceback_level': 'DEBUG',
'max_queue_size': 5,
'dir': 'foo',
'file_size': 4096,
'file_num': 5,
'loggers': {
'foo.bar': 'INFO'
}
},
'restapi': {},
'postgresql': {
'data_dir': 'foo'
}
}
sys.argv = ['patroni.py']
os.environ[Config.PATRONI_CONFIG_VARIABLE] = yaml.dump(
config, default_flow_style=False)
logger = PatroniLogger()
patroni_config = Config(None)
logger.reload_config(patroni_config['log'])
_LOG.exception('test')
logger.start()
with patch.object(logging.Handler, 'format', Mock(side_effect=Exception)),\
patch('_pytest.logging.LogCaptureHandler.emit', Mock()):
logging.error('test')
self.assertEqual(logger.log_handler.maxBytes,
config['log']['file_size'])
self.assertEqual(logger.log_handler.backupCount,
config['log']['file_num'])
config['log']['level'] = 'DEBUG'
config['log'].pop('dir')
with patch('logging.Handler.close', Mock(side_effect=Exception)):
logger.reload_config(config['log'])
with patch.object(
logging.Logger, 'makeRecord',
Mock(side_effect=[
logging.LogRecord('', logging.INFO, '', 0, '', (
), None), Exception
])):
logging.exception('test')
logging.error('test')
with patch.object(Queue, 'put_nowait', Mock(side_effect=Full)):
self.assertRaises(SystemExit, logger.shutdown)
self.assertRaises(Exception, logger.shutdown)
self.assertLessEqual(
logger.queue_size, 2
) # "Failed to close the old log handler" could be still in the queue
self.assertEqual(logger.records_lost, 0)
def test_reload_local_configuration(self):
os.environ.update({
'PATRONI_NAME': 'postgres0',
'PATRONI_NAMESPACE': '/patroni/',
'PATRONI_SCOPE': 'batman2',
'PATRONI_LOGLEVEL': 'ERROR',
'PATRONI_LOG_LOGGERS':
'patroni.postmaster: WARNING, urllib3: DEBUG',
'PATRONI_RESTAPI_USERNAME': '******',
'PATRONI_RESTAPI_PASSWORD': '******',
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
'PATRONI_RESTAPI_CERTFILE': '/certfile',
'PATRONI_RESTAPI_KEYFILE': '/keyfile',
'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_CONFIG_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_PGPASS': '******',
'PATRONI_ETCD_HOST': '127.0.0.1:2379',
'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',
'PATRONI_ETCD_PROXY': 'http://127.0.0.1:2379',
'PATRONI_ETCD_SRV': 'test',
'PATRONI_ETCD_CACERT': '/cacert',
'PATRONI_ETCD_CERT': '/cert',
'PATRONI_ETCD_KEY': '/key',
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
'PATRONI_CONSUL_REGISTER_SERVICE': 'on',
'PATRONI_KUBERNETES_LABELS': 'a: b: c',
'PATRONI_KUBERNETES_SCOPE_LABEL': 'a',
'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]',
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
'PATRONI_EXHIBITOR_PORT': '8181',
'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list
'PATRONI_SUPERUSER_USERNAME': '******',
'PATRONI_SUPERUSER_PASSWORD': '******',
'PATRONI_REPLICATION_USERNAME': '******',
'PATRONI_REPLICATION_PASSWORD': '******',
'PATRONI_admin_PASSWORD': '******',
'PATRONI_admin_OPTIONS': 'createrole,createdb'
})
sys.argv = ['patroni.py', 'postgres0.yml']
config = Config()
with patch.object(Config, '_load_config_file',
Mock(return_value={'restapi': {}})):
with patch.object(Config, '_build_effective_configuration',
Mock(side_effect=Exception)):
self.assertRaises(Exception, config.reload_local_configuration,
True)
self.assertTrue(config.reload_local_configuration(True))
self.assertTrue(config.reload_local_configuration())
self.assertIsNone(config.reload_local_configuration())
def main():
from pg_upgrade import PostgresqlUpgrade
from patroni.config import Config
from spilo_commons import get_binary_version
config = Config(sys.argv[1])
upgrade = PostgresqlUpgrade(config)
bin_version = get_binary_version(upgrade.pgcommand(''))
cluster_version = upgrade.get_cluster_version()
logger.info('Cluster version: %s, bin version: %s', cluster_version,
bin_version)
assert float(cluster_version) <= float(bin_version)
perform_pitr(upgrade, cluster_version, bin_version, config['bootstrap'])
if cluster_version == bin_version:
return 0
if not upgrade.bootstrap.call_post_bootstrap(config['bootstrap']):
upgrade.stop(block_callbacks=True, checkpoint=False)
raise Exception('Failed to run bootstrap.post_init')
if not upgrade.prepare_new_pgdata(bin_version):
raise Exception('initdb failed')
try:
upgrade.drop_possibly_incompatible_objects()
except Exception:
upgrade.stop(block_callbacks=True, checkpoint=False)
raise
logger.info('Doing a clean shutdown of the cluster before pg_upgrade')
if not upgrade.stop(block_callbacks=True, checkpoint=False):
raise Exception('Failed to stop the cluster with old postgres')
if not upgrade.do_upgrade():
raise Exception('Failed to upgrade cluster from {0} to {1}'.format(
cluster_version, bin_version))
logger.info('Starting the cluster with new postgres after upgrade')
if not upgrade.start():
raise Exception('Failed to start the cluster with new postgres')
try:
upgrade.update_extensions()
except Exception as e:
logger.error('Failed to update extensions: %r', e)
upgrade.analyze()
def __init__(self):
self.setup_signal_handlers()
self.version = __version__
self.config = Config()
self.dcs = get_dcs(self.config)
self.load_dynamic_configuration()
self.postgresql = Postgresql(self.config['postgresql'])
self.api = RestApiServer(self, self.config['restapi'])
self.ha = Ha(self)
self.tags = self.get_tags()
self.next_run = time.time()
self.scheduled_restart = {}
def main():
from patroni.config import Config
from spilo_commons import PATRONI_CONFIG_FILE
config = Config(PATRONI_CONFIG_FILE)
if len(sys.argv) == 4:
desired_version = sys.argv[1]
primary_ip = sys.argv[2]
pid = int(sys.argv[3])
return rsync_replica(config, desired_version, primary_ip, pid)
elif len(sys.argv) == 2:
replica_count = int(sys.argv[1])
upgrade = InplaceUpgrade(config)
return 0 if upgrade.try_upgrade(replica_count) else 1
else:
return 2
def __init__(self):
self.version = __version__
self.config = Config()
self.dcs = get_dcs(self.config)
self.load_dynamic_configuration()
self.postgresql = Postgresql(self.config['postgresql'])
self.api = RestApiServer(self, self.config['restapi'])
self.ha = Ha(self)
self.tags = self.get_tags()
self.nap_time = self.config['loop_wait']
self.next_run = time.time()
self._reload_config_scheduled = False
self._received_sighup = False
self._received_sigterm = False
def load_config(path, dcs):
from patroni.config import Config
if not (os.path.exists(path) and os.access(path, os.R_OK)):
if path != CONFIG_FILE_PATH: # bail if non-default config location specified but file not found / readable
raise PatroniCtlException('Provided config file {0} not existing or no read rights.'
' Check the -c/--config-file parameter'.format(path))
else:
logging.debug('Ignoring configuration file "%s". It does not exists or is not readable.', path)
else:
logging.debug('Loading configuration from file %s', path)
config = Config(path, validator=None).copy()
dcs = parse_dcs(dcs) or parse_dcs(config.get('dcs_api')) or {}
if dcs:
for d in DCS_DEFAULTS:
config.pop(d, None)
config.update(dcs)
return config
def load_config(path, dcs):
logging.debug('Loading configuration from file %s', path)
config = {}
old_argv = list(sys.argv)
try:
sys.argv[1] = path
if Config.PATRONI_CONFIG_VARIABLE not in os.environ:
for p in ('PATRONI_RESTAPI_LISTEN', 'PATRONI_POSTGRESQL_DATA_DIR'):
if p not in os.environ:
os.environ[p] = '.'
config = Config().copy()
finally:
sys.argv = old_argv
dcs = parse_dcs(dcs) or parse_dcs(config.get('dcs_api')) or {}
if dcs:
for d in DCS_DEFAULTS:
config.pop(d, None)
config.update(dcs)
return config
def __init__(self, p, d):
os.environ[Config.PATRONI_CONFIG_VARIABLE] = """
restapi:
listen: 0.0.0.0:8008
bootstrap:
users:
replicator:
password: rep-pass
options:
- replication
postgresql:
name: foo
data_dir: data/postgresql0
pg_rewind:
username: postgres
password: postgres
watchdog:
mode: off
zookeeper:
exhibitor:
hosts: [localhost]
port: 8181
"""
# We rely on sys.argv in Config, so it's necessary to reset
# all the extra values that are coming from py.test
sys.argv = sys.argv[:1]
self.config = Config()
self.postgresql = p
self.dcs = d
self.api = Mock()
self.tags = {'foo': 'bar'}
self.nofailover = None
self.replicatefrom = None
self.api.connection_string = 'http://127.0.0.1:8008'
self.clonefrom = None
self.nosync = False
self.scheduled_restart = {'schedule': future_restart_time,
'postmaster_start_time': str(postmaster_start_time)}
self.watchdog = Watchdog(self.config)
def test_reload_local_configuration(self):
os.environ.update({
'PATRONI_NAME': 'postgres0',
'PATRONI_NAMESPACE': '/patroni/',
'PATRONI_SCOPE': 'batman2',
'PATRONI_RESTAPI_USERNAME': '******',
'PATRONI_RESTAPI_PASSWORD': '******',
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
'PATRONI_RESTAPI_CERTFILE': '/certfile',
'PATRONI_RESTAPI_KEYFILE': '/keyfile',
'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
'PATRONI_POSTGRESQL_PGPASS': '******',
'PATRONI_ETCD_HOST': '127.0.0.1:2379',
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
'PATRONI_EXHIBITOR_PORT': '8181',
'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list
'PATRONI_SUPERUSER_USERNAME': '******',
'PATRONI_SUPERUSER_PASSWORD': '******',
'PATRONI_REPLICATION_USERNAME': '******',
'PATRONI_REPLICATION_PASSWORD': '******',
'PATRONI_admin_PASSWORD': '******',
'PATRONI_admin_OPTIONS': 'createrole,createdb'
})
sys.argv = ['patroni.py', 'postgres0.yml']
config = Config()
with patch.object(Config, '_load_config_file',
Mock(return_value={'restapi': {}})):
with patch.object(Config, '_build_effective_configuration',
Mock(side_effect=Exception)):
self.assertRaises(Exception, config.reload_local_configuration,
True)
self.assertTrue(config.reload_local_configuration(True))
self.assertTrue(config.reload_local_configuration())
def setUp(self):
sys.argv = ['patroni.py']
os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
self.config = Config()
def setUp(self):
self._handlers = logging.getLogger().handlers[:]
self.remove_files()
os.environ['PATRONI_RAFT_SELF_ADDR'] = self.SELF_ADDR
config = Config('postgres0.yml', validator=None)
self.rc = RaftController(config)
def main():
from patroni.config import Config
from patroni.utils import polling_loop
from pg_upgrade import PostgresqlUpgrade
config = Config()
config['postgresql'].update({
'callbacks': {},
'pg_ctl_timeout': 3600 * 24 * 7
})
upgrade = PostgresqlUpgrade(config['postgresql'])
bin_version = upgrade.get_binary_version()
cluster_version = upgrade.get_cluster_version()
if cluster_version == bin_version:
return 0
logger.info('Cluster version: %s, bin version: %s', cluster_version,
bin_version)
assert float(cluster_version) < float(bin_version)
logger.info('Trying to start the cluster with old postgres')
if not upgrade.start_old_cluster(config['bootstrap'], cluster_version):
raise Exception('Failed to start the cluster with old postgres')
for _ in polling_loop(upgrade.config.get('pg_ctl_timeout'), 10):
upgrade.reset_cluster_info_state()
if upgrade.is_leader():
break
logger.info('waiting for end of recovery of the old cluster')
if not upgrade.bootstrap.call_post_bootstrap(config['bootstrap']):
upgrade.stop(block_callbacks=True, checkpoint=False)
raise Exception('Failed to run bootstrap.post_init')
locale = upgrade.query('SHOW lc_collate').fetchone()[0]
encoding = upgrade.query('SHOW server_encoding').fetchone()[0]
initdb_config = [{'locale': locale}, {'encoding': encoding}]
if upgrade.query(
"SELECT current_setting('data_checksums')::bool").fetchone()[0]:
initdb_config.append('data-checksums')
logger.info(
'Dropping objects from the cluster which could be incompatible')
try:
upgrade.drop_possibly_incompatible_objects()
except Exception:
upgrade.stop(block_callbacks=True, checkpoint=False)
raise
logger.info('Doing a clean shutdown of the cluster before pg_upgrade')
if not upgrade.stop(block_callbacks=True, checkpoint=False):
raise Exception('Failed to stop the cluster with old postgres')
logger.info('initdb config: %s', initdb_config)
logger.info('Executing pg_upgrade')
if not upgrade.do_upgrade(bin_version, initdb_config):
raise Exception('Failed to upgrade cluster from {0} to {1}'.format(
cluster_version, bin_version))
logger.info('Starting the cluster with new postgres after upgrade')
if not upgrade.start():
raise Exception('Failed to start the cluster with new postgres')
upgrade.analyze()
