def remove_data_directory(self):
self.set_role('uninitialized')
logger.info('Removing data directory: %s', self._data_dir)
try:
if os.path.islink(self._data_dir):
os.unlink(self._data_dir)
elif not os.path.exists(self._data_dir):
return
elif os.path.isfile(self._data_dir):
os.remove(self._data_dir)
elif os.path.isdir(self._data_dir):
# let's see if pg_xlog|pg_wal is a symlink, in this case we
# should clean the target
for pg_wal_dir in ('pg_xlog', 'pg_wal'):
pg_wal_path = os.path.join(self._data_dir, pg_wal_dir)
if os.path.exists(pg_wal_path) and os.path.islink(pg_wal_path):
pg_wal_realpath = os.path.realpath(pg_wal_path)
logger.info('Removing WAL directory: %s', pg_wal_realpath)
shutil.rmtree(pg_wal_realpath)
# Remove user defined tablespace directory
pg_tblsp_dir = os.path.join(self._data_dir, 'pg_tblspc')
if os.path.exists(pg_tblsp_dir):
for tsdn in os.listdir(pg_tblsp_dir):
pg_tsp_path = os.path.join(pg_tblsp_dir, tsdn)
if parse_int(tsdn) and os.path.islink(pg_tsp_path):
pg_tsp_rpath = os.path.realpath(pg_tsp_path)
logger.info('Removing user defined tablespace directory: %s', pg_tsp_rpath)
shutil.rmtree(pg_tsp_rpath, ignore_errors=True)
shutil.rmtree(self._data_dir)
except (IOError, OSError):
logger.exception('Could not remove data directory %s', self._data_dir)
self.move_data_directory()
def do_POST_restart(self):
status_code = 500
data = 'restart failed'
request = self._read_json_content(body_is_optional=True)
cluster = self.server.patroni.dcs.get_cluster()
if request is None:
# failed to parse the json
return
if request:
logger.debug("received restart request: {0}".format(request))
if cluster.is_paused() and 'schedule' in request:
self._write_response(status_code,
"Can't schedule restart in the paused state")
return
for k in request:
if k == 'schedule':
(_, data,
request[k]) = self.parse_schedule(request[k], "restart")
if _:
status_code = _
break
elif k == 'role':
if request[k] not in ('master', 'replica'):
status_code = 400
data = "PostgreSQL role should be either master or replica"
break
elif k == 'postgres_version':
if not is_valid_pg_version(request[k]):
status_code = 400
data = "PostgreSQL version should be in the first.major.minor format"
break
elif k == 'timeout':
request[k] = parse_int(request[k], 's')
if request[k] is None or request[k] <= 0:
status_code = 400
data = "Timeout should be a positive number of seconds"
break
elif k != 'restart_pending':
status_code = 400
data = "Unknown filter for the scheduled restart: {0}".format(
k)
break
else:
if 'schedule' not in request:
try:
status, data = self.server.patroni.ha.restart(request)
status_code = 200 if status else 503
except Exception:
logger.exception('Exception during restart')
status_code = 400
else:
if self.server.patroni.ha.schedule_future_restart(request):
data = "Restart scheduled"
status_code = 202
else:
data = "Another restart is already scheduled"
status_code = 409
self._write_response(status_code, data)
def reload_config(self, config):
server_parameters = self.get_server_parameters(config)
listen_address_changed = pending_reload = pending_restart = False
if self.is_healthy():
changes = {p: v for p, v in server_parameters.items() if '.' not in p}
changes.update({p: None for p, v in self._server_parameters.items() if not ('.' in p or p in changes)})
if changes:
if 'wal_segment_size' not in changes:
changes['wal_segment_size'] = '16384kB'
# XXX: query can raise an exception
for r in self.query("""SELECT name, setting, unit, vartype, context
FROM pg_settings
WHERE name IN (""" + ', '.join(['%s'] * len(changes)) + """)
ORDER BY 1 DESC""", *(list(changes.keys()))):
if r[4] == 'internal':
if r[0] == 'wal_segment_size':
server_parameters.pop(r[0], None)
wal_segment_size = parse_int(r[2], 'kB')
if wal_segment_size is not None:
changes['wal_segment_size'] = '{0}kB'.format(int(r[1]) * wal_segment_size)
elif r[0] in changes:
unit = changes['wal_segment_size'] if r[0] in ('min_wal_size', 'max_wal_size') else r[2]
new_value = changes.pop(r[0])
if new_value is None or not compare_values(r[3], unit, r[1], new_value):
if r[4] == 'postmaster':
pending_restart = True
if r[0] in ('listen_addresses', 'port'):
listen_address_changed = True
else:
pending_reload = True
for param in changes:
if param in server_parameters:
logger.warning('Removing invalid parameter `%s` from postgresql.parameters', param)
server_parameters.pop(param)
# Check that user-defined-paramters have changed (parameters with period in name)
if not pending_reload:
for p, v in server_parameters.items():
if '.' in p and (p not in self._server_parameters or str(v) != str(self._server_parameters[p])):
pending_reload = True
break
if not pending_reload:
for p, v in self._server_parameters.items():
if '.' in p and (p not in server_parameters or str(v) != str(server_parameters[p])):
pending_reload = True
break
self.config = config
self._pending_restart = pending_restart
self._server_parameters = server_parameters
self._connect_address = config.get('connect_address')
if not listen_address_changed:
self.resolve_connection_addresses()
if pending_reload:
self._write_postgresql_conf()
self.reload()
self.retry.deadline = config['retry_timeout']/2.0
def reload_config(self, config):
server_parameters = self.get_server_parameters(config)
listen_address_changed = pending_reload = pending_restart = False
if self.is_healthy():
changes = {p: v for p, v in server_parameters.items() if '.' not in p}
changes.update({p: None for p, v in self._server_parameters.items() if not ('.' in p or p in changes)})
if changes:
if 'wal_segment_size' not in changes:
changes['wal_segment_size'] = '16384kB'
# XXX: query can raise an exception
for r in self.query("""SELECT name, setting, unit, vartype, context
FROM pg_settings
WHERE name IN (""" + ', '.join(['%s'] * len(changes)) + """)
ORDER BY 1 DESC""", *(list(changes.keys()))):
if r[4] == 'internal':
if r[0] == 'wal_segment_size':
server_parameters.pop(r[0], None)
wal_segment_size = parse_int(r[2], 'kB')
if wal_segment_size is not None:
changes['wal_segment_size'] = '{0}kB'.format(int(r[1]) * wal_segment_size)
elif r[0] in changes:
unit = changes['wal_segment_size'] if r[0] in ('min_wal_size', 'max_wal_size') else r[2]
new_value = changes.pop(r[0])
if new_value is None or not compare_values(r[3], unit, r[1], new_value):
if r[4] == 'postmaster':
pending_restart = True
if r[0] in ('listen_addresses', 'port'):
listen_address_changed = True
else:
pending_reload = True
for param in changes:
if param in server_parameters:
logger.warning('Removing invalid parameter `%s` from postgresql.parameters', param)
server_parameters.pop(param)
# Check that user-defined-paramters have changed (parameters with period in name)
if not pending_reload:
for p, v in server_parameters.items():
if '.' in p and (p not in self._server_parameters or str(v) != str(self._server_parameters[p])):
pending_reload = True
break
if not pending_reload:
for p, v in self._server_parameters.items():
if '.' in p and (p not in server_parameters or str(v) != str(server_parameters[p])):
pending_reload = True
break
self.config = config
self._pending_restart = pending_restart
self._server_parameters = server_parameters
self._connect_address = config.get('connect_address')
if not listen_address_changed:
self.resolve_connection_addresses()
if pending_reload:
self._write_postgresql_conf()
self.reload()
self.retry.deadline = config['retry_timeout']/2.0
def do_POST_restart(self):
status_code = 500
data = 'restart failed'
request = self._read_json_content(body_is_optional=True)
cluster = self.server.patroni.dcs.get_cluster()
if request is None:
# failed to parse the json
return
if request:
logger.debug("received restart request: {0}".format(request))
if cluster.is_paused() and 'schedule' in request:
self._write_response(status_code, "Can't schedule restart in the paused state")
return
for k in request:
if k == 'schedule':
(_, data, request[k]) = self.parse_schedule(request[k], "restart")
if _:
status_code = _
break
elif k == 'role':
if request[k] not in ('master', 'replica'):
status_code = 400
data = "PostgreSQL role should be either master or replica"
break
elif k == 'postgres_version':
try:
Postgresql.postgres_version_to_int(request[k])
except PostgresException as e:
status_code = 400
data = e.value
break
elif k == 'timeout':
request[k] = parse_int(request[k], 's')
if request[k] is None or request[k] <= 0:
status_code = 400
data = "Timeout should be a positive number of seconds"
break
elif k != 'restart_pending':
status_code = 400
data = "Unknown filter for the scheduled restart: {0}".format(k)
break
else:
if 'schedule' not in request:
try:
status, data = self.server.patroni.ha.restart(request)
status_code = 200 if status else 503
except Exception:
logger.exception('Exception during restart')
status_code = 400
else:
if self.server.patroni.ha.schedule_future_restart(request):
data = "Restart scheduled"
status_code = 202
else:
data = "Another restart is already scheduled"
status_code = 409
self._write_response(status_code, data)
def pg_tblspc_realpaths(self):
"""Returns a dict containing the symlink (key) and target (values) for the tablespaces"""
links = {}
pg_tblsp_dir = os.path.join(self._data_dir, 'pg_tblspc')
if os.path.exists(pg_tblsp_dir):
for tsdn in os.listdir(pg_tblsp_dir):
pg_tsp_path = os.path.join(pg_tblsp_dir, tsdn)
if parse_int(tsdn) and os.path.islink(pg_tsp_path):
pg_tsp_rpath = os.path.realpath(pg_tsp_path)
links[pg_tsp_path] = pg_tsp_rpath
return links
def _build_environment_configuration():
ret = defaultdict(dict)
def _popenv(name):
return os.environ.pop(PATRONI_ENV_PREFIX + name.upper(), None)
for param in ('name', 'namespace', 'scope'):
value = _popenv(param)
if value:
ret[param] = value
def _fix_log_env(name, oldname):
value = _popenv(oldname)
name = PATRONI_ENV_PREFIX + 'LOG_' + name.upper()
if value and name not in os.environ:
os.environ[name] = value
for name, oldname in (('level', 'loglevel'), ('format', 'logformat'),
('dateformat', 'log_datefmt')):
_fix_log_env(name, oldname)
def _set_section_values(section, params):
for param in params:
value = _popenv(section + '_' + param)
if value:
ret[section][param] = value
_set_section_values('restapi', [
'listen', 'connect_address', 'certfile', 'keyfile', 'cafile',
'verify_client'
])
_set_section_values('ctl',
['insecure', 'cacert', 'certfile', 'keyfile'])
_set_section_values('postgresql', [
'listen', 'connect_address', 'config_dir', 'data_dir', 'pgpass',
'bin_dir'
])
_set_section_values('log', [
'level', 'traceback_level', 'format', 'dateformat',
'max_queue_size', 'dir', 'file_size', 'file_num', 'loggers'
])
def _parse_dict(value):
if not value.strip().startswith('{'):
value = '{{{0}}}'.format(value)
try:
return yaml.safe_load(value)
except Exception:
logger.exception('Exception when parsing dict %s', value)
return None
value = ret.get('log', {}).pop('loggers', None)
if value:
value = _parse_dict(value)
if value:
ret['log']['loggers'] = value
def _get_auth(name, params=None):
ret = {}
for param in params or _AUTH_ALLOWED_PARAMETERS[:2]:
value = _popenv(name + '_' + param)
if value:
ret[param] = value
return ret
restapi_auth = _get_auth('restapi')
if restapi_auth:
ret['restapi']['authentication'] = restapi_auth
authentication = {}
for user_type in ('replication', 'superuser', 'rewind'):
entry = _get_auth(user_type, _AUTH_ALLOWED_PARAMETERS)
if entry:
authentication[user_type] = entry
if authentication:
ret['postgresql']['authentication'] = authentication
def _parse_list(value):
if not (value.strip().startswith('-') or '[' in value):
value = '[{0}]'.format(value)
try:
return yaml.safe_load(value)
except Exception:
logger.exception('Exception when parsing list %s', value)
return None
for param in list(os.environ.keys()):
if param.startswith(PATRONI_ENV_PREFIX):
# PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
name, suffix = (param[8:].split('_', 1) + [''])[:2]
if suffix in ('HOST', 'HOSTS', 'PORT', 'USE_PROXIES',
'PROTOCOL', 'SRV', 'URL', 'PROXY', 'CACERT',
'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC',
'CONSISTENCY', 'REGISTER_SERVICE',
'SERVICE_CHECK_INTERVAL', 'NAMESPACE', 'CONTEXT',
'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL',
'POD_IP', 'PORTS', 'LABELS') and name:
value = os.environ.pop(param)
if suffix == 'PORT':
value = value and parse_int(value)
elif suffix in ('HOSTS', 'PORTS', 'CHECKS'):
value = value and _parse_list(value)
elif suffix == 'LABELS':
value = _parse_dict(value)
elif suffix in ('USE_PROXIES', 'REGISTER_SERVICE'):
value = parse_bool(value)
if value:
ret[name.lower()][suffix.lower()] = value
if 'etcd' in ret:
ret['etcd'].update(_get_auth('etcd'))
users = {}
for param in list(os.environ.keys()):
if param.startswith(PATRONI_ENV_PREFIX):
name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
# PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
# CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
if name and suffix == 'PASSWORD':
password = os.environ.pop(param)
if password:
users[name] = {'password': password}
options = os.environ.pop(param[:-9] + '_OPTIONS', None)
options = options and _parse_list(options)
if options:
users[name]['options'] = options
if users:
ret['bootstrap']['users'] = users
return ret
def _build_environment_configuration():
ret = defaultdict(dict)
def _popenv(name):
return os.environ.pop(Config.PATRONI_ENV_PREFIX + name.upper(), None)
for param in ('name', 'namespace', 'scope'):
value = _popenv(param)
if value:
ret[param] = value
def _set_section_values(section, params):
for param in params:
value = _popenv(section + '_' + param)
if value:
ret[section][param] = value
_set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
_set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass', 'bin_dir'])
def _get_auth(name):
ret = {}
for param in ('username', 'password'):
value = _popenv(name + '_' + param)
if value:
ret[param] = value
return ret
restapi_auth = _get_auth('restapi')
if restapi_auth:
ret['restapi']['authentication'] = restapi_auth
authentication = {}
for user_type in ('replication', 'superuser'):
entry = _get_auth(user_type)
if entry:
authentication[user_type] = entry
if authentication:
ret['postgresql']['authentication'] = authentication
users = {}
def _parse_list(value):
if not (value.strip().startswith('-') or '[' in value):
value = '[{0}]'.format(value)
try:
return yaml.safe_load(value)
except Exception:
logger.exception('Exception when parsing list %s', value)
return None
for param in list(os.environ.keys()):
if param.startswith(Config.PATRONI_ENV_PREFIX):
name, suffix = (param[8:].split('_', 1) + [''])[:2]
if name and suffix:
# PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT',
'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'NAMESPACE', 'CONTEXT',
'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS'):
value = os.environ.pop(param)
if suffix == 'PORT':
value = value and parse_int(value)
elif suffix in ('HOSTS', 'PORTS', 'CHECKS'):
value = value and _parse_list(value)
elif suffix == 'LABELS':
if not value.strip().startswith('{'):
value = '{{{0}}}'.format(value)
try:
value = yaml.safe_load(value)
except Exception:
logger.exception('Exception when parsing dict %s', value)
value = None
if value:
ret[name.lower()][suffix.lower()] = value
# PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
# CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
elif suffix == 'PASSWORD':
password = os.environ.pop(param)
if password:
users[name] = {'password': password}
options = os.environ.pop(param[:-9] + '_OPTIONS', None)
options = options and _parse_list(options)
if options:
users[name]['options'] = options
if users:
ret['bootstrap']['users'] = users
return ret
def _build_environment_configuration():
ret = defaultdict(dict)
def _popenv(name):
return os.environ.pop(Config.PATRONI_ENV_PREFIX + name.upper(),
None)
for param in ('name', 'namespace', 'scope'):
value = _popenv(param)
if value:
ret[param] = value
def _set_section_values(section, params):
for param in params:
value = _popenv(section + '_' + param)
if value:
ret[section][param] = value
_set_section_values(
'restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
_set_section_values(
'postgresql',
['listen', 'connect_address', 'data_dir', 'pgpass', 'bin_dir'])
def _get_auth(name):
ret = {}
for param in ('username', 'password'):
value = _popenv(name + '_' + param)
if value:
ret[param] = value
return ret
restapi_auth = _get_auth('restapi')
if restapi_auth:
ret['restapi']['authentication'] = restapi_auth
authentication = {}
for user_type in ('replication', 'superuser'):
entry = _get_auth(user_type)
if entry:
authentication[user_type] = entry
if authentication:
ret['postgresql']['authentication'] = authentication
users = {}
def _parse_list(value):
if not (value.strip().startswith('-') or '[' in value):
value = '[{0}]'.format(value)
try:
return yaml.safe_load(value)
except Exception:
logger.exception('Exception when parsing list %s', value)
return None
for param in list(os.environ.keys()):
if param.startswith(Config.PATRONI_ENV_PREFIX):
name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
if name and suffix:
# PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL',
'PROXY', 'CACERT', 'CERT', 'KEY', 'VERIFY',
'TOKEN', 'CHECKS', 'DC') and '_' not in name:
value = os.environ.pop(param)
if suffix == 'PORT':
value = value and parse_int(value)
elif suffix in ('HOSTS', 'CHECKS'):
value = value and _parse_list(value)
if value:
ret[name.lower()][suffix.lower()] = value
# PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
# CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
elif suffix == 'PASSWORD':
password = os.environ.pop(param)
if password:
users[name] = {'password': password}
options = os.environ.pop(param[:-9] + '_OPTIONS',
None)
options = options and _parse_list(options)
if options:
users[name]['options'] = options
if users:
ret['bootstrap']['users'] = users
return ret