def reload_local_configuration(self, dry_run=False):
if self.config_file:
try:
configuration = self._load_config_file()
if not deep_compare(self._local_configuration, configuration):
new_configuration = self._build_effective_configuration(self._dynamic_configuration, configuration)
if dry_run:
return not deep_compare(new_configuration, self.__effective_configuration)
self._local_configuration = configuration
self.__effective_configuration = new_configuration
return True
except Exception:
logger.exception('Exception when reloading local configuration from %s', self.config_file)
if dry_run:
raise
def touch_member(self, data, ttl=None, permanent=False):
cluster = self.cluster
member = cluster and cluster.get_member(self._name,
fallback_to_leader=False)
create_member = not permanent and self.refresh_session()
if member and (create_member or member.session != self._session):
try:
self._client.kv.delete(self.member_path)
create_member = True
except Exception:
return False
if not create_member and member and deep_compare(
data, self._my_member_data):
return True
try:
args = {} if permanent else {'acquire': self._session}
self._client.kv.put(self.member_path,
json.dumps(data, separators=(',', ':')),
**args)
self._my_member_data = data
return True
except Exception:
logger.exception('touch_member')
return False
def touch_member(self, data, permanent=False):
cluster = self.cluster
if cluster and cluster.leader and cluster.leader.name == self._name:
role = 'promoted' if data['role'] in ('replica',
'promoted') else 'master'
elif data['state'] == 'running' and data['role'] != 'master':
role = data['role']
else:
role = None
member = cluster and cluster.get_member(self._name,
fallback_to_leader=False)
pod_labels = member and member.data.pop('pod_labels', None)
ret = pod_labels is not None and pod_labels.get(
self._role_label) == role and deep_compare(data, member.data)
if not ret:
metadata = {
'namespace': self._namespace,
'name': self._name,
'labels': {
self._role_label: role
},
'annotations': {
'status': json.dumps(data, separators=(',', ':'))
}
}
body = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(
**metadata))
ret = self._api.patch_namespaced_pod(self._name, self._namespace,
body)
if self.__subsets and self._should_create_config_service:
self._create_config_service()
return ret
def touch_member(self, data, permanent=False):
cluster = self.cluster
member = cluster and cluster.get_member(self._name,
fallback_to_leader=False)
create_member = not permanent and self.refresh_session()
if member and (create_member or member.session != self._session):
self._client.kv.delete(self.member_path)
create_member = True
if not create_member and member and deep_compare(data, member.data):
return True
try:
args = {} if permanent else {'acquire': self._session}
self._client.kv.put(self.member_path,
json.dumps(data, separators=(',', ':')),
**args)
if self._register_service:
self.update_service(
not create_member and member and member.data or {}, data)
return True
except InvalidSession:
self._session = None
logger.error(
'Our session disappeared from Consul, can not "touch_member"')
except Exception:
logger.exception('touch_member')
return False
def reload_local_configuration(self, dry_run=False):
if self.config_file:
try:
configuration = self._load_config_file()
if not deep_compare(self._local_configuration, configuration):
new_configuration = self._build_effective_configuration(self._dynamic_configuration, configuration)
if dry_run:
return not deep_compare(new_configuration, self.__effective_configuration)
self._local_configuration = configuration
self.__effective_configuration = new_configuration
return True
else:
logger.info('No configuration items changed, nothing to reload.')
except Exception:
logger.exception('Exception when reloading local configuration from %s', self.config_file)
if dry_run:
raise
def do_PUT_config(self):
request = self._read_json_content()
if request:
cluster = self.server.patroni.dcs.get_cluster()
if not deep_compare(request, cluster.config.data):
value = json.dumps(request, separators=(',', ':'))
if not self.server.patroni.dcs.set_config_value(value):
return self.send_error(502)
self._write_json_response(200, request)
def do_PUT_config(self):
request = self._read_json_content()
if request:
cluster = self.server.patroni.dcs.get_cluster()
if not deep_compare(request, cluster.config.data):
value = json.dumps(request, separators=(',', ':'))
if not self.server.patroni.dcs.set_config_value(value):
return self.send_error(502)
self._write_json_response(200, request)
def post_bootstrap(self, config, task):
try:
postgresql = self._postgresql
superuser = postgresql.config.superuser
if 'username' in superuser and 'password' in superuser:
self.create_or_update_role(superuser['username'], superuser['password'], ['SUPERUSER'])
task.complete(self.call_post_bootstrap(config))
if task.result:
replication = postgresql.config.replication
self.create_or_update_role(replication['username'], replication.get('password'), ['REPLICATION'])
rewind = postgresql.config.rewind_credentials
if not deep_compare(rewind, superuser):
self.create_or_update_role(rewind['username'], rewind.get('password'), [])
for f in ('pg_ls_dir(text, boolean, boolean)', 'pg_stat_file(text, boolean)',
'pg_read_binary_file(text)', 'pg_read_binary_file(text, bigint, bigint, boolean)'):
sql = """DO $$
BEGIN
SET local synchronous_commit = 'local';
GRANT EXECUTE ON function pg_catalog.{0} TO "{1}";
END;$$""".format(f, rewind['username'])
postgresql.query(sql)
for name, value in (config.get('users') or {}).items():
if all(name != a.get('username') for a in (superuser, replication, rewind)):
self.create_or_update_role(name, value.get('password'), value.get('options', []))
# We were doing a custom bootstrap instead of running initdb, therefore we opened trust
# access from certain addresses to be able to reach cluster and change password
if self._running_custom_bootstrap:
self._running_custom_bootstrap = False
# If we don't have custom configuration for pg_hba.conf we need to restore original file
if not postgresql.config.get('pg_hba'):
if os.path.exists(postgresql.config.pg_hba_conf):
os.unlink(postgresql.config.pg_hba_conf)
postgresql.config.restore_configuration_files()
postgresql.config.write_postgresql_conf()
postgresql.config.replace_pg_ident()
# at this point there should be no recovery.conf
postgresql.config.remove_recovery_conf()
if postgresql.config.hba_file:
postgresql.restart()
else:
postgresql.config.replace_pg_hba()
if postgresql.pending_restart:
postgresql.restart()
else:
postgresql.reload()
time.sleep(1) # give a time to postgres to "reload" configuration files
postgresql.connection().close() # close connection to reconnect with a new password
except Exception:
logger.exception('post_bootstrap')
task.complete(False)
return task.result
def touch_member(self, data, permanent=False):
cluster = self.cluster
member = cluster and cluster.get_member(self._name,
fallback_to_leader=False)
encoded_data = json.dumps(data, separators=(',', ':')).encode('utf-8')
if member and (self._client.client_id is not None
and member.session != self._client.client_id[0] or
not (deep_compare(member.data.get('tags', {}),
data.get('tags', {})) and
member.data.get('version') == data.get('version')
and member.data.get('checkpoint_after_promote')
== data.get('checkpoint_after_promote'))):
try:
self._client.delete_async(self.member_path).get(timeout=1)
except NoNodeError:
pass
except Exception:
return False
member = None
if member:
if deep_compare(data, member.data):
return True
else:
try:
self._client.create_async(
self.member_path,
encoded_data,
makepath=True,
ephemeral=not permanent).get(timeout=1)
return True
except Exception as e:
if not isinstance(e, NodeExistsError):
logger.exception('touch_member')
return False
try:
self._client.set_async(self.member_path,
encoded_data).get(timeout=1)
return True
except Exception:
logger.exception('touch_member')
return False
def reload_config(self, config):
if self._config is None or not deep_compare(self._config, config):
with self._queue_handler.queue.mutex:
self._queue_handler.queue.maxsize = config.get(
'max_queue_size', self.DEFAULT_MAX_QUEUE_SIZE)
self._root_logger.setLevel(
config.get('level', PatroniLogger.DEFAULT_LEVEL))
if config.get(
'traceback_level',
PatroniLogger.DEFAULT_TRACEBACK_LEVEL).lower() == 'debug':
logging.Logger.exception = debug_exception
else:
logging.Logger.exception = error_exception
new_handler = None
if 'dir' in config:
if not isinstance(self.log_handler, RotatingFileHandler):
new_handler = RotatingFileHandler(
os.path.join(config['dir'], __name__))
handler = new_handler or self.log_handler
handler.maxBytes = int(config.get('file_size', 25000000))
handler.backupCount = int(config.get('file_num', 4))
else:
if self.log_handler is None or isinstance(
self.log_handler, RotatingFileHandler):
new_handler = logging.StreamHandler()
handler = new_handler or self.log_handler
oldlogformat = (self._config
or {}).get('format', PatroniLogger.DEFAULT_FORMAT)
logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT)
olddateformat = (self._config or {}).get('dateformat') or None
dateformat = config.get(
'dateformat') or None # Convert empty string to `None`
if oldlogformat != logformat or olddateformat != dateformat or new_handler:
handler.setFormatter(logging.Formatter(logformat, dateformat))
if new_handler:
with self.log_handler_lock:
if self.log_handler:
self._old_handlers.append(self.log_handler)
self.log_handler = new_handler
self._config = config.copy()
self.update_loggers()
def set_dynamic_configuration(self, configuration):
if isinstance(configuration, ClusterConfig):
if self._modify_index == configuration.modify_index:
return False # If the index didn't changed there is nothing to do
self._modify_index = configuration.modify_index
configuration = configuration.data
if not deep_compare(self._dynamic_configuration, configuration):
try:
self.__effective_configuration = self._build_effective_configuration(configuration,
self._local_configuration)
self._dynamic_configuration = configuration
self._cache_needs_saving = True
return True
except Exception:
logger.exception('Exception when setting dynamic_configuration')
def set_dynamic_configuration(self, configuration):
if isinstance(configuration, ClusterConfig):
if self._modify_index == configuration.modify_index:
return False # If the index didn't changed there is nothing to do
self._modify_index = configuration.modify_index
configuration = configuration.data
if not deep_compare(self._dynamic_configuration, configuration):
try:
self.__effective_configuration = self._build_effective_configuration(configuration,
self._local_configuration)
self._dynamic_configuration = configuration
self._cache_needs_saving = True
return True
except Exception:
logger.exception('Exception when setting dynamic_configuration')
def reload_config(self, config):
if self.config is None or not deep_compare(self.config, config):
self.root_logger.setLevel(
config.get('level', PatroniLogger.DEFAULT_LEVEL))
add_handler = None
if 'dir' in config:
if not isinstance(self.handler, RotatingFileHandler):
add_handler = RotatingFileHandler(
os.path.join(config['dir'], __name__))
handler = add_handler or self.handler
handler.maxBytes = int(config.get('file_size', 25000000))
handler.backupCount = int(config.get('file_num', 4))
else:
if self.handler is None or isinstance(self.handler,
RotatingFileHandler):
add_handler = logging.StreamHandler()
handler = add_handler or self.handler
oldlogformat = (self.config
or {}).get('format', PatroniLogger.DEFAULT_FORMAT)
logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT)
olddateformat = (self.config or {}).get('dateformat') or None
dateformat = config.get(
'dateformat') or None # Convert empty string to `None`
if oldlogformat != logformat or olddateformat != dateformat or add_handler:
handler.setFormatter(logging.Formatter(logformat, dateformat))
if add_handler:
self.root_logger.addHandler(add_handler)
if self.handler is not None:
self.root_logger.removeHandler(self.handler)
self.handler.close()
self.handler = add_handler
self.config = config.copy()
self.update_loggers()
def touch_member(self, data, ttl=None, permanent=False):
cluster = self.cluster
member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
create_member = not permanent and self.refresh_session()
if member and (create_member or member.session != self._session):
try:
self._client.kv.delete(self.member_path)
create_member = True
except Exception:
return False
if not create_member and member and deep_compare(data, self._my_member_data):
return True
try:
args = {} if permanent else {'acquire': self._session}
self._client.kv.put(self.member_path, json.dumps(data, separators=(',', ':')), **args)
self._my_member_data = data
return True
except Exception:
logger.exception('touch_member')
return False
def touch_member(self, data, ttl=None, permanent=False):
cluster = self.cluster
if cluster and cluster.leader and cluster.leader.name == self._name:
role = 'master'
elif data['state'] == 'running' and data['role'] != 'master':
role = data['role']
else:
role = None
member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
pod_labels = member and member.data.pop('pod_labels', None)
ret = pod_labels is not None and pod_labels.get(self._role_label) == role and deep_compare(data, member.data)
if not ret:
metadata = {'namespace': self._namespace, 'name': self._name, 'labels': {self._role_label: role},
'annotations': {'status': json.dumps(data, separators=(',', ':'))}}
body = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(**metadata))
ret = self._api.patch_namespaced_pod(self._name, self._namespace, body)
return ret
