def __init__(self, config): super(Consul, self).__init__(config) self._scope = config['scope'] self._session = None self.__do_not_watch = False self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1, retry_exceptions=(ConsulInternalError, HTTPException, HTTPError, socket.error, socket.timeout)) kwargs = {} if 'url' in config: r = urlparse(config['url']) config.update({ 'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500 }) elif 'host' in config: host, port = split_host_port(config.get('host', '127.0.0.1:8500'), 8500) config['host'] = host if 'port' not in config: config['port'] = int(port) if config.get('cacert'): config['ca_cert'] = config.pop('cacert') if config.get('key') and config.get('cert'): config['cert'] = (config['cert'], config['key']) config_keys = ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert', 'dc') kwargs = {p: config.get(p) for p in config_keys if config.get(p)} verify = config.get('verify') if not isinstance(verify, bool): verify = parse_bool(verify) if isinstance(verify, bool): kwargs['verify'] = verify self._client = ConsulClient(**kwargs) self.set_retry_timeout(config['retry_timeout']) self.set_ttl(config.get('ttl') or 30) self._last_session_refresh = 0 self.__session_checks = config.get('checks') self._register_service = config.get('register_service', False) if self._register_service: self._service_name = service_name_from_scope_name(self._scope) if self._scope != self._service_name: logger.warning( 'Using %s as consul service name instead of scope name %s', self._service_name, self._scope) self._service_check_interval = config.get('service_check_interval', '5s') if not self._ctl: self.create_session()
def __init__(self, config): super(Consul, self).__init__(config) self._scope = config['scope'] self._session = None self.__do_not_watch = False self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1, retry_exceptions=(ConsulInternalError, HTTPException, HTTPError, socket.error, socket.timeout)) self._my_member_data = None kwargs = {} if 'url' in config: r = urlparse(config['url']) config.update({ 'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500 }) elif 'host' in config: host, port = (config.get('host', '127.0.0.1:8500') + ':8500').split(':')[:2] config['host'] = host if 'port' not in config: config['port'] = int(port) if config.get('cacert'): config['ca_cert'] = config.pop('cacert') if config.get('key') and config.get('cert'): config['cert'] = (config['cert'], config['key']) config_keys = ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert', 'dc') kwargs = {p: config.get(p) for p in config_keys if config.get(p)} verify = config.get('verify') if not isinstance(verify, bool): verify = parse_bool(verify) if isinstance(verify, bool): kwargs['verify'] = verify self._client = ConsulClient(**kwargs) self.set_retry_timeout(config['retry_timeout']) self.set_ttl(config.get('ttl') or 30) self._last_session_refresh = 0 self.__session_checks = config.get('checks') if not self._ctl: self.create_session()
def do_POST_reinitialize(self): request = self._read_json_content(body_is_optional=True) if request: logger.debug('received reinitialize request: %s', request) force = isinstance(request, dict) and parse_bool(request.get('force')) or False data = self.server.patroni.ha.reinitialize(force) if data is None: status_code = 200 data = 'reinitialize started' else: status_code = 503 self._write_response(status_code, data)
def do_POST_reinitialize(self): request = self._read_json_content(body_is_optional=True) if request: logger.debug('received reinitialize request: %s', request) force = isinstance(request, dict) and parse_bool(request.get('force')) or False data = self.server.patroni.ha.reinitialize(force) if data is None: status_code = 200 data = 'reinitialize started' else: status_code = 503 self._write_response(status_code, data)
def __init__(self, config): super(Consul, self).__init__(config) self._scope = config['scope'] self._session = None self.__do_not_watch = False self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1, retry_exceptions=(ConsulInternalError, HTTPException, HTTPError, socket.error, socket.timeout)) self._my_member_data = {} kwargs = {} if 'url' in config: r = urlparse(config['url']) config.update({'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500}) elif 'host' in config: host, port = split_host_port(config.get('host', '127.0.0.1:8500'), 8500) config['host'] = host if 'port' not in config: config['port'] = int(port) if config.get('cacert'): config['ca_cert'] = config.pop('cacert') if config.get('key') and config.get('cert'): config['cert'] = (config['cert'], config['key']) config_keys = ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert', 'dc') kwargs = {p: config.get(p) for p in config_keys if config.get(p)} verify = config.get('verify') if not isinstance(verify, bool): verify = parse_bool(verify) if isinstance(verify, bool): kwargs['verify'] = verify self._client = ConsulClient(**kwargs) self.set_retry_timeout(config['retry_timeout']) self.set_ttl(config.get('ttl') or 30) self._last_session_refresh = 0 self.__session_checks = config.get('checks') if not self._ctl: self.create_session()
def _build_environment_configuration(): ret = defaultdict(dict) def _popenv(name): return os.environ.pop(PATRONI_ENV_PREFIX + name.upper(), None) for param in ('name', 'namespace', 'scope'): value = _popenv(param) if value: ret[param] = value def _fix_log_env(name, oldname): value = _popenv(oldname) name = PATRONI_ENV_PREFIX + 'LOG_' + name.upper() if value and name not in os.environ: os.environ[name] = value for name, oldname in (('level', 'loglevel'), ('format', 'logformat'), ('dateformat', 'log_datefmt')): _fix_log_env(name, oldname) def _set_section_values(section, params): for param in params: value = _popenv(section + '_' + param) if value: ret[section][param] = value _set_section_values('restapi', [ 'listen', 'connect_address', 'certfile', 'keyfile', 'cafile', 'verify_client' ]) _set_section_values('ctl', ['insecure', 'cacert', 'certfile', 'keyfile']) _set_section_values('postgresql', [ 'listen', 'connect_address', 'config_dir', 'data_dir', 'pgpass', 'bin_dir' ]) _set_section_values('log', [ 'level', 'traceback_level', 'format', 'dateformat', 'max_queue_size', 'dir', 'file_size', 'file_num', 'loggers' ]) def _parse_dict(value): if not value.strip().startswith('{'): value = '{{{0}}}'.format(value) try: return yaml.safe_load(value) except Exception: logger.exception('Exception when parsing dict %s', value) return None value = ret.get('log', {}).pop('loggers', None) if value: value = _parse_dict(value) if value: ret['log']['loggers'] = value def _get_auth(name, params=None): ret = {} for param in params or _AUTH_ALLOWED_PARAMETERS[:2]: value = _popenv(name + '_' + param) if value: ret[param] = value return ret restapi_auth = _get_auth('restapi') if restapi_auth: ret['restapi']['authentication'] = restapi_auth authentication = {} for user_type in ('replication', 'superuser', 'rewind'): entry = _get_auth(user_type, _AUTH_ALLOWED_PARAMETERS) if entry: authentication[user_type] = entry if authentication: ret['postgresql']['authentication'] = authentication def _parse_list(value): if not (value.strip().startswith('-') or '[' in value): value = '[{0}]'.format(value) try: return yaml.safe_load(value) except Exception: logger.exception('Exception when parsing list %s', value) return None for param in list(os.environ.keys()): if param.startswith(PATRONI_ENV_PREFIX): # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..) name, suffix = (param[8:].split('_', 1) + [''])[:2] if suffix in ('HOST', 'HOSTS', 'PORT', 'USE_PROXIES', 'PROTOCOL', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'CONSISTENCY', 'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL', 'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS') and name: value = os.environ.pop(param) if suffix == 'PORT': value = value and parse_int(value) elif suffix in ('HOSTS', 'PORTS', 'CHECKS'): value = value and _parse_list(value) elif suffix == 'LABELS': value = _parse_dict(value) elif suffix in ('USE_PROXIES', 'REGISTER_SERVICE'): value = parse_bool(value) if value: ret[name.lower()][suffix.lower()] = value if 'etcd' in ret: ret['etcd'].update(_get_auth('etcd')) users = {} for param in list(os.environ.keys()): if param.startswith(PATRONI_ENV_PREFIX): name, suffix = (param[8:].rsplit('_', 1) + [''])[:2] # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...> # CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>' if name and suffix == 'PASSWORD': password = os.environ.pop(param) if password: users[name] = {'password': password} options = os.environ.pop(param[:-9] + '_OPTIONS', None) options = options and _parse_list(options) if options: users[name]['options'] = options if users: ret['bootstrap']['users'] = users return ret
def check_mode(self, mode): return bool(parse_bool(self._dynamic_configuration.get(mode)))
def check_mode(self, mode): return bool(self.config and parse_bool(self.config.data.get(mode)))
class Postgresql(object): # List of parameters which must be always passed to postmaster as command line options # to make it not possible to change them with 'ALTER SYSTEM'. # Some of these parameters have sane default value assigned and Patroni doesn't allow # to decrease this value. E.g. 'wal_level' can't be lower then 'hot_standby' and so on. # These parameters could be changed only globally, i.e. via DCS. # P.S. 'listen_addresses' and 'port' are added here just for convenience, to mark them # as a parameters which should always be passed through command line. # # Format: # key - parameter name # value - tuple(default_value, check_function, min_version) # default_value -- some sane default value # check_function -- if the new value is not correct must return `!False` # min_version -- major version of PostgreSQL when parameter was introduced CMDLINE_OPTIONS = { 'listen_addresses': (None, lambda _: False, 9.1), 'port': (None, lambda _: False, 9.1), 'cluster_name': (None, lambda _: False, 9.5), 'wal_level': ('hot_standby', lambda v: v.lower() in ('hot_standby', 'logical'), 9.1), 'hot_standby': ('on', lambda _: False, 9.1), 'max_connections': (100, lambda v: int(v) >= 100, 9.1), 'max_wal_senders': (5, lambda v: int(v) >= 5, 9.1), 'wal_keep_segments': (8, lambda v: int(v) >= 8, 9.1), 'max_prepared_transactions': (0, lambda v: int(v) >= 0, 9.1), 'max_locks_per_transaction': (64, lambda v: int(v) >= 64, 9.1), 'track_commit_timestamp': ('off', lambda v: parse_bool(v) is not None, 9.5), 'max_replication_slots': (5, lambda v: int(v) >= 5, 9.4), 'max_worker_processes': (8, lambda v: int(v) >= 8, 9.4), 'wal_log_hints': ('on', lambda _: False, 9.4) } def __init__(self, config): self.config = config self.name = config['name'] self.scope = config['scope'] self._database = config.get('database', 'postgres') self._data_dir = config['data_dir'] self._pending_restart = False self._server_parameters = self.get_server_parameters(config) self._connect_address = config.get('connect_address') self._superuser = config['authentication'].get('superuser', {}) self._replication = config['authentication']['replication'] self.resolve_connection_addresses() self._need_rewind = False self._use_slots = config.get('use_slots', True) self._version_file = os.path.join(self._data_dir, 'PG_VERSION') self._major_version = self.get_major_version() self._schedule_load_slots = self.use_slots self._pgpass = config.get('pgpass') or os.path.join( os.path.expanduser('~'), 'pgpass') self.callback = config.get('callbacks') or {} config_base_name = config.get('config_base_name', 'postgresql') self._postgresql_conf = os.path.join(self._data_dir, config_base_name + '.conf') self._postgresql_base_conf_name = config_base_name + '.base.conf' self._postgresql_base_conf = os.path.join( self._data_dir, self._postgresql_base_conf_name) self._recovery_conf = os.path.join(self._data_dir, 'recovery.conf') self._configuration_to_save = (self._postgresql_conf, self._postgresql_base_conf, os.path.join(self._data_dir, 'pg_hba.conf')) self._postmaster_pid = os.path.join(self._data_dir, 'postmaster.pid') self._trigger_file = config.get('recovery_conf', {}).get('trigger_file') or 'promote' self._trigger_file = os.path.abspath( os.path.join(self._data_dir, self._trigger_file)) self._connection = None self._cursor_holder = None self._sysid = None self._replication_slots = [ ] # list of already existing replication slots self.retry = Retry(max_tries=-1, deadline=config['retry_timeout'] / 2.0, max_delay=1, retry_exceptions=PostgresConnectionException) self._state_lock = Lock() self.set_state('stopped') self._role_lock = Lock() self.set_role(self.get_postgres_role_from_data_directory()) if self.is_running(): self.set_state('running') self.set_role('master' if self.is_leader() else 'replica') self._write_postgresql_conf( ) # we are "joining" already running postgres @property def use_slots(self): return self._use_slots and self._major_version >= 9.4 def _version_file_exists(self): return not self.data_directory_empty() and os.path.isfile( self._version_file) def get_major_version(self): if self._version_file_exists(): try: with open(self._version_file) as f: return float(f.read()) except Exception: logger.exception('Failed to read PG_VERSION from %s', self._data_dir) return 0.0 def get_server_parameters(self, config): parameters = config['parameters'].copy() listen_addresses, port = (config['listen'] + ':5432').split(':')[:2] parameters.update({ 'cluster_name': self.scope, 'listen_addresses': listen_addresses, 'port': port }) return parameters def resolve_connection_addresses(self): self._local_address = self.get_local_address() self.connection_string = 'postgres://{connect_address}/{database}'.format( connect_address=self._connect_address or self._local_address, database=self._database) def pg_ctl(self, cmd, *args, **kwargs): """Builds and executes pg_ctl command :returns: `!True` when return_code == 0, otherwise `!False`""" pg_ctl = ['pg_ctl', cmd] if cmd in ('start', 'stop', 'restart'): pg_ctl += ['-w'] timeout = self.config.get('pg_ctl_timeout') if timeout: try: pg_ctl += ['-t', str(int(timeout))] except Exception: logger.error('Bad value of pg_ctl_timeout: %s', timeout) return subprocess.call(pg_ctl + ['-D', self._data_dir] + list(args), **kwargs) == 0 def reload_config(self, config): server_parameters = self.get_server_parameters(config) listen_address_changed = pending_reload = pending_restart = False if self.is_healthy(): changes = { p: v for p, v in server_parameters.items() if '.' not in p } changes.update({ p: None for p, v in self._server_parameters.items() if not ('.' in p or p in changes) }) if changes: if 'wal_segment_size' not in changes: changes['wal_segment_size'] = '16384kB' # XXX: query can raise an exception for r in self.query( """SELECT name, setting, unit, vartype, context FROM pg_settings WHERE name IN (""" + ', '.join(['%s'] * len(changes)) + """) ORDER BY 1 DESC""", *(list(changes.keys()))): if r[4] == 'internal': if r[0] == 'wal_segment_size': server_parameters.pop(r[0], None) wal_segment_size = parse_int(r[2], 'kB') if wal_segment_size is not None: changes['wal_segment_size'] = '{0}kB'.format( int(r[1]) * wal_segment_size) elif r[0] in changes: unit = changes['wal_segment_size'] if r[0] in ( 'min_wal_size', 'max_wal_size') else r[2] new_value = changes.pop(r[0]) if new_value is None or not compare_values( r[3], unit, r[1], new_value): if r[4] == 'postmaster': pending_restart = True if r[0] in ('listen_addresses', 'port'): listen_address_changed = True else: pending_reload = True for param in changes: if param in server_parameters: logger.warning( 'Removing invalid parameter `%s` from postgresql.parameters', param) server_parameters.pop(param) # Check that user-defined-paramters have changed (parameters with period in name) if not pending_reload: for p, v in server_parameters.items(): if '.' in p and (p not in self._server_parameters or str(v) != str(self._server_parameters[p])): pending_reload = True break if not pending_reload: for p, v in self._server_parameters.items(): if '.' in p and (p not in server_parameters or str(v) != str(server_parameters[p])): pending_reload = True break self.config = config self._pending_restart = pending_restart self._server_parameters = server_parameters self._connect_address = config.get('connect_address') if not listen_address_changed: self.resolve_connection_addresses() if pending_reload: self._write_postgresql_conf() self.reload() self.retry.deadline = config['retry_timeout'] / 2.0 @property def pending_restart(self): return self._pending_restart @property def can_rewind(self): """ check if pg_rewind executable is there and that pg_controldata indicates we have either wal_log_hints or checksums turned on """ # low-hanging fruit: check if pg_rewind configuration is there if not (self.config.get('use_pg_rewind') and all( self._superuser.get(n) for n in ('username', 'password'))): return False cmd = ['pg_rewind', '--help'] try: ret = subprocess.call(cmd, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT) if ret != 0: # pg_rewind is not there, close up the shop and go home return False except OSError: return False # check if the cluster's configuration permits pg_rewind data = self.controldata() return data.get('wal_log_hints setting', 'off') == 'on' or data.get( 'Data page checksum version', '0') != '0' @property def sysid(self): if not self._sysid: data = self.controldata() self._sysid = data.get('Database system identifier', "") return self._sysid def get_local_address(self): listen_addresses = self._server_parameters['listen_addresses'].split( ',') local_address = listen_addresses[0].strip( ) # take first address from listen_addresses for la in listen_addresses: if la.strip().lower() in ( '*', '0.0.0.0', '127.0.0.1', 'localhost'): # we are listening on '*' or localhost local_address = 'localhost' # connection via localhost is preferred break return local_address + ':' + self._server_parameters['port'] def get_postgres_role_from_data_directory(self): if self.data_directory_empty(): return 'uninitialized' elif os.path.exists(self._recovery_conf): return 'replica' else: return 'master' @property def _connect_kwargs(self): return get_conn_kwargs( 'postgres://{0}/{1}'.format(self._local_address, self._database), self._superuser) def connection(self): if not self._connection or self._connection.closed != 0: self._connection = psycopg2.connect(**self._connect_kwargs) self._connection.autocommit = True self.server_version = self._connection.server_version return self._connection def _cursor(self): if not self._cursor_holder or self._cursor_holder.closed or self._cursor_holder.connection.closed != 0: logger.info( "establishing a new patroni connection to the postgres cluster" ) self._cursor_holder = self.connection().cursor() return self._cursor_holder def close_connection(self): if self._cursor_holder and self._cursor_holder.connection and self._cursor_holder.connection.closed == 0: self._cursor_holder.connection.close() logger.info("closed patroni connection to the postgresql cluster") def _query(self, sql, *params): cursor = None try: cursor = self._cursor() cursor.execute(sql, params) return cursor except psycopg2.Error as e: if cursor and cursor.connection.closed == 0: raise e if self.state == 'restarting': raise RetryFailedError('cluster is being restarted') raise PostgresConnectionException('connection problems') def query(self, sql, *params): try: return self.retry(self._query, sql, *params) except RetryFailedError as e: raise PostgresConnectionException(str(e)) def data_directory_empty(self): return not os.path.exists(self._data_dir) or os.listdir( self._data_dir) == [] @staticmethod def initdb_allowed_option(name): if name in ['pgdata', 'nosync', 'pwfile', 'sync-only']: raise Exception( '{0} option for initdb is not allowed'.format(name)) return True def get_initdb_options(self, config): options = [] for o in config: if isinstance(o, string_types) and self.initdb_allowed_option(o): options.append('--{0}'.format(o)) elif isinstance(o, dict): keys = list(o.keys()) if len(keys) != 1 or not isinstance( keys[0], string_types) or not self.initdb_allowed_option( keys[0]): raise Exception('Invalid option: {0}'.format(o)) options.append('--{0}={1}'.format(keys[0], o[keys[0]])) else: raise Exception('Unknown type of initdb option: {0}'.format(o)) return options def _initialize(self, config): self.set_state('initalizing new cluster') options = self.get_initdb_options(config.get('initdb') or []) pwfile = None if self._superuser: if 'username' in self._superuser: options.append('--username={0}'.format( self._superuser['username'])) if 'password' in self._superuser: (fd, pwfile) = tempfile.mkstemp() os.write(fd, self._superuser['password'].encode('utf-8')) os.close(fd) options.append('--pwfile={0}'.format(pwfile)) options = ['-o', ' '.join(options)] if options else [] ret = self.pg_ctl('initdb', *options) if pwfile: os.remove(pwfile) if ret: self.write_pg_hba(config.get('pg_hba', [])) self._major_version = self.get_major_version() else: self.set_state('initdb failed') return ret def delete_trigger_file(self): if os.path.exists(self._trigger_file): os.unlink(self._trigger_file) def write_pgpass(self, record): with open(self._pgpass, 'w') as f: os.fchmod(f.fileno(), 0o600) f.write('{host}:{port}:*:{user}:{password}\n'.format(**record)) env = os.environ.copy() env['PGPASSFILE'] = self._pgpass return env def replica_method_can_work_without_replication_connection(self, method): return method != 'basebackup' and self.config and self.config.get( method, {}).get('no_master') def can_create_replica_without_replication_connection(self): """ go through the replication methods to see if there are ones that does not require a working replication connection. """ replica_methods = self.config.get('create_replica_method', []) return any( self.replica_method_can_work_without_replication_connection(method) for method in replica_methods) def create_replica(self, clone_member): """ create the replica according to the replica_method defined by the user. this is a list, so we need to loop through all methods the user supplies """ self.set_state('creating replica') self._sysid = None # get list of replica methods from config. # If there is no configuration key, or no value is specified, use basebackup replica_methods = self.config.get('create_replica_method') or [ 'basebackup' ] if clone_member: r = get_conn_kwargs(clone_member.conn_url, self._replication) connstring = 'postgres://{user}@{host}:{port}/{database}'.format( **r) # add the credentials to connect to the replica origin to pgpass. env = self.write_pgpass(r) else: connstring = '' env = os.environ.copy() # if we don't have any source, leave only replica methods that work without it replica_methods = \ [r for r in replica_methods if self.replica_method_can_work_without_replication_connection(r)] # go through them in priority order ret = 1 for replica_method in replica_methods: # if the method is basebackup, then use the built-in if replica_method == "basebackup": ret = self.basebackup(connstring, env) if ret == 0: logger.info("replica has been created using basebackup") # if basebackup succeeds, exit with success break else: cmd = replica_method method_config = {} # user-defined method; check for configuration # not required, actually if replica_method in self.config: method_config = self.config[replica_method].copy() # look to see if the user has supplied a full command path # if not, use the method name as the command cmd = method_config.pop('command', cmd) # add the default parameters try: method_config.update({ "scope": self.scope, "role": "replica", "datadir": self._data_dir, "connstring": connstring }) params = [ "--{0}={1}".format(arg, val) for arg, val in method_config.items() ] # call script with the full set of parameters ret = subprocess.call(shlex.split(cmd) + params, env=env) # if we succeeded, stop if ret == 0: logger.info('replica has been created using %s', replica_method) break except Exception: logger.exception('Error creating replica using method %s', replica_method) ret = 1 self.set_state('stopped') return ret def is_leader(self): return not self.query('SELECT pg_is_in_recovery()').fetchone()[0] def is_running(self): if not (self._version_file_exists() and os.path.isfile(self._postmaster_pid)): return False try: with open(self._postmaster_pid) as f: pid = int(f.readline()) if pid < 0: pid = -pid return pid > 0 and pid != os.getpid() and pid != os.getppid( ) and (os.kill(pid, 0) or True) except Exception: return False def call_nowait(self, cb_name): """ pick a callback command and call it without waiting for it to finish """ if not self.callback or cb_name not in self.callback: return False cmd = self.callback[cb_name] try: subprocess.Popen( shlex.split(cmd) + [cb_name, self.role, self.scope]) except OSError: logger.exception('callback %s %s %s %s failed', cmd, cb_name, self.role, self.scope) return False return True @property def role(self): with self._role_lock: return self._role def set_role(self, value): with self._role_lock: self._role = value @property def state(self): with self._state_lock: return self._state def set_state(self, value): with self._state_lock: self._state = value def start(self, block_callbacks=False): if self.is_running(): logger.error( 'Cannot start PostgreSQL because one is already running.') return True self.set_role(self.get_postgres_role_from_data_directory()) if os.path.exists(self._postmaster_pid): os.remove(self._postmaster_pid) logger.info('Removed %s', self._postmaster_pid) if not block_callbacks: self.set_state('starting') env = {'PATH': os.environ.get('PATH')} # pg_ctl will write a FATAL if the username is incorrect. exporting PGUSER if necessary if 'username' in self._superuser and self._superuser[ 'username'] != os.environ.get('USER'): env['PGUSER'] = self._superuser['username'] self._write_postgresql_conf() self.resolve_connection_addresses() options = ' '.join("--{0}='{1}'".format(p, self._server_parameters[p]) for p, v in self.CMDLINE_OPTIONS.items() if self._major_version >= v[2]) ret = self.pg_ctl('start', '-o', options, env=env, preexec_fn=os.setsid) self._pending_restart = False self.set_state('running' if ret else 'start failed') self._schedule_load_slots = ret and self.use_slots self.save_configuration_files() # block_callbacks is used during restart to avoid # running start/stop callbacks in addition to restart ones if ret and not block_callbacks: self.call_nowait(ACTION_ON_START) return ret def checkpoint(self, connect_kwargs=None): check_not_is_in_recovery = connect_kwargs is not None connect_kwargs = connect_kwargs or self._connect_kwargs for p in ['connect_timeout', 'options']: connect_kwargs.pop(p, None) try: with psycopg2.connect(**connect_kwargs) as conn: conn.autocommit = True with conn.cursor() as cur: cur.execute("SET statement_timeout = 0") if check_not_is_in_recovery: cur.execute('SELECT pg_is_in_recovery()') if cur.fetchone()[0]: return 'is_in_recovery=true' return cur.execute('CHECKPOINT') except psycopg2.Error: logging.exception('Exception during CHECKPOINT') return 'not accessible or not healty' def stop(self, mode='fast', block_callbacks=False, checkpoint=True): # make sure we close all connections established against # the former node, otherwise, we might get a stalled one # after kill -9, which would report incorrect data to # patroni. self.close_connection() if not self.is_running(): if not block_callbacks: self.set_state('stopped') return True if checkpoint: self.checkpoint() if not block_callbacks: self.set_state('stopping') ret = self.pg_ctl('stop', '-m', mode) # block_callbacks is used during restart to avoid # running start/stop callbacks in addition to restart ones if not ret: logger.warning('pg_ctl stop failed') self.set_state('stop failed') elif not block_callbacks: self.set_state('stopped') self.call_nowait(ACTION_ON_STOP) return ret def reload(self): ret = self.pg_ctl('reload') if ret: self.call_nowait(ACTION_ON_RELOAD) return ret def restart(self): self.set_state('restarting') ret = self.stop(block_callbacks=True) and self.start( block_callbacks=True) if ret: self.call_nowait(ACTION_ON_RESTART) else: self.set_state('restart failed ({0})'.format(self.state)) return ret def _write_postgresql_conf(self): # rename the original configuration if it is necessary if not os.path.exists(self._postgresql_base_conf): os.rename(self._postgresql_conf, self._postgresql_base_conf) with open(self._postgresql_conf, 'w') as f: f.write( '# Do not edit this file manually!\n# It will be overwritten by Patroni!\n' ) f.write("include '{0}'\n\n".format( self._postgresql_base_conf_name)) for name, value in sorted(self._server_parameters.items()): if name not in self.CMDLINE_OPTIONS: f.write("{0} = '{1}'\n".format(name, value)) def is_healthy(self): if not self.is_running(): logger.warning('Postgresql is not running.') return False return True def check_replication_lag(self, last_leader_operation): return (last_leader_operation or 0) - self.xlog_position() <= self.config.get( 'maximum_lag_on_failover', 0) def write_pg_hba(self, config): with open(os.path.join(self._data_dir, 'pg_hba.conf'), 'a') as f: f.write('\n{}\n'.format('\n'.join(config))) def primary_conninfo(self, node_to_follow_url): r = get_conn_kwargs(node_to_follow_url, self._replication) r.update({ 'application_name': self.name, 'sslmode': 'prefer', 'sslcompression': '1' }) keywords = 'user password host port sslmode sslcompression application_name'.split( ) return ' '.join('{0}={{{0}}}'.format(kw) for kw in keywords).format(**r) def check_recovery_conf(self, node_to_follow): if not os.path.isfile(self._recovery_conf): return False pattern = node_to_follow and node_to_follow.conn_url and self.primary_conninfo( node_to_follow.conn_url) with open(self._recovery_conf, 'r') as f: for line in f: if line.startswith('primary_conninfo'): return pattern and (pattern in line) return not pattern def write_recovery_conf(self, node_to_follow): with open(self._recovery_conf, 'w') as f: f.write( "standby_mode = 'on'\nrecovery_target_timeline = 'latest'\n") if node_to_follow and node_to_follow.conn_url: f.write("primary_conninfo = '{0}'\n".format( self.primary_conninfo(node_to_follow.conn_url))) if self.use_slots: f.write("primary_slot_name = '{0}'\n".format(self.name)) for name, value in self.config.get('recovery_conf', {}).items(): if name not in ('standby_mode', 'recovery_target_timeline', 'primary_conninfo', 'primary_slot_name'): f.write("{0} = '{1}'\n".format(name, value)) def rewind(self, r): # prepare pg_rewind connection env = self.write_pgpass(r) dsn = 'user={user} host={host} port={port} dbname={database} sslmode=prefer sslcompression=1'.format( **r) logger.info('running pg_rewind from %s', dsn) try: return subprocess.call( ['pg_rewind', '-D', self._data_dir, '--source-server', dsn], env=env) == 0 except OSError: return False def controldata(self): """ return the contents of pg_controldata, or non-True value if pg_controldata call failed """ result = {} # Don't try to call pg_controldata during backup restore if self._version_file_exists() and self.state != 'creating replica': try: data = subprocess.check_output( ['pg_controldata', self._data_dir]) if data: data = data.decode('utf-8').splitlines() result = { l.split(':')[0].replace('Current ', '', 1): l.split(':')[1].strip() for l in data if l } except subprocess.CalledProcessError: logger.exception("Error when calling pg_controldata") return result def read_postmaster_opts(self): """ returns the list of option names/values from postgres.opts, Empty dict if read failed or no file """ result = {} try: with open(os.path.join(self._data_dir, "postmaster.opts")) as f: data = f.read() opts = [opt.strip('"\n') for opt in data.split(' "')] for opt in opts: if '=' in opt and opt.startswith('--'): name, val = opt.split('=', 1) name = name.strip('-') result[name] = val except IOError: logger.exception('Error when reading postmaster.opts') return result def single_user_mode(self, command=None, options=None): """ run a given command in a single-user mode. If the command is empty - then just start and stop """ cmd = ['postgres', '--single', '-D', self._data_dir] for opt, val in sorted((options or {}).items()): cmd.extend(['-c', '{0}={1}'.format(opt, val)]) # need a database name to connect cmd.append(self._database) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT) if p: if command: p.communicate('{0}\n'.format(command)) p.stdin.close() return p.wait() return 1 def cleanup_archive_status(self): status_dir = os.path.join(self._data_dir, 'pg_xlog', 'archive_status') try: for f in os.listdir(status_dir): path = os.path.join(status_dir, f) try: if os.path.islink(path): os.unlink(path) elif os.path.isfile(path): os.remove(path) except OSError: logger.exception("Unable to remove %s", path) except OSError: logger.exception("Unable to list %s", status_dir) def follow(self, member, leader, recovery=False): if self.check_recovery_conf(member) and not recovery: return True change_role = self.role == 'master' if change_role: if leader: if leader.name == self.name: self._need_rewind = False member = None if self.is_running(): return else: self._need_rewind = bool( leader.conn_url) and self.can_rewind else: self._need_rewind = False member = None if self._need_rewind: logger.info("set the rewind flag after demote") self.set_role('unknown') if self.is_running() and not self.stop(): return logger.warning( 'Can not run pg_rewind because postgres is still running') if not (leader and leader.conn_url): return logger.info('Leader unknown, can not rewind') # prepare pg_rewind connection r = get_conn_kwargs(leader.conn_url, self._superuser) # first make sure that we are really trying to rewind # from the master and run a checkpoint on a t in order to # make it store the new timeline ([email protected]) leader_status = self.checkpoint(r) if leader_status: return logger.warning('Can not use %s for rewind: %s', leader.name, leader_status) # at present, pg_rewind only runs when the cluster is shut down cleanly # and not shutdown in recovery. We have to remove the recovery.conf if present # and start/shutdown in a single user mode to emulate this. # XXX: if recovery.conf is linked, it will be written anew as a normal file. if os.path.isfile(self._recovery_conf) or os.path.islink( self._recovery_conf): os.unlink(self._recovery_conf) # Archived segments might be useful to pg_rewind, # clean the flags that tell we should remove them. self.cleanup_archive_status() # Start in a single user mode and stop to produce a clean shutdown opts = self.read_postmaster_opts() opts.update({'archive_mode': 'on', 'archive_command': 'false'}) self.single_user_mode(options=opts) if self.rewind(r) or not self.config.get( 'remove_data_directory_on_rewind_failure', False): self.write_recovery_conf(member) ret = self.start() else: logger.error('unable to rewind the former master') self.remove_data_directory() self.set_role('uninitialized') ret = True self._need_rewind = False else: self.write_recovery_conf(member) ret = self.restart() self.set_role('replica') if change_role: self.call_nowait(ACTION_ON_ROLE_CHANGE) return ret def save_configuration_files(self): """ copy postgresql.conf to postgresql.conf.backup to be able to retrive configuration files - originally stored as symlinks, those are normally skipped by pg_basebackup - in case of WAL-E basebackup (see http://comments.gmane.org/gmane.comp.db.postgresql.wal-e/239) """ try: for f in self._configuration_to_save: if os.path.isfile(f): shutil.copy(f, f + '.backup') except IOError: logger.exception( 'unable to create backup copies of configuration files') def restore_configuration_files(self): """ restore a previously saved postgresql.conf """ try: for f in self._configuration_to_save: if not os.path.isfile(f) and os.path.isfile(f + '.backup'): shutil.copy(f + '.backup', f) except IOError: logger.exception( 'unable to restore configuration files from backup') def promote(self): if self.role == 'master': return True ret = self.pg_ctl('promote') if ret: self.set_role('master') logger.info("cleared rewind flag after becoming the leader") self._need_rewind = False self.call_nowait(ACTION_ON_ROLE_CHANGE) return ret def create_or_update_role(self, name, password, options): options = list(map(str.upper, options)) if 'NOLOGIN' not in options and 'LOGIN' not in options: options.append('LOGIN') self.query( """DO $$ BEGIN SET local synchronous_commit = 'local'; PERFORM * FROM pg_authid WHERE rolname = %s; IF FOUND THEN ALTER ROLE "{0}" WITH {1} PASSWORD %s; ELSE CREATE ROLE "{0}" WITH {1} PASSWORD %s; END IF; END; $$""".format(name, ' '.join(options)), name, password, password) def xlog_position(self): return self.query( """SELECT pg_xlog_location_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_xlog_replay_location() ELSE pg_current_xlog_location() END, '0/0')::bigint""" ).fetchone()[0] def load_replication_slots(self): if self.use_slots and self._schedule_load_slots: cursor = self.query( "SELECT slot_name FROM pg_replication_slots WHERE slot_type='physical'" ) self._replication_slots = [r[0] for r in cursor] self._schedule_load_slots = False def postmaster_start_time(self): try: cursor = self.query( """SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ')""" ) return cursor.fetchone()[0] except psycopg2.Error: return None def sync_replication_slots(self, cluster): if self.use_slots: try: self.load_replication_slots() # if the replicatefrom tag is set on the member - we should not create the replication slot for it on # the current master, because that member would replicate from elsewhere. We still create the slot if # the replicatefrom destination member is currently not a member of the cluster (fallback to the # master), or if replicatefrom destination member happens to be the current master if self.role == 'master': slots = [ m.name for m in cluster.members if m.name != self.name and (m.replicatefrom is None or m.replicatefrom == self.name or not cluster.has_member(m.replicatefrom)) ] else: # only manage slots for replicas that replicate from this one, except for the leader among them slots = [ m.name for m in cluster.members if m.replicatefrom == self.name and m.name != cluster.leader.name ] # drop unused slots for slot in set(self._replication_slots) - set(slots): self.query( """SELECT pg_drop_replication_slot(%s) WHERE EXISTS(SELECT 1 FROM pg_replication_slots WHERE slot_name = %s)""", slot, slot) # create new slots for slot in set(slots) - set(self._replication_slots): self.query( """SELECT pg_create_physical_replication_slot(%s) WHERE NOT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = %s)""", slot, slot) self._replication_slots = slots except psycopg2.Error: logger.exception('Exception when changing replication slots') def last_operation(self): return str(self.xlog_position()) def clone(self, clone_member): """ - initialize the replica from an existing member (master or replica) - initialize the replica using the replica creation method that works without the replication connection (i.e. restore from on-disk base backup) """ ret = self.create_replica(clone_member) == 0 if ret: self._major_version = self.get_major_version() self.delete_trigger_file() self.restore_configuration_files() return ret def bootstrap(self, config): """ Initialize a new node from scratch and start it. """ if self._initialize(config) and self.start(): for name, value in config['users'].items(): if name not in (self._superuser.get('username'), self._replication['username']): self.create_or_update_role(name, value['password'], value.get('options', [])) self.create_or_update_role(self._replication['username'], self._replication['password'], ['REPLICATION']) else: raise PostgresException("Could not bootstrap master PostgreSQL") def move_data_directory(self): if os.path.isdir(self._data_dir) and not self.is_running(): try: new_name = '{0}_{1}'.format(self._data_dir, time.strftime('%Y-%m-%d-%H-%M-%S')) logger.info('renaming data directory to %s', new_name) os.rename(self._data_dir, new_name) except OSError: logger.exception("Could not rename data directory %s", self._data_dir) def remove_data_directory(self): logger.info('Removing data directory: %s', self._data_dir) try: if os.path.islink(self._data_dir): os.unlink(self._data_dir) elif not os.path.exists(self._data_dir): return elif os.path.isfile(self._data_dir): os.remove(self._data_dir) elif os.path.isdir(self._data_dir): shutil.rmtree(self._data_dir) except (IOError, OSError): logger.exception('Could not remove data directory %s', self._data_dir) self.move_data_directory() def basebackup(self, conn_url, env): # creates a replica data dir using pg_basebackup. # this is the default, built-in create_replica_method # tries twice, then returns failure (as 1) # uses "stream" as the xlog-method to avoid sync issues maxfailures = 2 ret = 1 for bbfailures in range(0, maxfailures): try: ret = subprocess.call([ 'pg_basebackup', '--pgdata=' + self._data_dir, '--xlog-method=stream', "--dbname=" + conn_url ], env=env) if ret == 0: break except Exception as e: logger.error( 'Error when fetching backup with pg_basebackup: {0}'. format(e)) if bbfailures < maxfailures - 1: logger.error('Trying again in 5 seconds') time.sleep(5) return ret @staticmethod def postgres_version_to_int(pg_version): """ Convert the server_version to integer >>> Postgresql.postgres_version_to_int('9.5.3') 90503 >>> Postgresql.postgres_version_to_int('9.3.13') 90313 >>> Postgresql.postgres_version_to_int('10.1') 100001 >>> Postgresql.postgres_version_to_int('10') Traceback (most recent call last): ... Exception: Invalid PostgreSQL format: X.Y or X.Y.Z is accepted: 10 >>> Postgresql.postgres_version_to_int('a.b.c') Traceback (most recent call last): ... Exception: Invalid PostgreSQL version: a.b.c """ components = pg_version.split('.') result = [] if len(components) < 2 or len(components) > 3: raise Exception( "Invalid PostgreSQL format: X.Y or X.Y.Z is accepted: {0}". format(pg_version)) if len(components) == 2: # new style verion numbers, i.e. 10.1 becomes 100001 components.insert(1, '0') try: result = [ c if int(c) > 10 else '0{0}'.format(c) for c in components ] result = int(''.join(result)) except ValueError: raise Exception( "Invalid PostgreSQL version: {0}".format(pg_version)) return result