コード例 #1
0
ファイル: __init__.py プロジェクト: kotsachin/patroni
    def remove_data_directory(self):
        self.set_role('uninitialized')
        logger.info('Removing data directory: %s', self._data_dir)
        try:
            if os.path.islink(self._data_dir):
                os.unlink(self._data_dir)
            elif not os.path.exists(self._data_dir):
                return
            elif os.path.isfile(self._data_dir):
                os.remove(self._data_dir)
            elif os.path.isdir(self._data_dir):

                # let's see if pg_xlog|pg_wal is a symlink, in this case we
                # should clean the target
                for pg_wal_dir in ('pg_xlog', 'pg_wal'):
                    pg_wal_path = os.path.join(self._data_dir, pg_wal_dir)
                    if os.path.exists(pg_wal_path) and os.path.islink(pg_wal_path):
                        pg_wal_realpath = os.path.realpath(pg_wal_path)
                        logger.info('Removing WAL directory: %s', pg_wal_realpath)
                        shutil.rmtree(pg_wal_realpath)
                # Remove user defined tablespace directory
                pg_tblsp_dir = os.path.join(self._data_dir, 'pg_tblspc')
                if os.path.exists(pg_tblsp_dir):
                    for tsdn in os.listdir(pg_tblsp_dir):
                        pg_tsp_path = os.path.join(pg_tblsp_dir, tsdn)
                        if parse_int(tsdn) and os.path.islink(pg_tsp_path):
                            pg_tsp_rpath = os.path.realpath(pg_tsp_path)
                            logger.info('Removing user defined tablespace directory: %s', pg_tsp_rpath)
                            shutil.rmtree(pg_tsp_rpath, ignore_errors=True)

                shutil.rmtree(self._data_dir)
        except (IOError, OSError):
            logger.exception('Could not remove data directory %s', self._data_dir)
            self.move_data_directory()
コード例 #2
0
ファイル: api.py プロジェクト: tom2jack/patroni
    def do_POST_restart(self):
        status_code = 500
        data = 'restart failed'
        request = self._read_json_content(body_is_optional=True)
        cluster = self.server.patroni.dcs.get_cluster()
        if request is None:
            # failed to parse the json
            return
        if request:
            logger.debug("received restart request: {0}".format(request))

        if cluster.is_paused() and 'schedule' in request:
            self._write_response(status_code,
                                 "Can't schedule restart in the paused state")
            return

        for k in request:
            if k == 'schedule':
                (_, data,
                 request[k]) = self.parse_schedule(request[k], "restart")
                if _:
                    status_code = _
                    break
            elif k == 'role':
                if request[k] not in ('master', 'replica'):
                    status_code = 400
                    data = "PostgreSQL role should be either master or replica"
                    break
            elif k == 'postgres_version':
                if not is_valid_pg_version(request[k]):
                    status_code = 400
                    data = "PostgreSQL version should be in the first.major.minor format"
                    break
            elif k == 'timeout':
                request[k] = parse_int(request[k], 's')
                if request[k] is None or request[k] <= 0:
                    status_code = 400
                    data = "Timeout should be a positive number of seconds"
                    break
            elif k != 'restart_pending':
                status_code = 400
                data = "Unknown filter for the scheduled restart: {0}".format(
                    k)
                break
        else:
            if 'schedule' not in request:
                try:
                    status, data = self.server.patroni.ha.restart(request)
                    status_code = 200 if status else 503
                except Exception:
                    logger.exception('Exception during restart')
                    status_code = 400
            else:
                if self.server.patroni.ha.schedule_future_restart(request):
                    data = "Restart scheduled"
                    status_code = 202
                else:
                    data = "Another restart is already scheduled"
                    status_code = 409
        self._write_response(status_code, data)
コード例 #3
0
ファイル: postgresql.py プロジェクト: alkersan/patroni
    def reload_config(self, config):
        server_parameters = self.get_server_parameters(config)

        listen_address_changed = pending_reload = pending_restart = False
        if self.is_healthy():
            changes = {p: v for p, v in server_parameters.items() if '.' not in p}
            changes.update({p: None for p, v in self._server_parameters.items() if not ('.' in p or p in changes)})
            if changes:
                if 'wal_segment_size' not in changes:
                    changes['wal_segment_size'] = '16384kB'
                # XXX: query can raise an exception
                for r in self.query("""SELECT name, setting, unit, vartype, context
                                         FROM pg_settings
                                        WHERE name IN (""" + ', '.join(['%s'] * len(changes)) + """)
                                        ORDER BY 1 DESC""", *(list(changes.keys()))):
                    if r[4] == 'internal':
                        if r[0] == 'wal_segment_size':
                            server_parameters.pop(r[0], None)
                            wal_segment_size = parse_int(r[2], 'kB')
                            if wal_segment_size is not None:
                                changes['wal_segment_size'] = '{0}kB'.format(int(r[1]) * wal_segment_size)
                    elif r[0] in changes:
                        unit = changes['wal_segment_size'] if r[0] in ('min_wal_size', 'max_wal_size') else r[2]
                        new_value = changes.pop(r[0])
                        if new_value is None or not compare_values(r[3], unit, r[1], new_value):
                            if r[4] == 'postmaster':
                                pending_restart = True
                                if r[0] in ('listen_addresses', 'port'):
                                    listen_address_changed = True
                            else:
                                pending_reload = True
                for param in changes:
                    if param in server_parameters:
                        logger.warning('Removing invalid parameter `%s` from postgresql.parameters', param)
                        server_parameters.pop(param)

            # Check that user-defined-paramters have changed (parameters with period in name)
            if not pending_reload:
                for p, v in server_parameters.items():
                    if '.' in p and (p not in self._server_parameters or str(v) != str(self._server_parameters[p])):
                        pending_reload = True
                        break
                if not pending_reload:
                    for p, v in self._server_parameters.items():
                        if '.' in p and (p not in server_parameters or str(v) != str(server_parameters[p])):
                            pending_reload = True
                            break

        self.config = config
        self._pending_restart = pending_restart
        self._server_parameters = server_parameters
        self._connect_address = config.get('connect_address')

        if not listen_address_changed:
            self.resolve_connection_addresses()

        if pending_reload:
            self._write_postgresql_conf()
            self.reload()
        self.retry.deadline = config['retry_timeout']/2.0
コード例 #4
0
ファイル: postgresql.py プロジェクト: zenitraM/patroni
    def reload_config(self, config):
        server_parameters = self.get_server_parameters(config)

        listen_address_changed = pending_reload = pending_restart = False
        if self.is_healthy():
            changes = {p: v for p, v in server_parameters.items() if '.' not in p}
            changes.update({p: None for p, v in self._server_parameters.items() if not ('.' in p or p in changes)})
            if changes:
                if 'wal_segment_size' not in changes:
                    changes['wal_segment_size'] = '16384kB'
                # XXX: query can raise an exception
                for r in self.query("""SELECT name, setting, unit, vartype, context
                                         FROM pg_settings
                                        WHERE name IN (""" + ', '.join(['%s'] * len(changes)) + """)
                                        ORDER BY 1 DESC""", *(list(changes.keys()))):
                    if r[4] == 'internal':
                        if r[0] == 'wal_segment_size':
                            server_parameters.pop(r[0], None)
                            wal_segment_size = parse_int(r[2], 'kB')
                            if wal_segment_size is not None:
                                changes['wal_segment_size'] = '{0}kB'.format(int(r[1]) * wal_segment_size)
                    elif r[0] in changes:
                        unit = changes['wal_segment_size'] if r[0] in ('min_wal_size', 'max_wal_size') else r[2]
                        new_value = changes.pop(r[0])
                        if new_value is None or not compare_values(r[3], unit, r[1], new_value):
                            if r[4] == 'postmaster':
                                pending_restart = True
                                if r[0] in ('listen_addresses', 'port'):
                                    listen_address_changed = True
                            else:
                                pending_reload = True
                for param in changes:
                    if param in server_parameters:
                        logger.warning('Removing invalid parameter `%s` from postgresql.parameters', param)
                        server_parameters.pop(param)

            # Check that user-defined-paramters have changed (parameters with period in name)
            if not pending_reload:
                for p, v in server_parameters.items():
                    if '.' in p and (p not in self._server_parameters or str(v) != str(self._server_parameters[p])):
                        pending_reload = True
                        break
                if not pending_reload:
                    for p, v in self._server_parameters.items():
                        if '.' in p and (p not in server_parameters or str(v) != str(server_parameters[p])):
                            pending_reload = True
                            break

        self.config = config
        self._pending_restart = pending_restart
        self._server_parameters = server_parameters
        self._connect_address = config.get('connect_address')

        if not listen_address_changed:
            self.resolve_connection_addresses()

        if pending_reload:
            self._write_postgresql_conf()
            self.reload()
        self.retry.deadline = config['retry_timeout']/2.0
コード例 #5
0
ファイル: api.py プロジェクト: jberkus/patroni
    def do_POST_restart(self):
        status_code = 500
        data = 'restart failed'
        request = self._read_json_content(body_is_optional=True)
        cluster = self.server.patroni.dcs.get_cluster()
        if request is None:
            # failed to parse the json
            return
        if request:
            logger.debug("received restart request: {0}".format(request))

        if cluster.is_paused() and 'schedule' in request:
            self._write_response(status_code, "Can't schedule restart in the paused state")
            return

        for k in request:
            if k == 'schedule':
                (_, data, request[k]) = self.parse_schedule(request[k], "restart")
                if _:
                    status_code = _
                    break
            elif k == 'role':
                if request[k] not in ('master', 'replica'):
                    status_code = 400
                    data = "PostgreSQL role should be either master or replica"
                    break
            elif k == 'postgres_version':
                try:
                    Postgresql.postgres_version_to_int(request[k])
                except PostgresException as e:
                    status_code = 400
                    data = e.value
                    break
            elif k == 'timeout':
                request[k] = parse_int(request[k], 's')
                if request[k] is None or request[k] <= 0:
                    status_code = 400
                    data = "Timeout should be a positive number of seconds"
                    break
            elif k != 'restart_pending':
                status_code = 400
                data = "Unknown filter for the scheduled restart: {0}".format(k)
                break
        else:
            if 'schedule' not in request:
                try:
                    status, data = self.server.patroni.ha.restart(request)
                    status_code = 200 if status else 503
                except Exception:
                    logger.exception('Exception during restart')
                    status_code = 400
            else:
                if self.server.patroni.ha.schedule_future_restart(request):
                    data = "Restart scheduled"
                    status_code = 202
                else:
                    data = "Another restart is already scheduled"
                    status_code = 409
        self._write_response(status_code, data)
コード例 #6
0
 def pg_tblspc_realpaths(self):
     """Returns a dict containing the symlink (key) and target (values) for the tablespaces"""
     links = {}
     pg_tblsp_dir = os.path.join(self._data_dir, 'pg_tblspc')
     if os.path.exists(pg_tblsp_dir):
         for tsdn in os.listdir(pg_tblsp_dir):
             pg_tsp_path = os.path.join(pg_tblsp_dir, tsdn)
             if parse_int(tsdn) and os.path.islink(pg_tsp_path):
                 pg_tsp_rpath = os.path.realpath(pg_tsp_path)
                 links[pg_tsp_path] = pg_tsp_rpath
     return links
コード例 #7
0
    def _build_environment_configuration():
        ret = defaultdict(dict)

        def _popenv(name):
            return os.environ.pop(PATRONI_ENV_PREFIX + name.upper(), None)

        for param in ('name', 'namespace', 'scope'):
            value = _popenv(param)
            if value:
                ret[param] = value

        def _fix_log_env(name, oldname):
            value = _popenv(oldname)
            name = PATRONI_ENV_PREFIX + 'LOG_' + name.upper()
            if value and name not in os.environ:
                os.environ[name] = value

        for name, oldname in (('level', 'loglevel'), ('format', 'logformat'),
                              ('dateformat', 'log_datefmt')):
            _fix_log_env(name, oldname)

        def _set_section_values(section, params):
            for param in params:
                value = _popenv(section + '_' + param)
                if value:
                    ret[section][param] = value

        _set_section_values('restapi', [
            'listen', 'connect_address', 'certfile', 'keyfile', 'cafile',
            'verify_client'
        ])
        _set_section_values('ctl',
                            ['insecure', 'cacert', 'certfile', 'keyfile'])
        _set_section_values('postgresql', [
            'listen', 'connect_address', 'config_dir', 'data_dir', 'pgpass',
            'bin_dir'
        ])
        _set_section_values('log', [
            'level', 'traceback_level', 'format', 'dateformat',
            'max_queue_size', 'dir', 'file_size', 'file_num', 'loggers'
        ])

        def _parse_dict(value):
            if not value.strip().startswith('{'):
                value = '{{{0}}}'.format(value)
            try:
                return yaml.safe_load(value)
            except Exception:
                logger.exception('Exception when parsing dict %s', value)
                return None

        value = ret.get('log', {}).pop('loggers', None)
        if value:
            value = _parse_dict(value)
            if value:
                ret['log']['loggers'] = value

        def _get_auth(name, params=None):
            ret = {}
            for param in params or _AUTH_ALLOWED_PARAMETERS[:2]:
                value = _popenv(name + '_' + param)
                if value:
                    ret[param] = value
            return ret

        restapi_auth = _get_auth('restapi')
        if restapi_auth:
            ret['restapi']['authentication'] = restapi_auth

        authentication = {}
        for user_type in ('replication', 'superuser', 'rewind'):
            entry = _get_auth(user_type, _AUTH_ALLOWED_PARAMETERS)
            if entry:
                authentication[user_type] = entry

        if authentication:
            ret['postgresql']['authentication'] = authentication

        def _parse_list(value):
            if not (value.strip().startswith('-') or '[' in value):
                value = '[{0}]'.format(value)
            try:
                return yaml.safe_load(value)
            except Exception:
                logger.exception('Exception when parsing list %s', value)
                return None

        for param in list(os.environ.keys()):
            if param.startswith(PATRONI_ENV_PREFIX):
                # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
                name, suffix = (param[8:].split('_', 1) + [''])[:2]
                if suffix in ('HOST', 'HOSTS', 'PORT', 'USE_PROXIES',
                              'PROTOCOL', 'SRV', 'URL', 'PROXY', 'CACERT',
                              'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC',
                              'CONSISTENCY', 'REGISTER_SERVICE',
                              'SERVICE_CHECK_INTERVAL', 'NAMESPACE', 'CONTEXT',
                              'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL',
                              'POD_IP', 'PORTS', 'LABELS') and name:
                    value = os.environ.pop(param)
                    if suffix == 'PORT':
                        value = value and parse_int(value)
                    elif suffix in ('HOSTS', 'PORTS', 'CHECKS'):
                        value = value and _parse_list(value)
                    elif suffix == 'LABELS':
                        value = _parse_dict(value)
                    elif suffix in ('USE_PROXIES', 'REGISTER_SERVICE'):
                        value = parse_bool(value)
                    if value:
                        ret[name.lower()][suffix.lower()] = value
        if 'etcd' in ret:
            ret['etcd'].update(_get_auth('etcd'))

        users = {}
        for param in list(os.environ.keys()):
            if param.startswith(PATRONI_ENV_PREFIX):
                name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
                # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
                # CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
                if name and suffix == 'PASSWORD':
                    password = os.environ.pop(param)
                    if password:
                        users[name] = {'password': password}
                        options = os.environ.pop(param[:-9] + '_OPTIONS', None)
                        options = options and _parse_list(options)
                        if options:
                            users[name]['options'] = options
        if users:
            ret['bootstrap']['users'] = users

        return ret
コード例 #8
0
ファイル: config.py プロジェクト: jberkus/patroni
    def _build_environment_configuration():
        ret = defaultdict(dict)

        def _popenv(name):
            return os.environ.pop(Config.PATRONI_ENV_PREFIX + name.upper(), None)

        for param in ('name', 'namespace', 'scope'):
            value = _popenv(param)
            if value:
                ret[param] = value

        def _set_section_values(section, params):
            for param in params:
                value = _popenv(section + '_' + param)
                if value:
                    ret[section][param] = value

        _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
        _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass', 'bin_dir'])

        def _get_auth(name):
            ret = {}
            for param in ('username', 'password'):
                value = _popenv(name + '_' + param)
                if value:
                    ret[param] = value
            return ret

        restapi_auth = _get_auth('restapi')
        if restapi_auth:
            ret['restapi']['authentication'] = restapi_auth

        authentication = {}
        for user_type in ('replication', 'superuser'):
            entry = _get_auth(user_type)
            if entry:
                authentication[user_type] = entry

        if authentication:
            ret['postgresql']['authentication'] = authentication

        users = {}

        def _parse_list(value):
            if not (value.strip().startswith('-') or '[' in value):
                value = '[{0}]'.format(value)
            try:
                return yaml.safe_load(value)
            except Exception:
                logger.exception('Exception when parsing list %s', value)
                return None

        for param in list(os.environ.keys()):
            if param.startswith(Config.PATRONI_ENV_PREFIX):
                name, suffix = (param[8:].split('_', 1) + [''])[:2]
                if name and suffix:
                    # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT',
                                  'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'NAMESPACE', 'CONTEXT',
                                  'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS'):
                        value = os.environ.pop(param)
                        if suffix == 'PORT':
                            value = value and parse_int(value)
                        elif suffix in ('HOSTS', 'PORTS', 'CHECKS'):
                            value = value and _parse_list(value)
                        elif suffix == 'LABELS':
                            if not value.strip().startswith('{'):
                                value = '{{{0}}}'.format(value)
                            try:
                                value = yaml.safe_load(value)
                            except Exception:
                                logger.exception('Exception when parsing dict %s', value)
                                value = None
                        if value:
                            ret[name.lower()][suffix.lower()] = value
                    # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
                    # CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
                    elif suffix == 'PASSWORD':
                        password = os.environ.pop(param)
                        if password:
                            users[name] = {'password': password}
                            options = os.environ.pop(param[:-9] + '_OPTIONS', None)
                            options = options and _parse_list(options)
                            if options:
                                users[name]['options'] = options
        if users:
            ret['bootstrap']['users'] = users

        return ret
コード例 #9
0
    def _build_environment_configuration():
        ret = defaultdict(dict)

        def _popenv(name):
            return os.environ.pop(Config.PATRONI_ENV_PREFIX + name.upper(),
                                  None)

        for param in ('name', 'namespace', 'scope'):
            value = _popenv(param)
            if value:
                ret[param] = value

        def _set_section_values(section, params):
            for param in params:
                value = _popenv(section + '_' + param)
                if value:
                    ret[section][param] = value

        _set_section_values(
            'restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
        _set_section_values(
            'postgresql',
            ['listen', 'connect_address', 'data_dir', 'pgpass', 'bin_dir'])

        def _get_auth(name):
            ret = {}
            for param in ('username', 'password'):
                value = _popenv(name + '_' + param)
                if value:
                    ret[param] = value
            return ret

        restapi_auth = _get_auth('restapi')
        if restapi_auth:
            ret['restapi']['authentication'] = restapi_auth

        authentication = {}
        for user_type in ('replication', 'superuser'):
            entry = _get_auth(user_type)
            if entry:
                authentication[user_type] = entry

        if authentication:
            ret['postgresql']['authentication'] = authentication

        users = {}

        def _parse_list(value):
            if not (value.strip().startswith('-') or '[' in value):
                value = '[{0}]'.format(value)
            try:
                return yaml.safe_load(value)
            except Exception:
                logger.exception('Exception when parsing list %s', value)
                return None

        for param in list(os.environ.keys()):
            if param.startswith(Config.PATRONI_ENV_PREFIX):
                name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
                if name and suffix:
                    # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL',
                                  'PROXY', 'CACERT', 'CERT', 'KEY', 'VERIFY',
                                  'TOKEN', 'CHECKS', 'DC') and '_' not in name:
                        value = os.environ.pop(param)
                        if suffix == 'PORT':
                            value = value and parse_int(value)
                        elif suffix in ('HOSTS', 'CHECKS'):
                            value = value and _parse_list(value)
                        if value:
                            ret[name.lower()][suffix.lower()] = value
                    # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
                    # CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
                    elif suffix == 'PASSWORD':
                        password = os.environ.pop(param)
                        if password:
                            users[name] = {'password': password}
                            options = os.environ.pop(param[:-9] + '_OPTIONS',
                                                     None)
                            options = options and _parse_list(options)
                            if options:
                                users[name]['options'] = options
        if users:
            ret['bootstrap']['users'] = users

        return ret