def test_get_os_debian(self):
with patch.object(os.path, 'isfile', side_effect=[False,
False,
False,
True]):
find_os = operating_system.get_os()
self.assertEqual('debian', find_os)
def __new__(meta, name, bases, dct):
if operating_system.get_os() == operating_system.REDHAT:
bases += (RedhatPackagerMixin, )
else:
# The default is debian
bases += (DebianPackagerMixin,)
return super(BasePackage, meta).__new__(meta, name, bases, dct)
def cassandra_conf_dir(self):
return {
operating_system.REDHAT: "/etc/dse/cassandra/",
operating_system.DEBIAN: "/etc/dse/cassandra/",
operating_system.SUSE: "/etc/dse/cassandra/",
operating_system.ORACLE: "/etc/dse/cassandra/"
}[operating_system.get_os()]
def __new__(meta, name, bases, dct):
if operating_system.get_os() == operating_system.REDHAT:
bases += (RedhatPackagerMixin, )
else:
# The default is debian
bases += (DebianPackagerMixin, )
return super(BasePackage, meta).__new__(meta, name, bases, dct)
def cassandra_conf_dir(self):
return {
operating_system.REDHAT: "/etc/dse/cassandra/",
operating_system.DEBIAN: "/etc/dse/cassandra/",
operating_system.SUSE: "/etc/dse/cassandra/",
operating_system.ORACLE: "/etc/dse/cassandra/"
}[operating_system.get_os()]
def cassandra_conf(self):
return {
operating_system.REDHAT:
"/etc/cassandra/default.conf/cassandra.yaml",
operating_system.DEBIAN: "/etc/cassandra/cassandra.yaml",
operating_system.SUSE: "/etc/cassandra/default.conf/cassandra.yaml"
}[operating_system.get_os()]
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Determines operating system version and OS dependent commands.
"""
from trove.guestagent.common.operating_system import get_os
REDIS_OWNER = 'redis'
REDIS_CONFIG = '/etc/redis/redis.conf'
REDIS_PID_FILE = '/var/run/redis/redis-server.pid'
REDIS_LOG_FILE = '/var/log/redis/server.log'
REDIS_CONF_DIR = '/etc/redis'
REDIS_DATA_DIR = '/var/lib/redis'
REDIS_PORT = '6379'
REDIS_INIT = '/etc/init/redis-server.conf'
REDIS_PACKAGE = ''
SERVICE_CANDIDATES = ['redis-server', 'redis']
OS = get_os()
if OS == 'redhat':
REDIS_CONFIG = '/etc/redis.conf'
REDIS_PACKAGE = 'redis'
def test_get_os_redhat(self):
with patch.object(os.path, 'isfile', side_effect=[True]):
find_os = operating_system.get_os()
self.assertEqual('redhat', find_os)
CONNECTION_STR_FORMAT = "mysql+pymysql://%s:%[email protected]:3306" LOG = logging.getLogger(__name__) FLUSH = text(sql_query.FLUSH) ENGINE = None DATADIR = None PREPARING = False UUID = False TMP_MYCNF = "/tmp/my.cnf.tmp" MYSQL_BASE_DIR = "/var/lib/mysql" CONF = cfg.CONF INCLUDE_MARKER_OPERATORS = {True: ">=", False: ">"} OS_NAME = operating_system.get_os() MYSQL_CONFIG = { operating_system.REDHAT: "/etc/my.cnf", operating_system.DEBIAN: "/etc/mysql/my.cnf", operating_system.SUSE: "/etc/my.cnf" }[OS_NAME] MYSQL_BIN_CANDIDATES = ["/usr/sbin/mysqld", "/usr/libexec/mysqld"] MYSQL_OWNER = 'mysql' CNF_EXT = 'cnf' CNF_INCLUDE_DIR = '/etc/mysql/conf.d/' CNF_MASTER = 'master-replication' CNF_SLAVE = 'slave-replication' # Create a package impl packager = pkg.Package()
def test_get_os_redhat(self):
with patch.object(os.path, 'isfile', side_effect=[True]):
find_os = operating_system.get_os()
self.assertEqual('redhat', find_os)
class PgSqlConfig(PgSqlProcess):
"""Mixin that implements the config API.
This mixin has a dependency on the PgSqlProcess mixin.
"""
OS = operating_system.get_os()
CONFIG_BASE = {
operating_system.DEBIAN: '/etc/postgresql/',
operating_system.REDHAT: '/var/lib/postgresql/',
operating_system.SUSE: '/var/lib/pgsql/'
}[OS]
LISTEN_ADDRESSES = ['*'] # Listen on all available IP (v4/v6) interfaces.
def __init__(self):
self._configuration_manager = ConfigurationManager(
self.PGSQL_CONFIG,
self.PGSQL_OWNER,
self.PGSQL_OWNER,
PropertiesCodec(delimiter='=',
string_mappings={
'on': True,
'off': False,
"''": None
}),
requires_root=True,
override_strategy=OneFileOverrideStrategy(
self._init_overrides_dir()))
# TODO(pmalik): To be removed when
# 'https://review.openstack.org/#/c/218382/' merges.
def _init_overrides_dir(self):
"""Initialize a directory for configuration overrides.
"""
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(self.PGSQL_CONFIG),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
if not os.path.exists(revision_dir):
operating_system.create_directory(revision_dir,
user=self.PGSQL_OWNER,
group=self.PGSQL_OWNER,
force=True,
as_root=True)
return revision_dir
@property
def PGSQL_EXTRA_BIN_DIR(self):
"""Redhat and Ubuntu packages for PgSql do not place 'extra' important
binaries in /usr/bin, but rather in a directory like /usr/pgsql-9.4/bin
in the case of PostgreSQL 9.4 for RHEL/CentOS
"""
version = self.pg_version[1]
return {
operating_system.DEBIAN: '/usr/lib/postgresql/%s/bin',
operating_system.REDHAT: '/usr/pgsql-%s/bin',
operating_system.SUSE: '/usr/bin'
}[self.OS] % version
@property
def PGSQL_CONFIG(self):
return self._find_config_file('postgresql.conf')
@property
def PGSQL_HBA_CONFIG(self):
return self._find_config_file('pg_hba.conf')
@property
def PGSQL_IDENT_CONFIG(self):
return self._find_config_file('pg_ident.conf')
def _find_config_file(self, name_pattern):
version_base = guestagent_utils.build_file_path(
self.CONFIG_BASE, self.pg_version[1])
return sorted(operating_system.list_files_in_directory(
version_base, recursive=True, pattern=name_pattern, as_root=True),
key=len)[0]
def update_overrides(self, context, overrides, remove=False):
if remove:
self.configuration_manager.remove_user_override()
elif overrides:
self.configuration_manager.apply_user_override(overrides)
def apply_overrides(self, context, overrides):
# Send a signal to the server, causing configuration files to be
# reloaded by all server processes.
# Active queries or connections to the database will not be
# interrupted.
#
# NOTE: Do not use the 'SET' command as it only affects the current
# session.
pgutil.psql("SELECT pg_reload_conf()")
def reset_configuration(self, context, configuration):
"""Reset the PgSql configuration to the one given.
"""
config_contents = configuration['config_contents']
self.configuration_manager.save_configuration(config_contents)
def start_db_with_conf_changes(self, context, config_contents):
"""Starts the PgSql instance with a new configuration."""
if PgSqlAppStatus.get().is_running:
raise RuntimeError(_("The service is still running."))
self.configuration_manager.save_configuration(config_contents)
# The configuration template has to be updated with
# guestagent-controlled settings.
self.apply_initial_guestagent_configuration()
self.start_db(context)
def apply_initial_guestagent_configuration(self):
"""Update guestagent-controlled configuration properties.
"""
LOG.debug("Applying initial guestagent configuration.")
file_locations = {
'data_directory': self._quote(self.PGSQL_DATA_DIR),
'hba_file': self._quote(self.PGSQL_HBA_CONFIG),
'ident_file': self._quote(self.PGSQL_IDENT_CONFIG),
'external_pid_file': self._quote(self.PID_FILE),
'unix_socket_directories': self._quote(self.UNIX_SOCKET_DIR),
'listen_addresses': self._quote(','.join(self.LISTEN_ADDRESSES)),
'port': CONF.postgresql.postgresql_port
}
self.configuration_manager.apply_system_override(file_locations)
self._apply_access_rules()
@staticmethod
def _quote(value):
return "'%s'" % value
def _apply_access_rules(self):
LOG.debug("Applying database access rules.")
# Connections to all resources are granted.
#
# Local access from administrative users is implicitly trusted.
#
# Remote access from the Trove's account is always rejected as
# it is not needed and could be used by malicious users to hijack the
# instance.
#
# Connections from other accounts always require a double-MD5-hashed
# password.
#
# Make the rules readable only by the Postgres service.
#
# NOTE: The order of entries is important.
# The first failure to authenticate stops the lookup.
# That is why the 'local' connections validate first.
# The OrderedDict is necessary to guarantee the iteration order.
access_rules = OrderedDict([
('local', [['all', 'postgres,os_admin', None, 'trust'],
['all', 'all', None, 'md5'],
['replication', 'postgres,os_admin', None, 'trust']]),
('host', [['all', 'postgres,os_admin', '127.0.0.1/32', 'trust'],
['all', 'postgres,os_admin', '::1/128', 'trust'],
['all', 'postgres,os_admin', 'localhost', 'trust'],
['all', 'os_admin', '0.0.0.0/0', 'reject'],
['all', 'os_admin', '::/0', 'reject'],
['all', 'all', '0.0.0.0/0', 'md5'],
['all', 'all', '::/0', 'md5']])
])
operating_system.write_file(
self.PGSQL_HBA_CONFIG,
access_rules,
PropertiesCodec(string_mappings={'\t': None}),
as_root=True)
operating_system.chown(self.PGSQL_HBA_CONFIG,
self.PGSQL_OWNER,
self.PGSQL_OWNER,
as_root=True)
operating_system.chmod(self.PGSQL_HBA_CONFIG,
FileMode.SET_USR_RO,
as_root=True)
def disable_backups(self):
"""Reverse overrides applied by PgBaseBackup strategy"""
if not self.configuration_manager.has_system_override(
BACKUP_CFG_OVERRIDE):
return
LOG.info("Removing configuration changes for backups")
self.configuration_manager.remove_system_override(BACKUP_CFG_OVERRIDE)
self.remove_wal_archive_dir()
self.restart(context=None)
def enable_backups(self):
"""Apply necessary changes to config to enable WAL-based backups
if we are using the PgBaseBackup strategy
"""
LOG.info("Checking if we need to apply changes to WAL config")
if not CONF.postgresql.backup_strategy == 'PgBaseBackup':
return
if self.configuration_manager.has_system_override(BACKUP_CFG_OVERRIDE):
return
LOG.info("Applying changes to WAL config for use by base backups")
arch_cmd = "'test ! -f {wal_arch}/%f && cp %p {wal_arch}/%f'".format(
wal_arch=CONF.postgresql.wal_archive_location)
opts = {
# FIXME(atomic77) These spaces after the options are needed until
# DBAAS-949 is fixed
'wal_level ': 'hot_standby',
'archive_mode ': 'on',
'max_wal_senders': 8,
# 'checkpoint_segments ': 8,
'wal_keep_segments': 8,
'archive_command': arch_cmd
}
if self.pg_version[1] in ('9.4', '9.5'):
opts['wal_log_hints'] = 'on'
self.configuration_manager.apply_system_override(
opts, BACKUP_CFG_OVERRIDE)
# self.enable_debugging(level=1)
self.restart(None)
def disable_debugging(self, level=1):
"""Enable debug-level logging in postgres"""
self.configuration_manager.remove_system_override(DEBUG_MODE_OVERRIDE)
def enable_debugging(self, level=1):
"""Enable debug-level logging in postgres"""
opt = {'log_min_messages': 'DEBUG%s' % level}
self.configuration_manager.apply_system_override(
opt, DEBUG_MODE_OVERRIDE)
LOG.debug("Package %s already installed.", package_name)
else:
return False
return True
def pkg_remove(self, package_name, time_out):
"""Removes a package."""
if self.pkg_version(package_name) is None:
return
result = self._remove(package_name, time_out)
if result != OK:
if result == REINSTALL_FIRST:
self._install(package_name, time_out)
elif result == RUN_DPKG_FIRST:
self._fix(time_out)
result = self._remove(package_name, time_out)
if result != OK:
raise PkgPackageStateError(
_("Package %s is in a bad state.") % package_name)
if operating_system.get_os() == operating_system.REDHAT:
class Package(RedhatPackagerMixin):
pass
else:
class Package(DebianPackagerMixin):
pass
def test_get_os_suse(self):
with patch.object(os.path, "isfile", side_effect=[False, True]):
find_os = operating_system.get_os()
self.assertEqual("suse", find_os)
class PgSqlApp(object):
OS = operating_system.get_os()
LISTEN_ADDRESSES = ['*'] # Listen on all available IP (v4/v6) interfaces.
ADMIN_USER = '******' # Trove's administrative user.
def __init__(self):
super(PgSqlApp, self).__init__()
self._current_admin_user = None
self.status = PgSqlAppStatus(self.pgsql_extra_bin_dir)
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(self.pgsql_config),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
self.configuration_manager = ConfigurationManager(
self.pgsql_config,
self.pgsql_owner,
self.pgsql_owner,
PropertiesCodec(delimiter='=',
string_mappings={
'on': True,
'off': False,
"''": None
}),
requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
@property
def service_candidates(self):
return ['postgresql']
@property
def pgsql_owner(self):
return 'postgres'
@property
def default_superuser_name(self):
return "postgres"
@property
def pgsql_base_data_dir(self):
return '/var/lib/postgresql/'
@property
def pgsql_pid_file(self):
return guestagent_utils.build_file_path(self.pgsql_run_dir,
'postgresql.pid')
@property
def pgsql_run_dir(self):
return '/var/run/postgresql/'
@property
def pgsql_extra_bin_dir(self):
"""Redhat and Ubuntu packages for PgSql do not place 'extra' important
binaries in /usr/bin, but rather in a directory like /usr/pgsql-9.4/bin
in the case of PostgreSQL 9.4 for RHEL/CentOS
"""
return {
operating_system.DEBIAN: '/usr/lib/postgresql/%s/bin/',
operating_system.REDHAT: '/usr/pgsql-%s/bin/',
operating_system.SUSE: '/usr/bin/'
}[self.OS] % self.pg_version[1]
@property
def pgsql_config(self):
return self._find_config_file('postgresql.conf')
@property
def pgsql_hba_config(self):
return self._find_config_file('pg_hba.conf')
@property
def pgsql_ident_config(self):
return self._find_config_file('pg_ident.conf')
def _find_config_file(self, name_pattern):
version_base = guestagent_utils.build_file_path(
self.pgsql_config_dir, self.pg_version[1])
return sorted(operating_system.list_files_in_directory(
version_base, recursive=True, pattern=name_pattern, as_root=True),
key=len)[0]
@property
def pgsql_config_dir(self):
return {
operating_system.DEBIAN: '/etc/postgresql/',
operating_system.REDHAT: '/var/lib/postgresql/',
operating_system.SUSE: '/var/lib/pgsql/'
}[self.OS]
@property
def pgsql_log_dir(self):
return "/var/log/postgresql/"
def build_admin(self):
return PgSqlAdmin(self.get_current_admin_user())
def update_overrides(self, context, overrides, remove=False):
if remove:
self.configuration_manager.remove_user_override()
elif overrides:
self.configuration_manager.apply_user_override(overrides)
def set_current_admin_user(self, user):
self._current_admin_user = user
def get_current_admin_user(self):
if self._current_admin_user is not None:
return self._current_admin_user
if self.status.is_installed:
return models.PostgreSQLUser(self.ADMIN_USER)
return models.PostgreSQLUser(self.default_superuser_name)
def apply_overrides(self, context, overrides):
self.reload_configuration()
def reload_configuration(self):
"""Send a signal to the server, causing configuration files to be
reloaded by all server processes.
Active queries or connections to the database will not be
interrupted.
NOTE: Do not use the 'SET' command as it only affects the current
session.
"""
self.build_admin().psql("SELECT pg_reload_conf()")
def reset_configuration(self, context, configuration):
"""Reset the PgSql configuration to the one given.
"""
config_contents = configuration['config_contents']
self.configuration_manager.save_configuration(config_contents)
def start_db_with_conf_changes(self, context, config_contents):
"""Starts the PgSql instance with a new configuration."""
if self.status.is_running:
raise RuntimeError(_("The service is still running."))
self.configuration_manager.save_configuration(config_contents)
# The configuration template has to be updated with
# guestagent-controlled settings.
self.apply_initial_guestagent_configuration()
self.start_db()
def apply_initial_guestagent_configuration(self):
"""Update guestagent-controlled configuration properties.
"""
LOG.debug("Applying initial guestagent configuration.")
file_locations = {
'data_directory': self._quote(self.pgsql_data_dir),
'hba_file': self._quote(self.pgsql_hba_config),
'ident_file': self._quote(self.pgsql_ident_config),
'external_pid_file': self._quote(self.pgsql_pid_file),
'unix_socket_directories': self._quote(self.pgsql_run_dir),
'listen_addresses': self._quote(','.join(self.LISTEN_ADDRESSES)),
'port': cfg.get_configuration_property('postgresql_port')
}
self.configuration_manager.apply_system_override(file_locations)
self._apply_access_rules()
@staticmethod
def _quote(value):
return "'%s'" % value
def _apply_access_rules(self):
LOG.debug("Applying database access rules.")
# Connections to all resources are granted.
#
# Local access from administrative users is implicitly trusted.
#
# Remote access from the Trove's account is always rejected as
# it is not needed and could be used by malicious users to hijack the
# instance.
#
# Connections from other accounts always require a double-MD5-hashed
# password.
#
# Make the rules readable only by the Postgres service.
#
# NOTE: The order of entries is important.
# The first failure to authenticate stops the lookup.
# That is why the 'local' connections validate first.
# The OrderedDict is necessary to guarantee the iteration order.
local_admins = ','.join([self.default_superuser_name, self.ADMIN_USER])
remote_admins = self.ADMIN_USER
access_rules = OrderedDict([
('local', [['all', local_admins, None, 'trust'],
['replication', local_admins, None, 'trust'],
['all', 'all', None, 'md5']]),
('host', [['all', local_admins, '127.0.0.1/32', 'trust'],
['all', local_admins, '::1/128', 'trust'],
['all', local_admins, 'localhost', 'trust'],
['all', remote_admins, '0.0.0.0/0', 'reject'],
['all', remote_admins, '::/0', 'reject'],
['all', 'all', '0.0.0.0/0', 'md5'],
['all', 'all', '::/0', 'md5']])
])
operating_system.write_file(
self.pgsql_hba_config,
access_rules,
PropertiesCodec(string_mappings={'\t': None}),
as_root=True)
operating_system.chown(self.pgsql_hba_config,
self.pgsql_owner,
self.pgsql_owner,
as_root=True)
operating_system.chmod(self.pgsql_hba_config,
FileMode.SET_USR_RO,
as_root=True)
def disable_backups(self):
"""Reverse overrides applied by PgBaseBackup strategy"""
if not self.configuration_manager.has_system_override(
BACKUP_CFG_OVERRIDE):
return
LOG.info("Removing configuration changes for backups")
self.configuration_manager.remove_system_override(BACKUP_CFG_OVERRIDE)
self.remove_wal_archive_dir()
self.restart()
def enable_backups(self):
"""Apply necessary changes to config to enable WAL-based backups
if we are using the PgBaseBackup strategy
"""
LOG.info("Checking if we need to apply changes to WAL config")
if 'PgBaseBackup' not in self.backup_strategy:
return
if self.configuration_manager.has_system_override(BACKUP_CFG_OVERRIDE):
return
LOG.info("Applying changes to WAL config for use by base backups")
wal_arch_loc = self.wal_archive_location
if not os.path.isdir(wal_arch_loc):
raise RuntimeError(
_("Cannot enable backup as WAL dir '%s' does "
"not exist.") % wal_arch_loc)
arch_cmd = "'test ! -f {wal_arch}/%f && cp %p {wal_arch}/%f'".format(
wal_arch=wal_arch_loc)
# Only support pg version > 9.6, wal_level set to replica, and
# remove parameter "checkpoint_segments".
opts = {
'wal_level': 'replica',
'archive_mode': 'on',
'max_wal_senders': 8,
'wal_log_hints': 'on',
'wal_keep_segments': 8,
'archive_command': arch_cmd
}
self.configuration_manager.apply_system_override(
opts, BACKUP_CFG_OVERRIDE)
self.restart()
def disable_debugging(self, level=1):
"""Disable debug-level logging in postgres"""
self.configuration_manager.remove_system_override(DEBUG_MODE_OVERRIDE)
def enable_debugging(self, level=1):
"""Enable debug-level logging in postgres"""
opt = {'log_min_messages': 'DEBUG%s' % level}
self.configuration_manager.apply_system_override(
opt, DEBUG_MODE_OVERRIDE)
def install(self, context, packages):
"""Install one or more packages that postgresql needs to run.
The packages parameter is a string representing the package names that
should be given to the system's package manager.
"""
LOG.debug("{guest_id}: Beginning PgSql package installation.".format(
guest_id=CONF.guest_id))
self.recreate_wal_archive_dir()
packager = pkg.Package()
if not packager.pkg_is_installed(packages):
try:
LOG.info("{guest_id}: Installing ({packages}).".format(
guest_id=CONF.guest_id,
packages=packages,
))
packager.pkg_install(packages, {}, 1000)
except (pkg.PkgAdminLockError, pkg.PkgPermissionError,
pkg.PkgPackageStateError, pkg.PkgNotFoundError,
pkg.PkgTimeout, pkg.PkgScriptletError,
pkg.PkgDownloadError, pkg.PkgSignError,
pkg.PkgBrokenError):
LOG.exception(
"{guest_id}: There was a package manager error while "
"trying to install ({packages}).".format(
guest_id=CONF.guest_id,
packages=packages,
))
raise
except Exception:
LOG.exception(
"{guest_id}: The package manager encountered an unknown "
"error while trying to install ({packages}).".format(
guest_id=CONF.guest_id,
packages=packages,
))
raise
else:
self.start_db()
LOG.debug("{guest_id}: Completed package installation.".format(
guest_id=CONF.guest_id, ))
@property
def pgsql_recovery_config(self):
return os.path.join(self.pgsql_data_dir, "recovery.conf")
@property
def pgsql_data_dir(self):
return os.path.dirname(self.pg_version[0])
@property
def pg_version(self):
"""Find the database version file stored in the data directory.
:returns: A tuple with the path to the version file
(in the root of the data directory) and the version string.
"""
version_files = operating_system.list_files_in_directory(
self.pgsql_base_data_dir,
recursive=True,
pattern='PG_VERSION',
as_root=True)
version_file = sorted(version_files, key=len)[0]
version = operating_system.read_file(version_file, as_root=True)
return version_file, version.strip()
def restart(self):
self.status.restart_db_service(self.service_candidates,
CONF.state_change_wait_time)
def start_db(self, enable_on_boot=True, update_db=False):
self.status.start_db_service(self.service_candidates,
CONF.state_change_wait_time,
enable_on_boot=enable_on_boot,
update_db=update_db)
def stop_db(self, do_not_start_on_reboot=False, update_db=False):
self.status.stop_db_service(self.service_candidates,
CONF.state_change_wait_time,
disable_on_boot=do_not_start_on_reboot,
update_db=update_db)
def secure(self, context):
"""Create an administrative user for Trove.
Force password encryption.
Also disable the built-in superuser
"""
password = utils.generate_random_password()
os_admin_db = models.PostgreSQLSchema(self.ADMIN_USER)
os_admin = models.PostgreSQLUser(self.ADMIN_USER, password)
os_admin.databases.append(os_admin_db.serialize())
postgres = models.PostgreSQLUser(self.default_superuser_name)
admin = PgSqlAdmin(postgres)
admin._create_database(context, os_admin_db)
admin._create_admin_user(context, os_admin, encrypt_password=True)
PgSqlAdmin(os_admin).alter_user(context, postgres, None, 'NOSUPERUSER',
'NOLOGIN')
self.set_current_admin_user(os_admin)
def pg_current_xlog_location(self):
"""Wrapper for pg_current_xlog_location()
Cannot be used against a running slave
"""
version = int(self.pg_version[1])
if version < 10:
query = "SELECT pg_current_xlog_location()"
else:
query = "SELECT pg_current_wal_lsn()"
r = self.build_admin().query(query)
return r[0][0]
def pg_last_xlog_replay_location(self):
"""Wrapper for pg_last_xlog_replay_location()
For use on standby servers
"""
version = int(self.pg_version[1])
if version < 10:
query = "SELECT pg_last_xlog_replay_location()"
else:
query = "SELECT pg_last_wal_replay_lsn()"
r = self.build_admin().query(query)
return r[0][0]
def pg_is_in_recovery(self):
"""Wrapper for pg_is_in_recovery() for detecting a server in
standby mode
"""
r = self.build_admin().query("SELECT pg_is_in_recovery()")
return r[0][0]
def pg_primary_host(self):
"""There seems to be no way to programmatically determine this
on a hot standby, so grab what we have written to the recovery
file
"""
r = operating_system.read_file(self.pgsql_recovery_config,
as_root=True)
regexp = re.compile(r"host=(\d+.\d+.\d+.\d+) ")
m = regexp.search(r)
return m.group(1)
def recreate_wal_archive_dir(self):
wal_archive_dir = self.wal_archive_location
operating_system.remove(wal_archive_dir,
force=True,
recursive=True,
as_root=True)
operating_system.create_directory(wal_archive_dir,
user=self.pgsql_owner,
group=self.pgsql_owner,
force=True,
as_root=True)
def remove_wal_archive_dir(self):
wal_archive_dir = self.wal_archive_location
operating_system.remove(wal_archive_dir,
force=True,
recursive=True,
as_root=True)
def is_root_enabled(self, context):
"""Return True if there is a superuser account enabled.
"""
results = self.build_admin().query(
pgsql_query.UserQuery.list_root(),
timeout=30,
)
# There should be only one superuser (Trove's administrative account).
return len(results) > 1 or (results[0][0] != self.ADMIN_USER)
def enable_root(self, context, root_password=None):
"""Create a superuser user or reset the superuser password.
The default PostgreSQL administration account is 'postgres'.
This account always exists and cannot be removed.
Its attributes and access can however be altered.
Clients can connect from the localhost or remotely via TCP/IP:
Local clients (e.g. psql) can connect from a preset *system* account
called 'postgres'.
This system account has no password and is *locked* by default,
so that it can be used by *local* users only.
It should *never* be enabled (or its password set)!!!
That would just open up a new attack vector on the system account.
Remote clients should use a build-in *database* account of the same
name. It's password can be changed using the "ALTER USER" statement.
Access to this account is disabled by Trove exposed only once the
superuser access is requested.
Trove itself creates its own administrative account.
{"_name": "postgres", "_password": "******"}
"""
user = self.build_root_user(root_password)
self.build_admin().alter_user(context, user, None,
*PgSqlAdmin.ADMIN_OPTIONS)
return user.serialize()
def build_root_user(self, password=None):
return models.PostgreSQLUser.root(password=password)
def pg_start_backup(self, backup_label):
r = self.build_admin().query("SELECT pg_start_backup('%s', true)" %
backup_label)
return r[0][0]
def pg_xlogfile_name(self, start_segment):
version = int(self.pg_version[1])
if version < 10:
query = "SELECT pg_xlogfile_name('%s')"
else:
query = "SELECT pg_walfile_name('%s')"
r = self.build_admin().query(query % start_segment)
return r[0][0]
def pg_stop_backup(self):
r = self.build_admin().query("SELECT pg_stop_backup()")
return r[0][0]
def disable_root(self, context):
"""Generate a new random password for the public superuser account.
Do not disable its access rights. Once enabled the account should
stay that way.
"""
self.enable_root(context)
def enable_root_with_password(self, context, root_password=None):
return self.enable_root(context, root_password)
@property
def wal_archive_location(self):
return cfg.get_configuration_property('wal_archive_location')
@property
def backup_strategy(self):
return cfg.get_configuration_property('backup_strategy')
def save_files_pre_upgrade(self, mount_point):
LOG.debug('Saving files pre-upgrade.')
mnt_etc_dir = os.path.join(mount_point, 'save_etc')
if self.OS not in [operating_system.REDHAT]:
# No need to store the config files away for Redhat because
# they are already stored in the data volume.
operating_system.remove(mnt_etc_dir, force=True, as_root=True)
operating_system.copy(self.pgsql_config_dir,
mnt_etc_dir,
preserve=True,
recursive=True,
as_root=True)
return {'save_etc': mnt_etc_dir}
def restore_files_post_upgrade(self, upgrade_info):
LOG.debug('Restoring files post-upgrade.')
if self.OS not in [operating_system.REDHAT]:
# No need to restore the config files for Redhat because
# they are already in the data volume.
operating_system.copy('%s/.' % upgrade_info['save_etc'],
self.pgsql_config_dir,
preserve=True,
recursive=True,
force=True,
as_root=True)
operating_system.remove(upgrade_info['save_etc'],
force=True,
as_root=True)
self.configuration_manager.refresh_cache()
self.status.set_ready()
if ((package_version and installed_version == package_version) or
(installed_version and not package_version)):
LOG.debug("Package %s already installed." % package_name)
else:
return False
return True
def pkg_remove(self, package_name, time_out):
"""Removes a package."""
if self.pkg_version(package_name) is None:
return
result = self._remove(package_name, time_out)
if result != OK:
if result == REINSTALL_FIRST:
self._install(package_name, time_out)
elif result == RUN_DPKG_FIRST:
self._fix(time_out)
result = self._remove(package_name, time_out)
if result != OK:
raise PkgPackageStateError("Package %s is in a bad state."
% package_name)
if operating_system.get_os() == operating_system.REDHAT:
class Package(RedhatPackagerMixin):
pass
else:
class Package(DebianPackagerMixin):
pass
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Determines operating system version and os depended commands.
"""
from trove.guestagent.common.operating_system import get_os
OS = get_os()
REDIS_CONFIG = '/etc/redis/redis.conf'
REDIS_CONF_DIR = '/etc/redis'
REDIS_INIT = '/etc/init/redis-server.conf'
REDIS_CLI = '/usr/bin/redis-cli'
REDIS_BIN = '/usr/bin/redis-server'
REDIS_CMD_ENABLE = 'update-rc.d redis-server enable'
REDIS_CMD_DISABLE = 'update-rc.d redis-server disable'
REDIS_CMD_START = 'service redis-server start || /bin/true'
REDIS_CMD_STOP = 'service redis-server stop || /bin/true'
REDIS_PACKAGE = 'redis-server'
REDIS_BASE_DIR = '/var/lib/redis'
if OS is 'redhat':
REDIS_BIN = '/usr/libexec/redis-server'
REDIS_CMD_ENABLE = 'chkconfig redis-server on'
ENGINE = None
DATADIR = None
PREPARING = False
UUID = False
TMP_MYCNF = "/tmp/my.cnf.tmp"
MYSQL_BASE_DIR = "/var/lib/mysql"
CONF = cfg.CONF
INCLUDE_MARKER_OPERATORS = {
True: ">=",
False: ">"
}
OS_NAME = operating_system.get_os()
MYSQL_CONFIG = {operating_system.REDHAT: "/etc/my.cnf",
operating_system.DEBIAN: "/etc/mysql/my.cnf",
operating_system.SUSE: "/etc/my.cnf"}[OS_NAME]
MYSQL_SERVICE_CANDIDATES = ["mysql", "mysqld", "mysql-server"]
MYSQL_BIN_CANDIDATES = ["/usr/sbin/mysqld", "/usr/libexec/mysqld"]
MYSQL_OWNER = 'mysql'
CNF_EXT = 'cnf'
CNF_INCLUDE_DIR = '/etc/mysql/conf.d/'
CNF_MASTER = 'master-replication'
CNF_SLAVE = 'slave-replication'
# Create a package impl
packager = pkg.Package()
class PgSqlConfig(PgSqlProcess):
"""Mixin that implements the config API.
This mixin has a dependency on the PgSqlProcess mixin.
"""
OS = operating_system.get_os()
CONFIG_BASE = {
operating_system.DEBIAN: '/etc/postgresql/',
operating_system.REDHAT: '/var/lib/postgresql/',
operating_system.SUSE: '/var/lib/pgsql/'}[OS]
LISTEN_ADDRESSES = ['*'] # Listen on all available IP (v4/v6) interfaces.
def __init__(self, *args, **kwargs):
super(PgSqlConfig, self).__init__(*args, **kwargs)
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(self.pgsql_config),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
self._configuration_manager = ConfigurationManager(
self.pgsql_config, self.PGSQL_OWNER, self.PGSQL_OWNER,
PropertiesCodec(
delimiter='=',
string_mappings={'on': True, 'off': False, "''": None}),
requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
@property
def pgsql_config(self):
return self._find_config_file('postgresql.conf')
@property
def pgsql_hba_config(self):
return self._find_config_file('pg_hba.conf')
@property
def pgsql_ident_config(self):
return self._find_config_file('pg_ident.conf')
def _find_config_file(self, name_pattern):
version_base = guestagent_utils.build_file_path(self.CONFIG_BASE,
self.pg_version[1])
return sorted(operating_system.list_files_in_directory(
version_base, recursive=True, pattern=name_pattern,
as_root=True), key=len)[0]
def update_overrides(self, context, overrides, remove=False):
if remove:
self.configuration_manager.remove_user_override()
elif overrides:
self.configuration_manager.apply_user_override(overrides)
def apply_overrides(self, context, overrides):
# Send a signal to the server, causing configuration files to be
# reloaded by all server processes.
# Active queries or connections to the database will not be
# interrupted.
#
# NOTE: Do not use the 'SET' command as it only affects the current
# session.
pgutil.psql("SELECT pg_reload_conf()")
def reset_configuration(self, context, configuration):
"""Reset the PgSql configuration to the one given.
"""
config_contents = configuration['config_contents']
self.configuration_manager.save_configuration(config_contents)
def start_db_with_conf_changes(self, context, config_contents):
"""Starts the PgSql instance with a new configuration."""
if PgSqlAppStatus.get().is_running:
raise RuntimeError(_("The service is still running."))
self.configuration_manager.save_configuration(config_contents)
# The configuration template has to be updated with
# guestagent-controlled settings.
self.apply_initial_guestagent_configuration()
self.start_db(context)
def apply_initial_guestagent_configuration(self):
"""Update guestagent-controlled configuration properties.
"""
LOG.debug("Applying initial guestagent configuration.")
file_locations = {
'data_directory': self._quote(self.pgsql_data_dir),
'hba_file': self._quote(self.pgsql_hba_config),
'ident_file': self._quote(self.pgsql_ident_config),
'external_pid_file': self._quote(self.PID_FILE),
'unix_socket_directories': self._quote(self.UNIX_SOCKET_DIR),
'listen_addresses': self._quote(','.join(self.LISTEN_ADDRESSES)),
'port': CONF.postgresql.postgresql_port}
self.configuration_manager.apply_system_override(file_locations)
self._apply_access_rules()
@staticmethod
def _quote(value):
return "'%s'" % value
def _apply_access_rules(self):
LOG.debug("Applying database access rules.")
# Connections to all resources are granted.
#
# Local access from administrative users is implicitly trusted.
#
# Remote access from the Trove's account is always rejected as
# it is not needed and could be used by malicious users to hijack the
# instance.
#
# Connections from other accounts always require a double-MD5-hashed
# password.
#
# Make the rules readable only by the Postgres service.
#
# NOTE: The order of entries is important.
# The first failure to authenticate stops the lookup.
# That is why the 'local' connections validate first.
# The OrderedDict is necessary to guarantee the iteration order.
access_rules = OrderedDict(
[('local', [['all', 'postgres,os_admin', None, 'trust'],
['all', 'all', None, 'md5']]),
('host', [['all', 'postgres,os_admin', '127.0.0.1/32', 'trust'],
['all', 'postgres,os_admin', '::1/128', 'trust'],
['all', 'postgres,os_admin', 'localhost', 'trust'],
['all', 'os_admin', '0.0.0.0/0', 'reject'],
['all', 'os_admin', '::/0', 'reject'],
['all', 'all', '0.0.0.0/0', 'md5'],
['all', 'all', '::/0', 'md5']])
])
operating_system.write_file(self.pgsql_hba_config, access_rules,
PropertiesCodec(
string_mappings={'\t': None}),
as_root=True)
operating_system.chown(self.pgsql_hba_config,
self.PGSQL_OWNER, self.PGSQL_OWNER,
as_root=True)
operating_system.chmod(self.pgsql_hba_config, FileMode.SET_USR_RO,
as_root=True)
def test_get_os_debian(self):
with patch.object(os.path, 'isfile', side_effect=[False, False]):
find_os = operating_system.get_os()
self.assertEqual('debian', find_os)
class MariaDBApp(galera_service.GaleraApp):
OS = operating_system.get_os()
def __init__(self, status):
super(MariaDBApp,
self).__init__(status, mysql_service.BaseLocalSqlClient,
mysql_service.BaseKeepAliveConnection)
@property
def service_candidates(self):
service_candidates = super(MariaDBApp, self).service_candidates
return {
operating_system.DEBIAN: ["mariadb"] + service_candidates,
operating_system.REDHAT: ["mariadb"],
operating_system.SUSE: service_candidates
}[self.OS]
@property
def mysql_service(self):
result = super(MariaDBApp, self).mysql_service
if result['type'] == 'sysvinit':
result['cmd_bootstrap_galera_cluster'] = (
"sudo service %s bootstrap" % result['service'])
elif result['type'] == 'systemd':
# TODO(mwj 2016/01/28): determine RHEL start for MariaDB Cluster
result['cmd_bootstrap_galera_cluster'] = (
"sudo systemctl start %[email protected]" %
result['service'])
return result
@property
def cluster_configuration(self):
return self.configuration_manager.get_value('galera')
def _get_slave_status(self):
with self.local_sql_client(self.get_engine()) as client:
return client.execute('SHOW SLAVE STATUS').first()
def _get_master_UUID(self):
slave_status = self._get_slave_status()
return slave_status and slave_status['Master_Server_Id'] or None
def _get_gtid_executed(self):
with self.local_sql_client(self.get_engine()) as client:
return client.execute('SELECT @@global.gtid_binlog_pos').first()[0]
def get_last_txn(self):
master_UUID = self._get_master_UUID()
last_txn_id = '0'
gtid_executed = self._get_gtid_executed()
for gtid_set in gtid_executed.split(','):
uuid_set = gtid_set.split('-')
if uuid_set[1] == master_UUID:
last_txn_id = uuid_set[-1]
break
return master_UUID, int(last_txn_id)
def get_latest_txn_id(self):
LOG.info(_("Retrieving latest txn id."))
return self._get_gtid_executed()
def wait_for_txn(self, txn):
LOG.info(_("Waiting on txn '%s'."), txn)
with self.local_sql_client(self.get_engine()) as client:
client.execute("SELECT MASTER_GTID_WAIT('%s')" % txn)
return False
return True
def pkg_remove(self, package_name, time_out):
"""Removes a package."""
if self.pkg_version(package_name) is None:
return
result = self._remove(package_name, time_out)
if result != OK:
if result == REINSTALL_FIRST:
self._install(package_name, time_out)
elif result == RUN_DPKG_FIRST:
self._fix(time_out)
result = self._remove(package_name, time_out)
if result != OK:
raise PkgPackageStateError("Package %s is in a bad state." %
package_name)
if operating_system.get_os() in [
operating_system.REDHAT, operating_system.ORACLE
]:
class Package(RedhatPackagerMixin):
pass
else:
class Package(DebianPackagerMixin):
pass
(installed_version and not package_version)):
LOG.debug("Package %s already installed." % package_name)
else:
return False
return True
def pkg_remove(self, package_name, time_out):
"""Removes a package."""
if self.pkg_version(package_name) is None:
return
result = self._remove(package_name, time_out)
if result != OK:
if result == REINSTALL_FIRST:
self._install(package_name, time_out)
elif result == RUN_DPKG_FIRST:
self._fix(time_out)
result = self._remove(package_name, time_out)
if result != OK:
raise PkgPackageStateError(_("Package %s is in a bad state.")
% package_name)
if operating_system.get_os() in [operating_system.REDHAT,
operating_system.ORACLE]:
class Package(RedhatPackagerMixin):
pass
else:
class Package(DebianPackagerMixin):
pass
