def enable_as_master(self, service, master_config, for_failover=False):
"""For a server to be a master in postgres, we need to enable
the replication user in pg_hba and ensure that WAL logging is
at the appropriate level (use the same settings as backups)
"""
LOG.debug("Enabling as master, with cfg: %s " % master_config)
self._get_or_create_replication_user()
hba_entry = "host replication replicator 0.0.0.0/0 md5 \n"
# TODO(atomic77) Remove this hack after adding cfg manager for pg_hba
tmp_hba = '/tmp/pg_hba'
operating_system.copy(self.PGSQL_HBA_CONFIG, tmp_hba,
force=True, as_root=True)
operating_system.chmod(tmp_hba, FileMode.OCTAL_MODE("0777"),
as_root=True)
with open(tmp_hba, 'a+') as hba_file:
hba_file.write(hba_entry)
operating_system.copy(tmp_hba, self.PGSQL_HBA_CONFIG,
force=True, as_root=True)
operating_system.chmod(self.PGSQL_HBA_CONFIG,
FileMode.OCTAL_MODE("0600"),
as_root=True)
operating_system.remove(tmp_hba, as_root=True)
pgutil.psql("SELECT pg_reload_conf()")
def enable_as_master(self, service, master_config, for_failover=False):
# For a server to be a master in postgres, we need to enable
# replication user in pg_hba and ensure that WAL logging is
# the appropriate level (use the same settings as backups)
self._get_or_create_replication_user()
hba_entry = "host replication replicator 0.0.0.0/0 md5 \n"
tmp_hba = '/tmp/pg_hba'
operating_system.copy(self.pgsql_hba_config,
tmp_hba,
force=True,
as_root=True)
operating_system.chmod(tmp_hba, FileMode.SET_ALL_RWX(), as_root=True)
with open(tmp_hba, 'a+') as hba_file:
hba_file.write(hba_entry)
operating_system.copy(tmp_hba,
self.pgsql_hba_config,
force=True,
as_root=True)
operating_system.chmod(self.pgsql_hba_config,
FileMode.SET_USR_RWX(),
as_root=True)
operating_system.remove(tmp_hba, as_root=True)
pgutil.psql("SELECT pg_reload_conf()")
def enable_as_master(self, service, master_config, for_failover=False):
"""For a server to be a master in postgres, we need to enable
the replication user in pg_hba and ensure that WAL logging is
at the appropriate level (use the same settings as backups)
"""
LOG.debug("Enabling as master, with cfg: %s " % master_config)
self._get_or_create_replication_user(service)
hba_entry = "host replication replicator 0.0.0.0/0 md5 \n"
# TODO(atomic77) Remove this hack after adding cfg manager for pg_hba
tmp_hba = '/tmp/pg_hba'
operating_system.copy(service.pgsql_hba_config,
tmp_hba,
force=True,
as_root=True)
operating_system.chmod(tmp_hba,
FileMode.OCTAL_MODE("0777"),
as_root=True)
with open(tmp_hba, 'a+') as hba_file:
hba_file.write(hba_entry)
operating_system.copy(tmp_hba,
service.pgsql_hba_config,
force=True,
as_root=True)
operating_system.chmod(service.pgsql_hba_config,
FileMode.OCTAL_MODE("0600"),
as_root=True)
operating_system.remove(tmp_hba, as_root=True)
service.reload_configuration()
def write_oracle_user_file(self, filepath, contents,
filemode=operating_system.FileMode.SET_USR_RW):
operating_system.write_file(filepath, contents, as_root=True)
operating_system.chown(filepath, INSTANCE_OWNER, INSTANCE_OWNER_GROUP,
force=True, as_root=True)
operating_system.chmod(filepath, filemode,
force=True, as_root=True)
def reset_root_password(self):
"""Reset the password of the localhost root account used by Trove
for initial datastore configuration.
"""
try:
# Do not attempt to delete these files as the 'trove' user.
# The process writing into it may have assumed its ownership.
# Only owners can delete temporary files (restricted deletion).
init_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
operating_system.write_file(init_file.name,
self.RESET_ROOT_MYSQL_COMMANDS)
operating_system.chmod(init_file.name,
FileMode.ADD_READ_ALL,
as_root=True)
err_log_file = tempfile.NamedTemporaryFile(
suffix=self._ERROR_LOG_SUFFIX, delete=False)
self._start_mysqld_safe_with_init_file(init_file, err_log_file)
finally:
init_file.close()
err_log_file.close()
operating_system.remove(init_file.name, force=True, as_root=True)
operating_system.remove(err_log_file.name,
force=True,
as_root=True)
def base_backup_metadata(self, f):
"""Parse the contents of the .backup file"""
meta = {}
operating_system.chmod(f, FileMode(add=[stat.S_IROTH]), as_root=True)
start_re = re.compile("START WAL LOCATION: (.*) \(file (.*)\)")
stop_re = re.compile("STOP WAL LOCATION: (.*) \(file (.*)\)")
checkpt_re = re.compile("CHECKPOINT LOCATION: (.*)")
label_re = re.compile("LABEL: (.*)")
with open(f, 'r') as base_metadata:
lines = "\n".join(base_metadata.readlines())
match = start_re.search(lines)
if match:
self.start_segment = meta['start-segment'] = match.group(1)
self.start_wal_file = meta['start-wal-file'] = match.group(2)
match = stop_re.search(lines)
if match:
self.stop_segment = meta['stop-segment'] = match.group(1)
self.stop_wal_file = meta['stop-wal-file'] = match.group(2)
match = checkpt_re.search(lines)
if match:
self.checkpoint_location \
= meta['checkpoint-location'] = match.group(1)
match = label_re.search(lines)
if match:
self.label = meta['label'] = match.group(1)
return meta
def reset_configuration(self, options, remove_overrides=False):
"""Write given contents to the base configuration file.
Remove all existing overrides (both system and user) as required.
:param options: Contents of the configuration file (string or dict).
:param remove_overrides: Remove the overrides or not.
"""
if isinstance(options, dict):
# Serialize a dict of options for writing.
self.reset_configuration(self._codec.serialize(options),
remove_overrides=remove_overrides)
else:
if remove_overrides:
self._override_strategy.remove(self.USER_GROUP)
self._override_strategy.remove(self.SYSTEM_PRE_USER_GROUP)
self._override_strategy.remove(self.SYSTEM_POST_USER_GROUP)
operating_system.write_file(self._base_config_path,
options,
as_root=self._requires_root)
operating_system.chown(self._base_config_path,
self._owner,
self._group,
as_root=self._requires_root)
operating_system.chmod(self._base_config_path,
FileMode.ADD_READ_ALL,
as_root=self._requires_root)
self.refresh_cache()
def reset_root_password(self):
"""Reset the password of the localhost root account used by Trove
for initial datastore configuration.
"""
with tempfile.NamedTemporaryFile(mode='w') as init_file:
operating_system.write_file(init_file.name,
self.RESET_ROOT_MYSQL_COMMANDS)
operating_system.chmod(init_file.name,
FileMode.ADD_READ_ALL,
as_root=True)
# Do not attempt to delete the file as the 'trove' user.
# The process writing into it may have assumed its ownership.
# Only owners can delete temporary
# files (restricted deletion).
err_log_file = tempfile.NamedTemporaryFile(
suffix=self._ERROR_LOG_SUFFIX, delete=False)
try:
# As of MySQL 5.7.6, for MySQL installation using an RPM
# distribution, server startup and shutdown is managed by
# systemd on several Linux platforms. On these platforms,
# mysqld_safe is no longer installed because it is
# unnecessary.
if self._mysqld_safe_cmd_exists():
self._start_mysqld_safe_with_init_file(
init_file, err_log_file)
else:
self._start_mysqld_with_init_file(init_file)
finally:
err_log_file.close()
operating_system.remove(err_log_file.name,
force=True,
as_root=True)
def _create_replication_user(self, pwfile):
"""Create the replication user. Unfortunately, to be able to
run pg_rewind, we need SUPERUSER, not just REPLICATION privilege
"""
pw = utils.generate_random_password()
operating_system.write_file(pwfile, pw, as_root=True)
operating_system.chown(pwfile,
user=self.PGSQL_OWNER,
group=self.PGSQL_OWNER,
as_root=True)
operating_system.chmod(pwfile,
FileMode.OCTAL_MODE("0600"),
as_root=True)
# TODO(atomic77) Alter user is swallowing the replication
# option for some reason -- enable this code when the
# underlying issue is fixed
# repl_user = models.PostgreSQLUser(name=REPL_USER,
# password=REPL_PW)
# self._create_user(context=None, user=repl_user)
# self.alter_user(None, repl_user, 'REPLICATION', 'LOGIN')
pgutil.psql("CREATE USER %s SUPERUSER ENCRYPTED "
"password '%s';" % (REPL_USER, pw))
return pw
def pre_restore(self):
self.app.stop_db()
LOG.info("Cleaning out restore location: %s.",
self.restore_location)
operating_system.chmod(self.restore_location, FileMode.SET_FULL,
as_root=True)
utils.clean_out(self.restore_location)
def __init__(self, log_context, log_name, log_type, log_user, log_file,
log_exposed):
self._context = log_context
self._name = log_name
self._type = log_type
self._user = log_user
self._file = log_file
self._exposed = log_exposed
self._size = None
self._published_size = None
self._header_digest = 'abc'
self._published_header_digest = None
self._status = None
self._cached_context = None
self._cached_swift_client = None
self._enabled = log_type == LogType.SYS
self._file_readable = False
self._container_name = None
self._codec = stream_codecs.JsonCodec()
self._set_status(self._type == LogType.USER, LogStatus.Disabled,
LogStatus.Enabled)
# The directory should already exist - make sure we have access to it
log_dir = os.path.dirname(self._file)
operating_system.chmod(log_dir,
FileMode.ADD_GRP_RX_OTH_RX,
as_root=True)
def apply(self, group_name, change_id, options):
self._initialize_import_directory()
revision_file = self._find_revision_file(group_name, change_id)
if revision_file is None:
# Create a new file.
last_revision_index = self._get_last_file_index(group_name)
revision_file = guestagent_utils.build_file_path(
self._revision_dir,
'%s-%03d-%s' % (group_name, last_revision_index + 1,
change_id),
self._revision_ext)
else:
# Update the existing file.
current = operating_system.read_file(
revision_file, codec=self._codec, as_root=self._requires_root)
options = guestagent_utils.update_dict(options, current)
operating_system.write_file(
revision_file, options, codec=self._codec,
as_root=self._requires_root)
operating_system.chown(
revision_file, self._owner, self._group,
as_root=self._requires_root)
operating_system.chmod(
revision_file, FileMode.ADD_READ_ALL, as_root=self._requires_root)
def _update_details(self):
if operating_system.exists(self._file, as_root=True):
file_path = Path(self._file)
# Make sure guest agent can read the log file.
if not os.access(self._file, os.R_OK):
operating_system.chmod(self._file,
FileMode.ADD_ALL_R,
as_root=True)
operating_system.chmod(str(file_path.parent),
FileMode.ADD_GRP_RX_OTH_RX,
as_root=True)
self._size = file_path.stat().st_size
self._update_log_header_digest(self._file)
if self.status != LogStatus.Disabled:
if self._log_rotated():
self.status = LogStatus.Rotated
# See if we have stuff to publish
elif self._size > self._published_size:
self._set_status(self._published_size, LogStatus.Partial,
LogStatus.Ready)
# We've published everything so far
elif self._size == self._published_size:
self._set_status(self._published_size, LogStatus.Published,
LogStatus.Enabled)
# We've already handled this case (log rotated) so what gives?
else:
raise Exception(_("Bug in _log_rotated ?"))
else:
LOG.warning(f"File {self._file} does not exist")
self._published_size = 0
self._size = 0
def apply(self, group_name, change_id, options):
revision_file = self._find_revision_file(group_name, change_id)
if revision_file is None:
# Create a new file.
last_revision_index = self._get_last_file_index(group_name)
revision_file = guestagent_utils.build_file_path(
self._revision_dir, '%s-%03d-%s' %
(group_name, last_revision_index + 1, change_id),
self._revision_ext)
else:
# Update the existing file.
current = operating_system.read_file(revision_file,
codec=self._codec)
options = guestagent_utils.update_dict(options, current)
operating_system.write_file(revision_file,
options,
codec=self._codec,
as_root=self._requires_root)
operating_system.chown(revision_file,
self._owner,
self._group,
as_root=self._requires_root)
operating_system.chmod(revision_file,
FileMode.ADD_READ_ALL,
as_root=self._requires_root)
def enable_as_master(self, service, master_config):
"""Primary postgredql settings.
For a server to be a master in postgres, we need to enable
the replication user in pg_hba.conf
"""
self._get_or_create_replication_user(service)
hba_entry = f"host replication {REPL_USER} 0.0.0.0/0 md5\n"
tmp_hba = '/tmp/pg_hba'
operating_system.copy(pg_service.HBA_CONFIG_FILE, tmp_hba,
force=True, as_root=True)
operating_system.chmod(tmp_hba, FileMode.SET_ALL_RWX(),
as_root=True)
with open(tmp_hba, 'a+') as hba_file:
hba_file.write(hba_entry)
operating_system.copy(tmp_hba, pg_service.HBA_CONFIG_FILE,
force=True, as_root=True)
operating_system.chown(pg_service.HBA_CONFIG_FILE,
user=CONF.database_service_uid,
group=CONF.database_service_uid, as_root=True)
operating_system.chmod(pg_service.HBA_CONFIG_FILE,
FileMode.SET_USR_RWX(),
as_root=True)
operating_system.remove(tmp_hba, as_root=True)
LOG.debug(f"{pg_service.HBA_CONFIG_FILE} changed")
service.restart()
def pre_restore(self):
self.app.stop_db()
LOG.info(_("Cleaning out restore location: %s."),
self.restore_location)
operating_system.chmod(self.restore_location, FileMode.SET_FULL,
as_root=True)
utils.clean_out(self.restore_location)
def __init__(self, status):
self.state_change_wait_time = CONF.state_change_wait_time
self.status = status
revision_dir = \
guestagent_utils.build_file_path(
os.path.join(MOUNT_POINT,
os.path.dirname(system.VERTICA_ADMIN)),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
if not operating_system.exists(FAKE_CFG):
operating_system.write_file(FAKE_CFG, '', as_root=True)
operating_system.chown(FAKE_CFG,
system.VERTICA_ADMIN,
system.VERTICA_ADMIN_GRP,
as_root=True)
operating_system.chmod(FAKE_CFG,
FileMode.ADD_GRP_RX_OTH_RX(),
as_root=True)
self.configuration_manager = \
ConfigurationManager(FAKE_CFG, system.VERTICA_ADMIN,
system.VERTICA_ADMIN_GRP,
PropertiesCodec(delimiter='='),
requires_root=True,
override_strategy=ImportOverrideStrategy(
revision_dir, "cnf"))
def reset_root_password(self):
"""Reset the password of the localhost root account used by Trove
for initial datastore configuration.
"""
with tempfile.NamedTemporaryFile(mode='w') as init_file:
operating_system.write_file(init_file.name,
self.RESET_ROOT_MYSQL_COMMANDS)
operating_system.chmod(init_file.name, FileMode.ADD_READ_ALL,
as_root=True)
# Do not attempt to delete the file as the 'trove' user.
# The process writing into it may have assumed its ownership.
# Only owners can delete temporary
# files (restricted deletion).
err_log_file = tempfile.NamedTemporaryFile(
suffix=self._ERROR_LOG_SUFFIX,
delete=False)
try:
# As of MySQL 5.7.6, for MySQL installation using an RPM
# distribution, server startup and shutdown is managed by
# systemd on several Linux platforms. On these platforms,
# mysqld_safe is no longer installed because it is
# unnecessary.
if self._mysqld_safe_cmd_exists():
self._start_mysqld_safe_with_init_file(
init_file, err_log_file)
else:
self._start_mysqld_with_init_file(init_file)
finally:
err_log_file.close()
operating_system.remove(
err_log_file.name, force=True, as_root=True)
def _incremental_restore(self, location, checksum):
metadata = self.storage.load_metadata(location, checksum)
LOG.info(_("Metadata for inc backup: %s") % str(metadata))
if 'parent_location' in metadata:
LOG.info("Found parent at %s", metadata['parent_location'])
parent_location = metadata['parent_location']
parent_checksum = metadata['parent_checksum']
self._incremental_restore(parent_location, parent_checksum)
cmd = self._incremental_restore_cmd(incr=True)
self.content_length += self._unpack(location, checksum, cmd)
else:
# For the parent base backup, revert to the default restore cmd
LOG.info("Recursed back to full backup ")
super(PgBaseBackupIncremental, self).pre_restore()
cmd = self._incremental_restore_cmd(incr=False)
self.content_length += self._unpack(location, checksum, cmd)
operating_system.chmod(self.PGSQL_DATA_DIR,
FileMode.OCTAL_MODE("0700"),
as_root=True,
recursive=True,
force=True)
def __init__(self, log_context, log_name, log_type, log_user, log_file,
log_exposed):
self._context = log_context
self._name = log_name
self._type = log_type
self._user = log_user
self._file = log_file
self._exposed = log_exposed
self._size = None
self._published_size = None
self._header_digest = 'abc'
self._published_header_digest = None
self._status = None
self._cached_context = None
self._cached_swift_client = None
self._enabled = log_type == LogType.SYS
self._file_readable = False
self._container_name = None
self._codec = stream_codecs.JsonCodec()
self._set_status(self._type == LogType.USER,
LogStatus.Disabled, LogStatus.Enabled)
# The directory should already exist - make sure we have access to it
log_dir = os.path.dirname(self._file)
operating_system.chmod(
log_dir, FileMode.ADD_GRP_RX_OTH_RX, as_root=True)
def write_config(
self,
config_contents,
execute_function=utils.execute_with_timeout,
mkstemp_function=tempfile.mkstemp,
unlink_function=os.unlink,
):
# first securely create a temp file. mkstemp() will set
# os.O_EXCL on the open() call, and we get a file with
# permissions of 600 by default.
(conf_fd, conf_path) = mkstemp_function()
LOG.debug("Storing temporary configuration at %s." % conf_path)
# write config and close the file, delete it if there is an
# error. only unlink if there is a problem. In normal course,
# we move the file.
try:
os.write(conf_fd, config_contents)
operating_system.move(conf_path, system.CASSANDRA_CONF, as_root=True)
# TODO(denis_makogon): figure out the dynamic way to discover
# configs owner since it can cause errors if there is
# no cassandra user in operating system
operating_system.chown(system.CASSANDRA_CONF, "cassandra", "cassandra", recursive=False, as_root=True)
operating_system.chmod(system.CASSANDRA_CONF, FileMode.ADD_READ_ALL, as_root=True)
except Exception:
LOG.exception(_("Exception generating Cassandra configuration %s.") % conf_path)
unlink_function(conf_path)
raise
finally:
os.close(conf_fd)
LOG.info(_("Wrote new Cassandra configuration."))
def clear_file(filename):
LOG.debug("Creating clean file %s" % filename)
if operating_system.file_discovery([filename]):
operating_system.remove(filename)
# force file creation by just opening it
open(filename, "wb")
operating_system.chmod(filename, operating_system.FileMode.SET_USR_RW, as_root=True)
def base_backup_metadata(self, f):
"""Parse the contents of the .backup file"""
meta = {}
operating_system.chmod(f, FileMode(add=[stat.S_IROTH]), as_root=True)
start_re = re.compile("START WAL LOCATION: (.*) \(file (.*)\)")
stop_re = re.compile("STOP WAL LOCATION: (.*) \(file (.*)\)")
checkpt_re = re.compile("CHECKPOINT LOCATION: (.*)")
label_re = re.compile("LABEL: (.*)")
with open(f, 'r') as base_metadata:
lines = "\n".join(base_metadata.readlines())
match = start_re.search(lines)
if match:
self.start_segment = meta['start-segment'] = match.group(1)
self.start_wal_file = meta['start-wal-file'] = match.group(2)
match = stop_re.search(lines)
if match:
self.stop_segment = meta['stop-segment'] = match.group(1)
self.stop_wal_file = meta['stop-wal-file'] = match.group(2)
match = checkpt_re.search(lines)
if match:
self.checkpoint_location \
= meta['checkpoint-location'] = match.group(1)
match = label_re.search(lines)
if match:
self.label = meta['label'] = match.group(1)
return meta
def save_configuration(self, options):
"""Write given contents to the base configuration file.
Remove all existing overrides (both system and user).
:param contents Contents of the configuration file.
:type contents string or dict
"""
if isinstance(options, dict):
# Serialize a dict of options for writing.
self.save_configuration(self._codec.serialize(options))
else:
self._override_strategy.remove(self.USER_GROUP)
self._override_strategy.remove(self.SYSTEM_PRE_USER_GROUP)
self._override_strategy.remove(self.SYSTEM_POST_USER_GROUP)
operating_system.write_file(
self._base_config_path, options, as_root=self._requires_root)
operating_system.chown(
self._base_config_path, self._owner, self._group,
as_root=self._requires_root)
operating_system.chmod(
self._base_config_path, FileMode.ADD_READ_ALL,
as_root=self._requires_root)
self.refresh_cache()
def save_configuration(self, options):
"""Write given contents to the base configuration file.
Remove all existing overrides (both system and user).
:param contents Contents of the configuration file.
:type contents string or dict
"""
if isinstance(options, dict):
# Serialize a dict of options for writing.
self.save_configuration(self._codec.serialize(options))
else:
self._override_strategy.remove(self.USER_GROUP)
self._override_strategy.remove(self.SYSTEM_GROUP)
operating_system.write_file(self._base_config_path,
options,
as_root=self._requires_root)
operating_system.chown(self._base_config_path,
self._owner,
self._group,
as_root=self._requires_root)
operating_system.chmod(self._base_config_path,
FileMode.ADD_READ_ALL,
as_root=self._requires_root)
self._refresh_cache()
def apply_access_rules(self):
"""PostgreSQL Client authentication settings
The order of entries is important. The first failure to authenticate
stops the lookup. That is why the 'local' connections validate first.
The OrderedDict is necessary to guarantee the iteration order.
"""
LOG.debug("Applying client authentication access rules.")
access_rules = OrderedDict([
('local', [['all', SUPER_USER_NAME, None, 'trust'],
['replication', SUPER_USER_NAME, None, 'trust'],
['all', 'all', None, 'md5']]),
('host', [['all', SUPER_USER_NAME, '127.0.0.1/32', 'trust'],
['all', SUPER_USER_NAME, '::1/128', 'trust'],
['all', SUPER_USER_NAME, 'localhost', 'trust'],
['all', SUPER_USER_NAME, '0.0.0.0/0', 'reject'],
['all', SUPER_USER_NAME, '::/0', 'reject'],
['all', 'all', '0.0.0.0/0', 'md5'],
['all', 'all', '::/0', 'md5']])
])
operating_system.write_file(
HBA_CONFIG_FILE,
access_rules,
stream_codecs.PropertiesCodec(string_mappings={'\t': None}),
as_root=True)
operating_system.chown(HBA_CONFIG_FILE,
CONF.database_service_uid,
CONF.database_service_uid,
as_root=True)
operating_system.chmod(HBA_CONFIG_FILE,
operating_system.FileMode.SET_USR_RO,
as_root=True)
def reset_password_for_restore(self,
ds_version=None,
data_dir='/var/lib/mysql/data'):
"""Reset the root password after restore the db data.
We create a temporary database container by running mysqld_safe to
reset the root password.
"""
LOG.info('Starting to reset password for restore')
try:
root_pass = self.app.get_auth_password(file="root.cnf")
except exception.UnprocessableEntity:
root_pass = utils.generate_random_password()
self.app.save_password('root', root_pass)
with tempfile.NamedTemporaryFile(mode='w') as init_file, \
tempfile.NamedTemporaryFile(suffix='.err') as err_file:
operating_system.write_file(
init_file.name,
f"ALTER USER 'root'@'localhost' IDENTIFIED BY '{root_pass}';")
command = (f'mysqld_safe --init-file={init_file.name} '
f'--log-error={err_file.name} '
f'--datadir={data_dir}')
extra_volumes = {
init_file.name: {
"bind": init_file.name,
"mode": "rw"
},
err_file.name: {
"bind": err_file.name,
"mode": "rw"
},
}
# Allow database service user to access the temporary files.
for file in [init_file.name, err_file.name]:
operating_system.chmod(file,
operating_system.FileMode.SET_ALL_RWX(),
force=True,
as_root=True)
try:
self.app.start_db(ds_version=ds_version,
command=command,
extra_volumes=extra_volumes)
except Exception as err:
LOG.error('Failed to reset password for restore, error: %s',
str(err))
LOG.debug('Content in init error log file: %s',
err_file.read())
raise err
finally:
LOG.debug(
'The init container log: %s',
docker_util.get_container_logs(self.app.docker_client))
docker_util.remove_container(self.app.docker_client)
LOG.info('Finished to reset password for restore')
def pre_restore(self):
app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())
app.stop_db()
LOG.info(_("Cleaning out restore location: %s."),
self.restore_location)
operating_system.chmod(self.restore_location, FileMode.SET_FULL,
as_root=True)
utils.clean_out(self.restore_location)
def apply_next(self, options):
revision_num = self.count_revisions() + 1
revision_file_path = guestagent_utils.build_file_path(
self._revision_dir, self._base_config_name, str(revision_num), self._revision_ext
)
operating_system.write_file(revision_file_path, options, codec=self._codec, as_root=self._requires_root)
operating_system.chown(revision_file_path, self._owner, self._group, as_root=self._requires_root)
operating_system.chmod(revision_file_path, FileMode.ADD_READ_ALL, as_root=self._requires_root)
def pre_restore(self):
app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())
app.stop_db()
LOG.info(_("Cleaning out restore location: %s."),
self.restore_location)
operating_system.chmod(self.restore_location, FileMode.SET_FULL,
as_root=True)
utils.clean_out(self.restore_location)
def store_key(self, key):
"""Store the cluster key."""
LOG.debug("Storing key for MongoDB cluster.")
with tempfile.NamedTemporaryFile() as f:
f.write(key)
f.flush()
operating_system.copy(f.name, system.MONGO_KEY_FILE, force=True, as_root=True)
operating_system.chmod(system.MONGO_KEY_FILE, operating_system.FileMode.SET_USR_RO, as_root=True)
operating_system.chown(system.MONGO_KEY_FILE, system.MONGO_USER, system.MONGO_USER, as_root=True)
def clear_file(filename):
LOG.debug("Creating clean file %s", filename)
if operating_system.file_discovery([filename]):
operating_system.remove(filename)
# force file creation by just opening it
open(filename, 'wb')
operating_system.chmod(filename,
operating_system.FileMode.SET_USR_RW,
as_root=True)
def store_key(self, key):
"""Store the cluster key."""
LOG.debug('Storing key for MongoDB cluster.')
operating_system.write_file(system.MONGO_KEY_FILE, key, as_root=True)
operating_system.chmod(system.MONGO_KEY_FILE,
operating_system.FileMode.SET_USR_RO,
as_root=True)
operating_system.chown(system.MONGO_KEY_FILE,
system.MONGO_USER, system.MONGO_USER,
as_root=True)
def _write_replication_overrides(self, overrideValues, cnf_file):
LOG.info(_("Writing replication.cnf file."))
with open(MYCNF_REPLCONFIG_TMP, 'w') as overrides:
overrides.write(overrideValues)
LOG.debug("Moving temp replication.cnf into correct location.")
operating_system.move(MYCNF_REPLCONFIG_TMP, cnf_file, as_root=True)
LOG.debug("Setting permissions on replication.cnf.")
operating_system.chmod(cnf_file, FileMode.SET_GRP_RW_OTH_R,
as_root=True)
def store_key(self, key):
"""Store the cluster key."""
LOG.debug('Storing key for MongoDB cluster.')
operating_system.write_file(system.MONGO_KEY_FILE, key, as_root=True)
operating_system.chmod(system.MONGO_KEY_FILE,
operating_system.FileMode.SET_USR_RO,
as_root=True)
operating_system.chown(system.MONGO_KEY_FILE,
system.MONGO_USER, system.MONGO_USER,
as_root=True)
def pre_restore(self):
self.app.stop_db()
LOG.info(_("Cleaning out restore location: %s."),
self.restore_location)
operating_system.chmod(self.restore_location, FileMode.SET_FULL,
as_root=True)
utils.clean_out(self.restore_location)
# IF AOF is set, we need to turn it off temporarily
if self.aof_set:
self.app.configuration_manager.apply_system_override(
self.aof_off_cfg, change_id=self.CONF_LABEL_AOF_TEMP_OFF)
def _write_config_overrides(self, overrideValues):
LOG.info(_("Writing new temp overrides.cnf file."))
with open(MYCNF_OVERRIDES_TMP, 'w') as overrides:
overrides.write(overrideValues)
LOG.info(_("Moving overrides.cnf into correct location."))
operating_system.move(MYCNF_OVERRIDES_TMP, MYCNF_OVERRIDES,
as_root=True)
LOG.info(_("Setting permissions on overrides.cnf."))
operating_system.chmod(MYCNF_OVERRIDES, FileMode.SET_GRP_RW_OTH_R,
as_root=True)
def _write_replication_overrides(self, overrideValues, cnf_file):
LOG.info(_("Writing replication.cnf file."))
with open(MYCNF_REPLCONFIG_TMP, 'w') as overrides:
overrides.write(overrideValues)
LOG.debug("Moving temp replication.cnf into correct location.")
operating_system.move(MYCNF_REPLCONFIG_TMP, cnf_file, as_root=True)
LOG.debug("Setting permissions on replication.cnf.")
operating_system.chmod(cnf_file,
FileMode.SET_GRP_RW_OTH_R,
as_root=True)
def write_oracle_user_file(self,
filepath,
contents,
filemode=operating_system.FileMode.SET_USR_RW):
operating_system.write_file(filepath, contents, as_root=True)
operating_system.chown(filepath,
INSTANCE_OWNER,
INSTANCE_OWNER_GROUP,
force=True,
as_root=True)
operating_system.chmod(filepath, filemode, force=True, as_root=True)
def restore_backup(self, context, backup_info, restore_location):
backup_id = backup_info['id']
storage_driver = CONF.storage_strategy
backup_driver = self.get_backup_strategy()
user_token = context.auth_token
auth_url = CONF.service_credentials.auth_url
user_tenant = context.project_id
image = self.get_backup_image()
name = 'db_restore'
volumes = {'/var/lib/mysql': {'bind': '/var/lib/mysql', 'mode': 'rw'}}
command = (
f'/usr/bin/python3 main.py --nobackup '
f'--storage-driver={storage_driver} --driver={backup_driver} '
f'--os-token={user_token} --os-auth-url={auth_url} '
f'--os-tenant-id={user_tenant} '
f'--restore-from={backup_info["location"]} '
f'--restore-checksum={backup_info["checksum"]}')
if CONF.backup_aes_cbc_key:
command = (f"{command} "
f"--backup-encryption-key={CONF.backup_aes_cbc_key}")
LOG.debug(
'Stop the database and clean up the data before restore '
'from %s', backup_id)
self.stop_db()
operating_system.chmod(restore_location,
operating_system.FileMode.SET_FULL,
as_root=True)
utils.clean_out(restore_location)
# Start to run restore inside a separate docker container
LOG.info('Starting to restore backup %s, command: %s', backup_id,
command)
output, ret = docker_util.run_container(self.docker_client,
image,
name,
volumes=volumes,
command=command)
result = output[-1]
if not ret:
msg = f'Failed to run restore container, error: {result}'
LOG.error(msg)
raise Exception(msg)
LOG.debug('Deleting ib_logfile files after restore from backup %s',
backup_id)
operating_system.chown(restore_location,
CONF.database_service_uid,
CONF.database_service_uid,
force=True,
as_root=True)
self.wipe_ib_logfiles()
def validate_log_file(self, log_file, owner):
"""Make sure the log file exists and is accessible by owner.
"""
if not operating_system.exists(log_file, as_root=True):
operating_system.write_file(log_file, '', as_root=True)
operating_system.chown(log_file, user=owner, group=owner, as_root=True)
operating_system.chmod(log_file,
FileMode.ADD_USR_RW_GRP_RW_OTH_R,
as_root=True)
return log_file
def validate_log_file(self, log_file, owner):
"""Make sure the log file exists and is accessible by owner.
"""
if not operating_system.exists(log_file, as_root=True):
operating_system.write_file(log_file, '', as_root=True)
operating_system.chown(log_file, user=owner, group=owner,
as_root=True)
operating_system.chmod(log_file, FileMode.ADD_USR_RW_GRP_RW_OTH_R,
as_root=True)
LOG.debug("Set log file '%s' as readable" % log_file)
return log_file
def _read_last_master_gtid(self):
INFO_FILE = ('%s/xtrabackup_binlog_info' % MySqlApp.get_data_dir())
LOG.info(_("Setting read permissions on %s") % INFO_FILE)
operating_system.chmod(INFO_FILE, FileMode.ADD_READ_ALL, as_root=True)
LOG.info(_("Reading last master GTID from %s") % INFO_FILE)
try:
with open(INFO_FILE, 'rb') as f:
row = f.read().split('\t')
return row[2]
except (IOError, IndexError) as ex:
LOG.exception(ex)
raise self.UnableToDetermineLastMasterGTID(
{'binlog_file': INFO_FILE})
def pre_restore(self):
self.app.stop_db()
LOG.info(_("Removing old persistence file: %s."),
self.restore_location)
operating_system.remove(self.restore_location, force=True,
as_root=True)
dir = os.path.dirname(self.restore_location)
operating_system.create_directory(dir, as_root=True)
operating_system.chmod(dir, FileMode.SET_FULL, as_root=True)
# IF AOF is set, we need to turn it off temporarily
if self.aof_set:
self.app.configuration_manager.apply_system_override(
self.aof_off_cfg, change_id=self.CONF_LABEL_AOF_TEMP_OFF)
def _read_last_master_gtid(self):
INFO_FILE = ('%s/xtrabackup_binlog_info' % MySqlApp.get_data_dir())
LOG.info(_("Setting read permissions on %s") % INFO_FILE)
operating_system.chmod(INFO_FILE, FileMode.ADD_READ_ALL, as_root=True)
LOG.info(_("Reading last master GTID from %s") % INFO_FILE)
try:
with open(INFO_FILE, 'rb') as f:
row = f.read().split('\t')
return row[2]
except (IOError, IndexError) as ex:
LOG.exception(ex)
raise self.UnableToDetermineLastMasterGTID(
{'binlog_file': INFO_FILE})
def save_configuration(self, contents):
"""Write given contents to the base configuration file.
Remove all existing revisions.
:param contents Plain-text contents of the configuration file.
:type contents string
"""
if self._override_strategy:
self._override_strategy.remove_last(self._current_revision + 1)
operating_system.write_file(self._base_config_path, contents, as_root=self._requires_root)
operating_system.chown(self._base_config_path, self._owner, self._group, as_root=self._requires_root)
operating_system.chmod(self._base_config_path, FileMode.ADD_READ_ALL, as_root=self._requires_root)
def apply_next(self, options):
revision_num = self.count_revisions() + 1
old_revision_backup = guestagent_utils.build_file_path(
self._revision_backup_dir, self._base_config_name, str(revision_num), self._BACKUP_EXT
)
operating_system.copy(
self._base_config_path, old_revision_backup, force=True, preserve=True, as_root=self._requires_root
)
current = operating_system.read_file(self._base_config_path, codec=self._codec)
guestagent_utils.update_dict(options, current)
operating_system.write_file(self._base_config_path, current, codec=self._codec, as_root=self._requires_root)
operating_system.chown(self._base_config_path, self._owner, self._group, as_root=self._requires_root)
operating_system.chmod(self._base_config_path, FileMode.ADD_READ_ALL, as_root=self._requires_root)
def pre_restore(self):
self.app.stop_db()
LOG.info("Removing old persistence file: %s.", self.restore_location)
operating_system.remove(self.restore_location,
force=True,
as_root=True)
dir = os.path.dirname(self.restore_location)
operating_system.create_directory(dir, as_root=True)
operating_system.chmod(dir, FileMode.SET_FULL, as_root=True)
# IF AOF is set, we need to turn it off temporarily
if self.aof_set:
self.app.configuration_manager.apply_system_override(
self.aof_off_cfg, change_id=self.CONF_LABEL_AOF_TEMP_OFF)
def _write_config_overrides(self, overrideValues):
LOG.info(_("Writing new temp overrides.cnf file."))
with open(MYCNF_OVERRIDES_TMP, 'w') as overrides:
overrides.write(overrideValues)
LOG.info(_("Moving overrides.cnf into correct location."))
operating_system.move(MYCNF_OVERRIDES_TMP,
MYCNF_OVERRIDES,
as_root=True)
LOG.info(_("Setting permissions on overrides.cnf."))
operating_system.chmod(MYCNF_OVERRIDES,
FileMode.SET_GRP_RW_OTH_R,
as_root=True)
def _read_log_position(self):
INFO_FILE = ('%s/xtrabackup_binlog_info' % MySqlApp.get_data_dir())
LOG.info(_("Setting read permissions on %s") % INFO_FILE)
operating_system.chmod(INFO_FILE, FileMode.ADD_READ_ALL, as_root=True)
LOG.info(_("Reading log position from %s") % INFO_FILE)
try:
with open(INFO_FILE, 'rb') as f:
row = next(csv.reader(f, delimiter='\t',
skipinitialspace=True))
return {'log_file': row[0], 'log_position': int(row[1])}
except (IOError, IndexError) as ex:
LOG.exception(ex)
raise self.UnableToDetermineBinlogPosition(
{'binlog_file': INFO_FILE})
def change_permissions(self):
"""
When CouchDB is installed, a default user 'couchdb' is created.
Inorder to start/stop/restart CouchDB service as the current
OS user, add this user to the 'couchdb' group and provide read/
write access to the 'couchdb' group.
"""
try:
LOG.debug("Changing permissions.")
operating_system.update_owner('couchdb', 'couchdb',
COUCHDB_LIB_DIR)
operating_system.update_owner('couchdb', 'couchdb',
COUCHDB_LOG_DIR)
operating_system.update_owner('couchdb', 'couchdb',
COUCHDB_BIN_DIR)
operating_system.update_owner('couchdb', 'couchdb',
COUCHDB_CONFIG_DIR)
operating_system.chmod(COUCHDB_LIB_DIR,
FileMode.ADD_GRP_RW,
as_root=True)
operating_system.chmod(COUCHDB_LOG_DIR,
FileMode.ADD_GRP_RW,
as_root=True)
operating_system.chmod(COUCHDB_BIN_DIR,
FileMode.ADD_GRP_RW,
as_root=True)
operating_system.chmod(COUCHDB_CONFIG_DIR,
FileMode.ADD_GRP_RW,
as_root=True)
self.execute_change_permission_commands(
system.UPDATE_GROUP_MEMBERSHIP)
LOG.debug("Successfully changed permissions.")
except exception.ProcessExecutionError:
LOG.exception(_("Error changing permissions."))
def change_permissions(self):
"""
When CouchDB is installed, a default user 'couchdb' is created.
Inorder to start/stop/restart CouchDB service as the current
OS user, add this user to the 'couchdb' group and provide read/
write access to the 'couchdb' group.
"""
try:
LOG.debug("Changing permissions.")
operating_system.chown(
COUCHDB_LIB_DIR, 'couchdb', 'couchdb', as_root=True
)
operating_system.chown(
COUCHDB_LOG_DIR, 'couchdb', 'couchdb', as_root=True
)
operating_system.chown(
COUCHDB_BIN_DIR, 'couchdb', 'couchdb', as_root=True
)
operating_system.chown(
COUCHDB_CONFIG_DIR, 'couchdb', 'couchdb', as_root=True
)
operating_system.chmod(COUCHDB_LIB_DIR, FileMode.ADD_GRP_RW,
as_root=True)
operating_system.chmod(COUCHDB_LOG_DIR, FileMode.ADD_GRP_RW,
as_root=True)
operating_system.chmod(COUCHDB_BIN_DIR, FileMode.ADD_GRP_RW,
as_root=True)
operating_system.chmod(COUCHDB_CONFIG_DIR, FileMode.ADD_GRP_RW,
as_root=True)
self.execute_change_permission_commands(
system.UPDATE_GROUP_MEMBERSHIP
)
LOG.debug("Successfully changed permissions.")
except exception.ProcessExecutionError:
LOG.exception(_("Error changing permissions."))
def write_cluster_topology(self, data_center, rack, prefer_local=True):
LOG.info(_('Saving Cassandra cluster topology configuration.'))
config = {'dc': data_center,
'rack': rack,
'prefer_local': prefer_local}
operating_system.write_file(self.cassandra_topology, config,
codec=self._TOPOLOGY_CODEC, as_root=True)
operating_system.chown(
self.cassandra_topology,
self.cassandra_owner, self.cassandra_owner, as_root=True)
operating_system.chmod(
self.cassandra_topology, FileMode.ADD_READ_ALL, as_root=True)
def _create_replication_user(self, service, admin, pwfile):
"""Create the replication user. Unfortunately, to be able to
run pg_rewind, we need SUPERUSER, not just REPLICATION privilege
"""
pw = utils.generate_random_password()
operating_system.write_file(pwfile, pw, as_root=True)
operating_system.chown(pwfile, user=service.pgsql_owner, group=service.pgsql_owner, as_root=True)
operating_system.chmod(pwfile, FileMode.OCTAL_MODE("0600"), as_root=True)
repl_user = models.PostgreSQLUser(name=REPL_USER, password=pw)
admin._create_user(context=None, user=repl_user)
admin.alter_user(None, repl_user, True, "REPLICATION", "LOGIN")
return pw
def _read_log_position(self):
INFO_FILE = ('%s/xtrabackup_binlog_info' % get_datadir())
LOG.info(_("Setting read permissions on %s") % INFO_FILE)
operating_system.chmod(INFO_FILE, FileMode.ADD_READ_ALL, as_root=True)
LOG.info(_("Reading log position from %s") % INFO_FILE)
try:
with open(INFO_FILE, 'rb') as f:
row = csv.reader(f, delimiter='\t',
skipinitialspace=True).next()
return {
'log_file': row[0],
'log_position': int(row[1])
}
except (IOError, IndexError) as ex:
LOG.exception(ex)
raise self.UnableToDetermineBinlogPosition(
{'info_file': INFO_FILE})
def reset_root_password(self):
with tempfile.NamedTemporaryFile() as init_file:
operating_system.chmod(init_file.name, FileMode.ADD_READ_ALL,
as_root=True)
self._writelines_one_per_line(init_file,
self.RESET_ROOT_MYSQL_COMMANDS)
# Do not attempt to delete the file as the 'trove' user.
# The process writing into it may have assumed its ownership.
# Only owners can delete temporary
# files (restricted deletion).
err_log_file = tempfile.NamedTemporaryFile(
suffix=self._ERROR_LOG_SUFFIX,
delete=False)
try:
self._start_mysqld_safe_with_init_file(init_file, err_log_file)
finally:
err_log_file.close()
MySQLRestoreMixin._delete_file(err_log_file.name)
def change_permissions(self):
"""
When CouchDB is installed, a default user 'couchdb' is created.
Inorder to start/stop/restart CouchDB service as the current
OS user, add the current OS user to the 'couchdb' group and provide
read/write access to the 'couchdb' group.
"""
try:
LOG.debug("Changing permissions.")
for dir in [COUCHDB_LIB_DIR, COUCHDB_LOG_DIR,
COUCHDB_BIN_DIR, COUCHDB_CONFIG_DIR]:
operating_system.chown(dir, 'couchdb', 'couchdb', as_root=True)
operating_system.chmod(dir, FileMode.ADD_GRP_RW, as_root=True)
operating_system.change_user_group(getpass.getuser(), 'couchdb',
as_root=True)
LOG.debug("Successfully changed permissions.")
except exception.ProcessExecutionError:
LOG.exception(_("Error changing permissions."))
def _apply_access_rules(self):
LOG.debug("Applying database access rules.")
# Connections to all resources are granted.
#
# Local access from administrative users is implicitly trusted.
#
# Remote access from the Trove's account is always rejected as
# it is not needed and could be used by malicious users to hijack the
# instance.
#
# Connections from other accounts always require a double-MD5-hashed
# password.
#
# Make the rules readable only by the Postgres service.
#
# NOTE: The order of entries is important.
# The first failure to authenticate stops the lookup.
# That is why the 'local' connections validate first.
# The OrderedDict is necessary to guarantee the iteration order.
local_admins = ','.join([self.default_superuser_name, self.ADMIN_USER])
remote_admins = self.ADMIN_USER
access_rules = OrderedDict(
[('local', [['all', local_admins, None, 'trust'],
['replication', local_admins, None, 'trust'],
['all', 'all', None, 'md5']]),
('host', [['all', local_admins, '127.0.0.1/32', 'trust'],
['all', local_admins, '::1/128', 'trust'],
['all', local_admins, 'localhost', 'trust'],
['all', remote_admins, '0.0.0.0/0', 'reject'],
['all', remote_admins, '::/0', 'reject'],
['all', 'all', '0.0.0.0/0', 'md5'],
['all', 'all', '::/0', 'md5']])
])
operating_system.write_file(self.pgsql_hba_config, access_rules,
PropertiesCodec(
string_mappings={'\t': None}),
as_root=True)
operating_system.chown(self.pgsql_hba_config,
self.pgsql_owner, self.pgsql_owner,
as_root=True)
operating_system.chmod(self.pgsql_hba_config, FileMode.SET_USR_RO,
as_root=True)
def __init__(self, status):
self.state_change_wait_time = CONF.state_change_wait_time
self.status = status
revision_dir = \
guestagent_utils.build_file_path(
os.path.join(MOUNT_POINT,
os.path.dirname(system.VERTICA_ADMIN)),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
if not operating_system.exists(FAKE_CFG):
operating_system.write_file(FAKE_CFG, '', as_root=True)
operating_system.chown(FAKE_CFG, system.VERTICA_ADMIN,
system.VERTICA_ADMIN_GRP, as_root=True)
operating_system.chmod(FAKE_CFG, FileMode.ADD_GRP_RX_OTH_RX(),
as_root=True)
self.configuration_manager = \
ConfigurationManager(FAKE_CFG, system.VERTICA_ADMIN,
system.VERTICA_ADMIN_GRP,
PropertiesCodec(delimiter='='),
requires_root=True,
override_strategy=ImportOverrideStrategy(
revision_dir, "cnf"))
