def initial_setup(self):
self.ip_address = netutils.get_my_ipv4()
mount_point = CONF.couchbase.mount_point
try:
LOG.info(_('Couchbase Server change data dir path.'))
operating_system.chown(mount_point, 'couchbase', 'couchbase',
as_root=True)
pwd = CouchbaseRootAccess.get_password()
utils.execute_with_timeout(
(system.cmd_node_init
% {'data_path': mount_point,
'IP': self.ip_address,
'PWD': pwd}), shell=True)
operating_system.remove(system.INSTANCE_DATA_DIR, force=True,
as_root=True)
LOG.debug('Couchbase Server initialize cluster.')
utils.execute_with_timeout(
(system.cmd_cluster_init
% {'IP': self.ip_address, 'PWD': pwd}),
shell=True)
utils.execute_with_timeout(system.cmd_set_swappiness, shell=True)
utils.execute_with_timeout(system.cmd_update_sysctl_conf,
shell=True)
LOG.info(_('Couchbase Server initial setup finished.'))
except exception.ProcessExecutionError:
LOG.exception(_('Error performing initial Couchbase setup.'))
raise RuntimeError("Couchbase Server initial setup failed")
def recreate_wal_archive_dir():
operating_system.remove(WAL_ARCHIVE_DIR, force=True, recursive=True,
as_root=True)
operating_system.create_directory(WAL_ARCHIVE_DIR,
user=PgSqlProcess.PGSQL_OWNER,
group=PgSqlProcess.PGSQL_OWNER,
force=True, as_root=True)
def enable_as_master(self, service, master_config, for_failover=False):
"""For a server to be a master in postgres, we need to enable
the replication user in pg_hba and ensure that WAL logging is
at the appropriate level (use the same settings as backups)
"""
LOG.debug("Enabling as master, with cfg: %s " % master_config)
self._get_or_create_replication_user()
hba_entry = "host replication replicator 0.0.0.0/0 md5 \n"
# TODO(atomic77) Remove this hack after adding cfg manager for pg_hba
tmp_hba = '/tmp/pg_hba'
operating_system.copy(self.PGSQL_HBA_CONFIG, tmp_hba,
force=True, as_root=True)
operating_system.chmod(tmp_hba, FileMode.OCTAL_MODE("0777"),
as_root=True)
with open(tmp_hba, 'a+') as hba_file:
hba_file.write(hba_entry)
operating_system.copy(tmp_hba, self.PGSQL_HBA_CONFIG,
force=True, as_root=True)
operating_system.chmod(self.PGSQL_HBA_CONFIG,
FileMode.OCTAL_MODE("0600"),
as_root=True)
operating_system.remove(tmp_hba, as_root=True)
pgutil.psql("SELECT pg_reload_conf()")
def clear_storage(self):
mount_point = "/var/lib/mongodb/*"
LOG.debug("Clearing storage at %s." % mount_point)
try:
operating_system.remove(mount_point, force=True, as_root=True)
except exception.ProcessExecutionError:
LOG.exception(_("Error clearing storage."))
def clear_file(filename):
LOG.debug("Creating clean file %s" % filename)
if operating_system.file_discovery([filename]):
operating_system.remove(filename)
# force file creation by just opening it
open(filename, "wb")
operating_system.chmod(filename, operating_system.FileMode.SET_USR_RW, as_root=True)
def assert_module_retrieve(
self, client, instance_id, expected_count, expected_http_code=200, expected_results=None
):
try:
temp_dir = tempfile.mkdtemp()
prefix = "contents"
modretrieve_list = client.instances.module_retrieve(instance_id, directory=temp_dir, prefix=prefix)
self.assert_client_code(expected_http_code, client)
count = len(modretrieve_list)
self.assert_equal(expected_count, count, "Wrong number of modules from retrieve")
expected_results = expected_results or {}
for module_name, filename in modretrieve_list.items():
if module_name in expected_results:
expected = expected_results[module_name]
contents_name = "%s_%s_%s_%s" % (
prefix,
module_name,
expected["datastore"],
expected["datastore_version"],
)
expected_filename = guestagent_utils.build_file_path(temp_dir, contents_name, "dat")
self.assert_equal(expected_filename, filename, "Unexpected retrieve filename")
if "contents" in expected and expected["contents"]:
with open(filename, "rb") as fh:
contents = fh.read()
# convert contents into bytearray to work with py27
# and py34
contents = bytes([ord(item) for item in contents])
expected_contents = bytes([ord(item) for item in expected["contents"]])
self.assert_equal(expected_contents, contents, "Unexpected contents for %s" % module_name)
finally:
operating_system.remove(temp_dir)
def remove_last(self, num_revisions):
revision_files = self._collect_revisions()
deleted_files = []
if num_revisions > 0:
deleted_files = revision_files[-num_revisions:]
for path in deleted_files:
operating_system.remove(path, force=True, as_root=self._requires_root)
def reset_root_password(self):
"""Reset the password of the localhost root account used by Trove
for initial datastore configuration.
"""
with tempfile.NamedTemporaryFile(mode='w') as init_file:
operating_system.write_file(init_file.name,
self.RESET_ROOT_MYSQL_COMMANDS)
operating_system.chmod(init_file.name, FileMode.ADD_READ_ALL,
as_root=True)
# Do not attempt to delete the file as the 'trove' user.
# The process writing into it may have assumed its ownership.
# Only owners can delete temporary
# files (restricted deletion).
err_log_file = tempfile.NamedTemporaryFile(
suffix=self._ERROR_LOG_SUFFIX,
delete=False)
try:
# As of MySQL 5.7.6, for MySQL installation using an RPM
# distribution, server startup and shutdown is managed by
# systemd on several Linux platforms. On these platforms,
# mysqld_safe is no longer installed because it is
# unnecessary.
if self._mysqld_safe_cmd_exists():
self._start_mysqld_safe_with_init_file(
init_file, err_log_file)
else:
self._start_mysqld_with_init_file(init_file)
finally:
err_log_file.close()
operating_system.remove(
err_log_file.name, force=True, as_root=True)
def clear_expired_password():
"""
Some mysql installations generate random root password
and save it in /root/.mysql_secret, this password is
expired and should be changed by client that supports expired passwords.
"""
LOG.debug("Removing expired password.")
secret_file = "/root/.mysql_secret"
try:
out, err = utils.execute("cat", secret_file,
run_as_root=True, root_helper="sudo")
except exception.ProcessExecutionError:
LOG.exception(_("/root/.mysql_secret does not exist."))
return
m = re.match('# The random password set for the root user at .*: (.*)',
out)
if m:
try:
out, err = utils.execute("mysqladmin", "-p%s" % m.group(1),
"password", "", run_as_root=True,
root_helper="sudo")
except exception.ProcessExecutionError:
LOG.exception(_("Cannot change mysql password."))
return
operating_system.remove(secret_file, force=True, as_root=True)
LOG.debug("Expired password removed.")
def recreate_wal_archive_dir(cls):
wal_archive_dir = CONF.postgresql.wal_archive_location
operating_system.remove(wal_archive_dir, force=True, recursive=True,
as_root=True)
operating_system.create_directory(wal_archive_dir,
user=cls.PGSQL_OWNER,
group=cls.PGSQL_OWNER,
force=True, as_root=True)
def post_restore(self):
"""
Restore from the directory that we untarred into
"""
utils.execute_with_timeout("mongorestore", MONGO_DUMP_DIR,
timeout=LARGE_TIMEOUT)
operating_system.remove(MONGO_DUMP_DIR, force=True, as_root=True)
def recreate_wal_archive_dir(self):
wal_archive_dir = self.wal_archive_location
operating_system.remove(wal_archive_dir, force=True, recursive=True,
as_root=True)
operating_system.create_directory(wal_archive_dir,
user=self.pgsql_owner,
group=self.pgsql_owner,
force=True, as_root=True)
def _delete_revisions(self, num_revisions):
revision_files = self._collect_revisions()
deleted_files = []
if num_revisions > 0:
deleted_files = revision_files[-num_revisions:]
for path in deleted_files:
operating_system.remove(path, force=True, as_root=self._requires_root)
return [path for path in revision_files if path not in deleted_files]
def remove(self, group_name, change_id=None):
self._import_strategy.remove(group_name, change_id=change_id)
self._regenerate_base_configuration()
if not self._import_strategy.has_revisions:
# The base revision file is no longer needed if there are no
# overrides. It will be regenerated based on the current
# configuration file on the first 'apply()'.
operating_system.remove(self._base_revision_file, force=True,
as_root=self._requires_root)
def pre_restore(self):
self.stop_db(context=None)
PgBaseBackupUtil.recreate_wal_archive_dir()
datadir = self.pgsql_data_dir
operating_system.remove(datadir, force=True, recursive=True,
as_root=True)
operating_system.create_directory(datadir, user=self.PGSQL_OWNER,
group=self.PGSQL_OWNER, force=True,
as_root=True)
def _delete_file(self, file_path):
"""Force-remove a given file as root.
Do not raise an exception on failure.
"""
if os.path.isfile(file_path):
try:
operating_system.remove(file_path, force=True, as_root=True)
except Exception:
LOG.exception("Could not remove file: '%s'" % file_path)
def pre_restore(self):
self.stop_db(context=None)
LOG.info("Preparing WAL archive dir")
PgSqlProcess.recreate_wal_archive_dir()
datadir = self.pgsql_data_dir
operating_system.remove(datadir, force=True, recursive=True,
as_root=True)
operating_system.create_directory(datadir, user=self.PGSQL_OWNER,
group=self.PGSQL_OWNER, force=True,
as_root=True)
def post_restore(self):
"""
Restore from the directory that we untarred into
"""
params = self.app.admin_cmd_auth_params()
params.append(MONGO_DUMP_DIR)
utils.execute_with_timeout('mongorestore', *params,
timeout=LARGE_TIMEOUT)
operating_system.remove(MONGO_DUMP_DIR, force=True, as_root=True)
def pre_restore(self):
self.app.stop_db()
LOG.info("Preparing WAL archive dir")
self.app.recreate_wal_archive_dir()
datadir = self.app.pgsql_data_dir
operating_system.remove(datadir, force=True, recursive=True,
as_root=True)
operating_system.create_directory(datadir, user=self.app.pgsql_owner,
group=self.app.pgsql_owner,
force=True, as_root=True)
def save_files_pre_upgrade(self, mount_point):
LOG.debug('Saving files pre-upgrade.')
mnt_etc_dir = os.path.join(mount_point, 'save_etc')
if self.OS != operating_system.REDHAT:
# No need to store the config files away for Redhat because
# they are already stored in the data volume.
operating_system.remove(mnt_etc_dir, force=True, as_root=True)
operating_system.copy(self.pgsql_config_dir, mnt_etc_dir,
preserve=True, recursive=True, as_root=True)
return {'save_etc': mnt_etc_dir}
def restore_files_post_upgrade(self, upgrade_info):
LOG.debug('Restoring files post-upgrade.')
if self.OS != operating_system.REDHAT:
# No need to restore the config files for Redhat because
# they are already in the data volume.
operating_system.copy('%s/.' % upgrade_info['save_etc'],
self.pgsql_config_dir,
preserve=True, recursive=True,
force=True, as_root=True)
operating_system.remove(upgrade_info['save_etc'], force=True,
as_root=True)
def write_mongos_upstart(self):
upstart_contents = system.MONGOS_UPSTART_CONTENTS.format(config_file_placeholder=CONFIG_FILE)
LOG.info(_("Writing %s.") % system.TMP_MONGOS_UPSTART)
with open(system.TMP_MONGOS_UPSTART, "w") as t:
t.write(upstart_contents)
LOG.info(_("Moving %(a)s to %(b)s.") % {"a": system.TMP_MONGOS_UPSTART, "b": system.MONGOS_UPSTART})
operating_system.move(system.TMP_MONGOS_UPSTART, system.MONGOS_UPSTART, as_root=True)
operating_system.remove("/etc/init/mongodb.conf", force=True, as_root=True)
def save_files_pre_upgrade(self, mount_point):
LOG.debug('Saving files pre-upgrade.')
for save_dir, save_dir_info in self.upgrade_copy_info.items():
is_dir = save_dir_info['is_dir']
from_path = save_dir_info['path']
to_path = os.path.join(mount_point, save_dir)
operating_system.remove(to_path, force=True, as_root=True)
operating_system.copy(from_path, to_path,
preserve=True, recursive=is_dir,
as_root=True)
return {'copy_info': self.upgrade_copy_info}
def restore_files_post_upgrade(self, upgrade_info):
operating_system.copy('%s/.' % upgrade_info['save_confs'],
system.CONFIG_DIR,
preserve=True, recursive=True,
force=True, as_root=True)
operating_system.copy('%s/.' % upgrade_info['save_creds'],
os.path.expanduser('~'),
preserve=True, recursive=True,
force=True, as_root=True)
for save_dir in [upgrade_info['save_confs'],
upgrade_info['save_creds']]:
operating_system.remove(save_dir, force=True, as_root=True)
def restore_files_post_upgrade(self, upgrade_info):
LOG.debug('Restoring files post-upgrade.')
operating_system.copy('%s/.' % upgrade_info['save_opt_etc'],
self.couchbase_opt_etc_dir,
preserve=True, recursive=True,
force=True, as_root=True)
operating_system.copy(upgrade_info['save_etc'],
self.couchbase_conf_dir,
preserve=True, force=True, as_root=True)
for save_dir in [upgrade_info['save_opt_etc'],
upgrade_info['save_etc']]:
operating_system.remove(save_dir, force=True, as_root=True)
def save_files_pre_upgrade(self, mount_point):
LOG.debug('Saving files pre-upgrade.')
mnt_opt_etc = os.path.join(mount_point, 'save_opt_etc')
mnt_etc = os.path.join(mount_point, 'save_etc')
for save_dir in [mnt_opt_etc, mnt_etc]:
operating_system.remove(save_dir, force=True, as_root=True)
operating_system.copy(self.couchbase_opt_etc_dir, mnt_opt_etc,
preserve=True, recursive=True, as_root=True)
operating_system.copy(self.couchbase_conf_dir,
mnt_etc, preserve=True, recursive=True,
as_root=True)
return {'save_opt_etc': mnt_opt_etc,
'save_etc': mnt_etc}
def restore_files_post_upgrade(self, upgrade_info):
LOG.debug('Restoring files post-upgrade.')
mount_point = upgrade_info['mount_point']
upgrade_copy_info = upgrade_info['copy_info']
for save_dir, save_dir_info in upgrade_copy_info.items():
is_dir = save_dir_info['is_dir']
from_path = os.path.join(mount_point, save_dir)
files = os.path.join(from_path, '.') if is_dir else from_path
operating_system.copy(files, save_dir_info['path'],
preserve=True, recursive=is_dir,
force=True, as_root=True)
operating_system.remove(from_path, force=True, as_root=True)
self.status.set_ready()
def pre_restore(self):
self.app.stop_db()
LOG.info(_("Removing old persistence file: %s."),
self.restore_location)
operating_system.remove(self.restore_location, force=True,
as_root=True)
dir = os.path.dirname(self.restore_location)
operating_system.create_directory(dir, as_root=True)
operating_system.chmod(dir, FileMode.SET_FULL, as_root=True)
# IF AOF is set, we need to turn it off temporarily
if self.aof_set:
self.app.configuration_manager.apply_system_override(
self.aof_off_cfg, change_id=self.CONF_LABEL_AOF_TEMP_OFF)
def post_restore(self):
"""
Restore from the directory that we untarred into
"""
out, err = utils.execute_with_timeout(system.GET_DB_NAMES,
shell=True)
dbNames = out.split()
for dbName in dbNames:
service.run_command(system.RESTORE_DB % {'dbname': dbName,
'dir': DB2_BACKUP_DIR})
LOG.info(_("Cleaning out restore location post: %s."), DB2_BACKUP_DIR)
operating_system.remove(DB2_BACKUP_DIR, force=True, as_root=True)
def restore_files_post_upgrade(self, upgrade_info):
LOG.debug('Restoring files post-upgrade.')
if self.OS not in [operating_system.REDHAT, operating_system.ORACLE]:
# No need to restore the config files for Redhat because
# they are already in the data volume.
operating_system.copy('%s/.' % upgrade_info['save_etc'],
self.pgsql_config_dir,
preserve=True, recursive=True,
force=True, as_root=True)
operating_system.remove(upgrade_info['save_etc'], force=True,
as_root=True)
self.configuration_manager.refresh_cache()
self.status.set_ready()
def wipe_ib_logfiles(self):
"""Destroys the iblogfiles.
If for some reason the selected log size in the conf changes from the
current size of the files MySQL will fail to start, so we delete the
files to be safe.
"""
LOG.info("Wiping ib_logfiles.")
for index in range(2):
try:
# On restarts, sometimes these are wiped. So it can be a race
# to have MySQL start up before it's restarted and these have
# to be deleted. That's why its ok if they aren't found and
# that is why we use the "force" option to "remove".
operating_system.remove("%s/ib_logfile%d" %
(self.get_data_dir(), index),
force=True,
as_root=True)
except exception.ProcessExecutionError:
LOG.exception("Could not delete logfile.")
raise
def _incremental_restore(self, location, checksum):
"""Recursively apply backups from all parents.
If we are the parent then we restore to the restore_location and
we apply the logs to the restore_location only.
Otherwise if we are an incremental we restore to a subfolder to
prevent stomping on the full restore data. Then we run apply log
with the '--incremental-dir' flag
"""
metadata = self.storage.load_metadata(location, checksum)
incremental_dir = None
if 'parent_location' in metadata:
LOG.info(
_("Restoring parent: %(parent_location)s"
" checksum: %(parent_checksum)s.") % metadata)
parent_location = metadata['parent_location']
parent_checksum = metadata['parent_checksum']
# Restore parents recursively so backup are applied sequentially
self._incremental_restore(parent_location, parent_checksum)
# for *this* backup set the incremental_dir
# just use the checksum for the incremental path as it is
# sufficiently unique /var/lib/mysql/<checksum>
incremental_dir = os.path.join(
cfg.get_configuration_property('mount_point'), checksum)
operating_system.create_directory(incremental_dir, as_root=True)
command = self._incremental_restore_cmd(incremental_dir)
else:
# The parent (full backup) use the same command from InnobackupEx
# super class and do not set an incremental_dir.
command = self.restore_cmd
self.content_length += self._unpack(location, checksum, command)
self._incremental_prepare(incremental_dir)
# Delete unpacked incremental backup metadata
if incremental_dir:
operating_system.remove(incremental_dir, force=True, as_root=True)
def result(filename):
"""A generator representing the results of a query.
This generator produces result records of a query by iterating over a
CSV file created by the query. When the file is out of records it is
removed.
The purpose behind this abstraction is to provide a record set interface
with minimal memory consumption without requiring an active DB connection.
This makes it possible to iterate over any sized record set without
allocating memory for the entire record set and without using a DB cursor.
Each row is returned as an iterable of column values. The order of these
values is determined by the query.
"""
operating_system.chmod(filename, FileMode.SET_FULL, as_root=True)
with open(filename, 'r+') as file_handle:
for line in file_handle:
if line != "":
yield line.split(',')
operating_system.remove(filename, as_root=True)
raise StopIteration()
def reset_root_password(self):
"""Reset the password of the localhost root account used by Trove
for initial datastore configuration.
"""
with tempfile.NamedTemporaryFile(mode='w') as init_file:
operating_system.write_file(init_file.name,
self.RESET_ROOT_MYSQL_COMMANDS)
operating_system.chmod(init_file.name, FileMode.ADD_READ_ALL,
as_root=True)
# Do not attempt to delete the file as the 'trove' user.
# The process writing into it may have assumed its ownership.
# Only owners can delete temporary
# files (restricted deletion).
err_log_file = tempfile.NamedTemporaryFile(
suffix=self._ERROR_LOG_SUFFIX,
delete=False)
try:
self._start_mysqld_safe_with_init_file(init_file, err_log_file)
finally:
err_log_file.close()
operating_system.remove(
err_log_file.name, force=True, as_root=True)
def enable_as_master(self, service, master_config, for_failover=False):
"""For a server to be a master in postgres, we need to enable
the replication user in pg_hba and ensure that WAL logging is
at the appropriate level (use the same settings as backups)
"""
LOG.debug("Enabling as master, with cfg: %s " % master_config)
self._get_or_create_replication_user(service)
hba_entry = "host replication replicator 0.0.0.0/0 md5 \n"
tmp_hba = '/tmp/pg_hba'
operating_system.copy(service.pgsql_hba_config, tmp_hba,
force=True, as_root=True)
operating_system.chmod(tmp_hba, FileMode.SET_ALL_RWX(),
as_root=True)
with open(tmp_hba, 'a+') as hba_file:
hba_file.write(hba_entry)
operating_system.copy(tmp_hba, service.pgsql_hba_config,
force=True, as_root=True)
operating_system.chmod(service.pgsql_hba_config,
FileMode.SET_USR_RWX(),
as_root=True)
operating_system.remove(tmp_hba, as_root=True)
service.reload_configuration()
def post_restore(self):
self._create_oratab_entry()
operating_system.remove(ORA_BACKUP_PATH,
force=True,
as_root=True,
recursive=True)
def _remove_recovery_file(self, service):
operating_system.remove(service.pgsql_recovery_config, as_root=True)
def remove_wal_archive_dir(self):
wal_archive_dir = self.wal_archive_location
operating_system.remove(wal_archive_dir,
force=True,
recursive=True,
as_root=True)
def cleanup(self):
operating_system.remove(DB2_BACKUP_DIR, force=True, as_root=True)
def remove_overrides(self):
LOG.info(_("Removing overrides configuration file."))
if os.path.exists(MYCNF_OVERRIDES):
operating_system.remove(MYCNF_OVERRIDES, as_root=True)
def _remove_replication_overrides(self, cnf_file):
LOG.info(_("Removing replication configuration file."))
if os.path.exists(cnf_file):
operating_system.remove(cnf_file, as_root=True)
def pre_restore(self):
try:
operating_system.remove(system.COUCHBASE_DUMP_DIR, force=True)
except exception.ProcessExecutionError as p:
LOG.error(p)
raise p
def pre_restore(self):
try:
operating_system.remove(system.COUCHBASE_DUMP_DIR, force=True)
except exception.ProcessExecutionError:
LOG.exception("Error during pre-restore phase.")
raise
def remove_db2_dir(dir_name):
operating_system.remove(dir_name,
force=True,
as_root=True)
def _remove_recovery_file(self):
operating_system.remove(self.PGSQL_RECOVERY_CONFIG, as_root=True)
def cleanup(self):
operating_system.remove(MONGO_DUMP_DIR, force=True, as_root=True)
def cleanup(self):
operating_system.remove(BACKUP_DIR,
force=True,
as_root=True,
recursive=True)
def _cleanup_tmp_files():
operating_system.remove(ctlfile, force=True, as_root=True)
operating_system.remove(pfile, force=True, as_root=True)
operating_system.remove(datafile, force=True, as_root=True)
def remove_wal_archive_dir(cls):
wal_archive_dir = CONF.postgresql.wal_archive_location
operating_system.remove(wal_archive_dir, force=True, recursive=True,
as_root=True)
def cleanup(self):
operating_system.remove(self.app.paths.backup_dir,
force=True,
as_root=True,
recursive=True)
def clear_storage(self, mount_point):
LOG.debug("Clearing storage at %s." % mount_point)
try:
operating_system.remove(mount_point, force=True, as_root=True)
except exception.ProcessExecutionError:
LOG.exception(_("Error clearing storage."))
def post_restore(self):
self._create_oratab_entry()
operating_system.remove(self.app.paths.backup_dir,
force=True,
as_root=True,
recursive=True)
