def cluster_meet(self, ip, port):
try:
utils.execute_with_timeout('redis-cli', 'cluster', 'meet',
ip, port)
except exception.ProcessExecutionError:
LOG.exception(_('Error joining node to cluster at %s.'), ip)
raise
def _pre_restore(self):
LOG.debug("Cleaning out restore location: %s", self.restore_location)
utils.execute_with_timeout("sudo", "chmod", "-R",
"0777", self.restore_location)
utils.clean_out(self.restore_location)
app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())
app.stop_db()
def _get_actual_db_status(self):
global MYSQLD_ARGS
try:
out, err = utils.execute_with_timeout("/usr/bin/mysqladmin", "ping", run_as_root=True, root_helper="sudo")
LOG.info("Service Status is RUNNING.")
return rd_models.ServiceStatuses.RUNNING
except exception.ProcessExecutionError as e:
LOG.error("Process execution ")
try:
out, err = utils.execute_with_timeout("/bin/ps", "-C", "mysqld", "h")
pid = out.split()[0]
# TODO(rnirmal): Need to create new statuses for instances
# where the mysql service is up, but unresponsive
LOG.info("Service Status is BLOCKED.")
return rd_models.ServiceStatuses.BLOCKED
except exception.ProcessExecutionError as e:
if not MYSQLD_ARGS:
MYSQLD_ARGS = load_mysqld_options()
pid_file = MYSQLD_ARGS.get("pid_file", "/var/run/mysqld/mysqld.pid")
if os.path.exists(pid_file):
LOG.info("Service Status is CRASHED.")
return rd_models.ServiceStatuses.CRASHED
else:
LOG.info("Service Status is SHUTDOWN.")
return rd_models.ServiceStatuses.SHUTDOWN
def start_mysql(self, update_db=False):
LOG.info(_("Starting mysql..."))
# This is the site of all the trouble in the restart tests.
# Essentially what happens is that mysql start fails, but does not
# die. It is then impossible to kill the original, so
self._enable_mysql_on_boot()
try:
mysql_service = operating_system.service_discovery(
MYSQL_SERVICE_CANDIDATES)
utils.execute_with_timeout(mysql_service['cmd_start'], shell=True)
except KeyError:
raise RuntimeError("Service is not discovered.")
except exception.ProcessExecutionError:
# it seems mysql (percona, at least) might come back with [Fail]
# but actually come up ok. we're looking into the timing issue on
# parallel, but for now, we'd like to give it one more chance to
# come up. so regardless of the execute_with_timeout() response,
# we'll assume mysql comes up and check it's status for a while.
pass
if not self.status.wait_for_real_status_to_change_to(
rd_instance.ServiceStatuses.RUNNING,
self.state_change_wait_time, update_db):
LOG.error(_("Start up of MySQL failed!"))
# If it won't start, but won't die either, kill it by hand so we
# don't let a rouge process wander around.
try:
utils.execute_with_timeout("sudo", "pkill", "-9", "mysql")
except exception.ProcessExecutionError as p:
LOG.error("Error killing stalled mysql start command.")
LOG.error(p)
# There's nothing more we can do...
self.status.end_install_or_restart()
raise RuntimeError("Could not start MySQL!")
def clear_storage(self):
mount_point = "/var/lib/mongodb/*"
try:
cmd = "sudo rm -rf %s" % mount_point
utils.execute_with_timeout(cmd, shell=True)
except exception.ProcessExecutionError as e:
LOG.error(_("Process execution %s") % e)
def start_db(self, update_db=False):
LOG.info(_("Starting MongoDB"))
self._enable_db_on_boot()
try:
mongo_service = self._get_service()
utils.execute_with_timeout(mongo_service['cmd_start'],
shell=True)
except ProcessExecutionError:
pass
except KeyError:
raise RuntimeError("MongoDB service is not discovered.")
if not self.status.wait_for_real_status_to_change_to(
ds_instance.ServiceStatuses.RUNNING,
self.state_change_wait_time, update_db):
LOG.error(_("Start up of MongoDB failed"))
# If it won't start, but won't die either, kill it by hand so we
# don't let a rouge process wander around.
try:
out, err = utils.execute_with_timeout(
system.FIND_PID, shell=True)
pid = "".join(out.split(" ")[1:2])
utils.execute_with_timeout(
system.MONGODB_KILL % pid, shell=True)
except exception.ProcessExecutionError as p:
LOG.error("Error killing stalled MongoDB start command.")
LOG.error(p)
# There's nothing more we can do...
self.status.end_install_or_restart()
raise RuntimeError("Could not start MongoDB")
def _post_restore(self):
utils.execute_with_timeout("sudo", "chown", "-R", "-f",
"mysql", self.restore_location)
self._delete_old_binlogs()
self._reset_root_password()
app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())
app.start_mysql()
def _truncate_backup_chain(self):
"""Truncate all backups in the backup chain after the
specified parent backup."""
with LocalOracleClient(self.db_name, service=True) as client:
max_recid = sql_query.Query()
max_recid.columns = ["max(recid)"]
max_recid.tables = ["v$backup_piece"]
max_recid.where = ["handle like '%%%s%%'" % self.parent_id]
q = sql_query.Query()
q.columns = ["recid"]
q.tables = ["v$backup_piece"]
q.where = ["recid > (%s)" % str(max_recid)]
client.execute(str(q))
delete_list = [ str(row[0]) for row in client ]
if delete_list:
cmd = ("""\"\
rman target %(admin_user)s/%(admin_pswd)s@localhost/%(db_name)s <<EOF
run {
delete force noprompt backupset %(delete_list)s;
}
EXIT;
EOF\"
""" % {'admin_user': ADMIN_USER, 'admin_pswd': self.oracnf.admin_password,
'db_name': self.db_name, 'delete_list': ",".join(delete_list)})
utils.execute_with_timeout("su - oracle -c " + cmd,
run_as_root=True,
root_helper='sudo',
timeout=LARGE_TIMEOUT,
shell=True,
log_output_on_error=True)
def update_owner(self, path):
LOG.info(_("Set owner to 'mongodb' for %s ") % system.CONFIG)
utils.execute_with_timeout("chown", "-R", "mongodb", path,
run_as_root=True, root_helper="sudo")
LOG.info(_("Set group to 'mongodb' for %s ") % system.CONFIG)
utils.execute_with_timeout("chgrp", "-R", "mongodb", path,
run_as_root=True, root_helper="sudo")
def _get_actual_db_status(self):
try:
out, err = utils.execute_with_timeout(
"/usr/bin/mysqladmin",
"ping", run_as_root=True, root_helper="sudo",
log_output_on_error=True)
LOG.info(_("MySQL Service Status is RUNNING."))
return rd_instance.ServiceStatuses.RUNNING
except exception.ProcessExecutionError:
LOG.exception(_("Failed to get database status."))
try:
out, err = utils.execute_with_timeout("/bin/ps", "-C",
"mysqld", "h")
pid = out.split()[0]
# TODO(rnirmal): Need to create new statuses for instances
# where the mysql service is up, but unresponsive
LOG.info(_('MySQL Service Status %(pid)s is BLOCKED.') %
{'pid': pid})
return rd_instance.ServiceStatuses.BLOCKED
except exception.ProcessExecutionError:
LOG.exception(_("Process execution failed."))
mysql_args = load_mysqld_options()
pid_file = mysql_args.get('pid_file',
['/var/run/mysqld/mysqld.pid'])[0]
if os.path.exists(pid_file):
LOG.info(_("MySQL Service Status is CRASHED."))
return rd_instance.ServiceStatuses.CRASHED
else:
LOG.info(_("MySQL Service Status is SHUTDOWN."))
return rd_instance.ServiceStatuses.SHUTDOWN
def init_storage_structure(self, mount_point):
try:
cmd = system.INIT_FS % mount_point
utils.execute_with_timeout(cmd, shell=True)
except exception.ProcessExecutionError as e:
LOG.error(_("Error while initiating storage structure."))
LOG.error(e)
def start_db(self, update_db=False):
"""
Start the Couchbase Server.
"""
LOG.info(_("Starting Couchbase Server..."))
self._enable_db_on_boot()
try:
couchbase_service = operating_system.service_discovery(
system.SERVICE_CANDIDATES)
utils.execute_with_timeout(
couchbase_service['cmd_start'], shell=True)
except exception.ProcessExecutionError:
pass
except KeyError:
raise RuntimeError("Command to start Couchbase Server not found.")
if not self.status.wait_for_real_status_to_change_to(
rd_instance.ServiceStatuses.RUNNING,
self.state_change_wait_time, update_db):
LOG.error(_("Start up of Couchbase Server failed!"))
try:
utils.execute_with_timeout(system.cmd_kill)
except exception.ProcessExecutionError as p:
LOG.error('Error killing stalled Couchbase start command.')
LOG.error(p)
self.status.end_install_or_restart()
raise RuntimeError("Could not start Couchbase Server")
def _run_pre_backup(self):
try:
for cmd in self.pre_backup_commands:
utils.execute_with_timeout(*cmd)
root = service.CouchbaseRootAccess()
pw = root.get_password()
self._save_buckets_config(pw)
with open(OUTFILE, "r") as f:
out = f.read()
if out != "[]":
d = json.loads(out)
all_memcached = True
for i in range(len(d)):
bucket_type = d[i]["bucketType"]
if bucket_type != "memcached":
all_memcached = False
break
if not all_memcached:
self._backup(pw)
else:
LOG.info(_("All buckets are memcached. "
"Skipping backup."))
operating_system.move(OUTFILE, system.COUCHBASE_DUMP_DIR)
if pw != "password":
# Not default password, backup generated root password
operating_system.copy(system.pwd_file,
system.COUCHBASE_DUMP_DIR,
preserve=True, as_root=True)
except exception.ProcessExecutionError as p:
LOG.error(p)
raise p
def _spawn_with_init_file(self, temp_file):
child = pexpect.spawn("sudo mysqld_safe --init-file=%s" %
temp_file.name)
try:
i = child.expect(['Starting mysqld daemon'])
if i == 0:
LOG.info(_("Starting MySQL"))
except pexpect.TIMEOUT:
LOG.exception(_("Got a timeout launching mysqld_safe"))
finally:
# There is a race condition here where we kill mysqld before
# the init file been executed. We need to ensure mysqld is up.
self.poll_until_then_raise(
self.mysql_is_running,
base.RestoreError("Reset root password failed: "
"mysqld did not start!"))
LOG.info(_("Root password reset successfully."))
LOG.debug("Cleaning up the temp mysqld process.")
utils.execute_with_timeout("mysqladmin", "-uroot",
"--protocol=tcp", "shutdown")
utils.execute_with_timeout("killall", "mysqld_safe",
root_helper="sudo", run_as_root=True)
self.poll_until_then_raise(
self.mysql_is_not_running,
base.RestoreError("Reset root password failed: "
"mysqld did not stop!"))
def _disable_db_on_boot(self):
LOG.info(_("Disabling MongoDB on boot."))
try:
mongo_service = self._get_service()
utils.execute_with_timeout(mongo_service["cmd_disable"], shell=True)
except KeyError:
raise RuntimeError("MongoDB service is not discovered.")
def _execute_shell_cmd(cmd, options, *args, **kwargs):
"""Execute a given shell command passing it
given options (flags) and arguments.
Takes optional keyword arguments:
:param as_root: Execute as root.
:type as_root: boolean
:param timeout: Number of seconds if specified,
default if not.
There is no timeout if set to None.
:type timeout: integer
:raises: class:`UnknownArgumentError` if passed unknown args.
"""
exec_args = {}
if kwargs.pop('as_root', False):
exec_args['run_as_root'] = True
exec_args['root_helper'] = 'sudo'
if 'timeout' in kwargs:
exec_args['timeout'] = kwargs.pop('timeout')
if kwargs:
raise UnknownArgumentError(_("Got unknown keyword args: %r") % kwargs)
cmd_flags = _build_command_options(options)
cmd_args = cmd_flags + list(args)
utils.execute_with_timeout(cmd, *cmd_args, **exec_args)
def _reset_root_password(self):
#Create temp file with reset root password
with tempfile.NamedTemporaryFile() as fp:
fp.write(RESET_ROOT_MYSQL_COMMAND)
fp.flush()
utils.execute_with_timeout("sudo", "chmod", "a+r", fp.name)
self._spawn_with_init_file(fp)
def _run_post_backup(self):
try:
for cmd in self.post_backup_commands:
utils.execute_with_timeout(*cmd)
except exception.ProcessExecutionError:
LOG.exception(_("Error during post-backup phase."))
raise
def _disable_mysql_on_boot(self):
try:
utils.execute_with_timeout(self.mysql_service['cmd_disable'],
shell=True)
except KeyError:
LOG.exception(_("Error disabling MySQL start on boot."))
raise RuntimeError(_("Service is not discovered."))
def _write_mycnf(self, admin_password, config_contents, overrides=None):
"""
Install the set of mysql my.cnf templates.
Update the os_admin user and password to the my.cnf
file for direct login from localhost.
"""
LOG.info(_("Writing my.cnf templates."))
if admin_password is None:
admin_password = get_auth_password()
with open(TMP_MYCNF, 'w') as t:
t.write(config_contents)
utils.execute_with_timeout("sudo", "mv", TMP_MYCNF,
MYSQL_CONFIG)
self._write_temp_mycnf_with_admin_account(MYSQL_CONFIG,
TMP_MYCNF,
admin_password)
utils.execute_with_timeout("sudo", "mv", TMP_MYCNF,
MYSQL_CONFIG)
self.wipe_ib_logfiles()
# write configuration file overrides
if overrides:
self._write_config_overrides(overrides)
def pre_restore(self):
try:
utils.execute_with_timeout("rm -rf " + system.COUCHBASE_DUMP_DIR,
shell=True)
except exception.ProcessExecutionError as p:
LOG.error(p)
raise p
def create_user(self, users):
LOG.debug("Creating user(s) for accessing DB2 database(s).")
try:
for item in users:
user = models.DatastoreUser.deserialize(item)
user.check_create()
try:
LOG.debug("Creating OS user: %s." % user.name)
utils.execute_with_timeout(
system.CREATE_USER_COMMAND % {
'login': user.name, 'login': user.name,
'passwd': user.password}, shell=True)
except exception.ProcessExecutionError as pe:
LOG.exception(_("Error creating user: %s.") % user.name)
continue
for database in user.databases:
mydb = models.DatastoreSchema.deserialize(database)
try:
LOG.debug("Granting user: %s access to database: %s."
% (user.name, mydb.name))
run_command(system.GRANT_USER_ACCESS % {
'dbname': mydb.name, 'login': user.name})
except exception.ProcessExecutionError as pe:
LOG.debug(
"Error granting user: %s access to database: %s."
% (user.name, mydb.name))
LOG.debug(pe)
pass
except exception.ProcessExecutionError as pe:
LOG.exception(_("An error occurred creating users: %s.") %
pe.message)
pass
def delete_user(self, user):
LOG.debug("Delete a given user.")
db2_user = models.DatastoreUser.deserialize(user)
db2_user.check_delete()
userName = db2_user.name
user_dbs = db2_user.databases
LOG.debug("For user %s, databases to be deleted = %r." % (
userName, user_dbs))
if len(user_dbs) == 0:
databases = self.list_access(db2_user.name, None)
else:
databases = user_dbs
LOG.debug("databases for user = %r." % databases)
for database in databases:
mydb = models.DatastoreSchema.deserialize(database)
try:
run_command(system.REVOKE_USER_ACCESS % {
'dbname': mydb.name,
'login': userName})
LOG.debug("Revoked access for user:%s on database:%s." % (
userName, mydb.name))
except exception.ProcessExecutionError as pe:
LOG.debug("Error occurred while revoking access to %s." %
mydb.name)
pass
try:
utils.execute_with_timeout(system.DELETE_USER_COMMAND % {
'login': db2_user.name.lower()}, shell=True)
except exception.ProcessExecutionError as pe:
LOG.exception(_(
"There was an error while deleting user: %s.") % pe)
raise exception.GuestError(original_message=_(
"Unable to delete user: %s.") % userName)
def _perform_recover(self):
recover_cmd = ("""\"export ORACLE_SID=%(db_name)s
rman target %(admin_user)s/%(admin_pswd)s <<EOF
run {
recover database;
}
EXIT;
EOF\"
""" % {'admin_user': ADMIN_USER, 'admin_pswd': ADMIN_PSWD,
'db_name': self.db_name})
cmd = "su - oracle -c " + recover_cmd
try:
utils.execute_with_timeout(cmd,
run_as_root=True, root_helper='sudo',
timeout=LARGE_TIMEOUT,
shell=True, log_output_on_error=True)
except exception.ProcessExecutionError as p:
# Ignore the "media recovery requesting unknown archived log" error
# because RMAN would throw this error even when recovery is
# successful.
# If there are in fact errors when recovering the database, the
# database open step following will fail anyway.
if str(p).find('media recovery requesting unknown archived log') != -1:
pass
else:
raise(p)
def _disable_db_on_boot(self):
try:
LOG.debug("Disable CouchDB on boot.")
couchdb_service = operating_system.service_discovery(system.SERVICE_CANDIDATES)
utils.execute_with_timeout(couchdb_service["cmd_disable"], shell=True)
except KeyError:
raise RuntimeError("Command to disable CouchDB server on boot not found.")
def _run_post_backup(self):
try:
for cmd in self.post_backup_commands:
utils.execute_with_timeout(*cmd)
except exception.ProcessExecutionError as p:
LOG.error(p)
raise p
def _run_pre_backup(self):
"""Create archival contents in dump dir"""
try:
est_dump_size = self.estimate_dump_size()
avail = operating_system.get_bytes_free_on_fs(MONGODB_DBPATH)
if est_dump_size > avail:
self.cleanup()
# TODO(atomic77) Though we can fully recover from this error
# BackupRunner will leave the trove instance in a BACKUP state
raise OSError(_("Need more free space to run mongodump, "
"estimated %(est_dump_size)s"
" and found %(avail)s bytes free ") %
{'est_dump_size': est_dump_size,
'avail': avail})
operating_system.create_directory(MONGO_DUMP_DIR, as_root=True)
operating_system.chown(MONGO_DUMP_DIR, mongo_system.MONGO_USER,
mongo_system.MONGO_USER, as_root=True)
# high timeout here since mongodump can take a long time
utils.execute_with_timeout(
'mongodump', '--out', MONGO_DUMP_DIR,
*(self.app.admin_cmd_auth_params()),
run_as_root=True, root_helper='sudo',
timeout=LARGE_TIMEOUT
)
except exception.ProcessExecutionError as e:
LOG.debug("Caught exception when creating the dump")
self.cleanup()
raise e
def _spawn_with_init_file(self, temp_file):
child = pexpect.spawn("sudo mysqld_safe --init-file=%s" %
temp_file.name)
try:
i = child.expect(['Starting mysqld daemon'])
if i == 0:
LOG.info(_("Starting mysqld daemon"))
except pexpect.TIMEOUT:
LOG.exception(_("wait_and_close_proc failed"))
finally:
# There is a race condition here where we kill mysqld before
# the init file been executed. We need to ensure mysqld is up.
self.poll_until_then_raise(
self.mysql_is_running,
base.RestoreError("Reset root password failed: "
"mysqld did not start!"))
LOG.info(_("Root password reset successfully!"))
LOG.info(_("Cleaning up the temp mysqld process..."))
child.delayafterclose = 1
child.delayafterterminate = 1
child.close(force=True)
utils.execute_with_timeout("sudo", "killall", "mysqld")
self.poll_until_then_raise(
self.mysql_is_not_running,
base.RestoreError("Reset root password failed: "
"mysqld did not stop!"))
def set_db_to_listen(self, context):
"""Allow remote connections with encrypted passwords."""
# Using cat to read file due to read permissions issues.
out, err = utils.execute_with_timeout(
'sudo', 'cat',
PGSQL_HBA_CONFIG.format(
version=self._get_psql_version(),
),
timeout=30,
)
LOG.debug(
"{guest_id}: Writing hba file to /tmp/pgsql_hba_config.".format(
guest_id=CONF.guest_id,
)
)
with open('/tmp/pgsql_hba_config', 'w+') as config_file:
config_file.write(out)
config_file.write("host all all 0.0.0.0/0 md5\n")
utils.execute_with_timeout(
'sudo', 'chown', 'postgres', '/tmp/pgsql_hba_config',
timeout=30,
)
utils.execute_with_timeout(
'sudo', 'mv', '/tmp/pgsql_hba_config',
PGSQL_HBA_CONFIG.format(
version=self._get_psql_version(),
),
timeout=30,
)
def reset_configuration(self, context, configuration):
"""Reset the PgSql configuration file to the one given.
The configuration parameter is a string containing the full
configuration file that should be used.
"""
config_location = PGSQL_CONFIG.format(
version=self._get_psql_version(),
)
LOG.debug(
"{guest_id}: Writing configuration file to /tmp/pgsql_config."
.format(
guest_id=CONF.guest_id,
)
)
with open('/tmp/pgsql_config', 'w+') as config_file:
config_file.write(configuration)
utils.execute_with_timeout(
'sudo', 'chown', 'postgres', '/tmp/pgsql_config',
timeout=30,
)
utils.execute_with_timeout(
'sudo', 'mv', '/tmp/pgsql_config', config_location,
timeout=30,
)
def cleanup_stalled_db_services(self):
utils.execute_with_timeout(CassandraApp.CASSANDRA_KILL_CMD, shell=True)
def cleanup_stalled_db_services(self):
utils.execute_with_timeout(system.cmd_kill)
def delete_user(self, user):
LOG.debug("Delete a given CouchDB user.")
couchdb_user = models.CouchDBUser.deserialize(user)
db_names = self.list_database_names()
for db in db_names:
userlist = []
try:
out, err = utils.execute_with_timeout(
system.DB_ACCESS_COMMAND %
{'admin_name': self._admin_user().name,
'admin_password': self._admin_user().password,
'dbname': db},
shell=True)
except exception.ProcessExecutionError:
LOG.debug(
"Error while trying to get the users for database: %s.",
db)
continue
evalout = ast.literal_eval(out)
if evalout:
members = evalout['members']
names = members['names']
for i in range(0, len(names)):
couchdb_user.databases = db
userlist.append(names[i])
if couchdb_user.name in userlist:
userlist.remove(couchdb_user.name)
out2, err2 = utils.execute_with_timeout(
system.REVOKE_ACCESS_COMMAND % {
'admin_name': self._admin_user().name,
'admin_password': self._admin_user().password,
'dbname': db,
'username': userlist},
shell=True)
try:
out2, err = utils.execute_with_timeout(
system.DELETE_REV_ID %
{'admin_name': self._admin_user().name,
'admin_password': self._admin_user().password},
shell=True)
evalout2 = ast.literal_eval(out2)
rows = evalout2['rows']
userlist = []
for i in range(0, len(rows)):
row = rows[i]
username = "******" + couchdb_user.name
if row['key'] == username:
rev = row['value']
revid = rev['rev']
utils.execute_with_timeout(
system.DELETE_USER_COMMAND % {
'admin_name': self._admin_user().name,
'admin_password': self._admin_user().password,
'username': couchdb_user.name,
'revid': revid},
shell=True)
except exception.ProcessExecutionError as pe:
LOG.exception(_(
"There was an error while deleting user: %s."), pe)
raise exception.GuestError(original_message=_(
"Unable to delete user: %s.") % couchdb_user.name)
def prepare_slave(self, snapshot):
"""Prepare the environment needed for starting the slave Oracle
processes.
"""
master_info = snapshot['master']
db_name = master_info['db_name']
tmp_dir = '/tmp'
tmp_data_path = path.join(tmp_dir, 'oradata.tar.gz')
orabase_path = CONF.get(MANAGER).oracle_base
orahome_path = CONF.get(MANAGER).oracle_home
db_data_path = path.join(orabase_path, 'oradata', db_name)
fast_recovery_path = path.join(orabase_path, 'fast_recovery_area')
db_fast_recovery_path = path.join(fast_recovery_path, db_name)
audit_path = path.join(orabase_path, 'admin', db_name, 'adump')
admin_path = path.join(orabase_path, 'admin')
# Create necessary directories and set permissions
directories = [db_data_path, db_fast_recovery_path, audit_path]
for directory in directories:
operating_system.create_directory(directory,
system.ORACLE_INSTANCE_OWNER,
system.ORACLE_GROUP_OWNER,
as_root=True)
operating_system.chown(fast_recovery_path,
system.ORACLE_INSTANCE_OWNER,
system.ORACLE_GROUP_OWNER,
as_root=True)
operating_system.chown(admin_path,
system.ORACLE_INSTANCE_OWNER,
system.ORACLE_GROUP_OWNER,
as_root=True)
# Install on the slave files extracted from the master
# (e.g. the control, pfile, password, oracle.cnf file ... etc)
oradata = master_info['oradata']
operating_system.write_file(tmp_data_path,
oradata,
codec=stream_codecs.Base64Codec())
utils.execute_with_timeout('tar',
'-Pxzvf',
tmp_data_path,
run_as_root=True,
root_helper='sudo')
# Put the control file in place
tmp_ctlfile_path = path.join(tmp_dir, '%s_stby.ctl' % db_name)
ctlfile1_path = path.join(db_data_path, 'control01.ctl')
ctlfile2_path = path.join(db_fast_recovery_path, 'control02.ctl')
operating_system.move(tmp_ctlfile_path, ctlfile1_path, as_root=True)
operating_system.copy(ctlfile1_path,
ctlfile2_path,
preserve=True,
as_root=True)
db_unique_name = ('%(db_name)s_%(replica_label)s' % {
'db_name': db_name,
'replica_label': utils.generate_random_string(6)
})
# Customize the pfile for slave and put it in the right place.
# The pfile that came from master is owned by the 'oracle' user,
# so we need to change ownership first before editing it.
tmp_pfile_path = path.join(tmp_dir, 'init%s_stby.ora' % db_name)
pfile_path = path.join(orahome_path, 'dbs', 'init%s.ora' % db_name)
operating_system.chown(tmp_pfile_path,
getpass.getuser(),
None,
as_root=True)
with open(tmp_pfile_path, 'a') as pfile:
pfile.write("*.db_unique_name='%s'\n" % db_unique_name)
# Finished editing pfile, put it in the proper directory and chown
# back to oracle user and group
operating_system.move(tmp_pfile_path,
pfile_path,
force=True,
as_root=True)
operating_system.chown(pfile_path,
system.ORACLE_INSTANCE_OWNER,
system.ORACLE_GROUP_OWNER,
as_root=True)
self.ORA_CONF.db_name = db_name
self.ORA_CONF.db_unique_name = db_unique_name
# Set proper permissions on the oratab file
operating_system.chown('/etc/oratab',
system.ORACLE_INSTANCE_OWNER,
system.ORACLE_GROUP_OWNER,
as_root=True)
# Create the listener.ora file
self._create_lsnr_file()
# Restart the listener
utils.execute_with_timeout("sudo",
"su",
"-",
"oracle",
"-c",
"lsnrctl reload",
timeout=CONF.usage_timeout)
def cleanup_stalled_db_services(self):
pid, err = utils.execute_with_timeout(system.FIND_PID, shell=True)
utils.execute_with_timeout(system.MONGODB_KILL % pid, shell=True)
def post_restore(self):
try:
# Root enabled for the backup
pwd_file = system.COUCHBASE_DUMP_DIR + system.SECRET_KEY
if os.path.exists(pwd_file):
with open(pwd_file, "r") as f:
pw = f.read().rstrip("\n")
root = service.CouchbaseRootAccess()
root.set_password(pw)
# Get current root password
root = service.CouchbaseRootAccess()
root_pwd = root.get_password()
# Iterate through each bucket config
buckets_json = system.COUCHBASE_DUMP_DIR + system.BUCKETS_JSON
with open(buckets_json, "r") as f:
out = f.read()
if out == "[]":
# No buckets or data to restore. Done.
return
d = json.loads(out)
for i in range(len(d)):
bucket_name = d[i]["name"]
bucket_type = d[i]["bucketType"]
if bucket_type == "membase":
bucket_type = "couchbase"
ram = int(dbaas.to_mb(d[i]["quota"]["ram"]))
auth_type = d[i]["authType"]
password = d[i]["saslPassword"]
port = d[i]["proxyPort"]
replica_number = d[i]["replicaNumber"]
replica_index = 1 if d[i]["replicaIndex"] else 0
threads = d[i]["threadsNumber"]
flush = 1 if "flush" in d[i]["controllers"] else 0
# cbrestore requires you to manually create dest buckets
create_bucket_cmd = (
'curl -X POST -u root:' + root_pwd + ' -d name="' +
bucket_name + '"' + ' -d bucketType="' + bucket_type +
'"' + ' -d ramQuotaMB="' + str(ram) + '"' +
' -d authType="' + auth_type + '"' +
' -d saslPassword="******"' +
' -d proxyPort="' + str(port) + '"' +
' -d replicaNumber="' + str(replica_number) + '"' +
' -d replicaIndex="' + str(replica_index) + '"' +
' -d threadsNumber="' + str(threads) + '"' +
' -d flushEnabled="' + str(flush) + '" ' +
system.COUCHBASE_REST_API + '/pools/default/buckets')
utils.execute_with_timeout(create_bucket_cmd,
shell=True,
timeout=300)
if bucket_type == "memcached":
continue
# Wait for couchbase (membase) bucket creation to complete
# (follows same logic as --wait for couchbase-cli)
timeout_in_seconds = 120
start = time.time()
bucket_exist = False
while ((time.time() - start) <= timeout_in_seconds
and not bucket_exist):
url = (system.COUCHBASE_REST_API +
'/pools/default/buckets/')
outfile = system.COUCHBASE_DUMP_DIR + '/buckets.all'
utils.execute_with_timeout('curl -u root:' + root_pwd +
' ' + url + ' > ' + outfile,
shell=True,
timeout=300)
with open(outfile, "r") as file:
out = file.read()
buckets = json.loads(out)
for bucket in buckets:
if bucket["name"] == bucket_name:
bucket_exist = True
break
if not bucket_exist:
time.sleep(2)
if not bucket_exist:
raise base.RestoreError(
"Failed to create bucket '%s' "
"within %s seconds" %
(bucket_name, timeout_in_seconds))
# Query status
# (follows same logic as --wait for couchbase-cli)
healthy = False
while ((time.time() - start) <= timeout_in_seconds):
url = (system.COUCHBASE_REST_API +
'/pools/default/buckets/' + bucket_name)
outfile = system.COUCHBASE_DUMP_DIR + '/' + bucket_name
utils.execute_with_timeout('curl -u root:' + root_pwd +
' ' + url + ' > ' + outfile,
shell=True,
timeout=300)
all_node_ready = True
with open(outfile, "r") as file:
out = file.read()
bucket = json.loads(out)
for node in bucket["nodes"]:
if node["status"] != "healthy":
all_node_ready = False
break
if not all_node_ready:
time.sleep(2)
else:
healthy = True
break
if not healthy:
raise base.RestoreError(
"Bucket '%s' is created but "
"not ready to use within %s "
"seconds" % (bucket_name, timeout_in_seconds))
# Restore
restore_cmd = ('/opt/couchbase/bin/cbrestore ' +
system.COUCHBASE_DUMP_DIR + ' ' +
system.COUCHBASE_REST_API +
' --bucket-source=' + bucket_name +
' --bucket-destination=' + bucket_name +
' -u root' + ' -p ' + root_pwd)
try:
utils.execute_with_timeout(restore_cmd,
shell=True,
timeout=300)
except exception.ProcessExecutionError as p:
# cbrestore fails or hangs at times:
# http://www.couchbase.com/issues/browse/MB-10832
# Retrying typically works
LOG.error(p)
LOG.error("cbrestore failed. Retrying...")
utils.execute_with_timeout(restore_cmd,
shell=True,
timeout=300)
except exception.ProcessExecutionError as p:
LOG.error(p)
raise base.RestoreError("Couchbase restore failed.")
def cleanup_stalled_db_services(self):
out, err = utils.execute_with_timeout(system.FIND_PID, shell=True)
pid = "".join(out.split(" ")[1:2])
utils.execute_with_timeout(system.MONGODB_KILL % pid, shell=True)
def init_storage_structure(self, mount_point):
try:
cmd = system.INIT_FS % mount_point
utils.execute_with_timeout(cmd, shell=True)
except exception.ProcessExecutionError:
LOG.exception(_("Error while initiating storage structure."))
def execute(*command, **kwargs):
"""Execute a command as the 'postgres' user."""
LOG.debug('Running as postgres: {0}'.format(command))
return utils.execute_with_timeout("sudo", "-u", "postgres", *command,
**kwargs)
def _create_mysql_confd_dir(self):
conf_dir = "/etc/mysql/conf.d"
LOG.debug("Creating %s." % conf_dir)
command = "sudo mkdir -p %s" % conf_dir
utils.execute_with_timeout(command, shell=True)
def _backup(self, password):
utils.execute_with_timeout('/opt/couchbase/bin/cbbackup ' +
system.COUCHBASE_REST_API + ' ' +
system.COUCHBASE_DUMP_DIR +
' -u root -p ' + password,
shell=True, timeout=600)
def cleanup_stalled_db_services(self):
utils.execute_with_timeout('pkill',
'-9',
'redis-server',
run_as_root=True,
root_helper='sudo')
def _remove_overrides(self):
LOG.info(_("Removing overrides configuration file."))
if os.path.exists(MYCNF_OVERRIDES):
utils.execute_with_timeout("sudo", "rm", MYCNF_OVERRIDES)
def execute_change_permission_commands(self, chng_perm_cmd):
out, err = utils.execute_with_timeout(chng_perm_cmd, shell=True)
if err:
raise exception.ProcessExecutionError(cmd=chng_perm_cmd,
stderr=err,
stdout=out)
def remove_replication_overrides(self):
LOG.info(_("Removing replication configuration file."))
if os.path.exists(MYCNF_REPLMASTER):
utils.execute_with_timeout("sudo", "rm", MYCNF_REPLMASTER)
def _disable_db_on_boot(self):
utils.execute_with_timeout(system.DISABLE_CASSANDRA_ON_BOOT,
shell=True)
def _start_mysqld_safe_with_init_file(self, init_file, err_log_file):
# This directory is added and removed by the mysql systemd service
# as the database is started and stopped. The restore operation
# takes place when the database is stopped, so the directory does
# not exist, but it is assumed to exist by the mysqld_safe command
# which starts the database. This command used to create this
# directory if it didn't exist, but it was changed recently to
# simply fail in this case.
run_dir = "/var/run/mysqld"
if not os.path.exists(run_dir):
utils.execute("mkdir",
run_dir,
run_as_root=True,
root_helper="sudo")
utils.execute("chown",
"mysql:mysql",
run_dir,
err_log_file.name,
init_file.name,
run_as_root=True,
root_helper="sudo")
command_mysql_safe = ("sudo mysqld_safe"
" --init-file=%s"
" --log-error=%s" %
(init_file.name, err_log_file.name))
LOG.debug("Spawning: %s" % command_mysql_safe)
child = pexpect.spawn(command_mysql_safe)
try:
index = child.expect(['Starting mysqld daemon'])
if index == 0:
LOG.info("Starting MySQL")
except pexpect.TIMEOUT:
LOG.exception("Got a timeout launching mysqld_safe")
finally:
# There is a race condition here where we kill mysqld before
# the init file been executed. We need to ensure mysqld is up.
#
# mysqld_safe will start even if init-file statement(s) fail.
# We therefore also check for errors in the log file.
self.poll_until_then_raise(
self.mysql_is_running,
base.RestoreError("Reset root password failed:"
" mysqld did not start!"))
first_err_message = self._find_first_error_message(err_log_file)
if first_err_message:
raise base.RestoreError("Reset root password failed: %s" %
first_err_message)
LOG.info("Root password reset successfully.")
LOG.debug("Cleaning up the temp mysqld process.")
utils.execute_with_timeout("mysqladmin", "-uroot",
"--protocol=tcp", "shutdown")
LOG.debug("Polling for shutdown to complete.")
try:
utils.poll_until(self.mysql_is_not_running,
sleep_time=self.RESET_ROOT_SLEEP_INTERVAL,
time_out=self.RESET_ROOT_RETRY_TIMEOUT)
LOG.debug("Database successfully shutdown")
except exception.PollTimeOut:
LOG.debug("Timeout shutting down database "
"- performing killall on mysqld_safe.")
utils.execute_with_timeout("killall",
"mysqld_safe",
root_helper="sudo",
run_as_root=True)
self.poll_until_then_raise(
self.mysql_is_not_running,
base.RestoreError("Reset root password failed: "
"mysqld did not stop!"))
def _save_buckets_config(self, password):
url = system.COUCHBASE_REST_API + '/pools/default/buckets'
utils.execute_with_timeout('curl -u root:' + password + ' ' + url +
' > ' + OUTFILE,
shell=True,
timeout=300)
def cleanup_stalled_db_services(self):
utils.execute_with_timeout(system.CASSANDRA_KILL, shell=True)
def _backup(self, password):
utils.execute_with_timeout(['/opt/couchbase/bin/cbbackup',
system.COUCHBASE_REST_API,
system.COUCHBASE_DUMP_DIR,
'-u', 'root', '-p', password],
timeout=600)
def initial_setup(self):
self.ip_address = operating_system.get_ip_address()
mount_point = CONF.get('couchbase').mount_point
try:
LOG.info(_('Couchbase Server change data dir path'))
utils.execute_with_timeout(system.cmd_own_data_dir, shell=True)
utils.execute_with_timeout((system.cmd_node_init % {
'data_path': mount_point,
'IP': self.ip_address
}),
shell=True)
utils.execute_with_timeout(system.cmd_rm_old_data_dir, shell=True)
LOG.info(_('Couchbase Server initialize cluster'))
utils.execute_with_timeout((system.cmd_cluster_init % {
'IP': self.ip_address
}),
shell=True)
utils.execute_with_timeout(system.cmd_set_swappiness, shell=True)
utils.execute_with_timeout(system.cmd_update_sysctl_conf,
shell=True)
LOG.info(_('Couchbase Server initial setup finished'))
except exception.ProcessExecutionError as e:
LOG.error(_('Process execution error %s') % e)
raise RuntimeError("Couchbase Server initial setup failed")
def prepare_slave(self, service, snapshot):
"""Prepare the environment needed for starting the slave Oracle
processes.
"""
master_info = snapshot['master']
db_name = master_info['db_name']
db_unique_name = ('%(db_name)s_%(replica_label)s' % {
'db_name': db_name,
'replica_label': utils.generate_random_string(6)
})
service.paths.update_db_name(db_name)
# Create necessary directories and set necessary permissions
new_dirs = [
service.paths.db_data_dir, service.paths.db_fast_recovery_logs_dir,
service.paths.db_fast_recovery_dir, service.paths.audit_dir
]
for directory in new_dirs:
operating_system.create_directory(directory,
service.instance_owner,
service.instance_owner_group,
as_root=True)
chown_dirs = [
service.paths.fast_recovery_area, service.paths.admin_dir
]
for directory in chown_dirs:
operating_system.chown(directory,
service.instance_owner,
service.instance_owner_group,
as_root=True)
# Install on the slave files extracted from the master
# (e.g. the control, pfile, password, oracle.cnf file ... etc)
oradata_encoded = master_info['oradata']
tmp_data_path = path.join(TMP_DIR, 'oradata.tar.gz')
operating_system.write_file(tmp_data_path,
oradata_encoded,
codec=stream_codecs.Base64Codec(),
encode=False)
utils.execute_with_timeout('tar',
'-Pxzvf',
tmp_data_path,
run_as_root=True,
root_helper='sudo')
# Put the control file in place
tmp_ctlfile_path = path.join(TMP_DIR, '%s_stby.ctl' % db_name)
operating_system.move(tmp_ctlfile_path,
service.paths.ctlfile1_file,
as_root=True)
operating_system.copy(service.paths.ctlfile1_file,
service.paths.ctlfile2_file,
preserve=True,
as_root=True)
# Set the db_name and db_unique_name via the PFILE which will be
# removed later
operating_system.write_file(service.paths.pfile,
"*.db_unique_name='%s'\n"
"*.db_name='%s'\n" %
(db_unique_name, db_name),
as_root=True)
operating_system.chown(service.paths.pfile,
service.instance_owner,
service.instance_owner_group,
as_root=True,
force=True)
service.admin.delete_conf_cache()
service.admin.ora_config.db_name = db_name
service.admin.ora_config.db_unique_name = db_unique_name
# Set proper permissions on the oratab file
operating_system.chown(service.paths.oratab_file,
service.instance_owner,
service.instance_owner_group,
as_root=True,
force=True)
# Create the listener.ora file and restart
service.configure_listener()
def run_command(command, superuser=system.DB2_INSTANCE_OWNER,
timeout=system.TIMEOUT):
return utils.execute_with_timeout("sudo", "su", "-", superuser, "-c",
command, timeout=timeout)
def _get_if_addr_pairs(self):
stdout, stderr = utils.execute_with_timeout('ip', '-4', '-o', 'a')
return [
(words[1], words[3])
for words in [line.split() for line in stdout.strip().split('\n')]
]
def cleanup_stalled_db_services(self):
utils.execute_with_timeout(CouchbaseApp.COUCHBASE_KILL_CMD,
run_as_root=True,
root_helper='sudo')
