def _spawn_with_init_file(self, temp_file):
child = pexpect.spawn("sudo mysqld_safe --init-file=%s" %
temp_file.name)
try:
i = child.expect(['Starting mysqld daemon'])
if i == 0:
LOG.info(_("Starting MySQL"))
except pexpect.TIMEOUT:
LOG.exception(_("Got a timeout launching mysqld_safe"))
finally:
# There is a race condition here where we kill mysqld before
# the init file been executed. We need to ensure mysqld is up.
self.poll_until_then_raise(
self.mysql_is_running,
base.RestoreError("Reset root password failed: "
"mysqld did not start!"))
LOG.info(_("Root password reset successfully."))
LOG.debug("Cleaning up the temp mysqld process.")
utils.execute_with_timeout("mysqladmin", "-uroot",
"--protocol=tcp", "shutdown")
utils.execute_with_timeout("killall", "mysqld_safe",
root_helper="sudo", run_as_root=True)
self.poll_until_then_raise(
self.mysql_is_not_running,
base.RestoreError("Reset root password failed: "
"mysqld did not stop!"))
def _spawn_with_init_file(self, temp_file):
child = pexpect.spawn("sudo mysqld_safe --init-file=%s" %
temp_file.name)
try:
i = child.expect(['Starting mysqld daemon'])
if i == 0:
LOG.info(_("Starting mysqld daemon"))
except pexpect.TIMEOUT:
LOG.exception(_("wait_and_close_proc failed"))
finally:
# There is a race condition here where we kill mysqld before
# the init file been executed. We need to ensure mysqld is up.
self.poll_until_then_raise(
self.mysql_is_running,
base.RestoreError("Reset root password failed: "
"mysqld did not start!"))
LOG.info(_("Root password reset successfully!"))
LOG.info(_("Cleaning up the temp mysqld process..."))
child.delayafterclose = 1
child.delayafterterminate = 1
child.close(force=True)
utils.execute_with_timeout("sudo", "killall", "mysqld")
self.poll_until_then_raise(
self.mysql_is_not_running,
base.RestoreError("Reset root password failed: "
"mysqld did not stop!"))
def _start_mysqld_safe_with_init_file(self, init_file, err_log_file):
# This directory is added and removed by the mysql systemd service
# as the database is started and stopped. The restore operation
# takes place when the database is stopped, so the directory does
# not exist, but it is assumed to exist by the mysqld_safe command
# which starts the database. This command used to create this
# directory if it didn't exist, but it was changed recently to
# simply fail in this case.
run_dir = "/var/run/mysqld"
if not os.path.exists(run_dir):
utils.execute("mkdir", run_dir,
run_as_root=True, root_helper="sudo")
utils.execute("chown", "mysql:mysql", run_dir, err_log_file.name,
run_as_root=True, root_helper="sudo")
child = pexpect.spawn(
"sudo mysqld_safe --init-file=%s --log-error=%s" %
(init_file.name, err_log_file.name))
try:
index = child.expect(['Starting mysqld daemon'])
if index == 0:
LOG.info("Starting MySQL")
except pexpect.TIMEOUT:
LOG.exception("Got a timeout launching mysqld_safe")
finally:
# There is a race condition here where we kill mysqld before
# the init file been executed. We need to ensure mysqld is up.
#
# mysqld_safe will start even if init-file statement(s) fail.
# We therefore also check for errors in the log file.
self.poll_until_then_raise(
self.mysql_is_running,
base.RestoreError("Reset root password failed:"
" mysqld did not start!"))
first_err_message = self._find_first_error_message(err_log_file)
if first_err_message:
raise base.RestoreError("Reset root password failed: %s"
% first_err_message)
LOG.info("Root password reset successfully.")
LOG.debug("Cleaning up the temp mysqld process.")
utils.execute_with_timeout("mysqladmin", "-uroot",
"--protocol=tcp", "shutdown")
LOG.debug("Polling for shutdown to complete.")
try:
utils.poll_until(self.mysql_is_not_running,
sleep_time=self.RESET_ROOT_SLEEP_INTERVAL,
time_out=self.RESET_ROOT_RETRY_TIMEOUT)
LOG.debug("Database successfully shutdown")
except exception.PollTimeOut:
LOG.debug("Timeout shutting down database "
"- performing killall on mysqld_safe.")
utils.execute_with_timeout("killall", "mysqld_safe",
root_helper="sudo",
run_as_root=True)
self.poll_until_then_raise(
self.mysql_is_not_running,
base.RestoreError("Reset root password failed: "
"mysqld did not stop!"))
def _start_mysqld_safe_with_init_file(self, init_file, err_log_file):
child = pexpect.spawn("sudo mysqld_safe"
" --skip-grant-tables"
" --skip-networking"
" --init-file='%s'"
" --log-error='%s'" %
(init_file.name, err_log_file.name))
try:
i = child.expect(['Starting mysqld daemon'])
if i == 0:
LOG.info(_("Starting MySQL"))
except pexpect.TIMEOUT:
LOG.exception(_("Got a timeout launching mysqld_safe"))
finally:
# There is a race condition here where we kill mysqld before
# the init file been executed. We need to ensure mysqld is up.
#
# mysqld_safe will start even if init-file statement(s) fail.
# We therefore also check for errors in the log file.
self.poll_until_then_raise(
self.mysql_is_running,
base.RestoreError("Reset root password failed:"
" mysqld did not start!"))
first_err_message = self._find_first_error_message(err_log_file)
if first_err_message:
raise base.RestoreError("Reset root password failed: %s" %
first_err_message)
LOG.info(_("Root password reset successfully."))
LOG.debug("Cleaning up the temp mysqld process.")
utils.execute_with_timeout("mysqladmin", "-uroot", "shutdown")
LOG.debug("Polling for shutdown to complete.")
try:
utils.poll_until(self.mysql_is_not_running,
sleep_time=self.RESET_ROOT_SLEEP_INTERVAL,
time_out=self.RESET_ROOT_RETRY_TIMEOUT)
LOG.debug("Database successfully shutdown")
except exception.PollTimeOut:
LOG.debug("Timeout shutting down database "
"- performing killall on mysqld_safe.")
utils.execute_with_timeout("killall",
"mysqld_safe",
root_helper="sudo",
run_as_root=True)
self.poll_until_then_raise(
self.mysql_is_not_running,
base.RestoreError("Reset root password failed: "
"mysqld did not stop!"))
def test_restore_failed_due_to_run_restore(self):
self.restore_runner.pre_restore = mock.Mock()
self.restore_runner._run_restore = mock.Mock(
side_effect=restoreBase.RestoreError('Error'))
self.restore_runner.post_restore = mock.Mock()
self.assertRaises(restoreBase.RestoreError,
self.restore_runner.restore)
def post_restore(self):
try:
# Root enabled for the backup
pwd_file = guestagent_utils.build_file_path(
system.COUCHBASE_DUMP_DIR, self.app.SECRET_KEY_FILE)
if os.path.exists(pwd_file):
with open(pwd_file, "r") as f:
pw = f.read().rstrip("\n")
self.app.reset_admin_credentials(password=pw)
# Iterate through each bucket config
buckets_json = system.COUCHBASE_DUMP_DIR + system.BUCKETS_JSON
with open(buckets_json, "r") as f:
out = f.read()
if out == "[]":
# No buckets or data to restore. Done.
return
d = json.loads(out)
for i in range(len(d)):
bucket_name = d[i]["name"]
bucket_type = d[i]["bucketType"]
if bucket_type == "membase":
bucket_type = "couchbase"
if d[i]["authType"] != "none":
bucket_password = d[i]["saslPassword"]
# SASL buckets can be only on this port.
bucket_port = "11211"
else:
bucket_password = None
bucket_port = d[i]["proxyPort"]
replica_count = d[i]["replicaNumber"]
enable_index_replica = 1 if d[i]["replicaIndex"] else 0
self._create_restore_bucket(bucket_name, bucket_password,
bucket_port, bucket_type,
enable_index_replica,
CONF.couchbase.eviction_policy,
replica_count)
self.run_cbrestore(bucket_name)
except exception.ProcessExecutionError as p:
LOG.error(p)
raise base.RestoreError("Couchbase restore failed.")
def post_restore(self):
try:
# Root enabled for the backup
pwd_file = system.COUCHBASE_DUMP_DIR + system.SECRET_KEY
if os.path.exists(pwd_file):
with open(pwd_file, "r") as f:
pw = f.read().rstrip("\n")
root = service.CouchbaseRootAccess()
root.set_password(pw)
# Get current root password
root = service.CouchbaseRootAccess()
root_pwd = root.get_password()
# Iterate through each bucket config
buckets_json = system.COUCHBASE_DUMP_DIR + system.BUCKETS_JSON
with open(buckets_json, "r") as f:
out = f.read()
if out == "[]":
# No buckets or data to restore. Done.
return
d = json.loads(out)
for i in range(len(d)):
bucket_name = d[i]["name"]
bucket_type = d[i]["bucketType"]
if bucket_type == "membase":
bucket_type = "couchbase"
ram = int(dbaas.to_mb(d[i]["quota"]["ram"]))
auth_type = d[i]["authType"]
password = d[i]["saslPassword"]
port = d[i]["proxyPort"]
replica_number = d[i]["replicaNumber"]
replica_index = 1 if d[i]["replicaIndex"] else 0
threads = d[i]["threadsNumber"]
flush = 1 if "flush" in d[i]["controllers"] else 0
# cbrestore requires you to manually create dest buckets
create_bucket_cmd = (
'curl -X POST -u root:' + root_pwd + ' -d name="' +
bucket_name + '"' + ' -d bucketType="' + bucket_type +
'"' + ' -d ramQuotaMB="' + str(ram) + '"' +
' -d authType="' + auth_type + '"' +
' -d saslPassword="******"' +
' -d proxyPort="' + str(port) + '"' +
' -d replicaNumber="' + str(replica_number) + '"' +
' -d replicaIndex="' + str(replica_index) + '"' +
' -d threadsNumber="' + str(threads) + '"' +
' -d flushEnabled="' + str(flush) + '" ' +
system.COUCHBASE_REST_API + '/pools/default/buckets')
utils.execute_with_timeout(create_bucket_cmd,
shell=True,
timeout=300)
if bucket_type == "memcached":
continue
# Wait for couchbase (membase) bucket creation to complete
# (follows same logic as --wait for couchbase-cli)
timeout_in_seconds = 120
start = time.time()
bucket_exist = False
while ((time.time() - start) <= timeout_in_seconds
and not bucket_exist):
url = (system.COUCHBASE_REST_API +
'/pools/default/buckets/')
outfile = system.COUCHBASE_DUMP_DIR + '/buckets.all'
utils.execute_with_timeout('curl -u root:' + root_pwd +
' ' + url + ' > ' + outfile,
shell=True,
timeout=300)
with open(outfile, "r") as file:
out = file.read()
buckets = json.loads(out)
for bucket in buckets:
if bucket["name"] == bucket_name:
bucket_exist = True
break
if not bucket_exist:
time.sleep(2)
if not bucket_exist:
raise base.RestoreError(
"Failed to create bucket '%s' "
"within %s seconds" %
(bucket_name, timeout_in_seconds))
# Query status
# (follows same logic as --wait for couchbase-cli)
healthy = False
while ((time.time() - start) <= timeout_in_seconds):
url = (system.COUCHBASE_REST_API +
'/pools/default/buckets/' + bucket_name)
outfile = system.COUCHBASE_DUMP_DIR + '/' + bucket_name
utils.execute_with_timeout('curl -u root:' + root_pwd +
' ' + url + ' > ' + outfile,
shell=True,
timeout=300)
all_node_ready = True
with open(outfile, "r") as file:
out = file.read()
bucket = json.loads(out)
for node in bucket["nodes"]:
if node["status"] != "healthy":
all_node_ready = False
break
if not all_node_ready:
time.sleep(2)
else:
healthy = True
break
if not healthy:
raise base.RestoreError(
"Bucket '%s' is created but "
"not ready to use within %s "
"seconds" % (bucket_name, timeout_in_seconds))
# Restore
restore_cmd = ('/opt/couchbase/bin/cbrestore ' +
system.COUCHBASE_DUMP_DIR + ' ' +
system.COUCHBASE_REST_API +
' --bucket-source=' + bucket_name +
' --bucket-destination=' + bucket_name +
' -u root' + ' -p ' + root_pwd)
try:
utils.execute_with_timeout(restore_cmd,
shell=True,
timeout=300)
except exception.ProcessExecutionError as p:
# cbrestore fails or hangs at times:
# http://www.couchbase.com/issues/browse/MB-10832
# Retrying typically works
LOG.error(p)
LOG.error("cbrestore failed. Retrying...")
utils.execute_with_timeout(restore_cmd,
shell=True,
timeout=300)
except exception.ProcessExecutionError as p:
LOG.error(p)
raise base.RestoreError("Couchbase restore failed.")
