def _parse_bucket_metadata(self, meta):
bucket_name = meta["name"]
bucket_type = meta["bucketType"]
if bucket_type == "membase":
bucket_type = "couchbase"
if meta["authType"] != "none":
bucket_password = meta["saslPassword"]
# SASL buckets can be only on this port.
bucket_port = "11211"
else:
bucket_password = None
bucket_port = meta["proxyPort"]
replica_count = meta["replicaNumber"]
bucket_ramsize_mb = int(dbaas.to_mb(meta["quota"]["ram"]))
enable_index_replica = 1 if meta["replicaIndex"] else 0
return guest_models.CouchbaseUser(
bucket_name,
password=bucket_password,
bucket_ramsize_mb=bucket_ramsize_mb,
bucket_replica_count=replica_count,
enable_index_replica=enable_index_replica,
bucket_eviction_policy=None,
bucket_priority=None,
bucket_port=bucket_port)
def post_restore(self):
try:
# Root enabled for the backup
pwd_file = system.COUCHBASE_DUMP_DIR + system.SECRET_KEY
if os.path.exists(pwd_file):
with open(pwd_file, "r") as f:
pw = f.read().rstrip("\n")
root = service.CouchbaseRootAccess()
root.set_password(pw)
# Get current root password
root = service.CouchbaseRootAccess()
root_pwd = root.get_password()
# Iterate through each bucket config
buckets_json = system.COUCHBASE_DUMP_DIR + system.BUCKETS_JSON
with open(buckets_json, "r") as f:
out = f.read()
if out == "[]":
# No buckets or data to restore. Done.
return
d = json.loads(out)
for i in range(len(d)):
bucket_name = d[i]["name"]
bucket_type = d[i]["bucketType"]
if bucket_type == "membase":
bucket_type = "couchbase"
ram = int(dbaas.to_mb(d[i]["quota"]["ram"]))
auth_type = d[i]["authType"]
password = d[i]["saslPassword"]
port = d[i]["proxyPort"]
replica_number = d[i]["replicaNumber"]
replica_index = 1 if d[i]["replicaIndex"] else 0
threads = d[i]["threadsNumber"]
flush = 1 if "flush" in d[i]["controllers"] else 0
# cbrestore requires you to manually create dest buckets
create_bucket_cmd = (
'curl -X POST -u root:' + root_pwd + ' -d name="' +
bucket_name + '"' + ' -d bucketType="' + bucket_type +
'"' + ' -d ramQuotaMB="' + str(ram) + '"' +
' -d authType="' + auth_type + '"' +
' -d saslPassword="******"' +
' -d proxyPort="' + str(port) + '"' +
' -d replicaNumber="' + str(replica_number) + '"' +
' -d replicaIndex="' + str(replica_index) + '"' +
' -d threadsNumber="' + str(threads) + '"' +
' -d flushEnabled="' + str(flush) + '" ' +
system.COUCHBASE_REST_API + '/pools/default/buckets')
utils.execute_with_timeout(create_bucket_cmd,
shell=True,
timeout=300)
if bucket_type == "memcached":
continue
# Wait for couchbase (membase) bucket creation to complete
# (follows same logic as --wait for couchbase-cli)
timeout_in_seconds = 120
start = time.time()
bucket_exist = False
while ((time.time() - start) <= timeout_in_seconds
and not bucket_exist):
url = (system.COUCHBASE_REST_API +
'/pools/default/buckets/')
outfile = system.COUCHBASE_DUMP_DIR + '/buckets.all'
utils.execute_with_timeout('curl -u root:' + root_pwd +
' ' + url + ' > ' + outfile,
shell=True,
timeout=300)
with open(outfile, "r") as file:
out = file.read()
buckets = json.loads(out)
for bucket in buckets:
if bucket["name"] == bucket_name:
bucket_exist = True
break
if not bucket_exist:
time.sleep(2)
if not bucket_exist:
raise base.RestoreError(
"Failed to create bucket '%s' "
"within %s seconds" %
(bucket_name, timeout_in_seconds))
# Query status
# (follows same logic as --wait for couchbase-cli)
healthy = False
while ((time.time() - start) <= timeout_in_seconds):
url = (system.COUCHBASE_REST_API +
'/pools/default/buckets/' + bucket_name)
outfile = system.COUCHBASE_DUMP_DIR + '/' + bucket_name
utils.execute_with_timeout('curl -u root:' + root_pwd +
' ' + url + ' > ' + outfile,
shell=True,
timeout=300)
all_node_ready = True
with open(outfile, "r") as file:
out = file.read()
bucket = json.loads(out)
for node in bucket["nodes"]:
if node["status"] != "healthy":
all_node_ready = False
break
if not all_node_ready:
time.sleep(2)
else:
healthy = True
break
if not healthy:
raise base.RestoreError(
"Bucket '%s' is created but "
"not ready to use within %s "
"seconds" % (bucket_name, timeout_in_seconds))
# Restore
restore_cmd = ('/opt/couchbase/bin/cbrestore ' +
system.COUCHBASE_DUMP_DIR + ' ' +
system.COUCHBASE_REST_API +
' --bucket-source=' + bucket_name +
' --bucket-destination=' + bucket_name +
' -u root' + ' -p ' + root_pwd)
try:
utils.execute_with_timeout(restore_cmd,
shell=True,
timeout=300)
except exception.ProcessExecutionError as p:
# cbrestore fails or hangs at times:
# http://www.couchbase.com/issues/browse/MB-10832
# Retrying typically works
LOG.error(p)
LOG.error("cbrestore failed. Retrying...")
utils.execute_with_timeout(restore_cmd,
shell=True,
timeout=300)
except exception.ProcessExecutionError as p:
LOG.error(p)
raise base.RestoreError("Couchbase restore failed.")
def post_restore(self):
try:
# Root enabled for the backup
pwd_file = system.COUCHBASE_DUMP_DIR + system.SECRET_KEY
if os.path.exists(pwd_file):
with open(pwd_file, "r") as f:
pw = f.read().rstrip("\n")
root = service.CouchbaseRootAccess()
root.set_password(pw)
# Get current root password
root = service.CouchbaseRootAccess()
root_pwd = root.get_password()
# Iterate through each bucket config
buckets_json = system.COUCHBASE_DUMP_DIR + system.BUCKETS_JSON
with open(buckets_json, "r") as f:
out = f.read()
if out == "[]":
# No buckets or data to restore. Done.
return
d = json.loads(out)
for i in range(len(d)):
bucket_name = d[i]["name"]
bucket_type = d[i]["bucketType"]
if bucket_type == "membase":
bucket_type = "couchbase"
ram = int(dbaas.to_mb(d[i]["quota"]["ram"]))
auth_type = d[i]["authType"]
password = d[i]["saslPassword"]
port = d[i]["proxyPort"]
replica_number = d[i]["replicaNumber"]
replica_index = 1 if d[i]["replicaIndex"] else 0
threads = d[i]["threadsNumber"]
flush = 1 if "flush" in d[i]["controllers"] else 0
# cbrestore requires you to manually create dest buckets
create_bucket_cmd = (
"curl -X POST -u root:"
+ root_pwd
+ ' -d name="'
+ bucket_name
+ '"'
+ ' -d bucketType="'
+ bucket_type
+ '"'
+ ' -d ramQuotaMB="'
+ str(ram)
+ '"'
+ ' -d authType="'
+ auth_type
+ '"'
+ ' -d saslPassword="******"'
+ ' -d proxyPort="'
+ str(port)
+ '"'
+ ' -d replicaNumber="'
+ str(replica_number)
+ '"'
+ ' -d replicaIndex="'
+ str(replica_index)
+ '"'
+ ' -d threadsNumber="'
+ str(threads)
+ '"'
+ ' -d flushEnabled="'
+ str(flush)
+ '" '
+ system.COUCHBASE_REST_API
+ "/pools/default/buckets"
)
utils.execute_with_timeout(create_bucket_cmd, shell=True, timeout=300)
if bucket_type == "memcached":
continue
# Wait for couchbase (membase) bucket creation to complete
# (follows same logic as --wait for couchbase-cli)
timeout_in_seconds = 120
start = time.time()
bucket_exist = False
while (time.time() - start) <= timeout_in_seconds and not bucket_exist:
url = system.COUCHBASE_REST_API + "/pools/default/buckets/"
outfile = system.COUCHBASE_DUMP_DIR + "/buckets.all"
utils.execute_with_timeout(
"curl -u root:" + root_pwd + " " + url + " > " + outfile, shell=True, timeout=300
)
with open(outfile, "r") as file:
out = file.read()
buckets = json.loads(out)
for bucket in buckets:
if bucket["name"] == bucket_name:
bucket_exist = True
break
if not bucket_exist:
time.sleep(2)
if not bucket_exist:
raise base.RestoreError(
"Failed to create bucket '%s' " "within %s seconds" % (bucket_name, timeout_in_seconds)
)
# Query status
# (follows same logic as --wait for couchbase-cli)
healthy = False
while (time.time() - start) <= timeout_in_seconds:
url = system.COUCHBASE_REST_API + "/pools/default/buckets/" + bucket_name
outfile = system.COUCHBASE_DUMP_DIR + "/" + bucket_name
utils.execute_with_timeout(
"curl -u root:" + root_pwd + " " + url + " > " + outfile, shell=True, timeout=300
)
all_node_ready = True
with open(outfile, "r") as file:
out = file.read()
bucket = json.loads(out)
for node in bucket["nodes"]:
if node["status"] != "healthy":
all_node_ready = False
break
if not all_node_ready:
time.sleep(2)
else:
healthy = True
break
if not healthy:
raise base.RestoreError(
"Bucket '%s' is created but "
"not ready to use within %s "
"seconds" % (bucket_name, timeout_in_seconds)
)
# Restore
restore_cmd = (
"/opt/couchbase/bin/cbrestore "
+ system.COUCHBASE_DUMP_DIR
+ " "
+ system.COUCHBASE_REST_API
+ " --bucket-source="
+ bucket_name
+ " --bucket-destination="
+ bucket_name
+ " -u root"
+ " -p "
+ root_pwd
)
try:
utils.execute_with_timeout(restore_cmd, shell=True, timeout=300)
except exception.ProcessExecutionError as p:
# cbrestore fails or hangs at times:
# http://www.couchbase.com/issues/browse/MB-10832
# Retrying typically works
LOG.error(p)
LOG.error("cbrestore failed. Retrying...")
utils.execute_with_timeout(restore_cmd, shell=True, timeout=300)
except exception.ProcessExecutionError as p:
LOG.error(p)
raise base.RestoreError("Couchbase restore failed.")