def _parse_bucket_metadata(self, meta): bucket_name = meta["name"] bucket_type = meta["bucketType"] if bucket_type == "membase": bucket_type = "couchbase" if meta["authType"] != "none": bucket_password = meta["saslPassword"] # SASL buckets can be only on this port. bucket_port = "11211" else: bucket_password = None bucket_port = meta["proxyPort"] replica_count = meta["replicaNumber"] bucket_ramsize_mb = int(dbaas.to_mb(meta["quota"]["ram"])) enable_index_replica = 1 if meta["replicaIndex"] else 0 return guest_models.CouchbaseUser( bucket_name, password=bucket_password, bucket_ramsize_mb=bucket_ramsize_mb, bucket_replica_count=replica_count, enable_index_replica=enable_index_replica, bucket_eviction_policy=None, bucket_priority=None, bucket_port=bucket_port)
def post_restore(self): try: # Root enabled for the backup pwd_file = system.COUCHBASE_DUMP_DIR + system.SECRET_KEY if os.path.exists(pwd_file): with open(pwd_file, "r") as f: pw = f.read().rstrip("\n") root = service.CouchbaseRootAccess() root.set_password(pw) # Get current root password root = service.CouchbaseRootAccess() root_pwd = root.get_password() # Iterate through each bucket config buckets_json = system.COUCHBASE_DUMP_DIR + system.BUCKETS_JSON with open(buckets_json, "r") as f: out = f.read() if out == "[]": # No buckets or data to restore. Done. return d = json.loads(out) for i in range(len(d)): bucket_name = d[i]["name"] bucket_type = d[i]["bucketType"] if bucket_type == "membase": bucket_type = "couchbase" ram = int(dbaas.to_mb(d[i]["quota"]["ram"])) auth_type = d[i]["authType"] password = d[i]["saslPassword"] port = d[i]["proxyPort"] replica_number = d[i]["replicaNumber"] replica_index = 1 if d[i]["replicaIndex"] else 0 threads = d[i]["threadsNumber"] flush = 1 if "flush" in d[i]["controllers"] else 0 # cbrestore requires you to manually create dest buckets create_bucket_cmd = ( 'curl -X POST -u root:' + root_pwd + ' -d name="' + bucket_name + '"' + ' -d bucketType="' + bucket_type + '"' + ' -d ramQuotaMB="' + str(ram) + '"' + ' -d authType="' + auth_type + '"' + ' -d saslPassword="******"' + ' -d proxyPort="' + str(port) + '"' + ' -d replicaNumber="' + str(replica_number) + '"' + ' -d replicaIndex="' + str(replica_index) + '"' + ' -d threadsNumber="' + str(threads) + '"' + ' -d flushEnabled="' + str(flush) + '" ' + system.COUCHBASE_REST_API + '/pools/default/buckets') utils.execute_with_timeout(create_bucket_cmd, shell=True, timeout=300) if bucket_type == "memcached": continue # Wait for couchbase (membase) bucket creation to complete # (follows same logic as --wait for couchbase-cli) timeout_in_seconds = 120 start = time.time() bucket_exist = False while ((time.time() - start) <= timeout_in_seconds and not bucket_exist): url = (system.COUCHBASE_REST_API + '/pools/default/buckets/') outfile = system.COUCHBASE_DUMP_DIR + '/buckets.all' utils.execute_with_timeout('curl -u root:' + root_pwd + ' ' + url + ' > ' + outfile, shell=True, timeout=300) with open(outfile, "r") as file: out = file.read() buckets = json.loads(out) for bucket in buckets: if bucket["name"] == bucket_name: bucket_exist = True break if not bucket_exist: time.sleep(2) if not bucket_exist: raise base.RestoreError( "Failed to create bucket '%s' " "within %s seconds" % (bucket_name, timeout_in_seconds)) # Query status # (follows same logic as --wait for couchbase-cli) healthy = False while ((time.time() - start) <= timeout_in_seconds): url = (system.COUCHBASE_REST_API + '/pools/default/buckets/' + bucket_name) outfile = system.COUCHBASE_DUMP_DIR + '/' + bucket_name utils.execute_with_timeout('curl -u root:' + root_pwd + ' ' + url + ' > ' + outfile, shell=True, timeout=300) all_node_ready = True with open(outfile, "r") as file: out = file.read() bucket = json.loads(out) for node in bucket["nodes"]: if node["status"] != "healthy": all_node_ready = False break if not all_node_ready: time.sleep(2) else: healthy = True break if not healthy: raise base.RestoreError( "Bucket '%s' is created but " "not ready to use within %s " "seconds" % (bucket_name, timeout_in_seconds)) # Restore restore_cmd = ('/opt/couchbase/bin/cbrestore ' + system.COUCHBASE_DUMP_DIR + ' ' + system.COUCHBASE_REST_API + ' --bucket-source=' + bucket_name + ' --bucket-destination=' + bucket_name + ' -u root' + ' -p ' + root_pwd) try: utils.execute_with_timeout(restore_cmd, shell=True, timeout=300) except exception.ProcessExecutionError as p: # cbrestore fails or hangs at times: # http://www.couchbase.com/issues/browse/MB-10832 # Retrying typically works LOG.error(p) LOG.error("cbrestore failed. Retrying...") utils.execute_with_timeout(restore_cmd, shell=True, timeout=300) except exception.ProcessExecutionError as p: LOG.error(p) raise base.RestoreError("Couchbase restore failed.")
def post_restore(self): try: # Root enabled for the backup pwd_file = system.COUCHBASE_DUMP_DIR + system.SECRET_KEY if os.path.exists(pwd_file): with open(pwd_file, "r") as f: pw = f.read().rstrip("\n") root = service.CouchbaseRootAccess() root.set_password(pw) # Get current root password root = service.CouchbaseRootAccess() root_pwd = root.get_password() # Iterate through each bucket config buckets_json = system.COUCHBASE_DUMP_DIR + system.BUCKETS_JSON with open(buckets_json, "r") as f: out = f.read() if out == "[]": # No buckets or data to restore. Done. return d = json.loads(out) for i in range(len(d)): bucket_name = d[i]["name"] bucket_type = d[i]["bucketType"] if bucket_type == "membase": bucket_type = "couchbase" ram = int(dbaas.to_mb(d[i]["quota"]["ram"])) auth_type = d[i]["authType"] password = d[i]["saslPassword"] port = d[i]["proxyPort"] replica_number = d[i]["replicaNumber"] replica_index = 1 if d[i]["replicaIndex"] else 0 threads = d[i]["threadsNumber"] flush = 1 if "flush" in d[i]["controllers"] else 0 # cbrestore requires you to manually create dest buckets create_bucket_cmd = ( "curl -X POST -u root:" + root_pwd + ' -d name="' + bucket_name + '"' + ' -d bucketType="' + bucket_type + '"' + ' -d ramQuotaMB="' + str(ram) + '"' + ' -d authType="' + auth_type + '"' + ' -d saslPassword="******"' + ' -d proxyPort="' + str(port) + '"' + ' -d replicaNumber="' + str(replica_number) + '"' + ' -d replicaIndex="' + str(replica_index) + '"' + ' -d threadsNumber="' + str(threads) + '"' + ' -d flushEnabled="' + str(flush) + '" ' + system.COUCHBASE_REST_API + "/pools/default/buckets" ) utils.execute_with_timeout(create_bucket_cmd, shell=True, timeout=300) if bucket_type == "memcached": continue # Wait for couchbase (membase) bucket creation to complete # (follows same logic as --wait for couchbase-cli) timeout_in_seconds = 120 start = time.time() bucket_exist = False while (time.time() - start) <= timeout_in_seconds and not bucket_exist: url = system.COUCHBASE_REST_API + "/pools/default/buckets/" outfile = system.COUCHBASE_DUMP_DIR + "/buckets.all" utils.execute_with_timeout( "curl -u root:" + root_pwd + " " + url + " > " + outfile, shell=True, timeout=300 ) with open(outfile, "r") as file: out = file.read() buckets = json.loads(out) for bucket in buckets: if bucket["name"] == bucket_name: bucket_exist = True break if not bucket_exist: time.sleep(2) if not bucket_exist: raise base.RestoreError( "Failed to create bucket '%s' " "within %s seconds" % (bucket_name, timeout_in_seconds) ) # Query status # (follows same logic as --wait for couchbase-cli) healthy = False while (time.time() - start) <= timeout_in_seconds: url = system.COUCHBASE_REST_API + "/pools/default/buckets/" + bucket_name outfile = system.COUCHBASE_DUMP_DIR + "/" + bucket_name utils.execute_with_timeout( "curl -u root:" + root_pwd + " " + url + " > " + outfile, shell=True, timeout=300 ) all_node_ready = True with open(outfile, "r") as file: out = file.read() bucket = json.loads(out) for node in bucket["nodes"]: if node["status"] != "healthy": all_node_ready = False break if not all_node_ready: time.sleep(2) else: healthy = True break if not healthy: raise base.RestoreError( "Bucket '%s' is created but " "not ready to use within %s " "seconds" % (bucket_name, timeout_in_seconds) ) # Restore restore_cmd = ( "/opt/couchbase/bin/cbrestore " + system.COUCHBASE_DUMP_DIR + " " + system.COUCHBASE_REST_API + " --bucket-source=" + bucket_name + " --bucket-destination=" + bucket_name + " -u root" + " -p " + root_pwd ) try: utils.execute_with_timeout(restore_cmd, shell=True, timeout=300) except exception.ProcessExecutionError as p: # cbrestore fails or hangs at times: # http://www.couchbase.com/issues/browse/MB-10832 # Retrying typically works LOG.error(p) LOG.error("cbrestore failed. Retrying...") utils.execute_with_timeout(restore_cmd, shell=True, timeout=300) except exception.ProcessExecutionError as p: LOG.error(p) raise base.RestoreError("Couchbase restore failed.")