def move_bucket(self, bucket_name, direction):
logger.debug("Rename folder")
if direction == 'save':
src = join(self.virtual_source.parameters.mount_path, 'data',
bucket_name)
dst = join(self.virtual_source.parameters.mount_path, 'data',
".{}.delphix".format(bucket_name))
command = CommandFactory.os_mv(src, dst, self.need_sudo, self.uid)
logger.debug("rename command: {}".format(command))
stdout, error, exit_code = utilities.execute_bash(
self.connection, command)
elif direction == 'restore':
dst = join(self.virtual_source.parameters.mount_path, 'data',
bucket_name)
src = join(self.virtual_source.parameters.mount_path, 'data',
".{}.delphix".format(bucket_name))
command = CommandFactory.delete_dir(dst, self.need_sudo, self.uid)
logger.debug("delete command: {}".format(command))
stdout, error, exit_code = utilities.execute_bash(
self.connection, command)
command = CommandFactory.os_mv(src, dst, self.need_sudo, self.uid)
logger.debug("rename command: {}".format(command))
stdout, error, exit_code = utilities.execute_bash(
self.connection, command)
def make_directory(self, directory_path):
"""
Create a directory and set the permission level 775
:param directory_path: The directory path
:return: None
"""
logger.debug("Creating Directory {} ".format(directory_path))
env = {'directory_path': directory_path}
command = CommandFactory.make_directory(directory_path)
utilities.execute_bash(self.connection, command)
logger.debug("Changing permission of directory path {}".format(directory_path))
command = CommandFactory.change_permission(directory_path)
utilities.execute_bash(self.connection, command)
logger.debug("Changed the permission of directory")
def delete_config(self):
# TODO:
# error handling
logger.debug("start delete_config")
filename = "{}/../var/lib/couchbase/config/config.dat".format(
helper_lib.get_base_directory_of_given_path(
self.repository.cb_shell_path))
cmd = CommandFactory.check_file(filename, self.need_sudo, self.uid)
logger.debug("check file cmd: {}".format(cmd))
command_output, std_err, exit_code = utilities.execute_bash(
self.connection, command_name=cmd, callback_func=self.ignore_err)
if exit_code == 0 and "Found" in command_output:
cmd = CommandFactory.os_mv(filename, "{}.bak".format(filename),
self.need_sudo, self.uid)
logger.debug("rename config cmd: {}".format(cmd))
command_output, std_err, exit_code = utilities.execute_bash(
self.connection, command_name=cmd)
logger.debug(
"rename config.dat to bak - exit_code: {} stdout: {} std_err: {}"
.format(exit_code, command_output, std_err))
filename = "{}/../etc/couchdb/local.ini".format(
helper_lib.get_base_directory_of_given_path(
self.repository.cb_shell_path))
command_output, std_err, exit_code = self.run_os_command(
os_command='sed', filename=filename, regex='s/view_index_dir.*//')
logger.debug(
"clean local.ini index - exit_code: {} stdout: {} std_err: {}".
format(exit_code, command_output, std_err))
command_output, std_err, exit_code = self.run_os_command(
os_command='sed', filename=filename, regex='s/database_dir.*//')
logger.debug(
"clean local.ini data - exit_code: {} stdout: {} std_err: {}".
format(exit_code, command_output, std_err))
command_output, std_err, exit_code = self.run_os_command(
os_command='change_permission', path=filename)
logger.debug(
"fix local.ini permission - exit_code: {} stdout: {} std_err: {}".
format(exit_code, command_output, std_err))
def get_replication_uuid(self):
# False for string
logger.debug("Finding the replication uuid through host name")
is_ip_or_string = False
kwargs = {
ENV_VAR_KEY: {
'source_password': self.parameters.xdcr_admin_password
}
}
cluster_name = self.parameters.stg_cluster_name
env = _ReplicationMixin.generate_environment_map(self)
cmd = CommandFactory.get_replication_uuid(**env)
try:
stdout, stderr, exit_code = utilities.execute_bash(
self.connection, cmd, **kwargs)
if stdout is None or stdout == "":
logger.debug("No Replication ID identified")
return None, None
logger.debug("xdcr remote references : {}".format(stdout))
hostname = self.connection.environment.host.name
logger.debug("Environment hostname {}".format(hostname))
host_ip = ""
if not re.search(r"{}".format(hostname), stdout):
logger.debug(
"cluster for hostname {} doesn't exist".format(hostname))
logger.debug("Finding the ip for this host")
host_ip = self.get_ip()
logger.debug("Finding the replication uuid through host ip")
if not re.search(r"{}".format(host_ip), stdout):
logger.debug("cluster for host_ip {} doesn't exist".format(
hostname))
return None, None
else:
is_ip_or_string = True
logger.debug(
"cluster for host_ip {} exist".format(host_ip))
else:
logger.debug("cluster for hostname {} exist".format(host_ip))
if is_ip_or_string == False:
uuid = re.search(r"uuid:.*(?=\s.*{})".format(hostname),
stdout).group()
else:
uuid = re.search(r"uuid:.*(?=\s.*{})".format(host_ip),
stdout).group()
uuid = uuid.split(":")[1].strip()
cluster_name_staging = re.search(
r"cluster name:.*(?=\s.*{})".format(uuid), stdout).group()
cluster_name_staging = cluster_name_staging.split(":")[1].strip()
logger.debug("uuid for {} cluster : {}".format(
uuid, cluster_name_staging))
if cluster_name_staging == cluster_name:
return uuid, cluster_name
else:
return uuid, cluster_name_staging
except Exception as err:
logger.warn("Error identified: {} ".format(err.message))
logger.warn("UUID is None. Not able to find any cluster")
return None, None
def bucket_list(self, return_type=list):
# See the all bucket.
# It will return also other information like ramused, ramsize etc
logger.debug("Finding staged bucket list")
env = _BucketMixin.generate_environment_map(self)
command = CommandFactory.bucket_list(**env)
kwargs = {
ENV_VAR_KEY: {
'password': self.parameters.couchbase_admin_password
}
}
logger.debug("list bucket {}".format(command))
bucket_list, error, exit_code = utilities.execute_bash(
self.connection, command, **kwargs)
logger.debug("list bucket output{}".format(bucket_list))
if return_type == list:
#bucket_list = bucket_list.split("\n")
if bucket_list == "[]" or bucket_list is None:
logger.debug("empty list")
return []
else:
logger.debug("clean up json")
bucket_list = bucket_list.replace("u'", "'")
bucket_list = bucket_list.replace("'", "\"")
bucket_list = bucket_list.replace("True", "\"True\"")
bucket_list = bucket_list.replace("False", "\"False\"")
logger.debug("parse json")
bucket_list_dict = json.loads(bucket_list)
logger.debug("remap json")
bucket_list_dict = map(helper_lib.remap_bucket_json,
bucket_list_dict)
logger.debug(
"Bucket details in staged environment: {}".format(bucket_list))
return bucket_list_dict
def monitor_bucket(self, bucket_name, staging_UUID):
# To monitor the replication
logger.debug(
"Monitoring the replication for bucket {} ".format(bucket_name))
kwargs = {
ENV_VAR_KEY: {
'password': self.staged_source.parameters.xdcr_admin_password
}
}
command = CommandFactory.monitor_replication(
source_username=self.staged_source.parameters.xdcr_admin,
source_hostname=self.source_config.couchbase_src_host,
source_port=self.source_config.couchbase_src_port,
bucket_name=bucket_name,
uuid=staging_UUID)
stdout, stderr, exit_code = utilities.execute_bash(
self.connection, command, **kwargs)
logger.debug("stdout: {}".format(stdout))
content = json.loads(stdout)
pending_docs = self._get_last_value_of_node_stats(
content["nodeStats"].values()[0])
while pending_docs != 0:
logger.debug(
"Documents pending for replication: {}".format(pending_docs))
helper_lib.sleepForSecond(30)
stdout, stderr, exit_code = utilities.execute_bash(
self.connection, command, **kwargs)
content = json.loads(stdout)
pending_docs = self._get_last_value_of_node_stats(
content["nodeStats"].values()[0])
else:
logger.debug(
"Replication for bucket {} completed".format(bucket_name))
def check_server_is_used(connection, path):
ret = Status.INACTIVE
output, stderr, exit_code = utilities.execute_bash(connection,
CommandFactory.mount())
if exit_code != 0:
logger.error("mount retured error")
logger.error("stdout: {} stderr: {} exit_code: {}".format(
output.encode('utf-8'), stderr.encode('utf-8'), exit_code))
raise UserError("Problem with reading mounted file systems",
"Ask OS admin to check mount", stderr)
else:
# parse a mount output to find another Delphix mount points
fs_re = re.compile(r'(\S*)\son\s(\S*)\stype\s(\S*)')
for i in output.split("\n"):
match = re.search(fs_re, i)
if match is not None:
groups = match.groups()
if groups[2] == 'nfs':
if path == groups[1]:
# this is our mount point - skip it
ret = Status.ACTIVE
continue
if "domain0" in groups[0] and "timeflow" in groups[0]:
# this is a delphix mount point but it's not ours
# raise an exception
raise UserError(
"Another database (VDB or staging) is using this server.",
"Disable another one to provision or enable this one",
"{} {}".format(groups[0], groups[1]))
return ret
def delete_replication(self):
logger.debug("Deleting replication...")
stream_id, cluster_name = self.get_stream_id()
logger.debug("stream_id: {} and cluster_name : {}".format(
stream_id, cluster_name))
if stream_id is None or stream_id == "":
logger.debug("No Replication is found to delete.")
return False, cluster_name
kwargs = {
ENV_VAR_KEY: {
'source_password': self.parameters.xdcr_admin_password
}
}
env = _ReplicationMixin.generate_environment_map(self)
for id in stream_id:
cmd = CommandFactory.delete_replication(cluster_name=cluster_name,
id=id,
**env)
stdout, stderr, exit_code = utilities.execute_bash(
self.connection, cmd, **kwargs)
if exit_code != 0:
logger.warn("stream_id: {} deletion failed".format(id))
else:
logger.debug("stream_id: {} deletion succeeded".format(id))
return True, cluster_name
def find_version(source_connection, install_path):
""" return the couchbase version installed on the host"""
cb_version, std_err, exit_code = utilities.execute_bash(
source_connection, CommandFactory.get_version(install_path))
version = re.search(r"\d.*$", cb_version).group()
logger.debug("Couchbase version installed {}".format(version))
return version
def check_duplicate_replication(self, cluster_name):
logger.debug("Searching cluster name")
kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}}
env = _ReplicationMixin.generate_environment_map(self)
cmd = CommandFactory.get_replication_uuid(**env)
try:
stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
all_clusters = re.findall(r'cluster name:.*', stdout)
stream_id, cluster = self.get_stream_id()
logger.debug("stream_id:{} and cluster s:{} ".format(stream_id, cluster))
if stream_id:
# cluster is already set up between these nodes# No setup and no mis match
logger.debug("Already XDCR set up have been between source and staging server")
return True, False
logger.debug("No XDCR for staging host. Now validating the cluster name... ")
for each_cluster_pair in all_clusters:
each_cluster = each_cluster_pair.split(':')[1].strip()
logger.debug("Listed cluster: {} and input is:{} ".format(each_cluster, cluster_name))
if each_cluster == cluster_name:
logger.debug("Duplicate cluster name issue identified ")
# no setup but mismatch
return False, True
return False, False
except Exception as err:
logger.debug("Failed to verify the duplicate name: {} ".format(err.message))
def unmount_file_system(rx_connection, path):
""" unmount the file system which will use in cbbackup manager after post snapshot"""
try:
utilities.execute_bash(rx_connection,
CommandFactory.unmount_file_system(path))
except Exception as err:
logger.debug("error here {}".format(err.message))
raise UnmountFileSystemError(err.message)
def build_index(self, index_name):
logger.debug("Building indexes....")
env = _CBBackupMixin.generate_environment_map(self)
cmd = CommandFactory.build_index(index_name=index_name, **env)
command_output, std_err, exit_code = utilities.execute_bash(
self.connection, command_name=cmd, **env)
logger.debug("command_output is ".format(command_output))
return command_output
def pause_replication(self):
logger.debug("Pausing replication ...")
stream_id, cluster_name = self.get_stream_id()
kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}}
env = _ReplicationMixin.generate_environment_map(self)
cmd = CommandFactory.pause_replication(cluster_name=cluster_name, **env)
for replication_id in stream_id:
kwargs["id"] = replication_id
stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
logger.debug(stdout)
def write_file(connection, content, filename):
"""Add given data into passed filename"""
logger.debug("writing data {} in file {}".format(content, filename))
try:
utilities.execute_bash(
connection,
CommandFactory.write_file(data=content, filename=filename))
except Exception as e:
logger.debug("Failed to Write into file")
raise FileIOError("Failed to Write into file ")
def clean_stale_mountpoint(connection, path):
umount_std, umount_stderr, umount_exit_code = utilities.execute_bash(
connection,
CommandFactory.unmount_file_system(mount_path=path, options='-lf'))
if umount_exit_code != 0:
logger.error("Problem with cleaning mount path")
logger.error("stderr {}".format(umount_stderr))
raise UserError("Problem with cleaning mount path",
"Ask OS admin to check mount points", umount_stderr)
def check_file_present(connection, config_file_path):
""" return True if file is present else return False"""
try:
stdout, stderr, exit_code = utilities.execute_bash(
connection, CommandFactory.check_file(config_file_path))
if stdout == "Found":
logger.debug("file path exist {}".format(config_file_path))
return True
except Exception as e:
logger.debug("File path not exist {}".format(config_file_path))
return False
def check_dir_present(connection, dir):
""" return True if directory is present else return False"""
try:
stdout, stderr, exit_code = utilities.execute_bash(
connection, CommandFactory.check_directory(dir))
if stdout == "Found":
logger.debug("dir path found {} ".format(dir))
return True
except Exception as err:
logger.debug("directory path is absent: {}".format(err.message))
return False
def node_init(self):
"""
This method initializes couchbase server node. Where user sets different required paths
:return: None
"""
logger.debug("Initializing the NODE")
kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
command = CommandFactory.node_init(self.repository.cb_shell_path, self.parameters.couchbase_port,
self.parameters.couchbase_admin, self.parameters.mount_path)
command_output, std_err, exit_code = utilities.execute_bash(self.connection, command, **kwargs)
logger.debug("Command Output {} ".format(command_output))
def check_config(self):
filename = os.path.join(self.get_config_directory(), "config.dat")
cmd = CommandFactory.check_file(filename)
logger.debug("check file cmd: {}".format(cmd))
command_output, std_err, exit_code = utilities.execute_bash(
self.connection, command_name=cmd, callback_func=self.ignore_err)
if exit_code == 0 and "Found" in command_output:
return True
else:
return False
def bucket_flush(self, bucket_name):
# It requires the before bucket delete
logger.debug("Flushing bucket: {} ".format(bucket_name))
self.__validate_bucket_name(bucket_name)
env = _BucketMixin.generate_environment_map(self)
command = CommandFactory.bucket_flush(bucket_name=bucket_name, **env)
kwargs = {
ENV_VAR_KEY: {
'password': self.parameters.couchbase_admin_password
}
}
return utilities.execute_bash(self.connection, command, **kwargs)
def bucket_delete(self, bucket_name):
# To delete the bucket
logger.debug("Deleting bucket: {} ".format(bucket_name))
self.__validate_bucket_name(bucket_name)
env = _BucketMixin.generate_environment_map(self)
command = CommandFactory.bucket_delete(bucket_name=bucket_name, **env)
kwargs = {
ENV_VAR_KEY: {
'password': self.parameters.couchbase_admin_password
}
}
logger.debug("delete bucket {}".format(command))
return utilities.execute_bash(self.connection, command, **kwargs)
def find_shell_path(source_connection, binary_path):
"""
:param source_connection:Connection for the source environment
:param binary_path: Couchbase binary path
:return:path of cluster management utility: {couchbase-cli}
"""
logger.debug("Finding Shell Path...")
shell_path, std_err, exit_code = utilities.execute_bash(
source_connection, CommandFactory.find_shell_path(binary_path))
if shell_path == "":
message = "Shell path {}/couchbase-cli not found".format(binary_path)
raise RepositoryDiscoveryError(message)
return shell_path
def xdcr_setup(self):
logger.debug("Started XDCR set up ...")
kwargs = {
ENV_VAR_KEY: {
'source_password': self.parameters.xdcr_admin_password,
'password': self.parameters.couchbase_admin_password
}
}
env = _XDCrMixin.generate_environment_map(self)
cmd = CommandFactory.xdcr_setup(
cluster_name=self.parameters.stg_cluster_name, **env)
stdout, stderr, exit_code = utilities.execute_bash(
self.connection, cmd, **kwargs)
helper_lib.sleepForSecond(3)
def find_ids(source_connection, install_path):
""" return the couchbase uid and gid"""
std_out, std_err, exit_code = utilities.execute_bash(
source_connection, CommandFactory.get_ids(install_path))
logger.debug("find ids output: {}".format(std_out))
ids = re.search(r"[-rwx.]+\s\d\s([\d]+)\s([\d]+).*", std_out)
if ids:
uid = int(ids.group(1))
gid = int(ids.group(2))
else:
uid = -1
gid = -1
logger.debug("Couchbase user uid {} gid {}".format(uid, gid))
return (uid, gid)
def get_indexes_name(self, index_name):
logger.debug("Finding indexes....")
env = {
ENV_VAR_KEY: {
'password': self.parameters.couchbase_admin_password
}
}
env = _CBBackupMixin.generate_environment_map(self)
cmd = CommandFactory.get_indexes_name(**env)
logger.debug("env detail is : ".format(env))
command_output, std_err, exit_code = utilities.execute_bash(
self.connection, command_name=cmd, **env)
logger.debug("Indexes are {}".format(command_output))
return command_output
def find_whoami(source_connection):
""" return the user env id"""
std_out, std_err, exit_code = utilities.execute_bash(
source_connection, CommandFactory.whoami())
logger.debug("find whoami output: {}".format(std_out))
ids = re.search(r"uid=([\d]+).*gid=([\d]+)", std_out)
if ids:
uid = int(ids.group(1))
gid = int(ids.group(2))
else:
uid = -1
gid = -1
logger.debug("Delphix user uid {} gid {}".format(uid, gid))
return (uid, gid)
def check_stale_mountpoint(connection, path):
output, stderr, exit_code = utilities.execute_bash(connection,
CommandFactory.df(path))
if exit_code != 0:
if "No such file or directory" in stderr:
# this is actually OK
return False
else:
logger.error("df retured error - stale mount point or other error")
logger.error("stdout: {} stderr: {} exit_code: {}".format(
output.encode('utf-8'), stderr.encode('utf-8'), exit_code))
return True
else:
return False
def cb_backup_full(self, csv_bucket):
logger.debug("Starting Restore via Backup file...")
logger.debug("csv_bucket_list: {}".format(csv_bucket))
kwargs = {
ENV_VAR_KEY: {
'password': self.parameters.couchbase_admin_password
}
}
env = _CBBackupMixin.generate_environment_map(self)
cmd = CommandFactory.cb_backup_full(
backup_location=self.parameters.couchbase_bak_loc,
csv_bucket_list=csv_bucket,
backup_repo=self.parameters.couchbase_bak_repo,
**env)
utilities.execute_bash(self.connection, cmd, **kwargs)
def get_stream_id(self):
logger.debug("Finding the stream id for provided cluster name")
kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}}
uuid, cluster_name = self.get_replication_uuid()
if uuid is None:
return None, None
env = _ReplicationMixin.generate_environment_map(self)
cmd = CommandFactory.get_stream_id(cluster_name=cluster_name, **env)
stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
if stdout is None or stdout == "":
logger.debug("No stream ID identified")
return None, None
else:
stream_id = re.findall(r"(?<=stream id: ){}.*".format(uuid), stdout)
logger.debug("Stream id found: {}".format(stream_id))
return stream_id, cluster_name
def create_config_dir(self):
"""create and return the hidden folder directory with name 'delphix'"""
logger.debug("Finding toolkit Path...")
command = CommandFactory.get_dlpx_bin()
bin_directory, std_err, exit_code = utilities.execute_bash(self.connection, command)
if bin_directory is None or bin_directory == "":
raise Exception("Failed to find the toolkit directory")
# Toolkit directory tested on linux x86_64Bit is 6 level below jq path
loop_var = 6
while loop_var:
bin_directory = os.path.dirname(bin_directory)
loop_var = loop_var - 1
dir_name = bin_directory + "/" + DELPHIX_HIDDEN_FOLDER
if not helper_lib.check_dir_present(self.connection, dir_name):
self.make_directory(dir_name)
return dir_name