Ejemplo n.º 1
0
    def move_bucket(self, bucket_name, direction):
        logger.debug("Rename folder")

        if direction == 'save':
            src = join(self.virtual_source.parameters.mount_path, 'data',
                       bucket_name)
            dst = join(self.virtual_source.parameters.mount_path, 'data',
                       ".{}.delphix".format(bucket_name))
            command = CommandFactory.os_mv(src, dst, self.need_sudo, self.uid)
            logger.debug("rename command: {}".format(command))
            stdout, error, exit_code = utilities.execute_bash(
                self.connection, command)
        elif direction == 'restore':
            dst = join(self.virtual_source.parameters.mount_path, 'data',
                       bucket_name)
            src = join(self.virtual_source.parameters.mount_path, 'data',
                       ".{}.delphix".format(bucket_name))
            command = CommandFactory.delete_dir(dst, self.need_sudo, self.uid)
            logger.debug("delete command: {}".format(command))
            stdout, error, exit_code = utilities.execute_bash(
                self.connection, command)
            command = CommandFactory.os_mv(src, dst, self.need_sudo, self.uid)
            logger.debug("rename command: {}".format(command))
            stdout, error, exit_code = utilities.execute_bash(
                self.connection, command)
Ejemplo n.º 2
0
 def make_directory(self, directory_path):
     """
     Create a directory and set the permission level 775
     :param directory_path: The directory path
     :return: None
     """
     logger.debug("Creating Directory {} ".format(directory_path))
     env = {'directory_path': directory_path}
     command = CommandFactory.make_directory(directory_path)
     utilities.execute_bash(self.connection, command)
     logger.debug("Changing permission of directory path {}".format(directory_path))
     command = CommandFactory.change_permission(directory_path)
     utilities.execute_bash(self.connection, command)
     logger.debug("Changed the permission of directory")
Ejemplo n.º 3
0
    def delete_config(self):

        # TODO:
        # error handling

        logger.debug("start delete_config")

        filename = "{}/../var/lib/couchbase/config/config.dat".format(
            helper_lib.get_base_directory_of_given_path(
                self.repository.cb_shell_path))

        cmd = CommandFactory.check_file(filename, self.need_sudo, self.uid)
        logger.debug("check file cmd: {}".format(cmd))
        command_output, std_err, exit_code = utilities.execute_bash(
            self.connection, command_name=cmd, callback_func=self.ignore_err)

        if exit_code == 0 and "Found" in command_output:
            cmd = CommandFactory.os_mv(filename, "{}.bak".format(filename),
                                       self.need_sudo, self.uid)
            logger.debug("rename config cmd: {}".format(cmd))
            command_output, std_err, exit_code = utilities.execute_bash(
                self.connection, command_name=cmd)
            logger.debug(
                "rename config.dat to bak - exit_code: {} stdout: {} std_err: {}"
                .format(exit_code, command_output, std_err))

        filename = "{}/../etc/couchdb/local.ini".format(
            helper_lib.get_base_directory_of_given_path(
                self.repository.cb_shell_path))
        command_output, std_err, exit_code = self.run_os_command(
            os_command='sed', filename=filename, regex='s/view_index_dir.*//')

        logger.debug(
            "clean local.ini index - exit_code: {} stdout: {} std_err: {}".
            format(exit_code, command_output, std_err))

        command_output, std_err, exit_code = self.run_os_command(
            os_command='sed', filename=filename, regex='s/database_dir.*//')

        logger.debug(
            "clean local.ini data - exit_code: {} stdout: {} std_err: {}".
            format(exit_code, command_output, std_err))
        command_output, std_err, exit_code = self.run_os_command(
            os_command='change_permission', path=filename)

        logger.debug(
            "fix local.ini permission - exit_code: {} stdout: {} std_err: {}".
            format(exit_code, command_output, std_err))
Ejemplo n.º 4
0
    def get_replication_uuid(self):
        # False for string
        logger.debug("Finding the replication uuid through host name")
        is_ip_or_string = False
        kwargs = {
            ENV_VAR_KEY: {
                'source_password': self.parameters.xdcr_admin_password
            }
        }
        cluster_name = self.parameters.stg_cluster_name
        env = _ReplicationMixin.generate_environment_map(self)
        cmd = CommandFactory.get_replication_uuid(**env)
        try:

            stdout, stderr, exit_code = utilities.execute_bash(
                self.connection, cmd, **kwargs)
            if stdout is None or stdout == "":
                logger.debug("No Replication ID identified")
                return None, None
            logger.debug("xdcr remote references : {}".format(stdout))
            hostname = self.connection.environment.host.name
            logger.debug("Environment hostname {}".format(hostname))
            host_ip = ""
            if not re.search(r"{}".format(hostname), stdout):
                logger.debug(
                    "cluster for hostname {} doesn't exist".format(hostname))
                logger.debug("Finding the ip for this host")
                host_ip = self.get_ip()
                logger.debug("Finding the replication uuid through host ip")
                if not re.search(r"{}".format(host_ip), stdout):
                    logger.debug("cluster for host_ip {} doesn't exist".format(
                        hostname))
                    return None, None
                else:
                    is_ip_or_string = True
                    logger.debug(
                        "cluster for host_ip {} exist".format(host_ip))
            else:
                logger.debug("cluster for hostname {}  exist".format(host_ip))
            if is_ip_or_string == False:
                uuid = re.search(r"uuid:.*(?=\s.*{})".format(hostname),
                                 stdout).group()
            else:
                uuid = re.search(r"uuid:.*(?=\s.*{})".format(host_ip),
                                 stdout).group()

            uuid = uuid.split(":")[1].strip()
            cluster_name_staging = re.search(
                r"cluster name:.*(?=\s.*{})".format(uuid), stdout).group()
            cluster_name_staging = cluster_name_staging.split(":")[1].strip()
            logger.debug("uuid for {} cluster : {}".format(
                uuid, cluster_name_staging))
            if cluster_name_staging == cluster_name:
                return uuid, cluster_name
            else:
                return uuid, cluster_name_staging
        except Exception as err:
            logger.warn("Error identified: {} ".format(err.message))
            logger.warn("UUID is None. Not able to find any cluster")
            return None, None
Ejemplo n.º 5
0
 def bucket_list(self, return_type=list):
     # See the all bucket.
     # It will return also other information like ramused, ramsize etc
     logger.debug("Finding staged bucket list")
     env = _BucketMixin.generate_environment_map(self)
     command = CommandFactory.bucket_list(**env)
     kwargs = {
         ENV_VAR_KEY: {
             'password': self.parameters.couchbase_admin_password
         }
     }
     logger.debug("list bucket {}".format(command))
     bucket_list, error, exit_code = utilities.execute_bash(
         self.connection, command, **kwargs)
     logger.debug("list bucket output{}".format(bucket_list))
     if return_type == list:
         #bucket_list = bucket_list.split("\n")
         if bucket_list == "[]" or bucket_list is None:
             logger.debug("empty list")
             return []
         else:
             logger.debug("clean up json")
             bucket_list = bucket_list.replace("u'", "'")
             bucket_list = bucket_list.replace("'", "\"")
             bucket_list = bucket_list.replace("True", "\"True\"")
             bucket_list = bucket_list.replace("False", "\"False\"")
             logger.debug("parse json")
             bucket_list_dict = json.loads(bucket_list)
             logger.debug("remap json")
             bucket_list_dict = map(helper_lib.remap_bucket_json,
                                    bucket_list_dict)
     logger.debug(
         "Bucket details in staged environment: {}".format(bucket_list))
     return bucket_list_dict
Ejemplo n.º 6
0
 def monitor_bucket(self, bucket_name, staging_UUID):
     # To monitor the replication
     logger.debug(
         "Monitoring the replication for bucket {} ".format(bucket_name))
     kwargs = {
         ENV_VAR_KEY: {
             'password': self.staged_source.parameters.xdcr_admin_password
         }
     }
     command = CommandFactory.monitor_replication(
         source_username=self.staged_source.parameters.xdcr_admin,
         source_hostname=self.source_config.couchbase_src_host,
         source_port=self.source_config.couchbase_src_port,
         bucket_name=bucket_name,
         uuid=staging_UUID)
     stdout, stderr, exit_code = utilities.execute_bash(
         self.connection, command, **kwargs)
     logger.debug("stdout: {}".format(stdout))
     content = json.loads(stdout)
     pending_docs = self._get_last_value_of_node_stats(
         content["nodeStats"].values()[0])
     while pending_docs != 0:
         logger.debug(
             "Documents pending for replication: {}".format(pending_docs))
         helper_lib.sleepForSecond(30)
         stdout, stderr, exit_code = utilities.execute_bash(
             self.connection, command, **kwargs)
         content = json.loads(stdout)
         pending_docs = self._get_last_value_of_node_stats(
             content["nodeStats"].values()[0])
     else:
         logger.debug(
             "Replication for bucket {} completed".format(bucket_name))
Ejemplo n.º 7
0
def check_server_is_used(connection, path):

    ret = Status.INACTIVE

    output, stderr, exit_code = utilities.execute_bash(connection,
                                                       CommandFactory.mount())
    if exit_code != 0:
        logger.error("mount retured error")
        logger.error("stdout: {} stderr: {} exit_code: {}".format(
            output.encode('utf-8'), stderr.encode('utf-8'), exit_code))
        raise UserError("Problem with reading mounted file systems",
                        "Ask OS admin to check mount", stderr)
    else:
        # parse a mount output to find another Delphix mount points
        fs_re = re.compile(r'(\S*)\son\s(\S*)\stype\s(\S*)')
        for i in output.split("\n"):
            match = re.search(fs_re, i)
            if match is not None:
                groups = match.groups()
                if groups[2] == 'nfs':
                    if path == groups[1]:
                        # this is our mount point - skip it
                        ret = Status.ACTIVE
                        continue
                    if "domain0" in groups[0] and "timeflow" in groups[0]:
                        # this is a delphix mount point but it's not ours
                        # raise an exception
                        raise UserError(
                            "Another database (VDB or staging) is using this server.",
                            "Disable another one to provision or enable this one",
                            "{} {}".format(groups[0], groups[1]))

    return ret
Ejemplo n.º 8
0
    def delete_replication(self):
        logger.debug("Deleting replication...")
        stream_id, cluster_name = self.get_stream_id()
        logger.debug("stream_id: {} and cluster_name : {}".format(
            stream_id, cluster_name))
        if stream_id is None or stream_id == "":
            logger.debug("No Replication is found to delete.")
            return False, cluster_name
        kwargs = {
            ENV_VAR_KEY: {
                'source_password': self.parameters.xdcr_admin_password
            }
        }
        env = _ReplicationMixin.generate_environment_map(self)

        for id in stream_id:
            cmd = CommandFactory.delete_replication(cluster_name=cluster_name,
                                                    id=id,
                                                    **env)
            stdout, stderr, exit_code = utilities.execute_bash(
                self.connection, cmd, **kwargs)
            if exit_code != 0:
                logger.warn("stream_id: {} deletion failed".format(id))
            else:
                logger.debug("stream_id: {} deletion succeeded".format(id))
        return True, cluster_name
Ejemplo n.º 9
0
def find_version(source_connection, install_path):
    """ return the couchbase version installed on the host"""
    cb_version, std_err, exit_code = utilities.execute_bash(
        source_connection, CommandFactory.get_version(install_path))
    version = re.search(r"\d.*$", cb_version).group()
    logger.debug("Couchbase version installed {}".format(version))
    return version
Ejemplo n.º 10
0
    def check_duplicate_replication(self, cluster_name):
        logger.debug("Searching cluster name")
        kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}}
        env = _ReplicationMixin.generate_environment_map(self)
        cmd = CommandFactory.get_replication_uuid(**env)
        try:
            stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
            all_clusters = re.findall(r'cluster name:.*', stdout)
            stream_id, cluster = self.get_stream_id()
            logger.debug("stream_id:{} and cluster s:{} ".format(stream_id, cluster))
            if stream_id:
                # cluster is already set up between these nodes# No setup and no mis match
                logger.debug("Already XDCR set up have been between source and staging server")
                return True, False
            logger.debug("No XDCR for staging host. Now validating the cluster name... ")
            for each_cluster_pair in all_clusters:
                each_cluster = each_cluster_pair.split(':')[1].strip()
                logger.debug("Listed cluster: {} and input is:{} ".format(each_cluster, cluster_name))
                if each_cluster == cluster_name:
                    logger.debug("Duplicate cluster name issue identified ")
                    # no setup but mismatch
                    return False, True
            return False, False

        except Exception as err:
            logger.debug("Failed to verify the duplicate name: {} ".format(err.message))
Ejemplo n.º 11
0
def unmount_file_system(rx_connection, path):
    """ unmount the file system which will use in cbbackup manager after post snapshot"""
    try:
        utilities.execute_bash(rx_connection,
                               CommandFactory.unmount_file_system(path))
    except Exception as err:
        logger.debug("error here {}".format(err.message))
        raise UnmountFileSystemError(err.message)
Ejemplo n.º 12
0
 def build_index(self, index_name):
     logger.debug("Building indexes....")
     env = _CBBackupMixin.generate_environment_map(self)
     cmd = CommandFactory.build_index(index_name=index_name, **env)
     command_output, std_err, exit_code = utilities.execute_bash(
         self.connection, command_name=cmd, **env)
     logger.debug("command_output is ".format(command_output))
     return command_output
Ejemplo n.º 13
0
 def pause_replication(self):
     logger.debug("Pausing replication ...")
     stream_id, cluster_name = self.get_stream_id()
     kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}}
     env = _ReplicationMixin.generate_environment_map(self)
     cmd = CommandFactory.pause_replication(cluster_name=cluster_name, **env)
     for replication_id in stream_id:
         kwargs["id"] = replication_id
         stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
         logger.debug(stdout)
Ejemplo n.º 14
0
def write_file(connection, content, filename):
    """Add given data into passed filename"""
    logger.debug("writing data {} in file {}".format(content, filename))
    try:
        utilities.execute_bash(
            connection,
            CommandFactory.write_file(data=content, filename=filename))
    except Exception as e:
        logger.debug("Failed to Write into file")
        raise FileIOError("Failed to Write into file ")
Ejemplo n.º 15
0
def clean_stale_mountpoint(connection, path):

    umount_std, umount_stderr, umount_exit_code = utilities.execute_bash(
        connection,
        CommandFactory.unmount_file_system(mount_path=path, options='-lf'))
    if umount_exit_code != 0:
        logger.error("Problem with cleaning mount path")
        logger.error("stderr {}".format(umount_stderr))
        raise UserError("Problem with cleaning mount path",
                        "Ask OS admin to check mount points", umount_stderr)
Ejemplo n.º 16
0
def check_file_present(connection, config_file_path):
    """ return True if file is present else return False"""
    try:
        stdout, stderr, exit_code = utilities.execute_bash(
            connection, CommandFactory.check_file(config_file_path))
        if stdout == "Found":
            logger.debug("file path exist {}".format(config_file_path))
            return True
    except Exception as e:
        logger.debug("File path not exist {}".format(config_file_path))
    return False
Ejemplo n.º 17
0
def check_dir_present(connection, dir):
    """ return True if directory is present else return False"""
    try:
        stdout, stderr, exit_code = utilities.execute_bash(
            connection, CommandFactory.check_directory(dir))
        if stdout == "Found":
            logger.debug("dir path found {} ".format(dir))
            return True
    except Exception as err:
        logger.debug("directory path is absent: {}".format(err.message))
    return False
Ejemplo n.º 18
0
 def node_init(self):
     """
     This method initializes couchbase server node. Where user sets different required paths
     :return: None
     """
     logger.debug("Initializing the NODE")
     kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}}
     command = CommandFactory.node_init(self.repository.cb_shell_path, self.parameters.couchbase_port,
                                        self.parameters.couchbase_admin, self.parameters.mount_path)
     command_output, std_err, exit_code = utilities.execute_bash(self.connection, command, **kwargs)
     logger.debug("Command Output {} ".format(command_output))
Ejemplo n.º 19
0
    def check_config(self):

        filename = os.path.join(self.get_config_directory(), "config.dat")
        cmd = CommandFactory.check_file(filename)
        logger.debug("check file cmd: {}".format(cmd))
        command_output, std_err, exit_code = utilities.execute_bash(
            self.connection, command_name=cmd, callback_func=self.ignore_err)

        if exit_code == 0 and "Found" in command_output:
            return True
        else:
            return False
Ejemplo n.º 20
0
 def bucket_flush(self, bucket_name):
     # It requires the before bucket delete
     logger.debug("Flushing bucket: {} ".format(bucket_name))
     self.__validate_bucket_name(bucket_name)
     env = _BucketMixin.generate_environment_map(self)
     command = CommandFactory.bucket_flush(bucket_name=bucket_name, **env)
     kwargs = {
         ENV_VAR_KEY: {
             'password': self.parameters.couchbase_admin_password
         }
     }
     return utilities.execute_bash(self.connection, command, **kwargs)
Ejemplo n.º 21
0
 def bucket_delete(self, bucket_name):
     # To delete the bucket
     logger.debug("Deleting bucket: {} ".format(bucket_name))
     self.__validate_bucket_name(bucket_name)
     env = _BucketMixin.generate_environment_map(self)
     command = CommandFactory.bucket_delete(bucket_name=bucket_name, **env)
     kwargs = {
         ENV_VAR_KEY: {
             'password': self.parameters.couchbase_admin_password
         }
     }
     logger.debug("delete bucket {}".format(command))
     return utilities.execute_bash(self.connection, command, **kwargs)
Ejemplo n.º 22
0
def find_shell_path(source_connection, binary_path):
    """
    :param source_connection:Connection for the source environment
    :param binary_path: Couchbase binary path
    :return:path of cluster management utility: {couchbase-cli}
    """
    logger.debug("Finding Shell Path...")
    shell_path, std_err, exit_code = utilities.execute_bash(
        source_connection, CommandFactory.find_shell_path(binary_path))
    if shell_path == "":
        message = "Shell path {}/couchbase-cli not found".format(binary_path)
        raise RepositoryDiscoveryError(message)
    return shell_path
Ejemplo n.º 23
0
 def xdcr_setup(self):
     logger.debug("Started XDCR set up ...")
     kwargs = {
         ENV_VAR_KEY: {
             'source_password': self.parameters.xdcr_admin_password,
             'password': self.parameters.couchbase_admin_password
         }
     }
     env = _XDCrMixin.generate_environment_map(self)
     cmd = CommandFactory.xdcr_setup(
         cluster_name=self.parameters.stg_cluster_name, **env)
     stdout, stderr, exit_code = utilities.execute_bash(
         self.connection, cmd, **kwargs)
     helper_lib.sleepForSecond(3)
Ejemplo n.º 24
0
def find_ids(source_connection, install_path):
    """ return the couchbase uid and gid"""
    std_out, std_err, exit_code = utilities.execute_bash(
        source_connection, CommandFactory.get_ids(install_path))
    logger.debug("find ids output: {}".format(std_out))
    ids = re.search(r"[-rwx.]+\s\d\s([\d]+)\s([\d]+).*", std_out)
    if ids:
        uid = int(ids.group(1))
        gid = int(ids.group(2))
    else:
        uid = -1
        gid = -1
    logger.debug("Couchbase user uid {} gid {}".format(uid, gid))
    return (uid, gid)
Ejemplo n.º 25
0
 def get_indexes_name(self, index_name):
     logger.debug("Finding indexes....")
     env = {
         ENV_VAR_KEY: {
             'password': self.parameters.couchbase_admin_password
         }
     }
     env = _CBBackupMixin.generate_environment_map(self)
     cmd = CommandFactory.get_indexes_name(**env)
     logger.debug("env detail is : ".format(env))
     command_output, std_err, exit_code = utilities.execute_bash(
         self.connection, command_name=cmd, **env)
     logger.debug("Indexes are {}".format(command_output))
     return command_output
Ejemplo n.º 26
0
def find_whoami(source_connection):
    """ return the user env id"""
    std_out, std_err, exit_code = utilities.execute_bash(
        source_connection, CommandFactory.whoami())
    logger.debug("find whoami output: {}".format(std_out))
    ids = re.search(r"uid=([\d]+).*gid=([\d]+)", std_out)
    if ids:
        uid = int(ids.group(1))
        gid = int(ids.group(2))
    else:
        uid = -1
        gid = -1
    logger.debug("Delphix user uid {} gid {}".format(uid, gid))
    return (uid, gid)
Ejemplo n.º 27
0
def check_stale_mountpoint(connection, path):

    output, stderr, exit_code = utilities.execute_bash(connection,
                                                       CommandFactory.df(path))
    if exit_code != 0:
        if "No such file or directory" in stderr:
            # this is actually OK
            return False
        else:
            logger.error("df retured error - stale mount point or other error")
            logger.error("stdout: {} stderr: {} exit_code: {}".format(
                output.encode('utf-8'), stderr.encode('utf-8'), exit_code))
            return True
    else:
        return False
Ejemplo n.º 28
0
 def cb_backup_full(self, csv_bucket):
     logger.debug("Starting Restore via Backup file...")
     logger.debug("csv_bucket_list: {}".format(csv_bucket))
     kwargs = {
         ENV_VAR_KEY: {
             'password': self.parameters.couchbase_admin_password
         }
     }
     env = _CBBackupMixin.generate_environment_map(self)
     cmd = CommandFactory.cb_backup_full(
         backup_location=self.parameters.couchbase_bak_loc,
         csv_bucket_list=csv_bucket,
         backup_repo=self.parameters.couchbase_bak_repo,
         **env)
     utilities.execute_bash(self.connection, cmd, **kwargs)
Ejemplo n.º 29
0
 def get_stream_id(self):
     logger.debug("Finding the stream id for provided cluster name")
     kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}}
     uuid, cluster_name = self.get_replication_uuid()
     if uuid is None:
         return None, None
     env = _ReplicationMixin.generate_environment_map(self)
     cmd = CommandFactory.get_stream_id(cluster_name=cluster_name, **env)
     stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs)
     if stdout is None or stdout == "":
         logger.debug("No stream ID identified")
         return None, None
     else:
         stream_id = re.findall(r"(?<=stream id: ){}.*".format(uuid), stdout)
         logger.debug("Stream id found: {}".format(stream_id))
         return stream_id, cluster_name
Ejemplo n.º 30
0
 def create_config_dir(self):
     """create and return the hidden folder directory with name 'delphix'"""
     logger.debug("Finding toolkit Path...")
     command = CommandFactory.get_dlpx_bin()
     bin_directory, std_err, exit_code = utilities.execute_bash(self.connection, command)
     if bin_directory is None or bin_directory == "":
         raise Exception("Failed to find the toolkit directory")
     # Toolkit directory tested on linux x86_64Bit is 6 level below jq path
     loop_var = 6
     while loop_var:
         bin_directory = os.path.dirname(bin_directory)
         loop_var = loop_var - 1
     dir_name = bin_directory + "/" + DELPHIX_HIDDEN_FOLDER
     if not helper_lib.check_dir_present(self.connection, dir_name):
         self.make_directory(dir_name)
     return dir_name