def save_buckets_tree(self, buckets):
        ceph_api = CephAPI()
        buckets_types = ceph_api.get_bucket_types()
        buckets_tree = json.loads(buckets)
        crush_map = []
        for bucket_tree in buckets_tree:
            if "children" in bucket_tree.keys():
                get_children_info(bucket_tree, crush_map, buckets_types)
            else:
                crush_map.append({
                    "name":
                    bucket_tree["name"],
                    "id":
                    bucket_tree["id"],
                    "class_ids":
                    bucket_tree["class_ids"],
                    "hash":
                    bucket_tree["hash"],
                    "alg":
                    bucket_tree["alg"],
                    "type_id":
                    buckets_types.index(bucket_tree["type"])
                })

        ceph_api.save_buckets(crush_map)
    def detach_disk(self, disk_id, pool):
        ceph_api = CephAPI()
        disk_meta = DiskMeta()
        disk_meta.id = disk_id
        disk_meta.disk_name = None

        return ceph_api.add_disk(disk_meta, False, pool)
def get_disk_snapshots(args):
    disk_id = args.disk_id
    try:
        ceph_api = CephAPI()
        image_name = "image-" + disk_id
        pool_name = ceph_api.get_pool_bydisk(disk_id)

        # If pool inactive #
        if pool_name is None:
            print("Error : Cannot get pool of disk {}".format(str(disk_id)))
            sys.exit(-1)

        else:
            snaps_ls = ceph_api.get_disk_snapshots(pool_name, image_name)
            print(json.dumps(snaps_ls))
            sys.exit(0)

    except CephException as e:
        if e.id == CephException.GENERAL_EXCEPTION:
            print("Error : CephException - {} , Cannot get disk {} snapshots.".
                  format(str(e.message), disk_id))
            sys.exit(-1)

    except Exception as e:
        print("Error : Exception - {}".format(str(e.message)))
        sys.exit(-1)
Exemple #4
0
def rollback_snapshot(args):
    disk_id = args.disk_id
    snap_name = args.snapshot_name
    try:
        ceph_api = CephAPI()
        image_name = str(ceph_api.conf_api.get_image_name_prefix() + disk_id)
        pool_name = ceph_api.get_pool_bydisk(
            disk_id)  # getting pool_name from disk_id

        confirm = ceph_api.rollback_to_snapshot(pool_name, image_name,
                                                snap_name)

        if not confirm:
            print("Error : Cannot rollback disk " + disk_id + " to snapshot " +
                  snap_name)
            sys.exit(-1)

        sys.exit(0)

    except CephException as e:
        if e.id == CephException.GENERAL_EXCEPTION:
            print("Error : " + str(e.message) + " , Cannot rollback disk " +
                  disk_id + " to snapshot " + snap_name)
            sys.exit(-1)

    except Exception as e:
        print("Error : Exception , {}".format(e.message))
        sys.exit(-1)
def delete_snapshot(args):
    disk_id = args.disk_id
    snap_name = args.snapshot_name
    try:
        ceph_api = CephAPI()
        image_name = str(ceph_api.conf_api.get_image_name_prefix() + disk_id)
        pool_name = ceph_api.get_pool_bydisk(
            disk_id)  # getting pool_name from disk_id

        # If pool inactive #
        if pool_name is None:
            print("Error : Cannot get pool of disk {}".format(str(disk_id)))
            sys.exit(-1)

        else:
            confirm = ceph_api.delete_snapshot(pool_name, image_name,
                                               snap_name)

            if not confirm:
                print("Error : Cannot delete snapshot \"" + snap_name +
                      "\" of disk " + disk_id)
                sys.exit(-1)

        sys.exit(0)

    except CephException as e:
        if e.id == CephException.GENERAL_EXCEPTION:
            print("Error : CephException - " + str(e.message) +
                  " , Cannot delete snapshot \"" + snap_name + "\" of disk " +
                  disk_id)
            sys.exit(-1)

    except Exception as e:
        print("Error : Exception - {}".format(str(e.message)))
        sys.exit(-1)
Exemple #6
0
def get_diskmeta(args):
    # Getting a specific disk metadata from "Remote Cluster" giving a disk_id (DiskMeta Objects):
    try:
        disk_id = args.disk_id
        ceph_api = CephAPI()
        disk_metadata = ceph_api.get_diskmeta(disk_id)
        diskmeta_dict = disk_metadata.__dict__
        print(json.dumps(diskmeta_dict, indent=4, sort_keys=False))
        sys.exit(0)

    except DiskListException as e:
        print("Error : DiskListException , {}".format(e.message))
        sys.exit(-1)

    except CephException as e:
        if e.id == CephException.GENERAL_EXCEPTION:
            print("Error : {} , Cannot get disk meta.".format(e.message))
            sys.exit(-1)

    except MetadataException as e:
        print("Error : MetadataException , {}".format(e.message))
        sys.exit(-1)

    except Exception as e:
        print("Error : Exception , {}".format(e.message))
        sys.exit(-1)
    def run(self):
        try:
            result = Result()
            ceph_api = CephAPI()
            cluster_status = ceph_api.get_ceph_cluster_status()
            if cluster_status is not None:
                cluster_status = json.loads(cluster_status)

                available_size = 0
                used_size = 0

                if cluster_status['pgmap']['bytes_total'] > 0:
                    available_size = cluster_status['pgmap']['bytes_avail'] * 100.0 / cluster_status['pgmap']['bytes_total']
                    used_size = cluster_status['pgmap']['bytes_used'] * 100.0 / cluster_status['pgmap']['bytes_total']

                notify_cluster_space_percent = ConfigAPI().get_notify_cluster_used_space_percent()

                if float(used_size) > float(notify_cluster_space_percent):
                    check_state = self.__context.state.get(self.get_plugin_name(), False)

                    if check_state == False:
                        result.title = gettext("core_message_notify_title_cluster_out_space")
                        result.message = '\n'.join(gettext("core_message_notify_cluster_out_space").split("\\n")).format(int(available_size))
                        # logger.warning(result.message)
                        result.plugin_name = str(self.get_plugin_name())
                        self.__context.results.append(result)
                        self.__context.state[self.get_plugin_name()] = True
                        logger.warning("Cluster is running out of disk space")
                    return
                self.__context.state[self.get_plugin_name()] = False

        except:
            logger.exception("Error occur during get cluster state")
Exemple #8
0
def manage_cluster_setting():
    if request.method == 'GET':
        result = ""
        if list_err in session:
            result = session[list_err]
            session.pop(list_err)
            return render_template('admin/configuration/cluster_settings.html', err=result, form=request.form)
        elif list_warning in session:
            result = session[list_warning]
            session.pop(list_warning)
            return render_template('admin/configuration/cluster_settings.html', warning=result, form=request.form)
        elif list_success in session:
            result = session[list_success]
            session.pop(list_success)
            cluster_config_model = ClusterConfigForm()
            manage_configuration = ManageConfig()
            new_cephAPI = CephAPI()
            # replica_no = new_cephAPI.get_replicas()
            # cluster_config_model.replica_no = replica_no
            # compression_algorithm = manage_configuration.get_compression_algorithm()
            # cluster_config_model.compression_algorithm = compression_algorithm
            # compression_mode = manage_configuration.get_compression_mode()
            # cluster_config_model.compression_mode = compression_mode
            internet_time_server = manage_configuration.get_ntp_server()
            app_config = ManageConfig().get_app_config()
            if app_config.email_notify_smtp_server != "":
                cluster_config_model.smtp_server = app_config.email_notify_smtp_server
                cluster_config_model.smtp_port = app_config.email_notify_smtp_port
                cluster_config_model.smtp_email = app_config.email_notify_smtp_email
                cluster_config_model.smtp_passwod = app_config.email_notify_smtp_password
                cluster_config_model.authentication_value = app_config.email_notify_smtp_security
            if internet_time_server == None:
                cluster_config_model.internet_time_server = ""
            else:
                cluster_config_model.internet_time_server = internet_time_server
            return render_template('admin/configuration/cluster_settings.html', success=result,
                                   form=cluster_config_model)
        else:
            cluster_config_model = ClusterConfigForm
            manage_configuration = ManageConfig()
            new_cephAPI = CephAPI()
            # replica_no = new_cephAPI.get_replicas()
            # cluster_config_model.replica_no = replica_no
            # compression_algorithm = manage_configuration.get_compression_algorithm()
            # cluster_config_model.compression_algorithm = compression_algorithm
            # compression_mode = manage_configuration.get_compression_mode()
            # cluster_config_model.compression_mode = compression_mode
            internet_time_server = manage_configuration.get_ntp_server()
            app_config = ManageConfig().get_app_config()
            if app_config.email_notify_smtp_server != "":
                cluster_config_model.smtp_server = app_config.email_notify_smtp_server
                cluster_config_model.smtp_port = app_config.email_notify_smtp_port
                cluster_config_model.smtp_email = app_config.email_notify_smtp_email
                cluster_config_model.smtp_passwod = app_config.email_notify_smtp_password
                cluster_config_model.authentication_value = app_config.email_notify_smtp_security
            if internet_time_server == None:
                cluster_config_model.internet_time_server = ""
            else:
                cluster_config_model.internet_time_server = internet_time_server
            return render_template('/admin/configuration/cluster_settings.html', form=cluster_config_model)
Exemple #9
0
def delete_snapshots(args):
    disk_id = args.disk_id
    try:
        ceph_api = CephAPI()
        image_name = str(ceph_api.conf_api.get_image_name_prefix() + disk_id)
        pool_name = ceph_api.get_pool_bydisk(
            disk_id)  # getting pool_name from disk_id

        confirm = ceph_api.delete_snapshots(pool_name, image_name)

        if not confirm:
            print("Error : Cannot delete all snapshots of disk " + disk_id)
            sys.exit(-1)

        sys.exit(0)

    except CephException as e:
        if e.id == CephException.GENERAL_EXCEPTION:
            print("Error : " + str(e.message) +
                  " , Cannot delete all snapshots of disk " + disk_id)
            sys.exit(-1)

    except Exception as e:
        print("Error : Exception , {}".format(e.message))
        sys.exit(-1)
    def get_disks_meta(self):
        ceph_api = CephAPI()
        consul_api = ConsulAPI()
        ls = ceph_api.get_disks_meta()
        for disk in ls:
            if disk and hasattr(disk, "paths") and not disk.paths:
                disk.status = DisplayDiskStatus.unattached
            elif disk and hasattr(disk, "paths") and disk.paths:
                data = consul_api.find_disk(disk.id)
                if data is not None:
                    disk.status = DisplayDiskStatus.starting
                    if str(data.Flags) == "1":
                        disk.status = DisplayDiskStatus.stopping
                    elif consul_api.is_path_locked(disk.id):
                        disk.status = DisplayDiskStatus.started

                else:
                    disk.status = DisplayDiskStatus.stopped

                job_manager = JobManager()
                job_list = job_manager.get_running_job_list()

                for j in job_list:

                    # Check if the status running
                    if j.is_running:
                        # Set disk status [deleting]
                        if j.type == JobType.DELETE_DISK and str(
                                j.params).find(str(disk.id)) > -1:
                            disk.status = DisplayDiskStatus.deleting

        return ls
Exemple #11
0
 def set_compression_mode(self, mode, pool="rbd"):
     api = CephAPI()
     if mode == CompressionMode.none:
         mode_str = 'none'
     elif mode == CompressionMode.force:
         mode_str = 'force'
     return api.set_compression_mode(mode_str, pool)
    def start(self, disk_id, pool):
        try:
            ceph_api = CephAPI()
            consul_api = ConsulAPI()

            attr = ceph_api.read_image_metadata(
                ConfigAPI().get_image_name_prefix() + disk_id, pool)
            petasan_meta = attr.get(ConfigAPI().get_image_meta_key())
            disk_meta = DiskMeta()
            if petasan_meta:
                disk_meta.load_json(petasan_meta)
            else:
                return Status.error

            consul_api.add_disk_resource(disk_meta.id, "disk")
            consul_api.add_disk_pool(disk_meta.id, pool)
            i = 0
            for p in disk_meta.paths:
                i += 1
                consul_api.add_disk_resource(
                    "/".join(["", disk_meta.id, str(i)]), None)

        except Exception as e:
            logger.error("Can not start disk %s" % disk_id)
            logger.exception(e.message)
            return Status.error
        return Status.done
    def get_templates(self):
        templates = {}
        path = ConfigAPI().get_crush_rule_templates_path()
        ceph_api = CephAPI()
        next_id = ceph_api.get_next_rule_id()

        for f in os.listdir(path):
            if os.path.isdir(path + f):
                continue

            with open(path + f, 'r') as file:
                lines = file.readlines()

            body = ''
            for line in lines:
                if len(line) == 0:
                    continue
                if line.startswith('id '):
                    body += '# Auto generated id. do not modify \n'
                    body += 'id ' + next_id + '\n'
                    continue
                body += line

            templates[f] = body

        return templates
Exemple #14
0
 def get_compression_mode(self, pool='rbd'):
     api = CephAPI()
     mode = api.get_compression_mode(pool)
     if mode is None or mode is '' or "none" in mode:
         return CompressionMode.none
     if 'force' in mode:
         return CompressionMode.force
     raise Exception("Compression mode is not valid")
Exemple #15
0
def get_list():
    ceph_api = CephAPI()
    for i in ceph_api.get_disks_meta():

        print i.user, i.disk_name, i.password
        for p in i.get_paths():

            print p.ip, p.subnet_mask, p.eth
Exemple #16
0
def set_disk_metadata(args):
    io_ctx = None
    ceph_api = CephAPI()
    cluster = None

    try:
        cluster = ceph_api.connect()
        io_ctx = cluster.open_ioctx(args.pool)

        # Get which ceph user is using this function & get his keyring file path #
        ceph_auth = CephAuthenticator()

        config = configuration()
        cluster_name = config.get_cluster_name()

        if args.file:
            with open(str(args.file), 'r') as file:
                disk_metadata_str = file.read()

        else:
            disk_metadata = sys.stdin.readlines()
            disk_metadata_str = ''.join(
                str(line)
                for line in disk_metadata)  # converting list to string

        # read object meta :
        cmd = "rbd info " + args.pool + "/" + str(
            args.image) + " " + ceph_auth.get_authentication_string(
            ) + " --cluster " + cluster_name + " | grep rbd_data"
        ret, stdout, stderr = exec_command_ex(cmd)

        if ret != 0:
            if stderr:
                cluster.shutdown()
                print("Cannot get image meta object from rbd header.")

        rbd_data = stdout.rstrip().strip()
        dot_indx = rbd_data.rfind(".")

        image_id = rbd_data[(dot_indx + 1):]

        meta_object = "rbd_header." + image_id
        attr_object = meta_object

        io_ctx.set_xattr(str(attr_object),
                         str(ConfigAPI().get_image_meta_key()),
                         disk_metadata_str)
        io_ctx.close()
        cluster.shutdown()
        sys.exit(0)

    except Exception as e:
        print("Error in executing script function : set_disk_metadata , " +
              str(e.message))
        io_ctx.close()
        cluster.shutdown()
        sys.exit(-1)
Exemple #17
0
 def get_active_replicated_pools(self):
     ceph_api = CephAPI()
     active_pools = ceph_api.get_active_pools()
     pools_info = ceph_api.get_pools_info()
     replicated_active_pools = []
     for pool in pools_info:
         if pool.type == 'replicated' and pool.name in active_pools:
             replicated_active_pools.append(pool.name)
     return replicated_active_pools
    def rados_benchmark(self, block_size, is_type_write, duration_sec, threads,
                        pool):
        ceph_api = CephAPI()

        if is_type_write:
            return ceph_api.rados_write(duration_sec, threads, block_size,
                                        pool)
        else:
            return ceph_api.rados_read(duration_sec, threads, pool)
Exemple #19
0
def read_disks_metadata(args):
    io_ctx = None
    ceph_api = CephAPI()
    cluster = None

    try:
        cluster = ceph_api.connect()
        io_ctx = cluster.open_ioctx(args.pool)

        # Get which ceph user is using this function & get his keyring file path #
        ceph_auth = CephAuthenticator()

        config = configuration()
        cluster_name = config.get_cluster_name()

        cmd = "rbd info " + args.pool + "/" + str(
            args.image) + " " + ceph_auth.get_authentication_string(
            ) + " --cluster " + cluster_name + " | grep rbd_data"

        ret, stdout, stderr = exec_command_ex(cmd)

        if ret != 0:
            if stderr:
                cluster.shutdown()
                print("Cannot get image meta object from rbd header.")
                sys.exit(-1)

        rbd_data = stdout.rstrip().strip()
        dot_indx = rbd_data.rfind(".")

        image_id = rbd_data[(dot_indx + 1):]

        rbd_header_object = "rbd_header." + image_id

        try:
            ret = io_ctx.get_xattr(rbd_header_object, meta_key)
        except:
            ret = io_ctx.get_xattr(rbd_header_object[:-1], meta_key)

        io_ctx.close()
        cluster.shutdown()

        if ret:
            print(ret)
            sys.stdout.flush()
            sys.exit(0)
        else:
            # Non-PetaSAN Disk :
            sys.exit(-1)

    except Exception as e:
        print("Error in executing script function : read_disks_metadata , " +
              str(e.message))
        io_ctx.close()
        cluster.shutdown()
        sys.exit(-1)
Exemple #20
0
 def __clean_unused_rbd_images(self):
     ceph_api = CephAPI()
     rbd_images = ceph_api.get_mapped_images()
     if rbd_images is None:
         return
     for image, mapped_count in rbd_images.iteritems():
         if image not in self.__backstore:
             if int(mapped_count) > 0:
                 for i in range(0, int(mapped_count)):
                     ceph_api.unmap_image(image)
                     logger.debug("Unmapped unused image {}.".format(image))
Exemple #21
0
def get_meta():
    ceph_confg = CephAPI()
    ceph_api = CephAPI()
    logger.info("done , create disk")
    attr = ceph_api.read_image_metadata("000001")
    xx = attr.get(app_conf.get_image_meta_key())
    logger.info(xx)
    disk = DiskMeta()
    disk.load_json(xx)

    logger.info("disk user is %s" % (disk.user))
def get_pool_by_disk(disk_id):

    consul_api = ConsulAPI()
    ceph_api = CephAPI()
    pool = consul_api.get_disk_pool(disk_id)
    if pool:
        return pool
    pool = ceph_api.get_pool_bydisk(disk_id)
    if pool:
        return pool

    return None
Exemple #23
0
 def set_compression_algorithm(self, algorithm, pool="rbd"):
     api = CephAPI()
     if algorithm == CompressionAlgorithm.none:
         algorithm_str = 'none'
     elif algorithm == CompressionAlgorithm.zlib:
         algorithm_str = 'zlib'
     elif algorithm == CompressionAlgorithm.snappy:
         algorithm_str = 'snappy'
     elif algorithm == CompressionAlgorithm.zstd:
         algorithm_str = 'zstd'
     elif algorithm == CompressionAlgorithm.lz4:
         algorithm_str = 'lz4'
     return api.set_compression_algorithm(algorithm_str, pool)
Exemple #24
0
def stop_all_disks():
    ceph_api = CephAPI()
    disks = ceph_api.get_disks_meta()
    if disks is None:
        print('Error reading disk metadata from Ceph')
        return False
    disks.sort(key=lambda disk: disk.id)
    for disk_meta in disks:
        if is_disk_started(disk_meta.id):
            if stop_disk(disk_meta.id, disk_meta.pool):
                print "Stopped disk " + disk_meta.id + " " + disk_meta.disk_name
            else:
                print "Error stopping disk " + disk_meta.id + " " + disk_meta.disk_name
    return True
Exemple #25
0
    def get_compression_algorithm(self, pool="rbd"):
        api = CephAPI()
        algorithm = api.get_compression_algorithm(pool)

        if algorithm is None or algorithm is '' or "none" in algorithm:
            return CompressionAlgorithm.none
        if 'zlib' in algorithm:
            return CompressionAlgorithm.zlib
        if 'snappy' in algorithm:
            return CompressionAlgorithm.snappy
        if 'zstd' in algorithm:
            return CompressionAlgorithm.zstd
        if 'lz4' in algorithm:
            return CompressionAlgorithm.lz4
        raise Exception("Compression algorithm is not valid")
Exemple #26
0
    def run(self):
        try:
            result = Result()
            ceph_status_overall = ""

            ceph_api = CephAPI()
            cluster_status = ceph_api.get_ceph_cluster_status()                       # ceph status --format json-pretty

            if cluster_status is not None:
                cluster_status = json.loads(cluster_status)

                # Ceph 12 :
                if "overall_status" in cluster_status["health"] and cluster_status["health"]["overall_status"] is not None:
                    ceph_status_overall = cluster_status["health"]["overall_status"]
                else:
                    ceph_status_overall = cluster_status["health"]["status"]

                if ceph_status_overall == "HEALTH_ERR":
                    prv_err  = self.__context.state.get(self.get_plugin_name(), False)

                    if not prv_err:
                        ceph_health_obj = cluster_status["health"]
                        summary_messages = ""
                        summary_messages_ls = []

                        if "checks" in ceph_health_obj:
                            for key in ceph_health_obj["checks"]:
                                if ceph_health_obj["checks"][key] is not None:
                                    msg = ceph_health_obj["checks"][key]["summary"]["message"]
                                    summary_messages_ls.append(msg)

                        summary_messages = '\n    '.join(summary_messages_ls)

                        result.title = gettext("core_message_notify_cluster_status_title")
                        result.message = '\n'.join(gettext("core_message_notify_cluster_status_body").split("\\n")).format(summary_messages)

                        result.plugin_name = str(self.get_plugin_name())
                        self.__context.results.append(result)
                        self.__context.state[self.get_plugin_name()] = True
                        logger.warning("Cluster overall health status is HEALTH_ERR")

                    return

                self.__context.state[self.get_plugin_name()] = False

        except Exception as e:
            logger.exception(e)
            logger.error("An error occurred while ClusterStatusPlugin was running.")
Exemple #27
0
    def validate_new_iscsi2_ip(self, ip):

        if not self.is_ip_in_iscsi2_subnet(ip):
            return NewIPValidation.wrong_subnet

        ips_used = set()
        # get the image metada from ceph
        ceph_api = CephAPI()
        for i in ceph_api.get_disks_meta():
            for p in i.get_paths():
                ips_used.add(p.ip)

        if ip in ips_used:
            return NewIPValidation.used_already

        return NewIPValidation.valid
    def get_disk_paths(self, disk_id, pool):
        paths_list = CephAPI().get_disk_meta(disk_id, pool).paths
        paths_list_with_node = []
        sessions_dict = ConsulAPI().get_sessions_dict(
            ConfigAPI().get_iscsi_service_session_name())

        # in case consul lock on disk
        for kv in ConsulAPI().get_disk_paths(disk_id):
            path = Path()
            path_str = paths_list[int(str(kv.Key).split(disk_id + "/")[1]) - 1]
            path.load_json(json.dumps(path_str))
            if hasattr(kv, "Session") and sessions_dict.has_key(kv.Session):

                path.locked_by = sessions_dict.get(kv.Session).Node
                paths_list_with_node.append(path)
            else:
                paths_list_with_node.append(path)

        # in case disk is stopped
        if not paths_list_with_node:

            for path_str in paths_list:
                path = Path()
                path.load_json(json.dumps(path_str))
                paths_list_with_node.append(path)
        return paths_list_with_node
Exemple #29
0
def get_all_images(args):
    try:
        ceph_api = CephAPI()
        pool_name = args.pool
        images_ls = ceph_api.get_all_images(pool_name)
        print(json.dumps(images_ls))
        sys.exit(0)

    except CephException as e:
        if e.id == CephException.GENERAL_EXCEPTION:
            print("Error : {} , Cannot get images list.".format(e.message))
            sys.exit(-1)

    except Exception as e:
        print("Error : Exception , {}".format(e.message))
        sys.exit(-1)
Exemple #30
0
    def _get_pool_by_disk(self, disk_id):
        consul_api = ConsulAPI()
        ceph_api = CephAPI()
        pool = consul_api.get_disk_pool(disk_id)
        if pool:
            logger.info('Found pool:{} for disk:{} via consul'.format(
                pool, disk_id))
            return pool
        pool = ceph_api.get_pool_bydisk(disk_id)
        if pool:
            logger.info('Found pool:{} for disk:{} via ceph'.format(
                pool, disk_id))
            return pool

        logger.error('Could not find pool for disk ' + disk_id)
        return None