예제 #1
0
파일: limits.py 프로젝트: Byron/bit
    def generate_fix_script(self, report, writer):
        """Generate a script with progress and zfs destroy commands
        @note will only generate commands for one host"""
        mss = self._config.min_snapshot_size or 0
        if mss:
            mss = size_to_int(mss)
        # end handle type conversion
        fss = self._config.min_filesystem_size or 0
        if fss:
            fss = size_to_int(fss)
        # end handle type conversion
        sotd = self._config.snapshots_older_than_days
        date = datetime_days_ago(time(), sotd)

        def predicate(rec):
            tn = rec[3]
            ctime = rec[4] # creation delta time
            size = rec[6]

            # We don't know what's part of the Report, and why those values are there.
            # This is why we have to re-evaluate the inverse logic to prevent us from 
            # generating code that would mean something different
            return (tn == self.TYPE_SNAPSHOT   and (size < mss or ctime <= date)) or (tn == self.TYPE_FILESYSTEM and size < fss)
        # end predicate

        header_line = "SETTING UP ZFS DESTROY COMMANDS FOR SELECTED DATASETS ON HOST %s\n" % ', '.join(self._config.hosts)
        return self._create_zfs_destroy_script(predicate, header_line, report, writer)
예제 #2
0
파일: test_utility.py 프로젝트: Byron/bit
 def test_size(self):
     ki = 1024
     for size, res in (('124K', 124.0*ki), 
                       ('24.2M', 24.2*ki**2),
                       ('10.6g', 10.6*ki**3), # caps don't matter
                       ('2.75T', 2.75*ki**4),
                       ('5.234P', 5.234*ki**5) ): 
         assert size_to_int(size) == int(res)
예제 #3
0
파일: io_stat.py 프로젝트: Byron/bit
    def _sanitize_configuration(self):
        """@return verified and sanitized configuration"""
        config = self.configuration()
        for size_attr in ('write_chunk_size', 'random_read_volume', 'random_read_chunk_size', 'file_size'):
            setattr(config, size_attr, size_to_int(getattr(config, size_attr)))
        # end for each attribute

        assert config.num_threads >= 1, "Must set at least 1 or more workers"
        assert config.output_dir, "Output directory (output_dir) must be set"
        assert config.output_dir.isdir(), "output directory at '%s' must be an accessible directory" % self.output_dir

        return config
예제 #4
0
파일: test_utility.py 프로젝트: Byron/bit
    def test_conversion(self):
          """Assure we do it right !"""
          size_str = '59.6T'
          size = size_to_int(size_str)
          assert round(float(size) / 1024**4, 3) == 59.6

          now_time = time()
          offset = 83921
          delta = seconds_to_datetime(now_time) - seconds_to_datetime(now_time - offset)
          assert delta_to_seconds(delta) == offset

          # just make the call, no check for now
          utc_datetime_to_date_time_string(datetime.utcnow())
예제 #5
0
파일: reserve.py 프로젝트: Byron/bit
    def generate(self):
        # Create an initial query and map filesystems by basename
        rep = self.ReportType(copy.deepcopy(self.report_schema))
        now = datetime.now()
        config = self.settings_value()
        if config.mode not in self.valid_modes:
            raise ValueError("Can only support the following modes: %s" % ', '.join(self.valid_modes))
        # end handle

        rep.columns[4][0] = config.mode == self.MODE_RESERVATION and 'reserved' or 'quota'

        query = self._session.query(ZDataset).filter(ZDataset.avail != None).\
                                              filter(ZDataset.zfs_priority != None).\
                                              filter(ZDataset.name.like(config.pool_name + '%/%'))

        query = host_filter(config.hosts, ZDataset.host, query)
        fs_map = dict()
        for fs in query:
            if fs.property_is_inherited('zfs_priority'):
                continue
            fs_map.setdefault((fs.host, fs.url().pool()), list()).append(fs)
        # end for each filesystem

        distribute_space = config.distribute_space
        if distribute_space:
            distribute_space = size_to_int(distribute_space)
        # end convert space

        if distribute_space and config.max_cap:
            raise ValueError("Please specify either 'max_cap or 'distribute_space', or set one of them to 0")
        # end assure we don't see both

        if config.debug_priorities and len(fs_map) > 1:
            raise AssertionError("If debug_priorities are used, you muse limit the amount of involved hosts to one")
        # end make sure debugging makes sense

        for (host, pool), fs_list in fs_map.iteritems():
            if config.debug_priorities and len(config.debug_priorities) != len(fs_list):
                raise AssertionError("Please specify exactly %i priorities, one for each filesystem, got %i" % (len(fs_list), len(config.debug_priorities)))
            # end verify priorities

            priorities = config.debug_priorities or [fs.zfs_priority for fs in fs_list]
            total_parts = sum(priorities)
            pool = self._session.instance_by_url(ZFSURL.new(host, pool))
            if distribute_space:
                total_alloc = distribute_space
            else:
                total_alloc = pool.size * (config.max_cap / 100.0)
            # end handle total_alloc

            for fs, prio in zip(fs_list, priorities):
                reserve = (total_alloc / float(total_parts)) * prio
                rep.records.append([now - fs.updated_at,
                                    fs,
                                    prio,
                                    fs.used,
                                    reserve,
                                    reserve - fs.used,
                                    reserve - fs.avail,
                                    (fs.used / float(reserve)) * 100.0
                                    ])
            # end for each filesystem
        # end for each pool-host pair
        if len(rep.records) > 1:
            rep.records.append(rep.aggregate_record())
        # end aggregate only if it makes sense

        return rep
예제 #6
0
파일: limits.py 프로젝트: Byron/bit
    def generate(self):
        """See class description and schema for more information"""
        rep = self.ReportType()
        rep.columns.extend(self.report_schema)

        max_pool_cap,\
        fs_cap,\
        min_snapshot_size,\
        min_filesystem_size,\
        snapshots_older_than_days,\
        filesystems_older_than_days,\
        name_like,\
        hosts                       = ( self._config.max_pool_cap, 
                                        self._config.max_filesystem_cap,
                                        self._config.min_snapshot_size,
                                        self._config.min_filesystem_size,
                                        self._config.snapshots_older_than_days,
                                        self._config.filesystems_older_than_days,
                                        self._config.name_like,
                                        self._config.hosts)
        assert name_like, "Name matching must be at least '%%', got nothing"
        rappend = rep.records.append
        now = datetime.now()
        nows = time()
        percent_comment = '%s.cap > %.0f%%'

        # POOL CAPACITIES
        ###################
        if max_pool_cap <= 100.0:
            fill_capage = (ZPool.size - ZPool.free) / (ZPool.size * 1.0)
            percent_condition = fill_capage >= max_pool_cap / 100.0
            comment = percent_comment % (self.TYPE_POOL, max_pool_cap)
            for pool in host_filter(hosts, ZPool.host, self._session.query(ZPool).\
                                            filter(percent_condition).\
                                            filter(ZPool.name.like(name_like)).\
                                            order_by(fill_capage.desc())): 
                rec = [now - pool.updated_at, pool.host, pool.name, self.TYPE_POOL,'NA', 'NA', pool.size, pool.free, 
                        ((pool.size - pool.free) / float(pool.size)) * 100, comment]
                rappend(rec)
            # end for each that matches
        # end check enabled

        # FILESYSTEM CAPACITIES
        ########################
        if fs_cap <= 100.0:
            filesystem = ZDataset.avail != None
            fill_capage = 1.0 - (ZDataset.avail / (ZDataset.avail + ZDataset.used) * 1.0)
            percent_condition = fill_capage >= fs_cap / 100.0
            comment = percent_comment % (self.TYPE_FILESYSTEM, fs_cap)
            for fs in host_filter(hosts, ZDataset.host, self._session.query(ZDataset).\
                                                filter(filesystem).\
                                                filter(ZDataset.name.like(name_like)).\
                                                filter(percent_condition).\
                                                order_by(fill_capage.desc())):
                rec = [ now - fs.updated_at, fs.host, fs.name, self.TYPE_FILESYSTEM, fs.creation, now - fs.creation, fs.used, fs.avail, 
                        (1.0 - (fs.avail / float((fs.avail + fs.used)))) * 100.0, comment]
                rappend(rec)
            # end for each record
        # end check enabled

        # DATASETS TOO SMALL
        #####################
        # TODO: implement protection against deleting the last snapshot ! In case it's on, we must assure
        # the latest snapshot of a filesystem is not deleted. Could be a complex group + join
        def make_records(type_condition, condition, type_name, comment):
            for ds in host_filter(hosts, ZDataset.host, self._session.query(ZDataset).\
                                        filter(type_condition).\
                                        filter(ZDataset.name.like(name_like)).\
                                        filter(condition).\
                                        order_by(ZDataset.host)):
                rec = [now - ds.updated_at, ds.host, ds.name, type_name, ds.creation, now - ds.creation, 
                        ds.used, ds.avail or 0,
                        ds.avail and ((ds.used / float(ds.used + ds.avail)) * 100.0) or 100.0, comment]
                rappend(rec)
            # end for each snapshot
        # end make records utility

        min_size_fmt = '%s.size < %s'
        if min_snapshot_size:
            condition = ZDataset.used < size_to_int(min_snapshot_size)
            make_records(ZDataset.avail == None, condition, self.TYPE_SNAPSHOT, 
                            min_size_fmt % (self.TYPE_SNAPSHOT, min_snapshot_size))
        if min_filesystem_size:
            condition = ZDataset.used < size_to_int(min_filesystem_size)
            make_records(ZDataset.avail != None, condition, self.TYPE_FILESYSTEM, 
                            min_size_fmt % (self.TYPE_FILESYSTEM, min_filesystem_size))
        # end handle item sizes


        # DATASET AGE
        ##############
        if snapshots_older_than_days:
            condition = ZDataset.creation <= datetime_days_ago(nows, snapshots_older_than_days)
            make_records(ZDataset.avail == None, condition, self.TYPE_SNAPSHOT, 
                            '%s.creation older %id' % (self.TYPE_SNAPSHOT, snapshots_older_than_days))

        if filesystems_older_than_days:
            condition = ZDataset.creation <= datetime_days_ago(nows, filesystems_older_than_days)
            make_records(ZDataset.avail != None, condition, self.TYPE_FILESYSTEM, 
                            '%s.creation older %id' % (self.TYPE_FILESYSTEM, filesystems_older_than_days))
        # end ignore age if time is 0, which is essentially just no


        # FILESYSTEM FREE SPACE
        ########################
        min_filesystem_avail_size = size_to_int(self._config.min_filesystem_avail_size)
        if min_filesystem_avail_size:
            condition = ZDataset.avail >= min_filesystem_avail_size
            make_records(ZDataset.avail != None, condition, self.TYPE_FILESYSTEM, 
                            '%s.avail >= %s' % (self.TYPE_FILESYSTEM, self._config.min_filesystem_avail_size))
        # end ignore this value if zero
        

        # AGGREGATE
        #############
        self._aggregate_records(rep.records, now)
        return rep
예제 #7
0
파일: duplication.py 프로젝트: Byron/bit
    def generate(self):
        # Create an initial query and map filesystems by basename
        rep = self.ReportType(self.report_schema)
        now = datetime.now()
        config = self.settings_value()
        query = self._session.query(ZDataset).filter(ZDataset.type == self.TYPE_FILESYSTEM).\
                                              filter(ZDataset.name.like(config.name_like)).\
                                              order_by(ZDataset.name, ZDataset.creation)

        bmap = dict()
        for inst in query:
            # Ignore pools and non-leaf filesystems
            if inst.is_pool_filesystem() or list(inst.children()):
                continue
            # end ignore non-leafs
            bmap.setdefault(os.path.basename(inst.name), list()).append(inst)
        # end build lookup


        min_copies, hosts, min_equivalence, ignore_size =   config.min_copies,\
                                                            config.hosts,\
                                                            config.min_equivalence,\
                                                            size_to_int(config.ignore_filesystems_smaller)
        radd = lambda r: rep.records.append(r) or r

        # Sort the list by num-copies-descending
        slist = sorted(bmap.items(), key=lambda t: len(t[1]), reverse=True)

        # Then compute the equivalence and build the report
        for basename, filesystems in slist:
            # For now, the oldest one is the master, which owns the copies
            # The host filter is applied here
            master = filesystems.pop(0)
            if min_copies and len(filesystems) + 1 >= min_copies or\
               hosts and master.host not in hosts or\
               master.used < ignore_size:
                continue
            # end skip if we are below threshold

            mrec = radd([now - master.updated_at,
                          (not len(filesystems) and '+' or '+-'),
                          master,
                          len(filesystems) + 1,
                          0.0,      # done later as avg
                          None,     # never set
                          master.used,
                          master.ratio,
                          master.used * master.ratio])

            crecs = list()
            msnapshots = list(master.snapshots())
            msnapshot_names = [ss.snapshot_name() for ss in msnapshots]
            for shadow in filesystems:
                equivalence, ss = self._compute_equivalence(msnapshots, msnapshot_names, shadow)
                crecs.append(radd([now - shadow.updated_at,
                                  '  `-',
                                  shadow,
                                  0,
                                  equivalence * 100.0,
                                  ss,
                                  shadow.used,
                                  shadow.ratio,
                                  shadow.used * shadow.ratio,
                            ]))
            # end for each copy

            if filesystems:
                mrec[4] = sum(r[4] for r in crecs) / len(crecs)
            # end compute average equivalence

            # drop previous records if equivalence is good enough
            if min_equivalence and mrec[4] >= min_equivalence:
                rep.records = rep.records[:-len(crecs) - 1]
        # end handle    

        return rep